superlocalmemory 4.1.13 → 4.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/CHANGELOG.md +48 -0
- package/README.md +3 -3
- package/package.json +1 -8
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/agents/slm-memory-advisor.md +1 -1
- package/plugin-src/agents/slm-optimize-advisor.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-governance/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-loop/SKILL.md +1 -1
- package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
- package/plugin-src/skills/slm-profile/SKILL.md +1 -1
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-scope/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/scripts/postinstall.js +71 -2
- package/pyproject.toml +0 -250
- package/src/superlocalmemory/__init__.py +0 -82
- package/src/superlocalmemory/access/__init__.py +0 -3
- package/src/superlocalmemory/access/rbac.py +0 -575
- package/src/superlocalmemory/attribution/__init__.py +0 -9
- package/src/superlocalmemory/attribution/signer.py +0 -173
- package/src/superlocalmemory/attribution/watermark.py +0 -189
- package/src/superlocalmemory/brain/__init__.py +0 -5
- package/src/superlocalmemory/brain/truth.py +0 -418
- package/src/superlocalmemory/cli/__init__.py +0 -5
- package/src/superlocalmemory/cli/__main__.py +0 -17
- package/src/superlocalmemory/cli/_lazy_init.py +0 -115
- package/src/superlocalmemory/cli/cache_cmd.py +0 -198
- package/src/superlocalmemory/cli/commands.py +0 -4710
- package/src/superlocalmemory/cli/compress_cmd.py +0 -151
- package/src/superlocalmemory/cli/context_commands.py +0 -193
- package/src/superlocalmemory/cli/daemon.py +0 -909
- package/src/superlocalmemory/cli/db_migrate.py +0 -150
- package/src/superlocalmemory/cli/diagnostics_cmd.py +0 -101
- package/src/superlocalmemory/cli/escape_hatch.py +0 -220
- package/src/superlocalmemory/cli/evidence_cmd.py +0 -103
- package/src/superlocalmemory/cli/gdpr_cmd.py +0 -792
- package/src/superlocalmemory/cli/gdpr_io.py +0 -109
- package/src/superlocalmemory/cli/help_cmd.py +0 -197
- package/src/superlocalmemory/cli/host_upgrades.py +0 -189
- package/src/superlocalmemory/cli/ingest_cmd.py +0 -327
- package/src/superlocalmemory/cli/json_output.py +0 -81
- package/src/superlocalmemory/cli/loop_cmd.py +0 -187
- package/src/superlocalmemory/cli/main.py +0 -1145
- package/src/superlocalmemory/cli/mesh_cmd.py +0 -38
- package/src/superlocalmemory/cli/migrate_cmd.py +0 -55
- package/src/superlocalmemory/cli/ops_cmd.py +0 -281
- package/src/superlocalmemory/cli/optimize_cmd.py +0 -179
- package/src/superlocalmemory/cli/optimize_constants.py +0 -31
- package/src/superlocalmemory/cli/pending_store.py +0 -296
- package/src/superlocalmemory/cli/proxy_cmd.py +0 -108
- package/src/superlocalmemory/cli/scale_engine_cmd.py +0 -56
- package/src/superlocalmemory/cli/service_installer.py +0 -373
- package/src/superlocalmemory/cli/setup_wizard.py +0 -1162
- package/src/superlocalmemory/cli/summary_cmd.py +0 -215
- package/src/superlocalmemory/cli/version_banner.py +0 -202
- package/src/superlocalmemory/cli/wizard_v3426_options.py +0 -129
- package/src/superlocalmemory/code_graph/__init__.py +0 -46
- package/src/superlocalmemory/code_graph/blast_radius.py +0 -177
- package/src/superlocalmemory/code_graph/bridge/__init__.py +0 -36
- package/src/superlocalmemory/code_graph/bridge/entity_resolver.py +0 -490
- package/src/superlocalmemory/code_graph/bridge/event_listeners.py +0 -206
- package/src/superlocalmemory/code_graph/bridge/fact_enricher.py +0 -159
- package/src/superlocalmemory/code_graph/bridge/hebbian_linker.py +0 -170
- package/src/superlocalmemory/code_graph/bridge/maintenance.py +0 -220
- package/src/superlocalmemory/code_graph/bridge/temporal_checker.py +0 -152
- package/src/superlocalmemory/code_graph/changes.py +0 -363
- package/src/superlocalmemory/code_graph/communities.py +0 -299
- package/src/superlocalmemory/code_graph/config.py +0 -154
- package/src/superlocalmemory/code_graph/database.py +0 -526
- package/src/superlocalmemory/code_graph/extractors/__init__.py +0 -95
- package/src/superlocalmemory/code_graph/extractors/python.py +0 -413
- package/src/superlocalmemory/code_graph/extractors/typescript.py +0 -556
- package/src/superlocalmemory/code_graph/flows.py +0 -350
- package/src/superlocalmemory/code_graph/git_hooks.py +0 -226
- package/src/superlocalmemory/code_graph/graph_engine.py +0 -295
- package/src/superlocalmemory/code_graph/graph_store.py +0 -335
- package/src/superlocalmemory/code_graph/incremental.py +0 -200
- package/src/superlocalmemory/code_graph/models.py +0 -130
- package/src/superlocalmemory/code_graph/parser.py +0 -687
- package/src/superlocalmemory/code_graph/resolver.py +0 -321
- package/src/superlocalmemory/code_graph/search.py +0 -460
- package/src/superlocalmemory/code_graph/service.py +0 -95
- package/src/superlocalmemory/code_graph/watcher.py +0 -207
- package/src/superlocalmemory/compliance/__init__.py +0 -0
- package/src/superlocalmemory/compliance/abac.py +0 -204
- package/src/superlocalmemory/compliance/audit.py +0 -385
- package/src/superlocalmemory/compliance/eu_ai_act.py +0 -101
- package/src/superlocalmemory/compliance/gdpr.py +0 -1479
- package/src/superlocalmemory/compliance/lifecycle.py +0 -158
- package/src/superlocalmemory/compliance/retention.py +0 -415
- package/src/superlocalmemory/compliance/scheduler.py +0 -217
- package/src/superlocalmemory/contracts/__init__.py +0 -1
- package/src/superlocalmemory/contracts/schemas/agent-experience-v1.schema.json +0 -92
- package/src/superlocalmemory/contracts/schemas/agent-integration-contract-v2.schema.json +0 -46
- package/src/superlocalmemory/contracts/schemas/cognitive-turn-receipt-v1.schema.json +0 -59
- package/src/superlocalmemory/contracts/v402.py +0 -62
- package/src/superlocalmemory/core/__init__.py +0 -0
- package/src/superlocalmemory/core/actor_context.py +0 -166
- package/src/superlocalmemory/core/admission.py +0 -769
- package/src/superlocalmemory/core/backend_orchestrator.py +0 -637
- package/src/superlocalmemory/core/block_hygiene.py +0 -147
- package/src/superlocalmemory/core/community_summary.py +0 -267
- package/src/superlocalmemory/core/component_healer.py +0 -144
- package/src/superlocalmemory/core/component_registry.py +0 -514
- package/src/superlocalmemory/core/config.py +0 -2204
- package/src/superlocalmemory/core/consolidation_engine.py +0 -983
- package/src/superlocalmemory/core/context_cache.py +0 -574
- package/src/superlocalmemory/core/derivation_lineage.py +0 -246
- package/src/superlocalmemory/core/embedding_worker.py +0 -208
- package/src/superlocalmemory/core/embeddings.py +0 -1052
- package/src/superlocalmemory/core/engine.py +0 -1395
- package/src/superlocalmemory/core/engine_capabilities.py +0 -24
- package/src/superlocalmemory/core/engine_ingestion.py +0 -983
- package/src/superlocalmemory/core/engine_lock.py +0 -75
- package/src/superlocalmemory/core/engine_wiring.py +0 -776
- package/src/superlocalmemory/core/entity_community.py +0 -178
- package/src/superlocalmemory/core/error_envelope.py +0 -60
- package/src/superlocalmemory/core/evidence_bundle.py +0 -528
- package/src/superlocalmemory/core/fact_consolidator.py +0 -812
- package/src/superlocalmemory/core/file_lock.py +0 -92
- package/src/superlocalmemory/core/graph_analyzer.py +0 -456
- package/src/superlocalmemory/core/graph_metrics.py +0 -597
- package/src/superlocalmemory/core/graph_pruner.py +0 -939
- package/src/superlocalmemory/core/health_monitor.py +0 -338
- package/src/superlocalmemory/core/hooks.py +0 -65
- package/src/superlocalmemory/core/ingest_gate.py +0 -133
- package/src/superlocalmemory/core/ingest_policy.py +0 -38
- package/src/superlocalmemory/core/ingestion_command.py +0 -1042
- package/src/superlocalmemory/core/injection.py +0 -434
- package/src/superlocalmemory/core/install_detector.py +0 -131
- package/src/superlocalmemory/core/key_expander.py +0 -138
- package/src/superlocalmemory/core/lifecycle_state.py +0 -153
- package/src/superlocalmemory/core/maintenance.py +0 -777
- package/src/superlocalmemory/core/maintenance_scheduler.py +0 -507
- package/src/superlocalmemory/core/materialization_control.py +0 -20
- package/src/superlocalmemory/core/mcp_embedder_proxy.py +0 -89
- package/src/superlocalmemory/core/memory_health.py +0 -266
- package/src/superlocalmemory/core/mode_capability.py +0 -111
- package/src/superlocalmemory/core/modes.py +0 -168
- package/src/superlocalmemory/core/mutations.py +0 -688
- package/src/superlocalmemory/core/ollama_embedder.py +0 -266
- package/src/superlocalmemory/core/ollama_validator.py +0 -315
- package/src/superlocalmemory/core/operation_policy.py +0 -92
- package/src/superlocalmemory/core/operation_policy_registry.py +0 -542
- package/src/superlocalmemory/core/operation_request.py +0 -127
- package/src/superlocalmemory/core/ops_remediation.py +0 -542
- package/src/superlocalmemory/core/pii.py +0 -105
- package/src/superlocalmemory/core/platform_utils.py +0 -138
- package/src/superlocalmemory/core/profiles.py +0 -234
- package/src/superlocalmemory/core/progressive_abstraction.py +0 -208
- package/src/superlocalmemory/core/projection_drain.py +0 -380
- package/src/superlocalmemory/core/queue_consumer.py +0 -168
- package/src/superlocalmemory/core/ram_lock.py +0 -160
- package/src/superlocalmemory/core/rate_limit.py +0 -151
- package/src/superlocalmemory/core/recall_gate.py +0 -95
- package/src/superlocalmemory/core/recall_pipeline.py +0 -1337
- package/src/superlocalmemory/core/recall_queue.py +0 -377
- package/src/superlocalmemory/core/recall_worker.py +0 -414
- package/src/superlocalmemory/core/registry.py +0 -121
- package/src/superlocalmemory/core/remember_admission.py +0 -161
- package/src/superlocalmemory/core/remember_runtime.py +0 -1190
- package/src/superlocalmemory/core/remote_mode.py +0 -214
- package/src/superlocalmemory/core/reranker_worker.py +0 -338
- package/src/superlocalmemory/core/safe_fs.py +0 -108
- package/src/superlocalmemory/core/scale_autopromote.py +0 -196
- package/src/superlocalmemory/core/scale_engine.py +0 -915
- package/src/superlocalmemory/core/score_contract.py +0 -82
- package/src/superlocalmemory/core/security_primitives.py +0 -672
- package/src/superlocalmemory/core/session_identity.py +0 -98
- package/src/superlocalmemory/core/shadow_router.py +0 -319
- package/src/superlocalmemory/core/slm_disabled.py +0 -85
- package/src/superlocalmemory/core/status_contract.py +0 -108
- package/src/superlocalmemory/core/store_pipeline.py +0 -1404
- package/src/superlocalmemory/core/summarizer.py +0 -200
- package/src/superlocalmemory/core/tier_manager.py +0 -461
- package/src/superlocalmemory/core/topic_signature.py +0 -156
- package/src/superlocalmemory/core/transactions/__init__.py +0 -78
- package/src/superlocalmemory/core/transactions/concrete_owners.py +0 -604
- package/src/superlocalmemory/core/transactions/erasure.py +0 -825
- package/src/superlocalmemory/core/transactions/manifest.py +0 -255
- package/src/superlocalmemory/core/transactions/manifest_key.py +0 -155
- package/src/superlocalmemory/core/transactions/obligations.py +0 -272
- package/src/superlocalmemory/core/transactions/owners.py +0 -114
- package/src/superlocalmemory/core/transactions/reconciler.py +0 -285
- package/src/superlocalmemory/core/transactions/service.py +0 -330
- package/src/superlocalmemory/core/worker_pool.py +0 -377
- package/src/superlocalmemory/core/working_memory.py +0 -288
- package/src/superlocalmemory/dynamics/__init__.py +0 -0
- package/src/superlocalmemory/dynamics/activation_guided_quantization.py +0 -374
- package/src/superlocalmemory/dynamics/eap_scheduler.py +0 -294
- package/src/superlocalmemory/dynamics/ebbinghaus_langevin_coupling.py +0 -171
- package/src/superlocalmemory/dynamics/fisher_langevin_coupling.py +0 -227
- package/src/superlocalmemory/encoding/__init__.py +0 -0
- package/src/superlocalmemory/encoding/auto_linker.py +0 -308
- package/src/superlocalmemory/encoding/cognitive_consolidator.py +0 -899
- package/src/superlocalmemory/encoding/consolidator.py +0 -472
- package/src/superlocalmemory/encoding/context_generator.py +0 -175
- package/src/superlocalmemory/encoding/emotional.py +0 -189
- package/src/superlocalmemory/encoding/entity_reflexion.py +0 -200
- package/src/superlocalmemory/encoding/entity_resolver.py +0 -687
- package/src/superlocalmemory/encoding/entropy_gate.py +0 -101
- package/src/superlocalmemory/encoding/fact_extractor.py +0 -877
- package/src/superlocalmemory/encoding/foresight.py +0 -93
- package/src/superlocalmemory/encoding/graph_builder.py +0 -346
- package/src/superlocalmemory/encoding/observation_builder.py +0 -177
- package/src/superlocalmemory/encoding/prospective_markers.py +0 -262
- package/src/superlocalmemory/encoding/scene_builder.py +0 -410
- package/src/superlocalmemory/encoding/signal_inference.py +0 -90
- package/src/superlocalmemory/encoding/temporal_parser.py +0 -432
- package/src/superlocalmemory/encoding/temporal_validator.py +0 -572
- package/src/superlocalmemory/encoding/type_router.py +0 -237
- package/src/superlocalmemory/evaluation/__init__.py +0 -13
- package/src/superlocalmemory/evaluation/calibration.py +0 -308
- package/src/superlocalmemory/evolution/__init__.py +0 -29
- package/src/superlocalmemory/evolution/blind_verifier.py +0 -122
- package/src/superlocalmemory/evolution/budget.py +0 -356
- package/src/superlocalmemory/evolution/evolution_store.py +0 -619
- package/src/superlocalmemory/evolution/llm_dispatch.py +0 -559
- package/src/superlocalmemory/evolution/model_selection.py +0 -175
- package/src/superlocalmemory/evolution/mutation_generator.py +0 -226
- package/src/superlocalmemory/evolution/skill_activator.py +0 -270
- package/src/superlocalmemory/evolution/skill_evolver.py +0 -928
- package/src/superlocalmemory/evolution/triggers.py +0 -376
- package/src/superlocalmemory/evolution/types.py +0 -114
- package/src/superlocalmemory/graph/__init__.py +0 -9
- package/src/superlocalmemory/graph/cozo_adjacency.py +0 -122
- package/src/superlocalmemory/graph/cozo_backend.py +0 -751
- package/src/superlocalmemory/hooks/__init__.py +0 -3
- package/src/superlocalmemory/hooks/_outcome_common.py +0 -523
- package/src/superlocalmemory/hooks/adapter_base.py +0 -347
- package/src/superlocalmemory/hooks/antigravity_adapter.py +0 -171
- package/src/superlocalmemory/hooks/auto_capture.py +0 -133
- package/src/superlocalmemory/hooks/auto_invoker.py +0 -521
- package/src/superlocalmemory/hooks/auto_parameterize.py +0 -147
- package/src/superlocalmemory/hooks/auto_recall.py +0 -191
- package/src/superlocalmemory/hooks/auto_recall_hook.py +0 -251
- package/src/superlocalmemory/hooks/before_web_hook.py +0 -131
- package/src/superlocalmemory/hooks/claude_code_hooks.py +0 -637
- package/src/superlocalmemory/hooks/codex_assets.py +0 -251
- package/src/superlocalmemory/hooks/codex_hooks.py +0 -186
- package/src/superlocalmemory/hooks/context_payload.py +0 -311
- package/src/superlocalmemory/hooks/copilot_adapter.py +0 -208
- package/src/superlocalmemory/hooks/cross_platform_connector.py +0 -90
- package/src/superlocalmemory/hooks/cursor_adapter.py +0 -173
- package/src/superlocalmemory/hooks/hook_daemon.py +0 -295
- package/src/superlocalmemory/hooks/hook_handlers.py +0 -822
- package/src/superlocalmemory/hooks/ide_connector.py +0 -246
- package/src/superlocalmemory/hooks/memory_protocol.py +0 -158
- package/src/superlocalmemory/hooks/portable_kit.py +0 -755
- package/src/superlocalmemory/hooks/post_tool_async_hook.py +0 -183
- package/src/superlocalmemory/hooks/post_tool_outcome_hook.py +0 -351
- package/src/superlocalmemory/hooks/prewarm_auth.py +0 -187
- package/src/superlocalmemory/hooks/rules_engine.py +0 -99
- package/src/superlocalmemory/hooks/session_registry.py +0 -330
- package/src/superlocalmemory/hooks/stop_outcome_hook.py +0 -138
- package/src/superlocalmemory/hooks/sync_loop.py +0 -114
- package/src/superlocalmemory/hooks/topic_shift_hook.py +0 -302
- package/src/superlocalmemory/hooks/user_prompt_hook.py +0 -131
- package/src/superlocalmemory/hooks/user_prompt_rehash_hook.py +0 -210
- package/src/superlocalmemory/infra/__init__.py +0 -3
- package/src/superlocalmemory/infra/auth_middleware.py +0 -145
- package/src/superlocalmemory/infra/backup.py +0 -974
- package/src/superlocalmemory/infra/backup_obligations.py +0 -423
- package/src/superlocalmemory/infra/cache_manager.py +0 -267
- package/src/superlocalmemory/infra/cloud_backup.py +0 -788
- package/src/superlocalmemory/infra/daemon_identity.py +0 -300
- package/src/superlocalmemory/infra/data_root.py +0 -238
- package/src/superlocalmemory/infra/event_bus.py +0 -637
- package/src/superlocalmemory/infra/local_diagnostics.py +0 -327
- package/src/superlocalmemory/infra/pid_manager.py +0 -193
- package/src/superlocalmemory/infra/process_identity.py +0 -180
- package/src/superlocalmemory/infra/process_reaper.py +0 -624
- package/src/superlocalmemory/infra/rate_limiter.py +0 -228
- package/src/superlocalmemory/infra/self_heal.py +0 -401
- package/src/superlocalmemory/infra/version_integrity.py +0 -229
- package/src/superlocalmemory/ingestion/__init__.py +0 -13
- package/src/superlocalmemory/ingestion/adapter_manager.py +0 -255
- package/src/superlocalmemory/ingestion/base_adapter.py +0 -171
- package/src/superlocalmemory/ingestion/calendar_adapter.py +0 -349
- package/src/superlocalmemory/ingestion/credentials.py +0 -125
- package/src/superlocalmemory/ingestion/gmail_adapter.py +0 -378
- package/src/superlocalmemory/ingestion/parsers.py +0 -100
- package/src/superlocalmemory/ingestion/transcript_adapter.py +0 -161
- package/src/superlocalmemory/integrations/__init__.py +0 -1
- package/src/superlocalmemory/integrations/bounded_loops_mcp.py +0 -431
- package/src/superlocalmemory/integrations/bounded_loops_v051.py +0 -236
- package/src/superlocalmemory/learning/__init__.py +0 -0
- package/src/superlocalmemory/learning/adaptive.py +0 -172
- package/src/superlocalmemory/learning/arm_catalog.py +0 -97
- package/src/superlocalmemory/learning/assertion_miner.py +0 -403
- package/src/superlocalmemory/learning/bandit.py +0 -654
- package/src/superlocalmemory/learning/bandit_cache.py +0 -131
- package/src/superlocalmemory/learning/behavioral.py +0 -542
- package/src/superlocalmemory/learning/bootstrap.py +0 -298
- package/src/superlocalmemory/learning/consolidation_cycle.py +0 -398
- package/src/superlocalmemory/learning/consolidation_quantization_worker.py +0 -115
- package/src/superlocalmemory/learning/consolidation_worker.py +0 -261
- package/src/superlocalmemory/learning/cross_project.py +0 -408
- package/src/superlocalmemory/learning/database.py +0 -698
- package/src/superlocalmemory/learning/dedup_hnsw.py +0 -413
- package/src/superlocalmemory/learning/engagement.py +0 -487
- package/src/superlocalmemory/learning/engagement_features.py +0 -279
- package/src/superlocalmemory/learning/ensemble.py +0 -309
- package/src/superlocalmemory/learning/entity_compiler.py +0 -356
- package/src/superlocalmemory/learning/fact_outcome_joins.py +0 -207
- package/src/superlocalmemory/learning/features.py +0 -138
- package/src/superlocalmemory/learning/feedback.py +0 -724
- package/src/superlocalmemory/learning/forgetting_scheduler.py +0 -375
- package/src/superlocalmemory/learning/hnsw_dedup.py +0 -69
- package/src/superlocalmemory/learning/labeler.py +0 -85
- package/src/superlocalmemory/learning/legacy_migration.py +0 -316
- package/src/superlocalmemory/learning/lightgbm_subprocess.py +0 -236
- package/src/superlocalmemory/learning/memory_merge.py +0 -175
- package/src/superlocalmemory/learning/model_cache.py +0 -267
- package/src/superlocalmemory/learning/model_rollback.py +0 -281
- package/src/superlocalmemory/learning/outcome_queue.py +0 -306
- package/src/superlocalmemory/learning/outcomes.py +0 -286
- package/src/superlocalmemory/learning/pattern_miner.py +0 -465
- package/src/superlocalmemory/learning/pattern_miner_constants.py +0 -90
- package/src/superlocalmemory/learning/pcos.py +0 -291
- package/src/superlocalmemory/learning/project_context.py +0 -366
- package/src/superlocalmemory/learning/propensity.py +0 -131
- package/src/superlocalmemory/learning/ranker.py +0 -300
- package/src/superlocalmemory/learning/ranker_common.py +0 -163
- package/src/superlocalmemory/learning/ranker_retrain_legacy.py +0 -210
- package/src/superlocalmemory/learning/ranker_retrain_online.py +0 -423
- package/src/superlocalmemory/learning/reward.py +0 -888
- package/src/superlocalmemory/learning/reward_archive.py +0 -223
- package/src/superlocalmemory/learning/reward_boost.py +0 -211
- package/src/superlocalmemory/learning/reward_from_outcomes.py +0 -365
- package/src/superlocalmemory/learning/reward_model.py +0 -144
- package/src/superlocalmemory/learning/reward_proxy.py +0 -578
- package/src/superlocalmemory/learning/shadow_test.py +0 -524
- package/src/superlocalmemory/learning/signal_kinds.py +0 -79
- package/src/superlocalmemory/learning/signal_worker.py +0 -268
- package/src/superlocalmemory/learning/signals.py +0 -646
- package/src/superlocalmemory/learning/skill_performance_miner.py +0 -422
- package/src/superlocalmemory/learning/source_quality.py +0 -828
- package/src/superlocalmemory/learning/trigram_index.py +0 -548
- package/src/superlocalmemory/learning/workflows.py +0 -309
- package/src/superlocalmemory/llm/__init__.py +0 -0
- package/src/superlocalmemory/llm/backbone.py +0 -364
- package/src/superlocalmemory/loops/__init__.py +0 -58
- package/src/superlocalmemory/loops/budget.py +0 -58
- package/src/superlocalmemory/loops/engine.py +0 -174
- package/src/superlocalmemory/loops/ledger.py +0 -298
- package/src/superlocalmemory/loops/models.py +0 -152
- package/src/superlocalmemory/loops/rules.py +0 -52
- package/src/superlocalmemory/math/__init__.py +0 -0
- package/src/superlocalmemory/math/ebbinghaus.py +0 -352
- package/src/superlocalmemory/math/fisher.py +0 -356
- package/src/superlocalmemory/math/fisher_quantized.py +0 -255
- package/src/superlocalmemory/math/hopfield.py +0 -282
- package/src/superlocalmemory/math/langevin.py +0 -411
- package/src/superlocalmemory/math/polar_quant.py +0 -414
- package/src/superlocalmemory/math/qjl.py +0 -115
- package/src/superlocalmemory/math/sheaf.py +0 -261
- package/src/superlocalmemory/math/turbo_quant.py +0 -318
- package/src/superlocalmemory/mcp/__init__.py +0 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +0 -203
- package/src/superlocalmemory/mcp/_pool_adapter.py +0 -181
- package/src/superlocalmemory/mcp/_stdin_guard.py +0 -60
- package/src/superlocalmemory/mcp/agent_context.py +0 -115
- package/src/superlocalmemory/mcp/cli_fallback.py +0 -602
- package/src/superlocalmemory/mcp/http_transport.py +0 -85
- package/src/superlocalmemory/mcp/profiles.py +0 -154
- package/src/superlocalmemory/mcp/resources.py +0 -281
- package/src/superlocalmemory/mcp/server.py +0 -480
- package/src/superlocalmemory/mcp/session_binding.py +0 -98
- package/src/superlocalmemory/mcp/shared.py +0 -112
- package/src/superlocalmemory/mcp/tools.py +0 -18
- package/src/superlocalmemory/mcp/tools_active.py +0 -958
- package/src/superlocalmemory/mcp/tools_brain.py +0 -298
- package/src/superlocalmemory/mcp/tools_code_graph.py +0 -1717
- package/src/superlocalmemory/mcp/tools_context.py +0 -239
- package/src/superlocalmemory/mcp/tools_core.py +0 -1112
- package/src/superlocalmemory/mcp/tools_evolution.py +0 -343
- package/src/superlocalmemory/mcp/tools_learning.py +0 -393
- package/src/superlocalmemory/mcp/tools_loops.py +0 -345
- package/src/superlocalmemory/mcp/tools_mesh.py +0 -429
- package/src/superlocalmemory/mcp/tools_ops.py +0 -115
- package/src/superlocalmemory/mcp/tools_optimize.py +0 -322
- package/src/superlocalmemory/mcp/tools_summaries.py +0 -147
- package/src/superlocalmemory/mcp/tools_v28.py +0 -292
- package/src/superlocalmemory/mcp/tools_v3.py +0 -398
- package/src/superlocalmemory/mcp/tools_v33.py +0 -507
- package/src/superlocalmemory/mesh/__init__.py +0 -12
- package/src/superlocalmemory/mesh/broker.py +0 -812
- package/src/superlocalmemory/mesh/broker_security.py +0 -470
- package/src/superlocalmemory/mesh/discovery.py +0 -365
- package/src/superlocalmemory/mesh/lock_protocol.py +0 -313
- package/src/superlocalmemory/mesh/node_identity.py +0 -97
- package/src/superlocalmemory/mesh/outbox_remote.py +0 -429
- package/src/superlocalmemory/mesh/remote_sync.py +0 -829
- package/src/superlocalmemory/mesh/state_sync.py +0 -286
- package/src/superlocalmemory/migrations/__init__.py +0 -5
- package/src/superlocalmemory/migrations/v3_4_25_to_v3_4_26.py +0 -144
- package/src/superlocalmemory/optimize/NOTICE +0 -6
- package/src/superlocalmemory/optimize/__init__.py +0 -0
- package/src/superlocalmemory/optimize/adapters/__init__.py +0 -68
- package/src/superlocalmemory/optimize/adapters/_agent_registry.py +0 -120
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +0 -112
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +0 -122
- package/src/superlocalmemory/optimize/adapters/wrap.py +0 -228
- package/src/superlocalmemory/optimize/cache/__init__.py +0 -31
- package/src/superlocalmemory/optimize/cache/boundary_store.py +0 -488
- package/src/superlocalmemory/optimize/cache/centroid_store.py +0 -199
- package/src/superlocalmemory/optimize/cache/context_key.py +0 -67
- package/src/superlocalmemory/optimize/cache/exact.py +0 -88
- package/src/superlocalmemory/optimize/cache/invalidation.py +0 -36
- package/src/superlocalmemory/optimize/cache/key_builder.py +0 -111
- package/src/superlocalmemory/optimize/cache/manager.py +0 -737
- package/src/superlocalmemory/optimize/cache/semantic.py +0 -637
- package/src/superlocalmemory/optimize/cache/stampede.py +0 -50
- package/src/superlocalmemory/optimize/compress/__init__.py +0 -17
- package/src/superlocalmemory/optimize/compress/align.py +0 -159
- package/src/superlocalmemory/optimize/compress/ccr.py +0 -116
- package/src/superlocalmemory/optimize/compress/prose_llmlingua.py +0 -71
- package/src/superlocalmemory/optimize/compress/router.py +0 -667
- package/src/superlocalmemory/optimize/config/__init__.py +0 -56
- package/src/superlocalmemory/optimize/config/defaults.py +0 -43
- package/src/superlocalmemory/optimize/config/schema.py +0 -327
- package/src/superlocalmemory/optimize/config/store.py +0 -270
- package/src/superlocalmemory/optimize/metrics/__init__.py +0 -8
- package/src/superlocalmemory/optimize/metrics/counters.py +0 -155
- package/src/superlocalmemory/optimize/metrics/estimator.py +0 -87
- package/src/superlocalmemory/optimize/metrics/exporters.py +0 -77
- package/src/superlocalmemory/optimize/metrics/persistence.py +0 -115
- package/src/superlocalmemory/optimize/proxy/__init__.py +0 -28
- package/src/superlocalmemory/optimize/proxy/_helpers.py +0 -730
- package/src/superlocalmemory/optimize/proxy/anthropic_surface.py +0 -375
- package/src/superlocalmemory/optimize/proxy/capture.py +0 -550
- package/src/superlocalmemory/optimize/proxy/gemini_surface.py +0 -528
- package/src/superlocalmemory/optimize/proxy/lifecycle.py +0 -126
- package/src/superlocalmemory/optimize/proxy/openai_surface.py +0 -465
- package/src/superlocalmemory/optimize/proxy/server.py +0 -199
- package/src/superlocalmemory/optimize/proxy/vertex_surface.py +0 -246
- package/src/superlocalmemory/optimize/storage/__init__.py +0 -0
- package/src/superlocalmemory/optimize/storage/db.py +0 -1185
- package/src/superlocalmemory/optimize/storage/schema.py +0 -205
- package/src/superlocalmemory/parameterization/__init__.py +0 -47
- package/src/superlocalmemory/parameterization/cross_project.py +0 -12
- package/src/superlocalmemory/parameterization/pattern_extractor.py +0 -584
- package/src/superlocalmemory/parameterization/pii_filter.py +0 -106
- package/src/superlocalmemory/parameterization/prompt_injector.py +0 -219
- package/src/superlocalmemory/parameterization/prompt_lifecycle.py +0 -281
- package/src/superlocalmemory/parameterization/soft_prompt_generator.py +0 -542
- package/src/superlocalmemory/parameterization/workflow_miner.py +0 -17
- package/src/superlocalmemory/reliability/__init__.py +0 -45
- package/src/superlocalmemory/reliability/join_liveness.py +0 -301
- package/src/superlocalmemory/reliability/prior_distance.py +0 -243
- package/src/superlocalmemory/retrieval/__init__.py +0 -0
- package/src/superlocalmemory/retrieval/agentic.py +0 -367
- package/src/superlocalmemory/retrieval/ann_index.py +0 -235
- package/src/superlocalmemory/retrieval/bm25_channel.py +0 -451
- package/src/superlocalmemory/retrieval/bridge_discovery.py +0 -253
- package/src/superlocalmemory/retrieval/channel_registry.py +0 -154
- package/src/superlocalmemory/retrieval/channel_status.py +0 -117
- package/src/superlocalmemory/retrieval/engine.py +0 -1615
- package/src/superlocalmemory/retrieval/entity_channel.py +0 -994
- package/src/superlocalmemory/retrieval/forgetting_filter.py +0 -160
- package/src/superlocalmemory/retrieval/fusion.py +0 -81
- package/src/superlocalmemory/retrieval/graph_adjacency.py +0 -219
- package/src/superlocalmemory/retrieval/hopfield_channel.py +0 -465
- package/src/superlocalmemory/retrieval/profile_channel.py +0 -105
- package/src/superlocalmemory/retrieval/quantization_aware_search.py +0 -147
- package/src/superlocalmemory/retrieval/remote_reranker.py +0 -758
- package/src/superlocalmemory/retrieval/reranker.py +0 -674
- package/src/superlocalmemory/retrieval/scope_policy.py +0 -126
- package/src/superlocalmemory/retrieval/semantic_channel.py +0 -638
- package/src/superlocalmemory/retrieval/spreading.py +0 -288
- package/src/superlocalmemory/retrieval/spreading_activation.py +0 -616
- package/src/superlocalmemory/retrieval/strategy.py +0 -248
- package/src/superlocalmemory/retrieval/temporal_channel.py +0 -433
- package/src/superlocalmemory/retrieval/temporal_frame.py +0 -102
- package/src/superlocalmemory/retrieval/temporal_utils.py +0 -122
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +0 -499
- package/src/superlocalmemory/retrieval/time_window.py +0 -181
- package/src/superlocalmemory/retrieval/vector_store.py +0 -863
- package/src/superlocalmemory/server/__init__.py +0 -1
- package/src/superlocalmemory/server/api.py +0 -310
- package/src/superlocalmemory/server/asset_versions.py +0 -171
- package/src/superlocalmemory/server/bandit_loops.py +0 -158
- package/src/superlocalmemory/server/config_file.py +0 -90
- package/src/superlocalmemory/server/consolidation_runner.py +0 -140
- package/src/superlocalmemory/server/egress_policy.py +0 -258
- package/src/superlocalmemory/server/loopback.py +0 -85
- package/src/superlocalmemory/server/middleware/__init__.py +0 -11
- package/src/superlocalmemory/server/middleware/security_headers.py +0 -144
- package/src/superlocalmemory/server/origin.py +0 -55
- package/src/superlocalmemory/server/profile_runtime.py +0 -515
- package/src/superlocalmemory/server/rbac_enforce.py +0 -194
- package/src/superlocalmemory/server/recall_health.py +0 -343
- package/src/superlocalmemory/server/recall_serializer.py +0 -320
- package/src/superlocalmemory/server/route_mutations.py +0 -104
- package/src/superlocalmemory/server/routes/__init__.py +0 -4
- package/src/superlocalmemory/server/routes/abstraction.py +0 -314
- package/src/superlocalmemory/server/routes/adapters.py +0 -63
- package/src/superlocalmemory/server/routes/agents.py +0 -303
- package/src/superlocalmemory/server/routes/backup.py +0 -869
- package/src/superlocalmemory/server/routes/behavioral.py +0 -659
- package/src/superlocalmemory/server/routes/brain.py +0 -1892
- package/src/superlocalmemory/server/routes/chat.py +0 -393
- package/src/superlocalmemory/server/routes/compliance.py +0 -533
- package/src/superlocalmemory/server/routes/config_api.py +0 -703
- package/src/superlocalmemory/server/routes/data_io.py +0 -329
- package/src/superlocalmemory/server/routes/entity.py +0 -237
- package/src/superlocalmemory/server/routes/events.py +0 -214
- package/src/superlocalmemory/server/routes/evolution.py +0 -510
- package/src/superlocalmemory/server/routes/helpers.py +0 -499
- package/src/superlocalmemory/server/routes/ingest.py +0 -137
- package/src/superlocalmemory/server/routes/insights.py +0 -366
- package/src/superlocalmemory/server/routes/learning.py +0 -834
- package/src/superlocalmemory/server/routes/learning_telemetry.py +0 -154
- package/src/superlocalmemory/server/routes/lifecycle.py +0 -184
- package/src/superlocalmemory/server/routes/memories.py +0 -1661
- package/src/superlocalmemory/server/routes/mesh.py +0 -517
- package/src/superlocalmemory/server/routes/mesh_lock.py +0 -54
- package/src/superlocalmemory/server/routes/mesh_state.py +0 -63
- package/src/superlocalmemory/server/routes/optimize.py +0 -197
- package/src/superlocalmemory/server/routes/prewarm.py +0 -173
- package/src/superlocalmemory/server/routes/profiles.py +0 -292
- package/src/superlocalmemory/server/routes/ratelimit.py +0 -132
- package/src/superlocalmemory/server/routes/rbac.py +0 -366
- package/src/superlocalmemory/server/routes/stats.py +0 -385
- package/src/superlocalmemory/server/routes/tiers.py +0 -222
- package/src/superlocalmemory/server/routes/timeline.py +0 -258
- package/src/superlocalmemory/server/routes/token.py +0 -90
- package/src/superlocalmemory/server/routes/v3_api.py +0 -3023
- package/src/superlocalmemory/server/routes/ws.py +0 -171
- package/src/superlocalmemory/server/security_middleware.py +0 -89
- package/src/superlocalmemory/server/ui.py +0 -354
- package/src/superlocalmemory/server/unified_daemon.py +0 -6326
- package/src/superlocalmemory/server/write_identity.py +0 -195
- package/src/superlocalmemory/storage/__init__.py +0 -0
- package/src/superlocalmemory/storage/_migration_internals.py +0 -638
- package/src/superlocalmemory/storage/_schema_version.py +0 -174
- package/src/superlocalmemory/storage/access_log.py +0 -170
- package/src/superlocalmemory/storage/admission_codec.py +0 -129
- package/src/superlocalmemory/storage/admission_journal.py +0 -843
- package/src/superlocalmemory/storage/agent_experience.py +0 -546
- package/src/superlocalmemory/storage/backup.py +0 -531
- package/src/superlocalmemory/storage/correction_cases.py +0 -670
- package/src/superlocalmemory/storage/database.py +0 -3180
- package/src/superlocalmemory/storage/deferred_writes.py +0 -209
- package/src/superlocalmemory/storage/embedding_codec.py +0 -200
- package/src/superlocalmemory/storage/embedding_migrator.py +0 -672
- package/src/superlocalmemory/storage/erasure_fence.py +0 -45
- package/src/superlocalmemory/storage/execution_learning.py +0 -285
- package/src/superlocalmemory/storage/external_evidence.py +0 -359
- package/src/superlocalmemory/storage/generation_fence.py +0 -63
- package/src/superlocalmemory/storage/lineage_retention.py +0 -236
- package/src/superlocalmemory/storage/logical_edges.py +0 -86
- package/src/superlocalmemory/storage/memory_write.py +0 -115
- package/src/superlocalmemory/storage/migration_runner.py +0 -895
- package/src/superlocalmemory/storage/migration_v33.py +0 -140
- package/src/superlocalmemory/storage/migrations/M001_add_signal_features_columns.py +0 -67
- package/src/superlocalmemory/storage/migrations/M002_model_state_history.py +0 -107
- package/src/superlocalmemory/storage/migrations/M003_migration_log.py +0 -38
- package/src/superlocalmemory/storage/migrations/M004_cross_platform_sync_log.py +0 -46
- package/src/superlocalmemory/storage/migrations/M005_bandit_tables.py +0 -75
- package/src/superlocalmemory/storage/migrations/M006_action_outcomes_reward.py +0 -75
- package/src/superlocalmemory/storage/migrations/M007_pending_outcomes.py +0 -63
- package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +0 -94
- package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +0 -80
- package/src/superlocalmemory/storage/migrations/M011_archive_and_merge.py +0 -87
- package/src/superlocalmemory/storage/migrations/M012_shadow_observations.py +0 -72
- package/src/superlocalmemory/storage/migrations/M013_bi_temporal_columns.py +0 -55
- package/src/superlocalmemory/storage/migrations/M014_v345_scale_ready.py +0 -45
- package/src/superlocalmemory/storage/migrations/M015_add_pinned_column.py +0 -58
- package/src/superlocalmemory/storage/migrations/M016_add_scope_support.py +0 -120
- package/src/superlocalmemory/storage/migrations/M017_ccq_scope_column.py +0 -79
- package/src/superlocalmemory/storage/migrations/M018_ingestion_operations.py +0 -120
- package/src/superlocalmemory/storage/migrations/M019_derivation_lineage.py +0 -54
- package/src/superlocalmemory/storage/migrations/M020_model_state_integrity.py +0 -52
- package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py +0 -108
- package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py +0 -86
- package/src/superlocalmemory/storage/migrations/M023_mesh_profile_isolation.py +0 -194
- package/src/superlocalmemory/storage/migrations/M024_rbac_users_roles.py +0 -87
- package/src/superlocalmemory/storage/migrations/M025_perf_indexes.py +0 -90
- package/src/superlocalmemory/storage/migrations/M026_rbac_memberships_fk.py +0 -136
- package/src/superlocalmemory/storage/migrations/M027_transferable_patterns_profile.py +0 -163
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +0 -305
- package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +0 -137
- package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +0 -93
- package/src/superlocalmemory/storage/migrations/M031_dead_letter_operations.py +0 -80
- package/src/superlocalmemory/storage/migrations/M032_write_coordinator_admission.py +0 -188
- package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +0 -148
- package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +0 -58
- package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +0 -113
- package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +0 -107
- package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +0 -162
- package/src/superlocalmemory/storage/migrations/M038_learning_feedback_channel.py +0 -77
- package/src/superlocalmemory/storage/migrations/M039_scene_fact_members.py +0 -137
- package/src/superlocalmemory/storage/migrations/M040_agent_experience_receipts.py +0 -254
- package/src/superlocalmemory/storage/migrations/M041_external_evidence_receipts.py +0 -189
- package/src/superlocalmemory/storage/migrations/M042_correction_case_ledger.py +0 -245
- package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +0 -512
- package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +0 -127
- package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +0 -158
- package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +0 -620
- package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +0 -306
- package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +0 -229
- package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +0 -201
- package/src/superlocalmemory/storage/migrations/M050_execution_learning_v2.py +0 -70
- package/src/superlocalmemory/storage/migrations/__init__.py +0 -103
- package/src/superlocalmemory/storage/migrations.py +0 -333
- package/src/superlocalmemory/storage/models.py +0 -500
- package/src/superlocalmemory/storage/projection_outbox.py +0 -346
- package/src/superlocalmemory/storage/quantized_store.py +0 -280
- package/src/superlocalmemory/storage/read_connection.py +0 -115
- package/src/superlocalmemory/storage/retention_policy.py +0 -860
- package/src/superlocalmemory/storage/schema.py +0 -1108
- package/src/superlocalmemory/storage/schema_code_graph.py +0 -282
- package/src/superlocalmemory/storage/schema_v32.py +0 -382
- package/src/superlocalmemory/storage/schema_v3410.py +0 -159
- package/src/superlocalmemory/storage/schema_v3411.py +0 -149
- package/src/superlocalmemory/storage/schema_v343.py +0 -315
- package/src/superlocalmemory/storage/schema_v345.py +0 -109
- package/src/superlocalmemory/storage/schema_v347.py +0 -140
- package/src/superlocalmemory/storage/sqlite_vectors.py +0 -169
- package/src/superlocalmemory/storage/v2_migrator.py +0 -466
- package/src/superlocalmemory/storage/write_coordinator.py +0 -949
- package/src/superlocalmemory/storage/write_lock.py +0 -88
- package/src/superlocalmemory/summaries/__init__.py +0 -37
- package/src/superlocalmemory/summaries/base.py +0 -267
- package/src/superlocalmemory/summaries/daily_reflection.py +0 -340
- package/src/superlocalmemory/summaries/non_answer.py +0 -223
- package/src/superlocalmemory/summaries/project_work_log.py +0 -440
- package/src/superlocalmemory/summaries/session_summary.py +0 -311
- package/src/superlocalmemory/trust/__init__.py +0 -0
- package/src/superlocalmemory/trust/gate.py +0 -171
- package/src/superlocalmemory/trust/provenance.py +0 -124
- package/src/superlocalmemory/trust/scorer.py +0 -413
- package/src/superlocalmemory/trust/signals.py +0 -153
- package/src/superlocalmemory/ui/assets/slm-icon-white.svg +0 -64
- package/src/superlocalmemory/ui/assets/slm-icon.svg +0 -36
- package/src/superlocalmemory/ui/css/brain.css +0 -409
- package/src/superlocalmemory/ui/css/design-system.css +0 -696
- package/src/superlocalmemory/ui/css/legacy-dashboard.css +0 -663
- package/src/superlocalmemory/ui/css/neural-glass.css +0 -1599
- package/src/superlocalmemory/ui/css/od-bridge.css +0 -158
- package/src/superlocalmemory/ui/favicon.svg +0 -36
- package/src/superlocalmemory/ui/index.html +0 -1638
- package/src/superlocalmemory/ui/js/agents.js +0 -192
- package/src/superlocalmemory/ui/js/auto-settings.js +0 -624
- package/src/superlocalmemory/ui/js/brain.js +0 -1400
- package/src/superlocalmemory/ui/js/clusters.js +0 -326
- package/src/superlocalmemory/ui/js/compliance.js +0 -307
- package/src/superlocalmemory/ui/js/core.js +0 -566
- package/src/superlocalmemory/ui/js/dashboard.js +0 -503
- package/src/superlocalmemory/ui/js/event-delegation.js +0 -113
- package/src/superlocalmemory/ui/js/events.js +0 -178
- package/src/superlocalmemory/ui/js/fact-detail.js +0 -142
- package/src/superlocalmemory/ui/js/feedback.js +0 -339
- package/src/superlocalmemory/ui/js/graph-event-bus.js +0 -83
- package/src/superlocalmemory/ui/js/graph-filters.js +0 -220
- package/src/superlocalmemory/ui/js/graph-ui.js +0 -214
- package/src/superlocalmemory/ui/js/ide-status.js +0 -115
- package/src/superlocalmemory/ui/js/init.js +0 -54
- package/src/superlocalmemory/ui/js/knowledge-graph.js +0 -945
- package/src/superlocalmemory/ui/js/lifecycle.js +0 -387
- package/src/superlocalmemory/ui/js/math-health.js +0 -114
- package/src/superlocalmemory/ui/js/memories.js +0 -394
- package/src/superlocalmemory/ui/js/memory-chat.js +0 -371
- package/src/superlocalmemory/ui/js/memory-timeline.js +0 -265
- package/src/superlocalmemory/ui/js/modal.js +0 -733
- package/src/superlocalmemory/ui/js/ng-entities.js +0 -298
- package/src/superlocalmemory/ui/js/ng-health.js +0 -208
- package/src/superlocalmemory/ui/js/ng-ingestion.js +0 -203
- package/src/superlocalmemory/ui/js/ng-mesh.js +0 -374
- package/src/superlocalmemory/ui/js/ng-shell.js +0 -524
- package/src/superlocalmemory/ui/js/ng-skills.js +0 -663
- package/src/superlocalmemory/ui/js/od-agents.js +0 -588
- package/src/superlocalmemory/ui/js/od-auth-gate.js +0 -257
- package/src/superlocalmemory/ui/js/od-backup.js +0 -878
- package/src/superlocalmemory/ui/js/od-boundedloops.js +0 -324
- package/src/superlocalmemory/ui/js/od-brain.js +0 -1095
- package/src/superlocalmemory/ui/js/od-compliance-ext.js +0 -301
- package/src/superlocalmemory/ui/js/od-components.js +0 -147
- package/src/superlocalmemory/ui/js/od-entities.js +0 -622
- package/src/superlocalmemory/ui/js/od-graph.js +0 -776
- package/src/superlocalmemory/ui/js/od-health.js +0 -579
- package/src/superlocalmemory/ui/js/od-mcp.js +0 -508
- package/src/superlocalmemory/ui/js/od-memories.js +0 -1499
- package/src/superlocalmemory/ui/js/od-mesh.js +0 -645
- package/src/superlocalmemory/ui/js/od-operations.js +0 -1268
- package/src/superlocalmemory/ui/js/od-ops-health.js +0 -417
- package/src/superlocalmemory/ui/js/od-optimize.js +0 -828
- package/src/superlocalmemory/ui/js/od-settings.js +0 -1275
- package/src/superlocalmemory/ui/js/od-shell.js +0 -819
- package/src/superlocalmemory/ui/js/od-skills.js +0 -600
- package/src/superlocalmemory/ui/js/od-team.js +0 -265
- package/src/superlocalmemory/ui/js/optimize.js +0 -191
- package/src/superlocalmemory/ui/js/profiles.js +0 -362
- package/src/superlocalmemory/ui/js/quick-actions.js +0 -334
- package/src/superlocalmemory/ui/js/recall-lab.js +0 -373
- package/src/superlocalmemory/ui/js/search.js +0 -86
- package/src/superlocalmemory/ui/js/settings.js +0 -556
- package/src/superlocalmemory/ui/js/timeline.js +0 -62
- package/src/superlocalmemory/ui/js/trust-dashboard.js +0 -225
- package/src/superlocalmemory/ui/vendor/bootstrap-icons/bootstrap-icons.css +0 -2018
- package/src/superlocalmemory/ui/vendor/bootstrap-icons/fonts/bootstrap-icons.woff +0 -0
- package/src/superlocalmemory/ui/vendor/bootstrap-icons/fonts/bootstrap-icons.woff2 +0 -0
- package/src/superlocalmemory/ui/vendor/bootstrap.bundle.min.js +0 -7
- package/src/superlocalmemory/ui/vendor/bootstrap.min.css +0 -6
- package/src/superlocalmemory/ui/vendor/d3.v7.min.js +0 -2
- package/src/superlocalmemory/ui/vendor/graphology-library.min.js +0 -2
- package/src/superlocalmemory/ui/vendor/graphology.umd.min.js +0 -2
- package/src/superlocalmemory/ui/vendor/inter-ui/inter-variable.min.css +0 -8
- package/src/superlocalmemory/ui/vendor/inter-ui/variable/InterVariable-Italic.woff2 +0 -0
- package/src/superlocalmemory/ui/vendor/inter-ui/variable/InterVariable.woff2 +0 -0
- package/src/superlocalmemory/ui/vendor/sigma.min.js +0 -1
- package/src/superlocalmemory/vector/__init__.py +0 -9
- package/src/superlocalmemory/vector/lancedb_backend.py +0 -366
|
@@ -1,3180 +0,0 @@
|
|
|
1
|
-
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
-
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
-
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
-
|
|
5
|
-
"""SuperLocalMemory V3 — Database Manager.
|
|
6
|
-
|
|
7
|
-
SQLite with WAL, profile-scoped CRUD, FTS5 search, BM25 persistence.
|
|
8
|
-
Concurrent-safe: WAL mode + busy_timeout + retry on SQLITE_BUSY.
|
|
9
|
-
Multiple processes (MCP, CLI, integrations) can read/write safely.
|
|
10
|
-
|
|
11
|
-
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
12
|
-
"""
|
|
13
|
-
from __future__ import annotations
|
|
14
|
-
|
|
15
|
-
import json
|
|
16
|
-
import logging
|
|
17
|
-
import os
|
|
18
|
-
import platform
|
|
19
|
-
import sqlite3
|
|
20
|
-
import threading
|
|
21
|
-
import time
|
|
22
|
-
from contextlib import contextmanager
|
|
23
|
-
from pathlib import Path
|
|
24
|
-
from types import ModuleType
|
|
25
|
-
from typing import Any, Generator, NoReturn
|
|
26
|
-
|
|
27
|
-
from superlocalmemory.storage.models import (
|
|
28
|
-
AtomicFact,
|
|
29
|
-
CanonicalEntity,
|
|
30
|
-
ConsolidationAction,
|
|
31
|
-
EdgeType,
|
|
32
|
-
EntityAlias,
|
|
33
|
-
EntityProfile,
|
|
34
|
-
FactType,
|
|
35
|
-
GraphEdge,
|
|
36
|
-
MemoryLifecycle,
|
|
37
|
-
MemoryRecord,
|
|
38
|
-
MemoryScene,
|
|
39
|
-
SignalType,
|
|
40
|
-
TemporalEvent,
|
|
41
|
-
TrustScore,
|
|
42
|
-
)
|
|
43
|
-
from superlocalmemory.storage.embedding_codec import (
|
|
44
|
-
decode_embedding,
|
|
45
|
-
decode_float_vector,
|
|
46
|
-
encode_embedding,
|
|
47
|
-
encode_float_vector,
|
|
48
|
-
)
|
|
49
|
-
from superlocalmemory.storage.write_lock import get_write_lock
|
|
50
|
-
from superlocalmemory.storage import projection_outbox
|
|
51
|
-
|
|
52
|
-
logger = logging.getLogger(__name__)
|
|
53
|
-
|
|
54
|
-
_MISSING = object()
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
class ProfileOwnershipConflict(ValueError):
|
|
58
|
-
"""A store tried to hand one profile's existing row to another profile.
|
|
59
|
-
|
|
60
|
-
``memories.memory_id`` and ``atomic_facts.fact_id`` are bare
|
|
61
|
-
``TEXT PRIMARY KEY`` — global, not per-profile — while every row carries a
|
|
62
|
-
``profile_id``. So two profiles on one store can be handed the same
|
|
63
|
-
caller-chosen id (an importer keyed on an external record id syncing one
|
|
64
|
-
source into two workspaces does exactly this), and the upsert would
|
|
65
|
-
otherwise rewrite the owner and the content: the first profile's memory
|
|
66
|
-
silently became the second's.
|
|
67
|
-
"""
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
def _jl(raw: Any, default: Any = _MISSING) -> Any:
|
|
71
|
-
"""JSON-load a value, returning *default* on None/empty.
|
|
72
|
-
|
|
73
|
-
_jl(raw) -> [] when raw is None/empty (list fields)
|
|
74
|
-
_jl(raw, None) -> None when raw is None/empty (optional fields)
|
|
75
|
-
"""
|
|
76
|
-
if raw is None or raw == "":
|
|
77
|
-
return [] if default is _MISSING else default
|
|
78
|
-
return json.loads(raw)
|
|
79
|
-
|
|
80
|
-
def _jd(val: Any) -> str | None:
|
|
81
|
-
"""JSON-dump a list/dict, or return None."""
|
|
82
|
-
return json.dumps(val) if val is not None else None
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
def _env_int(name: str, default: int) -> int:
|
|
86
|
-
"""Read a positive int from the environment, falling back on bad/absent."""
|
|
87
|
-
try:
|
|
88
|
-
val = int(os.environ.get(name, "").strip())
|
|
89
|
-
return val if val > 0 else default
|
|
90
|
-
except (ValueError, AttributeError):
|
|
91
|
-
return default
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
def _env_float(name: str, default: float) -> float:
|
|
95
|
-
"""Read a positive float from the environment, falling back on bad/absent."""
|
|
96
|
-
try:
|
|
97
|
-
val = float(os.environ.get(name, "").strip())
|
|
98
|
-
return val if val > 0 else default
|
|
99
|
-
except (ValueError, AttributeError):
|
|
100
|
-
return default
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
# SQLite endurance tuning. Defaults preserve prior hard-coded behaviour exactly;
|
|
104
|
-
# operators on slow/contended I/O can raise them via env (issue #53) without a
|
|
105
|
-
# code change. Unset env => byte-identical to the previous constants.
|
|
106
|
-
_BUSY_TIMEOUT_MS = _env_int("SLM_DB_BUSY_TIMEOUT_MS", 10_000) # wait for writers
|
|
107
|
-
_MAX_RETRIES = _env_int("SLM_DB_MAX_RETRIES", 5) # retry on SQLITE_BUSY
|
|
108
|
-
_RETRY_BASE_DELAY = _env_float("SLM_DB_RETRY_BASE_DELAY", 0.1) # backoff base (s)
|
|
109
|
-
|
|
110
|
-
# Warn once per process, not once per connection, when the WAL close-path
|
|
111
|
-
# deadlock guard cannot be installed (Python < 3.12).
|
|
112
|
-
_NO_CKPT_WARNED = False
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def _unbounded_facts_ceiling() -> int:
|
|
116
|
-
"""Hard upper bound applied when a fact fetch is called with limit=None, so
|
|
117
|
-
a large tenant can never materialize the whole table into memory. Tunable
|
|
118
|
-
via SLM_MAX_FACTS_UNBOUNDED (default 50_000)."""
|
|
119
|
-
import os as _os
|
|
120
|
-
try:
|
|
121
|
-
return max(1, int(_os.environ.get("SLM_MAX_FACTS_UNBOUNDED", "50000")))
|
|
122
|
-
except (TypeError, ValueError):
|
|
123
|
-
return 50000
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
def _scope_where(
|
|
127
|
-
profile_id: str,
|
|
128
|
-
*,
|
|
129
|
-
include_global: bool = False,
|
|
130
|
-
include_shared: bool = False,
|
|
131
|
-
prefix: str = "",
|
|
132
|
-
) -> tuple[str, list]:
|
|
133
|
-
"""Build scope-filtering WHERE clause for multi-scope retrieval.
|
|
134
|
-
|
|
135
|
-
Returns ``(where_clause, params)`` for splicing into SQL queries.
|
|
136
|
-
|
|
137
|
-
When ``include_global=True``, facts with ``scope='global'`` are included
|
|
138
|
-
regardless of profile. When ``include_shared=True``, facts explicitly
|
|
139
|
-
shared with this profile (via ``shared_with`` JSON array) are also
|
|
140
|
-
included.
|
|
141
|
-
|
|
142
|
-
v3.6.15: defaults are SHARED-OFF (include_global/include_shared=False) so
|
|
143
|
-
any DIRECT caller (search, list_recent, fetch, resources) is private by
|
|
144
|
-
default — shared memory is opt-in. The recall channels pass explicit
|
|
145
|
-
resolved flags, so opt-in recall is unaffected. With both False the clause
|
|
146
|
-
collapses to ``profile_id = ?`` — identical to 3.6.14 isolation.
|
|
147
|
-
"""
|
|
148
|
-
table = f"{prefix}." if prefix else ""
|
|
149
|
-
clauses = [f"({table}profile_id = ?)"]
|
|
150
|
-
params: list = [profile_id]
|
|
151
|
-
|
|
152
|
-
if include_global:
|
|
153
|
-
clauses.append(f"({table}scope = 'global')")
|
|
154
|
-
|
|
155
|
-
if include_shared:
|
|
156
|
-
# Match the profile_id as a quoted JSON-array element. ESCAPE the LIKE
|
|
157
|
-
# metacharacters in profile_id so a profile id containing % or _ cannot
|
|
158
|
-
# false-positive-match another profile's shared_with list.
|
|
159
|
-
_esc = profile_id.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
160
|
-
clauses.append(
|
|
161
|
-
f"({table}scope = 'shared' AND {table}shared_with LIKE ? ESCAPE '\\')"
|
|
162
|
-
)
|
|
163
|
-
params.append(f'%"{_esc}"%')
|
|
164
|
-
|
|
165
|
-
where = "(" + " OR ".join(clauses) + ")"
|
|
166
|
-
return where, params
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
def _compose_visible_clause(
|
|
170
|
-
prefix: str,
|
|
171
|
-
*,
|
|
172
|
-
has_archive: bool,
|
|
173
|
-
has_quarantine: bool,
|
|
174
|
-
include_quarantined: bool = False,
|
|
175
|
-
) -> str:
|
|
176
|
-
"""Build the AND-clause from what the store actually has.
|
|
177
|
-
|
|
178
|
-
Split out from ``DatabaseManager.visible_fact_clause`` so that callers
|
|
179
|
-
holding a bare ``sqlite3.Connection`` produce a byte-identical predicate
|
|
180
|
-
rather than a second, drifting copy.
|
|
181
|
-
"""
|
|
182
|
-
table = f"{prefix}." if prefix else ""
|
|
183
|
-
clause = ""
|
|
184
|
-
if has_archive:
|
|
185
|
-
clause += f" AND COALESCE({table}archive_status, 'live') != 'archived'"
|
|
186
|
-
if not include_quarantined and has_quarantine:
|
|
187
|
-
clause += f" AND COALESCE({table}quarantined, 0) = 0"
|
|
188
|
-
return clause
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
def visible_fact_clause_for_connection(
|
|
192
|
-
conn: sqlite3.Connection,
|
|
193
|
-
prefix: str = "",
|
|
194
|
-
*,
|
|
195
|
-
include_quarantined: bool = False,
|
|
196
|
-
) -> str:
|
|
197
|
-
"""``visible_fact_clause`` for a caller that hand-rolls its SQL.
|
|
198
|
-
|
|
199
|
-
WHY THIS EXISTS. 4.0.10 put the withheld-row filter on every
|
|
200
|
-
``DatabaseManager`` read path and every retrieval channel, and enumerated
|
|
201
|
-
them in a test. The enumeration was of *methods*, and two HTTP routes never
|
|
202
|
-
call one: ``/api/memories`` and ``/api/v3/timeline/`` build their own SQL
|
|
203
|
-
against ``atomic_facts``. Measured against the author's store on 4.0.10,
|
|
204
|
-
``/api/memories?limit=50`` served **24 withheld rows on page one** and
|
|
205
|
-
reported a total of 5,218 against 3,919 real memories -- the release's
|
|
206
|
-
central claim, false on the dashboard's main list.
|
|
207
|
-
|
|
208
|
-
So the rule is not "read paths go through DatabaseManager"; plenty of
|
|
209
|
-
reasonable code will not. The rule is that anything answering "what does my
|
|
210
|
-
memory contain" resolves its predicate from here. Presence-guarded on the
|
|
211
|
-
passed connection, because a route may be pointed at a store the engine has
|
|
212
|
-
never opened, and filtering on an absent column would turn a cosmetic gap
|
|
213
|
-
into a 500 on every request.
|
|
214
|
-
|
|
215
|
-
Grep is not a sufficient test for this: ``mcp/tools_active.py`` also selects
|
|
216
|
-
from ``atomic_facts`` directly and was already clean, while these two routes
|
|
217
|
-
were not. The test that matters calls the surface and inspects what it
|
|
218
|
-
served -- tests/test_server/test_a_listed_memory_belongs_to_its_profile.py
|
|
219
|
-
"""
|
|
220
|
-
def _column_name(row: object) -> str:
|
|
221
|
-
"""PRAGMA row -> column name, whatever row_factory the caller set.
|
|
222
|
-
|
|
223
|
-
Not paranoia: ``server/routes/memories.py`` -- the first caller and the
|
|
224
|
-
route that was leaking -- sets ``row_factory = dict_factory``, so a
|
|
225
|
-
positional ``row[1]`` raises KeyError there, and the except below would
|
|
226
|
-
have swallowed it into "no such column" and silently dropped the filter.
|
|
227
|
-
That is the same shape of failure this function exists to close.
|
|
228
|
-
"""
|
|
229
|
-
if isinstance(row, dict):
|
|
230
|
-
return str(row.get("name", ""))
|
|
231
|
-
try:
|
|
232
|
-
return str(row["name"]) # sqlite3.Row
|
|
233
|
-
except (TypeError, IndexError, KeyError):
|
|
234
|
-
pass
|
|
235
|
-
try:
|
|
236
|
-
return str(row[1]) # plain tuple
|
|
237
|
-
except (TypeError, IndexError, KeyError):
|
|
238
|
-
return ""
|
|
239
|
-
|
|
240
|
-
def _has(column: str) -> bool:
|
|
241
|
-
try:
|
|
242
|
-
rows = list(conn.execute("PRAGMA table_info(atomic_facts)"))
|
|
243
|
-
except sqlite3.Error:
|
|
244
|
-
return False
|
|
245
|
-
return any(_column_name(r) == column for r in rows)
|
|
246
|
-
|
|
247
|
-
return _compose_visible_clause(
|
|
248
|
-
prefix,
|
|
249
|
-
has_archive=_has("archive_status"),
|
|
250
|
-
has_quarantine=_has("quarantined"),
|
|
251
|
-
include_quarantined=include_quarantined,
|
|
252
|
-
)
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
class DatabaseManager:
|
|
256
|
-
"""Concurrent-safe SQLite manager with WAL, profile isolation, and FTS5.
|
|
257
|
-
|
|
258
|
-
Designed for multi-process access: MCP server, CLI, LangChain, CrewAI,
|
|
259
|
-
and other integrations can all read/write the same database safely.
|
|
260
|
-
|
|
261
|
-
Concurrency model:
|
|
262
|
-
- WAL mode: readers never block writers, writers never block readers
|
|
263
|
-
- busy_timeout: writers wait up to 10s for other writers instead of failing
|
|
264
|
-
- Retry with backoff: transient SQLITE_BUSY errors are retried automatically
|
|
265
|
-
- Per-call connections: no shared state between processes
|
|
266
|
-
"""
|
|
267
|
-
|
|
268
|
-
def __init__(self, db_path: str | Path) -> None:
|
|
269
|
-
self.db_path = Path(db_path)
|
|
270
|
-
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
271
|
-
# Shared write-serialisation lock for this db_path.
|
|
272
|
-
# get_write_lock() returns the SAME RLock for every caller that
|
|
273
|
-
# passes the same resolved path, so DatabaseManager, VectorStore,
|
|
274
|
-
# adapter_base, consolidation, and all other in-process writers
|
|
275
|
-
# all share ONE lock → zero cross-connection SQLite WAL contention.
|
|
276
|
-
#
|
|
277
|
-
# RLock (re-entrant) is required: the self-heal backfill pattern
|
|
278
|
-
# with db._lock: # acquires write lock (count: 1→2)
|
|
279
|
-
# vs.upsert(...) # re-acquires same lock (count: 2→3)
|
|
280
|
-
# is safe because the same thread re-enters the RLock.
|
|
281
|
-
self._lock = get_write_lock(self.db_path)
|
|
282
|
-
# Transaction connections are thread-affine in sqlite3. A manager is
|
|
283
|
-
# shared across HTTP, materializer, and worker threads, so a process-
|
|
284
|
-
# global connection slot lets another thread accidentally execute on
|
|
285
|
-
# an uncommitted foreign connection. Keep the active connection local
|
|
286
|
-
# to the thread that owns the transaction.
|
|
287
|
-
self._txn_state = threading.local()
|
|
288
|
-
self._enable_wal()
|
|
289
|
-
|
|
290
|
-
def _enable_wal(self) -> None:
|
|
291
|
-
conn = sqlite3.connect(str(self.db_path))
|
|
292
|
-
try:
|
|
293
|
-
conn.execute(f"PRAGMA busy_timeout={_BUSY_TIMEOUT_MS}") # FIRST — so WAL pragma below uses configured timeout
|
|
294
|
-
conn.execute("PRAGMA journal_mode=WAL")
|
|
295
|
-
conn.execute("PRAGMA foreign_keys=ON")
|
|
296
|
-
# Fix D: synchronous=NORMAL is safe under WAL — the WAL ensures
|
|
297
|
-
# atomicity independently of the fsync level. Removing the
|
|
298
|
-
# fsync-on-every-commit penalty halves write latency under the
|
|
299
|
-
# 14 000+ edge-write bursts the materialiser produces per pass.
|
|
300
|
-
# Trade-off: on a power loss between COMMIT and WAL-checkpoint
|
|
301
|
-
# the last committed write to memory.db may be lost. Acceptable
|
|
302
|
-
# for an LLM memory system; NOT acceptable for financial records.
|
|
303
|
-
conn.execute("PRAGMA synchronous=NORMAL")
|
|
304
|
-
# Fix D: reduce WAL auto-checkpoint from the default 1000 to 400
|
|
305
|
-
# frames. Smaller checkpoints run more frequently and complete
|
|
306
|
-
# faster, preventing the WAL file growing unboundedly during
|
|
307
|
-
# high-ingestion bursts (which triggered checkpoint-starvation
|
|
308
|
-
# amplifying the lock storm).
|
|
309
|
-
conn.execute("PRAGMA wal_autocheckpoint=400")
|
|
310
|
-
conn.commit()
|
|
311
|
-
finally:
|
|
312
|
-
conn.close()
|
|
313
|
-
# C4 encryption-at-rest defense-in-depth: sensitive DB files shipped
|
|
314
|
-
# world-readable (0644). Restrict to owner-only now that the file (and
|
|
315
|
-
# its WAL sidecars) exist. Best-effort, never blocks DB open.
|
|
316
|
-
try:
|
|
317
|
-
from superlocalmemory.core.security_primitives import harden_db_perms
|
|
318
|
-
harden_db_perms(self.db_path)
|
|
319
|
-
except Exception:
|
|
320
|
-
pass
|
|
321
|
-
|
|
322
|
-
def initialize(self, schema_module: ModuleType) -> None:
|
|
323
|
-
"""Create all tables. *schema_module* must expose ``create_all_tables(conn)``."""
|
|
324
|
-
conn = self._connect()
|
|
325
|
-
try:
|
|
326
|
-
schema_module.create_all_tables(conn)
|
|
327
|
-
conn.commit()
|
|
328
|
-
logger.info("Schema initialized at %s", self.db_path)
|
|
329
|
-
finally:
|
|
330
|
-
conn.close()
|
|
331
|
-
|
|
332
|
-
def close(self) -> None:
|
|
333
|
-
"""No-op for per-call connection model."""
|
|
334
|
-
|
|
335
|
-
def __enter__(self) -> DatabaseManager:
|
|
336
|
-
return self
|
|
337
|
-
|
|
338
|
-
def __exit__(self, *args: Any) -> None:
|
|
339
|
-
self.close()
|
|
340
|
-
|
|
341
|
-
def _connect(self) -> sqlite3.Connection:
|
|
342
|
-
conn = sqlite3.connect(str(self.db_path), timeout=_BUSY_TIMEOUT_MS / 1000)
|
|
343
|
-
conn.row_factory = sqlite3.Row
|
|
344
|
-
conn.execute(f"PRAGMA busy_timeout={_BUSY_TIMEOUT_MS}")
|
|
345
|
-
conn.execute("PRAGMA foreign_keys=ON")
|
|
346
|
-
# wal_autocheckpoint is a PER-CONNECTION pragma and is NOT persisted in
|
|
347
|
-
# the database file (unlike journal_mode=WAL). Setting it only on the
|
|
348
|
-
# short-lived initialisation connection left every working connection
|
|
349
|
-
# on SQLite's default of 1000 frames. With checkpoint-on-close
|
|
350
|
-
# disabled below, autocheckpoint is the ONLY remaining checkpoint path,
|
|
351
|
-
# so the intended value must be set where the writes actually happen.
|
|
352
|
-
conn.execute("PRAGMA wal_autocheckpoint=400")
|
|
353
|
-
# Deadlock hardening (postmortem 2026-08-13, Option B): WAL close
|
|
354
|
-
# triggers a checkpoint that can wait indefinitely on reader marks
|
|
355
|
-
# pinned by another process/thread — while holding SQLite's
|
|
356
|
-
# process-global VFS mutex, which convoys every later connect().
|
|
357
|
-
# busy_timeout does NOT apply to the close path. NO_CKPT_ON_CLOSE
|
|
358
|
-
# makes close() checkpoint-free so it can never block; normal
|
|
359
|
-
# checkpointing continues via the wal_autocheckpoint set above.
|
|
360
|
-
# Available since Python 3.12 / SQLite 3.31; guarded for portability.
|
|
361
|
-
try:
|
|
362
|
-
conn.setconfig(sqlite3.SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, 1) # type: ignore[attr-defined]
|
|
363
|
-
except (AttributeError, sqlite3.OperationalError):
|
|
364
|
-
# Silent degradation would hide an inactive deadlock guard on a
|
|
365
|
-
# supported interpreter (requires-python allows 3.11, which
|
|
366
|
-
# predates Connection.setconfig). Warn once, not per connection.
|
|
367
|
-
global _NO_CKPT_WARNED
|
|
368
|
-
if not _NO_CKPT_WARNED:
|
|
369
|
-
_NO_CKPT_WARNED = True
|
|
370
|
-
logger.warning(
|
|
371
|
-
"SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE unavailable (Python %s, "
|
|
372
|
-
"SQLite %s); WAL close-path deadlock hardening is INACTIVE. "
|
|
373
|
-
"Python 3.12+ is required for this protection.",
|
|
374
|
-
platform.python_version(),
|
|
375
|
-
sqlite3.sqlite_version,
|
|
376
|
-
)
|
|
377
|
-
return conn
|
|
378
|
-
|
|
379
|
-
@contextmanager
|
|
380
|
-
def _bind_coordinator_connection(
|
|
381
|
-
self,
|
|
382
|
-
conn: sqlite3.Connection,
|
|
383
|
-
capability: Any,
|
|
384
|
-
) -> Generator[None, None, None]:
|
|
385
|
-
"""Reuse the coordinator's sole writable connection for one handler.
|
|
386
|
-
|
|
387
|
-
This deliberately stays internal: only ``WriteCoordinator`` can issue
|
|
388
|
-
a capability, and that capability is valid only for its worker thread
|
|
389
|
-
and the exact resolved database path. While bound, ``transaction``
|
|
390
|
-
and ``raw_connection`` become no-op ownership scopes: they may yield
|
|
391
|
-
the connection, but they must never commit, rollback, or close it.
|
|
392
|
-
The coordinator owns the enclosing ``BEGIN IMMEDIATE`` and final
|
|
393
|
-
commit/rollback together with the command receipt.
|
|
394
|
-
"""
|
|
395
|
-
from superlocalmemory.storage.write_coordinator import WriteCoordinatorError
|
|
396
|
-
|
|
397
|
-
if not isinstance(conn, sqlite3.Connection):
|
|
398
|
-
raise WriteCoordinatorError("coordinator binding requires a sqlite3 connection")
|
|
399
|
-
validate = getattr(capability, "_validate", None)
|
|
400
|
-
if not callable(validate):
|
|
401
|
-
raise WriteCoordinatorError("untrusted coordinator capability")
|
|
402
|
-
try:
|
|
403
|
-
validate(self.db_path.expanduser().resolve())
|
|
404
|
-
except Exception as exc:
|
|
405
|
-
# Import lazily so the storage manager retains its legacy import
|
|
406
|
-
# surface when the coordinator is not used.
|
|
407
|
-
if isinstance(exc, WriteCoordinatorError):
|
|
408
|
-
raise
|
|
409
|
-
raise WriteCoordinatorError("untrusted coordinator capability") from exc
|
|
410
|
-
|
|
411
|
-
attached = conn.execute("PRAGMA database_list").fetchall()
|
|
412
|
-
main_path = next((row[2] for row in attached if row[1] == "main"), "")
|
|
413
|
-
expected_path = self.db_path.expanduser().resolve()
|
|
414
|
-
if not main_path or Path(main_path).expanduser().resolve() != expected_path:
|
|
415
|
-
raise WriteCoordinatorError("coordinator connection targets a different database")
|
|
416
|
-
if getattr(self._txn_state, "conn", None) is not None:
|
|
417
|
-
raise WriteCoordinatorError("database manager is already bound to a transaction")
|
|
418
|
-
|
|
419
|
-
self._txn_state.conn = conn
|
|
420
|
-
self._txn_state.coordinator_bound = True
|
|
421
|
-
try:
|
|
422
|
-
yield
|
|
423
|
-
finally:
|
|
424
|
-
self._txn_state.conn = None
|
|
425
|
-
self._txn_state.coordinator_bound = False
|
|
426
|
-
|
|
427
|
-
@contextmanager
|
|
428
|
-
def transaction(self) -> Generator[None, None, None]:
|
|
429
|
-
"""Atomic transaction. All writes commit or rollback together."""
|
|
430
|
-
if getattr(self._txn_state, "coordinator_bound", False):
|
|
431
|
-
# The coordinator has already issued BEGIN IMMEDIATE. Do not
|
|
432
|
-
# create a nested transaction or steal its commit/close lifecycle.
|
|
433
|
-
yield
|
|
434
|
-
return
|
|
435
|
-
with self._lock:
|
|
436
|
-
conn = self._connect()
|
|
437
|
-
self._txn_state.conn = conn
|
|
438
|
-
try:
|
|
439
|
-
yield
|
|
440
|
-
conn.commit()
|
|
441
|
-
except Exception:
|
|
442
|
-
conn.rollback()
|
|
443
|
-
raise
|
|
444
|
-
finally:
|
|
445
|
-
self._txn_state.conn = None
|
|
446
|
-
conn.close()
|
|
447
|
-
|
|
448
|
-
@contextmanager
|
|
449
|
-
def raw_connection(self) -> Generator[sqlite3.Connection, None, None]:
|
|
450
|
-
"""Yield a live sqlite3.Connection for code that needs one directly.
|
|
451
|
-
|
|
452
|
-
For callers (e.g. schema migrations) that must hold a real connection
|
|
453
|
-
rather than going through execute(). Commits on success, rolls back on
|
|
454
|
-
error, and always closes — mirroring transaction(). This is the public
|
|
455
|
-
way to obtain a connection; there is no `.conn` attribute.
|
|
456
|
-
"""
|
|
457
|
-
coordinator_conn = getattr(self._txn_state, "conn", None)
|
|
458
|
-
if getattr(self._txn_state, "coordinator_bound", False):
|
|
459
|
-
if coordinator_conn is None: # pragma: no cover - binding invariant
|
|
460
|
-
raise RuntimeError("coordinator binding has no active connection")
|
|
461
|
-
yield coordinator_conn
|
|
462
|
-
return
|
|
463
|
-
with self._lock:
|
|
464
|
-
conn = self._connect()
|
|
465
|
-
self._txn_state.conn = conn
|
|
466
|
-
try:
|
|
467
|
-
yield conn
|
|
468
|
-
conn.commit()
|
|
469
|
-
except Exception:
|
|
470
|
-
conn.rollback()
|
|
471
|
-
raise
|
|
472
|
-
finally:
|
|
473
|
-
self._txn_state.conn = None
|
|
474
|
-
conn.close()
|
|
475
|
-
|
|
476
|
-
# DML prefixes that require the single-writer lock when executed outside
|
|
477
|
-
# a transaction() context. Checked case-insensitively against the first
|
|
478
|
-
# word of the stripped SQL statement.
|
|
479
|
-
_DML_PREFIXES: frozenset[str] = frozenset({
|
|
480
|
-
"INSERT", "UPDATE", "DELETE", "REPLACE", "UPSERT",
|
|
481
|
-
"CREATE", "DROP", "ALTER",
|
|
482
|
-
})
|
|
483
|
-
|
|
484
|
-
def _execute_one(self, sql: str, params: tuple[Any, ...]) -> list[sqlite3.Row]:
|
|
485
|
-
"""Open a per-call connection, execute, commit, close — with retry.
|
|
486
|
-
|
|
487
|
-
Never called when inside a transaction() context (that path uses the
|
|
488
|
-
context's existing connection directly). Factored out of execute() so
|
|
489
|
-
the RLock acquisition logic stays in one place.
|
|
490
|
-
"""
|
|
491
|
-
last_error: Exception | None = None
|
|
492
|
-
for attempt in range(_MAX_RETRIES):
|
|
493
|
-
conn = self._connect()
|
|
494
|
-
try:
|
|
495
|
-
rows = conn.execute(sql, params).fetchall()
|
|
496
|
-
conn.commit()
|
|
497
|
-
return rows
|
|
498
|
-
except sqlite3.OperationalError as exc:
|
|
499
|
-
last_error = exc
|
|
500
|
-
if "locked" in str(exc).lower() or "busy" in str(exc).lower():
|
|
501
|
-
delay = _RETRY_BASE_DELAY * (2 ** attempt)
|
|
502
|
-
logger.debug(
|
|
503
|
-
"DB busy (attempt %d/%d), retrying in %.1fs: %s",
|
|
504
|
-
attempt + 1, _MAX_RETRIES, delay, exc,
|
|
505
|
-
)
|
|
506
|
-
time.sleep(delay)
|
|
507
|
-
continue
|
|
508
|
-
raise
|
|
509
|
-
finally:
|
|
510
|
-
conn.close()
|
|
511
|
-
|
|
512
|
-
logger.warning("DB operation failed after %d retries: %s", _MAX_RETRIES, last_error)
|
|
513
|
-
raise last_error # type: ignore[misc]
|
|
514
|
-
|
|
515
|
-
def execute(self, sql: str, params: tuple[Any, ...] = ()) -> list[sqlite3.Row]:
|
|
516
|
-
"""Execute SQL with automatic retry on SQLITE_BUSY.
|
|
517
|
-
|
|
518
|
-
Fix B — single-writer serialisation:
|
|
519
|
-
• Inside a transaction(): uses the existing connection directly (no
|
|
520
|
-
lock acquisition — the transaction() context manager already holds
|
|
521
|
-
_lock for the duration of the whole transaction).
|
|
522
|
-
• Outside a transaction(), for DML (INSERT/UPDATE/DELETE/…): acquires
|
|
523
|
-
_lock before opening a per-call connection. This ensures that
|
|
524
|
-
concurrent callers — including background workers that bypass
|
|
525
|
-
transaction() entirely — do not race at the SQLite WAL layer.
|
|
526
|
-
• Outside a transaction(), for SELECTs: no lock needed — WAL mode
|
|
527
|
-
allows concurrent readers without stalling writers.
|
|
528
|
-
|
|
529
|
-
ORDERING INVARIANT: the _txn_state.conn check MUST come before the
|
|
530
|
-
lock acquisition. Reversing the order would deadlock threads that
|
|
531
|
-
call execute() from inside transaction() because threading.RLock is
|
|
532
|
-
re-entrant per-thread but a re-entering thread inside transaction()
|
|
533
|
-
would still try to re-acquire here (lock is already held by the same
|
|
534
|
-
thread, so RLock re-enters safely — but the old threading.Lock would
|
|
535
|
-
have deadlocked; that is exactly why we changed to RLock).
|
|
536
|
-
"""
|
|
537
|
-
# Fast path: already inside a transaction — use its connection directly.
|
|
538
|
-
transaction_conn = getattr(self._txn_state, "conn", None)
|
|
539
|
-
if transaction_conn is not None:
|
|
540
|
-
return transaction_conn.execute(sql, params).fetchall()
|
|
541
|
-
|
|
542
|
-
# Determine if this is a write operation that needs serialisation.
|
|
543
|
-
first_word = sql.strip().upper().split(None, 1)[0] if sql.strip() else ""
|
|
544
|
-
if first_word in self._DML_PREFIXES:
|
|
545
|
-
with self._lock:
|
|
546
|
-
return self._execute_one(sql, params)
|
|
547
|
-
else:
|
|
548
|
-
# Read-only path: concurrent reads are safe in WAL mode.
|
|
549
|
-
return self._execute_one(sql, params)
|
|
550
|
-
|
|
551
|
-
# The two tables whose primary key is global but whose rows are owned by a
|
|
552
|
-
# profile. Literal, internal, and closed — never built from caller input.
|
|
553
|
-
_OWNED_ROWS: dict[str, str] = {"memories": "memory_id", "atomic_facts": "fact_id"}
|
|
554
|
-
|
|
555
|
-
def _refuse_cross_profile_reown(
|
|
556
|
-
self, table: str, row_id: str, profile_id: str,
|
|
557
|
-
) -> NoReturn:
|
|
558
|
-
"""Raise ``ProfileOwnershipConflict``, naming the profile that owns it.
|
|
559
|
-
|
|
560
|
-
Called only when an upsert below returned no row. Both statements carry
|
|
561
|
-
``WHERE <table>.profile_id = excluded.profile_id`` on their
|
|
562
|
-
``DO UPDATE``, so SQLite skips a conflicting row owned by a different
|
|
563
|
-
profile and ``RETURNING`` yields nothing — which makes the ownership
|
|
564
|
-
check part of the write instead of a read in front of it. That costs no
|
|
565
|
-
extra query on the ordinary path and leaves no window between deciding
|
|
566
|
-
and writing. The lookup here runs only on the refusal, to say whose row
|
|
567
|
-
it is; a refusal nobody can read gets worked around.
|
|
568
|
-
|
|
569
|
-
Taking a row away from the profile that owns it is not last-write-wins,
|
|
570
|
-
it is a different tenant's write, so it is refused rather than merged.
|
|
571
|
-
Before this the outcome depended on something unrelated:
|
|
572
|
-
``scene_fact_members`` carries a composite
|
|
573
|
-
``(profile_id, fact_id) -> atomic_facts (profile_id, fact_id)`` foreign
|
|
574
|
-
key, so where a scene referenced the fact the ownership change orphaned
|
|
575
|
-
it and SQLite raised a bare ``FOREIGN KEY constraint failed`` — and
|
|
576
|
-
where no scene did, the identical write succeeded in silence and the
|
|
577
|
-
second profile kept the fact. Isolation cannot rest on whether a
|
|
578
|
-
projection happens to exist.
|
|
579
|
-
"""
|
|
580
|
-
# Interpolated, not parameterised: SQLite takes no parameter in a table
|
|
581
|
-
# or column position. Both come from _OWNED_ROWS, a closed literal map,
|
|
582
|
-
# and the row id stays bound.
|
|
583
|
-
id_column = self._OWNED_ROWS[table]
|
|
584
|
-
rows = self.execute(
|
|
585
|
-
f"SELECT profile_id FROM {table} WHERE {id_column} = ?",
|
|
586
|
-
(row_id,),
|
|
587
|
-
)
|
|
588
|
-
owner = dict(rows[0])["profile_id"] if rows else "<unknown>"
|
|
589
|
-
raise ProfileOwnershipConflict(
|
|
590
|
-
f"{id_column} {row_id!r} in {table} belongs to profile {owner!r}; "
|
|
591
|
-
f"refusing to re-own it as {profile_id!r}"
|
|
592
|
-
)
|
|
593
|
-
|
|
594
|
-
def store_memory(self, record: MemoryRecord) -> str:
|
|
595
|
-
"""Persist a raw memory record. Returns memory_id.
|
|
596
|
-
|
|
597
|
-
Upserts in place rather than replacing. ``INSERT OR REPLACE`` is a
|
|
598
|
-
DELETE followed by an INSERT, and ``atomic_facts.memory_id`` is a
|
|
599
|
-
foreign key with ``ON DELETE CASCADE`` — so storing a record whose
|
|
600
|
-
``memory_id`` already existed silently deleted every fact extracted from
|
|
601
|
-
it. Reproduced in isolation: three facts stored, one re-store of the same
|
|
602
|
-
memory_id, zero facts left, no error raised.
|
|
603
|
-
|
|
604
|
-
Most callers pass a freshly generated id, which is why this never fired.
|
|
605
|
-
But ``cognitive_consolidator`` supplies its own ``block_id``, and the
|
|
606
|
-
queryable-promotion path in ``run_store`` deliberately avoids calling
|
|
607
|
-
this at all for an existing memory — a rule that has to be remembered
|
|
608
|
-
rather than enforced. ``ON CONFLICT DO UPDATE`` keeps the same
|
|
609
|
-
last-write-wins semantics and takes the loaded gun out of the room.
|
|
610
|
-
|
|
611
|
-
``created_at`` is deliberately not overwritten: the row's first
|
|
612
|
-
observation is a historical fact, and a re-store is not a new one.
|
|
613
|
-
``profile_id`` is not in the update list either, and a store that would
|
|
614
|
-
change it is refused outright — the ``DO UPDATE`` is conditioned on the
|
|
615
|
-
owner matching, so SQLite skips the row and ``RETURNING`` comes back
|
|
616
|
-
empty. See ``_refuse_cross_profile_reown``.
|
|
617
|
-
"""
|
|
618
|
-
_scope = getattr(record, 'scope', None) or 'personal'
|
|
619
|
-
_shared = _jd(getattr(record, 'shared_with', None))
|
|
620
|
-
written = self.execute(
|
|
621
|
-
"""INSERT INTO memories
|
|
622
|
-
(memory_id, profile_id, content, session_id, speaker,
|
|
623
|
-
role, session_date, created_at, metadata_json,
|
|
624
|
-
scope, shared_with)
|
|
625
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?)
|
|
626
|
-
ON CONFLICT(memory_id) DO UPDATE SET
|
|
627
|
-
content = excluded.content,
|
|
628
|
-
session_id = excluded.session_id,
|
|
629
|
-
speaker = excluded.speaker,
|
|
630
|
-
role = excluded.role,
|
|
631
|
-
session_date = excluded.session_date,
|
|
632
|
-
metadata_json = excluded.metadata_json,
|
|
633
|
-
scope = excluded.scope,
|
|
634
|
-
shared_with = excluded.shared_with
|
|
635
|
-
WHERE memories.profile_id = excluded.profile_id
|
|
636
|
-
RETURNING profile_id""",
|
|
637
|
-
(record.memory_id, record.profile_id, record.content,
|
|
638
|
-
record.session_id, record.speaker, record.role,
|
|
639
|
-
record.session_date, record.created_at,
|
|
640
|
-
json.dumps(record.metadata), _scope, _shared),
|
|
641
|
-
)
|
|
642
|
-
if not written:
|
|
643
|
-
self._refuse_cross_profile_reown(
|
|
644
|
-
"memories", record.memory_id, record.profile_id,
|
|
645
|
-
)
|
|
646
|
-
return record.memory_id
|
|
647
|
-
|
|
648
|
-
def update_memory_summary(self, memory_id: str, summary: str) -> None:
|
|
649
|
-
"""Store a generated summary for a memory record."""
|
|
650
|
-
try:
|
|
651
|
-
self.execute(
|
|
652
|
-
"UPDATE memories SET metadata_json = json_set("
|
|
653
|
-
" COALESCE(metadata_json, '{}'), '$.summary', ?"
|
|
654
|
-
") WHERE memory_id = ?",
|
|
655
|
-
(summary, memory_id),
|
|
656
|
-
)
|
|
657
|
-
except Exception:
|
|
658
|
-
pass # Non-critical — summary is enhancement only
|
|
659
|
-
|
|
660
|
-
def get_memory_summary(self, memory_id: str) -> str:
|
|
661
|
-
"""Retrieve stored summary for a memory, or empty string."""
|
|
662
|
-
try:
|
|
663
|
-
rows = self.execute(
|
|
664
|
-
"SELECT json_extract(metadata_json, '$.summary') as s "
|
|
665
|
-
"FROM memories WHERE memory_id = ?",
|
|
666
|
-
(memory_id,),
|
|
667
|
-
)
|
|
668
|
-
if rows:
|
|
669
|
-
return dict(rows[0]).get("s") or ""
|
|
670
|
-
except Exception:
|
|
671
|
-
pass
|
|
672
|
-
return ""
|
|
673
|
-
|
|
674
|
-
def _atomically(self, work: Any) -> Any:
|
|
675
|
-
"""Run ``work`` in one transaction, joining an open one rather than nesting.
|
|
676
|
-
|
|
677
|
-
A caller already inside ``transaction()`` must not start a second one:
|
|
678
|
-
the lock is re-entrant but ``_connect`` is not, so a nested attempt opens
|
|
679
|
-
a separate connection to the same file while the first still holds its
|
|
680
|
-
write. ``store_fact`` carried this check inline; it is here because
|
|
681
|
-
three more methods now need the same thing, and a projection intent that
|
|
682
|
-
commits in a different transaction from the row it describes is exactly
|
|
683
|
-
the window this whole mechanism exists to close.
|
|
684
|
-
"""
|
|
685
|
-
if getattr(self._txn_state, "conn", None) is not None:
|
|
686
|
-
return work()
|
|
687
|
-
with self.transaction():
|
|
688
|
-
return work()
|
|
689
|
-
|
|
690
|
-
def store_fact(self, fact: AtomicFact) -> str:
|
|
691
|
-
"""Persist an atomic fact. Returns fact_id.
|
|
692
|
-
|
|
693
|
-
v3.6.4 — idempotent on content. If an ACTIVE fact with identical
|
|
694
|
-
content already exists for this profile, reinforce it (bump
|
|
695
|
-
evidence_count + access_count) and return its fact_id instead of
|
|
696
|
-
inserting a duplicate row. The passed fact's ``fact_id`` is rewritten
|
|
697
|
-
to the canonical id so downstream writes keyed on it (embeddings,
|
|
698
|
-
graph edges, context) target the real fact rather than orphaning.
|
|
699
|
-
|
|
700
|
-
This enforces the memory-system invariant "storing the same fact
|
|
701
|
-
twice is one fact" — preventing the duplicate explosion that poisons
|
|
702
|
-
importance ranking and core-memory promotion. Empty/whitespace
|
|
703
|
-
content is exempt (handled by placeholder filtering, not dedup).
|
|
704
|
-
|
|
705
|
-
The insert below upserts rather than replaces, for the reason
|
|
706
|
-
``store_memory`` does. ``INSERT OR REPLACE`` is a DELETE followed by an
|
|
707
|
-
INSERT, and eight tables hold
|
|
708
|
-
``FOREIGN KEY (fact_id) REFERENCES atomic_facts (fact_id) ON DELETE
|
|
709
|
-
CASCADE`` — so re-storing a fact under an occupied id dropped its
|
|
710
|
-
retention row, access history, context and importance, and raised
|
|
711
|
-
nothing.
|
|
712
|
-
|
|
713
|
-
The dedup above does not close this: it matches on *content*, so a
|
|
714
|
-
second store of the same id with *different* content falls straight
|
|
715
|
-
through to the insert. ``MemoryEngine.store_fact_direct`` reaches it —
|
|
716
|
-
``canonical_store_fact`` exists to persist a caller-chosen id and
|
|
717
|
-
raises if that id is not preserved. Within one profile an idempotency
|
|
718
|
-
key of ``prebuilt:<fact_id>`` catches the second store, but that key is
|
|
719
|
-
scoped ``(profile_id, source_type, idempotency_key)`` while
|
|
720
|
-
``atomic_facts.fact_id`` is a bare ``TEXT PRIMARY KEY``. Reproduced
|
|
721
|
-
across two profiles on one store: the first profile's fact was replaced
|
|
722
|
-
outright — new owner, new content — and its associations were gone.
|
|
723
|
-
|
|
724
|
-
``created_at`` is deliberately not overwritten: the row's first
|
|
725
|
-
observation is a historical fact, and a re-store is not a new one.
|
|
726
|
-
``pinned`` is not in the column list at all, so the upsert now leaves it
|
|
727
|
-
alone where the replace silently reset it to 0 — pinning is user intent,
|
|
728
|
-
not something a re-store gets to revoke. ``profile_id`` is out of the
|
|
729
|
-
update list for a stronger reason: a store that would change it is
|
|
730
|
-
refused rather than applied, because it is one profile taking another's
|
|
731
|
-
fact rather than a re-store at all. The refusal is a condition on the
|
|
732
|
-
``DO UPDATE`` itself, so there is no window between checking the owner
|
|
733
|
-
and writing the row.
|
|
734
|
-
|
|
735
|
-
``insert_fact_immutable`` remains the right call for a known-new fact
|
|
736
|
-
that must abort on a collision rather than win it.
|
|
737
|
-
"""
|
|
738
|
-
if fact.content and fact.content.strip():
|
|
739
|
-
# Dedup across all LIVE lifecycle zones (active/warm/cold). Excludes
|
|
740
|
-
# 'archived' — that is soft-deleted/forgotten, so re-storing the same
|
|
741
|
-
# content correctly re-learns it as a fresh fact. Matching only
|
|
742
|
-
# 'active' (pre-3.6.4) re-opened the duplication window for every
|
|
743
|
-
# fact that aged to warm/cold (the bulk of the KB).
|
|
744
|
-
existing = self.execute(
|
|
745
|
-
"SELECT fact_id FROM atomic_facts "
|
|
746
|
-
"WHERE profile_id = ? AND content = ? "
|
|
747
|
-
"AND lifecycle IN ('active', 'warm', 'cold') "
|
|
748
|
-
"ORDER BY created_at LIMIT 1",
|
|
749
|
-
(fact.profile_id, fact.content),
|
|
750
|
-
)
|
|
751
|
-
if existing:
|
|
752
|
-
canonical_id = dict(existing[0])["fact_id"]
|
|
753
|
-
self.execute(
|
|
754
|
-
"UPDATE atomic_facts "
|
|
755
|
-
"SET evidence_count = evidence_count + 1, "
|
|
756
|
-
" access_count = access_count + 1 "
|
|
757
|
-
"WHERE fact_id = ?",
|
|
758
|
-
(canonical_id,),
|
|
759
|
-
)
|
|
760
|
-
# Rewrite caller's id so downstream embedding/graph/context
|
|
761
|
-
# writes target the canonical fact (idempotent), not an
|
|
762
|
-
# orphaned id that was never inserted.
|
|
763
|
-
fact.fact_id = canonical_id
|
|
764
|
-
# Repair the only recoverable interrupted-write state from an
|
|
765
|
-
# early 4.0.2 attempt: a durable fact without its mandatory
|
|
766
|
-
# knowledge-time anchor. Do not manufacture a historical
|
|
767
|
-
# timestamp from ``created_at``; this retry is the earliest
|
|
768
|
-
# trustworthy observation that the anchor was missing.
|
|
769
|
-
if self.get_temporal_validity(canonical_id, fact.profile_id) is None:
|
|
770
|
-
self.store_temporal_validity(canonical_id, fact.profile_id)
|
|
771
|
-
# Re-storing known content is the natural moment to notice a
|
|
772
|
-
# projection that was never written — an upgraded store whose
|
|
773
|
-
# graph predates the migration reaches this branch, not the
|
|
774
|
-
# insert below.
|
|
775
|
-
projection_outbox.enqueue(self, canonical_id, fact.profile_id)
|
|
776
|
-
return canonical_id
|
|
777
|
-
_scope = getattr(fact, 'scope', None) or 'personal'
|
|
778
|
-
_shared = _jd(getattr(fact, 'shared_with', None))
|
|
779
|
-
def _insert_with_knowledge_anchor() -> None:
|
|
780
|
-
written = self.execute(
|
|
781
|
-
"""INSERT INTO atomic_facts
|
|
782
|
-
(fact_id, memory_id, profile_id, content, fact_type,
|
|
783
|
-
entities_json, canonical_entities_json,
|
|
784
|
-
observation_date, referenced_date, interval_start, interval_end,
|
|
785
|
-
confidence, importance, evidence_count, access_count,
|
|
786
|
-
source_turn_ids_json, session_id,
|
|
787
|
-
embedding, fisher_mean, fisher_variance,
|
|
788
|
-
lifecycle, langevin_position,
|
|
789
|
-
emotional_valence, emotional_arousal, signal_type, created_at,
|
|
790
|
-
scope, shared_with)
|
|
791
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
792
|
-
ON CONFLICT(fact_id) DO UPDATE SET
|
|
793
|
-
memory_id = excluded.memory_id,
|
|
794
|
-
content = excluded.content,
|
|
795
|
-
fact_type = excluded.fact_type,
|
|
796
|
-
entities_json = excluded.entities_json,
|
|
797
|
-
canonical_entities_json = excluded.canonical_entities_json,
|
|
798
|
-
observation_date = excluded.observation_date,
|
|
799
|
-
referenced_date = excluded.referenced_date,
|
|
800
|
-
interval_start = excluded.interval_start,
|
|
801
|
-
interval_end = excluded.interval_end,
|
|
802
|
-
confidence = excluded.confidence,
|
|
803
|
-
importance = excluded.importance,
|
|
804
|
-
evidence_count = excluded.evidence_count,
|
|
805
|
-
access_count = excluded.access_count,
|
|
806
|
-
source_turn_ids_json = excluded.source_turn_ids_json,
|
|
807
|
-
session_id = excluded.session_id,
|
|
808
|
-
embedding = excluded.embedding,
|
|
809
|
-
fisher_mean = excluded.fisher_mean,
|
|
810
|
-
fisher_variance = excluded.fisher_variance,
|
|
811
|
-
lifecycle = excluded.lifecycle,
|
|
812
|
-
langevin_position = excluded.langevin_position,
|
|
813
|
-
emotional_valence = excluded.emotional_valence,
|
|
814
|
-
emotional_arousal = excluded.emotional_arousal,
|
|
815
|
-
signal_type = excluded.signal_type,
|
|
816
|
-
scope = excluded.scope,
|
|
817
|
-
shared_with = excluded.shared_with
|
|
818
|
-
WHERE atomic_facts.profile_id = excluded.profile_id
|
|
819
|
-
RETURNING profile_id""",
|
|
820
|
-
(fact.fact_id, fact.memory_id, fact.profile_id, fact.content,
|
|
821
|
-
fact.fact_type.value,
|
|
822
|
-
json.dumps(fact.entities), json.dumps(fact.canonical_entities),
|
|
823
|
-
fact.observation_date, fact.referenced_date,
|
|
824
|
-
fact.interval_start, fact.interval_end,
|
|
825
|
-
fact.confidence, fact.importance, fact.evidence_count, fact.access_count,
|
|
826
|
-
json.dumps(fact.source_turn_ids), fact.session_id,
|
|
827
|
-
encode_embedding(fact.embedding),
|
|
828
|
-
encode_float_vector(fact.fisher_mean),
|
|
829
|
-
encode_float_vector(fact.fisher_variance),
|
|
830
|
-
fact.lifecycle.value, _jd(fact.langevin_position),
|
|
831
|
-
fact.emotional_valence, fact.emotional_arousal,
|
|
832
|
-
fact.signal_type.value, fact.created_at, _scope, _shared),
|
|
833
|
-
)
|
|
834
|
-
if not written:
|
|
835
|
-
self._refuse_cross_profile_reown(
|
|
836
|
-
"atomic_facts", fact.fact_id, fact.profile_id,
|
|
837
|
-
)
|
|
838
|
-
# Every fact written after 4.0.2 has an explicit transaction-time
|
|
839
|
-
# anchor. Absence deliberately represents pre-4.0.2
|
|
840
|
-
# ``legacy_unknown``; never backfill it from ``created_at``.
|
|
841
|
-
self.store_temporal_validity(fact.fact_id, fact.profile_id)
|
|
842
|
-
# The graph and the vectors live in other storage engines, so the
|
|
843
|
-
# intent to project this fact is queued here, in this transaction.
|
|
844
|
-
# Enqueueing in the storage layer rather than at each pipeline call
|
|
845
|
-
# site is deliberate: ingestion, the background materializer, the
|
|
846
|
-
# consolidator and the CLI all write facts through this method, and
|
|
847
|
-
# a call site that forgets would produce a memory that is stored
|
|
848
|
-
# and unrecallable.
|
|
849
|
-
projection_outbox.enqueue(self, fact.fact_id, fact.profile_id)
|
|
850
|
-
|
|
851
|
-
# The fact and its transaction-time anchor are one logical write. Do
|
|
852
|
-
# not open a nested transaction when an owner already holds one.
|
|
853
|
-
if getattr(self._txn_state, "conn", None) is not None:
|
|
854
|
-
_insert_with_knowledge_anchor()
|
|
855
|
-
else:
|
|
856
|
-
with self.transaction():
|
|
857
|
-
_insert_with_knowledge_anchor()
|
|
858
|
-
return fact.fact_id
|
|
859
|
-
|
|
860
|
-
def _row_to_fact(self, row: sqlite3.Row) -> AtomicFact:
|
|
861
|
-
"""Deserialize a row into AtomicFact."""
|
|
862
|
-
d = dict(row)
|
|
863
|
-
return AtomicFact(
|
|
864
|
-
fact_id=d["fact_id"], memory_id=d["memory_id"],
|
|
865
|
-
profile_id=d["profile_id"], content=d["content"],
|
|
866
|
-
fact_type=FactType(d["fact_type"]),
|
|
867
|
-
entities=_jl(d.get("entities_json")),
|
|
868
|
-
canonical_entities=_jl(d.get("canonical_entities_json")),
|
|
869
|
-
observation_date=d.get("observation_date"),
|
|
870
|
-
referenced_date=d.get("referenced_date"),
|
|
871
|
-
interval_start=d.get("interval_start"),
|
|
872
|
-
interval_end=d.get("interval_end"),
|
|
873
|
-
confidence=d["confidence"], importance=d["importance"],
|
|
874
|
-
evidence_count=d["evidence_count"], access_count=d["access_count"],
|
|
875
|
-
source_turn_ids=_jl(d.get("source_turn_ids_json")),
|
|
876
|
-
session_id=d.get("session_id", ""),
|
|
877
|
-
embedding=decode_embedding(d.get("embedding"), fact_id=d.get("fact_id", "<unknown>")),
|
|
878
|
-
fisher_mean=decode_float_vector(
|
|
879
|
-
d.get("fisher_mean"), field="fisher_mean",
|
|
880
|
-
fact_id=d.get("fact_id", "<unknown>")),
|
|
881
|
-
fisher_variance=decode_float_vector(
|
|
882
|
-
d.get("fisher_variance"), field="fisher_variance",
|
|
883
|
-
fact_id=d.get("fact_id", "<unknown>")),
|
|
884
|
-
lifecycle=MemoryLifecycle(d["lifecycle"]) if d.get("lifecycle") else MemoryLifecycle.ACTIVE,
|
|
885
|
-
langevin_position=_jl(d.get("langevin_position"), None),
|
|
886
|
-
emotional_valence=d.get("emotional_valence", 0.0),
|
|
887
|
-
emotional_arousal=d.get("emotional_arousal", 0.0),
|
|
888
|
-
signal_type=SignalType(d["signal_type"]) if d.get("signal_type") else SignalType.FACTUAL,
|
|
889
|
-
pinned=bool(d.get("pinned", 0)),
|
|
890
|
-
scope=d.get("scope", "personal"),
|
|
891
|
-
shared_with=_jl(d.get("shared_with"), None),
|
|
892
|
-
created_at=d["created_at"],
|
|
893
|
-
)
|
|
894
|
-
|
|
895
|
-
def insert_fact_immutable(self, fact: AtomicFact) -> str:
|
|
896
|
-
"""Insert one known-new fact without content deduplication or replacement.
|
|
897
|
-
|
|
898
|
-
Reviewed correction successors require a caller-chosen immutable
|
|
899
|
-
identity. Unlike normal remember ingestion, equal content must not
|
|
900
|
-
reinforce an existing row, and an occupied ID must abort the enclosing
|
|
901
|
-
transaction rather than overwrite history.
|
|
902
|
-
"""
|
|
903
|
-
scope = getattr(fact, "scope", None) or "personal"
|
|
904
|
-
shared = _jd(getattr(fact, "shared_with", None))
|
|
905
|
-
self.execute(
|
|
906
|
-
"""INSERT INTO atomic_facts
|
|
907
|
-
(fact_id, memory_id, profile_id, content, fact_type,
|
|
908
|
-
entities_json, canonical_entities_json,
|
|
909
|
-
observation_date, referenced_date, interval_start, interval_end,
|
|
910
|
-
confidence, importance, evidence_count, access_count,
|
|
911
|
-
source_turn_ids_json, session_id,
|
|
912
|
-
embedding, fisher_mean, fisher_variance,
|
|
913
|
-
lifecycle, langevin_position,
|
|
914
|
-
emotional_valence, emotional_arousal, signal_type, created_at,
|
|
915
|
-
scope, shared_with)
|
|
916
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
917
|
-
(
|
|
918
|
-
fact.fact_id,
|
|
919
|
-
fact.memory_id,
|
|
920
|
-
fact.profile_id,
|
|
921
|
-
fact.content,
|
|
922
|
-
fact.fact_type.value,
|
|
923
|
-
json.dumps(fact.entities),
|
|
924
|
-
json.dumps(fact.canonical_entities),
|
|
925
|
-
fact.observation_date,
|
|
926
|
-
fact.referenced_date,
|
|
927
|
-
fact.interval_start,
|
|
928
|
-
fact.interval_end,
|
|
929
|
-
fact.confidence,
|
|
930
|
-
fact.importance,
|
|
931
|
-
fact.evidence_count,
|
|
932
|
-
fact.access_count,
|
|
933
|
-
json.dumps(fact.source_turn_ids),
|
|
934
|
-
fact.session_id,
|
|
935
|
-
encode_embedding(fact.embedding),
|
|
936
|
-
encode_float_vector(fact.fisher_mean),
|
|
937
|
-
encode_float_vector(fact.fisher_variance),
|
|
938
|
-
fact.lifecycle.value,
|
|
939
|
-
_jd(fact.langevin_position),
|
|
940
|
-
fact.emotional_valence,
|
|
941
|
-
fact.emotional_arousal,
|
|
942
|
-
fact.signal_type.value,
|
|
943
|
-
fact.created_at,
|
|
944
|
-
scope,
|
|
945
|
-
shared,
|
|
946
|
-
),
|
|
947
|
-
)
|
|
948
|
-
self.store_temporal_validity(fact.fact_id, fact.profile_id)
|
|
949
|
-
projection_outbox.enqueue(self, fact.fact_id, fact.profile_id)
|
|
950
|
-
return fact.fact_id
|
|
951
|
-
|
|
952
|
-
def set_pinned(self, fact_id: str, pinned: bool) -> None:
|
|
953
|
-
"""Set or clear the pinned flag on a fact (v3.4.65 core-memory)."""
|
|
954
|
-
self.execute(
|
|
955
|
-
"UPDATE atomic_facts SET pinned = ? WHERE fact_id = ?",
|
|
956
|
-
(1 if pinned else 0, fact_id),
|
|
957
|
-
)
|
|
958
|
-
|
|
959
|
-
def get_pinned(
|
|
960
|
-
self, profile_id: str,
|
|
961
|
-
include_global: bool = False,
|
|
962
|
-
include_shared: bool = False,
|
|
963
|
-
) -> list[AtomicFact]:
|
|
964
|
-
"""Return currently admissible pinned facts, highest-importance first.
|
|
965
|
-
|
|
966
|
-
Pins are injection priority, not an override for a later correction.
|
|
967
|
-
A system-invalidated fact remains durable and historically queryable,
|
|
968
|
-
but it must not be inserted into the current session context.
|
|
969
|
-
"""
|
|
970
|
-
where, params = _scope_where(
|
|
971
|
-
profile_id,
|
|
972
|
-
include_global=include_global,
|
|
973
|
-
include_shared=include_shared,
|
|
974
|
-
prefix="f",
|
|
975
|
-
)
|
|
976
|
-
# Pins are injected straight into an agent's context, which makes this
|
|
977
|
-
# the most consequential display path in the class: a withheld row here
|
|
978
|
-
# is not merely shown, it is asserted as background truth.
|
|
979
|
-
rows = self.execute(
|
|
980
|
-
f"SELECT f.* FROM atomic_facts f WHERE {where} AND f.pinned = 1 "
|
|
981
|
-
f"{self.visible_fact_clause('f')} "
|
|
982
|
-
"AND NOT EXISTS ("
|
|
983
|
-
" SELECT 1 FROM fact_temporal_validity tv "
|
|
984
|
-
" WHERE tv.fact_id = f.fact_id "
|
|
985
|
-
" AND tv.profile_id = f.profile_id "
|
|
986
|
-
" AND tv.system_expired_at IS NOT NULL"
|
|
987
|
-
") "
|
|
988
|
-
"ORDER BY importance DESC",
|
|
989
|
-
(*params,),
|
|
990
|
-
)
|
|
991
|
-
facts = [self._row_to_fact(r) for r in rows]
|
|
992
|
-
if not facts:
|
|
993
|
-
return facts
|
|
994
|
-
try:
|
|
995
|
-
blocked = self.get_nonapplied_correction_successor_ids(
|
|
996
|
-
[fact.fact_id for fact in facts],
|
|
997
|
-
profile_id,
|
|
998
|
-
include_global=include_global,
|
|
999
|
-
include_shared=include_shared,
|
|
1000
|
-
)
|
|
1001
|
-
except Exception as exc:
|
|
1002
|
-
logger.warning("Pinned correction admission lookup failed: %s", exc)
|
|
1003
|
-
return []
|
|
1004
|
-
if not isinstance(blocked, set):
|
|
1005
|
-
logger.warning("Pinned correction admission returned malformed data")
|
|
1006
|
-
return []
|
|
1007
|
-
return [fact for fact in facts if fact.fact_id not in blocked]
|
|
1008
|
-
|
|
1009
|
-
def _has_archive_status(self) -> bool:
|
|
1010
|
-
"""Whether atomic_facts carries the M011 ``archive_status`` column.
|
|
1011
|
-
|
|
1012
|
-
M011 is a DEFERRED migration, so the column is absent until it runs;
|
|
1013
|
-
callers must not filter on a column that may not exist. Cached once True
|
|
1014
|
-
(a column never disappears); re-checked while absent so a later deferred
|
|
1015
|
-
migration is picked up.
|
|
1016
|
-
"""
|
|
1017
|
-
if getattr(self, "_archive_col_present", False):
|
|
1018
|
-
return True
|
|
1019
|
-
present = any(
|
|
1020
|
-
dict(row).get("name") == "archive_status"
|
|
1021
|
-
for row in self.execute("PRAGMA table_info(atomic_facts)")
|
|
1022
|
-
)
|
|
1023
|
-
if present:
|
|
1024
|
-
self._archive_col_present = True
|
|
1025
|
-
return present
|
|
1026
|
-
|
|
1027
|
-
def _has_quarantine_column(self) -> bool:
|
|
1028
|
-
"""Whether atomic_facts carries the ``quarantined`` column.
|
|
1029
|
-
|
|
1030
|
-
Same shape as ``_has_archive_status``: cached once True (a column never
|
|
1031
|
-
disappears), re-checked while absent so a later schema pass is picked
|
|
1032
|
-
up. ``storage.schema.create_all_tables`` adds the column at every engine
|
|
1033
|
-
init, so on any store the daemon has opened this is True — the guard
|
|
1034
|
-
exists for a bare DatabaseManager pointed at a store that engine init
|
|
1035
|
-
never touched, where filtering on the column would raise instead of
|
|
1036
|
-
returning results.
|
|
1037
|
-
"""
|
|
1038
|
-
if getattr(self, "_quarantine_col_present", False):
|
|
1039
|
-
return True
|
|
1040
|
-
present = any(
|
|
1041
|
-
dict(row).get("name") == "quarantined"
|
|
1042
|
-
for row in self.execute("PRAGMA table_info(atomic_facts)")
|
|
1043
|
-
)
|
|
1044
|
-
if present:
|
|
1045
|
-
self._quarantine_col_present = True
|
|
1046
|
-
return present
|
|
1047
|
-
|
|
1048
|
-
def visible_fact_clause(
|
|
1049
|
-
self, prefix: str = "", *, include_quarantined: bool = False,
|
|
1050
|
-
) -> str:
|
|
1051
|
-
"""AND-clause excluding rows no caller should be shown as a memory.
|
|
1052
|
-
|
|
1053
|
-
Two exclusions, one definition: soft-deleted (``archive_status``) and
|
|
1054
|
-
withheld (``quarantined``). Both are presence-guarded, because each
|
|
1055
|
-
column arrives with a migration and may be absent on a store the engine
|
|
1056
|
-
has not opened.
|
|
1057
|
-
|
|
1058
|
-
WHY THIS EXISTS AS A FUNCTION. 4.0.10 first put the quarantine filter in
|
|
1059
|
-
``get_facts_by_ids`` alone, reasoning that every retrieval channel
|
|
1060
|
-
re-authorises through it and the engine drops what it cannot hydrate.
|
|
1061
|
-
That reasoning was correct and the conclusion was wrong: it covered the
|
|
1062
|
-
RECALL pipeline, and ``search``, ``list_recent``, ``fetch``, the MCP
|
|
1063
|
-
resources and the dashboard's own search are not the recall pipeline.
|
|
1064
|
-
Measured on a copy of the author's store, ``search_facts_fts`` returned
|
|
1065
|
-
20 withheld rows out of 50 and ``get_all_facts`` 66 out of 400 — the
|
|
1066
|
-
exact defect the design was meant to prevent, in the paths the design
|
|
1067
|
-
never looked at.
|
|
1068
|
-
|
|
1069
|
-
There is no single SQL choke point in this codebase; ``_scope_where`` is
|
|
1070
|
-
spliced against six other tables and cannot carry a fact column. So the
|
|
1071
|
-
honest form of "one place" is one CLAUSE with an enumerable set of call
|
|
1072
|
-
sites, and a test that fails when a read path does not use it:
|
|
1073
|
-
tests/test_storage/test_no_read_path_shows_a_withheld_row.py
|
|
1074
|
-
|
|
1075
|
-
``include_quarantined=True`` is for repair, erasure and export — paths
|
|
1076
|
-
that must reach a withheld row to act on it.
|
|
1077
|
-
"""
|
|
1078
|
-
return _compose_visible_clause(
|
|
1079
|
-
prefix,
|
|
1080
|
-
has_archive=self._has_archive_status(),
|
|
1081
|
-
has_quarantine=self._has_quarantine_column(),
|
|
1082
|
-
include_quarantined=include_quarantined,
|
|
1083
|
-
)
|
|
1084
|
-
|
|
1085
|
-
def get_all_facts(
|
|
1086
|
-
self, profile_id: str, limit: int | None = None,
|
|
1087
|
-
*,
|
|
1088
|
-
include_global: bool = False,
|
|
1089
|
-
include_shared: bool = False,
|
|
1090
|
-
) -> list[AtomicFact]:
|
|
1091
|
-
"""All facts for a profile, newest first.
|
|
1092
|
-
|
|
1093
|
-
memory-bounding-02: optional SQL LIMIT so callers needing only the
|
|
1094
|
-
most-recent N (e.g. the Hopfield channel's 5000 cap) don't deserialize
|
|
1095
|
-
the entire table into AtomicFact objects. Default (None) = all facts.
|
|
1096
|
-
"""
|
|
1097
|
-
where, params = _scope_where(
|
|
1098
|
-
profile_id,
|
|
1099
|
-
include_global=include_global,
|
|
1100
|
-
include_shared=include_shared,
|
|
1101
|
-
)
|
|
1102
|
-
# memory-bounding-02 + perf M-04: an unbounded fetch materializes the
|
|
1103
|
-
# whole table (hundreds of MB with embeddings at 50k+ facts). Apply a
|
|
1104
|
-
# hard, env-tunable ceiling even when the caller passes limit=None.
|
|
1105
|
-
if limit is None:
|
|
1106
|
-
limit = _unbounded_facts_ceiling()
|
|
1107
|
-
# Soft-deleted and withheld rows are not memories a caller may see.
|
|
1108
|
-
rows = self.execute(
|
|
1109
|
-
f"SELECT * FROM atomic_facts WHERE {where}"
|
|
1110
|
-
f"{self.visible_fact_clause()} "
|
|
1111
|
-
"ORDER BY created_at DESC LIMIT ?",
|
|
1112
|
-
(*params, int(limit)),
|
|
1113
|
-
)
|
|
1114
|
-
return [self._row_to_fact(r) for r in rows]
|
|
1115
|
-
|
|
1116
|
-
def get_external_visible_facts(
|
|
1117
|
-
self,
|
|
1118
|
-
profile_id: str,
|
|
1119
|
-
*,
|
|
1120
|
-
include_global: bool = False,
|
|
1121
|
-
include_shared: bool = False,
|
|
1122
|
-
) -> list[AtomicFact]:
|
|
1123
|
-
"""Cross-profile facts visible to ``profile_id`` under scope policy.
|
|
1124
|
-
|
|
1125
|
-
This is the bounded supplement used by profile-partitioned candidate
|
|
1126
|
-
indexes. It deliberately excludes the requester's own partition so a
|
|
1127
|
-
fast local index can merge only the global/authorized-shared rows it
|
|
1128
|
-
cannot discover itself. The canonical scope predicate remains the
|
|
1129
|
-
sole authorization rule.
|
|
1130
|
-
"""
|
|
1131
|
-
if not include_global and not include_shared:
|
|
1132
|
-
return []
|
|
1133
|
-
where, params = _scope_where(
|
|
1134
|
-
profile_id,
|
|
1135
|
-
include_global=include_global,
|
|
1136
|
-
include_shared=include_shared,
|
|
1137
|
-
)
|
|
1138
|
-
# Crossing a profile boundary is the last place a withheld row should
|
|
1139
|
-
# appear: it would be a model's non-answer presented to somebody else
|
|
1140
|
-
# as one of this profile's shared memories.
|
|
1141
|
-
rows = self.execute(
|
|
1142
|
-
f"SELECT * FROM atomic_facts WHERE {where} AND profile_id != ?"
|
|
1143
|
-
f"{self.visible_fact_clause()} ORDER BY created_at DESC",
|
|
1144
|
-
(*params, profile_id),
|
|
1145
|
-
)
|
|
1146
|
-
return [self._row_to_fact(r) for r in rows]
|
|
1147
|
-
|
|
1148
|
-
_MAX_FACTS_PER_ENTITY_LOOKUP: int = 100
|
|
1149
|
-
|
|
1150
|
-
def get_facts_by_entity(
|
|
1151
|
-
self, entity_id: str, profile_id: str,
|
|
1152
|
-
include_global: bool = False,
|
|
1153
|
-
include_shared: bool = False,
|
|
1154
|
-
) -> list[AtomicFact]:
|
|
1155
|
-
"""Facts whose canonical_entities JSON array contains *entity_id*.
|
|
1156
|
-
|
|
1157
|
-
V3.3.14: LIMIT to _MAX_FACTS_PER_ENTITY_LOOKUP (100) to prevent
|
|
1158
|
-
unbounded memory growth during ingestion. Previously loaded ALL
|
|
1159
|
-
facts for popular entities (500+) causing 17GB+ memory usage.
|
|
1160
|
-
Ordered by created_at DESC so newest facts are always included.
|
|
1161
|
-
"""
|
|
1162
|
-
where, params = _scope_where(
|
|
1163
|
-
profile_id,
|
|
1164
|
-
include_global=include_global,
|
|
1165
|
-
include_shared=include_shared,
|
|
1166
|
-
)
|
|
1167
|
-
rows = self.execute(
|
|
1168
|
-
f"SELECT * FROM atomic_facts WHERE {where} AND canonical_entities_json LIKE ? "
|
|
1169
|
-
"ORDER BY created_at DESC LIMIT ?",
|
|
1170
|
-
(*params, f'%"{entity_id}"%', self._MAX_FACTS_PER_ENTITY_LOOKUP),
|
|
1171
|
-
)
|
|
1172
|
-
return [self._row_to_fact(r) for r in rows]
|
|
1173
|
-
|
|
1174
|
-
def get_facts_by_type(
|
|
1175
|
-
self, fact_type: FactType, profile_id: str,
|
|
1176
|
-
include_global: bool = False,
|
|
1177
|
-
include_shared: bool = False,
|
|
1178
|
-
) -> list[AtomicFact]:
|
|
1179
|
-
"""All facts of a given type for a profile."""
|
|
1180
|
-
where, params = _scope_where(
|
|
1181
|
-
profile_id,
|
|
1182
|
-
include_global=include_global,
|
|
1183
|
-
include_shared=include_shared,
|
|
1184
|
-
)
|
|
1185
|
-
rows = self.execute(
|
|
1186
|
-
f"SELECT * FROM atomic_facts WHERE {where} AND fact_type = ? "
|
|
1187
|
-
"ORDER BY created_at DESC",
|
|
1188
|
-
(*params, fact_type.value),
|
|
1189
|
-
)
|
|
1190
|
-
return [self._row_to_fact(r) for r in rows]
|
|
1191
|
-
|
|
1192
|
-
# Allowed columns for partial updates (prevents SQL injection via dict keys)
|
|
1193
|
-
_UPDATABLE_FACT_COLUMNS: frozenset[str] = frozenset({
|
|
1194
|
-
"content", "fact_type", "entities_json", "canonical_entities_json",
|
|
1195
|
-
"observation_date", "referenced_date", "interval_start", "interval_end",
|
|
1196
|
-
"confidence", "importance", "evidence_count", "access_count",
|
|
1197
|
-
"source_turn_ids_json", "session_id", "embedding",
|
|
1198
|
-
"fisher_mean", "fisher_variance", "lifecycle", "langevin_position",
|
|
1199
|
-
"emotional_valence", "emotional_arousal", "signal_type",
|
|
1200
|
-
# Multi-scope (M016): allow re-scoping a fact after creation so a memory
|
|
1201
|
-
# can be shared with a team or made global from the dashboard.
|
|
1202
|
-
"scope", "shared_with",
|
|
1203
|
-
})
|
|
1204
|
-
|
|
1205
|
-
def update_fact(self, fact_id: str, updates: dict[str, Any],
|
|
1206
|
-
profile_id: str | None = None) -> None:
|
|
1207
|
-
"""Partial update on a fact. JSON-serializes list/dict values.
|
|
1208
|
-
|
|
1209
|
-
Tenant safety: when ``profile_id`` is supplied the UPDATE is constrained
|
|
1210
|
-
to that tenant, so a fact_id belonging to another profile cannot be
|
|
1211
|
-
mutated. Authorized routes always pass it.
|
|
1212
|
-
"""
|
|
1213
|
-
if not updates:
|
|
1214
|
-
raise ValueError("updates dict must not be empty")
|
|
1215
|
-
bad_keys = set(updates) - self._UPDATABLE_FACT_COLUMNS
|
|
1216
|
-
if bad_keys:
|
|
1217
|
-
raise ValueError(f"Disallowed column(s): {bad_keys}")
|
|
1218
|
-
clean: dict[str, Any] = {}
|
|
1219
|
-
for k, v in updates.items():
|
|
1220
|
-
if k == "embedding":
|
|
1221
|
-
# Embeddings are stored in the canonical binary form. Falling
|
|
1222
|
-
# through to json.dumps here would write a text row back into a
|
|
1223
|
-
# converted store, one fact at a time, undoing the conversion
|
|
1224
|
-
# wherever a fact is updated.
|
|
1225
|
-
clean[k] = encode_embedding(v) if v is not None else None
|
|
1226
|
-
elif k in ("fisher_mean", "fisher_variance"):
|
|
1227
|
-
# Same hazard, same answer: these are float vectors of the same
|
|
1228
|
-
# width as the embedding and are stored the same way.
|
|
1229
|
-
clean[k] = encode_float_vector(v) if v is not None else None
|
|
1230
|
-
elif isinstance(v, (list, dict)):
|
|
1231
|
-
clean[k] = json.dumps(v)
|
|
1232
|
-
elif isinstance(v, (MemoryLifecycle, FactType, SignalType)):
|
|
1233
|
-
clean[k] = v.value
|
|
1234
|
-
else:
|
|
1235
|
-
clean[k] = v
|
|
1236
|
-
set_clause = ", ".join(f"{k} = ?" for k in clean)
|
|
1237
|
-
|
|
1238
|
-
def _write() -> None:
|
|
1239
|
-
if profile_id is not None:
|
|
1240
|
-
self.execute(
|
|
1241
|
-
f"UPDATE atomic_facts SET {set_clause} "
|
|
1242
|
-
"WHERE fact_id = ? AND profile_id = ?",
|
|
1243
|
-
(*clean.values(), fact_id, profile_id),
|
|
1244
|
-
)
|
|
1245
|
-
else:
|
|
1246
|
-
self.execute(
|
|
1247
|
-
f"UPDATE atomic_facts SET {set_clause} WHERE fact_id = ?",
|
|
1248
|
-
(*clean.values(), fact_id),
|
|
1249
|
-
)
|
|
1250
|
-
# Only an update that changes something a projection is derived
|
|
1251
|
-
# from needs re-projecting. Recall bumps access_count on every hit,
|
|
1252
|
-
# so queueing on any update at all would hand the drain worker one
|
|
1253
|
-
# row per returned memory per recall, for a column neither Cozo nor
|
|
1254
|
-
# Lance holds.
|
|
1255
|
-
if set(updates) & projection_outbox.PROJECTED_FACT_COLUMNS:
|
|
1256
|
-
if profile_id is not None:
|
|
1257
|
-
projection_outbox.enqueue(self, fact_id, profile_id)
|
|
1258
|
-
else:
|
|
1259
|
-
projection_outbox.enqueue_for_fact(self, fact_id)
|
|
1260
|
-
|
|
1261
|
-
self._atomically(_write)
|
|
1262
|
-
|
|
1263
|
-
def delete_fact(self, fact_id: str, profile_id: str | None = None) -> None:
|
|
1264
|
-
"""Hard-delete a fact.
|
|
1265
|
-
|
|
1266
|
-
DatabaseManager connections enforce FKs (PRAGMA foreign_keys=ON), so
|
|
1267
|
-
embedding_metadata / fact_retention cascade. The explicit
|
|
1268
|
-
embedding_metadata delete below is belt-and-suspenders for the case a
|
|
1269
|
-
future caller routes through a connection without FK enforcement.
|
|
1270
|
-
|
|
1271
|
-
``graph_edges`` does NOT cascade, whatever this docstring used to say.
|
|
1272
|
-
Its only foreign key is to ``profiles``; there is none to
|
|
1273
|
-
``atomic_facts``, because an edge's endpoints can be entity ids as well
|
|
1274
|
-
as fact ids and a single column cannot reference two tables. So the
|
|
1275
|
-
edges are deleted here, explicitly. Without that, deleting a memory left
|
|
1276
|
-
its connections behind pointing at nothing, and the graph tidy-up pass
|
|
1277
|
-
would not reach them until its next run -- during which a search could
|
|
1278
|
-
still follow an edge into a memory that no longer exists.
|
|
1279
|
-
|
|
1280
|
-
Tenant safety: when ``profile_id`` is supplied the delete is constrained
|
|
1281
|
-
to that tenant (the fact must belong to it), so a fact_id from another
|
|
1282
|
-
profile cannot be destroyed. Authorized routes always pass it.
|
|
1283
|
-
"""
|
|
1284
|
-
if profile_id is not None:
|
|
1285
|
-
row = self.execute(
|
|
1286
|
-
"SELECT 1 FROM atomic_facts WHERE fact_id = ? AND profile_id = ?",
|
|
1287
|
-
(fact_id, profile_id),
|
|
1288
|
-
)
|
|
1289
|
-
if not row:
|
|
1290
|
-
return # not this tenant's fact — no-op
|
|
1291
|
-
# A forgotten memory that survives in the graph or the vector index is
|
|
1292
|
-
# still recallable, which makes an erasure receipt a false statement. So
|
|
1293
|
-
# the deletes and the queued removal are one transaction: the process
|
|
1294
|
-
# can die immediately afterwards and the projections still get cleaned.
|
|
1295
|
-
def _write() -> None:
|
|
1296
|
-
# Read the tenant while the fact is still there. After the DELETE
|
|
1297
|
-
# there is nothing left to resolve it from, and a queued removal
|
|
1298
|
-
# filed under the wrong tenant is a fact id that outlives that
|
|
1299
|
-
# tenant's erasure.
|
|
1300
|
-
owner = profile_id or projection_outbox.resolve_profile(self, fact_id)
|
|
1301
|
-
self.execute(
|
|
1302
|
-
"DELETE FROM embedding_metadata WHERE fact_id = ?", (fact_id,),
|
|
1303
|
-
)
|
|
1304
|
-
if profile_id is not None:
|
|
1305
|
-
self.execute(
|
|
1306
|
-
"DELETE FROM atomic_facts WHERE fact_id = ? AND profile_id = ?",
|
|
1307
|
-
(fact_id, profile_id),
|
|
1308
|
-
)
|
|
1309
|
-
else:
|
|
1310
|
-
self.execute("DELETE FROM atomic_facts WHERE fact_id = ?", (fact_id,))
|
|
1311
|
-
# Both directions: an edge naming this fact at either end is now an
|
|
1312
|
-
# edge to nothing.
|
|
1313
|
-
self.execute(
|
|
1314
|
-
"DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?",
|
|
1315
|
-
(fact_id, fact_id),
|
|
1316
|
-
)
|
|
1317
|
-
projection_outbox.enqueue(
|
|
1318
|
-
self, fact_id, owner or "default", projection_outbox.OP_DELETE,
|
|
1319
|
-
)
|
|
1320
|
-
|
|
1321
|
-
self._atomically(_write)
|
|
1322
|
-
|
|
1323
|
-
def gc_orphaned_embedding_metadata(self) -> int:
|
|
1324
|
-
"""Remove embedding_metadata rows whose parent atomic_fact is gone.
|
|
1325
|
-
|
|
1326
|
-
P1-3 (embeddings-vector-02): orphans accumulate when facts are deleted
|
|
1327
|
-
through a connection that has FK enforcement OFF (the ON DELETE CASCADE
|
|
1328
|
-
never fires). Vector search maps a vec0 rowid → fact_id via this table;
|
|
1329
|
-
orphans return stale fact_ids that fail downstream fetch. This
|
|
1330
|
-
maintenance sweep removes them regardless of how they were created.
|
|
1331
|
-
Returns the number of rows deleted.
|
|
1332
|
-
"""
|
|
1333
|
-
rows = self.execute(
|
|
1334
|
-
"SELECT COUNT(*) AS c FROM embedding_metadata "
|
|
1335
|
-
"WHERE fact_id NOT IN (SELECT fact_id FROM atomic_facts)"
|
|
1336
|
-
)
|
|
1337
|
-
n = int(rows[0]["c"]) if rows else 0
|
|
1338
|
-
if n:
|
|
1339
|
-
self.execute(
|
|
1340
|
-
"DELETE FROM embedding_metadata "
|
|
1341
|
-
"WHERE fact_id NOT IN (SELECT fact_id FROM atomic_facts)"
|
|
1342
|
-
)
|
|
1343
|
-
return n
|
|
1344
|
-
|
|
1345
|
-
def get_fact_count(
|
|
1346
|
-
self, profile_id: str,
|
|
1347
|
-
include_global: bool = False,
|
|
1348
|
-
include_shared: bool = False,
|
|
1349
|
-
) -> int:
|
|
1350
|
-
"""Memories this profile has, as the owner would count them.
|
|
1351
|
-
|
|
1352
|
-
Counts what a caller can be shown, which is why it applies
|
|
1353
|
-
``visible_fact_clause``. It fed the dashboard's "All memories 5,093" and
|
|
1354
|
-
was counting 1,195 withheld summaries and every soft-deleted row into
|
|
1355
|
-
that figure -- a number the owner reads as "how much do I remember".
|
|
1356
|
-
"""
|
|
1357
|
-
where, params = _scope_where(
|
|
1358
|
-
profile_id,
|
|
1359
|
-
include_global=include_global,
|
|
1360
|
-
include_shared=include_shared,
|
|
1361
|
-
)
|
|
1362
|
-
rows = self.execute(
|
|
1363
|
-
f"SELECT COUNT(*) AS c FROM atomic_facts WHERE {where}"
|
|
1364
|
-
f"{self.visible_fact_clause()}", (*params,),
|
|
1365
|
-
)
|
|
1366
|
-
return int(rows[0]["c"]) if rows else 0
|
|
1367
|
-
|
|
1368
|
-
def store_entity(self, entity: CanonicalEntity) -> str:
|
|
1369
|
-
"""Persist a canonical entity. Returns entity_id."""
|
|
1370
|
-
self.execute(
|
|
1371
|
-
"""INSERT OR REPLACE INTO canonical_entities
|
|
1372
|
-
(entity_id, profile_id, canonical_name, entity_type,
|
|
1373
|
-
first_seen, last_seen, fact_count)
|
|
1374
|
-
VALUES (?,?,?,?,?,?,?)""",
|
|
1375
|
-
(entity.entity_id, entity.profile_id, entity.canonical_name,
|
|
1376
|
-
entity.entity_type, entity.first_seen, entity.last_seen,
|
|
1377
|
-
entity.fact_count),
|
|
1378
|
-
)
|
|
1379
|
-
return entity.entity_id
|
|
1380
|
-
|
|
1381
|
-
def get_entity_by_name(self, name: str, profile_id: str) -> CanonicalEntity | None:
|
|
1382
|
-
"""Look up entity by name (case-insensitive)."""
|
|
1383
|
-
rows = self.execute(
|
|
1384
|
-
"SELECT * FROM canonical_entities WHERE profile_id = ? AND LOWER(canonical_name) = LOWER(?)",
|
|
1385
|
-
(profile_id, name),
|
|
1386
|
-
)
|
|
1387
|
-
if not rows:
|
|
1388
|
-
return None
|
|
1389
|
-
d = dict(rows[0])
|
|
1390
|
-
return CanonicalEntity(
|
|
1391
|
-
entity_id=d["entity_id"], profile_id=d["profile_id"],
|
|
1392
|
-
canonical_name=d["canonical_name"], entity_type=d["entity_type"],
|
|
1393
|
-
first_seen=d["first_seen"], last_seen=d["last_seen"],
|
|
1394
|
-
fact_count=d["fact_count"],
|
|
1395
|
-
)
|
|
1396
|
-
|
|
1397
|
-
def store_alias(self, alias: EntityAlias, profile_id: str) -> str:
|
|
1398
|
-
"""Persist an entity alias under a profile. Returns alias_id.
|
|
1399
|
-
|
|
1400
|
-
profile_id scopes the alias so the same entity_id appearing in two
|
|
1401
|
-
profiles never shares aliases.
|
|
1402
|
-
"""
|
|
1403
|
-
self.execute(
|
|
1404
|
-
"INSERT OR REPLACE INTO entity_aliases "
|
|
1405
|
-
"(alias_id, profile_id, entity_id, alias, confidence, source) "
|
|
1406
|
-
"VALUES (?,?,?,?,?,?)",
|
|
1407
|
-
(alias.alias_id, profile_id, alias.entity_id, alias.alias,
|
|
1408
|
-
alias.confidence, alias.source),
|
|
1409
|
-
)
|
|
1410
|
-
return alias.alias_id
|
|
1411
|
-
|
|
1412
|
-
def get_aliases_for_entity(
|
|
1413
|
-
self, entity_id: str, profile_id: str,
|
|
1414
|
-
) -> list[EntityAlias]:
|
|
1415
|
-
"""All aliases for a canonical entity within one profile."""
|
|
1416
|
-
rows = self.execute(
|
|
1417
|
-
"SELECT * FROM entity_aliases WHERE entity_id = ? AND profile_id = ?",
|
|
1418
|
-
(entity_id, profile_id),
|
|
1419
|
-
)
|
|
1420
|
-
return [
|
|
1421
|
-
EntityAlias(**{k: dict(r)[k] for k in ("alias_id", "entity_id", "alias", "confidence", "source")})
|
|
1422
|
-
for r in rows
|
|
1423
|
-
]
|
|
1424
|
-
|
|
1425
|
-
def get_memory_content_batch(
|
|
1426
|
-
self, memory_ids: list[str], profile_id: str,
|
|
1427
|
-
include_global: bool = False, include_shared: bool = False,
|
|
1428
|
-
) -> dict[str, str]:
|
|
1429
|
-
"""Batch-fetch original memory text. Returns {memory_id: content}.
|
|
1430
|
-
|
|
1431
|
-
C4 hardening: this exposes raw memory *content* and is reachable from
|
|
1432
|
-
HTTP routes, so it is strictly tenant-scoped — a memory_id belonging to
|
|
1433
|
-
another profile is never resolved. Widen only via the scope flags.
|
|
1434
|
-
"""
|
|
1435
|
-
if not memory_ids:
|
|
1436
|
-
return {}
|
|
1437
|
-
unique_ids = list(set(memory_ids))
|
|
1438
|
-
ph = ','.join('?' * len(unique_ids))
|
|
1439
|
-
where, sparams = _scope_where(
|
|
1440
|
-
profile_id, include_global=include_global, include_shared=include_shared,
|
|
1441
|
-
)
|
|
1442
|
-
rows = self.execute(
|
|
1443
|
-
f"SELECT memory_id, content FROM memories "
|
|
1444
|
-
f"WHERE memory_id IN ({ph}) AND {where}",
|
|
1445
|
-
(*unique_ids, *sparams),
|
|
1446
|
-
)
|
|
1447
|
-
return {dict(r)["memory_id"]: dict(r)["content"] for r in rows}
|
|
1448
|
-
|
|
1449
|
-
def get_facts_by_memory_id(
|
|
1450
|
-
self, memory_id: str, profile_id: str,
|
|
1451
|
-
include_global: bool = False,
|
|
1452
|
-
include_shared: bool = False,
|
|
1453
|
-
) -> list[AtomicFact]:
|
|
1454
|
-
"""Get all atomic facts for a given memory_id."""
|
|
1455
|
-
where, params = _scope_where(
|
|
1456
|
-
profile_id,
|
|
1457
|
-
include_global=include_global,
|
|
1458
|
-
include_shared=include_shared,
|
|
1459
|
-
)
|
|
1460
|
-
rows = self.execute(
|
|
1461
|
-
f"SELECT * FROM atomic_facts WHERE memory_id = ? AND {where} "
|
|
1462
|
-
"ORDER BY confidence DESC",
|
|
1463
|
-
(memory_id, *params),
|
|
1464
|
-
)
|
|
1465
|
-
return [self._row_to_fact(r) for r in rows]
|
|
1466
|
-
|
|
1467
|
-
def store_edge(self, edge: GraphEdge) -> str:
|
|
1468
|
-
"""Persist a graph edge. Returns edge_id.
|
|
1469
|
-
|
|
1470
|
-
graph-integrity-02: dedup on the LOGICAL edge identity
|
|
1471
|
-
(profile, source, target, type). The PK is a random edge_id, so
|
|
1472
|
-
without this every re-link created a duplicate row, and NetworkX
|
|
1473
|
-
builds read last-weight-wins — corrupting PageRank/centrality. On a
|
|
1474
|
-
duplicate we keep the MAX weight (strongest association wins) and
|
|
1475
|
-
return the existing edge_id.
|
|
1476
|
-
"""
|
|
1477
|
-
# The edge and the re-projection of its endpoints are one logical write.
|
|
1478
|
-
# Left as separate statements they were three separate commits, which
|
|
1479
|
-
# cost three fsyncs on a path the materializer runs tens of thousands of
|
|
1480
|
-
# times, and left a window where the edge was durable and the intent to
|
|
1481
|
-
# project it was not.
|
|
1482
|
-
def _write() -> str:
|
|
1483
|
-
existing = self.execute(
|
|
1484
|
-
"SELECT edge_id FROM graph_edges "
|
|
1485
|
-
"WHERE profile_id = ? AND source_id = ? AND target_id = ? AND edge_type = ? "
|
|
1486
|
-
"LIMIT 1",
|
|
1487
|
-
(edge.profile_id, edge.source_id, edge.target_id, edge.edge_type.value),
|
|
1488
|
-
)
|
|
1489
|
-
if existing:
|
|
1490
|
-
canonical_id = dict(existing[0])["edge_id"]
|
|
1491
|
-
self.execute(
|
|
1492
|
-
"UPDATE graph_edges SET weight = MAX(weight, ?) WHERE edge_id = ?",
|
|
1493
|
-
(edge.weight, canonical_id),
|
|
1494
|
-
)
|
|
1495
|
-
self._enqueue_edge_endpoints(edge)
|
|
1496
|
-
return canonical_id
|
|
1497
|
-
_scope = getattr(edge, 'scope', None) or 'personal'
|
|
1498
|
-
_shared = _jd(getattr(edge, 'shared_with', None))
|
|
1499
|
-
self.execute(
|
|
1500
|
-
"""INSERT OR REPLACE INTO graph_edges
|
|
1501
|
-
(edge_id, profile_id, source_id, target_id, edge_type, weight, created_at,
|
|
1502
|
-
scope, shared_with)
|
|
1503
|
-
VALUES (?,?,?,?,?,?,?,?,?)""",
|
|
1504
|
-
(edge.edge_id, edge.profile_id, edge.source_id, edge.target_id,
|
|
1505
|
-
edge.edge_type.value, edge.weight, edge.created_at, _scope, _shared),
|
|
1506
|
-
)
|
|
1507
|
-
self._enqueue_edge_endpoints(edge)
|
|
1508
|
-
return edge.edge_id
|
|
1509
|
-
|
|
1510
|
-
return self._atomically(_write)
|
|
1511
|
-
|
|
1512
|
-
def _enqueue_edge_endpoints(self, edge: GraphEdge) -> None:
|
|
1513
|
-
"""Re-queue both ends of an edge for projection.
|
|
1514
|
-
|
|
1515
|
-
Ingestion is queryable-first: a fact commits immediately and its edges
|
|
1516
|
-
arrive afterwards, from the background materializer. A projection
|
|
1517
|
-
queued only when the fact was inserted would therefore be written
|
|
1518
|
-
before a single edge existed, leaving the node in the graph with none
|
|
1519
|
-
of its connections — which is precisely the adjacency the graph is
|
|
1520
|
-
consulted for.
|
|
1521
|
-
|
|
1522
|
-
An endpoint may be an entity id rather than a fact id. Those are queued
|
|
1523
|
-
too and the drain skips whatever it cannot find as a fact; filtering
|
|
1524
|
-
here would mean a lookup per endpoint on every edge write, which the
|
|
1525
|
-
materializer does tens of thousands of times.
|
|
1526
|
-
"""
|
|
1527
|
-
projection_outbox.enqueue_many(
|
|
1528
|
-
self, (edge.source_id, edge.target_id), edge.profile_id,
|
|
1529
|
-
)
|
|
1530
|
-
|
|
1531
|
-
def get_edges_for_node(
|
|
1532
|
-
self, node_id: str, profile_id: str,
|
|
1533
|
-
include_global: bool = False,
|
|
1534
|
-
include_shared: bool = False,
|
|
1535
|
-
) -> list[GraphEdge]:
|
|
1536
|
-
"""All edges where node_id is source or target."""
|
|
1537
|
-
where, params = _scope_where(
|
|
1538
|
-
profile_id,
|
|
1539
|
-
include_global=include_global,
|
|
1540
|
-
include_shared=include_shared,
|
|
1541
|
-
)
|
|
1542
|
-
rows = self.execute(
|
|
1543
|
-
f"SELECT * FROM graph_edges WHERE {where} "
|
|
1544
|
-
"AND (source_id = ? OR target_id = ?)",
|
|
1545
|
-
(*params, node_id, node_id),
|
|
1546
|
-
)
|
|
1547
|
-
return [
|
|
1548
|
-
GraphEdge(
|
|
1549
|
-
edge_id=(d := dict(r))["edge_id"], profile_id=d["profile_id"],
|
|
1550
|
-
source_id=d["source_id"], target_id=d["target_id"],
|
|
1551
|
-
edge_type=EdgeType(d["edge_type"]), weight=d["weight"],
|
|
1552
|
-
created_at=d["created_at"],
|
|
1553
|
-
)
|
|
1554
|
-
for r in rows
|
|
1555
|
-
]
|
|
1556
|
-
|
|
1557
|
-
def store_temporal_event(self, event: TemporalEvent) -> str:
|
|
1558
|
-
"""Persist a temporal event. Returns event_id."""
|
|
1559
|
-
_scope = getattr(event, 'scope', None) or 'personal'
|
|
1560
|
-
_shared = _jd(getattr(event, 'shared_with', None))
|
|
1561
|
-
self.execute(
|
|
1562
|
-
"""INSERT OR REPLACE INTO temporal_events
|
|
1563
|
-
(event_id, profile_id, entity_id, fact_id,
|
|
1564
|
-
observation_date, referenced_date, interval_start, interval_end,
|
|
1565
|
-
description, scope, shared_with)
|
|
1566
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
|
|
1567
|
-
(event.event_id, event.profile_id, event.entity_id, event.fact_id,
|
|
1568
|
-
event.observation_date, event.referenced_date,
|
|
1569
|
-
event.interval_start, event.interval_end, event.description,
|
|
1570
|
-
_scope, _shared),
|
|
1571
|
-
)
|
|
1572
|
-
return event.event_id
|
|
1573
|
-
|
|
1574
|
-
def get_temporal_events(
|
|
1575
|
-
self, entity_id: str, profile_id: str,
|
|
1576
|
-
include_global: bool = False,
|
|
1577
|
-
include_shared: bool = False,
|
|
1578
|
-
) -> list[TemporalEvent]:
|
|
1579
|
-
"""All temporal events for an entity, newest first."""
|
|
1580
|
-
where, params = _scope_where(
|
|
1581
|
-
profile_id,
|
|
1582
|
-
include_global=include_global,
|
|
1583
|
-
include_shared=include_shared,
|
|
1584
|
-
)
|
|
1585
|
-
rows = self.execute(
|
|
1586
|
-
f"SELECT * FROM temporal_events WHERE {where} AND entity_id = ? "
|
|
1587
|
-
"ORDER BY observation_date DESC",
|
|
1588
|
-
(*params, entity_id),
|
|
1589
|
-
)
|
|
1590
|
-
return [
|
|
1591
|
-
TemporalEvent(
|
|
1592
|
-
event_id=(d := dict(r))["event_id"], profile_id=d["profile_id"],
|
|
1593
|
-
entity_id=d["entity_id"], fact_id=d["fact_id"],
|
|
1594
|
-
observation_date=d.get("observation_date"),
|
|
1595
|
-
referenced_date=d.get("referenced_date"),
|
|
1596
|
-
interval_start=d.get("interval_start"),
|
|
1597
|
-
interval_end=d.get("interval_end"),
|
|
1598
|
-
description=d.get("description", ""),
|
|
1599
|
-
)
|
|
1600
|
-
for r in rows
|
|
1601
|
-
]
|
|
1602
|
-
|
|
1603
|
-
def store_bm25_tokens(self, fact_id: str, profile_id: str, tokens: list[str]) -> None:
|
|
1604
|
-
"""Persist BM25 tokens for a fact (survives restart)."""
|
|
1605
|
-
self.execute(
|
|
1606
|
-
"INSERT OR REPLACE INTO bm25_tokens (fact_id, profile_id, tokens) VALUES (?,?,?)",
|
|
1607
|
-
(fact_id, profile_id, json.dumps(tokens)),
|
|
1608
|
-
)
|
|
1609
|
-
|
|
1610
|
-
def get_all_bm25_tokens(
|
|
1611
|
-
self,
|
|
1612
|
-
profile_id: str,
|
|
1613
|
-
include_global: bool = False,
|
|
1614
|
-
include_shared: bool = False,
|
|
1615
|
-
) -> dict[str, list[str]]:
|
|
1616
|
-
"""Load the visible legacy BM25 index: fact_id -> token list."""
|
|
1617
|
-
if not include_global and not include_shared:
|
|
1618
|
-
# Preserve the historical token-store contract, including repair
|
|
1619
|
-
# tooling that can inspect orphaned token rows before facts exist.
|
|
1620
|
-
rows = self.execute(
|
|
1621
|
-
"SELECT fact_id, tokens FROM bm25_tokens WHERE profile_id = ?",
|
|
1622
|
-
(profile_id,),
|
|
1623
|
-
)
|
|
1624
|
-
return {
|
|
1625
|
-
dict(row)["fact_id"]: json.loads(dict(row)["tokens"])
|
|
1626
|
-
for row in rows
|
|
1627
|
-
}
|
|
1628
|
-
where, params = _scope_where(
|
|
1629
|
-
profile_id,
|
|
1630
|
-
include_global=include_global,
|
|
1631
|
-
include_shared=include_shared,
|
|
1632
|
-
prefix="af",
|
|
1633
|
-
)
|
|
1634
|
-
rows = self.execute(
|
|
1635
|
-
"SELECT bt.fact_id, bt.tokens FROM bm25_tokens AS bt "
|
|
1636
|
-
"JOIN atomic_facts AS af ON af.fact_id = bt.fact_id "
|
|
1637
|
-
f"WHERE {where}",
|
|
1638
|
-
(*params,),
|
|
1639
|
-
)
|
|
1640
|
-
return {dict(r)["fact_id"]: json.loads(dict(r)["tokens"]) for r in rows}
|
|
1641
|
-
|
|
1642
|
-
def search_facts_fts(
|
|
1643
|
-
self, query: str, profile_id: str, limit: int = 20,
|
|
1644
|
-
include_global: bool = False,
|
|
1645
|
-
include_shared: bool = False,
|
|
1646
|
-
) -> list[AtomicFact]:
|
|
1647
|
-
"""Full-text search via FTS5, joined to facts table for reconstruction."""
|
|
1648
|
-
# v3.6.12 (search-1): the raw query was passed straight into FTS5 MATCH,
|
|
1649
|
-
# so any '?', '-', quote, or trailing boolean keyword (AND/OR/NOT) raised
|
|
1650
|
-
# an FTS5 syntax error. Tokenize to word characters, quote each token,
|
|
1651
|
-
# and OR-join — mirrors the recall BM25 channel's safe MATCH expression.
|
|
1652
|
-
import re as _re
|
|
1653
|
-
tokens = [t for t in _re.findall(r"\w+", query.lower()) if t]
|
|
1654
|
-
if not tokens:
|
|
1655
|
-
return []
|
|
1656
|
-
match_expr = " OR ".join(f'"{t}"' for t in tokens)
|
|
1657
|
-
where, params = _scope_where(
|
|
1658
|
-
profile_id,
|
|
1659
|
-
include_global=include_global,
|
|
1660
|
-
include_shared=include_shared,
|
|
1661
|
-
prefix="f",
|
|
1662
|
-
)
|
|
1663
|
-
# Full-text search is a display path: the dashboard search box, the
|
|
1664
|
-
# `search` tool and `fetch` all land here, and none of them go through
|
|
1665
|
-
# the recall engine. Before 4.0.10 put the clause here it returned 20
|
|
1666
|
-
# withheld rows out of 50 on the author's store.
|
|
1667
|
-
rows = self.execute(
|
|
1668
|
-
f"""SELECT f.* FROM atomic_facts_fts AS fts
|
|
1669
|
-
JOIN atomic_facts AS f ON f.fact_id = fts.fact_id
|
|
1670
|
-
WHERE fts.atomic_facts_fts MATCH ? AND {where}
|
|
1671
|
-
{self.visible_fact_clause('f')}
|
|
1672
|
-
ORDER BY fts.rank LIMIT ?""",
|
|
1673
|
-
(match_expr, *params, limit),
|
|
1674
|
-
)
|
|
1675
|
-
return [self._row_to_fact(r) for r in rows]
|
|
1676
|
-
|
|
1677
|
-
def list_tables(self) -> set[str]:
|
|
1678
|
-
"""All table names in the database."""
|
|
1679
|
-
rows = self.execute(
|
|
1680
|
-
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
|
|
1681
|
-
)
|
|
1682
|
-
return {dict(r)["name"] for r in rows}
|
|
1683
|
-
|
|
1684
|
-
def get_config(self, key: str) -> str | None:
|
|
1685
|
-
"""Read a config value by key."""
|
|
1686
|
-
rows = self.execute("SELECT value FROM config WHERE key = ?", (key,))
|
|
1687
|
-
return str(rows[0]["value"]) if rows else None
|
|
1688
|
-
|
|
1689
|
-
def set_config(self, key: str, value: str) -> None:
|
|
1690
|
-
"""Write a config value (upsert)."""
|
|
1691
|
-
from datetime import UTC, datetime
|
|
1692
|
-
self.execute(
|
|
1693
|
-
"INSERT OR REPLACE INTO config (key, value, updated_at) VALUES (?,?,?)",
|
|
1694
|
-
(key, value, datetime.now(UTC).isoformat()),
|
|
1695
|
-
)
|
|
1696
|
-
|
|
1697
|
-
# ------------------------------------------------------------------
|
|
1698
|
-
# Phase 0.6: Missing methods (BLOCKER / CRITICAL / HIGH)
|
|
1699
|
-
# ------------------------------------------------------------------
|
|
1700
|
-
|
|
1701
|
-
def get_fact(self, fact_id: str, profile_id: str | None = None) -> AtomicFact | None:
|
|
1702
|
-
"""Get a single row by ID, exactly as stored. NOT a display path.
|
|
1703
|
-
|
|
1704
|
-
C4 defense-in-depth: when ``profile_id`` is provided the lookup is
|
|
1705
|
-
tenant-scoped so a fact_id from another profile cannot resolve. Left
|
|
1706
|
-
optional (fact_id is a random UUID sourced from already-scoped queries)
|
|
1707
|
-
to avoid destabilizing the core store/consolidation write path.
|
|
1708
|
-
|
|
1709
|
-
DELIBERATELY UNFILTERED, and this is load-bearing. It applies neither
|
|
1710
|
-
``archive_status`` nor ``quarantined`` because it is the primitive that
|
|
1711
|
-
write paths, correction handling and the 4.0.10 repair use to read a row
|
|
1712
|
-
they already hold the id of — including a withheld one, which they must
|
|
1713
|
-
be able to see in order to act on it. ``visible_fact_clause`` is for the
|
|
1714
|
-
paths that answer a question; this one answers "what is in that row".
|
|
1715
|
-
|
|
1716
|
-
A caller taking a fact_id from user input and rendering the result wants
|
|
1717
|
-
``get_facts_by_ids`` instead.
|
|
1718
|
-
"""
|
|
1719
|
-
if profile_id is not None:
|
|
1720
|
-
rows = self.execute(
|
|
1721
|
-
"SELECT * FROM atomic_facts WHERE fact_id = ? AND profile_id = ?",
|
|
1722
|
-
(fact_id, profile_id),
|
|
1723
|
-
)
|
|
1724
|
-
else:
|
|
1725
|
-
rows = self.execute(
|
|
1726
|
-
"SELECT * FROM atomic_facts WHERE fact_id = ?", (fact_id,),
|
|
1727
|
-
)
|
|
1728
|
-
return self._row_to_fact(rows[0]) if rows else None
|
|
1729
|
-
|
|
1730
|
-
def get_facts_by_ids(
|
|
1731
|
-
self, fact_ids: list[str], profile_id: str,
|
|
1732
|
-
include_global: bool = False,
|
|
1733
|
-
include_shared: bool = False,
|
|
1734
|
-
*,
|
|
1735
|
-
include_quarantined: bool = False,
|
|
1736
|
-
) -> list[AtomicFact]:
|
|
1737
|
-
"""Get multiple facts by their IDs, scoped to a profile.
|
|
1738
|
-
|
|
1739
|
-
THIS IS THE PLACE QUARANTINE IS ENFORCED, and the only one.
|
|
1740
|
-
|
|
1741
|
-
Every retrieval channel re-authorises its candidates through here
|
|
1742
|
-
(``retrieval/scope_policy.py`` — "candidate generators may use caches,
|
|
1743
|
-
approximate indexes, or graph stores that are not the authorization
|
|
1744
|
-
source of truth"), and the engine hydrates the fused set from here too.
|
|
1745
|
-
A fact this method does not return has no content to show, and
|
|
1746
|
-
``retrieval/engine.py`` drops it: ``if fact is None: continue``. So one
|
|
1747
|
-
clause here covers bm25, semantic, temporal, entity, hopfield and
|
|
1748
|
-
spreading activation, in normal and deep recall alike, whether or not
|
|
1749
|
-
the forgetting filter is registered.
|
|
1750
|
-
|
|
1751
|
-
The alternatives were checked and rejected. ``_scope_where`` looks like
|
|
1752
|
-
the natural home but is spliced against ``graph_edges``,
|
|
1753
|
-
``temporal_events``, ``memories``, ``bm25_tokens``,
|
|
1754
|
-
``fact_temporal_validity`` and ``correction_cases`` as well as
|
|
1755
|
-
``atomic_facts``, so a column reference there breaks eight call sites.
|
|
1756
|
-
``ForgettingFilter`` is optional (it no-ops when forgetting is
|
|
1757
|
-
disabled) and excludes nothing in deep recall.
|
|
1758
|
-
|
|
1759
|
-
``include_quarantined=True`` is for repair, export and erasure — paths
|
|
1760
|
-
that must be able to see a withheld row in order to act on it. It is
|
|
1761
|
-
keyword-only and greppable on purpose: every caller that opts in is
|
|
1762
|
-
meant to be found in one search.
|
|
1763
|
-
"""
|
|
1764
|
-
if not fact_ids:
|
|
1765
|
-
return []
|
|
1766
|
-
where, params = _scope_where(
|
|
1767
|
-
profile_id,
|
|
1768
|
-
include_global=include_global,
|
|
1769
|
-
include_shared=include_shared,
|
|
1770
|
-
)
|
|
1771
|
-
placeholders = ",".join("?" for _ in fact_ids)
|
|
1772
|
-
rows = self.execute(
|
|
1773
|
-
f"SELECT * FROM atomic_facts WHERE fact_id IN ({placeholders}) "
|
|
1774
|
-
f"AND {where}"
|
|
1775
|
-
f"{self.visible_fact_clause(include_quarantined=include_quarantined)} "
|
|
1776
|
-
"ORDER BY created_at DESC",
|
|
1777
|
-
(*fact_ids, *params),
|
|
1778
|
-
)
|
|
1779
|
-
return [self._row_to_fact(r) for r in rows]
|
|
1780
|
-
|
|
1781
|
-
def visible_fact_ids(
|
|
1782
|
-
self, fact_ids: list[str] | tuple[str, ...], profile_id: str,
|
|
1783
|
-
include_global: bool = False,
|
|
1784
|
-
include_shared: bool = False,
|
|
1785
|
-
*,
|
|
1786
|
-
include_quarantined: bool = False,
|
|
1787
|
-
) -> set[str]:
|
|
1788
|
-
"""Which of *fact_ids* this profile may see. Same rule, no hydration.
|
|
1789
|
-
|
|
1790
|
-
``get_facts_by_ids`` is the authorization source of truth and every
|
|
1791
|
-
retrieval channel re-authorises through it — but a channel deciding
|
|
1792
|
-
*which* of its candidates are allowed does not need their content, and
|
|
1793
|
-
paying for the content is most of what recall costs.
|
|
1794
|
-
|
|
1795
|
-
Measured on the author's store: the entity channel authorised 3,659
|
|
1796
|
-
candidates to return 20, and that single call was **374 ms of a 430 ms
|
|
1797
|
-
recall — 87%**. Not the graph walk, which is 3.8 ms. The cost is
|
|
1798
|
-
``_row_to_fact`` decoding a 768-float embedding and two 768-float Fisher
|
|
1799
|
-
vectors per row: about 8.4 million floats deserialised to answer a
|
|
1800
|
-
yes/no question about 3,659 ids. The same trap is recorded a few
|
|
1801
|
-
hundred lines up, where loading full facts "turned one new fact into a
|
|
1802
|
-
5-second recall stall".
|
|
1803
|
-
|
|
1804
|
-
The predicate is built by the same two calls as ``get_facts_by_ids``, in
|
|
1805
|
-
the same order, so the two cannot answer differently. A test asserts
|
|
1806
|
-
that on the same inputs. Batched because the id list is unbounded and a
|
|
1807
|
-
single ``IN`` clause is not.
|
|
1808
|
-
"""
|
|
1809
|
-
if not fact_ids:
|
|
1810
|
-
return set()
|
|
1811
|
-
where, params = _scope_where(
|
|
1812
|
-
profile_id,
|
|
1813
|
-
include_global=include_global,
|
|
1814
|
-
include_shared=include_shared,
|
|
1815
|
-
)
|
|
1816
|
-
visible = self.visible_fact_clause(include_quarantined=include_quarantined)
|
|
1817
|
-
unique = list(dict.fromkeys(fact_ids))
|
|
1818
|
-
allowed: set[str] = set()
|
|
1819
|
-
# Well inside SQLITE_MAX_VARIABLE_NUMBER once the scope parameters are
|
|
1820
|
-
# added, on every build this ships against.
|
|
1821
|
-
chunk = 800
|
|
1822
|
-
for start in range(0, len(unique), chunk):
|
|
1823
|
-
batch = unique[start:start + chunk]
|
|
1824
|
-
placeholders = ",".join("?" for _ in batch)
|
|
1825
|
-
rows = self.execute(
|
|
1826
|
-
f"SELECT fact_id FROM atomic_facts WHERE fact_id IN ({placeholders}) "
|
|
1827
|
-
f"AND {where}{visible}",
|
|
1828
|
-
(*batch, *params),
|
|
1829
|
-
)
|
|
1830
|
-
allowed.update(dict(row)["fact_id"] for row in rows)
|
|
1831
|
-
return allowed
|
|
1832
|
-
|
|
1833
|
-
def store_entity_profile(self, ep: EntityProfile) -> str:
|
|
1834
|
-
"""Persist an entity profile. Returns profile_entry_id."""
|
|
1835
|
-
self.execute(
|
|
1836
|
-
"""INSERT OR REPLACE INTO entity_profiles
|
|
1837
|
-
(profile_entry_id, entity_id, profile_id,
|
|
1838
|
-
knowledge_summary, fact_ids_json, last_updated)
|
|
1839
|
-
VALUES (?,?,?,?,?,?)""",
|
|
1840
|
-
(ep.profile_entry_id, ep.entity_id, ep.profile_id,
|
|
1841
|
-
ep.knowledge_summary, json.dumps(ep.fact_ids), ep.last_updated),
|
|
1842
|
-
)
|
|
1843
|
-
return ep.profile_entry_id
|
|
1844
|
-
|
|
1845
|
-
def get_entity_profiles_by_entity(
|
|
1846
|
-
self, entity_id: str, profile_id: str,
|
|
1847
|
-
) -> list[EntityProfile]:
|
|
1848
|
-
"""All profile entries for an entity within a profile scope."""
|
|
1849
|
-
rows = self.execute(
|
|
1850
|
-
"SELECT * FROM entity_profiles WHERE entity_id = ? AND profile_id = ? "
|
|
1851
|
-
"ORDER BY last_updated DESC",
|
|
1852
|
-
(entity_id, profile_id),
|
|
1853
|
-
)
|
|
1854
|
-
return [
|
|
1855
|
-
EntityProfile(
|
|
1856
|
-
profile_entry_id=(d := dict(r))["profile_entry_id"],
|
|
1857
|
-
entity_id=d["entity_id"], profile_id=d["profile_id"],
|
|
1858
|
-
knowledge_summary=d["knowledge_summary"],
|
|
1859
|
-
fact_ids=_jl(d.get("fact_ids_json")),
|
|
1860
|
-
last_updated=d["last_updated"],
|
|
1861
|
-
)
|
|
1862
|
-
for r in rows
|
|
1863
|
-
]
|
|
1864
|
-
|
|
1865
|
-
def store_scene(self, scene: MemoryScene) -> str:
|
|
1866
|
-
"""Persist a memory scene. Returns scene_id."""
|
|
1867
|
-
self.execute(
|
|
1868
|
-
"""INSERT OR REPLACE INTO memory_scenes
|
|
1869
|
-
(scene_id, profile_id, theme, fact_ids_json,
|
|
1870
|
-
entity_ids_json, created_at, last_updated)
|
|
1871
|
-
VALUES (?,?,?,?,?,?,?)""",
|
|
1872
|
-
(scene.scene_id, scene.profile_id, scene.theme,
|
|
1873
|
-
json.dumps(scene.fact_ids), json.dumps(scene.entity_ids),
|
|
1874
|
-
scene.created_at, scene.last_updated),
|
|
1875
|
-
)
|
|
1876
|
-
return scene.scene_id
|
|
1877
|
-
|
|
1878
|
-
def _row_to_scene(self, row: sqlite3.Row) -> MemoryScene:
|
|
1879
|
-
"""Deserialize a row into MemoryScene."""
|
|
1880
|
-
d = dict(row)
|
|
1881
|
-
return MemoryScene(
|
|
1882
|
-
scene_id=d["scene_id"], profile_id=d["profile_id"],
|
|
1883
|
-
theme=d.get("theme", ""),
|
|
1884
|
-
fact_ids=_jl(d.get("fact_ids_json")),
|
|
1885
|
-
entity_ids=_jl(d.get("entity_ids_json")),
|
|
1886
|
-
created_at=d["created_at"], last_updated=d["last_updated"],
|
|
1887
|
-
)
|
|
1888
|
-
|
|
1889
|
-
def get_scene(self, scene_id: str) -> MemoryScene | None:
|
|
1890
|
-
"""Get a scene by ID."""
|
|
1891
|
-
rows = self.execute(
|
|
1892
|
-
"SELECT * FROM memory_scenes WHERE scene_id = ?", (scene_id,),
|
|
1893
|
-
)
|
|
1894
|
-
return self._row_to_scene(rows[0]) if rows else None
|
|
1895
|
-
|
|
1896
|
-
def get_all_scenes(self, profile_id: str) -> list[MemoryScene]:
|
|
1897
|
-
"""All scenes for a profile, newest first."""
|
|
1898
|
-
rows = self.execute(
|
|
1899
|
-
"SELECT * FROM memory_scenes WHERE profile_id = ? "
|
|
1900
|
-
"ORDER BY last_updated DESC",
|
|
1901
|
-
(profile_id,),
|
|
1902
|
-
)
|
|
1903
|
-
return [self._row_to_scene(r) for r in rows]
|
|
1904
|
-
|
|
1905
|
-
def get_scenes_for_fact(
|
|
1906
|
-
self, fact_id: str, profile_id: str,
|
|
1907
|
-
) -> list[MemoryScene]:
|
|
1908
|
-
"""All scenes whose fact_ids JSON array contains *fact_id*."""
|
|
1909
|
-
rows = self.execute(
|
|
1910
|
-
"SELECT * FROM memory_scenes WHERE profile_id = ? "
|
|
1911
|
-
"AND fact_ids_json LIKE ? ORDER BY last_updated DESC",
|
|
1912
|
-
(profile_id, f'%"{fact_id}"%'),
|
|
1913
|
-
)
|
|
1914
|
-
return [self._row_to_scene(r) for r in rows]
|
|
1915
|
-
|
|
1916
|
-
def get_scenes_for_facts_batch(
|
|
1917
|
-
self, fact_ids: list[str], profile_id: str,
|
|
1918
|
-
) -> dict[str, list[MemoryScene]]:
|
|
1919
|
-
"""v3.5.0: batch scene lookup — one query replaces N individual LIKE scans.
|
|
1920
|
-
|
|
1921
|
-
The 20 individual ``get_scenes_for_fact`` calls in the retrieval engine's
|
|
1922
|
-
scene expansion path were the single largest recall latency source (~5.7s).
|
|
1923
|
-
This replaces them with a single multi-LIKE OR query. Returns
|
|
1924
|
-
``{fact_id: [scenes]}`` for facts that belong to at least one scene.
|
|
1925
|
-
"""
|
|
1926
|
-
if not fact_ids:
|
|
1927
|
-
return {}
|
|
1928
|
-
clauses = " OR ".join(
|
|
1929
|
-
'(profile_id = ? AND fact_ids_json LIKE ?)' for _ in fact_ids
|
|
1930
|
-
)
|
|
1931
|
-
params: list[str] = []
|
|
1932
|
-
for fid in fact_ids:
|
|
1933
|
-
params.extend((profile_id, f'%"{fid}"%'))
|
|
1934
|
-
rows = self.execute(
|
|
1935
|
-
f"SELECT * FROM memory_scenes WHERE {clauses} ORDER BY last_updated DESC",
|
|
1936
|
-
tuple(params),
|
|
1937
|
-
)
|
|
1938
|
-
out: dict[str, list[MemoryScene]] = {}
|
|
1939
|
-
for r in rows:
|
|
1940
|
-
scene = self._row_to_scene(r)
|
|
1941
|
-
for fid in fact_ids:
|
|
1942
|
-
if fid in (scene.fact_ids or []):
|
|
1943
|
-
out.setdefault(fid, []).append(scene)
|
|
1944
|
-
return out
|
|
1945
|
-
|
|
1946
|
-
def increment_entity_fact_count(self, entity_id: str, profile_id: str = "default") -> None:
|
|
1947
|
-
"""Atomically increment fact_count for a canonical entity scoped to profile.
|
|
1948
|
-
|
|
1949
|
-
L-01 fix: the original query had no profile_id guard; any entity_id match
|
|
1950
|
-
would be updated regardless of owner. The AND profile_id = ? clause prevents
|
|
1951
|
-
cross-profile mutations via shared entity_id values.
|
|
1952
|
-
"""
|
|
1953
|
-
self.execute(
|
|
1954
|
-
"UPDATE canonical_entities SET fact_count = fact_count + 1 "
|
|
1955
|
-
"WHERE entity_id = ? AND profile_id = ?",
|
|
1956
|
-
(entity_id, profile_id),
|
|
1957
|
-
)
|
|
1958
|
-
|
|
1959
|
-
def store_trust_score(self, ts: TrustScore) -> str:
|
|
1960
|
-
"""Persist a trust score. Returns trust_id."""
|
|
1961
|
-
self.execute(
|
|
1962
|
-
"""INSERT OR REPLACE INTO trust_scores
|
|
1963
|
-
(trust_id, profile_id, target_type, target_id,
|
|
1964
|
-
trust_score, evidence_count, last_updated)
|
|
1965
|
-
VALUES (?,?,?,?,?,?,?)""",
|
|
1966
|
-
(ts.trust_id, ts.profile_id, ts.target_type, ts.target_id,
|
|
1967
|
-
ts.trust_score, ts.evidence_count, ts.last_updated),
|
|
1968
|
-
)
|
|
1969
|
-
return ts.trust_id
|
|
1970
|
-
|
|
1971
|
-
def get_trust_score(
|
|
1972
|
-
self, target_type: str, target_id: str, profile_id: str,
|
|
1973
|
-
) -> TrustScore | None:
|
|
1974
|
-
"""Look up trust score for a specific target."""
|
|
1975
|
-
rows = self.execute(
|
|
1976
|
-
"SELECT * FROM trust_scores WHERE target_type = ? "
|
|
1977
|
-
"AND target_id = ? AND profile_id = ?",
|
|
1978
|
-
(target_type, target_id, profile_id),
|
|
1979
|
-
)
|
|
1980
|
-
if not rows:
|
|
1981
|
-
return None
|
|
1982
|
-
d = dict(rows[0])
|
|
1983
|
-
return TrustScore(
|
|
1984
|
-
trust_id=d["trust_id"], profile_id=d["profile_id"],
|
|
1985
|
-
target_type=d["target_type"], target_id=d["target_id"],
|
|
1986
|
-
trust_score=d["trust_score"], evidence_count=d["evidence_count"],
|
|
1987
|
-
last_updated=d["last_updated"],
|
|
1988
|
-
)
|
|
1989
|
-
|
|
1990
|
-
def store_consolidation_action(self, action: ConsolidationAction) -> str:
|
|
1991
|
-
"""Log a consolidation decision. Returns action_id."""
|
|
1992
|
-
self.execute(
|
|
1993
|
-
"""INSERT OR REPLACE INTO consolidation_log
|
|
1994
|
-
(action_id, profile_id, action_type, new_fact_id,
|
|
1995
|
-
existing_fact_id, reason, timestamp)
|
|
1996
|
-
VALUES (?,?,?,?,?,?,?)""",
|
|
1997
|
-
(action.action_id, action.profile_id,
|
|
1998
|
-
action.action_type.value, action.new_fact_id,
|
|
1999
|
-
action.existing_fact_id, action.reason, action.timestamp),
|
|
2000
|
-
)
|
|
2001
|
-
return action.action_id
|
|
2002
|
-
|
|
2003
|
-
def get_temporal_events_by_range(
|
|
2004
|
-
self, profile_id: str, start_date: str, end_date: str,
|
|
2005
|
-
include_global: bool = False,
|
|
2006
|
-
include_shared: bool = False,
|
|
2007
|
-
) -> list[TemporalEvent]:
|
|
2008
|
-
"""Temporal events within a date range (inclusive)."""
|
|
2009
|
-
where, params = _scope_where(
|
|
2010
|
-
profile_id,
|
|
2011
|
-
include_global=include_global,
|
|
2012
|
-
include_shared=include_shared,
|
|
2013
|
-
)
|
|
2014
|
-
rows = self.execute(
|
|
2015
|
-
f"SELECT * FROM temporal_events WHERE {where} "
|
|
2016
|
-
"AND (referenced_date BETWEEN ? AND ? "
|
|
2017
|
-
" OR observation_date BETWEEN ? AND ?) "
|
|
2018
|
-
"ORDER BY observation_date DESC",
|
|
2019
|
-
(*params, start_date, end_date, start_date, end_date),
|
|
2020
|
-
)
|
|
2021
|
-
return [
|
|
2022
|
-
TemporalEvent(
|
|
2023
|
-
event_id=(d := dict(r))["event_id"],
|
|
2024
|
-
profile_id=d["profile_id"],
|
|
2025
|
-
entity_id=d["entity_id"], fact_id=d["fact_id"],
|
|
2026
|
-
observation_date=d.get("observation_date"),
|
|
2027
|
-
referenced_date=d.get("referenced_date"),
|
|
2028
|
-
interval_start=d.get("interval_start"),
|
|
2029
|
-
interval_end=d.get("interval_end"),
|
|
2030
|
-
description=d.get("description", ""),
|
|
2031
|
-
)
|
|
2032
|
-
for r in rows
|
|
2033
|
-
]
|
|
2034
|
-
|
|
2035
|
-
# ------------------------------------------------------------------
|
|
2036
|
-
# Phase 2: fact_context CRUD (Auto-Invoke Engine)
|
|
2037
|
-
# ------------------------------------------------------------------
|
|
2038
|
-
|
|
2039
|
-
def store_fact_context(
|
|
2040
|
-
self,
|
|
2041
|
-
fact_id: str,
|
|
2042
|
-
profile_id: str,
|
|
2043
|
-
contextual_description: str,
|
|
2044
|
-
keywords: str,
|
|
2045
|
-
generated_by: str = "rules",
|
|
2046
|
-
) -> None:
|
|
2047
|
-
"""Store or replace contextual description for a fact."""
|
|
2048
|
-
self.execute(
|
|
2049
|
-
"INSERT OR REPLACE INTO fact_context "
|
|
2050
|
-
"(fact_id, profile_id, contextual_description, keywords, generated_by) "
|
|
2051
|
-
"VALUES (?, ?, ?, ?, ?)",
|
|
2052
|
-
(fact_id, profile_id, contextual_description, keywords, generated_by),
|
|
2053
|
-
)
|
|
2054
|
-
|
|
2055
|
-
def get_fact_context(self, fact_id: str, profile_id: str | None = None) -> dict | None:
|
|
2056
|
-
"""Get contextual description for a fact (C4: optionally tenant-scoped)."""
|
|
2057
|
-
if profile_id is not None:
|
|
2058
|
-
rows = self.execute(
|
|
2059
|
-
"SELECT * FROM fact_context WHERE fact_id = ? AND profile_id = ?",
|
|
2060
|
-
(fact_id, profile_id),
|
|
2061
|
-
)
|
|
2062
|
-
else:
|
|
2063
|
-
rows = self.execute(
|
|
2064
|
-
"SELECT * FROM fact_context WHERE fact_id = ?", (fact_id,),
|
|
2065
|
-
)
|
|
2066
|
-
return dict(rows[0]) if rows else None
|
|
2067
|
-
|
|
2068
|
-
def get_all_fact_contexts(self, profile_id: str) -> list[dict]:
|
|
2069
|
-
"""Get all contextual descriptions for a profile."""
|
|
2070
|
-
rows = self.execute(
|
|
2071
|
-
"SELECT * FROM fact_context WHERE profile_id = ?", (profile_id,),
|
|
2072
|
-
)
|
|
2073
|
-
return [dict(r) for r in rows]
|
|
2074
|
-
|
|
2075
|
-
def delete_fact_context(self, fact_id: str) -> None:
|
|
2076
|
-
"""Delete contextual description for a fact."""
|
|
2077
|
-
self.execute("DELETE FROM fact_context WHERE fact_id = ?", (fact_id,))
|
|
2078
|
-
|
|
2079
|
-
# ------------------------------------------------------------------
|
|
2080
|
-
# Phase 3: Association Graph CRUD (Rule 15)
|
|
2081
|
-
# ------------------------------------------------------------------
|
|
2082
|
-
|
|
2083
|
-
def store_association_edge(self, edge: dict) -> None:
|
|
2084
|
-
"""Persist an association edge."""
|
|
2085
|
-
self.execute(
|
|
2086
|
-
"INSERT OR IGNORE INTO association_edges "
|
|
2087
|
-
"(edge_id, profile_id, source_fact_id, target_fact_id, "
|
|
2088
|
-
" association_type, weight, co_access_count, created_at) "
|
|
2089
|
-
"VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))",
|
|
2090
|
-
(edge["edge_id"], edge["profile_id"],
|
|
2091
|
-
edge["source_fact_id"], edge["target_fact_id"],
|
|
2092
|
-
edge["association_type"], edge["weight"],
|
|
2093
|
-
edge.get("co_access_count", 0)),
|
|
2094
|
-
)
|
|
2095
|
-
|
|
2096
|
-
def get_association_edges(
|
|
2097
|
-
self, fact_id: str, profile_id: str,
|
|
2098
|
-
) -> list[dict]:
|
|
2099
|
-
"""All association edges where fact_id is source or target."""
|
|
2100
|
-
rows = self.execute(
|
|
2101
|
-
"SELECT * FROM association_edges WHERE profile_id = ? "
|
|
2102
|
-
"AND (source_fact_id = ? OR target_fact_id = ?)",
|
|
2103
|
-
(profile_id, fact_id, fact_id),
|
|
2104
|
-
)
|
|
2105
|
-
return [dict(r) for r in rows]
|
|
2106
|
-
|
|
2107
|
-
def get_all_association_edges(self, profile_id: str) -> list[dict]:
|
|
2108
|
-
"""All association edges for a profile."""
|
|
2109
|
-
rows = self.execute(
|
|
2110
|
-
"SELECT * FROM association_edges WHERE profile_id = ?",
|
|
2111
|
-
(profile_id,),
|
|
2112
|
-
)
|
|
2113
|
-
return [dict(r) for r in rows]
|
|
2114
|
-
|
|
2115
|
-
def delete_association_edges(self, profile_id: str) -> int:
|
|
2116
|
-
"""Delete all association edges for a profile. Returns count."""
|
|
2117
|
-
before = self.execute(
|
|
2118
|
-
"SELECT COUNT(*) AS c FROM association_edges WHERE profile_id = ?",
|
|
2119
|
-
(profile_id,),
|
|
2120
|
-
)
|
|
2121
|
-
count = int(before[0]["c"]) if before else 0
|
|
2122
|
-
self.execute(
|
|
2123
|
-
"DELETE FROM association_edges WHERE profile_id = ?",
|
|
2124
|
-
(profile_id,),
|
|
2125
|
-
)
|
|
2126
|
-
return count
|
|
2127
|
-
|
|
2128
|
-
def store_activation_cache(self, entry: dict) -> None:
|
|
2129
|
-
"""Persist an activation cache entry."""
|
|
2130
|
-
self.execute(
|
|
2131
|
-
"INSERT OR REPLACE INTO activation_cache "
|
|
2132
|
-
"(cache_id, profile_id, query_hash, node_id, activation_value, "
|
|
2133
|
-
" iteration, created_at, expires_at) "
|
|
2134
|
-
"VALUES (?, ?, ?, ?, ?, ?, datetime('now'), datetime('now', '+1 hour'))",
|
|
2135
|
-
(entry["cache_id"], entry["profile_id"],
|
|
2136
|
-
entry["query_hash"], entry["node_id"],
|
|
2137
|
-
entry["activation_value"], entry["iteration"]),
|
|
2138
|
-
)
|
|
2139
|
-
|
|
2140
|
-
def get_activation_cache(
|
|
2141
|
-
self, query_hash: str, profile_id: str,
|
|
2142
|
-
) -> list[dict]:
|
|
2143
|
-
"""Get cached activation results (non-expired)."""
|
|
2144
|
-
rows = self.execute(
|
|
2145
|
-
"SELECT node_id, activation_value FROM activation_cache "
|
|
2146
|
-
"WHERE profile_id = ? AND query_hash = ? "
|
|
2147
|
-
"AND expires_at > datetime('now') "
|
|
2148
|
-
"ORDER BY activation_value DESC",
|
|
2149
|
-
(profile_id, query_hash),
|
|
2150
|
-
)
|
|
2151
|
-
return [dict(r) for r in rows]
|
|
2152
|
-
|
|
2153
|
-
def cleanup_activation_cache(
|
|
2154
|
-
self, batch_size: int = 5000, max_batches: int = 500,
|
|
2155
|
-
) -> int:
|
|
2156
|
-
"""Delete expired activation_cache rows in bounded batches.
|
|
2157
|
-
|
|
2158
|
-
Wired into MaintenanceScheduler. Historically NEITHER cleanup path was
|
|
2159
|
-
ever called, so activation_cache grew without bound — observed 83,518
|
|
2160
|
-
rows on a real DB, all expired, oldest ~3.5 months old. That bloats the
|
|
2161
|
-
table and its ``idx_actcache_expires`` index and slows every cache
|
|
2162
|
-
INSERT OR REPLACE / lookup.
|
|
2163
|
-
|
|
2164
|
-
Batched so clearing a large backlog never holds the write lock for one
|
|
2165
|
-
long DELETE: each batch commits and yields, letting remember/materialize
|
|
2166
|
-
writers interleave. ``idx_actcache_expires`` makes the predicate
|
|
2167
|
-
index-backed. Steady state (30-min cycle) deletes only one cycle's
|
|
2168
|
-
worth, so the loop exits after a single small batch.
|
|
2169
|
-
"""
|
|
2170
|
-
total_deleted = 0
|
|
2171
|
-
for _ in range(max_batches):
|
|
2172
|
-
remaining = self.execute(
|
|
2173
|
-
"SELECT COUNT(*) AS c FROM activation_cache "
|
|
2174
|
-
"WHERE expires_at < datetime('now')"
|
|
2175
|
-
)
|
|
2176
|
-
n = int(remaining[0]["c"]) if remaining else 0
|
|
2177
|
-
if n <= 0:
|
|
2178
|
-
break
|
|
2179
|
-
self.execute(
|
|
2180
|
-
"DELETE FROM activation_cache WHERE cache_id IN ("
|
|
2181
|
-
" SELECT cache_id FROM activation_cache "
|
|
2182
|
-
" WHERE expires_at < datetime('now') LIMIT ?)",
|
|
2183
|
-
(batch_size,),
|
|
2184
|
-
)
|
|
2185
|
-
total_deleted += min(n, batch_size)
|
|
2186
|
-
return total_deleted
|
|
2187
|
-
|
|
2188
|
-
def store_fact_importance(self, entry: dict) -> None:
|
|
2189
|
-
"""Persist fact importance scores."""
|
|
2190
|
-
self.execute(
|
|
2191
|
-
"INSERT OR REPLACE INTO fact_importance "
|
|
2192
|
-
"(fact_id, profile_id, pagerank_score, community_id, "
|
|
2193
|
-
" degree_centrality, computed_at) "
|
|
2194
|
-
"VALUES (?, ?, ?, ?, ?, datetime('now'))",
|
|
2195
|
-
(entry["fact_id"], entry["profile_id"],
|
|
2196
|
-
entry["pagerank_score"], entry.get("community_id"),
|
|
2197
|
-
entry.get("degree_centrality", 0.0)),
|
|
2198
|
-
)
|
|
2199
|
-
|
|
2200
|
-
def get_fact_importance(
|
|
2201
|
-
self, fact_id: str, profile_id: str,
|
|
2202
|
-
) -> dict | None:
|
|
2203
|
-
"""Get importance scores for a fact."""
|
|
2204
|
-
rows = self.execute(
|
|
2205
|
-
"SELECT * FROM fact_importance "
|
|
2206
|
-
"WHERE fact_id = ? AND profile_id = ?",
|
|
2207
|
-
(fact_id, profile_id),
|
|
2208
|
-
)
|
|
2209
|
-
return dict(rows[0]) if rows else None
|
|
2210
|
-
|
|
2211
|
-
def get_top_facts_by_pagerank(
|
|
2212
|
-
self, profile_id: str, top_k: int = 20,
|
|
2213
|
-
) -> list[dict]:
|
|
2214
|
-
"""Top facts by PageRank score."""
|
|
2215
|
-
rows = self.execute(
|
|
2216
|
-
"SELECT * FROM fact_importance "
|
|
2217
|
-
"WHERE profile_id = ? "
|
|
2218
|
-
"ORDER BY pagerank_score DESC LIMIT ?",
|
|
2219
|
-
(profile_id, top_k),
|
|
2220
|
-
)
|
|
2221
|
-
return [dict(r) for r in rows]
|
|
2222
|
-
|
|
2223
|
-
# ------------------------------------------------------------------
|
|
2224
|
-
# Phase 4: Temporal Intelligence CRUD (Rule 15)
|
|
2225
|
-
# ------------------------------------------------------------------
|
|
2226
|
-
|
|
2227
|
-
def store_temporal_validity(
|
|
2228
|
-
self, fact_id: str, profile_id: str,
|
|
2229
|
-
valid_from: str | None = None,
|
|
2230
|
-
valid_until: str | None = None,
|
|
2231
|
-
) -> None:
|
|
2232
|
-
"""Create or enrich the temporal record for a fact.
|
|
2233
|
-
|
|
2234
|
-
The 4.0.2 fact writer creates the record immediately to anchor
|
|
2235
|
-
transaction time. A later temporal extraction may add event-time bounds;
|
|
2236
|
-
it must not be discarded merely because the anchor already exists.
|
|
2237
|
-
"""
|
|
2238
|
-
from datetime import UTC
|
|
2239
|
-
from datetime import datetime as _dt
|
|
2240
|
-
system_created_at = _dt.now(UTC).isoformat()
|
|
2241
|
-
self.execute(
|
|
2242
|
-
"INSERT OR IGNORE INTO fact_temporal_validity "
|
|
2243
|
-
"(fact_id, profile_id, valid_from, valid_until, system_created_at) "
|
|
2244
|
-
"VALUES (?, ?, ?, ?, ?)",
|
|
2245
|
-
(fact_id, profile_id, valid_from, valid_until, system_created_at),
|
|
2246
|
-
)
|
|
2247
|
-
if valid_from is not None or valid_until is not None:
|
|
2248
|
-
self.execute(
|
|
2249
|
-
"UPDATE fact_temporal_validity "
|
|
2250
|
-
"SET valid_from = COALESCE(?, valid_from), "
|
|
2251
|
-
" valid_until = COALESCE(?, valid_until) "
|
|
2252
|
-
"WHERE fact_id = ? AND profile_id = ?",
|
|
2253
|
-
(valid_from, valid_until, fact_id, profile_id),
|
|
2254
|
-
)
|
|
2255
|
-
|
|
2256
|
-
def get_temporal_validity(self, fact_id: str, profile_id: str | None = None) -> dict | None:
|
|
2257
|
-
"""Get temporal validity record for a fact (C4: optionally tenant-scoped)."""
|
|
2258
|
-
if profile_id is not None:
|
|
2259
|
-
rows = self.execute(
|
|
2260
|
-
"SELECT * FROM fact_temporal_validity WHERE fact_id = ? AND profile_id = ?",
|
|
2261
|
-
(fact_id, profile_id),
|
|
2262
|
-
)
|
|
2263
|
-
else:
|
|
2264
|
-
rows = self.execute(
|
|
2265
|
-
"SELECT * FROM fact_temporal_validity WHERE fact_id = ?",
|
|
2266
|
-
(fact_id,),
|
|
2267
|
-
)
|
|
2268
|
-
return dict(rows[0]) if rows else None
|
|
2269
|
-
|
|
2270
|
-
def get_all_temporal_validity(self, profile_id: str) -> list[dict]:
|
|
2271
|
-
"""Get all temporal validity records for a profile."""
|
|
2272
|
-
rows = self.execute(
|
|
2273
|
-
"SELECT * FROM fact_temporal_validity WHERE profile_id = ?",
|
|
2274
|
-
(profile_id,),
|
|
2275
|
-
)
|
|
2276
|
-
return [dict(r) for r in rows]
|
|
2277
|
-
|
|
2278
|
-
def invalidate_fact_temporal(
|
|
2279
|
-
self, fact_id: str, invalidated_by: str,
|
|
2280
|
-
invalidation_reason: str,
|
|
2281
|
-
*,
|
|
2282
|
-
event_valid_until: str | None = None,
|
|
2283
|
-
) -> None:
|
|
2284
|
-
"""Mark a fact as invalidated, preserving bi-temporal independence.
|
|
2285
|
-
|
|
2286
|
-
- valid_until (event-time): changed only when a reviewer supplies an
|
|
2287
|
-
independently validated real-world boundary. Review time, source
|
|
2288
|
-
timestamps, and a detector's conclusion are not a valid substitute.
|
|
2289
|
-
- system_expired_at (transaction-time): when the system learned the
|
|
2290
|
-
fact was invalid — always set to now.
|
|
2291
|
-
|
|
2292
|
-
The two dimensions must remain independent: a fact can be true until
|
|
2293
|
-
2020-06-30 in the real world (valid_until) while the system only
|
|
2294
|
-
discovers this in 2024 (system_expired_at).
|
|
2295
|
-
|
|
2296
|
-
Never deletes the fact (immutability).
|
|
2297
|
-
"""
|
|
2298
|
-
from datetime import UTC
|
|
2299
|
-
from datetime import datetime as _dt
|
|
2300
|
-
now = _dt.now(UTC).isoformat()
|
|
2301
|
-
|
|
2302
|
-
self.execute(
|
|
2303
|
-
"UPDATE fact_temporal_validity "
|
|
2304
|
-
"SET valid_until = COALESCE(?, valid_until), system_expired_at = ?, "
|
|
2305
|
-
" invalidated_by = ?, invalidation_reason = ? "
|
|
2306
|
-
"WHERE fact_id = ? AND system_expired_at IS NULL",
|
|
2307
|
-
(event_valid_until, now, invalidated_by, invalidation_reason, fact_id),
|
|
2308
|
-
)
|
|
2309
|
-
|
|
2310
|
-
def get_valid_facts(self, profile_id: str) -> list[str]:
|
|
2311
|
-
"""Get fact_ids that are currently valid (not expired).
|
|
2312
|
-
|
|
2313
|
-
Returns facts that either have no temporal record (assumed valid)
|
|
2314
|
-
or whose temporal record satisfies:
|
|
2315
|
-
- valid_until IS NULL (open-ended) OR valid_until > now() (still in window)
|
|
2316
|
-
- system_expired_at IS NULL (not system-invalidated)
|
|
2317
|
-
|
|
2318
|
-
M-02 fix: the original query used ``tv.valid_until IS NULL`` which
|
|
2319
|
-
incorrectly excluded future-dated (still valid) facts. The correct
|
|
2320
|
-
predicate is a date comparison against the current timestamp.
|
|
2321
|
-
"""
|
|
2322
|
-
rows = self.execute(
|
|
2323
|
-
"SELECT f.fact_id FROM atomic_facts f "
|
|
2324
|
-
"LEFT JOIN fact_temporal_validity tv ON f.fact_id = tv.fact_id "
|
|
2325
|
-
"WHERE f.profile_id = ? "
|
|
2326
|
-
" AND (tv.fact_id IS NULL "
|
|
2327
|
-
" OR ( "
|
|
2328
|
-
" (tv.valid_until IS NULL "
|
|
2329
|
-
" OR tv.valid_until > strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) "
|
|
2330
|
-
" AND tv.system_expired_at IS NULL "
|
|
2331
|
-
" ))",
|
|
2332
|
-
(profile_id,),
|
|
2333
|
-
)
|
|
2334
|
-
return [dict(r)["fact_id"] for r in rows]
|
|
2335
|
-
|
|
2336
|
-
def get_invalidated_fact_ids(
|
|
2337
|
-
self,
|
|
2338
|
-
fact_ids: list[str],
|
|
2339
|
-
profile_id: str,
|
|
2340
|
-
as_of: str | None = None,
|
|
2341
|
-
*,
|
|
2342
|
-
include_global: bool = False,
|
|
2343
|
-
include_shared: bool = False,
|
|
2344
|
-
) -> set[str]:
|
|
2345
|
-
"""Return the subset of ``fact_ids`` that are system-invalidated.
|
|
2346
|
-
|
|
2347
|
-
A fact is system-invalidated when ``system_expired_at`` is set — i.e.
|
|
2348
|
-
it was superseded/contradicted by a newer fact (see
|
|
2349
|
-
``invalidate_fact_temporal``). Such facts are wrong/outdated and must be
|
|
2350
|
-
excluded from default retrieval (T1, Phase 4).
|
|
2351
|
-
|
|
2352
|
-
Phase 4b — bi-temporal as_of:
|
|
2353
|
-
When ``as_of`` is None (default): returns ALL facts with
|
|
2354
|
-
``system_expired_at IS NOT NULL`` — existing behaviour, no regression.
|
|
2355
|
-
|
|
2356
|
-
When ``as_of`` is set (UTC ISO 8601, "+00:00" suffix): returns only
|
|
2357
|
-
facts where ``system_expired_at <= as_of`` (transaction-time boundary
|
|
2358
|
-
inclusive). This means supersessions that occurred AFTER ``as_of`` are
|
|
2359
|
-
excluded — at the historical query point the fact was still valid.
|
|
2360
|
-
|
|
2361
|
-
``as_of`` MUST be UTC-normalized via ``normalize_as_of()`` before this
|
|
2362
|
-
call. The stored ``system_expired_at`` values use Python's
|
|
2363
|
-
``datetime.now(UTC).isoformat()`` format ("...+00:00") so the
|
|
2364
|
-
``normalize_as_of()`` "+00:00" output produces correct lexicographic
|
|
2365
|
-
SQL comparisons.
|
|
2366
|
-
|
|
2367
|
-
Bounded + indexed: only the supplied candidate ids are queried (never a
|
|
2368
|
-
full-table scan), keyed on the ``fact_id`` PK with the
|
|
2369
|
-
``idx_temporal_system_expired`` index covering the predicate. Chunked to
|
|
2370
|
-
stay well under SQLite's ~999 bound-parameter limit. The visibility
|
|
2371
|
-
predicate is evaluated on the fact owner's row, so an opted-in global or
|
|
2372
|
-
shared fact is checked against *its owner's* temporal record rather than
|
|
2373
|
-
incorrectly against the requesting profile. Facts with no temporal
|
|
2374
|
-
record — or a record whose ``system_expired_at`` is NULL — are NOT
|
|
2375
|
-
returned (treated as valid), so existing DBs need no backfill.
|
|
2376
|
-
|
|
2377
|
-
Event-time expiry (``valid_until`` in the past) is intentionally NOT
|
|
2378
|
-
applied here: it is query-scoped (historical queries legitimately want
|
|
2379
|
-
expired facts, per ``include_expired_in_history``) and handled by the
|
|
2380
|
-
time-window path, not by this blanket admission filter.
|
|
2381
|
-
"""
|
|
2382
|
-
if not fact_ids:
|
|
2383
|
-
return set()
|
|
2384
|
-
invalid: set[str] = set()
|
|
2385
|
-
scope_where, scope_params = _scope_where(
|
|
2386
|
-
profile_id,
|
|
2387
|
-
include_global=include_global,
|
|
2388
|
-
include_shared=include_shared,
|
|
2389
|
-
prefix="f",
|
|
2390
|
-
)
|
|
2391
|
-
chunk = 900
|
|
2392
|
-
for start in range(0, len(fact_ids), chunk):
|
|
2393
|
-
batch = fact_ids[start:start + chunk]
|
|
2394
|
-
placeholders = ",".join("?" for _ in batch)
|
|
2395
|
-
if as_of is not None:
|
|
2396
|
-
# Transaction-time point-in-time: only supersessions that
|
|
2397
|
-
# occurred AT OR BEFORE as_of contribute to invalidation.
|
|
2398
|
-
# Supersessions after as_of are invisible at this query point.
|
|
2399
|
-
rows = self.execute(
|
|
2400
|
-
f"SELECT tv.fact_id FROM fact_temporal_validity tv "
|
|
2401
|
-
f"JOIN atomic_facts f ON f.fact_id = tv.fact_id "
|
|
2402
|
-
f"WHERE tv.fact_id IN ({placeholders}) "
|
|
2403
|
-
f" AND {scope_where} "
|
|
2404
|
-
f" AND tv.profile_id = f.profile_id "
|
|
2405
|
-
f" AND tv.system_expired_at IS NOT NULL "
|
|
2406
|
-
f" AND tv.system_expired_at <= ?",
|
|
2407
|
-
(*batch, *scope_params, as_of),
|
|
2408
|
-
)
|
|
2409
|
-
else:
|
|
2410
|
-
rows = self.execute(
|
|
2411
|
-
f"SELECT tv.fact_id FROM fact_temporal_validity tv "
|
|
2412
|
-
f"JOIN atomic_facts f ON f.fact_id = tv.fact_id "
|
|
2413
|
-
f"WHERE tv.fact_id IN ({placeholders}) "
|
|
2414
|
-
f" AND {scope_where} "
|
|
2415
|
-
f" AND tv.profile_id = f.profile_id "
|
|
2416
|
-
f" AND tv.system_expired_at IS NOT NULL",
|
|
2417
|
-
(*batch, *scope_params),
|
|
2418
|
-
)
|
|
2419
|
-
for r in rows:
|
|
2420
|
-
invalid.add(dict(r)["fact_id"])
|
|
2421
|
-
return invalid
|
|
2422
|
-
|
|
2423
|
-
def get_nonapplied_correction_successor_ids(
|
|
2424
|
-
self,
|
|
2425
|
-
fact_ids: list[str],
|
|
2426
|
-
profile_id: str,
|
|
2427
|
-
*,
|
|
2428
|
-
include_global: bool = False,
|
|
2429
|
-
include_shared: bool = False,
|
|
2430
|
-
) -> set[str]:
|
|
2431
|
-
"""Return candidate successors that are not current review truth.
|
|
2432
|
-
|
|
2433
|
-
M042 is optional for older databases. Its absence is safe because
|
|
2434
|
-
canonical proposal never commits a successor unless the same
|
|
2435
|
-
transaction also writes M042. Once present, a read failure must reach
|
|
2436
|
-
the retrieval fail-closed boundary rather than be converted to empty.
|
|
2437
|
-
"""
|
|
2438
|
-
if not fact_ids:
|
|
2439
|
-
return set()
|
|
2440
|
-
present = self.execute(
|
|
2441
|
-
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='correction_cases'"
|
|
2442
|
-
)
|
|
2443
|
-
if not present:
|
|
2444
|
-
return set()
|
|
2445
|
-
scope_where, scope_params = _scope_where(
|
|
2446
|
-
profile_id,
|
|
2447
|
-
include_global=include_global,
|
|
2448
|
-
include_shared=include_shared,
|
|
2449
|
-
prefix="f",
|
|
2450
|
-
)
|
|
2451
|
-
inadmissible: set[str] = set()
|
|
2452
|
-
for start in range(0, len(fact_ids), 900):
|
|
2453
|
-
batch = fact_ids[start:start + 900]
|
|
2454
|
-
placeholders = ",".join("?" for _ in batch)
|
|
2455
|
-
rows = self.execute(
|
|
2456
|
-
"SELECT c.successor_fact_id FROM correction_cases c "
|
|
2457
|
-
"JOIN atomic_facts f ON f.fact_id=c.successor_fact_id "
|
|
2458
|
-
f"WHERE c.successor_fact_id IN ({placeholders}) AND {scope_where} "
|
|
2459
|
-
"AND c.profile_id=f.profile_id "
|
|
2460
|
-
"AND c.status IN ('proposed', 'rejected', 'rolled_back')",
|
|
2461
|
-
(*batch, *scope_params),
|
|
2462
|
-
)
|
|
2463
|
-
inadmissible.update(str(row["successor_fact_id"]) for row in rows)
|
|
2464
|
-
return inadmissible
|
|
2465
|
-
|
|
2466
|
-
def get_correction_inadmissible_fact_ids(
|
|
2467
|
-
self,
|
|
2468
|
-
fact_ids: list[str],
|
|
2469
|
-
profile_id: str,
|
|
2470
|
-
as_of: str | None = None,
|
|
2471
|
-
*,
|
|
2472
|
-
include_global: bool = False,
|
|
2473
|
-
include_shared: bool = False,
|
|
2474
|
-
) -> set[str]:
|
|
2475
|
-
"""Return current-lifecycle exclusions with one bounded SQLite read.
|
|
2476
|
-
|
|
2477
|
-
Recall needs both sides of reviewed correction truth: an expired
|
|
2478
|
-
predecessor and a successor whose case is not applied. The older
|
|
2479
|
-
public helpers preserve their focused contracts, but invoking them
|
|
2480
|
-
consecutively opened two SQLite connections on every candidate stage.
|
|
2481
|
-
This read-model helper uses one connection and one UNION query while
|
|
2482
|
-
retaining the same profile/scope and historical ``as_of`` semantics.
|
|
2483
|
-
It is intentionally read-only and does not cache lifecycle state.
|
|
2484
|
-
"""
|
|
2485
|
-
if not fact_ids:
|
|
2486
|
-
return set()
|
|
2487
|
-
scope_where, scope_params = _scope_where(
|
|
2488
|
-
profile_id,
|
|
2489
|
-
include_global=include_global,
|
|
2490
|
-
include_shared=include_shared,
|
|
2491
|
-
prefix="f",
|
|
2492
|
-
)
|
|
2493
|
-
inadmissible: set[str] = set()
|
|
2494
|
-
with self.raw_connection() as conn:
|
|
2495
|
-
correction_table = conn.execute(
|
|
2496
|
-
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='correction_cases'"
|
|
2497
|
-
).fetchone()
|
|
2498
|
-
for start in range(0, len(fact_ids), 900):
|
|
2499
|
-
batch = fact_ids[start:start + 900]
|
|
2500
|
-
placeholders = ",".join("?" for _ in batch)
|
|
2501
|
-
temporal_conditions = (
|
|
2502
|
-
"AND tv.system_expired_at IS NOT NULL "
|
|
2503
|
-
+ ("AND tv.system_expired_at <= ?" if as_of is not None else "")
|
|
2504
|
-
)
|
|
2505
|
-
temporal_sql = (
|
|
2506
|
-
"SELECT tv.fact_id AS fact_id "
|
|
2507
|
-
"FROM fact_temporal_validity tv "
|
|
2508
|
-
"JOIN atomic_facts f ON f.fact_id=tv.fact_id "
|
|
2509
|
-
f"WHERE tv.fact_id IN ({placeholders}) AND {scope_where} "
|
|
2510
|
-
"AND tv.profile_id=f.profile_id "
|
|
2511
|
-
f"{temporal_conditions}"
|
|
2512
|
-
)
|
|
2513
|
-
temporal_params: tuple[Any, ...] = (
|
|
2514
|
-
*batch,
|
|
2515
|
-
*scope_params,
|
|
2516
|
-
*((as_of,) if as_of is not None else ()),
|
|
2517
|
-
)
|
|
2518
|
-
if correction_table is None:
|
|
2519
|
-
rows = conn.execute(temporal_sql, temporal_params).fetchall()
|
|
2520
|
-
else:
|
|
2521
|
-
correction_sql = (
|
|
2522
|
-
"SELECT c.successor_fact_id AS fact_id "
|
|
2523
|
-
"FROM correction_cases c "
|
|
2524
|
-
"JOIN atomic_facts f ON f.fact_id=c.successor_fact_id "
|
|
2525
|
-
f"WHERE c.successor_fact_id IN ({placeholders}) AND {scope_where} "
|
|
2526
|
-
"AND c.profile_id=f.profile_id "
|
|
2527
|
-
"AND c.status IN ('proposed', 'rejected', 'rolled_back')"
|
|
2528
|
-
)
|
|
2529
|
-
rows = conn.execute(
|
|
2530
|
-
f"{temporal_sql} UNION {correction_sql}",
|
|
2531
|
-
(*temporal_params, *batch, *scope_params),
|
|
2532
|
-
).fetchall()
|
|
2533
|
-
inadmissible.update(str(row["fact_id"]) for row in rows)
|
|
2534
|
-
return inadmissible
|
|
2535
|
-
|
|
2536
|
-
def get_strict_temporal_inadmissible_fact_ids(
|
|
2537
|
-
self,
|
|
2538
|
-
fact_ids: list[str],
|
|
2539
|
-
profile_id: str,
|
|
2540
|
-
*,
|
|
2541
|
-
known_as_of: str | None = None,
|
|
2542
|
-
valid_at: str | None = None,
|
|
2543
|
-
include_unknown: bool = False,
|
|
2544
|
-
include_global: bool = False,
|
|
2545
|
-
include_shared: bool = False,
|
|
2546
|
-
) -> set[str]:
|
|
2547
|
-
"""Return candidates excluded by an explicit two-clock query.
|
|
2548
|
-
|
|
2549
|
-
``known_as_of`` is transaction time: it asks what this SLM instance had
|
|
2550
|
-
learned by a timestamp. ``valid_at`` is event time: it asks what was
|
|
2551
|
-
true at a timestamp according to the requested knowledge state. The
|
|
2552
|
-
axes are independent and may be supplied separately or together.
|
|
2553
|
-
|
|
2554
|
-
A fact with no temporal row predates the 4.0.2 write invariant and is
|
|
2555
|
-
``legacy_unknown``. Strict time-travel excludes it unless the caller
|
|
2556
|
-
explicitly asks to include unknown history. This is intentionally
|
|
2557
|
-
bounded to the already-retrieved candidate pool.
|
|
2558
|
-
"""
|
|
2559
|
-
if not fact_ids or (known_as_of is None and valid_at is None):
|
|
2560
|
-
return set()
|
|
2561
|
-
from datetime import datetime as _dt
|
|
2562
|
-
from superlocalmemory.retrieval.temporal_utils import normalize_as_of
|
|
2563
|
-
|
|
2564
|
-
def _parse_timestamp(value: object) -> _dt | None:
|
|
2565
|
-
normalized = normalize_as_of(value)
|
|
2566
|
-
return _dt.fromisoformat(normalized) if normalized is not None else None
|
|
2567
|
-
|
|
2568
|
-
known_boundary = _parse_timestamp(known_as_of) if known_as_of is not None else None
|
|
2569
|
-
valid_boundary = _parse_timestamp(valid_at) if valid_at is not None else None
|
|
2570
|
-
inadmissible: set[str] = set()
|
|
2571
|
-
scope_where, scope_params = _scope_where(
|
|
2572
|
-
profile_id,
|
|
2573
|
-
include_global=include_global,
|
|
2574
|
-
include_shared=include_shared,
|
|
2575
|
-
prefix="f",
|
|
2576
|
-
)
|
|
2577
|
-
chunk = 900
|
|
2578
|
-
for start in range(0, len(fact_ids), chunk):
|
|
2579
|
-
batch = fact_ids[start:start + chunk]
|
|
2580
|
-
placeholders = ",".join("?" for _ in batch)
|
|
2581
|
-
rows = self.execute(
|
|
2582
|
-
f"SELECT f.fact_id, tv.fact_id AS temporal_fact_id, "
|
|
2583
|
-
f"tv.system_created_at, tv.system_expired_at, "
|
|
2584
|
-
f"tv.valid_from, tv.valid_until FROM atomic_facts f "
|
|
2585
|
-
f"LEFT JOIN fact_temporal_validity tv "
|
|
2586
|
-
f" ON tv.fact_id = f.fact_id AND tv.profile_id = f.profile_id "
|
|
2587
|
-
f"WHERE f.fact_id IN ({placeholders}) "
|
|
2588
|
-
f" AND {scope_where}",
|
|
2589
|
-
(*batch, *scope_params),
|
|
2590
|
-
)
|
|
2591
|
-
for row in rows:
|
|
2592
|
-
values = dict(row)
|
|
2593
|
-
has_temporal_record = values["temporal_fact_id"] is not None
|
|
2594
|
-
if not has_temporal_record:
|
|
2595
|
-
if not include_unknown:
|
|
2596
|
-
inadmissible.add(values["fact_id"])
|
|
2597
|
-
continue
|
|
2598
|
-
unknown = False
|
|
2599
|
-
if known_boundary is not None:
|
|
2600
|
-
created = _parse_timestamp(values["system_created_at"])
|
|
2601
|
-
expired = _parse_timestamp(values["system_expired_at"])
|
|
2602
|
-
if created is None:
|
|
2603
|
-
unknown = True
|
|
2604
|
-
elif created > known_boundary:
|
|
2605
|
-
inadmissible.add(values["fact_id"])
|
|
2606
|
-
continue
|
|
2607
|
-
elif expired is not None and expired <= known_boundary:
|
|
2608
|
-
inadmissible.add(values["fact_id"])
|
|
2609
|
-
continue
|
|
2610
|
-
if valid_boundary is not None:
|
|
2611
|
-
valid_from = _parse_timestamp(values["valid_from"])
|
|
2612
|
-
valid_until = _parse_timestamp(values["valid_until"])
|
|
2613
|
-
if values["valid_from"] is not None and valid_from is None:
|
|
2614
|
-
unknown = True
|
|
2615
|
-
elif valid_from is not None and valid_from > valid_boundary:
|
|
2616
|
-
inadmissible.add(values["fact_id"])
|
|
2617
|
-
continue
|
|
2618
|
-
if values["valid_until"] is not None and valid_until is None:
|
|
2619
|
-
unknown = True
|
|
2620
|
-
elif valid_until is not None and valid_until <= valid_boundary:
|
|
2621
|
-
inadmissible.add(values["fact_id"])
|
|
2622
|
-
continue
|
|
2623
|
-
if unknown and not include_unknown:
|
|
2624
|
-
inadmissible.add(values["fact_id"])
|
|
2625
|
-
return inadmissible
|
|
2626
|
-
|
|
2627
|
-
def get_event_time_expired_fact_ids(
|
|
2628
|
-
self,
|
|
2629
|
-
fact_ids: list[str],
|
|
2630
|
-
profile_id: str,
|
|
2631
|
-
as_of: str | None = None,
|
|
2632
|
-
) -> set[str]:
|
|
2633
|
-
"""Return the subset of ``fact_ids`` that are event-time out-of-range.
|
|
2634
|
-
|
|
2635
|
-
Returns fact_ids whose event-time validity window does not encompass
|
|
2636
|
-
``as_of`` (or the current wall-clock time when ``as_of`` is None):
|
|
2637
|
-
|
|
2638
|
-
1. **Already expired** — ``valid_until IS NOT NULL AND valid_until <= ref``
|
|
2639
|
-
where ``ref`` is ``as_of`` when provided or the current UTC time.
|
|
2640
|
-
The ``<=`` implements the half-open interval ``[valid_from, valid_until)``:
|
|
2641
|
-
a fact with ``valid_until == as_of`` has expired at that boundary (Phase 4b fix).
|
|
2642
|
-
2. **Not yet valid** — ``valid_from IS NOT NULL AND valid_from > as_of``
|
|
2643
|
-
(only when ``as_of`` is provided for explicit point-in-time recall).
|
|
2644
|
-
|
|
2645
|
-
Zero-regression guarantee: facts with ``valid_until = NULL`` are
|
|
2646
|
-
assumed open-ended (still valid) and are NEVER returned. Facts with no
|
|
2647
|
-
temporal record at all are NEVER returned (assumed valid). Because almost
|
|
2648
|
-
all existing facts have ``valid_until = NULL``, the default path
|
|
2649
|
-
(``as_of=None``) returns an empty set and causes no demotion.
|
|
2650
|
-
|
|
2651
|
-
Bounded + indexed: only the supplied candidate ids are queried (never a
|
|
2652
|
-
full-table scan), keyed on the ``fact_id`` PK. Chunked to stay under
|
|
2653
|
-
SQLite's ~999 bound-parameter limit (chunk size 900). The
|
|
2654
|
-
``idx_temporal_valid(profile_id, valid_until)`` index assists the
|
|
2655
|
-
``valid_until <`` range predicate after the PK IN-lookup. The
|
|
2656
|
-
``valid_from > as_of`` branch operates on the same bounded PK row set
|
|
2657
|
-
(≤ 900 rows per chunk) — no full-table scan occurs.
|
|
2658
|
-
|
|
2659
|
-
Fail-open: any DB error logs a warning and returns an empty set so
|
|
2660
|
-
retrieval can never break because of a validity lookup failure.
|
|
2661
|
-
|
|
2662
|
-
Args:
|
|
2663
|
-
fact_ids: Candidate fact IDs to check (bounded retrieval pool).
|
|
2664
|
-
profile_id: Current profile — scopes the lookup to one tenant.
|
|
2665
|
-
as_of: Optional ISO 8601 datetime string for point-in-time recall.
|
|
2666
|
-
When set, facts not yet valid at this time are also returned.
|
|
2667
|
-
When None (default), only facts past their ``valid_until`` are
|
|
2668
|
-
returned — the standard current-time path.
|
|
2669
|
-
|
|
2670
|
-
Note:
|
|
2671
|
-
``as_of`` must be in the same ISO 8601 format as the stored
|
|
2672
|
-
``valid_until`` / ``valid_from`` values so SQLite's lexicographic
|
|
2673
|
-
string comparison correctly orders the timestamps.
|
|
2674
|
-
"""
|
|
2675
|
-
if not fact_ids:
|
|
2676
|
-
return set()
|
|
2677
|
-
expired: set[str] = set()
|
|
2678
|
-
try:
|
|
2679
|
-
chunk = 900
|
|
2680
|
-
for start in range(0, len(fact_ids), chunk):
|
|
2681
|
-
batch = fact_ids[start:start + chunk]
|
|
2682
|
-
placeholders = ",".join("?" for _ in batch)
|
|
2683
|
-
if as_of is not None:
|
|
2684
|
-
# Time-travel: expired-before-as_of OR not-yet-started-at-as_of.
|
|
2685
|
-
rows = self.execute(
|
|
2686
|
-
f"SELECT fact_id FROM fact_temporal_validity "
|
|
2687
|
-
f"WHERE fact_id IN ({placeholders}) "
|
|
2688
|
-
f" AND profile_id = ? "
|
|
2689
|
-
f" AND ("
|
|
2690
|
-
f" (valid_until IS NOT NULL AND valid_until <= ?) "
|
|
2691
|
-
f" OR (valid_from IS NOT NULL AND valid_from > ?)"
|
|
2692
|
-
f" )",
|
|
2693
|
-
(*batch, profile_id, as_of, as_of),
|
|
2694
|
-
)
|
|
2695
|
-
else:
|
|
2696
|
-
# Default path: only facts whose valid_until has passed.
|
|
2697
|
-
# idx_temporal_valid(profile_id, valid_until) assists range scan.
|
|
2698
|
-
rows = self.execute(
|
|
2699
|
-
f"SELECT fact_id FROM fact_temporal_validity "
|
|
2700
|
-
f"WHERE fact_id IN ({placeholders}) "
|
|
2701
|
-
f" AND profile_id = ? "
|
|
2702
|
-
f" AND valid_until IS NOT NULL "
|
|
2703
|
-
f" AND valid_until < strftime('%Y-%m-%dT%H:%M:%SZ', 'now')",
|
|
2704
|
-
(*batch, profile_id),
|
|
2705
|
-
)
|
|
2706
|
-
for r in rows:
|
|
2707
|
-
expired.add(dict(r)["fact_id"])
|
|
2708
|
-
except Exception as exc:
|
|
2709
|
-
logger.warning(
|
|
2710
|
-
"Event-time expiry lookup failed (fail-open): %s", exc,
|
|
2711
|
-
)
|
|
2712
|
-
return set()
|
|
2713
|
-
return expired
|
|
2714
|
-
|
|
2715
|
-
def get_fact_event_times(
|
|
2716
|
-
self, fact_ids: list[str], profile_id: str,
|
|
2717
|
-
) -> dict[str, str]:
|
|
2718
|
-
"""Map each candidate fact_id to its best-available event time.
|
|
2719
|
-
|
|
2720
|
-
Priority (most specific first): ``referenced_date`` (the date the fact
|
|
2721
|
-
is *about*) → ``observation_date`` (when it was observed) →
|
|
2722
|
-
``valid_from`` (bi-temporal event start) → ``created_at`` (storage
|
|
2723
|
-
time, always present). Used by time-window recall to prune candidates
|
|
2724
|
-
by when the underlying event happened, falling back to capture time for
|
|
2725
|
-
undated facts.
|
|
2726
|
-
|
|
2727
|
-
Bounded + indexed (candidate ids only, ``fact_id`` PK), chunked under
|
|
2728
|
-
SQLite's bound-parameter limit. Facts absent from the result (unknown
|
|
2729
|
-
id / wrong profile) are simply omitted.
|
|
2730
|
-
"""
|
|
2731
|
-
if not fact_ids:
|
|
2732
|
-
return {}
|
|
2733
|
-
out: dict[str, str] = {}
|
|
2734
|
-
chunk = 900
|
|
2735
|
-
for start in range(0, len(fact_ids), chunk):
|
|
2736
|
-
batch = fact_ids[start:start + chunk]
|
|
2737
|
-
placeholders = ",".join("?" for _ in batch)
|
|
2738
|
-
rows = self.execute(
|
|
2739
|
-
f"SELECT f.fact_id AS fact_id, "
|
|
2740
|
-
f"COALESCE(f.referenced_date, f.observation_date, "
|
|
2741
|
-
f" tv.valid_from, f.created_at) AS event_time "
|
|
2742
|
-
f"FROM atomic_facts f "
|
|
2743
|
-
f"LEFT JOIN fact_temporal_validity tv ON f.fact_id = tv.fact_id "
|
|
2744
|
-
f"WHERE f.fact_id IN ({placeholders}) "
|
|
2745
|
-
f"AND (f.profile_id = ? OR f.scope = 'global')",
|
|
2746
|
-
(*batch, profile_id),
|
|
2747
|
-
)
|
|
2748
|
-
for r in rows:
|
|
2749
|
-
d = dict(r)
|
|
2750
|
-
if d.get("event_time"):
|
|
2751
|
-
out[d["fact_id"]] = d["event_time"]
|
|
2752
|
-
return out
|
|
2753
|
-
|
|
2754
|
-
def delete_temporal_validity(self, fact_id: str) -> None:
|
|
2755
|
-
"""Delete temporal validity record (for testing/rollback only)."""
|
|
2756
|
-
self.execute(
|
|
2757
|
-
"DELETE FROM fact_temporal_validity WHERE fact_id = ?",
|
|
2758
|
-
(fact_id,),
|
|
2759
|
-
)
|
|
2760
|
-
|
|
2761
|
-
# ------------------------------------------------------------------
|
|
2762
|
-
# Phase 4 (T3b): fact-augmented key expansion (BM25 alt-keys)
|
|
2763
|
-
# ------------------------------------------------------------------
|
|
2764
|
-
|
|
2765
|
-
def upsert_fact_expansion(self, fact_id: str, alt_keys: str) -> None:
|
|
2766
|
-
"""Store/replace a fact's alternate keys in ``fact_expansion_fts``.
|
|
2767
|
-
|
|
2768
|
-
Standalone FTS5 (no external-content triggers), so we replace by hand:
|
|
2769
|
-
delete any prior row for the fact, then insert the new alt-keys. An
|
|
2770
|
-
empty/blank ``alt_keys`` clears the fact's expansion entry. Fail-soft:
|
|
2771
|
-
a missing FTS table (legacy DB) never breaks the write path.
|
|
2772
|
-
"""
|
|
2773
|
-
try:
|
|
2774
|
-
self.execute(
|
|
2775
|
-
"DELETE FROM fact_expansion_fts WHERE fact_id = ?", (fact_id,)
|
|
2776
|
-
)
|
|
2777
|
-
if alt_keys and alt_keys.strip():
|
|
2778
|
-
self.execute(
|
|
2779
|
-
"INSERT INTO fact_expansion_fts (fact_id, alt_keys) "
|
|
2780
|
-
"VALUES (?, ?)",
|
|
2781
|
-
(fact_id, alt_keys.strip()),
|
|
2782
|
-
)
|
|
2783
|
-
except Exception as exc: # pragma: no cover — legacy/missing FTS table
|
|
2784
|
-
logger.debug("upsert_fact_expansion skipped for %s: %s", fact_id, exc)
|
|
2785
|
-
|
|
2786
|
-
def reset_fact_expansion(self, fact_id: str, alt_keys: str = "") -> None:
|
|
2787
|
-
"""Replace the expansion row unconditionally, keeping it alive with new alt_keys.
|
|
2788
|
-
|
|
2789
|
-
Unlike ``upsert_fact_expansion``, this always inserts (even when
|
|
2790
|
-
``alt_keys`` is empty) so the row survives as a cleared placeholder.
|
|
2791
|
-
Used by update paths that must guarantee the expansion entry exists but
|
|
2792
|
-
holds no stale tokens. Fail-soft: a missing FTS table is a no-op.
|
|
2793
|
-
"""
|
|
2794
|
-
try:
|
|
2795
|
-
self.execute(
|
|
2796
|
-
"DELETE FROM fact_expansion_fts WHERE fact_id = ?", (fact_id,)
|
|
2797
|
-
)
|
|
2798
|
-
self.execute(
|
|
2799
|
-
"INSERT INTO fact_expansion_fts (fact_id, alt_keys) VALUES (?, ?)",
|
|
2800
|
-
(fact_id, alt_keys),
|
|
2801
|
-
)
|
|
2802
|
-
except Exception as exc:
|
|
2803
|
-
logger.debug("reset_fact_expansion skipped for %s: %s", fact_id, exc)
|
|
2804
|
-
|
|
2805
|
-
def update_temporal_event_description(
|
|
2806
|
-
self, fact_id: str, description: str
|
|
2807
|
-
) -> None:
|
|
2808
|
-
"""Update the description column in ``temporal_events`` for a fact.
|
|
2809
|
-
|
|
2810
|
-
Fail-soft: absent table (pre-migration DB) is silently skipped.
|
|
2811
|
-
"""
|
|
2812
|
-
try:
|
|
2813
|
-
self.execute(
|
|
2814
|
-
"UPDATE temporal_events SET description = ? WHERE fact_id = ?",
|
|
2815
|
-
(description, fact_id),
|
|
2816
|
-
)
|
|
2817
|
-
except Exception as exc:
|
|
2818
|
-
logger.debug(
|
|
2819
|
-
"update_temporal_event_description skipped for %s: %s", fact_id, exc
|
|
2820
|
-
)
|
|
2821
|
-
|
|
2822
|
-
def delete_bm25_tokens_for_fact(self, fact_id: str) -> None:
|
|
2823
|
-
"""Delete persisted BM25 tokens for a fact from the ``bm25_tokens`` table."""
|
|
2824
|
-
try:
|
|
2825
|
-
self.execute(
|
|
2826
|
-
"DELETE FROM bm25_tokens WHERE fact_id = ?", (fact_id,)
|
|
2827
|
-
)
|
|
2828
|
-
except Exception as exc:
|
|
2829
|
-
logger.debug("delete_bm25_tokens_for_fact skipped for %s: %s", fact_id, exc)
|
|
2830
|
-
|
|
2831
|
-
def delete_graph_edges_for_fact(self, fact_id: str) -> None:
|
|
2832
|
-
"""Delete all graph edges where this fact is the source or the target."""
|
|
2833
|
-
try:
|
|
2834
|
-
self.execute(
|
|
2835
|
-
"DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?",
|
|
2836
|
-
(fact_id, fact_id),
|
|
2837
|
-
)
|
|
2838
|
-
except Exception as exc:
|
|
2839
|
-
logger.debug("delete_graph_edges_for_fact skipped for %s: %s", fact_id, exc)
|
|
2840
|
-
|
|
2841
|
-
def remove_fact_from_scenes(self, fact_id: str, profile_id: str) -> None:
|
|
2842
|
-
"""Remove a fact_id from every ``memory_scenes`` JSON array in the profile.
|
|
2843
|
-
|
|
2844
|
-
Scenes that become empty after removal are deleted entirely.
|
|
2845
|
-
Fail-soft: any exception is logged and ignored.
|
|
2846
|
-
"""
|
|
2847
|
-
try:
|
|
2848
|
-
scenes = self.get_scenes_for_fact(fact_id, profile_id)
|
|
2849
|
-
for scene in scenes:
|
|
2850
|
-
new_ids = [fid for fid in (scene.fact_ids or []) if fid != fact_id]
|
|
2851
|
-
if new_ids:
|
|
2852
|
-
self.execute(
|
|
2853
|
-
"UPDATE memory_scenes SET fact_ids_json = ? "
|
|
2854
|
-
"WHERE scene_id = ?",
|
|
2855
|
-
(json.dumps(new_ids), scene.scene_id),
|
|
2856
|
-
)
|
|
2857
|
-
else:
|
|
2858
|
-
self.execute(
|
|
2859
|
-
"DELETE FROM memory_scenes WHERE scene_id = ?",
|
|
2860
|
-
(scene.scene_id,),
|
|
2861
|
-
)
|
|
2862
|
-
except Exception as exc:
|
|
2863
|
-
logger.debug("remove_fact_from_scenes skipped for %s: %s", fact_id, exc)
|
|
2864
|
-
|
|
2865
|
-
def delete_memory_for_fact(self, fact_id: str, profile_id: str) -> None:
|
|
2866
|
-
"""Delete the raw ``memories`` record that sourced this fact.
|
|
2867
|
-
|
|
2868
|
-
Reads the ``memory_id`` from ``atomic_facts`` before the fact row is
|
|
2869
|
-
gone, then deletes the memory. Fail-soft: any exception is logged.
|
|
2870
|
-
"""
|
|
2871
|
-
try:
|
|
2872
|
-
rows = self.execute(
|
|
2873
|
-
"SELECT memory_id FROM atomic_facts "
|
|
2874
|
-
"WHERE fact_id = ? AND profile_id = ? LIMIT 1",
|
|
2875
|
-
(fact_id, profile_id),
|
|
2876
|
-
)
|
|
2877
|
-
if rows:
|
|
2878
|
-
memory_id = dict(rows[0]).get("memory_id") or ""
|
|
2879
|
-
if memory_id:
|
|
2880
|
-
self.execute(
|
|
2881
|
-
"DELETE FROM memories "
|
|
2882
|
-
"WHERE memory_id = ? AND profile_id = ?",
|
|
2883
|
-
(memory_id, profile_id),
|
|
2884
|
-
)
|
|
2885
|
-
except Exception as exc:
|
|
2886
|
-
logger.debug("delete_memory_for_fact skipped for %s: %s", fact_id, exc)
|
|
2887
|
-
|
|
2888
|
-
# ------------------------------------------------------------------
|
|
2889
|
-
# Phase 5: Core Memory Blocks CRUD (Rule 15)
|
|
2890
|
-
# ------------------------------------------------------------------
|
|
2891
|
-
|
|
2892
|
-
def store_core_block(
|
|
2893
|
-
self,
|
|
2894
|
-
block_id: str,
|
|
2895
|
-
profile_id: str,
|
|
2896
|
-
block_type: str,
|
|
2897
|
-
content: str,
|
|
2898
|
-
source_fact_ids: str = "[]",
|
|
2899
|
-
char_count: int = 0,
|
|
2900
|
-
version: int = 1,
|
|
2901
|
-
compiled_by: str = "rules",
|
|
2902
|
-
) -> None:
|
|
2903
|
-
"""Store or replace a Core Memory block.
|
|
2904
|
-
|
|
2905
|
-
Uses INSERT OR REPLACE on UNIQUE(profile_id, block_type)
|
|
2906
|
-
to guarantee idempotency (L18).
|
|
2907
|
-
"""
|
|
2908
|
-
self.execute(
|
|
2909
|
-
"INSERT OR REPLACE INTO core_memory_blocks "
|
|
2910
|
-
"(block_id, profile_id, block_type, content, source_fact_ids, "
|
|
2911
|
-
" char_count, version, compiled_by, created_at, updated_at) "
|
|
2912
|
-
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
|
|
2913
|
-
(block_id, profile_id, block_type, content,
|
|
2914
|
-
source_fact_ids, char_count, version, compiled_by),
|
|
2915
|
-
)
|
|
2916
|
-
|
|
2917
|
-
def get_core_blocks(self, profile_id: str) -> list[dict]:
|
|
2918
|
-
"""Get all Core Memory blocks for a profile."""
|
|
2919
|
-
rows = self.execute(
|
|
2920
|
-
"SELECT * FROM core_memory_blocks "
|
|
2921
|
-
"WHERE profile_id = ? ORDER BY block_type",
|
|
2922
|
-
(profile_id,),
|
|
2923
|
-
)
|
|
2924
|
-
return [dict(r) for r in rows]
|
|
2925
|
-
|
|
2926
|
-
def get_core_block(self, profile_id: str, block_type: str) -> dict | None:
|
|
2927
|
-
"""Get a single Core Memory block by profile and type."""
|
|
2928
|
-
rows = self.execute(
|
|
2929
|
-
"SELECT * FROM core_memory_blocks "
|
|
2930
|
-
"WHERE profile_id = ? AND block_type = ?",
|
|
2931
|
-
(profile_id, block_type),
|
|
2932
|
-
)
|
|
2933
|
-
return dict(rows[0]) if rows else None
|
|
2934
|
-
|
|
2935
|
-
def delete_core_blocks(self, profile_id: str) -> None:
|
|
2936
|
-
"""Delete all Core Memory blocks for a profile."""
|
|
2937
|
-
self.execute(
|
|
2938
|
-
"DELETE FROM core_memory_blocks WHERE profile_id = ?",
|
|
2939
|
-
(profile_id,),
|
|
2940
|
-
)
|
|
2941
|
-
|
|
2942
|
-
# ------------------------------------------------------------------
|
|
2943
|
-
# Phase A: Fact Retention CRUD (Forgetting Brain)
|
|
2944
|
-
# ------------------------------------------------------------------
|
|
2945
|
-
|
|
2946
|
-
def get_retention(self, fact_id: str, profile_id: str) -> dict | None:
|
|
2947
|
-
"""Get retention data for a single fact.
|
|
2948
|
-
|
|
2949
|
-
Returns dict with column names as keys, or None if not found.
|
|
2950
|
-
All SQL parameterized (HR-05).
|
|
2951
|
-
"""
|
|
2952
|
-
rows = self.execute(
|
|
2953
|
-
"SELECT fact_id, retention_score, memory_strength, access_count, "
|
|
2954
|
-
" last_accessed_at, lifecycle_zone, last_computed_at "
|
|
2955
|
-
"FROM fact_retention WHERE fact_id = ? AND profile_id = ?",
|
|
2956
|
-
(fact_id, profile_id),
|
|
2957
|
-
)
|
|
2958
|
-
return dict(rows[0]) if rows else None
|
|
2959
|
-
|
|
2960
|
-
def batch_get_retention(
|
|
2961
|
-
self, fact_ids: list[str], profile_id: str,
|
|
2962
|
-
) -> list[dict]:
|
|
2963
|
-
"""Get retention data for a batch of facts.
|
|
2964
|
-
|
|
2965
|
-
Uses dynamic ? placeholders for IN clause (never string concat).
|
|
2966
|
-
Missing fact_ids are simply absent from results.
|
|
2967
|
-
All SQL parameterized (HR-05).
|
|
2968
|
-
"""
|
|
2969
|
-
if not fact_ids:
|
|
2970
|
-
return []
|
|
2971
|
-
placeholders = ",".join("?" for _ in fact_ids)
|
|
2972
|
-
rows = self.execute(
|
|
2973
|
-
f"SELECT fact_id, retention_score, lifecycle_zone "
|
|
2974
|
-
f"FROM fact_retention "
|
|
2975
|
-
f"WHERE fact_id IN ({placeholders}) AND profile_id = ?",
|
|
2976
|
-
(*fact_ids, profile_id),
|
|
2977
|
-
)
|
|
2978
|
-
return [dict(r) for r in rows]
|
|
2979
|
-
|
|
2980
|
-
def upsert_retention(
|
|
2981
|
-
self,
|
|
2982
|
-
fact_id: str,
|
|
2983
|
-
profile_id: str,
|
|
2984
|
-
retention_score: float,
|
|
2985
|
-
memory_strength: float,
|
|
2986
|
-
access_count: int,
|
|
2987
|
-
last_accessed_at: str,
|
|
2988
|
-
lifecycle_zone: str,
|
|
2989
|
-
) -> None:
|
|
2990
|
-
"""UPSERT retention data for a fact.
|
|
2991
|
-
|
|
2992
|
-
Retries 3x on SQLITE_BUSY (handled by execute()).
|
|
2993
|
-
All SQL parameterized (HR-05).
|
|
2994
|
-
"""
|
|
2995
|
-
from superlocalmemory.core.lifecycle_state import atomic_lifecycle_for
|
|
2996
|
-
|
|
2997
|
-
with self.transaction():
|
|
2998
|
-
self._upsert_retention_in_transaction(
|
|
2999
|
-
fact_id=fact_id,
|
|
3000
|
-
profile_id=profile_id,
|
|
3001
|
-
retention_score=retention_score,
|
|
3002
|
-
memory_strength=memory_strength,
|
|
3003
|
-
access_count=access_count,
|
|
3004
|
-
last_accessed_at=last_accessed_at,
|
|
3005
|
-
lifecycle_zone=lifecycle_zone,
|
|
3006
|
-
)
|
|
3007
|
-
self.execute(
|
|
3008
|
-
"UPDATE atomic_facts SET lifecycle = ? "
|
|
3009
|
-
"WHERE fact_id = ? AND profile_id = ?",
|
|
3010
|
-
(atomic_lifecycle_for(lifecycle_zone), fact_id, profile_id),
|
|
3011
|
-
)
|
|
3012
|
-
|
|
3013
|
-
def _upsert_retention_in_transaction(
|
|
3014
|
-
self,
|
|
3015
|
-
*,
|
|
3016
|
-
fact_id: str,
|
|
3017
|
-
profile_id: str,
|
|
3018
|
-
retention_score: float,
|
|
3019
|
-
memory_strength: float,
|
|
3020
|
-
access_count: int,
|
|
3021
|
-
last_accessed_at: str,
|
|
3022
|
-
lifecycle_zone: str,
|
|
3023
|
-
) -> None:
|
|
3024
|
-
"""Write one retention row using the caller's active transaction."""
|
|
3025
|
-
self.execute(
|
|
3026
|
-
"INSERT INTO fact_retention "
|
|
3027
|
-
"(fact_id, profile_id, retention_score, memory_strength, "
|
|
3028
|
-
" access_count, last_accessed_at, lifecycle_zone, last_computed_at) "
|
|
3029
|
-
"VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now')) "
|
|
3030
|
-
"ON CONFLICT(fact_id) DO UPDATE SET "
|
|
3031
|
-
" retention_score = excluded.retention_score, "
|
|
3032
|
-
" memory_strength = excluded.memory_strength, "
|
|
3033
|
-
" access_count = excluded.access_count, "
|
|
3034
|
-
" lifecycle_zone = excluded.lifecycle_zone, "
|
|
3035
|
-
" last_computed_at = excluded.last_computed_at",
|
|
3036
|
-
(fact_id, profile_id, retention_score, memory_strength,
|
|
3037
|
-
access_count, last_accessed_at, lifecycle_zone),
|
|
3038
|
-
)
|
|
3039
|
-
|
|
3040
|
-
def batch_upsert_retention(
|
|
3041
|
-
self, facts: list[dict], profile_id: str,
|
|
3042
|
-
) -> int:
|
|
3043
|
-
"""Batch UPSERT retention data. Wraps in transaction for atomicity.
|
|
3044
|
-
|
|
3045
|
-
Each dict must contain: fact_id, retention, strength,
|
|
3046
|
-
access_count, last_accessed_at, zone.
|
|
3047
|
-
|
|
3048
|
-
Returns count of successfully upserted rows.
|
|
3049
|
-
"""
|
|
3050
|
-
count = 0
|
|
3051
|
-
from superlocalmemory.core.lifecycle_state import atomic_lifecycle_for
|
|
3052
|
-
|
|
3053
|
-
with self.transaction():
|
|
3054
|
-
for f in facts:
|
|
3055
|
-
self._upsert_retention_in_transaction(
|
|
3056
|
-
fact_id=f["fact_id"],
|
|
3057
|
-
profile_id=profile_id,
|
|
3058
|
-
retention_score=f["retention"],
|
|
3059
|
-
memory_strength=f["strength"],
|
|
3060
|
-
access_count=f["access_count"],
|
|
3061
|
-
last_accessed_at=f["last_accessed_at"],
|
|
3062
|
-
lifecycle_zone=f["zone"],
|
|
3063
|
-
)
|
|
3064
|
-
self.execute(
|
|
3065
|
-
"UPDATE atomic_facts SET lifecycle = ? "
|
|
3066
|
-
"WHERE fact_id = ? AND profile_id = ?",
|
|
3067
|
-
(atomic_lifecycle_for(f["zone"]), f["fact_id"], profile_id),
|
|
3068
|
-
)
|
|
3069
|
-
count += 1
|
|
3070
|
-
return count
|
|
3071
|
-
|
|
3072
|
-
def get_facts_needing_decay(self, profile_id: str) -> list[dict]:
|
|
3073
|
-
"""Get facts that need decay computation (excludes core memory).
|
|
3074
|
-
|
|
3075
|
-
Core memory facts are immune to forgetting (HR-01).
|
|
3076
|
-
All SQL parameterized (HR-05).
|
|
3077
|
-
"""
|
|
3078
|
-
rows = self.execute(
|
|
3079
|
-
"SELECT f.fact_id, f.created_at, f.profile_id "
|
|
3080
|
-
"FROM atomic_facts f "
|
|
3081
|
-
"LEFT JOIN fact_retention r ON f.fact_id = r.fact_id "
|
|
3082
|
-
"WHERE f.profile_id = ? "
|
|
3083
|
-
"AND f.fact_id NOT IN ("
|
|
3084
|
-
" SELECT json_each.value "
|
|
3085
|
-
" FROM core_memory_blocks, json_each(core_memory_blocks.source_fact_ids) "
|
|
3086
|
-
" WHERE core_memory_blocks.profile_id = ?"
|
|
3087
|
-
")",
|
|
3088
|
-
(profile_id, profile_id),
|
|
3089
|
-
)
|
|
3090
|
-
return [dict(r) for r in rows]
|
|
3091
|
-
|
|
3092
|
-
def soft_delete_fact(self, fact_id: str, profile_id: str) -> None:
|
|
3093
|
-
"""Soft-delete a forgotten fact.
|
|
3094
|
-
|
|
3095
|
-
Sets fact_retention.lifecycle_zone to 'forgotten' and
|
|
3096
|
-
atomic_facts.lifecycle to 'archived' (valid enum value).
|
|
3097
|
-
Never physically deletes (HR-04).
|
|
3098
|
-
|
|
3099
|
-
Idempotent: if fact not found, logs warning and returns.
|
|
3100
|
-
"""
|
|
3101
|
-
# Check existence first (idempotent)
|
|
3102
|
-
rows = self.execute(
|
|
3103
|
-
"SELECT fact_id FROM fact_retention WHERE fact_id = ? AND profile_id = ?",
|
|
3104
|
-
(fact_id, profile_id),
|
|
3105
|
-
)
|
|
3106
|
-
if not rows:
|
|
3107
|
-
logger.warning(
|
|
3108
|
-
"soft_delete_fact: fact_id=%s not found in fact_retention, skipping",
|
|
3109
|
-
fact_id,
|
|
3110
|
-
)
|
|
3111
|
-
return
|
|
3112
|
-
|
|
3113
|
-
from superlocalmemory.core.lifecycle_state import set_fact_lifecycle_zone
|
|
3114
|
-
|
|
3115
|
-
with self.transaction():
|
|
3116
|
-
set_fact_lifecycle_zone(
|
|
3117
|
-
self, [fact_id], "forgotten", profile_id=profile_id,
|
|
3118
|
-
)
|
|
3119
|
-
self.execute(
|
|
3120
|
-
"UPDATE fact_retention SET retention_score = 0.0 "
|
|
3121
|
-
"WHERE fact_id = ? AND profile_id = ?",
|
|
3122
|
-
(fact_id, profile_id),
|
|
3123
|
-
)
|
|
3124
|
-
|
|
3125
|
-
# ------------------------------------------------------------------
|
|
3126
|
-
# Phase E: CCQ Consolidated Blocks & Audit CRUD
|
|
3127
|
-
# ------------------------------------------------------------------
|
|
3128
|
-
|
|
3129
|
-
def store_ccq_block(
|
|
3130
|
-
self,
|
|
3131
|
-
block_id: str,
|
|
3132
|
-
profile_id: str,
|
|
3133
|
-
content: str,
|
|
3134
|
-
source_fact_ids: str,
|
|
3135
|
-
gist_embedding_rowid: int | None,
|
|
3136
|
-
char_count: int,
|
|
3137
|
-
cluster_id: str,
|
|
3138
|
-
) -> None:
|
|
3139
|
-
"""Store a CCQ consolidated block. Parameterized SQL only."""
|
|
3140
|
-
self.execute(
|
|
3141
|
-
"INSERT INTO ccq_consolidated_blocks "
|
|
3142
|
-
"(block_id, profile_id, content, source_fact_ids, "
|
|
3143
|
-
" gist_embedding_rowid, char_count, compiled_by, cluster_id, created_at) "
|
|
3144
|
-
"VALUES (?, ?, ?, ?, ?, ?, 'ccq', ?, datetime('now'))",
|
|
3145
|
-
(block_id, profile_id, content, source_fact_ids,
|
|
3146
|
-
gist_embedding_rowid, char_count, cluster_id),
|
|
3147
|
-
)
|
|
3148
|
-
|
|
3149
|
-
def get_ccq_blocks(self, profile_id: str) -> list[dict]:
|
|
3150
|
-
"""Get all CCQ consolidated blocks for a profile."""
|
|
3151
|
-
rows = self.execute(
|
|
3152
|
-
"SELECT * FROM ccq_consolidated_blocks "
|
|
3153
|
-
"WHERE profile_id = ? ORDER BY created_at DESC",
|
|
3154
|
-
(profile_id,),
|
|
3155
|
-
)
|
|
3156
|
-
return [dict(r) for r in rows]
|
|
3157
|
-
|
|
3158
|
-
def store_ccq_audit(self, entry: dict) -> None:
|
|
3159
|
-
"""Store a CCQ audit log entry. Parameterized SQL only."""
|
|
3160
|
-
self.execute(
|
|
3161
|
-
"INSERT INTO ccq_audit_log "
|
|
3162
|
-
"(audit_id, profile_id, cluster_id, block_id, fact_ids, fact_count, "
|
|
3163
|
-
" gist_text, extraction_mode, bytes_before, bytes_after, "
|
|
3164
|
-
" compression_ratio, shared_entities, created_at) "
|
|
3165
|
-
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))",
|
|
3166
|
-
(entry["audit_id"], entry["profile_id"], entry["cluster_id"],
|
|
3167
|
-
entry["block_id"], entry["fact_ids"], entry["fact_count"],
|
|
3168
|
-
entry["gist_text"], entry["extraction_mode"],
|
|
3169
|
-
entry["bytes_before"], entry["bytes_after"],
|
|
3170
|
-
entry["compression_ratio"], entry["shared_entities"]),
|
|
3171
|
-
)
|
|
3172
|
-
|
|
3173
|
-
def get_ccq_audit(self, profile_id: str, limit: int = 50) -> list[dict]:
|
|
3174
|
-
"""Get CCQ audit log entries for a profile."""
|
|
3175
|
-
rows = self.execute(
|
|
3176
|
-
"SELECT * FROM ccq_audit_log "
|
|
3177
|
-
"WHERE profile_id = ? ORDER BY created_at DESC LIMIT ?",
|
|
3178
|
-
(profile_id, limit),
|
|
3179
|
-
)
|
|
3180
|
-
return [dict(r) for r in rows]
|