superlocalmemory 3.7.8 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/ATTRIBUTION.md +1 -3
- package/CHANGELOG.md +129 -0
- package/README.md +205 -123
- package/package.json +12 -3
- package/plugin/.claude-plugin/plugin.json +2 -3
- package/plugin/CLAUDE.md +8 -8
- package/plugin/agents/slm-governance-advisor.md +80 -0
- package/plugin/agents/slm-loop-runner.md +71 -0
- package/plugin/agents/slm-memory-advisor.md +10 -5
- package/plugin/agents/slm-optimize-advisor.md +9 -3
- package/plugin/commands/slm-loop.md +31 -0
- package/plugin/hooks/hooks.json +79 -0
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/slm-launch +46 -7
- package/plugin/settings.json +9 -0
- package/plugin/skills/slm-cache/SKILL.md +9 -1
- package/plugin/skills/slm-compress/SKILL.md +8 -1
- package/plugin/skills/slm-governance/SKILL.md +248 -0
- package/plugin/skills/slm-graph/SKILL.md +17 -3
- package/plugin/skills/slm-loop/SKILL.md +99 -0
- package/plugin/skills/slm-mesh/SKILL.md +282 -0
- package/plugin/skills/slm-profile/SKILL.md +148 -0
- package/plugin/skills/slm-recall/SKILL.md +46 -10
- package/plugin/skills/slm-remember/SKILL.md +48 -1
- package/plugin/skills/slm-scope/SKILL.md +176 -0
- package/plugin/skills/slm-session/SKILL.md +24 -1
- package/plugin/skills/slm-status/SKILL.md +18 -1
- package/plugin-src/rules/AGENTS.md +57 -18
- package/plugin-src/skills/slm-cache/SKILL.md +9 -1
- package/plugin-src/skills/slm-compress/SKILL.md +8 -1
- package/plugin-src/skills/slm-graph/SKILL.md +17 -3
- package/plugin-src/skills/slm-recall/SKILL.md +46 -10
- package/plugin-src/skills/slm-remember/SKILL.md +48 -1
- package/plugin-src/skills/slm-session/SKILL.md +24 -1
- package/plugin-src/skills/slm-status/SKILL.md +18 -1
- package/pyproject.toml +2 -1
- package/scripts/postinstall/validation.js +2 -0
- package/scripts/postinstall-interactive.js +74 -2
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/__init__.py +3 -0
- package/src/superlocalmemory/access/rbac.py +477 -0
- package/src/superlocalmemory/cli/commands.py +228 -17
- package/src/superlocalmemory/cli/compress_cmd.py +17 -7
- package/src/superlocalmemory/cli/daemon.py +7 -0
- package/src/superlocalmemory/cli/loop_cmd.py +187 -0
- package/src/superlocalmemory/cli/main.py +49 -8
- package/src/superlocalmemory/cli/mesh_cmd.py +38 -0
- package/src/superlocalmemory/cli/optimize_cmd.py +3 -0
- package/src/superlocalmemory/cli/pending_store.py +49 -13
- package/src/superlocalmemory/cli/proxy_cmd.py +4 -0
- package/src/superlocalmemory/cli/scale_engine_cmd.py +6 -0
- package/src/superlocalmemory/cli/setup_wizard.py +22 -13
- package/src/superlocalmemory/cli/version_banner.py +17 -3
- package/src/superlocalmemory/compliance/audit.py +6 -0
- package/src/superlocalmemory/compliance/gdpr.py +128 -138
- package/src/superlocalmemory/compliance/retention.py +176 -45
- package/src/superlocalmemory/core/backend_orchestrator.py +23 -59
- package/src/superlocalmemory/core/community_summary.py +267 -0
- package/src/superlocalmemory/core/config.py +216 -3
- package/src/superlocalmemory/core/consolidation_engine.py +95 -22
- package/src/superlocalmemory/core/context_cache.py +61 -18
- package/src/superlocalmemory/core/embedding_worker.py +21 -7
- package/src/superlocalmemory/core/embeddings.py +131 -46
- package/src/superlocalmemory/core/engine.py +41 -22
- package/src/superlocalmemory/core/engine_ingestion.py +359 -43
- package/src/superlocalmemory/core/engine_wiring.py +13 -0
- package/src/superlocalmemory/core/entity_community.py +178 -0
- package/src/superlocalmemory/core/graph_analyzer.py +39 -2
- package/src/superlocalmemory/core/graph_pruner.py +13 -8
- package/src/superlocalmemory/core/ingestion_command.py +134 -25
- package/src/superlocalmemory/core/injection.py +12 -7
- package/src/superlocalmemory/core/key_expander.py +138 -0
- package/src/superlocalmemory/core/maintenance.py +23 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +17 -7
- package/src/superlocalmemory/core/modes.py +1 -1
- package/src/superlocalmemory/core/mutations.py +2 -2
- package/src/superlocalmemory/core/pii.py +105 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +208 -0
- package/src/superlocalmemory/core/recall_pipeline.py +7 -3
- package/src/superlocalmemory/core/recall_worker.py +20 -6
- package/src/superlocalmemory/core/scale_engine.py +60 -1
- package/src/superlocalmemory/core/security_primitives.py +40 -2
- package/src/superlocalmemory/core/store_pipeline.py +186 -29
- package/src/superlocalmemory/core/worker_pool.py +21 -6
- package/src/superlocalmemory/encoding/entity_reflexion.py +200 -0
- package/src/superlocalmemory/encoding/entity_resolver.py +34 -24
- package/src/superlocalmemory/encoding/fact_extractor.py +26 -1
- package/src/superlocalmemory/encoding/temporal_validator.py +64 -1
- package/src/superlocalmemory/evolution/evolution_store.py +122 -45
- package/src/superlocalmemory/evolution/llm_dispatch.py +12 -1
- package/src/superlocalmemory/evolution/model_selection.py +160 -0
- package/src/superlocalmemory/evolution/mutation_generator.py +16 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +127 -42
- package/src/superlocalmemory/evolution/triggers.py +22 -13
- package/src/superlocalmemory/graph/cozo_backend.py +43 -20
- package/src/superlocalmemory/hooks/adapter_base.py +5 -1
- package/src/superlocalmemory/hooks/auto_recall.py +13 -1
- package/src/superlocalmemory/hooks/claude_code_hooks.py +11 -0
- package/src/superlocalmemory/hooks/codex_assets.py +64 -5
- package/src/superlocalmemory/hooks/hook_daemon.py +20 -3
- package/src/superlocalmemory/hooks/hook_handlers.py +6 -1
- package/src/superlocalmemory/hooks/memory_protocol.py +54 -0
- package/src/superlocalmemory/hooks/portable_kit.py +148 -3
- package/src/superlocalmemory/infra/backup.py +12 -1
- package/src/superlocalmemory/infra/daemon_identity.py +40 -4
- package/src/superlocalmemory/infra/data_root.py +43 -4
- package/src/superlocalmemory/infra/event_bus.py +107 -24
- package/src/superlocalmemory/infra/rate_limiter.py +93 -0
- package/src/superlocalmemory/ingestion/adapter_manager.py +4 -1
- package/src/superlocalmemory/ingestion/credentials.py +1 -1
- package/src/superlocalmemory/learning/cross_project.py +28 -19
- package/src/superlocalmemory/learning/model_rollback.py +3 -0
- package/src/superlocalmemory/learning/ranker_retrain_online.py +2 -0
- package/src/superlocalmemory/learning/reward.py +50 -0
- package/src/superlocalmemory/learning/reward_proxy.py +42 -9
- package/src/superlocalmemory/learning/source_quality.py +523 -1
- package/src/superlocalmemory/loops/__init__.py +56 -0
- package/src/superlocalmemory/loops/budget.py +58 -0
- package/src/superlocalmemory/loops/engine.py +164 -0
- package/src/superlocalmemory/loops/ledger.py +263 -0
- package/src/superlocalmemory/loops/models.py +152 -0
- package/src/superlocalmemory/loops/rules.py +52 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/profiles.py +103 -0
- package/src/superlocalmemory/mcp/server.py +32 -79
- package/src/superlocalmemory/mcp/tools_active.py +4 -7
- package/src/superlocalmemory/mcp/tools_code_graph.py +51 -5
- package/src/superlocalmemory/mcp/tools_core.py +12 -4
- package/src/superlocalmemory/mcp/tools_evolution.py +6 -3
- package/src/superlocalmemory/mcp/tools_learning.py +2 -2
- package/src/superlocalmemory/mcp/tools_loops.py +300 -0
- package/src/superlocalmemory/mcp/tools_mesh.py +140 -4
- package/src/superlocalmemory/mcp/tools_optimize.py +15 -8
- package/src/superlocalmemory/mesh/broker.py +237 -129
- package/src/superlocalmemory/mesh/remote_sync.py +50 -8
- package/src/superlocalmemory/optimize/NOTICE +1 -6
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +1 -4
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +1 -4
- package/src/superlocalmemory/optimize/cache/semantic.py +27 -19
- package/src/superlocalmemory/optimize/compress/align.py +32 -26
- package/src/superlocalmemory/optimize/compress/ccr.py +14 -71
- package/src/superlocalmemory/optimize/compress/router.py +105 -22
- package/src/superlocalmemory/optimize/config/defaults.py +1 -1
- package/src/superlocalmemory/optimize/config/schema.py +87 -4
- package/src/superlocalmemory/optimize/metrics/counters.py +13 -4
- package/src/superlocalmemory/optimize/metrics/estimator.py +0 -3
- package/src/superlocalmemory/optimize/proxy/_helpers.py +31 -4
- package/src/superlocalmemory/optimize/storage/db.py +38 -9
- package/src/superlocalmemory/optimize/storage/schema.py +10 -0
- package/src/superlocalmemory/parameterization/pattern_extractor.py +6 -3
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/bm25_channel.py +68 -10
- package/src/superlocalmemory/retrieval/engine.py +221 -47
- package/src/superlocalmemory/retrieval/entity_channel.py +7 -5
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -2
- package/src/superlocalmemory/retrieval/reranker.py +3 -4
- package/src/superlocalmemory/retrieval/semantic_channel.py +114 -21
- package/src/superlocalmemory/retrieval/spreading_activation.py +11 -2
- package/src/superlocalmemory/retrieval/temporal_channel.py +48 -9
- package/src/superlocalmemory/retrieval/temporal_frame.py +102 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +135 -0
- package/src/superlocalmemory/retrieval/time_window.py +181 -0
- package/src/superlocalmemory/server/api.py +4 -4
- package/src/superlocalmemory/server/config_file.py +90 -0
- package/src/superlocalmemory/server/origin.py +50 -0
- package/src/superlocalmemory/server/profile_runtime.py +125 -8
- package/src/superlocalmemory/server/rbac_enforce.py +142 -0
- package/src/superlocalmemory/server/recall_health.py +24 -3
- package/src/superlocalmemory/server/recall_serializer.py +19 -1
- package/src/superlocalmemory/server/routes/abstraction.py +115 -0
- package/src/superlocalmemory/server/routes/agents.py +128 -38
- package/src/superlocalmemory/server/routes/backup.py +317 -70
- package/src/superlocalmemory/server/routes/behavioral.py +349 -71
- package/src/superlocalmemory/server/routes/brain.py +69 -12
- package/src/superlocalmemory/server/routes/chat.py +10 -5
- package/src/superlocalmemory/server/routes/compliance.py +171 -21
- package/src/superlocalmemory/server/routes/config_api.py +438 -0
- package/src/superlocalmemory/server/routes/data_io.py +30 -8
- package/src/superlocalmemory/server/routes/entity.py +108 -26
- package/src/superlocalmemory/server/routes/events.py +24 -8
- package/src/superlocalmemory/server/routes/evolution.py +189 -68
- package/src/superlocalmemory/server/routes/helpers.py +16 -1
- package/src/superlocalmemory/server/routes/ingest.py +7 -4
- package/src/superlocalmemory/server/routes/insights.py +3 -3
- package/src/superlocalmemory/server/routes/learning.py +289 -118
- package/src/superlocalmemory/server/routes/learning_telemetry.py +153 -0
- package/src/superlocalmemory/server/routes/lifecycle.py +59 -8
- package/src/superlocalmemory/server/routes/memories.py +182 -57
- package/src/superlocalmemory/server/routes/mesh.py +200 -31
- package/src/superlocalmemory/server/routes/optimize.py +33 -1
- package/src/superlocalmemory/server/routes/prewarm.py +2 -0
- package/src/superlocalmemory/server/routes/profiles.py +63 -17
- package/src/superlocalmemory/server/routes/ratelimit.py +132 -0
- package/src/superlocalmemory/server/routes/rbac.py +367 -0
- package/src/superlocalmemory/server/routes/stats.py +103 -158
- package/src/superlocalmemory/server/routes/tiers.py +11 -9
- package/src/superlocalmemory/server/routes/token.py +3 -13
- package/src/superlocalmemory/server/routes/v3_api.py +247 -89
- package/src/superlocalmemory/server/routes/ws.py +5 -2
- package/src/superlocalmemory/server/security_middleware.py +12 -5
- package/src/superlocalmemory/server/ui.py +20 -5
- package/src/superlocalmemory/server/unified_daemon.py +827 -72
- package/src/superlocalmemory/server/write_identity.py +38 -8
- package/src/superlocalmemory/storage/database.py +265 -53
- package/src/superlocalmemory/storage/migration_runner.py +132 -1
- package/src/superlocalmemory/storage/migrations/M010_evolution_config.py +5 -0
- package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py +108 -0
- package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py +86 -0
- package/src/superlocalmemory/storage/migrations/M023_mesh_profile_isolation.py +194 -0
- package/src/superlocalmemory/storage/migrations/M024_rbac_users_roles.py +87 -0
- package/src/superlocalmemory/storage/migrations/M025_perf_indexes.py +90 -0
- package/src/superlocalmemory/storage/migrations/M026_rbac_memberships_fk.py +136 -0
- package/src/superlocalmemory/storage/migrations/M027_transferable_patterns_profile.py +163 -0
- package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py +270 -0
- package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py +137 -0
- package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py +93 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -0
- package/src/superlocalmemory/storage/models.py +4 -0
- package/src/superlocalmemory/storage/schema.py +136 -1
- package/src/superlocalmemory/storage/schema_v32.py +2 -0
- package/src/superlocalmemory/storage/schema_v343.py +24 -12
- package/src/superlocalmemory/storage/schema_v347.py +4 -0
- package/src/superlocalmemory/trust/gate.py +49 -8
- package/src/superlocalmemory/ui/assets/slm-icon-white.svg +64 -0
- package/src/superlocalmemory/ui/assets/slm-icon.svg +36 -0
- package/src/superlocalmemory/ui/css/design-system.css +621 -0
- package/src/superlocalmemory/ui/css/neural-glass.css +6 -0
- package/src/superlocalmemory/ui/css/od-bridge.css +158 -0
- package/src/superlocalmemory/ui/favicon.svg +35 -4
- package/src/superlocalmemory/ui/index.html +303 -173
- package/src/superlocalmemory/ui/js/brain.js +5 -20
- package/src/superlocalmemory/ui/js/core.js +100 -41
- package/src/superlocalmemory/ui/js/dashboard.js +403 -65
- package/src/superlocalmemory/ui/js/event-delegation.js +102 -0
- package/src/superlocalmemory/ui/js/knowledge-graph.js +11 -11
- package/src/superlocalmemory/ui/js/math-health.js +1 -1
- package/src/superlocalmemory/ui/js/memories.js +15 -4
- package/src/superlocalmemory/ui/js/memory-chat.js +7 -7
- package/src/superlocalmemory/ui/js/ng-entities.js +6 -8
- package/src/superlocalmemory/ui/js/ng-ingestion.js +4 -4
- package/src/superlocalmemory/ui/js/ng-mesh.js +4 -9
- package/src/superlocalmemory/ui/js/ng-shell.js +8 -8
- package/src/superlocalmemory/ui/js/ng-skills.js +54 -2
- package/src/superlocalmemory/ui/js/od-agents.js +544 -0
- package/src/superlocalmemory/ui/js/od-auth-gate.js +257 -0
- package/src/superlocalmemory/ui/js/od-backup.js +871 -0
- package/src/superlocalmemory/ui/js/od-brain.js +816 -0
- package/src/superlocalmemory/ui/js/od-entities.js +579 -0
- package/src/superlocalmemory/ui/js/od-graph.js +600 -0
- package/src/superlocalmemory/ui/js/od-health.js +539 -0
- package/src/superlocalmemory/ui/js/od-mcp.js +508 -0
- package/src/superlocalmemory/ui/js/od-memories.js +929 -0
- package/src/superlocalmemory/ui/js/od-mesh.js +553 -0
- package/src/superlocalmemory/ui/js/od-operations.js +1250 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +787 -0
- package/src/superlocalmemory/ui/js/od-settings.js +1107 -0
- package/src/superlocalmemory/ui/js/od-shell.js +809 -0
- package/src/superlocalmemory/ui/js/od-skills.js +600 -0
- package/src/superlocalmemory/ui/js/od-team.js +258 -0
- package/src/superlocalmemory/ui/js/profiles.js +159 -46
- package/src/superlocalmemory/ui/js/settings.js +17 -3
- package/src/superlocalmemory/ui/js/timeline.js +34 -5
- package/src/superlocalmemory/ui/js/trust-dashboard.js +2 -2
- package/src/superlocalmemory/vector/lancedb_backend.py +8 -6
- package/plugin-src/.mcp.json +0 -12
- package/plugin-src/agents/slm-memory-advisor.md +0 -44
- package/plugin-src/agents/slm-optimize-advisor.md +0 -38
- package/plugin-src/hooks/.gitkeep +0 -0
- package/plugin-src/hooks/hooks.json +0 -23
- package/plugin-src/manifest.json +0 -25
- package/plugin-src/requirements.txt +0 -1
- package/plugin-src/rules/CLAUDE.md.fragment +0 -44
- package/plugin-src/scripts/ensure-venv.bat +0 -122
- package/plugin-src/scripts/ensure-venv.sh +0 -105
- package/plugin-src/scripts/slm-launch +0 -23
- package/plugin-src/scripts/slm-launch.bat +0 -23
- package/plugin-src/settings.json +0 -16
- package/src/superlocalmemory/learning/behavioral_listener.py +0 -94
|
@@ -120,6 +120,8 @@ class SemanticChannel:
|
|
|
120
120
|
query_embedding: list[float],
|
|
121
121
|
profile_id: str,
|
|
122
122
|
top_k: int = 50,
|
|
123
|
+
include_global: bool | None = None,
|
|
124
|
+
include_shared: bool | None = None,
|
|
123
125
|
) -> list[tuple[str, float]]:
|
|
124
126
|
"""Search for semantically similar facts.
|
|
125
127
|
|
|
@@ -130,11 +132,23 @@ class SemanticChannel:
|
|
|
130
132
|
query_embedding: Dense vector for the query.
|
|
131
133
|
profile_id: Scope to this profile.
|
|
132
134
|
top_k: Maximum results to return.
|
|
135
|
+
include_global: Include global-scope facts. Defaults to the
|
|
136
|
+
``include_global`` instance attribute when not supplied,
|
|
137
|
+
preserving backward compatibility for callers that still
|
|
138
|
+
set the attribute directly.
|
|
139
|
+
include_shared: Include shared-scope facts. Same fallback.
|
|
133
140
|
|
|
134
141
|
Returns:
|
|
135
142
|
List of (fact_id, score) sorted by score descending.
|
|
136
143
|
Score is in [0, 1] range.
|
|
137
144
|
"""
|
|
145
|
+
# Resolve scope flags: explicit param takes priority; fall back to the
|
|
146
|
+
# legacy attribute-based path so existing callers keep working.
|
|
147
|
+
if include_global is None:
|
|
148
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
149
|
+
if include_shared is None:
|
|
150
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
151
|
+
|
|
138
152
|
if not query_embedding:
|
|
139
153
|
return []
|
|
140
154
|
|
|
@@ -142,16 +156,26 @@ class SemanticChannel:
|
|
|
142
156
|
|
|
143
157
|
# Lance is a derived projection. It is never an authorization source
|
|
144
158
|
# and it never silently replaces the canonical sqlite-vec path: every
|
|
145
|
-
# promoted query is shadowed and falls back if membership
|
|
159
|
+
# promoted query is shadowed and falls back if the result *membership*
|
|
160
|
+
# differs. Order differences within the same fact set are tolerated —
|
|
161
|
+
# float-score ties would otherwise force a fallback on every query,
|
|
162
|
+
# leaving the promoted backend permanently unused. Once membership
|
|
163
|
+
# matches, the projected backend's own ranking is authoritative.
|
|
146
164
|
if (
|
|
147
165
|
self._scale_vector_backend is not None
|
|
148
|
-
and not
|
|
149
|
-
and not
|
|
166
|
+
and not include_global
|
|
167
|
+
and not include_shared
|
|
150
168
|
):
|
|
151
|
-
projected = self._search_via_lance(
|
|
152
|
-
|
|
169
|
+
projected = self._search_via_lance(
|
|
170
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
171
|
+
include_global=include_global, include_shared=include_shared,
|
|
172
|
+
)
|
|
173
|
+
canonical = self._search_without_lance(
|
|
174
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
175
|
+
include_global=include_global, include_shared=include_shared,
|
|
176
|
+
)
|
|
153
177
|
self._scale_shadow_checks += 1
|
|
154
|
-
if
|
|
178
|
+
if {fid for fid, _ in projected} == {fid for fid, _ in canonical}:
|
|
155
179
|
return projected
|
|
156
180
|
self._scale_shadow_mismatches += 1
|
|
157
181
|
logger.warning("Lance semantic projection diverged from SQLite; using SQLite")
|
|
@@ -161,13 +185,17 @@ class SemanticChannel:
|
|
|
161
185
|
if self._vector_store and self._vector_store.available:
|
|
162
186
|
results = self._search_via_vector_store(
|
|
163
187
|
query_embedding, q_vec, profile_id, top_k,
|
|
188
|
+
include_global=include_global, include_shared=include_shared,
|
|
164
189
|
)
|
|
165
190
|
if results: # If vec0 returned results, use them
|
|
166
191
|
return results
|
|
167
192
|
# If vec0 is empty (cold start), fall through to full scan
|
|
168
193
|
|
|
169
194
|
# --- FALLBACK: full-table scan (original code, unchanged) ---
|
|
170
|
-
return self._search_full_scan(
|
|
195
|
+
return self._search_full_scan(
|
|
196
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
197
|
+
include_global=include_global, include_shared=include_shared,
|
|
198
|
+
)
|
|
171
199
|
|
|
172
200
|
def set_scale_vector_backend(self, backend: Any | None) -> None:
|
|
173
201
|
"""Attach a parity-verified Lance projection without replacing SQLite."""
|
|
@@ -180,14 +208,27 @@ class SemanticChannel:
|
|
|
180
208
|
}
|
|
181
209
|
|
|
182
210
|
def _search_via_lance(
|
|
183
|
-
self,
|
|
211
|
+
self,
|
|
212
|
+
query_embedding: list[float],
|
|
213
|
+
q_vec: np.ndarray,
|
|
214
|
+
profile_id: str,
|
|
215
|
+
top_k: int,
|
|
216
|
+
include_global: bool | None = None,
|
|
217
|
+
include_shared: bool | None = None,
|
|
184
218
|
) -> list[tuple[str, float]]:
|
|
219
|
+
if include_global is None:
|
|
220
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
221
|
+
if include_shared is None:
|
|
222
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
185
223
|
original_store, original_qas = self._vector_store, self._qas
|
|
186
224
|
try:
|
|
187
225
|
self._vector_store = _LanceCandidateSource(self._scale_vector_backend)
|
|
188
226
|
# QAS indexes SQLite/quantized records and cannot represent Lance.
|
|
189
227
|
self._qas = None
|
|
190
|
-
return self._search_via_vector_store(
|
|
228
|
+
return self._search_via_vector_store(
|
|
229
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
230
|
+
include_global=include_global, include_shared=include_shared,
|
|
231
|
+
)
|
|
191
232
|
except Exception as exc:
|
|
192
233
|
logger.warning("Lance semantic projection failed closed to SQLite: %s", exc)
|
|
193
234
|
return []
|
|
@@ -195,15 +236,31 @@ class SemanticChannel:
|
|
|
195
236
|
self._vector_store, self._qas = original_store, original_qas
|
|
196
237
|
|
|
197
238
|
def _search_without_lance(
|
|
198
|
-
self,
|
|
239
|
+
self,
|
|
240
|
+
query_embedding: list[float],
|
|
241
|
+
q_vec: np.ndarray,
|
|
242
|
+
profile_id: str,
|
|
243
|
+
top_k: int,
|
|
244
|
+
include_global: bool | None = None,
|
|
245
|
+
include_shared: bool | None = None,
|
|
199
246
|
) -> list[tuple[str, float]]:
|
|
247
|
+
if include_global is None:
|
|
248
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
249
|
+
if include_shared is None:
|
|
250
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
200
251
|
backend, self._scale_vector_backend = self._scale_vector_backend, None
|
|
201
252
|
try:
|
|
202
253
|
if self._vector_store and self._vector_store.available:
|
|
203
|
-
results = self._search_via_vector_store(
|
|
254
|
+
results = self._search_via_vector_store(
|
|
255
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
256
|
+
include_global=include_global, include_shared=include_shared,
|
|
257
|
+
)
|
|
204
258
|
if results:
|
|
205
259
|
return results
|
|
206
|
-
return self._search_full_scan(
|
|
260
|
+
return self._search_full_scan(
|
|
261
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
262
|
+
include_global=include_global, include_shared=include_shared,
|
|
263
|
+
)
|
|
207
264
|
finally:
|
|
208
265
|
self._scale_vector_backend = backend
|
|
209
266
|
|
|
@@ -213,8 +270,14 @@ class SemanticChannel:
|
|
|
213
270
|
q_vec: np.ndarray,
|
|
214
271
|
profile_id: str,
|
|
215
272
|
top_k: int,
|
|
273
|
+
include_global: bool | None = None,
|
|
274
|
+
include_shared: bool | None = None,
|
|
216
275
|
) -> list[tuple[str, float]]:
|
|
217
276
|
"""KNN via VectorStore (or QAS 3-tier), then Fisher-Rao re-scoring."""
|
|
277
|
+
if include_global is None:
|
|
278
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
279
|
+
if include_shared is None:
|
|
280
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
218
281
|
# V3.3.19: Try TurboQuant 3-tier search first (float32 + int8 + polar)
|
|
219
282
|
if self._qas is not None:
|
|
220
283
|
try:
|
|
@@ -235,12 +298,19 @@ class SemanticChannel:
|
|
|
235
298
|
query_embedding, top_k=top_k * 2, profile_id=profile_id,
|
|
236
299
|
)
|
|
237
300
|
|
|
301
|
+
# M-01: Normalize KNN scores to [0.5, 1.0] via (score + 1.0) / 2.0 so
|
|
302
|
+
# they are on the same scale as full-scan scores computed from
|
|
303
|
+
# (_cosine_similarity(q, f) + 1.0) / 2.0 in the external_scores path.
|
|
304
|
+
# vector_store.search() clips at 0 (max(0, cosine)), which maps [–1,1]
|
|
305
|
+
# to [0,1]; the canonical formula maps to [0.5, 1.0] for positives.
|
|
306
|
+
# Both are [0,1] but different scales — without this normalization KNN
|
|
307
|
+
# scores are systematically lower and external facts always win max().
|
|
308
|
+
knn_results = [(fid, (score + 1.0) / 2.0) for fid, score in knn_results]
|
|
309
|
+
|
|
238
310
|
# The vector index is partitioned by owner profile. An opted-in global
|
|
239
311
|
# or authorized shared fact owned by another profile cannot enter the
|
|
240
312
|
# local KNN candidate set, so merge the bounded cross-profile visible
|
|
241
313
|
# supplement using the same canonical DB scope predicate as fallback.
|
|
242
|
-
include_global = bool(getattr(self, "include_global", False))
|
|
243
|
-
include_shared = bool(getattr(self, "include_shared", False))
|
|
244
314
|
external_facts = self._db.get_external_visible_facts(
|
|
245
315
|
profile_id,
|
|
246
316
|
include_global=include_global,
|
|
@@ -272,8 +342,8 @@ class SemanticChannel:
|
|
|
272
342
|
knn_scores = {fid: score for fid, score in knn_results}
|
|
273
343
|
facts = self._db.get_facts_by_ids(
|
|
274
344
|
candidate_ids, profile_id,
|
|
275
|
-
include_global=
|
|
276
|
-
include_shared=
|
|
345
|
+
include_global=include_global,
|
|
346
|
+
include_shared=include_shared,
|
|
277
347
|
)
|
|
278
348
|
|
|
279
349
|
if not facts:
|
|
@@ -294,15 +364,32 @@ class SemanticChannel:
|
|
|
294
364
|
|
|
295
365
|
scored: list[tuple[str, float]] = []
|
|
296
366
|
for fact in facts:
|
|
297
|
-
|
|
367
|
+
# C2-ret H-01: recompute the final cosine from the canonical
|
|
368
|
+
# full-precision embedding using the SAME formula as the full-scan
|
|
369
|
+
# fallback, so the fast path is observationally equivalent — identical
|
|
370
|
+
# score MAGNITUDES, not merely identical rankings. The KNN/vector-store
|
|
371
|
+
# score is a candidate-SELECTION signal only (it decides which facts
|
|
372
|
+
# are Fisher-rescored), never the final magnitude; trusting it here
|
|
373
|
+
# leaked the vector store's negative-cosine clamp into public scores.
|
|
374
|
+
# When a candidate carries no usable embedding (index-only rows), fall
|
|
375
|
+
# back to the normalized KNN score, preserving the M-01 contract.
|
|
376
|
+
f_vec: np.ndarray | None = None
|
|
377
|
+
if fact.embedding is not None:
|
|
378
|
+
candidate = np.array(fact.embedding, dtype=np.float32)
|
|
379
|
+
if candidate.shape == q_vec.shape:
|
|
380
|
+
f_vec = candidate
|
|
381
|
+
|
|
382
|
+
if f_vec is not None:
|
|
383
|
+
cos_sim = (_cosine_similarity(q_vec, f_vec) + 1.0) / 2.0
|
|
384
|
+
else:
|
|
385
|
+
cos_sim = knn_scores.get(fact.fact_id, 0.0)
|
|
298
386
|
|
|
299
387
|
fisher_weight = self._fisher_weight(fact.access_count)
|
|
300
388
|
|
|
301
389
|
if (fisher_weight > 0.01
|
|
302
390
|
and fact.fisher_variance is not None
|
|
303
|
-
and
|
|
391
|
+
and f_vec is not None
|
|
304
392
|
and len(fact.fisher_variance) == len(q_vec)):
|
|
305
|
-
f_vec = np.array(fact.embedding, dtype=np.float32)
|
|
306
393
|
var_vec = np.array(fact.fisher_variance, dtype=np.float32)
|
|
307
394
|
f_sim = self._compute_fisher_sim(
|
|
308
395
|
q_vec, f_vec, var_vec, fact, q_mean, q_var,
|
|
@@ -323,10 +410,16 @@ class SemanticChannel:
|
|
|
323
410
|
q_vec: np.ndarray,
|
|
324
411
|
profile_id: str,
|
|
325
412
|
top_k: int,
|
|
413
|
+
include_global: bool | None = None,
|
|
414
|
+
include_shared: bool | None = None,
|
|
326
415
|
) -> list[tuple[str, float]]:
|
|
327
416
|
"""Original full-table-scan search. Used as fallback when VectorStore
|
|
328
417
|
is unavailable or empty (cold start).
|
|
329
418
|
"""
|
|
419
|
+
if include_global is None:
|
|
420
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
421
|
+
if include_shared is None:
|
|
422
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
330
423
|
# Compute query Fisher params for Bayesian comparison (F45 fix)
|
|
331
424
|
q_mean: np.ndarray | None = None
|
|
332
425
|
q_var: np.ndarray | None = None
|
|
@@ -337,8 +430,8 @@ class SemanticChannel:
|
|
|
337
430
|
|
|
338
431
|
facts = self._db.get_all_facts(
|
|
339
432
|
profile_id,
|
|
340
|
-
include_global=
|
|
341
|
-
include_shared=
|
|
433
|
+
include_global=include_global,
|
|
434
|
+
include_shared=include_shared,
|
|
342
435
|
)
|
|
343
436
|
|
|
344
437
|
scored: list[tuple[str, float]] = []
|
|
@@ -110,16 +110,25 @@ class SpreadingActivation:
|
|
|
110
110
|
query: Any,
|
|
111
111
|
profile_id: str = "",
|
|
112
112
|
top_k: int = 7,
|
|
113
|
+
include_global: bool | None = None,
|
|
114
|
+
include_shared: bool | None = None,
|
|
113
115
|
) -> list[tuple[str, float]]:
|
|
114
116
|
"""Channel-compatible interface: (query, top_k) -> [(fact_id, score)].
|
|
115
117
|
|
|
116
118
|
Matches ANNSearchable protocol (Rule 07).
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
include_global: Include global-scope facts. Falls back to the
|
|
122
|
+
instance attribute when not supplied.
|
|
123
|
+
include_shared: Include shared-scope facts. Same fallback.
|
|
117
124
|
"""
|
|
118
125
|
if not self._config.enabled:
|
|
119
126
|
return []
|
|
120
127
|
|
|
121
|
-
include_global
|
|
122
|
-
|
|
128
|
+
if include_global is None:
|
|
129
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
130
|
+
if include_shared is None:
|
|
131
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
123
132
|
try:
|
|
124
133
|
# Step 0: Get seed nodes from VectorStore KNN
|
|
125
134
|
seed_results = self._seed_search(
|
|
@@ -62,7 +62,14 @@ class TemporalChannel:
|
|
|
62
62
|
def __init__(self, db: DatabaseManager) -> None:
|
|
63
63
|
self._db = db
|
|
64
64
|
|
|
65
|
-
def search(
|
|
65
|
+
def search(
|
|
66
|
+
self,
|
|
67
|
+
query: str,
|
|
68
|
+
profile_id: str,
|
|
69
|
+
top_k: int = 30,
|
|
70
|
+
include_global: bool | None = None,
|
|
71
|
+
include_shared: bool | None = None,
|
|
72
|
+
) -> list[tuple[str, float]]:
|
|
66
73
|
"""Search for temporally relevant facts.
|
|
67
74
|
|
|
68
75
|
Two strategies:
|
|
@@ -72,7 +79,17 @@ class TemporalChannel:
|
|
|
72
79
|
|
|
73
80
|
Returns empty only when query has no temporal signal AND no
|
|
74
81
|
entity-temporal matches.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
include_global: Include global-scope facts. Falls back to the
|
|
85
|
+
instance attribute when not supplied.
|
|
86
|
+
include_shared: Include shared-scope facts. Same fallback.
|
|
75
87
|
"""
|
|
88
|
+
if include_global is None:
|
|
89
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
90
|
+
if include_shared is None:
|
|
91
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
92
|
+
|
|
76
93
|
parser = TemporalParser()
|
|
77
94
|
dates = parser.extract_dates_from_text(query)
|
|
78
95
|
query_dt = _parse_iso(dates.get("referenced_date"))
|
|
@@ -81,13 +98,18 @@ class TemporalChannel:
|
|
|
81
98
|
|
|
82
99
|
# Strategy 1: Entity-temporal metadata search
|
|
83
100
|
# "When did Alice...?" → find all temporal events for Alice
|
|
84
|
-
entity_results = self._entity_temporal_search(
|
|
101
|
+
entity_results = self._entity_temporal_search(
|
|
102
|
+
query, profile_id,
|
|
103
|
+
include_global=include_global, include_shared=include_shared,
|
|
104
|
+
)
|
|
85
105
|
|
|
86
106
|
# Strategy 2: Date proximity search
|
|
87
107
|
if query_dt is None and not entity_results:
|
|
88
108
|
return []
|
|
89
109
|
|
|
90
|
-
events = self._load_events(
|
|
110
|
+
events = self._load_events(
|
|
111
|
+
profile_id, include_global=include_global, include_shared=include_shared,
|
|
112
|
+
)
|
|
91
113
|
scored: dict[str, float] = {}
|
|
92
114
|
|
|
93
115
|
# Include entity-temporal results with high base score
|
|
@@ -124,13 +146,21 @@ class TemporalChannel:
|
|
|
124
146
|
return results[:top_k]
|
|
125
147
|
|
|
126
148
|
def _entity_temporal_search(
|
|
127
|
-
self,
|
|
149
|
+
self,
|
|
150
|
+
query: str,
|
|
151
|
+
profile_id: str,
|
|
152
|
+
include_global: bool | None = None,
|
|
153
|
+
include_shared: bool | None = None,
|
|
128
154
|
) -> list[tuple[str, float]]:
|
|
129
155
|
"""Metadata-first: find temporal events for entities mentioned in query.
|
|
130
156
|
|
|
131
157
|
"When did Alice do X?" → SQL filter by entity_id for Alice → return
|
|
132
158
|
all temporal facts about Alice. High precision for entity+time queries.
|
|
133
159
|
"""
|
|
160
|
+
if include_global is None:
|
|
161
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
162
|
+
if include_shared is None:
|
|
163
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
134
164
|
import re
|
|
135
165
|
_PROPER_RE = re.compile(r"\b([A-Z][a-z]+)\b")
|
|
136
166
|
names = [m.group(1) for m in _PROPER_RE.finditer(query)]
|
|
@@ -151,8 +181,8 @@ class TemporalChannel:
|
|
|
151
181
|
seen: set[str] = set()
|
|
152
182
|
where, params = _scope_where(
|
|
153
183
|
profile_id,
|
|
154
|
-
include_global=
|
|
155
|
-
include_shared=
|
|
184
|
+
include_global=include_global,
|
|
185
|
+
include_shared=include_shared,
|
|
156
186
|
prefix="af",
|
|
157
187
|
)
|
|
158
188
|
|
|
@@ -179,11 +209,20 @@ class TemporalChannel:
|
|
|
179
209
|
|
|
180
210
|
return results
|
|
181
211
|
|
|
182
|
-
def _load_events(
|
|
212
|
+
def _load_events(
|
|
213
|
+
self,
|
|
214
|
+
profile_id: str,
|
|
215
|
+
include_global: bool | None = None,
|
|
216
|
+
include_shared: bool | None = None,
|
|
217
|
+
) -> list[dict]:
|
|
218
|
+
if include_global is None:
|
|
219
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
220
|
+
if include_shared is None:
|
|
221
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
183
222
|
where, params = _scope_where(
|
|
184
223
|
profile_id,
|
|
185
|
-
include_global=
|
|
186
|
-
include_shared=
|
|
224
|
+
include_global=include_global,
|
|
225
|
+
include_shared=include_shared,
|
|
187
226
|
prefix="af",
|
|
188
227
|
)
|
|
189
228
|
rows = self._db.execute(
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Temporal-context injection helpers (Phase 4, T-inject).
|
|
6
|
+
|
|
7
|
+
LLMs have no innate sense of time — a recalled fact reads the same whether it
|
|
8
|
+
was stored an hour ago or two years ago. These pure helpers give every recalled
|
|
9
|
+
memory a human-relative age label and give the whole result set a "temporal
|
|
10
|
+
frame" header anchoring it to *now*, so the model can weigh recency and treat
|
|
11
|
+
aged facts as possibly stale.
|
|
12
|
+
|
|
13
|
+
Reuses ``time_window.parse_timestamp`` for tolerant timestamp parsing (SQLite
|
|
14
|
+
space form, ISO ``T``/``Z``, date-only) — comparisons are always on parsed
|
|
15
|
+
datetimes, never strings.
|
|
16
|
+
|
|
17
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
18
|
+
License: AGPL-3.0-or-later
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from datetime import datetime, timezone
|
|
24
|
+
from typing import Iterable
|
|
25
|
+
|
|
26
|
+
from superlocalmemory.retrieval.time_window import parse_timestamp
|
|
27
|
+
|
|
28
|
+
__all__ = ["relative_age", "temporal_frame"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _plural(n: int, unit: str) -> str:
|
|
32
|
+
return f"{n} {unit}" + ("" if n == 1 else "s")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def relative_age(timestamp: str | None, now: datetime | None = None) -> str:
|
|
36
|
+
"""Human-relative age of ``timestamp`` vs now, e.g. "3 days ago".
|
|
37
|
+
|
|
38
|
+
Returns "" when the timestamp is missing/unparseable. Future timestamps
|
|
39
|
+
(e.g. a referenced event date ahead of now) read as "in N …".
|
|
40
|
+
"""
|
|
41
|
+
dt = parse_timestamp(timestamp)
|
|
42
|
+
if dt is None:
|
|
43
|
+
return ""
|
|
44
|
+
_now = now or datetime.now(timezone.utc)
|
|
45
|
+
if _now.tzinfo is None:
|
|
46
|
+
_now = _now.replace(tzinfo=timezone.utc)
|
|
47
|
+
secs = (_now - dt).total_seconds()
|
|
48
|
+
future = secs < 0
|
|
49
|
+
secs = abs(secs)
|
|
50
|
+
|
|
51
|
+
if secs < 45:
|
|
52
|
+
return "just now"
|
|
53
|
+
minutes = secs / 60.0
|
|
54
|
+
hours = minutes / 60.0
|
|
55
|
+
days = hours / 24.0
|
|
56
|
+
if minutes < 45:
|
|
57
|
+
phrase = _plural(round(minutes), "minute")
|
|
58
|
+
elif hours < 24:
|
|
59
|
+
phrase = _plural(round(hours), "hour")
|
|
60
|
+
elif days < 14:
|
|
61
|
+
phrase = _plural(round(days), "day")
|
|
62
|
+
elif days < 60:
|
|
63
|
+
phrase = _plural(round(days / 7.0), "week")
|
|
64
|
+
elif days < 365:
|
|
65
|
+
phrase = _plural(round(days / 30.0), "month")
|
|
66
|
+
else:
|
|
67
|
+
phrase = _plural(round(days / 365.0), "year")
|
|
68
|
+
return f"in {phrase}" if future else f"{phrase} ago"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def temporal_frame(
|
|
72
|
+
timestamps: Iterable[str | None],
|
|
73
|
+
now: datetime | None = None,
|
|
74
|
+
) -> str:
|
|
75
|
+
"""A one-line "now" anchor + age span for a set of recalled timestamps.
|
|
76
|
+
|
|
77
|
+
Example: ``"Now: 2026-07-22T12:00:00+00:00. Recalled memories span 2 years
|
|
78
|
+
ago → just now. Treat undated or aged facts as possibly stale."``
|
|
79
|
+
|
|
80
|
+
With no dated timestamps, returns just the now-anchor + an undated note.
|
|
81
|
+
"""
|
|
82
|
+
_now = now or datetime.now(timezone.utc)
|
|
83
|
+
if _now.tzinfo is None:
|
|
84
|
+
_now = _now.replace(tzinfo=timezone.utc)
|
|
85
|
+
now_iso = _now.replace(microsecond=0).isoformat()
|
|
86
|
+
|
|
87
|
+
dts = [d for d in (parse_timestamp(t) for t in timestamps) if d is not None]
|
|
88
|
+
if not dts:
|
|
89
|
+
return f"Now: {now_iso}. Recalled memories are undated."
|
|
90
|
+
|
|
91
|
+
oldest = min(dts)
|
|
92
|
+
newest = max(dts)
|
|
93
|
+
span = (
|
|
94
|
+
relative_age(oldest.isoformat(), _now)
|
|
95
|
+
if oldest == newest
|
|
96
|
+
else f"{relative_age(oldest.isoformat(), _now)} → "
|
|
97
|
+
f"{relative_age(newest.isoformat(), _now)}"
|
|
98
|
+
)
|
|
99
|
+
return (
|
|
100
|
+
f"Now: {now_iso}. Recalled memories span {span}. "
|
|
101
|
+
f"Treat undated or aged facts as possibly stale."
|
|
102
|
+
)
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
|
+
|
|
5
|
+
"""Bi-temporal validity filter for the retrieval pipeline (Phase 4, T1).
|
|
6
|
+
|
|
7
|
+
Post-retrieval filter for *system-invalidated* facts — a fact whose temporal
|
|
8
|
+
record has ``system_expired_at`` set was superseded/contradicted by a newer
|
|
9
|
+
fact (see ``invalidate_fact_temporal`` / conflict-resolution supersession).
|
|
10
|
+
|
|
11
|
+
P5-INT-01 (non-destructive supersession): such a fact is DEMOTED, not hidden.
|
|
12
|
+
Its per-channel score is multiplied by ``superseded_demotion_factor`` (default
|
|
13
|
+
0.25) and the channel lists are re-sorted, so currently-valid facts rank above
|
|
14
|
+
it — but nothing valid silently vanishes. This is the Mem0-2026 design that
|
|
15
|
+
wins long-term-memory benchmarks: keep every fact recallable and let
|
|
16
|
+
retrieval-time recency resolve conflicts, rather than destructively deleting on
|
|
17
|
+
a write-time contradiction guess (which over-fires: two complementary facts
|
|
18
|
+
about the same entity diverge past the coboundary threshold and one would be
|
|
19
|
+
wrongly hidden). A factor of 0.0 restores the legacy hide behaviour (a demoted
|
|
20
|
+
score of 0 is gated out by the evidence floor).
|
|
21
|
+
|
|
22
|
+
The filter runs on the per-channel candidate dict BEFORE RRF fusion, so fused
|
|
23
|
+
ranks reflect the demotion. It queries validity only for the bounded candidate
|
|
24
|
+
set (never the full ``get_valid_facts`` set) — an indexed, O(candidates) lookup
|
|
25
|
+
on the hot path, no full-table scan.
|
|
26
|
+
|
|
27
|
+
Pure SQL, no LLM → safe in every mode including Mode A. A no-op when nothing is
|
|
28
|
+
invalidated and when config.enabled is False.
|
|
29
|
+
|
|
30
|
+
Integrates with ChannelRegistry.register_filter() using the FilterFn signature:
|
|
31
|
+
(all_channel_results, profile_id, context) -> filtered_results
|
|
32
|
+
|
|
33
|
+
Part of Qualixar | Author: Varun Pratap Bhardwaj
|
|
34
|
+
License: AGPL-3.0-or-later
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import logging
|
|
40
|
+
from typing import TYPE_CHECKING, Any
|
|
41
|
+
|
|
42
|
+
if TYPE_CHECKING:
|
|
43
|
+
from superlocalmemory.core.config import TemporalValidatorConfig
|
|
44
|
+
from superlocalmemory.retrieval.channel_registry import ChannelRegistry
|
|
45
|
+
from superlocalmemory.storage.database import DatabaseManager
|
|
46
|
+
|
|
47
|
+
logger = logging.getLogger(__name__)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class TemporalValidityFilter:
|
|
51
|
+
"""Demotes system-invalidated (superseded) facts in retrieval candidates."""
|
|
52
|
+
|
|
53
|
+
__slots__ = ("_db", "_demotion_factor")
|
|
54
|
+
|
|
55
|
+
def __init__(self, db: DatabaseManager, demotion_factor: float = 0.25) -> None:
|
|
56
|
+
self._db = db
|
|
57
|
+
# Clamp to [0, 1]. 0.0 = legacy hide (evidence floor drops zero-score
|
|
58
|
+
# facts); 1.0 = no demotion.
|
|
59
|
+
self._demotion_factor = max(0.0, min(1.0, float(demotion_factor)))
|
|
60
|
+
|
|
61
|
+
def filter(
|
|
62
|
+
self,
|
|
63
|
+
all_results: dict[str, list[tuple[str, float]]],
|
|
64
|
+
profile_id: str,
|
|
65
|
+
context: Any,
|
|
66
|
+
) -> dict[str, list[tuple[str, float]]]:
|
|
67
|
+
"""Demote superseded fact_ids in every channel's candidate list.
|
|
68
|
+
|
|
69
|
+
Matches FilterFn signature from channel_registry.py.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
all_results: Channel name -> [(fact_id, score)] dict.
|
|
73
|
+
profile_id: Current profile.
|
|
74
|
+
context: Optional context (unused).
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
A new dict where system-invalidated facts keep their channel
|
|
78
|
+
presence but have their score scaled by the demotion factor and the
|
|
79
|
+
channel lists re-sorted (so valid facts rank above them). Inputs are
|
|
80
|
+
never mutated (immutability). Unchanged when nothing is invalidated.
|
|
81
|
+
"""
|
|
82
|
+
# Collect all unique candidate fact_ids across every channel.
|
|
83
|
+
all_fact_ids: set[str] = set()
|
|
84
|
+
for channel_results in all_results.values():
|
|
85
|
+
for fact_id, _ in channel_results:
|
|
86
|
+
all_fact_ids.add(fact_id)
|
|
87
|
+
|
|
88
|
+
if not all_fact_ids:
|
|
89
|
+
return all_results
|
|
90
|
+
|
|
91
|
+
try:
|
|
92
|
+
invalid = self._db.get_invalidated_fact_ids(
|
|
93
|
+
list(all_fact_ids), profile_id,
|
|
94
|
+
)
|
|
95
|
+
except Exception as exc:
|
|
96
|
+
# Fail-open: a validity-lookup error must never break retrieval.
|
|
97
|
+
logger.warning("Temporal validity lookup failed: %s", exc)
|
|
98
|
+
return all_results
|
|
99
|
+
|
|
100
|
+
if not invalid:
|
|
101
|
+
return all_results
|
|
102
|
+
|
|
103
|
+
factor = self._demotion_factor
|
|
104
|
+
demoted: dict[str, list[tuple[str, float]]] = {}
|
|
105
|
+
for channel_name, channel_results in all_results.items():
|
|
106
|
+
new_list = [
|
|
107
|
+
(fact_id, score * factor if fact_id in invalid else score)
|
|
108
|
+
for fact_id, score in channel_results
|
|
109
|
+
]
|
|
110
|
+
# Re-sort descending so demoted (superseded) facts fall below
|
|
111
|
+
# currently-valid facts in this channel's rank order.
|
|
112
|
+
new_list.sort(key=lambda pair: pair[1], reverse=True)
|
|
113
|
+
demoted[channel_name] = new_list
|
|
114
|
+
return demoted
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def register_temporal_validity_filter(
|
|
118
|
+
registry: ChannelRegistry,
|
|
119
|
+
db: DatabaseManager,
|
|
120
|
+
config: TemporalValidatorConfig,
|
|
121
|
+
) -> None:
|
|
122
|
+
"""Register the bi-temporal validity filter into the channel registry.
|
|
123
|
+
|
|
124
|
+
Does nothing if config.enabled is False.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
registry: Channel registry to register with.
|
|
128
|
+
db: Database manager for validity queries.
|
|
129
|
+
config: Temporal-validator configuration.
|
|
130
|
+
"""
|
|
131
|
+
if not getattr(config, "enabled", True):
|
|
132
|
+
return
|
|
133
|
+
factor = getattr(config, "superseded_demotion_factor", 0.25)
|
|
134
|
+
f = TemporalValidityFilter(db, demotion_factor=factor)
|
|
135
|
+
registry.register_filter(f.filter)
|