superlocalmemory 3.7.8 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/ATTRIBUTION.md +1 -3
- package/CHANGELOG.md +69 -0
- package/README.md +199 -29
- package/package.json +4 -2
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/CLAUDE.md +8 -8
- package/plugin/agents/slm-governance-advisor.md +80 -0
- package/plugin/agents/slm-loop-runner.md +71 -0
- package/plugin/agents/slm-memory-advisor.md +10 -5
- package/plugin/agents/slm-optimize-advisor.md +9 -3
- package/plugin/commands/slm-loop.md +31 -0
- package/plugin/hooks/hooks.json +79 -0
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/slm-launch +46 -7
- package/plugin/settings.json +9 -0
- package/plugin/skills/slm-cache/SKILL.md +9 -1
- package/plugin/skills/slm-compress/SKILL.md +8 -1
- package/plugin/skills/slm-governance/SKILL.md +248 -0
- package/plugin/skills/slm-graph/SKILL.md +17 -3
- package/plugin/skills/slm-loop/SKILL.md +99 -0
- package/plugin/skills/slm-mesh/SKILL.md +282 -0
- package/plugin/skills/slm-profile/SKILL.md +148 -0
- package/plugin/skills/slm-recall/SKILL.md +46 -10
- package/plugin/skills/slm-remember/SKILL.md +48 -1
- package/plugin/skills/slm-scope/SKILL.md +176 -0
- package/plugin/skills/slm-session/SKILL.md +24 -1
- package/plugin/skills/slm-status/SKILL.md +18 -1
- package/plugin-src/agents/slm-governance-advisor.md +80 -0
- package/plugin-src/agents/slm-loop-runner.md +71 -0
- package/plugin-src/agents/slm-memory-advisor.md +10 -5
- package/plugin-src/agents/slm-optimize-advisor.md +9 -3
- package/plugin-src/commands/slm-loop.md +31 -0
- package/plugin-src/hooks/hooks.json +79 -0
- package/plugin-src/manifest.json +7 -2
- package/plugin-src/requirements.txt +1 -1
- package/plugin-src/rules/AGENTS.md +57 -18
- package/plugin-src/rules/CLAUDE.md.fragment +8 -8
- package/plugin-src/scripts/slm-launch +46 -7
- package/plugin-src/settings.json +9 -0
- package/plugin-src/skills/slm-cache/SKILL.md +9 -1
- package/plugin-src/skills/slm-compress/SKILL.md +8 -1
- package/plugin-src/skills/slm-governance/SKILL.md +248 -0
- package/plugin-src/skills/slm-graph/SKILL.md +17 -3
- package/plugin-src/skills/slm-loop/SKILL.md +99 -0
- package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
- package/plugin-src/skills/slm-profile/SKILL.md +148 -0
- package/plugin-src/skills/slm-recall/SKILL.md +46 -10
- package/plugin-src/skills/slm-remember/SKILL.md +48 -1
- package/plugin-src/skills/slm-scope/SKILL.md +176 -0
- package/plugin-src/skills/slm-session/SKILL.md +24 -1
- package/plugin-src/skills/slm-status/SKILL.md +18 -1
- package/pyproject.toml +1 -1
- package/scripts/postinstall/validation.js +2 -0
- package/scripts/postinstall-interactive.js +74 -2
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/access/__init__.py +3 -0
- package/src/superlocalmemory/access/rbac.py +477 -0
- package/src/superlocalmemory/cli/commands.py +94 -10
- package/src/superlocalmemory/cli/compress_cmd.py +17 -7
- package/src/superlocalmemory/cli/loop_cmd.py +192 -0
- package/src/superlocalmemory/cli/main.py +39 -4
- package/src/superlocalmemory/cli/mesh_cmd.py +38 -0
- package/src/superlocalmemory/cli/optimize_cmd.py +3 -0
- package/src/superlocalmemory/cli/pending_store.py +49 -13
- package/src/superlocalmemory/cli/proxy_cmd.py +4 -0
- package/src/superlocalmemory/cli/scale_engine_cmd.py +6 -0
- package/src/superlocalmemory/cli/setup_wizard.py +22 -13
- package/src/superlocalmemory/compliance/audit.py +6 -0
- package/src/superlocalmemory/compliance/gdpr.py +128 -138
- package/src/superlocalmemory/compliance/retention.py +176 -45
- package/src/superlocalmemory/core/backend_orchestrator.py +5 -43
- package/src/superlocalmemory/core/community_summary.py +267 -0
- package/src/superlocalmemory/core/config.py +216 -3
- package/src/superlocalmemory/core/consolidation_engine.py +95 -22
- package/src/superlocalmemory/core/context_cache.py +61 -18
- package/src/superlocalmemory/core/embedding_worker.py +17 -2
- package/src/superlocalmemory/core/embeddings.py +12 -1
- package/src/superlocalmemory/core/engine.py +17 -1
- package/src/superlocalmemory/core/engine_ingestion.py +29 -0
- package/src/superlocalmemory/core/engine_wiring.py +13 -0
- package/src/superlocalmemory/core/entity_community.py +178 -0
- package/src/superlocalmemory/core/graph_analyzer.py +39 -2
- package/src/superlocalmemory/core/graph_pruner.py +13 -8
- package/src/superlocalmemory/core/key_expander.py +138 -0
- package/src/superlocalmemory/core/maintenance.py +23 -0
- package/src/superlocalmemory/core/modes.py +1 -1
- package/src/superlocalmemory/core/mutations.py +2 -2
- package/src/superlocalmemory/core/pii.py +105 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +208 -0
- package/src/superlocalmemory/core/recall_pipeline.py +2 -0
- package/src/superlocalmemory/core/recall_worker.py +20 -6
- package/src/superlocalmemory/core/scale_engine.py +60 -1
- package/src/superlocalmemory/core/security_primitives.py +40 -2
- package/src/superlocalmemory/core/store_pipeline.py +35 -11
- package/src/superlocalmemory/core/worker_pool.py +21 -6
- package/src/superlocalmemory/encoding/entity_reflexion.py +200 -0
- package/src/superlocalmemory/encoding/entity_resolver.py +34 -24
- package/src/superlocalmemory/encoding/fact_extractor.py +26 -1
- package/src/superlocalmemory/encoding/temporal_validator.py +64 -1
- package/src/superlocalmemory/evolution/evolution_store.py +122 -45
- package/src/superlocalmemory/evolution/llm_dispatch.py +12 -1
- package/src/superlocalmemory/evolution/model_selection.py +160 -0
- package/src/superlocalmemory/evolution/mutation_generator.py +16 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +127 -42
- package/src/superlocalmemory/evolution/triggers.py +22 -13
- package/src/superlocalmemory/graph/cozo_backend.py +43 -20
- package/src/superlocalmemory/hooks/adapter_base.py +5 -1
- package/src/superlocalmemory/hooks/auto_recall.py +13 -1
- package/src/superlocalmemory/hooks/claude_code_hooks.py +11 -0
- package/src/superlocalmemory/hooks/codex_assets.py +64 -5
- package/src/superlocalmemory/hooks/hook_daemon.py +20 -3
- package/src/superlocalmemory/hooks/memory_protocol.py +54 -0
- package/src/superlocalmemory/hooks/portable_kit.py +114 -1
- package/src/superlocalmemory/infra/backup.py +12 -1
- package/src/superlocalmemory/infra/daemon_identity.py +40 -4
- package/src/superlocalmemory/infra/data_root.py +43 -4
- package/src/superlocalmemory/infra/event_bus.py +107 -24
- package/src/superlocalmemory/infra/rate_limiter.py +93 -0
- package/src/superlocalmemory/ingestion/adapter_manager.py +4 -1
- package/src/superlocalmemory/ingestion/credentials.py +1 -1
- package/src/superlocalmemory/learning/cross_project.py +28 -19
- package/src/superlocalmemory/learning/reward_proxy.py +42 -9
- package/src/superlocalmemory/loops/__init__.py +56 -0
- package/src/superlocalmemory/loops/budget.py +58 -0
- package/src/superlocalmemory/loops/engine.py +164 -0
- package/src/superlocalmemory/loops/ledger.py +243 -0
- package/src/superlocalmemory/loops/models.py +152 -0
- package/src/superlocalmemory/loops/rules.py +52 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/_pool_adapter.py +2 -0
- package/src/superlocalmemory/mcp/profiles.py +103 -0
- package/src/superlocalmemory/mcp/server.py +21 -49
- package/src/superlocalmemory/mcp/tools_active.py +4 -7
- package/src/superlocalmemory/mcp/tools_code_graph.py +51 -5
- package/src/superlocalmemory/mcp/tools_core.py +8 -1
- package/src/superlocalmemory/mcp/tools_evolution.py +6 -3
- package/src/superlocalmemory/mcp/tools_loops.py +300 -0
- package/src/superlocalmemory/mcp/tools_mesh.py +140 -4
- package/src/superlocalmemory/mcp/tools_optimize.py +15 -8
- package/src/superlocalmemory/mesh/broker.py +237 -129
- package/src/superlocalmemory/mesh/remote_sync.py +50 -8
- package/src/superlocalmemory/optimize/NOTICE +1 -6
- package/src/superlocalmemory/optimize/adapters/anthropic_adapter.py +1 -4
- package/src/superlocalmemory/optimize/adapters/openai_adapter.py +1 -4
- package/src/superlocalmemory/optimize/cache/semantic.py +27 -19
- package/src/superlocalmemory/optimize/compress/align.py +32 -26
- package/src/superlocalmemory/optimize/compress/ccr.py +14 -71
- package/src/superlocalmemory/optimize/compress/router.py +105 -22
- package/src/superlocalmemory/optimize/config/defaults.py +1 -1
- package/src/superlocalmemory/optimize/config/schema.py +87 -4
- package/src/superlocalmemory/optimize/metrics/counters.py +13 -4
- package/src/superlocalmemory/optimize/metrics/estimator.py +0 -3
- package/src/superlocalmemory/optimize/proxy/_helpers.py +31 -4
- package/src/superlocalmemory/optimize/storage/db.py +38 -9
- package/src/superlocalmemory/optimize/storage/schema.py +10 -0
- package/src/superlocalmemory/parameterization/pattern_extractor.py +6 -3
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/bm25_channel.py +68 -10
- package/src/superlocalmemory/retrieval/engine.py +168 -26
- package/src/superlocalmemory/retrieval/entity_channel.py +7 -5
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -2
- package/src/superlocalmemory/retrieval/semantic_channel.py +114 -21
- package/src/superlocalmemory/retrieval/spreading_activation.py +11 -2
- package/src/superlocalmemory/retrieval/temporal_channel.py +48 -9
- package/src/superlocalmemory/retrieval/temporal_frame.py +102 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +135 -0
- package/src/superlocalmemory/retrieval/time_window.py +181 -0
- package/src/superlocalmemory/server/api.py +4 -4
- package/src/superlocalmemory/server/profile_runtime.py +125 -8
- package/src/superlocalmemory/server/rbac_enforce.py +142 -0
- package/src/superlocalmemory/server/recall_health.py +24 -3
- package/src/superlocalmemory/server/recall_serializer.py +19 -1
- package/src/superlocalmemory/server/routes/abstraction.py +115 -0
- package/src/superlocalmemory/server/routes/agents.py +128 -38
- package/src/superlocalmemory/server/routes/backup.py +34 -10
- package/src/superlocalmemory/server/routes/behavioral.py +13 -12
- package/src/superlocalmemory/server/routes/brain.py +21 -5
- package/src/superlocalmemory/server/routes/chat.py +10 -5
- package/src/superlocalmemory/server/routes/compliance.py +171 -21
- package/src/superlocalmemory/server/routes/config_api.py +436 -0
- package/src/superlocalmemory/server/routes/data_io.py +30 -8
- package/src/superlocalmemory/server/routes/entity.py +9 -4
- package/src/superlocalmemory/server/routes/events.py +24 -8
- package/src/superlocalmemory/server/routes/evolution.py +135 -17
- package/src/superlocalmemory/server/routes/helpers.py +16 -1
- package/src/superlocalmemory/server/routes/ingest.py +7 -4
- package/src/superlocalmemory/server/routes/insights.py +3 -3
- package/src/superlocalmemory/server/routes/learning.py +14 -14
- package/src/superlocalmemory/server/routes/lifecycle.py +59 -8
- package/src/superlocalmemory/server/routes/memories.py +182 -57
- package/src/superlocalmemory/server/routes/mesh.py +95 -15
- package/src/superlocalmemory/server/routes/optimize.py +33 -1
- package/src/superlocalmemory/server/routes/prewarm.py +2 -0
- package/src/superlocalmemory/server/routes/profiles.py +63 -17
- package/src/superlocalmemory/server/routes/ratelimit.py +124 -0
- package/src/superlocalmemory/server/routes/rbac.py +367 -0
- package/src/superlocalmemory/server/routes/stats.py +13 -6
- package/src/superlocalmemory/server/routes/tiers.py +11 -9
- package/src/superlocalmemory/server/routes/v3_api.py +183 -69
- package/src/superlocalmemory/server/routes/ws.py +5 -2
- package/src/superlocalmemory/server/security_middleware.py +12 -5
- package/src/superlocalmemory/server/ui.py +20 -5
- package/src/superlocalmemory/server/unified_daemon.py +384 -56
- package/src/superlocalmemory/server/write_identity.py +38 -8
- package/src/superlocalmemory/storage/database.py +265 -53
- package/src/superlocalmemory/storage/migration_runner.py +53 -0
- package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py +108 -0
- package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py +86 -0
- package/src/superlocalmemory/storage/migrations/M023_mesh_profile_isolation.py +194 -0
- package/src/superlocalmemory/storage/migrations/M024_rbac_users_roles.py +87 -0
- package/src/superlocalmemory/storage/migrations/M025_perf_indexes.py +90 -0
- package/src/superlocalmemory/storage/migrations/M026_rbac_memberships_fk.py +136 -0
- package/src/superlocalmemory/storage/migrations/M027_transferable_patterns_profile.py +163 -0
- package/src/superlocalmemory/storage/models.py +4 -0
- package/src/superlocalmemory/storage/schema.py +87 -0
- package/src/superlocalmemory/storage/schema_v343.py +24 -12
- package/src/superlocalmemory/trust/gate.py +49 -8
- package/src/superlocalmemory/ui/assets/slm-icon-white.svg +64 -0
- package/src/superlocalmemory/ui/assets/slm-icon.svg +36 -0
- package/src/superlocalmemory/ui/css/design-system.css +621 -0
- package/src/superlocalmemory/ui/css/neural-glass.css +6 -0
- package/src/superlocalmemory/ui/css/od-bridge.css +158 -0
- package/src/superlocalmemory/ui/favicon.svg +35 -4
- package/src/superlocalmemory/ui/index.html +306 -173
- package/src/superlocalmemory/ui/js/brain.js +5 -20
- package/src/superlocalmemory/ui/js/core.js +47 -31
- package/src/superlocalmemory/ui/js/dashboard.js +314 -63
- package/src/superlocalmemory/ui/js/event-delegation.js +102 -0
- package/src/superlocalmemory/ui/js/knowledge-graph.js +11 -11
- package/src/superlocalmemory/ui/js/math-health.js +1 -1
- package/src/superlocalmemory/ui/js/memories.js +15 -4
- package/src/superlocalmemory/ui/js/memory-chat.js +7 -7
- package/src/superlocalmemory/ui/js/ng-entities.js +6 -8
- package/src/superlocalmemory/ui/js/ng-ingestion.js +4 -4
- package/src/superlocalmemory/ui/js/ng-mesh.js +4 -9
- package/src/superlocalmemory/ui/js/ng-shell.js +8 -8
- package/src/superlocalmemory/ui/js/ng-skills.js +54 -2
- package/src/superlocalmemory/ui/js/od-agents.js +544 -0
- package/src/superlocalmemory/ui/js/od-auth-gate.js +257 -0
- package/src/superlocalmemory/ui/js/od-backup.js +780 -0
- package/src/superlocalmemory/ui/js/od-brain.js +779 -0
- package/src/superlocalmemory/ui/js/od-entities.js +579 -0
- package/src/superlocalmemory/ui/js/od-graph.js +593 -0
- package/src/superlocalmemory/ui/js/od-health.js +539 -0
- package/src/superlocalmemory/ui/js/od-mcp.js +508 -0
- package/src/superlocalmemory/ui/js/od-memories.js +887 -0
- package/src/superlocalmemory/ui/js/od-mesh.js +539 -0
- package/src/superlocalmemory/ui/js/od-operations.js +1250 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +787 -0
- package/src/superlocalmemory/ui/js/od-settings.js +1053 -0
- package/src/superlocalmemory/ui/js/od-shell.js +593 -0
- package/src/superlocalmemory/ui/js/od-skills.js +573 -0
- package/src/superlocalmemory/ui/js/od-team.js +258 -0
- package/src/superlocalmemory/ui/js/profiles.js +159 -46
- package/src/superlocalmemory/ui/js/settings.js +2 -2
- package/src/superlocalmemory/ui/js/timeline.js +34 -5
- package/src/superlocalmemory/ui/js/trust-dashboard.js +2 -2
- package/src/superlocalmemory/vector/lancedb_backend.py +8 -6
- package/src/superlocalmemory/learning/behavioral_listener.py +0 -94
|
@@ -17,6 +17,7 @@ License: AGPL-3.0-or-later
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
19
|
import concurrent.futures
|
|
20
|
+
import functools
|
|
20
21
|
import logging
|
|
21
22
|
import math
|
|
22
23
|
import re
|
|
@@ -26,6 +27,11 @@ from typing import TYPE_CHECKING, Any, Protocol
|
|
|
26
27
|
|
|
27
28
|
from superlocalmemory.core.config import ChannelWeights, RetrievalConfig
|
|
28
29
|
from superlocalmemory.retrieval.fusion import FusionResult, weighted_rrf
|
|
30
|
+
from superlocalmemory.retrieval.time_window import (
|
|
31
|
+
in_window,
|
|
32
|
+
infer_window_from_query,
|
|
33
|
+
parse_window,
|
|
34
|
+
)
|
|
29
35
|
from superlocalmemory.retrieval.strategy import QueryStrategy, QueryStrategyClassifier
|
|
30
36
|
from superlocalmemory.storage.models import (
|
|
31
37
|
AtomicFact,
|
|
@@ -97,12 +103,11 @@ class RetrievalEngine:
|
|
|
97
103
|
self._profile_channel = profile_channel
|
|
98
104
|
self._bridge = bridge_discovery
|
|
99
105
|
self._trust_scorer = trust_scorer
|
|
100
|
-
# v3.
|
|
101
|
-
#
|
|
102
|
-
#
|
|
103
|
-
#
|
|
104
|
-
#
|
|
105
|
-
self._scope_lock = threading.Lock()
|
|
106
|
+
# v3.7.9: scope flags (include_global / include_shared) are now threaded
|
|
107
|
+
# as explicit call parameters into every channel's search() method, so
|
|
108
|
+
# concurrent recalls each carry their own flags — no shared mutable state,
|
|
109
|
+
# no lock needed. The _scope_lock and per-recall attribute-set loop have
|
|
110
|
+
# been removed. See defect S01 in the fix/3.7.9 branch notes.
|
|
106
111
|
# One executor belongs to one retrieval engine. Creating/destroying six
|
|
107
112
|
# worker threads on every recall caused allocator/thread-stack RSS churn
|
|
108
113
|
# under sustained sessions. The scope lock already serializes channel
|
|
@@ -149,6 +154,7 @@ class RetrievalEngine:
|
|
|
149
154
|
extra_disabled_channels: set[str] | None = None,
|
|
150
155
|
include_global: bool = False,
|
|
151
156
|
include_shared: bool = False,
|
|
157
|
+
window: str | tuple[str, str] | None = None,
|
|
152
158
|
) -> RecallResponse:
|
|
153
159
|
"""Full retrieval pipeline: strategy -> channels -> RRF -> rerank.
|
|
154
160
|
|
|
@@ -161,7 +167,11 @@ class RetrievalEngine:
|
|
|
161
167
|
for the ``--fast`` CLI flag) without mutating shared config.
|
|
162
168
|
"""
|
|
163
169
|
t0 = time.monotonic()
|
|
164
|
-
|
|
170
|
+
# NOTE: extra_disabled_channels is passed as an explicit local argument
|
|
171
|
+
# to _run_channels() — it is NOT stored on self. Storing it as a shared
|
|
172
|
+
# mutable instance attribute (the old self._extra_disabled = ...) caused
|
|
173
|
+
# a race condition where two concurrent recalls could overwrite each
|
|
174
|
+
# other's channel-disable set (v3.4.64 fix).
|
|
165
175
|
|
|
166
176
|
# v3.5.0 diagnostic: stage timing inside retrieval (SLM_RECALL_TIMING=1).
|
|
167
177
|
import os as _os_e
|
|
@@ -195,18 +205,14 @@ class RetrievalEngine:
|
|
|
195
205
|
# Dynamic top-k for aggregation queries
|
|
196
206
|
effective_limit = 100 if strat.query_type == "aggregation" else limit
|
|
197
207
|
|
|
198
|
-
# 3. Run channels.
|
|
199
|
-
#
|
|
200
|
-
#
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
if ch is not None:
|
|
207
|
-
ch.include_global = include_global
|
|
208
|
-
ch.include_shared = include_shared
|
|
209
|
-
ch_results = self._run_channels(query, profile_id, strat)
|
|
208
|
+
# 3. Run channels. Both scope flags AND extra_disabled_channels travel as
|
|
209
|
+
# explicit call parameters so concurrent recalls with different flags
|
|
210
|
+
# cannot corrupt each other. No lock needed — no shared mutable state.
|
|
211
|
+
ch_results = self._run_channels(
|
|
212
|
+
query, profile_id, strat,
|
|
213
|
+
extra_disabled_channels=extra_disabled_channels,
|
|
214
|
+
include_global=include_global, include_shared=include_shared,
|
|
215
|
+
)
|
|
210
216
|
_em("run_channels")
|
|
211
217
|
if profile_hits:
|
|
212
218
|
ch_results["profile"] = profile_hits
|
|
@@ -318,6 +324,32 @@ class RetrievalEngine:
|
|
|
318
324
|
logger.warning("Entity graph signal enhancement: %s", exc)
|
|
319
325
|
|
|
320
326
|
_em("expand+entity_enh")
|
|
327
|
+
|
|
328
|
+
# T-window: prune candidates to the requested event-time range.
|
|
329
|
+
# Event times are fetched for the bounded candidate set only (indexed),
|
|
330
|
+
# then in-range facts are kept — before fact load, so out-of-window facts
|
|
331
|
+
# are never materialized. T3: when the caller passes no explicit window,
|
|
332
|
+
# infer one from natural-language scope in the query ("last week").
|
|
333
|
+
# Safety: an EXPLICIT window is authoritative (honoured even if it empties
|
|
334
|
+
# the set — the user asked for that scope), but an INFERRED window is
|
|
335
|
+
# additive and never makes recall worse — if it would empty the results,
|
|
336
|
+
# fall back to the unwindowed set.
|
|
337
|
+
_explicit_window = window is not None
|
|
338
|
+
_window = window if _explicit_window else infer_window_from_query(query)
|
|
339
|
+
if _window is not None and fused:
|
|
340
|
+
bounds = parse_window(_window)
|
|
341
|
+
if bounds is not None:
|
|
342
|
+
etimes = self._db.get_fact_event_times(
|
|
343
|
+
[fr.fact_id for fr in fused], profile_id,
|
|
344
|
+
)
|
|
345
|
+
windowed = [
|
|
346
|
+
fr for fr in fused
|
|
347
|
+
if in_window(etimes.get(fr.fact_id), bounds)
|
|
348
|
+
]
|
|
349
|
+
if windowed or _explicit_window:
|
|
350
|
+
fused = windowed
|
|
351
|
+
_em("time_window")
|
|
352
|
+
|
|
321
353
|
# 4. Load facts for rerank pool
|
|
322
354
|
pool = min(len(fused), max(effective_limit * 3, 30))
|
|
323
355
|
top = fused[:pool]
|
|
@@ -411,8 +443,85 @@ class RetrievalEngine:
|
|
|
411
443
|
no_confident_match=no_match,
|
|
412
444
|
reranker_applied=reranker_applied,
|
|
413
445
|
reranker_status=reranker_status,
|
|
446
|
+
# Q2b: thematic context when the top results cluster in one
|
|
447
|
+
# community. Precomputed summary lookup only — no per-query LLM.
|
|
448
|
+
community_context=self._community_context(results, profile_id),
|
|
414
449
|
)
|
|
415
450
|
|
|
451
|
+
# -- Community context (Wave Q2b) --------------------------------------
|
|
452
|
+
|
|
453
|
+
def _community_context(
|
|
454
|
+
self, results: list[Any], profile_id: str, top_k: int = 8,
|
|
455
|
+
) -> dict | None:
|
|
456
|
+
"""Attach the precomputed community summary the top results fall into.
|
|
457
|
+
|
|
458
|
+
On-device-safe (market CRIT-1): a single read of the ≤N precomputed
|
|
459
|
+
community_summaries rows + a membership tally — never a per-query LLM
|
|
460
|
+
fan-out. Gated: fires only when >=2 of the top results AND >=40% of
|
|
461
|
+
them belong to one community, so precise factual queries are untouched.
|
|
462
|
+
Fail-open: any error returns None (recall is never affected).
|
|
463
|
+
"""
|
|
464
|
+
if not results or not getattr(
|
|
465
|
+
self._config, "enable_community_context", True,
|
|
466
|
+
):
|
|
467
|
+
return None
|
|
468
|
+
try:
|
|
469
|
+
import json
|
|
470
|
+
from collections import Counter
|
|
471
|
+
|
|
472
|
+
rows = [
|
|
473
|
+
dict(r) for r in self._db.execute(
|
|
474
|
+
"SELECT community_id, summary, keywords, fact_ids_json, "
|
|
475
|
+
"fact_count FROM community_summaries WHERE profile_id = ?",
|
|
476
|
+
(profile_id,),
|
|
477
|
+
)
|
|
478
|
+
]
|
|
479
|
+
if not rows:
|
|
480
|
+
return None
|
|
481
|
+
|
|
482
|
+
fact_to_cid: dict[str, int] = {}
|
|
483
|
+
summ_by_cid: dict[int, dict] = {}
|
|
484
|
+
for r in rows:
|
|
485
|
+
cid = int(r["community_id"])
|
|
486
|
+
summ_by_cid[cid] = r
|
|
487
|
+
try:
|
|
488
|
+
for fid in json.loads(r.get("fact_ids_json") or "[]"):
|
|
489
|
+
fact_to_cid[str(fid)] = cid
|
|
490
|
+
except (ValueError, TypeError):
|
|
491
|
+
continue
|
|
492
|
+
|
|
493
|
+
top_ids = [
|
|
494
|
+
res.fact.fact_id
|
|
495
|
+
for res in results[:top_k]
|
|
496
|
+
if getattr(res, "fact", None) is not None
|
|
497
|
+
]
|
|
498
|
+
tally = Counter(
|
|
499
|
+
fact_to_cid[fid] for fid in top_ids if fid in fact_to_cid
|
|
500
|
+
)
|
|
501
|
+
if not tally:
|
|
502
|
+
return None
|
|
503
|
+
best_cid, count = tally.most_common(1)[0]
|
|
504
|
+
coverage = count / len(top_ids) if top_ids else 0.0
|
|
505
|
+
if count < 2 or coverage < 0.4:
|
|
506
|
+
return None
|
|
507
|
+
|
|
508
|
+
row = summ_by_cid[best_cid]
|
|
509
|
+
try:
|
|
510
|
+
members = json.loads(row.get("fact_ids_json") or "[]")
|
|
511
|
+
except (ValueError, TypeError):
|
|
512
|
+
members = []
|
|
513
|
+
return {
|
|
514
|
+
"community_id": best_cid,
|
|
515
|
+
"summary": row.get("summary", ""),
|
|
516
|
+
"keywords": row.get("keywords", ""),
|
|
517
|
+
"member_fact_ids": members,
|
|
518
|
+
"coverage": round(coverage, 3),
|
|
519
|
+
"matched_results": count,
|
|
520
|
+
}
|
|
521
|
+
except Exception as exc:
|
|
522
|
+
logger.debug("community context skipped (fail-open): %s", exc)
|
|
523
|
+
return None
|
|
524
|
+
|
|
416
525
|
# -- Evidence floor (v3.6.6) -------------------------------------------
|
|
417
526
|
|
|
418
527
|
@staticmethod
|
|
@@ -628,7 +737,14 @@ class RetrievalEngine:
|
|
|
628
737
|
return emb
|
|
629
738
|
|
|
630
739
|
def _run_channels(
|
|
631
|
-
self,
|
|
740
|
+
self,
|
|
741
|
+
query: str,
|
|
742
|
+
profile_id: str,
|
|
743
|
+
strat: QueryStrategy,
|
|
744
|
+
*,
|
|
745
|
+
extra_disabled_channels: set[str] | None = None,
|
|
746
|
+
include_global: bool = False,
|
|
747
|
+
include_shared: bool = False,
|
|
632
748
|
) -> dict[str, list[tuple[str, float]]]:
|
|
633
749
|
"""Run active retrieval channels.
|
|
634
750
|
|
|
@@ -646,7 +762,9 @@ class RetrievalEngine:
|
|
|
646
762
|
out: dict[str, list[tuple[str, float]]] = {}
|
|
647
763
|
# Skip channels listed in disabled_channels (ablation support)
|
|
648
764
|
# V3.4.40: union with per-recall extra_disabled set (e.g. --fast skip)
|
|
649
|
-
|
|
765
|
+
# V3.4.64: extra_disabled is now a local parameter, not a shared instance
|
|
766
|
+
# attribute — eliminates the concurrent-recall race condition.
|
|
767
|
+
disabled = set(self._config.disabled_channels) | set(extra_disabled_channels or ())
|
|
650
768
|
|
|
651
769
|
# V3.3.4: Embed query ONCE, reuse for semantic + hopfield channels
|
|
652
770
|
q_emb: list[float] | None = None
|
|
@@ -688,22 +806,38 @@ class RetrievalEngine:
|
|
|
688
806
|
if self._semantic is not None and q_emb is not None and "semantic" not in disabled:
|
|
689
807
|
futures["semantic"] = executor.submit(
|
|
690
808
|
_safe_channel, "semantic",
|
|
691
|
-
|
|
809
|
+
functools.partial(
|
|
810
|
+
self._semantic.search,
|
|
811
|
+
include_global=include_global, include_shared=include_shared,
|
|
812
|
+
),
|
|
813
|
+
q_emb, profile_id, self._config.semantic_top_k,
|
|
692
814
|
)
|
|
693
815
|
if self._bm25 is not None and "bm25" not in disabled:
|
|
694
816
|
futures["bm25"] = executor.submit(
|
|
695
817
|
_safe_channel, "bm25",
|
|
696
|
-
|
|
818
|
+
functools.partial(
|
|
819
|
+
self._bm25.search,
|
|
820
|
+
include_global=include_global, include_shared=include_shared,
|
|
821
|
+
),
|
|
822
|
+
query, profile_id, self._config.bm25_top_k,
|
|
697
823
|
)
|
|
698
824
|
if self._temporal is not None and "temporal" not in disabled:
|
|
699
825
|
futures["temporal"] = executor.submit(
|
|
700
826
|
_safe_channel, "temporal",
|
|
701
|
-
|
|
827
|
+
functools.partial(
|
|
828
|
+
self._temporal.search,
|
|
829
|
+
include_global=include_global, include_shared=include_shared,
|
|
830
|
+
),
|
|
831
|
+
query, profile_id, self._config.bm25_top_k,
|
|
702
832
|
)
|
|
703
833
|
if self._hopfield is not None and q_emb is not None and "hopfield" not in disabled:
|
|
704
834
|
futures["hopfield"] = executor.submit(
|
|
705
835
|
_safe_channel, "hopfield",
|
|
706
|
-
|
|
836
|
+
functools.partial(
|
|
837
|
+
self._hopfield.search,
|
|
838
|
+
include_global=include_global, include_shared=include_shared,
|
|
839
|
+
),
|
|
840
|
+
q_emb, profile_id, self._config.hopfield_top_k,
|
|
707
841
|
)
|
|
708
842
|
if (
|
|
709
843
|
self._spreading_activation is not None
|
|
@@ -712,7 +846,11 @@ class RetrievalEngine:
|
|
|
712
846
|
):
|
|
713
847
|
futures["spreading_activation"] = executor.submit(
|
|
714
848
|
_safe_channel, "spreading_activation",
|
|
715
|
-
|
|
849
|
+
functools.partial(
|
|
850
|
+
self._spreading_activation.search,
|
|
851
|
+
include_global=include_global, include_shared=include_shared,
|
|
852
|
+
),
|
|
853
|
+
q_emb, profile_id, self._config.bm25_top_k,
|
|
716
854
|
)
|
|
717
855
|
|
|
718
856
|
# Collect results as channels complete.
|
|
@@ -981,6 +1119,10 @@ class RetrievalEngine:
|
|
|
981
1119
|
|
|
982
1120
|
_CHANNEL_KEYS: tuple[str, ...] = (
|
|
983
1121
|
"semantic", "bm25", "entity_graph", "temporal",
|
|
1122
|
+
# hopfield + spreading_activation are real retrieval channels (score
|
|
1123
|
+
# contract v2) with bandit-chosen weights; omitting them here silently
|
|
1124
|
+
# discarded adaptive reranking for multi-hop relational recall.
|
|
1125
|
+
"spreading_activation", "hopfield",
|
|
984
1126
|
)
|
|
985
1127
|
|
|
986
1128
|
|
|
@@ -835,13 +835,15 @@ class EntityGraphChannel:
|
|
|
835
835
|
)[:top_k]
|
|
836
836
|
# Shadow SQLite before accepting a projected answer. The graph
|
|
837
837
|
# channel has optional PageRank/community enrichments, so exact
|
|
838
|
-
#
|
|
839
|
-
#
|
|
840
|
-
#
|
|
838
|
+
# Score equality is neither required nor useful; result *membership*
|
|
839
|
+
# is the correctness contract. Order within the same fact set is
|
|
840
|
+
# tolerated — requiring identical ordering would fail closed on
|
|
841
|
+
# every query with score ties, leaving Cozo permanently unused.
|
|
842
|
+
# Any membership divergence is recorded and fails closed to SQLite.
|
|
841
843
|
sqlite_results = self._search_without_cozo(query, profile_id, top_k)
|
|
842
|
-
matches =
|
|
844
|
+
matches = {fact_id for fact_id, _ in cozo_results} == {
|
|
843
845
|
fact_id for fact_id, _ in sqlite_results
|
|
844
|
-
|
|
846
|
+
}
|
|
845
847
|
record = getattr(self._cozo, "record_shadow_comparison", None)
|
|
846
848
|
if callable(record):
|
|
847
849
|
record(matches=matches, projected=cozo_results, canonical=sqlite_results)
|
|
@@ -88,6 +88,8 @@ class HopfieldChannel:
|
|
|
88
88
|
query: Any,
|
|
89
89
|
profile_id: str,
|
|
90
90
|
top_k: int = 50,
|
|
91
|
+
include_global: bool | None = None,
|
|
92
|
+
include_shared: bool | None = None,
|
|
91
93
|
) -> list[tuple[str, float]]:
|
|
92
94
|
"""Search for facts using Hopfield associative retrieval.
|
|
93
95
|
|
|
@@ -95,6 +97,9 @@ class HopfieldChannel:
|
|
|
95
97
|
query: Query embedding (list[float] or np.ndarray).
|
|
96
98
|
profile_id: Scope search to this profile.
|
|
97
99
|
top_k: Maximum results to return.
|
|
100
|
+
include_global: Include global-scope facts. Falls back to the
|
|
101
|
+
instance attribute when not supplied.
|
|
102
|
+
include_shared: Include shared-scope facts. Same fallback.
|
|
98
103
|
|
|
99
104
|
Returns:
|
|
100
105
|
List of (fact_id, score) sorted by score descending.
|
|
@@ -104,8 +109,10 @@ class HopfieldChannel:
|
|
|
104
109
|
if not self._config.enabled:
|
|
105
110
|
return []
|
|
106
111
|
|
|
107
|
-
include_global
|
|
108
|
-
|
|
112
|
+
if include_global is None:
|
|
113
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
114
|
+
if include_shared is None:
|
|
115
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
109
116
|
try:
|
|
110
117
|
with self._cache_lock:
|
|
111
118
|
return self._search_inner(
|
|
@@ -120,6 +120,8 @@ class SemanticChannel:
|
|
|
120
120
|
query_embedding: list[float],
|
|
121
121
|
profile_id: str,
|
|
122
122
|
top_k: int = 50,
|
|
123
|
+
include_global: bool | None = None,
|
|
124
|
+
include_shared: bool | None = None,
|
|
123
125
|
) -> list[tuple[str, float]]:
|
|
124
126
|
"""Search for semantically similar facts.
|
|
125
127
|
|
|
@@ -130,11 +132,23 @@ class SemanticChannel:
|
|
|
130
132
|
query_embedding: Dense vector for the query.
|
|
131
133
|
profile_id: Scope to this profile.
|
|
132
134
|
top_k: Maximum results to return.
|
|
135
|
+
include_global: Include global-scope facts. Defaults to the
|
|
136
|
+
``include_global`` instance attribute when not supplied,
|
|
137
|
+
preserving backward compatibility for callers that still
|
|
138
|
+
set the attribute directly.
|
|
139
|
+
include_shared: Include shared-scope facts. Same fallback.
|
|
133
140
|
|
|
134
141
|
Returns:
|
|
135
142
|
List of (fact_id, score) sorted by score descending.
|
|
136
143
|
Score is in [0, 1] range.
|
|
137
144
|
"""
|
|
145
|
+
# Resolve scope flags: explicit param takes priority; fall back to the
|
|
146
|
+
# legacy attribute-based path so existing callers keep working.
|
|
147
|
+
if include_global is None:
|
|
148
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
149
|
+
if include_shared is None:
|
|
150
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
151
|
+
|
|
138
152
|
if not query_embedding:
|
|
139
153
|
return []
|
|
140
154
|
|
|
@@ -142,16 +156,26 @@ class SemanticChannel:
|
|
|
142
156
|
|
|
143
157
|
# Lance is a derived projection. It is never an authorization source
|
|
144
158
|
# and it never silently replaces the canonical sqlite-vec path: every
|
|
145
|
-
# promoted query is shadowed and falls back if membership
|
|
159
|
+
# promoted query is shadowed and falls back if the result *membership*
|
|
160
|
+
# differs. Order differences within the same fact set are tolerated —
|
|
161
|
+
# float-score ties would otherwise force a fallback on every query,
|
|
162
|
+
# leaving the promoted backend permanently unused. Once membership
|
|
163
|
+
# matches, the projected backend's own ranking is authoritative.
|
|
146
164
|
if (
|
|
147
165
|
self._scale_vector_backend is not None
|
|
148
|
-
and not
|
|
149
|
-
and not
|
|
166
|
+
and not include_global
|
|
167
|
+
and not include_shared
|
|
150
168
|
):
|
|
151
|
-
projected = self._search_via_lance(
|
|
152
|
-
|
|
169
|
+
projected = self._search_via_lance(
|
|
170
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
171
|
+
include_global=include_global, include_shared=include_shared,
|
|
172
|
+
)
|
|
173
|
+
canonical = self._search_without_lance(
|
|
174
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
175
|
+
include_global=include_global, include_shared=include_shared,
|
|
176
|
+
)
|
|
153
177
|
self._scale_shadow_checks += 1
|
|
154
|
-
if
|
|
178
|
+
if {fid for fid, _ in projected} == {fid for fid, _ in canonical}:
|
|
155
179
|
return projected
|
|
156
180
|
self._scale_shadow_mismatches += 1
|
|
157
181
|
logger.warning("Lance semantic projection diverged from SQLite; using SQLite")
|
|
@@ -161,13 +185,17 @@ class SemanticChannel:
|
|
|
161
185
|
if self._vector_store and self._vector_store.available:
|
|
162
186
|
results = self._search_via_vector_store(
|
|
163
187
|
query_embedding, q_vec, profile_id, top_k,
|
|
188
|
+
include_global=include_global, include_shared=include_shared,
|
|
164
189
|
)
|
|
165
190
|
if results: # If vec0 returned results, use them
|
|
166
191
|
return results
|
|
167
192
|
# If vec0 is empty (cold start), fall through to full scan
|
|
168
193
|
|
|
169
194
|
# --- FALLBACK: full-table scan (original code, unchanged) ---
|
|
170
|
-
return self._search_full_scan(
|
|
195
|
+
return self._search_full_scan(
|
|
196
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
197
|
+
include_global=include_global, include_shared=include_shared,
|
|
198
|
+
)
|
|
171
199
|
|
|
172
200
|
def set_scale_vector_backend(self, backend: Any | None) -> None:
|
|
173
201
|
"""Attach a parity-verified Lance projection without replacing SQLite."""
|
|
@@ -180,14 +208,27 @@ class SemanticChannel:
|
|
|
180
208
|
}
|
|
181
209
|
|
|
182
210
|
def _search_via_lance(
|
|
183
|
-
self,
|
|
211
|
+
self,
|
|
212
|
+
query_embedding: list[float],
|
|
213
|
+
q_vec: np.ndarray,
|
|
214
|
+
profile_id: str,
|
|
215
|
+
top_k: int,
|
|
216
|
+
include_global: bool | None = None,
|
|
217
|
+
include_shared: bool | None = None,
|
|
184
218
|
) -> list[tuple[str, float]]:
|
|
219
|
+
if include_global is None:
|
|
220
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
221
|
+
if include_shared is None:
|
|
222
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
185
223
|
original_store, original_qas = self._vector_store, self._qas
|
|
186
224
|
try:
|
|
187
225
|
self._vector_store = _LanceCandidateSource(self._scale_vector_backend)
|
|
188
226
|
# QAS indexes SQLite/quantized records and cannot represent Lance.
|
|
189
227
|
self._qas = None
|
|
190
|
-
return self._search_via_vector_store(
|
|
228
|
+
return self._search_via_vector_store(
|
|
229
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
230
|
+
include_global=include_global, include_shared=include_shared,
|
|
231
|
+
)
|
|
191
232
|
except Exception as exc:
|
|
192
233
|
logger.warning("Lance semantic projection failed closed to SQLite: %s", exc)
|
|
193
234
|
return []
|
|
@@ -195,15 +236,31 @@ class SemanticChannel:
|
|
|
195
236
|
self._vector_store, self._qas = original_store, original_qas
|
|
196
237
|
|
|
197
238
|
def _search_without_lance(
|
|
198
|
-
self,
|
|
239
|
+
self,
|
|
240
|
+
query_embedding: list[float],
|
|
241
|
+
q_vec: np.ndarray,
|
|
242
|
+
profile_id: str,
|
|
243
|
+
top_k: int,
|
|
244
|
+
include_global: bool | None = None,
|
|
245
|
+
include_shared: bool | None = None,
|
|
199
246
|
) -> list[tuple[str, float]]:
|
|
247
|
+
if include_global is None:
|
|
248
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
249
|
+
if include_shared is None:
|
|
250
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
200
251
|
backend, self._scale_vector_backend = self._scale_vector_backend, None
|
|
201
252
|
try:
|
|
202
253
|
if self._vector_store and self._vector_store.available:
|
|
203
|
-
results = self._search_via_vector_store(
|
|
254
|
+
results = self._search_via_vector_store(
|
|
255
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
256
|
+
include_global=include_global, include_shared=include_shared,
|
|
257
|
+
)
|
|
204
258
|
if results:
|
|
205
259
|
return results
|
|
206
|
-
return self._search_full_scan(
|
|
260
|
+
return self._search_full_scan(
|
|
261
|
+
query_embedding, q_vec, profile_id, top_k,
|
|
262
|
+
include_global=include_global, include_shared=include_shared,
|
|
263
|
+
)
|
|
207
264
|
finally:
|
|
208
265
|
self._scale_vector_backend = backend
|
|
209
266
|
|
|
@@ -213,8 +270,14 @@ class SemanticChannel:
|
|
|
213
270
|
q_vec: np.ndarray,
|
|
214
271
|
profile_id: str,
|
|
215
272
|
top_k: int,
|
|
273
|
+
include_global: bool | None = None,
|
|
274
|
+
include_shared: bool | None = None,
|
|
216
275
|
) -> list[tuple[str, float]]:
|
|
217
276
|
"""KNN via VectorStore (or QAS 3-tier), then Fisher-Rao re-scoring."""
|
|
277
|
+
if include_global is None:
|
|
278
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
279
|
+
if include_shared is None:
|
|
280
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
218
281
|
# V3.3.19: Try TurboQuant 3-tier search first (float32 + int8 + polar)
|
|
219
282
|
if self._qas is not None:
|
|
220
283
|
try:
|
|
@@ -235,12 +298,19 @@ class SemanticChannel:
|
|
|
235
298
|
query_embedding, top_k=top_k * 2, profile_id=profile_id,
|
|
236
299
|
)
|
|
237
300
|
|
|
301
|
+
# M-01: Normalize KNN scores to [0.5, 1.0] via (score + 1.0) / 2.0 so
|
|
302
|
+
# they are on the same scale as full-scan scores computed from
|
|
303
|
+
# (_cosine_similarity(q, f) + 1.0) / 2.0 in the external_scores path.
|
|
304
|
+
# vector_store.search() clips at 0 (max(0, cosine)), which maps [–1,1]
|
|
305
|
+
# to [0,1]; the canonical formula maps to [0.5, 1.0] for positives.
|
|
306
|
+
# Both are [0,1] but different scales — without this normalization KNN
|
|
307
|
+
# scores are systematically lower and external facts always win max().
|
|
308
|
+
knn_results = [(fid, (score + 1.0) / 2.0) for fid, score in knn_results]
|
|
309
|
+
|
|
238
310
|
# The vector index is partitioned by owner profile. An opted-in global
|
|
239
311
|
# or authorized shared fact owned by another profile cannot enter the
|
|
240
312
|
# local KNN candidate set, so merge the bounded cross-profile visible
|
|
241
313
|
# supplement using the same canonical DB scope predicate as fallback.
|
|
242
|
-
include_global = bool(getattr(self, "include_global", False))
|
|
243
|
-
include_shared = bool(getattr(self, "include_shared", False))
|
|
244
314
|
external_facts = self._db.get_external_visible_facts(
|
|
245
315
|
profile_id,
|
|
246
316
|
include_global=include_global,
|
|
@@ -272,8 +342,8 @@ class SemanticChannel:
|
|
|
272
342
|
knn_scores = {fid: score for fid, score in knn_results}
|
|
273
343
|
facts = self._db.get_facts_by_ids(
|
|
274
344
|
candidate_ids, profile_id,
|
|
275
|
-
include_global=
|
|
276
|
-
include_shared=
|
|
345
|
+
include_global=include_global,
|
|
346
|
+
include_shared=include_shared,
|
|
277
347
|
)
|
|
278
348
|
|
|
279
349
|
if not facts:
|
|
@@ -294,15 +364,32 @@ class SemanticChannel:
|
|
|
294
364
|
|
|
295
365
|
scored: list[tuple[str, float]] = []
|
|
296
366
|
for fact in facts:
|
|
297
|
-
|
|
367
|
+
# C2-ret H-01: recompute the final cosine from the canonical
|
|
368
|
+
# full-precision embedding using the SAME formula as the full-scan
|
|
369
|
+
# fallback, so the fast path is observationally equivalent — identical
|
|
370
|
+
# score MAGNITUDES, not merely identical rankings. The KNN/vector-store
|
|
371
|
+
# score is a candidate-SELECTION signal only (it decides which facts
|
|
372
|
+
# are Fisher-rescored), never the final magnitude; trusting it here
|
|
373
|
+
# leaked the vector store's negative-cosine clamp into public scores.
|
|
374
|
+
# When a candidate carries no usable embedding (index-only rows), fall
|
|
375
|
+
# back to the normalized KNN score, preserving the M-01 contract.
|
|
376
|
+
f_vec: np.ndarray | None = None
|
|
377
|
+
if fact.embedding is not None:
|
|
378
|
+
candidate = np.array(fact.embedding, dtype=np.float32)
|
|
379
|
+
if candidate.shape == q_vec.shape:
|
|
380
|
+
f_vec = candidate
|
|
381
|
+
|
|
382
|
+
if f_vec is not None:
|
|
383
|
+
cos_sim = (_cosine_similarity(q_vec, f_vec) + 1.0) / 2.0
|
|
384
|
+
else:
|
|
385
|
+
cos_sim = knn_scores.get(fact.fact_id, 0.0)
|
|
298
386
|
|
|
299
387
|
fisher_weight = self._fisher_weight(fact.access_count)
|
|
300
388
|
|
|
301
389
|
if (fisher_weight > 0.01
|
|
302
390
|
and fact.fisher_variance is not None
|
|
303
|
-
and
|
|
391
|
+
and f_vec is not None
|
|
304
392
|
and len(fact.fisher_variance) == len(q_vec)):
|
|
305
|
-
f_vec = np.array(fact.embedding, dtype=np.float32)
|
|
306
393
|
var_vec = np.array(fact.fisher_variance, dtype=np.float32)
|
|
307
394
|
f_sim = self._compute_fisher_sim(
|
|
308
395
|
q_vec, f_vec, var_vec, fact, q_mean, q_var,
|
|
@@ -323,10 +410,16 @@ class SemanticChannel:
|
|
|
323
410
|
q_vec: np.ndarray,
|
|
324
411
|
profile_id: str,
|
|
325
412
|
top_k: int,
|
|
413
|
+
include_global: bool | None = None,
|
|
414
|
+
include_shared: bool | None = None,
|
|
326
415
|
) -> list[tuple[str, float]]:
|
|
327
416
|
"""Original full-table-scan search. Used as fallback when VectorStore
|
|
328
417
|
is unavailable or empty (cold start).
|
|
329
418
|
"""
|
|
419
|
+
if include_global is None:
|
|
420
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
421
|
+
if include_shared is None:
|
|
422
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
330
423
|
# Compute query Fisher params for Bayesian comparison (F45 fix)
|
|
331
424
|
q_mean: np.ndarray | None = None
|
|
332
425
|
q_var: np.ndarray | None = None
|
|
@@ -337,8 +430,8 @@ class SemanticChannel:
|
|
|
337
430
|
|
|
338
431
|
facts = self._db.get_all_facts(
|
|
339
432
|
profile_id,
|
|
340
|
-
include_global=
|
|
341
|
-
include_shared=
|
|
433
|
+
include_global=include_global,
|
|
434
|
+
include_shared=include_shared,
|
|
342
435
|
)
|
|
343
436
|
|
|
344
437
|
scored: list[tuple[str, float]] = []
|
|
@@ -110,16 +110,25 @@ class SpreadingActivation:
|
|
|
110
110
|
query: Any,
|
|
111
111
|
profile_id: str = "",
|
|
112
112
|
top_k: int = 7,
|
|
113
|
+
include_global: bool | None = None,
|
|
114
|
+
include_shared: bool | None = None,
|
|
113
115
|
) -> list[tuple[str, float]]:
|
|
114
116
|
"""Channel-compatible interface: (query, top_k) -> [(fact_id, score)].
|
|
115
117
|
|
|
116
118
|
Matches ANNSearchable protocol (Rule 07).
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
include_global: Include global-scope facts. Falls back to the
|
|
122
|
+
instance attribute when not supplied.
|
|
123
|
+
include_shared: Include shared-scope facts. Same fallback.
|
|
117
124
|
"""
|
|
118
125
|
if not self._config.enabled:
|
|
119
126
|
return []
|
|
120
127
|
|
|
121
|
-
include_global
|
|
122
|
-
|
|
128
|
+
if include_global is None:
|
|
129
|
+
include_global = bool(getattr(self, "include_global", False))
|
|
130
|
+
if include_shared is None:
|
|
131
|
+
include_shared = bool(getattr(self, "include_shared", False))
|
|
123
132
|
try:
|
|
124
133
|
# Step 0: Get seed nodes from VectorStore KNN
|
|
125
134
|
seed_results = self._seed_search(
|