superlocalmemory 3.8.13 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ATTRIBUTION.md +4 -4
- package/CHANGELOG.md +113 -121
- package/README.md +65 -63
- package/docs/pi-dev-integration.md +1 -1
- package/package.json +6 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-governance/SKILL.md +248 -0
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-loop/SKILL.md +99 -0
- package/plugin-src/skills/slm-mesh/SKILL.md +282 -0
- package/plugin-src/skills/slm-profile/SKILL.md +148 -0
- package/plugin-src/skills/slm-recall/SKILL.md +1 -1
- package/plugin-src/skills/slm-remember/SKILL.md +1 -1
- package/plugin-src/skills/slm-scope/SKILL.md +176 -0
- package/plugin-src/skills/slm-session/SKILL.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +11 -4
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +125 -11
- package/src/superlocalmemory/cli/daemon.py +5 -1
- package/src/superlocalmemory/cli/main.py +35 -2
- package/src/superlocalmemory/cli/ops_cmd.py +281 -0
- package/src/superlocalmemory/cli/setup_wizard.py +1 -1
- package/src/superlocalmemory/compliance/audit.py +65 -0
- package/src/superlocalmemory/compliance/eu_ai_act.py +27 -57
- package/src/superlocalmemory/compliance/gdpr.py +416 -20
- package/src/superlocalmemory/compliance/retention.py +74 -22
- package/src/superlocalmemory/compliance/scheduler.py +78 -9
- package/src/superlocalmemory/core/actor_context.py +166 -0
- package/src/superlocalmemory/core/admission.py +549 -0
- package/src/superlocalmemory/core/backend_orchestrator.py +23 -10
- package/src/superlocalmemory/core/config.py +202 -24
- package/src/superlocalmemory/core/consolidation_engine.py +13 -13
- package/src/superlocalmemory/core/context_cache.py +28 -0
- package/src/superlocalmemory/core/embeddings.py +64 -2
- package/src/superlocalmemory/core/engine.py +7 -2
- package/src/superlocalmemory/core/engine_ingestion.py +65 -3
- package/src/superlocalmemory/core/engine_wiring.py +36 -9
- package/src/superlocalmemory/core/ingest_policy.py +38 -0
- package/src/superlocalmemory/core/maintenance.py +255 -0
- package/src/superlocalmemory/core/modes.py +40 -13
- package/src/superlocalmemory/core/mutations.py +437 -44
- package/src/superlocalmemory/core/operation_policy.py +92 -0
- package/src/superlocalmemory/core/operation_policy_registry.py +542 -0
- package/src/superlocalmemory/core/operation_request.py +127 -0
- package/src/superlocalmemory/core/ops_remediation.py +542 -0
- package/src/superlocalmemory/core/recall_pipeline.py +7 -0
- package/src/superlocalmemory/core/remember_runtime.py +202 -4
- package/src/superlocalmemory/core/remote_mode.py +20 -5
- package/src/superlocalmemory/core/store_pipeline.py +150 -0
- package/src/superlocalmemory/core/topic_signature.py +19 -4
- package/src/superlocalmemory/core/transactions/__init__.py +78 -0
- package/src/superlocalmemory/core/transactions/concrete_owners.py +597 -0
- package/src/superlocalmemory/core/transactions/erasure.py +825 -0
- package/src/superlocalmemory/core/transactions/manifest.py +255 -0
- package/src/superlocalmemory/core/transactions/manifest_key.py +155 -0
- package/src/superlocalmemory/core/transactions/obligations.py +272 -0
- package/src/superlocalmemory/core/transactions/owners.py +114 -0
- package/src/superlocalmemory/core/transactions/reconciler.py +285 -0
- package/src/superlocalmemory/core/transactions/service.py +330 -0
- package/src/superlocalmemory/core/worker_pool.py +33 -5
- package/src/superlocalmemory/encoding/cognitive_consolidator.py +70 -28
- package/src/superlocalmemory/encoding/emotional.py +75 -14
- package/src/superlocalmemory/encoding/scene_builder.py +115 -13
- package/src/superlocalmemory/encoding/temporal_parser.py +4 -0
- package/src/superlocalmemory/evolution/blind_verifier.py +11 -4
- package/src/superlocalmemory/evolution/evolution_store.py +244 -4
- package/src/superlocalmemory/evolution/llm_dispatch.py +40 -0
- package/src/superlocalmemory/evolution/model_selection.py +18 -3
- package/src/superlocalmemory/evolution/mutation_generator.py +3 -0
- package/src/superlocalmemory/evolution/skill_activator.py +270 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +281 -59
- package/src/superlocalmemory/evolution/types.py +30 -8
- package/src/superlocalmemory/graph/cozo_backend.py +17 -9
- package/src/superlocalmemory/hooks/auto_invoker.py +2 -1
- package/src/superlocalmemory/hooks/auto_recall.py +64 -30
- package/src/superlocalmemory/hooks/codex_assets.py +14 -1
- package/src/superlocalmemory/infra/backup.py +434 -7
- package/src/superlocalmemory/infra/process_reaper.py +18 -0
- package/src/superlocalmemory/infra/self_heal.py +401 -0
- package/src/superlocalmemory/learning/feedback.py +52 -9
- package/src/superlocalmemory/loops/engine.py +10 -0
- package/src/superlocalmemory/mcp/_daemon_proxy.py +3 -0
- package/src/superlocalmemory/mcp/http_transport.py +30 -331
- package/src/superlocalmemory/mcp/profiles.py +5 -0
- package/src/superlocalmemory/mcp/resources.py +8 -0
- package/src/superlocalmemory/mcp/server.py +51 -4
- package/src/superlocalmemory/mcp/shared.py +19 -0
- package/src/superlocalmemory/mcp/tools_active.py +25 -4
- package/src/superlocalmemory/mcp/tools_code_graph.py +26 -18
- package/src/superlocalmemory/mcp/tools_context.py +50 -8
- package/src/superlocalmemory/mcp/tools_core.py +69 -21
- package/src/superlocalmemory/mcp/tools_evolution.py +9 -2
- package/src/superlocalmemory/mcp/tools_learning.py +21 -10
- package/src/superlocalmemory/mcp/tools_loops.py +29 -18
- package/src/superlocalmemory/mcp/tools_mesh.py +8 -0
- package/src/superlocalmemory/mcp/tools_ops.py +115 -0
- package/src/superlocalmemory/mcp/tools_optimize.py +4 -0
- package/src/superlocalmemory/mcp/tools_v28.py +10 -3
- package/src/superlocalmemory/mcp/tools_v3.py +34 -14
- package/src/superlocalmemory/mcp/tools_v33.py +18 -33
- package/src/superlocalmemory/mesh/broker.py +124 -46
- package/src/superlocalmemory/mesh/broker_security.py +470 -0
- package/src/superlocalmemory/mesh/discovery.py +365 -0
- package/src/superlocalmemory/mesh/lock_protocol.py +313 -0
- package/src/superlocalmemory/mesh/node_identity.py +97 -0
- package/src/superlocalmemory/mesh/outbox_remote.py +429 -0
- package/src/superlocalmemory/mesh/remote_sync.py +511 -28
- package/src/superlocalmemory/mesh/state_sync.py +286 -0
- package/src/superlocalmemory/optimize/config/store.py +45 -0
- package/src/superlocalmemory/parameterization/cross_project.py +12 -0
- package/src/superlocalmemory/parameterization/prompt_injector.py +13 -11
- package/src/superlocalmemory/parameterization/prompt_lifecycle.py +8 -2
- package/src/superlocalmemory/parameterization/workflow_miner.py +17 -0
- package/src/superlocalmemory/retrieval/ann_index.py +5 -0
- package/src/superlocalmemory/retrieval/bm25_channel.py +49 -2
- package/src/superlocalmemory/retrieval/engine.py +19 -4
- package/src/superlocalmemory/retrieval/fusion.py +4 -1
- package/src/superlocalmemory/retrieval/hopfield_channel.py +9 -3
- package/src/superlocalmemory/retrieval/remote_reranker.py +47 -22
- package/src/superlocalmemory/retrieval/reranker.py +32 -1
- package/src/superlocalmemory/retrieval/temporal_channel.py +16 -3
- package/src/superlocalmemory/retrieval/temporal_utils.py +107 -0
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +155 -42
- package/src/superlocalmemory/retrieval/vector_store.py +214 -8
- package/src/superlocalmemory/server/api.py +5 -5
- package/src/superlocalmemory/server/egress_policy.py +258 -0
- package/src/superlocalmemory/server/rbac_enforce.py +32 -0
- package/src/superlocalmemory/server/route_mutations.py +20 -0
- package/src/superlocalmemory/server/routes/compliance.py +153 -7
- package/src/superlocalmemory/server/routes/data_io.py +43 -2
- package/src/superlocalmemory/server/routes/events.py +15 -0
- package/src/superlocalmemory/server/routes/memories.py +56 -3
- package/src/superlocalmemory/server/routes/mesh.py +82 -1
- package/src/superlocalmemory/server/routes/mesh_lock.py +54 -0
- package/src/superlocalmemory/server/routes/mesh_state.py +63 -0
- package/src/superlocalmemory/server/routes/v3_api.py +50 -27
- package/src/superlocalmemory/server/routes/ws.py +86 -0
- package/src/superlocalmemory/server/ui.py +6 -6
- package/src/superlocalmemory/server/unified_daemon.py +942 -119
- package/src/superlocalmemory/storage/_migration_internals.py +568 -0
- package/src/superlocalmemory/storage/_schema_version.py +110 -0
- package/src/superlocalmemory/storage/database.py +329 -24
- package/src/superlocalmemory/storage/embedding_migrator.py +246 -51
- package/src/superlocalmemory/storage/erasure_fence.py +45 -0
- package/src/superlocalmemory/storage/generation_fence.py +63 -0
- package/src/superlocalmemory/storage/migration_runner.py +140 -417
- package/src/superlocalmemory/storage/migrations/M009_model_lineage.py +40 -0
- package/src/superlocalmemory/storage/migrations/M033_projection_transactions.py +148 -0
- package/src/superlocalmemory/storage/migrations/M034_obligation_integrity.py +58 -0
- package/src/superlocalmemory/storage/migrations/M035_erasure_receipts.py +113 -0
- package/src/superlocalmemory/storage/migrations/M036_vector_row_map.py +107 -0
- package/src/superlocalmemory/storage/migrations/M037_manifest_hmac_version.py +162 -0
- package/src/superlocalmemory/storage/migrations/{M033_learning_feedback_channel.py → M038_learning_feedback_channel.py} +3 -3
- package/src/superlocalmemory/storage/migrations/M039_scene_fact_members.py +137 -0
- package/src/superlocalmemory/storage/migrations/__init__.py +4 -2
- package/src/superlocalmemory/storage/schema.py +67 -0
- package/src/superlocalmemory/storage/write_coordinator.py +125 -0
- package/src/superlocalmemory/trust/scorer.py +28 -4
- package/src/superlocalmemory/ui/index.html +14 -3
- package/src/superlocalmemory/ui/js/auto-settings.js +12 -1
- package/src/superlocalmemory/ui/js/brain.js +6 -4
- package/src/superlocalmemory/ui/js/compliance.js +66 -12
- package/src/superlocalmemory/ui/js/dashboard.js +13 -3
- package/src/superlocalmemory/ui/js/feedback.js +8 -2
- package/src/superlocalmemory/ui/js/lifecycle.js +7 -1
- package/src/superlocalmemory/ui/js/modal.js +272 -5
- package/src/superlocalmemory/ui/js/od-backup.js +9 -2
- package/src/superlocalmemory/ui/js/od-compliance-ext.js +301 -0
- package/src/superlocalmemory/ui/js/od-operations.js +154 -23
- package/src/superlocalmemory/ui/js/od-ops-health.js +417 -0
- package/src/superlocalmemory/ui/js/od-optimize.js +35 -21
- package/src/superlocalmemory/ui/js/od-team.js +9 -2
- package/src/superlocalmemory/ui/js/optimize.js +13 -16
- package/src/superlocalmemory/ui/js/profiles.js +7 -3
- package/src/superlocalmemory/ui/js/settings.js +7 -1
- package/src/superlocalmemory/vector/lancedb_backend.py +19 -9
- package/src/superlocalmemory/attribution/mathematical_dna.py +0 -235
- package/src/superlocalmemory/cli/post_install.py +0 -114
- package/src/superlocalmemory/core/clock_monitor.py +0 -45
- package/src/superlocalmemory/core/db_pool.py +0 -80
- package/src/superlocalmemory/core/error_catalog.py +0 -113
- package/src/superlocalmemory/core/loop_watchdog.py +0 -56
- package/src/superlocalmemory/core/priority_queue.py +0 -61
- package/src/superlocalmemory/core/pruning_engine.py +0 -216
- package/src/superlocalmemory/core/queue_dispatcher.py +0 -73
- package/src/superlocalmemory/core/slmignore.py +0 -125
- package/src/superlocalmemory/infra/heartbeat_monitor.py +0 -140
- package/src/superlocalmemory/infra/webhook_dispatcher.py +0 -247
- package/src/superlocalmemory/learning/quantization_scheduler.py +0 -320
- package/src/superlocalmemory/storage/access_control.py +0 -182
|
@@ -2,30 +2,55 @@
|
|
|
2
2
|
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
3
|
# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
|
|
4
4
|
|
|
5
|
-
"""Bi-temporal validity filter for the retrieval pipeline (Phase 4, T1).
|
|
6
|
-
|
|
7
|
-
Post-retrieval filter
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
P5-INT-01
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
5
|
+
"""Bi-temporal validity filter for the retrieval pipeline (Phase 4, T1 + T1b).
|
|
6
|
+
|
|
7
|
+
Post-retrieval filter that demotes facts whose bi-temporal validity window
|
|
8
|
+
makes them ineligible for the current recall point in time. Two independent
|
|
9
|
+
demotion axes:
|
|
10
|
+
|
|
11
|
+
**Axis 1 — Transaction-time supersession (P5-INT-01, existing):**
|
|
12
|
+
A fact whose temporal record has ``system_expired_at`` set was superseded or
|
|
13
|
+
contradicted by a newer fact (see ``invalidate_fact_temporal`` /
|
|
14
|
+
conflict-resolution supersession). Its per-channel score is multiplied by
|
|
15
|
+
``superseded_demotion_factor`` (default 0.25).
|
|
16
|
+
|
|
17
|
+
**Axis 2 — Event-time expiry (Phase 4 T1b, new):**
|
|
18
|
+
A fact whose ``valid_until`` has passed (in wall-clock time) was true in the
|
|
19
|
+
real world only up to that date. Its per-channel score is multiplied by
|
|
20
|
+
``event_time_demotion_factor`` (default 0.5, softer than supersession). A
|
|
21
|
+
separate "not-yet-valid" demotion applies when ``as_of`` is set for
|
|
22
|
+
point-in-time time-travel recall: facts whose ``valid_from > as_of`` are
|
|
23
|
+
also demoted.
|
|
24
|
+
|
|
25
|
+
**Zero-regression guarantee (default path):**
|
|
26
|
+
Almost all existing facts have ``valid_until = NULL`` (open-ended / still
|
|
27
|
+
valid). The event-time expiry lookup therefore returns an empty set on the
|
|
28
|
+
default path, so no scores change and default recall output is identical.
|
|
29
|
+
Additionally, when ``include_expired_in_history = True`` (the config default)
|
|
30
|
+
AND no explicit ``as_of`` is requested, event-time demotion is skipped
|
|
31
|
+
entirely — the guard ensures historical recall modes that intentionally want
|
|
32
|
+
to surface expired facts are unaffected.
|
|
33
|
+
|
|
34
|
+
**As-of time-travel:**
|
|
35
|
+
When ``as_of`` is packed into the filter context dict (``{"as_of": "..."}``),
|
|
36
|
+
event-time demotion is always applied regardless of
|
|
37
|
+
``include_expired_in_history``, because the caller explicitly requested a
|
|
38
|
+
point-in-time view. Facts not yet valid at ``as_of`` (``valid_from > as_of``)
|
|
39
|
+
and facts already expired at ``as_of`` (``valid_until <= as_of``, half-open) are
|
|
40
|
+
both demoted.
|
|
41
|
+
|
|
42
|
+
**Demotion priority:**
|
|
43
|
+
- System-invalidated facts: score × ``superseded_demotion_factor`` (0.25).
|
|
44
|
+
- Event-time-expired only (not system-invalidated): score × ``event_time_demotion_factor`` (0.5).
|
|
45
|
+
- Both: system-invalidated wins (the harder demotion, 0.25, is applied; event-time
|
|
46
|
+
demotion is not stacked to avoid excessive double-penalty).
|
|
47
|
+
|
|
48
|
+
All demotions are non-destructive (P5-INT-01): facts stay in the candidate
|
|
49
|
+
list but rank below valid facts. A factor of 0.0 restores the legacy hide
|
|
50
|
+
behaviour (a score of zero is gated out by the evidence floor).
|
|
51
|
+
|
|
52
|
+
Both lookups are bounded (candidate ids only), chunked, indexed, and
|
|
53
|
+
fail-open: a DB error returns results unchanged.
|
|
29
54
|
|
|
30
55
|
Integrates with ChannelRegistry.register_filter() using the FilterFn signature:
|
|
31
56
|
(all_channel_results, profile_id, context) -> filtered_results
|
|
@@ -46,17 +71,43 @@ if TYPE_CHECKING:
|
|
|
46
71
|
|
|
47
72
|
logger = logging.getLogger(__name__)
|
|
48
73
|
|
|
74
|
+
# Module-level fallback for event-time demotion factor.
|
|
75
|
+
# Used when the config object does not yet carry ``event_time_demotion_factor``
|
|
76
|
+
# (e.g. older serialised configs loaded from disk). Softer than the supersession
|
|
77
|
+
# factor (0.25) because event-time expiry is a softer signal — the boundary may
|
|
78
|
+
# be approximate or the fact may still be conceptually relevant.
|
|
79
|
+
_EVENT_TIME_DEMOTION_FACTOR: float = 0.5
|
|
80
|
+
|
|
49
81
|
|
|
50
82
|
class TemporalValidityFilter:
|
|
51
|
-
"""Demotes
|
|
83
|
+
"""Demotes bi-temporally invalid facts in retrieval candidates.
|
|
84
|
+
|
|
85
|
+
Two demotion axes:
|
|
86
|
+
- Axis 1: system-invalidated (superseded) facts — existing behaviour.
|
|
87
|
+
- Axis 2: event-time-expired facts — new in Phase 4 T1b.
|
|
88
|
+
|
|
89
|
+
Both are non-destructive: facts stay recallable but rank below valid ones.
|
|
90
|
+
"""
|
|
52
91
|
|
|
53
|
-
__slots__ = (
|
|
92
|
+
__slots__ = (
|
|
93
|
+
"_db",
|
|
94
|
+
"_demotion_factor",
|
|
95
|
+
"_event_time_factor",
|
|
96
|
+
"_include_expired_in_history",
|
|
97
|
+
)
|
|
54
98
|
|
|
55
|
-
def __init__(
|
|
99
|
+
def __init__(
|
|
100
|
+
self,
|
|
101
|
+
db: DatabaseManager,
|
|
102
|
+
demotion_factor: float = 0.25,
|
|
103
|
+
event_time_factor: float = _EVENT_TIME_DEMOTION_FACTOR,
|
|
104
|
+
include_expired_in_history: bool = True,
|
|
105
|
+
) -> None:
|
|
56
106
|
self._db = db
|
|
57
|
-
# Clamp to [0, 1]. 0.0 = legacy hide
|
|
58
|
-
# facts); 1.0 = no demotion.
|
|
107
|
+
# Clamp to [0, 1]. 0.0 = legacy hide; 1.0 = no demotion.
|
|
59
108
|
self._demotion_factor = max(0.0, min(1.0, float(demotion_factor)))
|
|
109
|
+
self._event_time_factor = max(0.0, min(1.0, float(event_time_factor)))
|
|
110
|
+
self._include_expired_in_history = bool(include_expired_in_history)
|
|
60
111
|
|
|
61
112
|
def filter(
|
|
62
113
|
self,
|
|
@@ -64,20 +115,23 @@ class TemporalValidityFilter:
|
|
|
64
115
|
profile_id: str,
|
|
65
116
|
context: Any,
|
|
66
117
|
) -> dict[str, list[tuple[str, float]]]:
|
|
67
|
-
"""Demote
|
|
118
|
+
"""Demote bi-temporally invalid fact_ids in every channel's candidate list.
|
|
68
119
|
|
|
69
120
|
Matches FilterFn signature from channel_registry.py.
|
|
70
121
|
|
|
71
122
|
Args:
|
|
72
123
|
all_results: Channel name -> [(fact_id, score)] dict.
|
|
73
124
|
profile_id: Current profile.
|
|
74
|
-
context: Optional
|
|
125
|
+
context: Optional dict that may carry ``as_of`` (ISO 8601 string)
|
|
126
|
+
for point-in-time time-travel recall. None or any non-dict value
|
|
127
|
+
means "current time, no time-travel" — the default path.
|
|
75
128
|
|
|
76
129
|
Returns:
|
|
77
|
-
A new dict where
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
130
|
+
A new dict where invalid facts keep their channel presence but have
|
|
131
|
+
their score scaled by the appropriate demotion factor and the channel
|
|
132
|
+
lists re-sorted so valid facts rank above them. Inputs are never
|
|
133
|
+
mutated (immutability contract). Returns the input unchanged when
|
|
134
|
+
nothing is invalid (fast path).
|
|
81
135
|
"""
|
|
82
136
|
# Collect all unique candidate fact_ids across every channel.
|
|
83
137
|
all_fact_ids: set[str] = set()
|
|
@@ -88,27 +142,73 @@ class TemporalValidityFilter:
|
|
|
88
142
|
if not all_fact_ids:
|
|
89
143
|
return all_results
|
|
90
144
|
|
|
145
|
+
# Extract as_of FIRST — needed for both Axis 1 and Axis 2. Must happen
|
|
146
|
+
# before any DB call so as_of is available for transaction-time gating.
|
|
147
|
+
# None context = legacy/default; non-dict = upstream caller without
|
|
148
|
+
# as_of support. Both treated as "no time-travel, current time".
|
|
149
|
+
as_of: str | None = (
|
|
150
|
+
context.get("as_of") if isinstance(context, dict) else None
|
|
151
|
+
)
|
|
152
|
+
# UTC-normalize at the filter boundary to guarantee format consistency
|
|
153
|
+
# before forwarding to DB comparisons.
|
|
154
|
+
if as_of is not None:
|
|
155
|
+
from superlocalmemory.retrieval.temporal_utils import normalize_as_of
|
|
156
|
+
as_of = normalize_as_of(as_of)
|
|
157
|
+
# normalize_as_of returns None on invalid input; treat as no as_of.
|
|
158
|
+
|
|
159
|
+
# --- Axis 1: Transaction-time supersession ---
|
|
160
|
+
# When as_of is set: only supersessions that occurred AT OR BEFORE
|
|
161
|
+
# as_of contribute (Phase 4b bi-temporal fix). Supersessions after
|
|
162
|
+
# as_of are invisible — the fact was still valid at the query point.
|
|
91
163
|
try:
|
|
92
164
|
invalid = self._db.get_invalidated_fact_ids(
|
|
93
|
-
list(all_fact_ids), profile_id,
|
|
165
|
+
list(all_fact_ids), profile_id, as_of=as_of,
|
|
94
166
|
)
|
|
95
167
|
except Exception as exc:
|
|
96
168
|
# Fail-open: a validity-lookup error must never break retrieval.
|
|
97
169
|
logger.warning("Temporal validity lookup failed: %s", exc)
|
|
98
170
|
return all_results
|
|
99
171
|
|
|
100
|
-
|
|
172
|
+
# --- Axis 2: Event-time expiry (Phase 4 T1b) ---
|
|
173
|
+
# Guard: skip event-time demotion when the caller signals it wants
|
|
174
|
+
# historical / all-expired-included mode AND no explicit as_of point
|
|
175
|
+
# is requested. Config default is include_expired_in_history=True,
|
|
176
|
+
# so the default recall path always takes this skip branch — the two
|
|
177
|
+
# new DB calls are never made, and no scores change.
|
|
178
|
+
apply_event_time = (as_of is not None) or (not self._include_expired_in_history)
|
|
179
|
+
|
|
180
|
+
event_expired: set[str] = set()
|
|
181
|
+
if apply_event_time:
|
|
182
|
+
try:
|
|
183
|
+
event_expired = self._db.get_event_time_expired_fact_ids(
|
|
184
|
+
list(all_fact_ids), profile_id, as_of=as_of,
|
|
185
|
+
)
|
|
186
|
+
except Exception as exc:
|
|
187
|
+
# Fail-open: continue with empty event_expired set.
|
|
188
|
+
logger.warning("Event-time expiry lookup failed: %s", exc)
|
|
189
|
+
|
|
190
|
+
# Fast path: nothing to demote — return input unchanged.
|
|
191
|
+
if not invalid and not event_expired:
|
|
101
192
|
return all_results
|
|
102
193
|
|
|
103
|
-
|
|
194
|
+
system_factor = self._demotion_factor
|
|
195
|
+
event_factor = self._event_time_factor
|
|
196
|
+
# event_time_only: expired by event-time but NOT system-invalidated.
|
|
197
|
+
# System-invalid already carries the harder demotion (0.25 < 0.5).
|
|
198
|
+
# We do not stack both factors to avoid excessive double-penalty.
|
|
199
|
+
event_time_only = event_expired - invalid
|
|
200
|
+
|
|
104
201
|
demoted: dict[str, list[tuple[str, float]]] = {}
|
|
105
202
|
for channel_name, channel_results in all_results.items():
|
|
106
203
|
new_list = [
|
|
107
|
-
(fact_id,
|
|
204
|
+
(fact_id,
|
|
205
|
+
score * system_factor if fact_id in invalid
|
|
206
|
+
else score * event_factor if fact_id in event_time_only
|
|
207
|
+
else score)
|
|
108
208
|
for fact_id, score in channel_results
|
|
109
209
|
]
|
|
110
|
-
# Re-sort descending so demoted
|
|
111
|
-
#
|
|
210
|
+
# Re-sort descending so demoted facts fall below currently-valid
|
|
211
|
+
# facts in this channel's rank order.
|
|
112
212
|
new_list.sort(key=lambda pair: pair[1], reverse=True)
|
|
113
213
|
demoted[channel_name] = new_list
|
|
114
214
|
return demoted
|
|
@@ -123,6 +223,10 @@ def register_temporal_validity_filter(
|
|
|
123
223
|
|
|
124
224
|
Does nothing if config.enabled is False.
|
|
125
225
|
|
|
226
|
+
Reads ``event_time_demotion_factor`` from the config with a fallback to the
|
|
227
|
+
module constant ``_EVENT_TIME_DEMOTION_FACTOR`` (0.5) so older serialised
|
|
228
|
+
configs without this field work without a migration.
|
|
229
|
+
|
|
126
230
|
Args:
|
|
127
231
|
registry: Channel registry to register with.
|
|
128
232
|
db: Database manager for validity queries.
|
|
@@ -131,5 +235,14 @@ def register_temporal_validity_filter(
|
|
|
131
235
|
if not getattr(config, "enabled", True):
|
|
132
236
|
return
|
|
133
237
|
factor = getattr(config, "superseded_demotion_factor", 0.25)
|
|
134
|
-
|
|
238
|
+
event_time_factor = getattr(
|
|
239
|
+
config, "event_time_demotion_factor", _EVENT_TIME_DEMOTION_FACTOR,
|
|
240
|
+
)
|
|
241
|
+
include_expired_in_history = getattr(config, "include_expired_in_history", True)
|
|
242
|
+
f = TemporalValidityFilter(
|
|
243
|
+
db,
|
|
244
|
+
demotion_factor=factor,
|
|
245
|
+
event_time_factor=event_time_factor,
|
|
246
|
+
include_expired_in_history=include_expired_in_history,
|
|
247
|
+
)
|
|
135
248
|
registry.register_filter(f.filter)
|
|
@@ -15,6 +15,7 @@ License: AGPL-3.0-or-later
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
|
+
import re
|
|
18
19
|
import sqlite3
|
|
19
20
|
import threading
|
|
20
21
|
from contextlib import contextmanager
|
|
@@ -137,8 +138,50 @@ class VectorStore:
|
|
|
137
138
|
|
|
138
139
|
# -- Table creation -----------------------------------------------------
|
|
139
140
|
|
|
141
|
+
@staticmethod
|
|
142
|
+
def _read_stored_dimension(conn: sqlite3.Connection) -> int | None:
|
|
143
|
+
"""Return the embedding dimension of the existing vec0 table, or None.
|
|
144
|
+
|
|
145
|
+
Reads the CREATE VIRTUAL TABLE DDL from sqlite_master and extracts the
|
|
146
|
+
float[N] declaration. Falls back to embedding_metadata.dimension if
|
|
147
|
+
the DDL is absent or unparseable. Returns None when the table does not
|
|
148
|
+
yet exist, which means no rebuild is needed.
|
|
149
|
+
"""
|
|
150
|
+
try:
|
|
151
|
+
row = conn.execute(
|
|
152
|
+
"SELECT sql FROM sqlite_master WHERE name = 'fact_embeddings'"
|
|
153
|
+
).fetchone()
|
|
154
|
+
if row is not None:
|
|
155
|
+
sql = row["sql"] or row[0]
|
|
156
|
+
if sql:
|
|
157
|
+
m = re.search(r'float\[(\d+)\]', sql, re.IGNORECASE)
|
|
158
|
+
if m:
|
|
159
|
+
return int(m.group(1))
|
|
160
|
+
except Exception:
|
|
161
|
+
pass
|
|
162
|
+
|
|
163
|
+
# Fallback: read dimension from the most recent metadata row.
|
|
164
|
+
try:
|
|
165
|
+
row = conn.execute(
|
|
166
|
+
"SELECT dimension FROM embedding_metadata LIMIT 1"
|
|
167
|
+
).fetchone()
|
|
168
|
+
if row is not None:
|
|
169
|
+
return int(row["dimension"])
|
|
170
|
+
except sqlite3.OperationalError:
|
|
171
|
+
pass # metadata table does not exist yet
|
|
172
|
+
|
|
173
|
+
return None
|
|
174
|
+
|
|
140
175
|
def _ensure_vec0_table(self) -> None:
|
|
141
|
-
"""Create the vec0 virtual table and embedding_metadata
|
|
176
|
+
"""Create or rebuild the vec0 virtual table and embedding_metadata.
|
|
177
|
+
|
|
178
|
+
If the existing vec0 table was built at a different embedding dimension
|
|
179
|
+
than self._config.dimension, both tables are dropped and recreated at
|
|
180
|
+
the new dimension. Old embeddings are lost; callers that want to
|
|
181
|
+
re-populate should call rebuild_from_facts() afterward.
|
|
182
|
+
|
|
183
|
+
Same-dimension opens are a no-op (IF NOT EXISTS guard).
|
|
184
|
+
"""
|
|
142
185
|
dim = self._config.dimension
|
|
143
186
|
vec0_ddl = (
|
|
144
187
|
f"CREATE VIRTUAL TABLE IF NOT EXISTS fact_embeddings USING vec0("
|
|
@@ -162,12 +205,41 @@ class VectorStore:
|
|
|
162
205
|
meta_idx_profile = (
|
|
163
206
|
"CREATE INDEX IF NOT EXISTS idx_embmeta_profile ON embedding_metadata (profile_id)"
|
|
164
207
|
)
|
|
208
|
+
row_map_ddl = (
|
|
209
|
+
"CREATE TABLE IF NOT EXISTS vector_row_map ("
|
|
210
|
+
"fact_id TEXT NOT NULL PRIMARY KEY, "
|
|
211
|
+
"profile_id TEXT NOT NULL, "
|
|
212
|
+
"vec_rowid INTEGER NOT NULL"
|
|
213
|
+
")"
|
|
214
|
+
)
|
|
215
|
+
row_map_idx = (
|
|
216
|
+
"CREATE INDEX IF NOT EXISTS idx_vector_row_map_profile "
|
|
217
|
+
"ON vector_row_map (profile_id)"
|
|
218
|
+
)
|
|
165
219
|
try:
|
|
166
220
|
with self._managed_connection() as conn:
|
|
221
|
+
stored_dim = self._read_stored_dimension(conn)
|
|
222
|
+
if stored_dim is not None and stored_dim != dim:
|
|
223
|
+
logger.info(
|
|
224
|
+
"Embedding dimension changed %d→%d: rebuilding vector index",
|
|
225
|
+
stored_dim,
|
|
226
|
+
dim,
|
|
227
|
+
)
|
|
228
|
+
# Drop metadata first (no FK enforcement, but cleaner ordering).
|
|
229
|
+
# Indexes on embedding_metadata are dropped automatically.
|
|
230
|
+
conn.execute("DROP TABLE IF EXISTS embedding_metadata")
|
|
231
|
+
conn.execute("DROP TABLE IF EXISTS vector_row_map")
|
|
232
|
+
conn.execute("DROP TABLE IF EXISTS fact_embeddings")
|
|
233
|
+
# Commit the drops before recreating so that the virtual-table
|
|
234
|
+
# shadow tables are fully removed before the new CREATE runs.
|
|
235
|
+
conn.commit()
|
|
236
|
+
|
|
167
237
|
conn.execute(vec0_ddl)
|
|
168
238
|
conn.execute(meta_ddl)
|
|
169
239
|
conn.execute(meta_idx_fact)
|
|
170
240
|
conn.execute(meta_idx_profile)
|
|
241
|
+
conn.execute(row_map_ddl)
|
|
242
|
+
conn.execute(row_map_idx)
|
|
171
243
|
conn.commit()
|
|
172
244
|
except Exception as exc:
|
|
173
245
|
logger.debug("vec0 table creation failed: %s", exc)
|
|
@@ -255,13 +327,30 @@ class VectorStore:
|
|
|
255
327
|
# Older self-heal code could insert metadata
|
|
256
328
|
# before sqlite-vec, or row-id drift could point
|
|
257
329
|
# metadata at another profile's vector. Neither
|
|
258
|
-
# is a valid projection pair. Remove
|
|
259
|
-
#
|
|
260
|
-
#
|
|
330
|
+
# is a valid projection pair. Remove the stale
|
|
331
|
+
# pointer and rebuild at a fresh rowid; never
|
|
332
|
+
# overwrite the other profile's payload.
|
|
261
333
|
conn.execute(
|
|
262
334
|
"DELETE FROM embedding_metadata WHERE fact_id = ?",
|
|
263
335
|
(fact_id,),
|
|
264
336
|
)
|
|
337
|
+
# The old vec0 row is orphaned unless another
|
|
338
|
+
# projection pair still references it. Reclaim it
|
|
339
|
+
# so drift-repair never abandons raw payload.
|
|
340
|
+
still_referenced = conn.execute(
|
|
341
|
+
"SELECT 1 FROM embedding_metadata "
|
|
342
|
+
"WHERE vec_rowid = ? LIMIT 1",
|
|
343
|
+
(rowid,),
|
|
344
|
+
).fetchone()
|
|
345
|
+
if still_referenced is None:
|
|
346
|
+
conn.execute(
|
|
347
|
+
"DELETE FROM fact_embeddings WHERE rowid = ?",
|
|
348
|
+
(rowid,),
|
|
349
|
+
)
|
|
350
|
+
conn.execute(
|
|
351
|
+
"DELETE FROM vector_row_map WHERE fact_id = ?",
|
|
352
|
+
(fact_id,),
|
|
353
|
+
)
|
|
265
354
|
row = None
|
|
266
355
|
|
|
267
356
|
if row is None:
|
|
@@ -301,6 +390,14 @@ class VectorStore:
|
|
|
301
390
|
),
|
|
302
391
|
)
|
|
303
392
|
|
|
393
|
+
conn.execute(
|
|
394
|
+
"INSERT INTO vector_row_map (fact_id, profile_id, vec_rowid) "
|
|
395
|
+
"VALUES (?, ?, ?) "
|
|
396
|
+
"ON CONFLICT(fact_id) DO UPDATE SET "
|
|
397
|
+
"profile_id = excluded.profile_id, "
|
|
398
|
+
"vec_rowid = excluded.vec_rowid",
|
|
399
|
+
(fact_id, profile_id, rowid),
|
|
400
|
+
)
|
|
304
401
|
conn.commit()
|
|
305
402
|
return True
|
|
306
403
|
except Exception as exc:
|
|
@@ -415,16 +512,33 @@ class VectorStore:
|
|
|
415
512
|
(fact_id,),
|
|
416
513
|
).fetchone()
|
|
417
514
|
|
|
418
|
-
|
|
515
|
+
rowid = None
|
|
516
|
+
owner_profile = None
|
|
517
|
+
if row is not None:
|
|
518
|
+
rowid = row["vec_rowid"]
|
|
519
|
+
owner_profile = str(row["profile_id"])
|
|
520
|
+
else:
|
|
521
|
+
# Metadata-less orphan: resolve the rowid via the
|
|
522
|
+
# fact-addressable map so the raw vec0 row is still
|
|
523
|
+
# removable by fact_id.
|
|
524
|
+
mrow = conn.execute(
|
|
525
|
+
"SELECT vec_rowid, profile_id "
|
|
526
|
+
"FROM vector_row_map WHERE fact_id = ?",
|
|
527
|
+
(fact_id,),
|
|
528
|
+
).fetchone()
|
|
529
|
+
if mrow is not None:
|
|
530
|
+
rowid = mrow["vec_rowid"]
|
|
531
|
+
owner_profile = str(mrow["profile_id"])
|
|
532
|
+
|
|
533
|
+
if rowid is None:
|
|
419
534
|
return False
|
|
420
535
|
|
|
421
|
-
rowid = row["vec_rowid"]
|
|
422
536
|
vector_row = conn.execute(
|
|
423
537
|
"SELECT profile_id FROM fact_embeddings WHERE rowid = ?",
|
|
424
538
|
(rowid,),
|
|
425
539
|
).fetchone()
|
|
426
|
-
if vector_row is not None and
|
|
427
|
-
|
|
540
|
+
if vector_row is not None and (
|
|
541
|
+
str(vector_row["profile_id"]) == owner_profile
|
|
428
542
|
):
|
|
429
543
|
conn.execute(
|
|
430
544
|
"DELETE FROM fact_embeddings WHERE rowid = ?",
|
|
@@ -434,12 +548,54 @@ class VectorStore:
|
|
|
434
548
|
"DELETE FROM embedding_metadata WHERE vec_rowid = ?",
|
|
435
549
|
(rowid,),
|
|
436
550
|
)
|
|
551
|
+
conn.execute(
|
|
552
|
+
"DELETE FROM vector_row_map WHERE fact_id = ?",
|
|
553
|
+
(fact_id,),
|
|
554
|
+
)
|
|
437
555
|
conn.commit()
|
|
438
556
|
return True
|
|
439
557
|
except Exception as exc:
|
|
440
558
|
logger.debug("delete failed for fact_id=%s: %s", fact_id, exc)
|
|
441
559
|
return False
|
|
442
560
|
|
|
561
|
+
def raw_vector_present(self, fact_id: str) -> bool:
|
|
562
|
+
if not self._available:
|
|
563
|
+
try:
|
|
564
|
+
conn = sqlite3.connect(str(self._db_path))
|
|
565
|
+
try:
|
|
566
|
+
row = conn.execute(
|
|
567
|
+
"SELECT 1 FROM vector_row_map WHERE fact_id = ? LIMIT 1",
|
|
568
|
+
(fact_id,),
|
|
569
|
+
).fetchone()
|
|
570
|
+
return row is not None
|
|
571
|
+
finally:
|
|
572
|
+
conn.close()
|
|
573
|
+
except Exception:
|
|
574
|
+
return False
|
|
575
|
+
try:
|
|
576
|
+
with self._managed_connection() as conn:
|
|
577
|
+
row = conn.execute(
|
|
578
|
+
"SELECT 1 FROM vector_row_map vrm "
|
|
579
|
+
"WHERE vrm.fact_id = ? "
|
|
580
|
+
"AND EXISTS (SELECT 1 FROM fact_embeddings fe "
|
|
581
|
+
"WHERE fe.rowid = vrm.vec_rowid)",
|
|
582
|
+
(fact_id,),
|
|
583
|
+
).fetchone()
|
|
584
|
+
return row is not None
|
|
585
|
+
except Exception:
|
|
586
|
+
try:
|
|
587
|
+
conn2 = sqlite3.connect(str(self._db_path))
|
|
588
|
+
try:
|
|
589
|
+
row = conn2.execute(
|
|
590
|
+
"SELECT 1 FROM vector_row_map WHERE fact_id = ? LIMIT 1",
|
|
591
|
+
(fact_id,),
|
|
592
|
+
).fetchone()
|
|
593
|
+
return row is not None
|
|
594
|
+
finally:
|
|
595
|
+
conn2.close()
|
|
596
|
+
except Exception:
|
|
597
|
+
return False
|
|
598
|
+
|
|
443
599
|
def count(self, profile_id: str | None = None) -> int:
|
|
444
600
|
"""Count complete metadata/vector pairs in the store.
|
|
445
601
|
|
|
@@ -493,6 +649,56 @@ class VectorStore:
|
|
|
493
649
|
logger.debug("indexed_fact_ids failed: %s", exc)
|
|
494
650
|
return set()
|
|
495
651
|
|
|
652
|
+
def gc_orphaned_vectors(self, profile_id: str | None = None) -> int:
|
|
653
|
+
"""Physically remove vec0 rows not referenced by any fact-addressable
|
|
654
|
+
mapping (neither embedding_metadata nor vector_row_map).
|
|
655
|
+
|
|
656
|
+
Such rows are unreachable by fact_id and would otherwise persist forever.
|
|
657
|
+
Returns the count removed. No-op when the vector backend is unavailable.
|
|
658
|
+
"""
|
|
659
|
+
if not self._available:
|
|
660
|
+
return 0
|
|
661
|
+
|
|
662
|
+
_wl = get_write_lock(self._db_path)
|
|
663
|
+
with _wl:
|
|
664
|
+
with self._lock:
|
|
665
|
+
try:
|
|
666
|
+
with self._managed_connection() as conn:
|
|
667
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
668
|
+
orphans = self._orphan_rowids(conn, profile_id)
|
|
669
|
+
for rowid in orphans:
|
|
670
|
+
conn.execute(
|
|
671
|
+
"DELETE FROM fact_embeddings WHERE rowid = ?",
|
|
672
|
+
(rowid,),
|
|
673
|
+
)
|
|
674
|
+
conn.commit()
|
|
675
|
+
return len(orphans)
|
|
676
|
+
except Exception as exc:
|
|
677
|
+
logger.debug("gc_orphaned_vectors failed: %s", exc)
|
|
678
|
+
return 0
|
|
679
|
+
|
|
680
|
+
@staticmethod
|
|
681
|
+
def _orphan_rowids(conn: sqlite3.Connection, profile_id: str | None) -> set[int]:
|
|
682
|
+
"""vec0 rowids not referenced by embedding_metadata or vector_row_map."""
|
|
683
|
+
if profile_id is not None:
|
|
684
|
+
all_rowids = {r[0] for r in conn.execute(
|
|
685
|
+
"SELECT rowid FROM fact_embeddings WHERE profile_id = ?",
|
|
686
|
+
(profile_id,)).fetchall()}
|
|
687
|
+
ref_meta = {r[0] for r in conn.execute(
|
|
688
|
+
"SELECT vec_rowid FROM embedding_metadata WHERE profile_id = ?",
|
|
689
|
+
(profile_id,)).fetchall()}
|
|
690
|
+
ref_map = {r[0] for r in conn.execute(
|
|
691
|
+
"SELECT vec_rowid FROM vector_row_map WHERE profile_id = ?",
|
|
692
|
+
(profile_id,)).fetchall()}
|
|
693
|
+
else:
|
|
694
|
+
all_rowids = {r[0] for r in conn.execute(
|
|
695
|
+
"SELECT rowid FROM fact_embeddings").fetchall()}
|
|
696
|
+
ref_meta = {r[0] for r in conn.execute(
|
|
697
|
+
"SELECT vec_rowid FROM embedding_metadata").fetchall()}
|
|
698
|
+
ref_map = {r[0] for r in conn.execute(
|
|
699
|
+
"SELECT vec_rowid FROM vector_row_map").fetchall()}
|
|
700
|
+
return all_rowids - ref_meta - ref_map
|
|
701
|
+
|
|
496
702
|
def rebuild_from_facts(
|
|
497
703
|
self,
|
|
498
704
|
facts: list[tuple[str, str, list[float]]],
|
|
@@ -109,8 +109,8 @@ async def lifespan(application: FastAPI):
|
|
|
109
109
|
def create_app() -> FastAPI:
|
|
110
110
|
"""Create and configure the FastAPI application."""
|
|
111
111
|
application = FastAPI(
|
|
112
|
-
title="SuperLocalMemory
|
|
113
|
-
description="
|
|
112
|
+
title="SuperLocalMemory V4 API",
|
|
113
|
+
description="V4 Memory Engine REST API",
|
|
114
114
|
version=SLM_VERSION,
|
|
115
115
|
lifespan=lifespan,
|
|
116
116
|
)
|
|
@@ -240,9 +240,9 @@ def create_app() -> FastAPI:
|
|
|
240
240
|
index_path = UI_DIR / "index.html"
|
|
241
241
|
if not index_path.exists():
|
|
242
242
|
return (
|
|
243
|
-
"<html><head><title>SuperLocalMemory
|
|
243
|
+
"<html><head><title>SuperLocalMemory V4</title></head>"
|
|
244
244
|
"<body style='font-family:Arial;padding:40px'>"
|
|
245
|
-
"<h1>SuperLocalMemory
|
|
245
|
+
"<h1>SuperLocalMemory V4 API Server Running</h1>"
|
|
246
246
|
"<p><a href='/docs'>API Documentation</a></p>"
|
|
247
247
|
"</body></html>"
|
|
248
248
|
)
|
|
@@ -272,7 +272,7 @@ def create_app() -> FastAPI:
|
|
|
272
272
|
if __name__ == "__main__":
|
|
273
273
|
app = create_app()
|
|
274
274
|
print("=" * 60)
|
|
275
|
-
print("SuperLocalMemory
|
|
275
|
+
print("SuperLocalMemory V4 - API Server (standalone mode)")
|
|
276
276
|
print("=" * 60)
|
|
277
277
|
print(f"Database: {DB_PATH}")
|
|
278
278
|
print(f"UI Directory: {UI_DIR}")
|