superlocalmemory 4.0.7 → 4.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +219 -1
- package/README.md +6 -6
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/scripts/ensure-venv.sh +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +3 -3
- package/plugin-src/rules/AGENTS.md +1 -1
- package/plugin-src/skills/slm-status/SKILL.md +2 -2
- package/pyproject.toml +1 -1
- package/scripts/postinstall.js +4 -0
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/_lazy_init.py +1 -1
- package/src/superlocalmemory/cli/commands.py +119 -9
- package/src/superlocalmemory/cli/db_migrate.py +0 -2
- package/src/superlocalmemory/cli/gdpr_io.py +1 -1
- package/src/superlocalmemory/cli/main.py +5 -5
- package/src/superlocalmemory/cli/service_installer.py +2 -1
- package/src/superlocalmemory/cli/setup_wizard.py +1 -1
- package/src/superlocalmemory/cli/summary_cmd.py +23 -3
- package/src/superlocalmemory/code_graph/bridge/maintenance.py +7 -1
- package/src/superlocalmemory/core/config.py +41 -7
- package/src/superlocalmemory/core/consolidation_engine.py +14 -15
- package/src/superlocalmemory/core/context_cache.py +0 -2
- package/src/superlocalmemory/core/engine.py +371 -63
- package/src/superlocalmemory/core/evidence_bundle.py +3 -1
- package/src/superlocalmemory/core/install_detector.py +131 -0
- package/src/superlocalmemory/core/progressive_abstraction.py +1 -1
- package/src/superlocalmemory/core/recall_worker.py +4 -0
- package/src/superlocalmemory/core/security_primitives.py +3 -6
- package/src/superlocalmemory/core/store_pipeline.py +94 -26
- package/src/superlocalmemory/core/topic_signature.py +0 -2
- package/src/superlocalmemory/core/transactions/concrete_owners.py +15 -8
- package/src/superlocalmemory/dynamics/eap_scheduler.py +17 -6
- package/src/superlocalmemory/encoding/graph_builder.py +2 -2
- package/src/superlocalmemory/encoding/scene_builder.py +8 -2
- package/src/superlocalmemory/evolution/skill_evolver.py +16 -1
- package/src/superlocalmemory/hooks/adapter_base.py +0 -2
- package/src/superlocalmemory/hooks/context_payload.py +0 -2
- package/src/superlocalmemory/hooks/hook_handlers.py +38 -11
- package/src/superlocalmemory/hooks/portable_kit.py +8 -8
- package/src/superlocalmemory/hooks/post_tool_async_hook.py +0 -2
- package/src/superlocalmemory/hooks/prewarm_auth.py +0 -2
- package/src/superlocalmemory/hooks/user_prompt_hook.py +0 -2
- package/src/superlocalmemory/infra/backup.py +44 -8
- package/src/superlocalmemory/integrations/bounded_loops_mcp.py +24 -7
- package/src/superlocalmemory/learning/arm_catalog.py +0 -2
- package/src/superlocalmemory/learning/bandit.py +0 -2
- package/src/superlocalmemory/learning/bandit_cache.py +0 -2
- package/src/superlocalmemory/learning/dedup_hnsw.py +11 -11
- package/src/superlocalmemory/learning/ensemble.py +0 -2
- package/src/superlocalmemory/learning/labeler.py +0 -2
- package/src/superlocalmemory/learning/legacy_migration.py +0 -2
- package/src/superlocalmemory/learning/model_cache.py +0 -2
- package/src/superlocalmemory/learning/pattern_miner.py +12 -7
- package/src/superlocalmemory/learning/ranker.py +0 -2
- package/src/superlocalmemory/learning/reward_archive.py +6 -1
- package/src/superlocalmemory/learning/reward_proxy.py +0 -2
- package/src/superlocalmemory/learning/signal_worker.py +0 -2
- package/src/superlocalmemory/math/fisher.py +1 -1
- package/src/superlocalmemory/math/hopfield.py +4 -1
- package/src/superlocalmemory/math/langevin.py +1 -1
- package/src/superlocalmemory/math/sheaf.py +7 -3
- package/src/superlocalmemory/mcp/cli_fallback.py +1 -1
- package/src/superlocalmemory/mcp/profiles.py +11 -4
- package/src/superlocalmemory/mcp/server.py +8 -1
- package/src/superlocalmemory/mcp/tools_active.py +56 -0
- package/src/superlocalmemory/mcp/tools_core.py +1 -1
- package/src/superlocalmemory/mcp/tools_summaries.py +147 -0
- package/src/superlocalmemory/optimize/cache/manager.py +2 -2
- package/src/superlocalmemory/optimize/compress/ccr.py +1 -1
- package/src/superlocalmemory/optimize/compress/router.py +1 -1
- package/src/superlocalmemory/optimize/proxy/_helpers.py +2 -2
- package/src/superlocalmemory/optimize/proxy/server.py +1 -1
- package/src/superlocalmemory/optimize/proxy/vertex_surface.py +2 -2
- package/src/superlocalmemory/optimize/storage/db.py +2 -2
- package/src/superlocalmemory/retrieval/agentic.py +1 -1
- package/src/superlocalmemory/retrieval/ann_index.py +9 -2
- package/src/superlocalmemory/retrieval/bm25_channel.py +2 -2
- package/src/superlocalmemory/retrieval/bridge_discovery.py +2 -2
- package/src/superlocalmemory/retrieval/engine.py +272 -43
- package/src/superlocalmemory/retrieval/entity_channel.py +1 -1
- package/src/superlocalmemory/retrieval/hopfield_channel.py +8 -2
- package/src/superlocalmemory/retrieval/profile_channel.py +1 -1
- package/src/superlocalmemory/retrieval/quantization_aware_search.py +1 -1
- package/src/superlocalmemory/retrieval/remote_reranker.py +2 -2
- package/src/superlocalmemory/retrieval/reranker.py +3 -3
- package/src/superlocalmemory/retrieval/semantic_channel.py +3 -3
- package/src/superlocalmemory/retrieval/spreading_activation.py +8 -8
- package/src/superlocalmemory/retrieval/strategy.py +94 -0
- package/src/superlocalmemory/retrieval/temporal_channel.py +167 -10
- package/src/superlocalmemory/retrieval/temporal_validity_filter.py +1 -1
- package/src/superlocalmemory/retrieval/vector_store.py +88 -10
- package/src/superlocalmemory/server/consolidation_runner.py +140 -0
- package/src/superlocalmemory/server/recall_serializer.py +44 -2
- package/src/superlocalmemory/server/routes/agents.py +52 -8
- package/src/superlocalmemory/server/routes/brain.py +110 -2
- package/src/superlocalmemory/server/routes/memories.py +153 -0
- package/src/superlocalmemory/server/routes/prewarm.py +4 -4
- package/src/superlocalmemory/server/routes/v3_api.py +24 -46
- package/src/superlocalmemory/server/unified_daemon.py +566 -7
- package/src/superlocalmemory/storage/_schema_version.py +46 -3
- package/src/superlocalmemory/storage/backup.py +531 -0
- package/src/superlocalmemory/storage/database.py +11 -4
- package/src/superlocalmemory/storage/embedding_codec.py +129 -0
- package/src/superlocalmemory/storage/embedding_migrator.py +5 -3
- package/src/superlocalmemory/storage/migration_runner.py +142 -2
- package/src/superlocalmemory/storage/migrations/__init__.py +1 -1
- package/src/superlocalmemory/storage/migrations.py +15 -1
- package/src/superlocalmemory/storage/models.py +7 -0
- package/src/superlocalmemory/storage/quantized_store.py +4 -2
- package/src/superlocalmemory/summaries/base.py +159 -0
- package/src/superlocalmemory/summaries/daily_reflection.py +55 -8
- package/src/superlocalmemory/summaries/project_work_log.py +23 -7
- package/src/superlocalmemory/summaries/session_summary.py +10 -6
- package/src/superlocalmemory/ui/css/legacy-dashboard.css +1 -1
- package/src/superlocalmemory/ui/css/neural-glass.css +1 -1
- package/src/superlocalmemory/ui/index.html +9 -3
- package/src/superlocalmemory/ui/js/core.js +1 -1
- package/src/superlocalmemory/ui/js/od-boundedloops.js +324 -0
- package/src/superlocalmemory/ui/js/od-brain.js +1 -1
- package/src/superlocalmemory/ui/js/od-memories.js +337 -12
- package/src/superlocalmemory/ui/js/od-mesh.js +97 -5
- package/src/superlocalmemory/ui/js/od-operations.js +1 -150
- package/src/superlocalmemory/ui/js/od-optimize.js +36 -9
- package/src/superlocalmemory/ui/js/od-shell.js +10 -0
|
@@ -20,6 +20,7 @@ from __future__ import annotations
|
|
|
20
20
|
import logging
|
|
21
21
|
import os
|
|
22
22
|
import threading
|
|
23
|
+
import time
|
|
23
24
|
from pathlib import Path
|
|
24
25
|
from typing import Any
|
|
25
26
|
|
|
@@ -49,7 +50,7 @@ def _verify_ingestion_schema(memory_db: Path) -> bool:
|
|
|
49
50
|
|
|
50
51
|
|
|
51
52
|
# ---------------------------------------------------------------------------
|
|
52
|
-
#
|
|
53
|
+
# Embedding a memory on the write path, before the receipt is returned
|
|
53
54
|
# ---------------------------------------------------------------------------
|
|
54
55
|
|
|
55
56
|
def _is_remote_embedder(embedder: object) -> bool:
|
|
@@ -132,11 +133,18 @@ class MemoryEngine:
|
|
|
132
133
|
self._consolidation_engine = None
|
|
133
134
|
self._maintenance_scheduler = None
|
|
134
135
|
self._hooks = HookRegistry()
|
|
135
|
-
#
|
|
136
|
-
#
|
|
136
|
+
# One single-worker pool reused across store_fast() calls, created on the
|
|
137
|
+
# first warm-guard attempt. Reusing it avoids a thread per call.
|
|
137
138
|
self._store_fast_embed_pool: object | None = None
|
|
138
|
-
#
|
|
139
|
+
# Guards the lazy creation so concurrent first calls cannot each build one.
|
|
139
140
|
self._store_fast_embed_pool_lock = threading.Lock()
|
|
141
|
+
# Set once the engine is closed, and never cleared. A caller already
|
|
142
|
+
# inside the warm-guard path when close() runs will reach the creation
|
|
143
|
+
# check afterwards and find None; without this it would build a
|
|
144
|
+
# replacement pool that nothing owns and nothing will shut down, so a
|
|
145
|
+
# burst of writes arriving during shutdown leaks a thread each and holds
|
|
146
|
+
# the process open past its shutdown budget.
|
|
147
|
+
self._store_fast_embed_pool_closed = False
|
|
140
148
|
|
|
141
149
|
# -- Public properties (Phase 2+ access) --------------------------------
|
|
142
150
|
|
|
@@ -535,6 +543,348 @@ class MemoryEngine:
|
|
|
535
543
|
trusted_actor_id=local_trusted_actor_id("python-api-prebuilt"),
|
|
536
544
|
)
|
|
537
545
|
|
|
546
|
+
def _warm_guard_embed(
|
|
547
|
+
self, text: str, *, timeout_s: float | None = None,
|
|
548
|
+
) -> tuple[list[float] | None, Any, Any]:
|
|
549
|
+
"""Embed now if it is cheap and safe to, otherwise leave it to the materializer.
|
|
550
|
+
|
|
551
|
+
A fact with no vector is invisible to the semantic channel, so a memory
|
|
552
|
+
written moments ago is the hardest thing in the store to find. Computing
|
|
553
|
+
it inline closes that window — but only when doing so cannot hurt anyone
|
|
554
|
+
else, because this daemon serves many sessions at once.
|
|
555
|
+
|
|
556
|
+
Four constraints, each load-bearing:
|
|
557
|
+
|
|
558
|
+
* only when the embedder is provably warm and local, so a cold start or a
|
|
559
|
+
network round-trip can never be paid on a write;
|
|
560
|
+
* on one shared single-worker pool, so N concurrent writers cannot spawn
|
|
561
|
+
N threads — the sixth caller simply misses its deadline and defers;
|
|
562
|
+
* inside ``background_work()``, so a foreground recall preempts it rather
|
|
563
|
+
than queueing behind it;
|
|
564
|
+
* behind a hard deadline, after which the caller proceeds with no
|
|
565
|
+
embedding and the background materializer finishes the job.
|
|
566
|
+
|
|
567
|
+
Returns ``(embedding, fisher_mean, fisher_variance)``, any of which may be
|
|
568
|
+
``None``. Never raises: every failure means "not now", not "write failed".
|
|
569
|
+
"""
|
|
570
|
+
emb = None
|
|
571
|
+
fmean = fvar = None
|
|
572
|
+
_embedder_ref = self._embedder
|
|
573
|
+
if (
|
|
574
|
+
_embedder_ref is None
|
|
575
|
+
or getattr(_embedder_ref, "_available", None) is not True
|
|
576
|
+
or _is_remote_embedder(_embedder_ref)
|
|
577
|
+
):
|
|
578
|
+
return None, None, None
|
|
579
|
+
|
|
580
|
+
import concurrent.futures as _cf
|
|
581
|
+
|
|
582
|
+
# Lazy-init once per engine — avoids per-call thread churn. Double-checked
|
|
583
|
+
# locking guards against TOCTOU on concurrent first calls.
|
|
584
|
+
if self._store_fast_embed_pool is None:
|
|
585
|
+
with self._store_fast_embed_pool_lock:
|
|
586
|
+
if self._store_fast_embed_pool_closed:
|
|
587
|
+
# Shutdown happened while this call was between the check
|
|
588
|
+
# above and this lock. Decline rather than resurrect the
|
|
589
|
+
# pool: the fact is already committed durably and keeps its
|
|
590
|
+
# place in the background queue.
|
|
591
|
+
return None, None, None
|
|
592
|
+
if self._store_fast_embed_pool is None:
|
|
593
|
+
self._store_fast_embed_pool = _cf.ThreadPoolExecutor(
|
|
594
|
+
max_workers=1,
|
|
595
|
+
thread_name_prefix="slm-sg-embed",
|
|
596
|
+
)
|
|
597
|
+
# A ceiling, not a target. Storing a memory is allowed up to 1.5 s in
|
|
598
|
+
# total, and giving up on the embedding early is not free: it is the
|
|
599
|
+
# difference between a memory that can be found by asking a question and
|
|
600
|
+
# one that can only be found by quoting its own words. Within the budget
|
|
601
|
+
# the more complete outcome wins, so this waits a full second rather than
|
|
602
|
+
# half of one. Both values below are milliseconds via the environment.
|
|
603
|
+
if timeout_s is None:
|
|
604
|
+
try:
|
|
605
|
+
timeout_s = int(os.environ.get("SLM_STORE_FAST_EMBED_TIMEOUT_MS", 1000)) / 1000.0
|
|
606
|
+
except (ValueError, TypeError):
|
|
607
|
+
timeout_s = 1.0
|
|
608
|
+
try:
|
|
609
|
+
def _best_effort_embed():
|
|
610
|
+
from superlocalmemory.core.recall_gate import background_work
|
|
611
|
+
with background_work():
|
|
612
|
+
return _embedder_ref.embed(text)
|
|
613
|
+
|
|
614
|
+
_future = self._store_fast_embed_pool.submit(_best_effort_embed)
|
|
615
|
+
try:
|
|
616
|
+
emb = _future.result(timeout=timeout_s)
|
|
617
|
+
if emb:
|
|
618
|
+
fmean, fvar = _embedder_ref.compute_fisher_params(emb)
|
|
619
|
+
except _cf.TimeoutError:
|
|
620
|
+
# Cancel rather than abandon. This pool has a single worker, so a
|
|
621
|
+
# task left running holds it for every caller behind us: each in
|
|
622
|
+
# turn times out and queues another task that runs after its
|
|
623
|
+
# caller has already answered. Cancel only stops it if it has not
|
|
624
|
+
# started, which is exactly the queued backlog we care about.
|
|
625
|
+
_future.cancel()
|
|
626
|
+
logger.debug(
|
|
627
|
+
"warm-guard embed timed out (>%.0fms) — deferring to materializer",
|
|
628
|
+
timeout_s * 1000,
|
|
629
|
+
)
|
|
630
|
+
emb = None
|
|
631
|
+
except Exception as _exc:
|
|
632
|
+
logger.debug("warm-guard embed failed (%s) — deferring to materializer", _exc)
|
|
633
|
+
emb = None
|
|
634
|
+
except Exception as _exc:
|
|
635
|
+
logger.debug("warm-guard pool submit failed (%s)", _exc)
|
|
636
|
+
emb = None
|
|
637
|
+
return emb, fmean, fvar
|
|
638
|
+
|
|
639
|
+
def _projection_has(self, fact_id: str) -> bool:
|
|
640
|
+
"""Can the meaning-based channel reach this fact?
|
|
641
|
+
|
|
642
|
+
Answered by the vector store, which owns the question, and never from
|
|
643
|
+
``atomic_facts.embedding``. The column and the projection can disagree,
|
|
644
|
+
and when they do it is the projection that decides whether a search
|
|
645
|
+
finds the fact. Reachability also needs more than a stored vector: the
|
|
646
|
+
search joins two projection tables, so a vector present in one of them
|
|
647
|
+
and absent from the other is not reachable. That predicate belongs to
|
|
648
|
+
the store; duplicating it here is how the column and the projection
|
|
649
|
+
drifted apart in the first place.
|
|
650
|
+
|
|
651
|
+
Fails closed. A store that cannot answer is treated as "not reachable",
|
|
652
|
+
which costs at most one redundant idempotent write and can never report
|
|
653
|
+
a fact as findable when it is not.
|
|
654
|
+
"""
|
|
655
|
+
store = getattr(self, "_vector_store", None)
|
|
656
|
+
answer = getattr(store, "is_searchable_by_meaning", None) if store else None
|
|
657
|
+
if not callable(answer):
|
|
658
|
+
return False
|
|
659
|
+
try:
|
|
660
|
+
return bool(answer(fact_id, self._profile_id))
|
|
661
|
+
except Exception:
|
|
662
|
+
return False
|
|
663
|
+
|
|
664
|
+
def _attach_vector(
|
|
665
|
+
self, fact_id: str, emb: list[float],
|
|
666
|
+
fmean: float | None, fvar: float | None,
|
|
667
|
+
) -> bool:
|
|
668
|
+
"""Attach a vector to a stored fact. True only if a search can now find it.
|
|
669
|
+
|
|
670
|
+
Two representations are involved and they answer different questions. The
|
|
671
|
+
vector projection is what a search on meaning queries, so it decides
|
|
672
|
+
whether the fact is *findable*. The ``atomic_facts.embedding`` column is
|
|
673
|
+
where the vector *lives* — it is what an export, a backup or a direct
|
|
674
|
+
fetch reads, and what a re-projection would be rebuilt from.
|
|
675
|
+
|
|
676
|
+
So:
|
|
677
|
+
|
|
678
|
+
* **The projection is attempted first**, because that is the ordering that
|
|
679
|
+
keeps the two consistent when both can succeed.
|
|
680
|
+
* **The column is written either way.** The vector has already been
|
|
681
|
+
computed and it is real data; withholding it because the projection was
|
|
682
|
+
refused would discard it and leave the repair pass paying for the model
|
|
683
|
+
again on every pass. Every reason a projection is refused — no search
|
|
684
|
+
extension on this platform, a dimension change, a store at a different
|
|
685
|
+
dimension — is a property of the installation and not of this fact, so
|
|
686
|
+
withholding one fact's column repairs nothing.
|
|
687
|
+
* **The return value is the honest answer to "can this be found by
|
|
688
|
+
meaning", and it is False whenever the projection did not accept the
|
|
689
|
+
vector.** That is what keeps a receipt from claiming a memory is
|
|
690
|
+
searchable when no search can reach it, which was the actual harm in
|
|
691
|
+
writing these two out of order — not the column itself.
|
|
692
|
+
* **If the projection succeeded and the column write then failed, the
|
|
693
|
+
projection is rolled back.** Otherwise a search finds a fact whose
|
|
694
|
+
vector nothing else can see, and nothing would ever reconcile them.
|
|
695
|
+
|
|
696
|
+
Never raises. The durable write has already happened before this runs; a
|
|
697
|
+
fact that is not projected keeps its place in the background queue.
|
|
698
|
+
|
|
699
|
+
Once started, the pair runs to completion and is not interrupted by a
|
|
700
|
+
caller deadline. Abandoning it half-way is what produces the disagreeing
|
|
701
|
+
states above, so a deadline can decide whether to *begin* the pair, never
|
|
702
|
+
whether to finish it. Callers check their budget immediately before
|
|
703
|
+
calling. Bounding the pair itself is a property of the write lock.
|
|
704
|
+
"""
|
|
705
|
+
projected = False
|
|
706
|
+
store = getattr(self, "_vector_store", None)
|
|
707
|
+
if store is not None and getattr(store, "available", False):
|
|
708
|
+
try:
|
|
709
|
+
projected = bool(store.upsert(fact_id, self._profile_id, emb))
|
|
710
|
+
if not projected:
|
|
711
|
+
logger.warning(
|
|
712
|
+
"vector projection refused for %s — stored but not "
|
|
713
|
+
"findable by meaning", fact_id[:12],
|
|
714
|
+
)
|
|
715
|
+
except Exception as exc:
|
|
716
|
+
logger.warning(
|
|
717
|
+
"vector projection failed for %s (%s: %s) — stored but not "
|
|
718
|
+
"findable by meaning",
|
|
719
|
+
fact_id[:12], type(exc).__name__, exc,
|
|
720
|
+
)
|
|
721
|
+
if projected:
|
|
722
|
+
index = getattr(self, "_ann_index", None)
|
|
723
|
+
if index is not None:
|
|
724
|
+
try:
|
|
725
|
+
index.add(fact_id, emb)
|
|
726
|
+
except Exception:
|
|
727
|
+
logger.warning(
|
|
728
|
+
"in-memory index rejected %s", fact_id[:12],
|
|
729
|
+
exc_info=True,
|
|
730
|
+
)
|
|
731
|
+
|
|
732
|
+
if not self._write_canonical_vector(fact_id, emb, fmean, fvar):
|
|
733
|
+
if projected:
|
|
734
|
+
logger.warning(
|
|
735
|
+
"canonical vector write failed for %s — undoing the "
|
|
736
|
+
"projection so the two representations cannot disagree",
|
|
737
|
+
fact_id[:12],
|
|
738
|
+
)
|
|
739
|
+
self._detach_vector(fact_id)
|
|
740
|
+
return False
|
|
741
|
+
return projected
|
|
742
|
+
|
|
743
|
+
def _write_canonical_vector(
|
|
744
|
+
self, fact_id: str, emb: list[float],
|
|
745
|
+
fmean: float | None, fvar: float | None,
|
|
746
|
+
) -> bool:
|
|
747
|
+
"""Write the vector to ``atomic_facts``. False if it did not land."""
|
|
748
|
+
try:
|
|
749
|
+
self._db.update_fact(
|
|
750
|
+
fact_id,
|
|
751
|
+
{"embedding": emb, "fisher_mean": fmean, "fisher_variance": fvar},
|
|
752
|
+
profile_id=self._profile_id,
|
|
753
|
+
)
|
|
754
|
+
return True
|
|
755
|
+
except Exception as exc:
|
|
756
|
+
logger.warning(
|
|
757
|
+
"canonical vector write failed for %s (%s: %s)",
|
|
758
|
+
fact_id[:12], type(exc).__name__, exc,
|
|
759
|
+
)
|
|
760
|
+
return False
|
|
761
|
+
|
|
762
|
+
def _detach_vector(self, fact_id: str) -> None:
|
|
763
|
+
"""Undo a projection whose canonical write did not land.
|
|
764
|
+
|
|
765
|
+
Returns the row to the one state that heals itself: no vector anywhere,
|
|
766
|
+
which is what the repair pass looks for.
|
|
767
|
+
"""
|
|
768
|
+
store = getattr(self, "_vector_store", None)
|
|
769
|
+
delete = getattr(store, "delete", None) if store else None
|
|
770
|
+
if callable(delete):
|
|
771
|
+
try:
|
|
772
|
+
delete(fact_id)
|
|
773
|
+
except Exception:
|
|
774
|
+
logger.warning(
|
|
775
|
+
"could not undo the projection for %s; it and the canonical "
|
|
776
|
+
"column now disagree", fact_id[:12], exc_info=True,
|
|
777
|
+
)
|
|
778
|
+
index = getattr(self, "_ann_index", None)
|
|
779
|
+
remove = getattr(index, "remove", None) if index else None
|
|
780
|
+
if callable(remove):
|
|
781
|
+
try:
|
|
782
|
+
remove(fact_id)
|
|
783
|
+
except Exception:
|
|
784
|
+
logger.warning(
|
|
785
|
+
"could not undo the index entry for %s", fact_id[:12],
|
|
786
|
+
exc_info=True,
|
|
787
|
+
)
|
|
788
|
+
|
|
789
|
+
def enrich_new_facts_now(
|
|
790
|
+
self, fact_ids: list[str], *, timeout_s: float | None = None,
|
|
791
|
+
) -> int:
|
|
792
|
+
"""Make just-written facts searchable by meaning, within a deadline.
|
|
793
|
+
|
|
794
|
+
Returns how many of ``fact_ids`` ARE searchable by meaning when this
|
|
795
|
+
returns — not how many were embedded here. A fact that already had a
|
|
796
|
+
vector counts, because the caller uses this number to tell the user
|
|
797
|
+
whether their memory can be found, and "not embedded by me" is not the
|
|
798
|
+
same statement as "not findable".
|
|
799
|
+
|
|
800
|
+
Every write path in the product — the command line, the tool interface and
|
|
801
|
+
the dashboard — commits through one durable receipt and then relies on a
|
|
802
|
+
background pass for enrichment. Until that pass runs, the memory can only
|
|
803
|
+
be found by quoting its own wording, which is not how anyone asks. This
|
|
804
|
+
closes that window for whichever facts it can reach in time, and reports
|
|
805
|
+
honestly how many it reached.
|
|
806
|
+
|
|
807
|
+
The durable write has already happened before this is called. Nothing here
|
|
808
|
+
can fail it: on any error or timeout the fact simply keeps its place in the
|
|
809
|
+
background queue.
|
|
810
|
+
"""
|
|
811
|
+
if not fact_ids:
|
|
812
|
+
return 0
|
|
813
|
+
# Components are built on demand. Without this the embedder is still
|
|
814
|
+
# None and every fact is silently skipped as "nothing to do" — which
|
|
815
|
+
# looks exactly like "there was nothing to enrich".
|
|
816
|
+
self._require_full("enrich_new_facts_now")
|
|
817
|
+
self._ensure_init()
|
|
818
|
+
# Matches the default the write path uses for its own embedding attempt;
|
|
819
|
+
# the two were allowed to drift apart, so a direct caller got half the
|
|
820
|
+
# budget the daemon gives.
|
|
821
|
+
deadline = time.monotonic() + (timeout_s if timeout_s is not None else 1.0)
|
|
822
|
+
enriched = 0
|
|
823
|
+
for fact_id in fact_ids:
|
|
824
|
+
try:
|
|
825
|
+
fact = self._db.get_fact(fact_id, self._profile_id)
|
|
826
|
+
if fact is None:
|
|
827
|
+
continue
|
|
828
|
+
already = self._projection_has(fact_id)
|
|
829
|
+
if already and getattr(fact, "embedding", None):
|
|
830
|
+
# Findable by meaning, and the two representations agree.
|
|
831
|
+
# Counting it keeps the caller's receipt truthful: saying
|
|
832
|
+
# "wording only" about a fact that is already fully
|
|
833
|
+
# searchable is its own kind of wrong.
|
|
834
|
+
#
|
|
835
|
+
# BOTH are required. A projection with no canonical vector is
|
|
836
|
+
# a fact a search can return while an export, a backup or a
|
|
837
|
+
# direct fetch sees nothing — and treating the projection
|
|
838
|
+
# alone as proof of health means no pass ever repairs it. A
|
|
839
|
+
# canonical vector with no projection is the opposite, and
|
|
840
|
+
# equally unrepaired. Requiring agreement is what makes both
|
|
841
|
+
# states heal instead of persisting silently.
|
|
842
|
+
enriched += 1
|
|
843
|
+
continue
|
|
844
|
+
# Answering "is it already findable" costs one indexed lookup and
|
|
845
|
+
# no model call, so it is done BEFORE the budget check: a caller
|
|
846
|
+
# that is out of time still deserves an honest answer about a
|
|
847
|
+
# fact that needs no work. Only the work below needs a budget.
|
|
848
|
+
remaining = deadline - time.monotonic()
|
|
849
|
+
if remaining <= 0:
|
|
850
|
+
break
|
|
851
|
+
# Re-checked immediately before the writes below as well. The
|
|
852
|
+
# caller awaits this on a thread and stops waiting at its own
|
|
853
|
+
# deadline, but the thread keeps running — so a late write could
|
|
854
|
+
# land while the next request already holds the write lock. Past
|
|
855
|
+
# the deadline this stops writing rather than racing a
|
|
856
|
+
# transaction nobody is waiting for.
|
|
857
|
+
emb = getattr(fact, "embedding", None)
|
|
858
|
+
fmean = getattr(fact, "fisher_mean", None)
|
|
859
|
+
fvar = getattr(fact, "fisher_variance", None)
|
|
860
|
+
if not emb:
|
|
861
|
+
emb, fmean, fvar = self._warm_guard_embed(
|
|
862
|
+
fact.content, timeout_s=remaining,
|
|
863
|
+
)
|
|
864
|
+
if not emb:
|
|
865
|
+
continue
|
|
866
|
+
if time.monotonic() >= deadline:
|
|
867
|
+
# The embed itself consumed the budget; the caller has
|
|
868
|
+
# moved on. Leave this fact to the background pass.
|
|
869
|
+
break
|
|
870
|
+
# The row already holds a vector that never reached the
|
|
871
|
+
# projection: reuse it rather than paying for the model again.
|
|
872
|
+
if time.monotonic() >= deadline:
|
|
873
|
+
# Checked here as well as after the embed, because reusing a
|
|
874
|
+
# vector already on the row skips that check entirely and the
|
|
875
|
+
# fetch above can itself be slow under write contention.
|
|
876
|
+
break
|
|
877
|
+
if self._attach_vector(fact_id, emb, fmean, fvar):
|
|
878
|
+
enriched += 1
|
|
879
|
+
except Exception as exc:
|
|
880
|
+
# Warning, not debug. A run that enriches nothing returns 0 in a
|
|
881
|
+
# millisecond and reads exactly like "there was nothing to do".
|
|
882
|
+
logger.warning(
|
|
883
|
+
"inline enrichment failed for %s (%s: %s) — left for the "
|
|
884
|
+
"background pass", fact_id[:12], type(exc).__name__, exc,
|
|
885
|
+
)
|
|
886
|
+
return enriched
|
|
887
|
+
|
|
538
888
|
def store_fast(
|
|
539
889
|
self, content: str, metadata: dict[str, Any] | None = None,
|
|
540
890
|
*, scope: str = "personal", shared_with: list[str] | None = None,
|
|
@@ -612,78 +962,31 @@ class MemoryEngine:
|
|
|
612
962
|
#
|
|
613
963
|
# 3.8.4 extension: when the embedder is PROVABLY warm (_available is True)
|
|
614
964
|
# AND is a local embedder (not a remote cloud/OpenAI endpoint), compute the
|
|
615
|
-
# embedding synchronously with a hard
|
|
965
|
+
# embedding synchronously with a hard 1s cap. On timeout or any
|
|
616
966
|
# exception, fall through to emb=None — the materializer fills it async.
|
|
617
967
|
# This preserves the 3.8.2 invariant for cold start while eliminating the
|
|
618
968
|
# semantic-channel blind spot on warm daemons (the top UX complaint).
|
|
619
|
-
emb =
|
|
620
|
-
fmean = fvar = None
|
|
621
|
-
_embedder_ref = self._embedder
|
|
622
|
-
if (
|
|
623
|
-
_embedder_ref is not None
|
|
624
|
-
and getattr(_embedder_ref, "_available", None) is True
|
|
625
|
-
and not _is_remote_embedder(_embedder_ref)
|
|
626
|
-
):
|
|
627
|
-
import concurrent.futures as _cf
|
|
628
|
-
# Lazy-init the pool once per engine instance — avoids per-call
|
|
629
|
-
# thread churn and the associated resource leak from discard-on-exit.
|
|
630
|
-
# Double-checked locking guards against TOCTOU on concurrent first calls.
|
|
631
|
-
if self._store_fast_embed_pool is None:
|
|
632
|
-
with self._store_fast_embed_pool_lock:
|
|
633
|
-
if self._store_fast_embed_pool is None:
|
|
634
|
-
self._store_fast_embed_pool = _cf.ThreadPoolExecutor(
|
|
635
|
-
max_workers=1,
|
|
636
|
-
thread_name_prefix="slm-sg-embed",
|
|
637
|
-
)
|
|
638
|
-
try:
|
|
639
|
-
_timeout_s = int(os.environ.get("SLM_STORE_FAST_EMBED_TIMEOUT_MS", 500)) / 1000.0
|
|
640
|
-
except (ValueError, TypeError):
|
|
641
|
-
_timeout_s = 0.5 # default 500 ms
|
|
642
|
-
try:
|
|
643
|
-
def _best_effort_embed():
|
|
644
|
-
from superlocalmemory.core.recall_gate import background_work
|
|
645
|
-
with background_work():
|
|
646
|
-
return _embedder_ref.embed(fact_text)
|
|
647
|
-
|
|
648
|
-
_future = self._store_fast_embed_pool.submit(_best_effort_embed)
|
|
649
|
-
try:
|
|
650
|
-
emb = _future.result(timeout=_timeout_s)
|
|
651
|
-
if emb:
|
|
652
|
-
fmean, fvar = _embedder_ref.compute_fisher_params(emb)
|
|
653
|
-
except _cf.TimeoutError:
|
|
654
|
-
logger.debug(
|
|
655
|
-
"store_fast: warm-guard embed timed out (>%.0fms) — deferring to materializer",
|
|
656
|
-
_timeout_s * 1000,
|
|
657
|
-
)
|
|
658
|
-
emb = None
|
|
659
|
-
except Exception as _exc:
|
|
660
|
-
logger.debug(
|
|
661
|
-
"store_fast: warm-guard embed failed (%s) — deferring to materializer",
|
|
662
|
-
_exc,
|
|
663
|
-
)
|
|
664
|
-
emb = None
|
|
665
|
-
except Exception as _exc:
|
|
666
|
-
logger.debug("store_fast: warm-guard pool submit failed (%s)", _exc)
|
|
667
|
-
emb = None
|
|
969
|
+
emb, fmean, fvar = self._warm_guard_embed(fact_text)
|
|
668
970
|
fact = AtomicFact(
|
|
669
971
|
fact_id=_uuid.uuid4().hex[:16], memory_id=record.memory_id,
|
|
670
972
|
profile_id=self._profile_id, content=fact_text,
|
|
671
973
|
fact_type=FactType.EPISODIC, entities=ents,
|
|
672
974
|
observation_date=session_date or now[:10],
|
|
673
975
|
confidence=0.7, importance=0.5,
|
|
674
|
-
|
|
976
|
+
# The vector is attached after this row exists, not with it. The
|
|
977
|
+
# projection that the meaning-based channel searches has a foreign
|
|
978
|
+
# key onto this row, so it cannot be written first; and writing this
|
|
979
|
+
# column first would hide the fact from the repair pass if the
|
|
980
|
+
# projection then failed. _attach_vector owns that ordering.
|
|
981
|
+
embedding=None, fisher_mean=None, fisher_variance=None,
|
|
675
982
|
created_at=now,
|
|
676
983
|
scope=scope, shared_with=shared_with,
|
|
677
984
|
)
|
|
678
985
|
self._db.store_fact(fact) # FTS5 trigger → immediately BM25-recallable
|
|
679
|
-
#
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
if emb and vs and getattr(vs, "available", False):
|
|
684
|
-
vs.upsert(fact.fact_id, self._profile_id, emb)
|
|
685
|
-
except Exception:
|
|
686
|
-
pass
|
|
986
|
+
# Attach the vector so the meaning-based channel finds it now, through
|
|
987
|
+
# the same ordering every other write path uses.
|
|
988
|
+
if emb and index_external:
|
|
989
|
+
self._attach_vector(fact.fact_id, emb, fmean, fvar)
|
|
687
990
|
# Persist BM25 tokens too (covers the in-memory rank_bm25 fallback path).
|
|
688
991
|
if index_external:
|
|
689
992
|
try:
|
|
@@ -828,8 +1131,13 @@ class MemoryEngine:
|
|
|
828
1131
|
embed_pool_lock = getattr(self, "_store_fast_embed_pool_lock", None)
|
|
829
1132
|
if embed_pool_lock is not None:
|
|
830
1133
|
with embed_pool_lock:
|
|
1134
|
+
# Marked closed under the same lock that guards creation, so a
|
|
1135
|
+
# call already past the unlocked check declines instead of
|
|
1136
|
+
# building an unowned replacement.
|
|
1137
|
+
self._store_fast_embed_pool_closed = True
|
|
831
1138
|
embed_pool, self._store_fast_embed_pool = self._store_fast_embed_pool, None
|
|
832
1139
|
else:
|
|
1140
|
+
self._store_fast_embed_pool_closed = True
|
|
833
1141
|
self._store_fast_embed_pool = None
|
|
834
1142
|
if embed_pool is not None:
|
|
835
1143
|
try:
|
|
@@ -12,6 +12,8 @@ from __future__ import annotations
|
|
|
12
12
|
|
|
13
13
|
import hashlib
|
|
14
14
|
import json
|
|
15
|
+
|
|
16
|
+
from superlocalmemory.storage.embedding_codec import encode_embedding
|
|
15
17
|
import re
|
|
16
18
|
from dataclasses import dataclass
|
|
17
19
|
from pathlib import Path
|
|
@@ -435,7 +437,7 @@ def rebuild_derived_state(
|
|
|
435
437
|
vector = embedder.embed(str(fact["content"]))
|
|
436
438
|
db.execute(
|
|
437
439
|
"UPDATE atomic_facts SET embedding=? WHERE fact_id=? AND profile_id=?",
|
|
438
|
-
(
|
|
440
|
+
(encode_embedding(vector), fact["fact_id"], profile_id),
|
|
439
441
|
)
|
|
440
442
|
embeddings += 1
|
|
441
443
|
return {"bm25_rows": bm25_rows, "embeddings": embeddings}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
|
|
2
|
+
# Licensed under AGPL-3.0-or-later - see LICENSE file
|
|
3
|
+
|
|
4
|
+
"""Detect all SuperLocalMemory installations present on this machine.
|
|
5
|
+
|
|
6
|
+
Supports three install types: pipx, venv (~/.slm-venv), and npm global.
|
|
7
|
+
All detection is read-only and fast (< 200 ms). No writes are ever performed.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import glob
|
|
13
|
+
import json
|
|
14
|
+
import subprocess
|
|
15
|
+
import sys
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Optional
|
|
18
|
+
|
|
19
|
+
# Roots for the two Python install types. Patched in tests.
|
|
20
|
+
_VENV_ROOT: Path = Path.home() / ".slm-venv"
|
|
21
|
+
_PIPX_ROOT: Path = Path.home() / ".local" / "pipx" / "venvs" / "superlocalmemory"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _npm_global_root() -> Optional[Path]:
|
|
25
|
+
"""Return the npm global node_modules root, or None on any failure."""
|
|
26
|
+
try:
|
|
27
|
+
result = subprocess.run(
|
|
28
|
+
["npm", "root", "-g"],
|
|
29
|
+
capture_output=True,
|
|
30
|
+
text=True,
|
|
31
|
+
timeout=5,
|
|
32
|
+
)
|
|
33
|
+
if result.returncode == 0:
|
|
34
|
+
root = result.stdout.strip()
|
|
35
|
+
if root:
|
|
36
|
+
return Path(root)
|
|
37
|
+
except Exception: # npm absent, timeout, etc.
|
|
38
|
+
pass
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _read_python_version(base: Path) -> Optional[str]:
|
|
43
|
+
"""Read __version__ from the first matching site-packages layout under base.
|
|
44
|
+
|
|
45
|
+
Both layouts are searched. POSIX virtualenvs use
|
|
46
|
+
``lib/python3.13/site-packages``; Windows uses ``Lib/site-packages`` with no
|
|
47
|
+
version component and a capitalised directory. Searching only the POSIX
|
|
48
|
+
shape made detection silently return None on Windows — which is precisely
|
|
49
|
+
where multi-install divergence between pip and npm is most likely, and where
|
|
50
|
+
the version-mismatch error would then name no installations at all.
|
|
51
|
+
"""
|
|
52
|
+
patterns = [
|
|
53
|
+
str(base / "lib" / "python*" / "site-packages" / "superlocalmemory" / "__init__.py"),
|
|
54
|
+
str(base / "Lib" / "site-packages" / "superlocalmemory" / "__init__.py"),
|
|
55
|
+
str(base / "lib" / "site-packages" / "superlocalmemory" / "__init__.py"),
|
|
56
|
+
]
|
|
57
|
+
matches: list[str] = []
|
|
58
|
+
for pattern in patterns:
|
|
59
|
+
matches.extend(glob.glob(pattern))
|
|
60
|
+
for init_path in sorted(set(matches)):
|
|
61
|
+
try:
|
|
62
|
+
text = Path(init_path).read_text(encoding="utf-8", errors="replace")
|
|
63
|
+
for line in text.splitlines():
|
|
64
|
+
line = line.strip()
|
|
65
|
+
if line.startswith("__version__"):
|
|
66
|
+
# __version__ = "4.1.0" or __version__ = '4.1.0'
|
|
67
|
+
parts = line.split("=", 1)
|
|
68
|
+
if len(parts) == 2:
|
|
69
|
+
return parts[1].strip().strip("\"'")
|
|
70
|
+
except OSError:
|
|
71
|
+
continue
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _read_npm_version(npm_root: Path) -> Optional[str]:
|
|
76
|
+
"""Read version from npm global package.json."""
|
|
77
|
+
pkg_json = npm_root / "superlocalmemory" / "package.json"
|
|
78
|
+
try:
|
|
79
|
+
data = json.loads(pkg_json.read_text(encoding="utf-8"))
|
|
80
|
+
return str(data.get("version", "")).strip() or None
|
|
81
|
+
except (OSError, json.JSONDecodeError):
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _detect_all_installs() -> list[dict]:
|
|
86
|
+
"""Return all SuperLocalMemory installs detected on this machine.
|
|
87
|
+
|
|
88
|
+
Each entry is a dict with keys:
|
|
89
|
+
- ``path`` (str) — directory of the install
|
|
90
|
+
- ``version`` (str) — version string read from package metadata
|
|
91
|
+
- ``type`` (str) — one of "pipx", "venv", "npm"
|
|
92
|
+
|
|
93
|
+
Detection is read-only and best-effort. A missing or unreadable install
|
|
94
|
+
produces no entry rather than an error. Subprocess calls are bounded to
|
|
95
|
+
5 seconds total.
|
|
96
|
+
"""
|
|
97
|
+
results: list[dict] = []
|
|
98
|
+
|
|
99
|
+
# --- pipx ---
|
|
100
|
+
pipx_version = _read_python_version(_PIPX_ROOT)
|
|
101
|
+
if pipx_version is not None:
|
|
102
|
+
results.append({
|
|
103
|
+
"path": str(_PIPX_ROOT) + "/",
|
|
104
|
+
"version": pipx_version,
|
|
105
|
+
"type": "pipx",
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
# --- ~/.slm-venv ---
|
|
109
|
+
venv_version = _read_python_version(_VENV_ROOT)
|
|
110
|
+
if venv_version is not None:
|
|
111
|
+
results.append({
|
|
112
|
+
"path": str(_VENV_ROOT) + "/",
|
|
113
|
+
"version": venv_version,
|
|
114
|
+
"type": "venv",
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
# --- npm global ---
|
|
118
|
+
npm_root = _npm_global_root()
|
|
119
|
+
if npm_root is not None:
|
|
120
|
+
npm_version = _read_npm_version(npm_root)
|
|
121
|
+
if npm_version is not None:
|
|
122
|
+
results.append({
|
|
123
|
+
"path": str(npm_root / "superlocalmemory") + "/",
|
|
124
|
+
"version": npm_version,
|
|
125
|
+
"type": "npm",
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
return results
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
__all__ = ["_detect_all_installs"]
|
|
@@ -16,7 +16,7 @@ The persona is one bounded roll-up per profile that consumes the top community
|
|
|
16
16
|
summaries. It is recall-GATED (never auto-injected into the hot recall path —
|
|
17
17
|
avoids the V3.4.40 summary-pollution regression) and SIZE-bounded. Drill-down
|
|
18
18
|
(``get_sources``) walks the hierarchy back down to the source atoms, matching
|
|
19
|
-
the
|
|
19
|
+
the design goal for summary->source provenance (Zep-style attribution).
|
|
20
20
|
|
|
21
21
|
Runs in the background consolidation lane after community summaries.
|
|
22
22
|
Fail-open throughout; recompute replaces a profile's row.
|
|
@@ -106,6 +106,10 @@ def _handle_recall(
|
|
|
106
106
|
memory_map=memory_map,
|
|
107
107
|
per_fact_max=getattr(_rc, "recall_per_fact_max_chars", 2400),
|
|
108
108
|
total_max=getattr(_rc, "recall_total_max_chars", 12000),
|
|
109
|
+
# Option B: markers only on session-bearing recalls. A marker can
|
|
110
|
+
# only buy a learning signal when a pending_outcomes row exists
|
|
111
|
+
# to settle, and those exist only when session_id is present.
|
|
112
|
+
include_marker=bool(session_id),
|
|
109
113
|
)
|
|
110
114
|
return {
|
|
111
115
|
"ok": True,
|