superlocalmemory 4.0.7 → 4.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/CHANGELOG.md +219 -1
  2. package/README.md +6 -6
  3. package/package.json +1 -1
  4. package/plugin/.claude-plugin/plugin.json +1 -1
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +1 -1
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/scripts/ensure-venv.sh +1 -1
  12. package/plugin/skills/slm-cache/SKILL.md +1 -1
  13. package/plugin/skills/slm-compress/SKILL.md +1 -1
  14. package/plugin/skills/slm-governance/SKILL.md +1 -1
  15. package/plugin/skills/slm-graph/SKILL.md +1 -1
  16. package/plugin/skills/slm-loop/SKILL.md +1 -1
  17. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  18. package/plugin/skills/slm-profile/SKILL.md +1 -1
  19. package/plugin/skills/slm-recall/SKILL.md +1 -1
  20. package/plugin/skills/slm-remember/SKILL.md +1 -1
  21. package/plugin/skills/slm-scope/SKILL.md +1 -1
  22. package/plugin/skills/slm-session/SKILL.md +1 -1
  23. package/plugin/skills/slm-status/SKILL.md +3 -3
  24. package/plugin-src/rules/AGENTS.md +1 -1
  25. package/plugin-src/skills/slm-status/SKILL.md +2 -2
  26. package/pyproject.toml +1 -1
  27. package/scripts/postinstall.js +4 -0
  28. package/src/superlocalmemory/__init__.py +1 -1
  29. package/src/superlocalmemory/cli/_lazy_init.py +1 -1
  30. package/src/superlocalmemory/cli/commands.py +119 -9
  31. package/src/superlocalmemory/cli/db_migrate.py +0 -2
  32. package/src/superlocalmemory/cli/gdpr_io.py +1 -1
  33. package/src/superlocalmemory/cli/main.py +5 -5
  34. package/src/superlocalmemory/cli/service_installer.py +2 -1
  35. package/src/superlocalmemory/cli/setup_wizard.py +1 -1
  36. package/src/superlocalmemory/cli/summary_cmd.py +23 -3
  37. package/src/superlocalmemory/code_graph/bridge/maintenance.py +7 -1
  38. package/src/superlocalmemory/core/config.py +41 -7
  39. package/src/superlocalmemory/core/consolidation_engine.py +14 -15
  40. package/src/superlocalmemory/core/context_cache.py +0 -2
  41. package/src/superlocalmemory/core/engine.py +371 -63
  42. package/src/superlocalmemory/core/evidence_bundle.py +3 -1
  43. package/src/superlocalmemory/core/install_detector.py +131 -0
  44. package/src/superlocalmemory/core/progressive_abstraction.py +1 -1
  45. package/src/superlocalmemory/core/recall_worker.py +4 -0
  46. package/src/superlocalmemory/core/security_primitives.py +3 -6
  47. package/src/superlocalmemory/core/store_pipeline.py +94 -26
  48. package/src/superlocalmemory/core/topic_signature.py +0 -2
  49. package/src/superlocalmemory/core/transactions/concrete_owners.py +15 -8
  50. package/src/superlocalmemory/dynamics/eap_scheduler.py +17 -6
  51. package/src/superlocalmemory/encoding/graph_builder.py +2 -2
  52. package/src/superlocalmemory/encoding/scene_builder.py +8 -2
  53. package/src/superlocalmemory/evolution/skill_evolver.py +16 -1
  54. package/src/superlocalmemory/hooks/adapter_base.py +0 -2
  55. package/src/superlocalmemory/hooks/context_payload.py +0 -2
  56. package/src/superlocalmemory/hooks/hook_handlers.py +38 -11
  57. package/src/superlocalmemory/hooks/portable_kit.py +8 -8
  58. package/src/superlocalmemory/hooks/post_tool_async_hook.py +0 -2
  59. package/src/superlocalmemory/hooks/prewarm_auth.py +0 -2
  60. package/src/superlocalmemory/hooks/user_prompt_hook.py +0 -2
  61. package/src/superlocalmemory/infra/backup.py +44 -8
  62. package/src/superlocalmemory/integrations/bounded_loops_mcp.py +24 -7
  63. package/src/superlocalmemory/learning/arm_catalog.py +0 -2
  64. package/src/superlocalmemory/learning/bandit.py +0 -2
  65. package/src/superlocalmemory/learning/bandit_cache.py +0 -2
  66. package/src/superlocalmemory/learning/dedup_hnsw.py +11 -11
  67. package/src/superlocalmemory/learning/ensemble.py +0 -2
  68. package/src/superlocalmemory/learning/labeler.py +0 -2
  69. package/src/superlocalmemory/learning/legacy_migration.py +0 -2
  70. package/src/superlocalmemory/learning/model_cache.py +0 -2
  71. package/src/superlocalmemory/learning/pattern_miner.py +12 -7
  72. package/src/superlocalmemory/learning/ranker.py +0 -2
  73. package/src/superlocalmemory/learning/reward_archive.py +6 -1
  74. package/src/superlocalmemory/learning/reward_proxy.py +0 -2
  75. package/src/superlocalmemory/learning/signal_worker.py +0 -2
  76. package/src/superlocalmemory/math/fisher.py +1 -1
  77. package/src/superlocalmemory/math/hopfield.py +4 -1
  78. package/src/superlocalmemory/math/langevin.py +1 -1
  79. package/src/superlocalmemory/math/sheaf.py +7 -3
  80. package/src/superlocalmemory/mcp/cli_fallback.py +1 -1
  81. package/src/superlocalmemory/mcp/profiles.py +11 -4
  82. package/src/superlocalmemory/mcp/server.py +8 -1
  83. package/src/superlocalmemory/mcp/tools_active.py +56 -0
  84. package/src/superlocalmemory/mcp/tools_core.py +1 -1
  85. package/src/superlocalmemory/mcp/tools_summaries.py +147 -0
  86. package/src/superlocalmemory/optimize/cache/manager.py +2 -2
  87. package/src/superlocalmemory/optimize/compress/ccr.py +1 -1
  88. package/src/superlocalmemory/optimize/compress/router.py +1 -1
  89. package/src/superlocalmemory/optimize/proxy/_helpers.py +2 -2
  90. package/src/superlocalmemory/optimize/proxy/server.py +1 -1
  91. package/src/superlocalmemory/optimize/proxy/vertex_surface.py +2 -2
  92. package/src/superlocalmemory/optimize/storage/db.py +2 -2
  93. package/src/superlocalmemory/retrieval/agentic.py +1 -1
  94. package/src/superlocalmemory/retrieval/ann_index.py +9 -2
  95. package/src/superlocalmemory/retrieval/bm25_channel.py +2 -2
  96. package/src/superlocalmemory/retrieval/bridge_discovery.py +2 -2
  97. package/src/superlocalmemory/retrieval/engine.py +272 -43
  98. package/src/superlocalmemory/retrieval/entity_channel.py +1 -1
  99. package/src/superlocalmemory/retrieval/hopfield_channel.py +8 -2
  100. package/src/superlocalmemory/retrieval/profile_channel.py +1 -1
  101. package/src/superlocalmemory/retrieval/quantization_aware_search.py +1 -1
  102. package/src/superlocalmemory/retrieval/remote_reranker.py +2 -2
  103. package/src/superlocalmemory/retrieval/reranker.py +3 -3
  104. package/src/superlocalmemory/retrieval/semantic_channel.py +3 -3
  105. package/src/superlocalmemory/retrieval/spreading_activation.py +8 -8
  106. package/src/superlocalmemory/retrieval/strategy.py +94 -0
  107. package/src/superlocalmemory/retrieval/temporal_channel.py +167 -10
  108. package/src/superlocalmemory/retrieval/temporal_validity_filter.py +1 -1
  109. package/src/superlocalmemory/retrieval/vector_store.py +88 -10
  110. package/src/superlocalmemory/server/consolidation_runner.py +140 -0
  111. package/src/superlocalmemory/server/recall_serializer.py +44 -2
  112. package/src/superlocalmemory/server/routes/agents.py +52 -8
  113. package/src/superlocalmemory/server/routes/brain.py +110 -2
  114. package/src/superlocalmemory/server/routes/memories.py +153 -0
  115. package/src/superlocalmemory/server/routes/prewarm.py +4 -4
  116. package/src/superlocalmemory/server/routes/v3_api.py +24 -46
  117. package/src/superlocalmemory/server/unified_daemon.py +566 -7
  118. package/src/superlocalmemory/storage/_schema_version.py +46 -3
  119. package/src/superlocalmemory/storage/backup.py +531 -0
  120. package/src/superlocalmemory/storage/database.py +11 -4
  121. package/src/superlocalmemory/storage/embedding_codec.py +129 -0
  122. package/src/superlocalmemory/storage/embedding_migrator.py +5 -3
  123. package/src/superlocalmemory/storage/migration_runner.py +142 -2
  124. package/src/superlocalmemory/storage/migrations/__init__.py +1 -1
  125. package/src/superlocalmemory/storage/migrations.py +15 -1
  126. package/src/superlocalmemory/storage/models.py +7 -0
  127. package/src/superlocalmemory/storage/quantized_store.py +4 -2
  128. package/src/superlocalmemory/summaries/base.py +159 -0
  129. package/src/superlocalmemory/summaries/daily_reflection.py +55 -8
  130. package/src/superlocalmemory/summaries/project_work_log.py +23 -7
  131. package/src/superlocalmemory/summaries/session_summary.py +10 -6
  132. package/src/superlocalmemory/ui/css/legacy-dashboard.css +1 -1
  133. package/src/superlocalmemory/ui/css/neural-glass.css +1 -1
  134. package/src/superlocalmemory/ui/index.html +9 -3
  135. package/src/superlocalmemory/ui/js/core.js +1 -1
  136. package/src/superlocalmemory/ui/js/od-boundedloops.js +324 -0
  137. package/src/superlocalmemory/ui/js/od-brain.js +1 -1
  138. package/src/superlocalmemory/ui/js/od-memories.js +337 -12
  139. package/src/superlocalmemory/ui/js/od-mesh.js +97 -5
  140. package/src/superlocalmemory/ui/js/od-operations.js +1 -150
  141. package/src/superlocalmemory/ui/js/od-optimize.js +36 -9
  142. package/src/superlocalmemory/ui/js/od-shell.js +10 -0
@@ -20,6 +20,7 @@ from __future__ import annotations
20
20
  import logging
21
21
  import os
22
22
  import threading
23
+ import time
23
24
  from pathlib import Path
24
25
  from typing import Any
25
26
 
@@ -49,7 +50,7 @@ def _verify_ingestion_schema(memory_db: Path) -> bool:
49
50
 
50
51
 
51
52
  # ---------------------------------------------------------------------------
52
- # Workstream D (3.8.4) warm-guard sync embed helpers
53
+ # Embedding a memory on the write path, before the receipt is returned
53
54
  # ---------------------------------------------------------------------------
54
55
 
55
56
  def _is_remote_embedder(embedder: object) -> bool:
@@ -132,11 +133,18 @@ class MemoryEngine:
132
133
  self._consolidation_engine = None
133
134
  self._maintenance_scheduler = None
134
135
  self._hooks = HookRegistry()
135
- # Workstream D (3.8.4): single-worker pool reused across store_fast() calls.
136
- # Lazy-created on first warm-guard attempt; avoids per-call thread churn.
136
+ # One single-worker pool reused across store_fast() calls, created on the
137
+ # first warm-guard attempt. Reusing it avoids a thread per call.
137
138
  self._store_fast_embed_pool: object | None = None
138
- # Lock guards the lazy-init to prevent TOCTOU race on concurrent first calls.
139
+ # Guards the lazy creation so concurrent first calls cannot each build one.
139
140
  self._store_fast_embed_pool_lock = threading.Lock()
141
+ # Set once the engine is closed, and never cleared. A caller already
142
+ # inside the warm-guard path when close() runs will reach the creation
143
+ # check afterwards and find None; without this it would build a
144
+ # replacement pool that nothing owns and nothing will shut down, so a
145
+ # burst of writes arriving during shutdown leaks a thread each and holds
146
+ # the process open past its shutdown budget.
147
+ self._store_fast_embed_pool_closed = False
140
148
 
141
149
  # -- Public properties (Phase 2+ access) --------------------------------
142
150
 
@@ -535,6 +543,348 @@ class MemoryEngine:
535
543
  trusted_actor_id=local_trusted_actor_id("python-api-prebuilt"),
536
544
  )
537
545
 
546
+ def _warm_guard_embed(
547
+ self, text: str, *, timeout_s: float | None = None,
548
+ ) -> tuple[list[float] | None, Any, Any]:
549
+ """Embed now if it is cheap and safe to, otherwise leave it to the materializer.
550
+
551
+ A fact with no vector is invisible to the semantic channel, so a memory
552
+ written moments ago is the hardest thing in the store to find. Computing
553
+ it inline closes that window — but only when doing so cannot hurt anyone
554
+ else, because this daemon serves many sessions at once.
555
+
556
+ Four constraints, each load-bearing:
557
+
558
+ * only when the embedder is provably warm and local, so a cold start or a
559
+ network round-trip can never be paid on a write;
560
+ * on one shared single-worker pool, so N concurrent writers cannot spawn
561
+ N threads — the sixth caller simply misses its deadline and defers;
562
+ * inside ``background_work()``, so a foreground recall preempts it rather
563
+ than queueing behind it;
564
+ * behind a hard deadline, after which the caller proceeds with no
565
+ embedding and the background materializer finishes the job.
566
+
567
+ Returns ``(embedding, fisher_mean, fisher_variance)``, any of which may be
568
+ ``None``. Never raises: every failure means "not now", not "write failed".
569
+ """
570
+ emb = None
571
+ fmean = fvar = None
572
+ _embedder_ref = self._embedder
573
+ if (
574
+ _embedder_ref is None
575
+ or getattr(_embedder_ref, "_available", None) is not True
576
+ or _is_remote_embedder(_embedder_ref)
577
+ ):
578
+ return None, None, None
579
+
580
+ import concurrent.futures as _cf
581
+
582
+ # Lazy-init once per engine — avoids per-call thread churn. Double-checked
583
+ # locking guards against TOCTOU on concurrent first calls.
584
+ if self._store_fast_embed_pool is None:
585
+ with self._store_fast_embed_pool_lock:
586
+ if self._store_fast_embed_pool_closed:
587
+ # Shutdown happened while this call was between the check
588
+ # above and this lock. Decline rather than resurrect the
589
+ # pool: the fact is already committed durably and keeps its
590
+ # place in the background queue.
591
+ return None, None, None
592
+ if self._store_fast_embed_pool is None:
593
+ self._store_fast_embed_pool = _cf.ThreadPoolExecutor(
594
+ max_workers=1,
595
+ thread_name_prefix="slm-sg-embed",
596
+ )
597
+ # A ceiling, not a target. Storing a memory is allowed up to 1.5 s in
598
+ # total, and giving up on the embedding early is not free: it is the
599
+ # difference between a memory that can be found by asking a question and
600
+ # one that can only be found by quoting its own words. Within the budget
601
+ # the more complete outcome wins, so this waits a full second rather than
602
+ # half of one. Both values below are milliseconds via the environment.
603
+ if timeout_s is None:
604
+ try:
605
+ timeout_s = int(os.environ.get("SLM_STORE_FAST_EMBED_TIMEOUT_MS", 1000)) / 1000.0
606
+ except (ValueError, TypeError):
607
+ timeout_s = 1.0
608
+ try:
609
+ def _best_effort_embed():
610
+ from superlocalmemory.core.recall_gate import background_work
611
+ with background_work():
612
+ return _embedder_ref.embed(text)
613
+
614
+ _future = self._store_fast_embed_pool.submit(_best_effort_embed)
615
+ try:
616
+ emb = _future.result(timeout=timeout_s)
617
+ if emb:
618
+ fmean, fvar = _embedder_ref.compute_fisher_params(emb)
619
+ except _cf.TimeoutError:
620
+ # Cancel rather than abandon. This pool has a single worker, so a
621
+ # task left running holds it for every caller behind us: each in
622
+ # turn times out and queues another task that runs after its
623
+ # caller has already answered. Cancel only stops it if it has not
624
+ # started, which is exactly the queued backlog we care about.
625
+ _future.cancel()
626
+ logger.debug(
627
+ "warm-guard embed timed out (>%.0fms) — deferring to materializer",
628
+ timeout_s * 1000,
629
+ )
630
+ emb = None
631
+ except Exception as _exc:
632
+ logger.debug("warm-guard embed failed (%s) — deferring to materializer", _exc)
633
+ emb = None
634
+ except Exception as _exc:
635
+ logger.debug("warm-guard pool submit failed (%s)", _exc)
636
+ emb = None
637
+ return emb, fmean, fvar
638
+
639
+ def _projection_has(self, fact_id: str) -> bool:
640
+ """Can the meaning-based channel reach this fact?
641
+
642
+ Answered by the vector store, which owns the question, and never from
643
+ ``atomic_facts.embedding``. The column and the projection can disagree,
644
+ and when they do it is the projection that decides whether a search
645
+ finds the fact. Reachability also needs more than a stored vector: the
646
+ search joins two projection tables, so a vector present in one of them
647
+ and absent from the other is not reachable. That predicate belongs to
648
+ the store; duplicating it here is how the column and the projection
649
+ drifted apart in the first place.
650
+
651
+ Fails closed. A store that cannot answer is treated as "not reachable",
652
+ which costs at most one redundant idempotent write and can never report
653
+ a fact as findable when it is not.
654
+ """
655
+ store = getattr(self, "_vector_store", None)
656
+ answer = getattr(store, "is_searchable_by_meaning", None) if store else None
657
+ if not callable(answer):
658
+ return False
659
+ try:
660
+ return bool(answer(fact_id, self._profile_id))
661
+ except Exception:
662
+ return False
663
+
664
+ def _attach_vector(
665
+ self, fact_id: str, emb: list[float],
666
+ fmean: float | None, fvar: float | None,
667
+ ) -> bool:
668
+ """Attach a vector to a stored fact. True only if a search can now find it.
669
+
670
+ Two representations are involved and they answer different questions. The
671
+ vector projection is what a search on meaning queries, so it decides
672
+ whether the fact is *findable*. The ``atomic_facts.embedding`` column is
673
+ where the vector *lives* — it is what an export, a backup or a direct
674
+ fetch reads, and what a re-projection would be rebuilt from.
675
+
676
+ So:
677
+
678
+ * **The projection is attempted first**, because that is the ordering that
679
+ keeps the two consistent when both can succeed.
680
+ * **The column is written either way.** The vector has already been
681
+ computed and it is real data; withholding it because the projection was
682
+ refused would discard it and leave the repair pass paying for the model
683
+ again on every pass. Every reason a projection is refused — no search
684
+ extension on this platform, a dimension change, a store at a different
685
+ dimension — is a property of the installation and not of this fact, so
686
+ withholding one fact's column repairs nothing.
687
+ * **The return value is the honest answer to "can this be found by
688
+ meaning", and it is False whenever the projection did not accept the
689
+ vector.** That is what keeps a receipt from claiming a memory is
690
+ searchable when no search can reach it, which was the actual harm in
691
+ writing these two out of order — not the column itself.
692
+ * **If the projection succeeded and the column write then failed, the
693
+ projection is rolled back.** Otherwise a search finds a fact whose
694
+ vector nothing else can see, and nothing would ever reconcile them.
695
+
696
+ Never raises. The durable write has already happened before this runs; a
697
+ fact that is not projected keeps its place in the background queue.
698
+
699
+ Once started, the pair runs to completion and is not interrupted by a
700
+ caller deadline. Abandoning it half-way is what produces the disagreeing
701
+ states above, so a deadline can decide whether to *begin* the pair, never
702
+ whether to finish it. Callers check their budget immediately before
703
+ calling. Bounding the pair itself is a property of the write lock.
704
+ """
705
+ projected = False
706
+ store = getattr(self, "_vector_store", None)
707
+ if store is not None and getattr(store, "available", False):
708
+ try:
709
+ projected = bool(store.upsert(fact_id, self._profile_id, emb))
710
+ if not projected:
711
+ logger.warning(
712
+ "vector projection refused for %s — stored but not "
713
+ "findable by meaning", fact_id[:12],
714
+ )
715
+ except Exception as exc:
716
+ logger.warning(
717
+ "vector projection failed for %s (%s: %s) — stored but not "
718
+ "findable by meaning",
719
+ fact_id[:12], type(exc).__name__, exc,
720
+ )
721
+ if projected:
722
+ index = getattr(self, "_ann_index", None)
723
+ if index is not None:
724
+ try:
725
+ index.add(fact_id, emb)
726
+ except Exception:
727
+ logger.warning(
728
+ "in-memory index rejected %s", fact_id[:12],
729
+ exc_info=True,
730
+ )
731
+
732
+ if not self._write_canonical_vector(fact_id, emb, fmean, fvar):
733
+ if projected:
734
+ logger.warning(
735
+ "canonical vector write failed for %s — undoing the "
736
+ "projection so the two representations cannot disagree",
737
+ fact_id[:12],
738
+ )
739
+ self._detach_vector(fact_id)
740
+ return False
741
+ return projected
742
+
743
+ def _write_canonical_vector(
744
+ self, fact_id: str, emb: list[float],
745
+ fmean: float | None, fvar: float | None,
746
+ ) -> bool:
747
+ """Write the vector to ``atomic_facts``. False if it did not land."""
748
+ try:
749
+ self._db.update_fact(
750
+ fact_id,
751
+ {"embedding": emb, "fisher_mean": fmean, "fisher_variance": fvar},
752
+ profile_id=self._profile_id,
753
+ )
754
+ return True
755
+ except Exception as exc:
756
+ logger.warning(
757
+ "canonical vector write failed for %s (%s: %s)",
758
+ fact_id[:12], type(exc).__name__, exc,
759
+ )
760
+ return False
761
+
762
+ def _detach_vector(self, fact_id: str) -> None:
763
+ """Undo a projection whose canonical write did not land.
764
+
765
+ Returns the row to the one state that heals itself: no vector anywhere,
766
+ which is what the repair pass looks for.
767
+ """
768
+ store = getattr(self, "_vector_store", None)
769
+ delete = getattr(store, "delete", None) if store else None
770
+ if callable(delete):
771
+ try:
772
+ delete(fact_id)
773
+ except Exception:
774
+ logger.warning(
775
+ "could not undo the projection for %s; it and the canonical "
776
+ "column now disagree", fact_id[:12], exc_info=True,
777
+ )
778
+ index = getattr(self, "_ann_index", None)
779
+ remove = getattr(index, "remove", None) if index else None
780
+ if callable(remove):
781
+ try:
782
+ remove(fact_id)
783
+ except Exception:
784
+ logger.warning(
785
+ "could not undo the index entry for %s", fact_id[:12],
786
+ exc_info=True,
787
+ )
788
+
789
+ def enrich_new_facts_now(
790
+ self, fact_ids: list[str], *, timeout_s: float | None = None,
791
+ ) -> int:
792
+ """Make just-written facts searchable by meaning, within a deadline.
793
+
794
+ Returns how many of ``fact_ids`` ARE searchable by meaning when this
795
+ returns — not how many were embedded here. A fact that already had a
796
+ vector counts, because the caller uses this number to tell the user
797
+ whether their memory can be found, and "not embedded by me" is not the
798
+ same statement as "not findable".
799
+
800
+ Every write path in the product — the command line, the tool interface and
801
+ the dashboard — commits through one durable receipt and then relies on a
802
+ background pass for enrichment. Until that pass runs, the memory can only
803
+ be found by quoting its own wording, which is not how anyone asks. This
804
+ closes that window for whichever facts it can reach in time, and reports
805
+ honestly how many it reached.
806
+
807
+ The durable write has already happened before this is called. Nothing here
808
+ can fail it: on any error or timeout the fact simply keeps its place in the
809
+ background queue.
810
+ """
811
+ if not fact_ids:
812
+ return 0
813
+ # Components are built on demand. Without this the embedder is still
814
+ # None and every fact is silently skipped as "nothing to do" — which
815
+ # looks exactly like "there was nothing to enrich".
816
+ self._require_full("enrich_new_facts_now")
817
+ self._ensure_init()
818
+ # Matches the default the write path uses for its own embedding attempt;
819
+ # the two were allowed to drift apart, so a direct caller got half the
820
+ # budget the daemon gives.
821
+ deadline = time.monotonic() + (timeout_s if timeout_s is not None else 1.0)
822
+ enriched = 0
823
+ for fact_id in fact_ids:
824
+ try:
825
+ fact = self._db.get_fact(fact_id, self._profile_id)
826
+ if fact is None:
827
+ continue
828
+ already = self._projection_has(fact_id)
829
+ if already and getattr(fact, "embedding", None):
830
+ # Findable by meaning, and the two representations agree.
831
+ # Counting it keeps the caller's receipt truthful: saying
832
+ # "wording only" about a fact that is already fully
833
+ # searchable is its own kind of wrong.
834
+ #
835
+ # BOTH are required. A projection with no canonical vector is
836
+ # a fact a search can return while an export, a backup or a
837
+ # direct fetch sees nothing — and treating the projection
838
+ # alone as proof of health means no pass ever repairs it. A
839
+ # canonical vector with no projection is the opposite, and
840
+ # equally unrepaired. Requiring agreement is what makes both
841
+ # states heal instead of persisting silently.
842
+ enriched += 1
843
+ continue
844
+ # Answering "is it already findable" costs one indexed lookup and
845
+ # no model call, so it is done BEFORE the budget check: a caller
846
+ # that is out of time still deserves an honest answer about a
847
+ # fact that needs no work. Only the work below needs a budget.
848
+ remaining = deadline - time.monotonic()
849
+ if remaining <= 0:
850
+ break
851
+ # Re-checked immediately before the writes below as well. The
852
+ # caller awaits this on a thread and stops waiting at its own
853
+ # deadline, but the thread keeps running — so a late write could
854
+ # land while the next request already holds the write lock. Past
855
+ # the deadline this stops writing rather than racing a
856
+ # transaction nobody is waiting for.
857
+ emb = getattr(fact, "embedding", None)
858
+ fmean = getattr(fact, "fisher_mean", None)
859
+ fvar = getattr(fact, "fisher_variance", None)
860
+ if not emb:
861
+ emb, fmean, fvar = self._warm_guard_embed(
862
+ fact.content, timeout_s=remaining,
863
+ )
864
+ if not emb:
865
+ continue
866
+ if time.monotonic() >= deadline:
867
+ # The embed itself consumed the budget; the caller has
868
+ # moved on. Leave this fact to the background pass.
869
+ break
870
+ # The row already holds a vector that never reached the
871
+ # projection: reuse it rather than paying for the model again.
872
+ if time.monotonic() >= deadline:
873
+ # Checked here as well as after the embed, because reusing a
874
+ # vector already on the row skips that check entirely and the
875
+ # fetch above can itself be slow under write contention.
876
+ break
877
+ if self._attach_vector(fact_id, emb, fmean, fvar):
878
+ enriched += 1
879
+ except Exception as exc:
880
+ # Warning, not debug. A run that enriches nothing returns 0 in a
881
+ # millisecond and reads exactly like "there was nothing to do".
882
+ logger.warning(
883
+ "inline enrichment failed for %s (%s: %s) — left for the "
884
+ "background pass", fact_id[:12], type(exc).__name__, exc,
885
+ )
886
+ return enriched
887
+
538
888
  def store_fast(
539
889
  self, content: str, metadata: dict[str, Any] | None = None,
540
890
  *, scope: str = "personal", shared_with: list[str] | None = None,
@@ -612,78 +962,31 @@ class MemoryEngine:
612
962
  #
613
963
  # 3.8.4 extension: when the embedder is PROVABLY warm (_available is True)
614
964
  # AND is a local embedder (not a remote cloud/OpenAI endpoint), compute the
615
- # embedding synchronously with a hard 500ms cap. On timeout or any
965
+ # embedding synchronously with a hard 1s cap. On timeout or any
616
966
  # exception, fall through to emb=None — the materializer fills it async.
617
967
  # This preserves the 3.8.2 invariant for cold start while eliminating the
618
968
  # semantic-channel blind spot on warm daemons (the top UX complaint).
619
- emb = None
620
- fmean = fvar = None
621
- _embedder_ref = self._embedder
622
- if (
623
- _embedder_ref is not None
624
- and getattr(_embedder_ref, "_available", None) is True
625
- and not _is_remote_embedder(_embedder_ref)
626
- ):
627
- import concurrent.futures as _cf
628
- # Lazy-init the pool once per engine instance — avoids per-call
629
- # thread churn and the associated resource leak from discard-on-exit.
630
- # Double-checked locking guards against TOCTOU on concurrent first calls.
631
- if self._store_fast_embed_pool is None:
632
- with self._store_fast_embed_pool_lock:
633
- if self._store_fast_embed_pool is None:
634
- self._store_fast_embed_pool = _cf.ThreadPoolExecutor(
635
- max_workers=1,
636
- thread_name_prefix="slm-sg-embed",
637
- )
638
- try:
639
- _timeout_s = int(os.environ.get("SLM_STORE_FAST_EMBED_TIMEOUT_MS", 500)) / 1000.0
640
- except (ValueError, TypeError):
641
- _timeout_s = 0.5 # default 500 ms
642
- try:
643
- def _best_effort_embed():
644
- from superlocalmemory.core.recall_gate import background_work
645
- with background_work():
646
- return _embedder_ref.embed(fact_text)
647
-
648
- _future = self._store_fast_embed_pool.submit(_best_effort_embed)
649
- try:
650
- emb = _future.result(timeout=_timeout_s)
651
- if emb:
652
- fmean, fvar = _embedder_ref.compute_fisher_params(emb)
653
- except _cf.TimeoutError:
654
- logger.debug(
655
- "store_fast: warm-guard embed timed out (>%.0fms) — deferring to materializer",
656
- _timeout_s * 1000,
657
- )
658
- emb = None
659
- except Exception as _exc:
660
- logger.debug(
661
- "store_fast: warm-guard embed failed (%s) — deferring to materializer",
662
- _exc,
663
- )
664
- emb = None
665
- except Exception as _exc:
666
- logger.debug("store_fast: warm-guard pool submit failed (%s)", _exc)
667
- emb = None
969
+ emb, fmean, fvar = self._warm_guard_embed(fact_text)
668
970
  fact = AtomicFact(
669
971
  fact_id=_uuid.uuid4().hex[:16], memory_id=record.memory_id,
670
972
  profile_id=self._profile_id, content=fact_text,
671
973
  fact_type=FactType.EPISODIC, entities=ents,
672
974
  observation_date=session_date or now[:10],
673
975
  confidence=0.7, importance=0.5,
674
- embedding=emb, fisher_mean=fmean, fisher_variance=fvar,
976
+ # The vector is attached after this row exists, not with it. The
977
+ # projection that the meaning-based channel searches has a foreign
978
+ # key onto this row, so it cannot be written first; and writing this
979
+ # column first would hide the fact from the repair pass if the
980
+ # projection then failed. _attach_vector owns that ordering.
981
+ embedding=None, fisher_mean=None, fisher_variance=None,
675
982
  created_at=now,
676
983
  scope=scope, shared_with=shared_with,
677
984
  )
678
985
  self._db.store_fact(fact) # FTS5 trigger → immediately BM25-recallable
679
- # Upsert to vector store so the semantic channel finds it now.
680
- if index_external:
681
- try:
682
- vs = getattr(self, "_vector_store", None)
683
- if emb and vs and getattr(vs, "available", False):
684
- vs.upsert(fact.fact_id, self._profile_id, emb)
685
- except Exception:
686
- pass
986
+ # Attach the vector so the meaning-based channel finds it now, through
987
+ # the same ordering every other write path uses.
988
+ if emb and index_external:
989
+ self._attach_vector(fact.fact_id, emb, fmean, fvar)
687
990
  # Persist BM25 tokens too (covers the in-memory rank_bm25 fallback path).
688
991
  if index_external:
689
992
  try:
@@ -828,8 +1131,13 @@ class MemoryEngine:
828
1131
  embed_pool_lock = getattr(self, "_store_fast_embed_pool_lock", None)
829
1132
  if embed_pool_lock is not None:
830
1133
  with embed_pool_lock:
1134
+ # Marked closed under the same lock that guards creation, so a
1135
+ # call already past the unlocked check declines instead of
1136
+ # building an unowned replacement.
1137
+ self._store_fast_embed_pool_closed = True
831
1138
  embed_pool, self._store_fast_embed_pool = self._store_fast_embed_pool, None
832
1139
  else:
1140
+ self._store_fast_embed_pool_closed = True
833
1141
  self._store_fast_embed_pool = None
834
1142
  if embed_pool is not None:
835
1143
  try:
@@ -12,6 +12,8 @@ from __future__ import annotations
12
12
 
13
13
  import hashlib
14
14
  import json
15
+
16
+ from superlocalmemory.storage.embedding_codec import encode_embedding
15
17
  import re
16
18
  from dataclasses import dataclass
17
19
  from pathlib import Path
@@ -435,7 +437,7 @@ def rebuild_derived_state(
435
437
  vector = embedder.embed(str(fact["content"]))
436
438
  db.execute(
437
439
  "UPDATE atomic_facts SET embedding=? WHERE fact_id=? AND profile_id=?",
438
- (json.dumps(vector, separators=(",", ":")), fact["fact_id"], profile_id),
440
+ (encode_embedding(vector), fact["fact_id"], profile_id),
439
441
  )
440
442
  embeddings += 1
441
443
  return {"bm25_rows": bm25_rows, "embeddings": embeddings}
@@ -0,0 +1,131 @@
1
+ # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
2
+ # Licensed under AGPL-3.0-or-later - see LICENSE file
3
+
4
+ """Detect all SuperLocalMemory installations present on this machine.
5
+
6
+ Supports three install types: pipx, venv (~/.slm-venv), and npm global.
7
+ All detection is read-only and fast (< 200 ms). No writes are ever performed.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import glob
13
+ import json
14
+ import subprocess
15
+ import sys
16
+ from pathlib import Path
17
+ from typing import Optional
18
+
19
+ # Roots for the two Python install types. Patched in tests.
20
+ _VENV_ROOT: Path = Path.home() / ".slm-venv"
21
+ _PIPX_ROOT: Path = Path.home() / ".local" / "pipx" / "venvs" / "superlocalmemory"
22
+
23
+
24
+ def _npm_global_root() -> Optional[Path]:
25
+ """Return the npm global node_modules root, or None on any failure."""
26
+ try:
27
+ result = subprocess.run(
28
+ ["npm", "root", "-g"],
29
+ capture_output=True,
30
+ text=True,
31
+ timeout=5,
32
+ )
33
+ if result.returncode == 0:
34
+ root = result.stdout.strip()
35
+ if root:
36
+ return Path(root)
37
+ except Exception: # npm absent, timeout, etc.
38
+ pass
39
+ return None
40
+
41
+
42
+ def _read_python_version(base: Path) -> Optional[str]:
43
+ """Read __version__ from the first matching site-packages layout under base.
44
+
45
+ Both layouts are searched. POSIX virtualenvs use
46
+ ``lib/python3.13/site-packages``; Windows uses ``Lib/site-packages`` with no
47
+ version component and a capitalised directory. Searching only the POSIX
48
+ shape made detection silently return None on Windows — which is precisely
49
+ where multi-install divergence between pip and npm is most likely, and where
50
+ the version-mismatch error would then name no installations at all.
51
+ """
52
+ patterns = [
53
+ str(base / "lib" / "python*" / "site-packages" / "superlocalmemory" / "__init__.py"),
54
+ str(base / "Lib" / "site-packages" / "superlocalmemory" / "__init__.py"),
55
+ str(base / "lib" / "site-packages" / "superlocalmemory" / "__init__.py"),
56
+ ]
57
+ matches: list[str] = []
58
+ for pattern in patterns:
59
+ matches.extend(glob.glob(pattern))
60
+ for init_path in sorted(set(matches)):
61
+ try:
62
+ text = Path(init_path).read_text(encoding="utf-8", errors="replace")
63
+ for line in text.splitlines():
64
+ line = line.strip()
65
+ if line.startswith("__version__"):
66
+ # __version__ = "4.1.0" or __version__ = '4.1.0'
67
+ parts = line.split("=", 1)
68
+ if len(parts) == 2:
69
+ return parts[1].strip().strip("\"'")
70
+ except OSError:
71
+ continue
72
+ return None
73
+
74
+
75
+ def _read_npm_version(npm_root: Path) -> Optional[str]:
76
+ """Read version from npm global package.json."""
77
+ pkg_json = npm_root / "superlocalmemory" / "package.json"
78
+ try:
79
+ data = json.loads(pkg_json.read_text(encoding="utf-8"))
80
+ return str(data.get("version", "")).strip() or None
81
+ except (OSError, json.JSONDecodeError):
82
+ return None
83
+
84
+
85
+ def _detect_all_installs() -> list[dict]:
86
+ """Return all SuperLocalMemory installs detected on this machine.
87
+
88
+ Each entry is a dict with keys:
89
+ - ``path`` (str) — directory of the install
90
+ - ``version`` (str) — version string read from package metadata
91
+ - ``type`` (str) — one of "pipx", "venv", "npm"
92
+
93
+ Detection is read-only and best-effort. A missing or unreadable install
94
+ produces no entry rather than an error. Subprocess calls are bounded to
95
+ 5 seconds total.
96
+ """
97
+ results: list[dict] = []
98
+
99
+ # --- pipx ---
100
+ pipx_version = _read_python_version(_PIPX_ROOT)
101
+ if pipx_version is not None:
102
+ results.append({
103
+ "path": str(_PIPX_ROOT) + "/",
104
+ "version": pipx_version,
105
+ "type": "pipx",
106
+ })
107
+
108
+ # --- ~/.slm-venv ---
109
+ venv_version = _read_python_version(_VENV_ROOT)
110
+ if venv_version is not None:
111
+ results.append({
112
+ "path": str(_VENV_ROOT) + "/",
113
+ "version": venv_version,
114
+ "type": "venv",
115
+ })
116
+
117
+ # --- npm global ---
118
+ npm_root = _npm_global_root()
119
+ if npm_root is not None:
120
+ npm_version = _read_npm_version(npm_root)
121
+ if npm_version is not None:
122
+ results.append({
123
+ "path": str(npm_root / "superlocalmemory") + "/",
124
+ "version": npm_version,
125
+ "type": "npm",
126
+ })
127
+
128
+ return results
129
+
130
+
131
+ __all__ = ["_detect_all_installs"]
@@ -16,7 +16,7 @@ The persona is one bounded roll-up per profile that consumes the top community
16
16
  summaries. It is recall-GATED (never auto-injected into the hot recall path —
17
17
  avoids the V3.4.40 summary-pollution regression) and SIZE-bounded. Drill-down
18
18
  (``get_sources``) walks the hierarchy back down to the source atoms, matching
19
- the market bar for summary->source provenance (Zep-style).
19
+ the design goal for summary->source provenance (Zep-style attribution).
20
20
 
21
21
  Runs in the background consolidation lane after community summaries.
22
22
  Fail-open throughout; recompute replaces a profile's row.
@@ -106,6 +106,10 @@ def _handle_recall(
106
106
  memory_map=memory_map,
107
107
  per_fact_max=getattr(_rc, "recall_per_fact_max_chars", 2400),
108
108
  total_max=getattr(_rc, "recall_total_max_chars", 12000),
109
+ # Option B: markers only on session-bearing recalls. A marker can
110
+ # only buy a learning signal when a pending_outcomes row exists
111
+ # to settle, and those exist only when session_id is present.
112
+ include_marker=bool(session_id),
109
113
  )
110
114
  return {
111
115
  "ok": True,