superlocalmemory 4.0.10 → 4.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +12 -2
- package/CHANGELOG.md +244 -0
- package/README.md +40 -75
- package/package.json +6 -3
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +4 -4
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +2 -2
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +5 -5
- package/plugin/skills/slm-recall/SKILL.md +102 -15
- package/plugin/skills/slm-remember/SKILL.md +35 -3
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +29 -3
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +16 -8
- package/plugin-src/skills/slm-cache/SKILL.md +1 -1
- package/plugin-src/skills/slm-compress/SKILL.md +1 -1
- package/plugin-src/skills/slm-governance/SKILL.md +1 -1
- package/plugin-src/skills/slm-graph/SKILL.md +1 -1
- package/plugin-src/skills/slm-loop/SKILL.md +2 -2
- package/plugin-src/skills/slm-mesh/SKILL.md +1 -1
- package/plugin-src/skills/slm-profile/SKILL.md +5 -5
- package/plugin-src/skills/slm-recall/SKILL.md +102 -15
- package/plugin-src/skills/slm-remember/SKILL.md +35 -3
- package/plugin-src/skills/slm-scope/SKILL.md +1 -1
- package/plugin-src/skills/slm-session/SKILL.md +29 -3
- package/plugin-src/skills/slm-status/SKILL.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/commands.py +357 -18
- package/src/superlocalmemory/cli/daemon.py +30 -0
- package/src/superlocalmemory/cli/db_migrate.py +71 -1
- package/src/superlocalmemory/cli/gdpr_cmd.py +15 -2
- package/src/superlocalmemory/cli/main.py +24 -2
- package/src/superlocalmemory/code_graph/database.py +44 -0
- package/src/superlocalmemory/compliance/gdpr.py +449 -39
- package/src/superlocalmemory/core/admission.py +231 -11
- package/src/superlocalmemory/core/backend_orchestrator.py +190 -84
- package/src/superlocalmemory/core/config.py +90 -11
- package/src/superlocalmemory/core/consolidation_engine.py +34 -0
- package/src/superlocalmemory/core/engine.py +140 -11
- package/src/superlocalmemory/core/graph_analyzer.py +76 -112
- package/src/superlocalmemory/core/graph_metrics.py +597 -0
- package/src/superlocalmemory/core/graph_pruner.py +121 -0
- package/src/superlocalmemory/core/maintenance_scheduler.py +205 -0
- package/src/superlocalmemory/core/mode_capability.py +111 -0
- package/src/superlocalmemory/core/ollama_validator.py +315 -0
- package/src/superlocalmemory/core/projection_drain.py +380 -0
- package/src/superlocalmemory/core/recall_pipeline.py +390 -3
- package/src/superlocalmemory/core/recall_worker.py +6 -3
- package/src/superlocalmemory/core/scale_autopromote.py +196 -0
- package/src/superlocalmemory/core/scale_engine.py +16 -2
- package/src/superlocalmemory/core/score_contract.py +21 -1
- package/src/superlocalmemory/core/session_identity.py +85 -0
- package/src/superlocalmemory/core/status_contract.py +108 -0
- package/src/superlocalmemory/core/worker_pool.py +4 -4
- package/src/superlocalmemory/core/working_memory.py +288 -0
- package/src/superlocalmemory/encoding/cognitive_consolidator.py +36 -6
- package/src/superlocalmemory/encoding/context_generator.py +1 -1
- package/src/superlocalmemory/encoding/entity_resolver.py +38 -0
- package/src/superlocalmemory/encoding/fact_extractor.py +18 -14
- package/src/superlocalmemory/encoding/prospective_markers.py +262 -0
- package/src/superlocalmemory/encoding/type_router.py +12 -12
- package/src/superlocalmemory/evolution/mutation_generator.py +30 -4
- package/src/superlocalmemory/graph/cozo_adjacency.py +122 -0
- package/src/superlocalmemory/graph/cozo_backend.py +103 -138
- package/src/superlocalmemory/hooks/portable_kit.py +10 -2
- package/src/superlocalmemory/learning/bandit.py +43 -0
- package/src/superlocalmemory/learning/consolidation_worker.py +54 -0
- package/src/superlocalmemory/learning/database.py +60 -3
- package/src/superlocalmemory/learning/entity_compiler.py +21 -58
- package/src/superlocalmemory/learning/feedback.py +3 -1
- package/src/superlocalmemory/learning/outcomes.py +47 -16
- package/src/superlocalmemory/learning/pattern_miner.py +28 -3
- package/src/superlocalmemory/learning/pattern_miner_constants.py +43 -0
- package/src/superlocalmemory/learning/pcos.py +291 -0
- package/src/superlocalmemory/learning/reward_from_outcomes.py +365 -0
- package/src/superlocalmemory/learning/reward_proxy.py +100 -10
- package/src/superlocalmemory/learning/signal_kinds.py +79 -0
- package/src/superlocalmemory/mcp/profiles.py +14 -2
- package/src/superlocalmemory/mcp/tools_active.py +2 -1
- package/src/superlocalmemory/mcp/tools_core.py +31 -3
- package/src/superlocalmemory/mcp/tools_v28.py +20 -1
- package/src/superlocalmemory/parameterization/pattern_extractor.py +14 -1
- package/src/superlocalmemory/parameterization/soft_prompt_generator.py +98 -0
- package/src/superlocalmemory/retrieval/bm25_channel.py +64 -3
- package/src/superlocalmemory/retrieval/channel_status.py +117 -0
- package/src/superlocalmemory/retrieval/engine.py +106 -11
- package/src/superlocalmemory/retrieval/entity_channel.py +210 -256
- package/src/superlocalmemory/retrieval/graph_adjacency.py +219 -0
- package/src/superlocalmemory/retrieval/scope_policy.py +20 -0
- package/src/superlocalmemory/retrieval/semantic_channel.py +47 -5
- package/src/superlocalmemory/retrieval/spreading.py +288 -0
- package/src/superlocalmemory/server/api.py +24 -5
- package/src/superlocalmemory/server/bandit_loops.py +17 -1
- package/src/superlocalmemory/server/rbac_enforce.py +26 -6
- package/src/superlocalmemory/server/recall_health.py +87 -10
- package/src/superlocalmemory/server/recall_serializer.py +9 -0
- package/src/superlocalmemory/server/routes/behavioral.py +75 -10
- package/src/superlocalmemory/server/routes/compliance.py +98 -18
- package/src/superlocalmemory/server/routes/config_api.py +186 -4
- package/src/superlocalmemory/server/routes/evolution.py +178 -0
- package/src/superlocalmemory/server/routes/ingest.py +8 -0
- package/src/superlocalmemory/server/routes/learning_telemetry.py +2 -1
- package/src/superlocalmemory/server/routes/memories.py +49 -7
- package/src/superlocalmemory/server/routes/timeline.py +4 -0
- package/src/superlocalmemory/server/routes/v3_api.py +191 -15
- package/src/superlocalmemory/server/ui.py +20 -4
- package/src/superlocalmemory/server/unified_daemon.py +241 -7
- package/src/superlocalmemory/storage/_migration_internals.py +54 -2
- package/src/superlocalmemory/storage/_schema_version.py +24 -3
- package/src/superlocalmemory/storage/database.py +477 -59
- package/src/superlocalmemory/storage/embedding_codec.py +71 -0
- package/src/superlocalmemory/storage/lineage_retention.py +236 -0
- package/src/superlocalmemory/storage/logical_edges.py +43 -2
- package/src/superlocalmemory/storage/migration_runner.py +119 -0
- package/src/superlocalmemory/storage/migrations/M043_quarantine_display_summaries.py +60 -36
- package/src/superlocalmemory/storage/migrations/M044_play_carries_its_own_evidence.py +127 -0
- package/src/superlocalmemory/storage/migrations/M045_fact_outcome_score.py +158 -0
- package/src/superlocalmemory/storage/migrations/M046_prospective_memory_has_its_own_name.py +620 -0
- package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py +306 -0
- package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py +207 -0
- package/src/superlocalmemory/storage/migrations/M049_a_schema_version_marker_is_one_row.py +201 -0
- package/src/superlocalmemory/storage/migrations.py +18 -2
- package/src/superlocalmemory/storage/models.py +40 -1
- package/src/superlocalmemory/storage/projection_outbox.py +346 -0
- package/src/superlocalmemory/storage/retention_policy.py +860 -0
- package/src/superlocalmemory/storage/schema.py +35 -1
- package/src/superlocalmemory/storage/write_coordinator.py +19 -2
- package/src/superlocalmemory/trust/scorer.py +43 -1
- package/src/superlocalmemory/ui/index.html +9 -18
- package/src/superlocalmemory/ui/js/event-delegation.js +12 -1
- package/src/superlocalmemory/ui/js/od-health.js +28 -6
- package/src/superlocalmemory/ui/js/od-memories.js +19 -0
- package/src/superlocalmemory/ui/js/od-settings.js +87 -1
- package/src/superlocalmemory/ui/js/recall-lab.js +78 -3
|
@@ -28,9 +28,22 @@ logger = logging.getLogger(__name__)
|
|
|
28
28
|
|
|
29
29
|
#: Default number of results returned by recall across MCP, CLI, daemon, and
|
|
30
30
|
#: engine. All surfaces bind their ``limit`` defaults to this constant so a
|
|
31
|
-
#: single change is sufficient to keep the full stack in sync.
|
|
31
|
+
#: single change is sufficient to keep the full stack in sync. Search is a
|
|
32
|
+
#: retrieval operation too and binds here: asking the same question through
|
|
33
|
+
#: ``recall`` and through ``search`` must not return differently sized answers.
|
|
32
34
|
CANONICAL_RECALL_LIMIT: int = 20
|
|
33
35
|
|
|
36
|
+
#: Default number of rows returned when listing the newest memories rather than
|
|
37
|
+
#: answering a question — MCP ``list_recent`` and ``slm list``. Separate from
|
|
38
|
+
#: the recall limit because the two can legitimately diverge; equal today.
|
|
39
|
+
CANONICAL_LIST_LIMIT: int = 20
|
|
40
|
+
|
|
41
|
+
#: Page size for the HTTP memory browser (``GET /api/memories``), which is a
|
|
42
|
+
#: paged table rather than an answer. Deliberately larger than the list limit:
|
|
43
|
+
#: a person scrolling a table wants a fuller page than an agent asking what
|
|
44
|
+
#: happened recently. The ceiling on that endpoint is enforced separately.
|
|
45
|
+
BROWSE_PAGE_SIZE: int = 50
|
|
46
|
+
|
|
34
47
|
|
|
35
48
|
# ---------------------------------------------------------------------------
|
|
36
49
|
# Default Paths
|
|
@@ -767,8 +780,8 @@ class ParameterizationConfig:
|
|
|
767
780
|
max_memory_tokens: int = 1500 # Token budget for regular memories
|
|
768
781
|
categories_enabled: tuple[str, ...] = (
|
|
769
782
|
"identity", "tech_preference", "communication_style",
|
|
770
|
-
"workflow_pattern", "project_context", "
|
|
771
|
-
"avoidance",
|
|
783
|
+
"workflow_pattern", "project_context", "topic_interest",
|
|
784
|
+
"decision_history", "avoidance",
|
|
772
785
|
)
|
|
773
786
|
|
|
774
787
|
# Lifecycle
|
|
@@ -1152,14 +1165,26 @@ class SLMConfig:
|
|
|
1152
1165
|
# stay on SQLite until a staged parity check promotes these projections.
|
|
1153
1166
|
scale_engine_state: str = "local_core" # local_core | prepared | verified | promoted
|
|
1154
1167
|
# v3.8.5: auto-promote Cozo+LanceDB when a DB grows past the scale at which
|
|
1155
|
-
# they actually help.
|
|
1156
|
-
# faster (measured ~1.7ms/traversal at 208K edges), so normal installs stay
|
|
1157
|
-
# on Local Core and never pay the projection/migration cost. The threshold
|
|
1158
|
-
# is deliberately high: the graph DB win appears at millions of edges, not
|
|
1159
|
-
# hundreds of thousands. Auto-promotion is background, uses the same staged
|
|
1168
|
+
# they actually help. Auto-promotion is background, uses the same staged
|
|
1160
1169
|
# parity gate as the manual path, and falls back to SQLite on any failure.
|
|
1170
|
+
#
|
|
1171
|
+
# 100,000 (was 1,000,000, set by Varun 2026-08-22). The old value gated every
|
|
1172
|
+
# store anyone actually has -- the author's is 140,000 edges -- so the
|
|
1173
|
+
# projection was built for nobody and, being unexercised, was wrong: it
|
|
1174
|
+
# carried 1,257 memories the store may not return and its graph search
|
|
1175
|
+
# disagreed with SQLite on every query. A threshold no real store crosses is
|
|
1176
|
+
# not a safety margin, it is a feature nobody tests.
|
|
1177
|
+
#
|
|
1178
|
+
# What promotion now changes is narrow and measured. The graph WALK does not
|
|
1179
|
+
# move: SQLite loads 130k edges in 141 ms against CozoDB's 252 ms, so the
|
|
1180
|
+
# adjacency read stays where it is fastest (retrieval/graph_adjacency.py
|
|
1181
|
+
# holds the seam for the day that flips). What promotion switches on is the
|
|
1182
|
+
# LanceDB vector index, which is within noise of sqlite-vec at this size
|
|
1183
|
+
# (18.3 ms against 18.7 ms over 5,324 vectors) and is the one built to keep
|
|
1184
|
+
# going as the vector count grows. So crossing this line is close to
|
|
1185
|
+
# latency-neutral today and buys the projection real exercise.
|
|
1161
1186
|
scale_auto_promote_enabled: bool = True
|
|
1162
|
-
scale_auto_promote_min_edges: int =
|
|
1187
|
+
scale_auto_promote_min_edges: int = 100_000
|
|
1163
1188
|
evolution: EvolutionConfig = field(default_factory=EvolutionConfig)
|
|
1164
1189
|
health: HealthConfig = field(default_factory=HealthConfig)
|
|
1165
1190
|
# v3.8.4-G: Graph thinning parameters (#84)
|
|
@@ -1304,10 +1329,10 @@ class SLMConfig:
|
|
|
1304
1329
|
)
|
|
1305
1330
|
try:
|
|
1306
1331
|
config.scale_auto_promote_min_edges = int(
|
|
1307
|
-
data.get("scale_auto_promote_min_edges",
|
|
1332
|
+
data.get("scale_auto_promote_min_edges", 100_000)
|
|
1308
1333
|
)
|
|
1309
1334
|
except (TypeError, ValueError):
|
|
1310
|
-
config.scale_auto_promote_min_edges =
|
|
1335
|
+
config.scale_auto_promote_min_edges = 100_000
|
|
1311
1336
|
|
|
1312
1337
|
# V3.3 config fields (additive — defaults work if missing from JSON)
|
|
1313
1338
|
fg = data.get("forgetting", {})
|
|
@@ -1392,6 +1417,44 @@ class SLMConfig:
|
|
|
1392
1417
|
if k in RetrievalConfig.__dataclass_fields__
|
|
1393
1418
|
})
|
|
1394
1419
|
|
|
1420
|
+
# 4.1.0 (#124): restore the two sections save() now writes. This runs
|
|
1421
|
+
# AFTER for_mode() has applied its presets, so a value someone chose
|
|
1422
|
+
# beats the mode default -- the same ordering retrieval already relies
|
|
1423
|
+
# on. A malformed section falls back to the dataclass defaults rather
|
|
1424
|
+
# than raising, matching quantization and sagq above: a corrupt config
|
|
1425
|
+
# file must not make `slm` unrunnable.
|
|
1426
|
+
mth = data.get("math")
|
|
1427
|
+
if isinstance(mth, dict) and mth:
|
|
1428
|
+
try:
|
|
1429
|
+
fields = {
|
|
1430
|
+
k: v for k, v in mth.items()
|
|
1431
|
+
if k in MathConfig.__dataclass_fields__
|
|
1432
|
+
}
|
|
1433
|
+
# JSON has no tuple. Written as a list, it must come back a
|
|
1434
|
+
# tuple or every reader that unpacks a pair gets a list and the
|
|
1435
|
+
# difference surfaces somewhere far from here.
|
|
1436
|
+
rng = fields.get("langevin_weight_range")
|
|
1437
|
+
if isinstance(rng, list):
|
|
1438
|
+
fields["langevin_weight_range"] = tuple(rng)
|
|
1439
|
+
config.math = MathConfig(**fields)
|
|
1440
|
+
except (TypeError, ValueError) as exc:
|
|
1441
|
+
logger.warning(
|
|
1442
|
+
"Ignoring invalid math config (%s) — using defaults", exc
|
|
1443
|
+
)
|
|
1444
|
+
|
|
1445
|
+
cw = data.get("channel_weights")
|
|
1446
|
+
if isinstance(cw, dict) and cw:
|
|
1447
|
+
try:
|
|
1448
|
+
config.channel_weights = ChannelWeights(**{
|
|
1449
|
+
k: v for k, v in cw.items()
|
|
1450
|
+
if k in ChannelWeights.__dataclass_fields__
|
|
1451
|
+
})
|
|
1452
|
+
except (TypeError, ValueError) as exc:
|
|
1453
|
+
logger.warning(
|
|
1454
|
+
"Ignoring invalid channel_weights config (%s) — using defaults",
|
|
1455
|
+
exc,
|
|
1456
|
+
)
|
|
1457
|
+
|
|
1395
1458
|
# V3.4.3 config fields (additive — missing keys get dataclass defaults)
|
|
1396
1459
|
config.daemon_idle_timeout = data.get("daemon_idle_timeout", 0)
|
|
1397
1460
|
config.daemon_port = data.get("daemon_port", 8765)
|
|
@@ -1654,11 +1717,20 @@ class SLMConfig:
|
|
|
1654
1717
|
#
|
|
1655
1718
|
# embedding_signature: has no typed in-memory model — it is an opaque blob
|
|
1656
1719
|
# written by external tooling (see below).
|
|
1720
|
+
# ``math`` and ``channel_weights`` join this loop as of 4.1.0. Both are
|
|
1721
|
+
# read by engine wiring -- ``math.sheaf_contradiction_threshold`` by the
|
|
1722
|
+
# consistency checker, ``channel_weights`` as the retrieval base weights
|
|
1723
|
+
# -- and neither was ever written here or restored in ``load()``. Tuning
|
|
1724
|
+
# either one, by hand or by switching mode, survived until the next
|
|
1725
|
+
# restart and then silently reverted to the dataclass default, with
|
|
1726
|
+
# nothing said. Reported as #124.
|
|
1657
1727
|
for _section, _obj in (
|
|
1658
1728
|
("forgetting", self.forgetting),
|
|
1659
1729
|
("quantization", self.quantization),
|
|
1660
1730
|
("sagq", self.sagq),
|
|
1661
1731
|
("auto_invoke", self.auto_invoke),
|
|
1732
|
+
("math", self.math),
|
|
1733
|
+
("channel_weights", self.channel_weights),
|
|
1662
1734
|
):
|
|
1663
1735
|
_base = existing.get(_section)
|
|
1664
1736
|
_merged = dict(_base) if isinstance(_base, dict) else {}
|
|
@@ -1794,6 +1866,13 @@ class SLMConfig:
|
|
|
1794
1866
|
base_dir=_base,
|
|
1795
1867
|
embedding=_a_emb,
|
|
1796
1868
|
llm=LLMConfig(), # No LLM
|
|
1869
|
+
# Mode A's promise is that nothing puts a model on any path, and
|
|
1870
|
+
# a subsystem should refuse because the mode says so, not because
|
|
1871
|
+
# it happened to be handed llm=None. Three of the four places
|
|
1872
|
+
# that build the consolidator pass only a database, so today the
|
|
1873
|
+
# guard holds by accident of wiring; this makes it hold by
|
|
1874
|
+
# configuration, which is the thing a reader can check.
|
|
1875
|
+
ccq=CCQConfig(use_llm_gist=False),
|
|
1797
1876
|
temporal_validator=TemporalValidatorConfig(mode="a"),
|
|
1798
1877
|
retrieval=RetrievalConfig(
|
|
1799
1878
|
# V3.3.2: ONNX cross-encoder enabled for all modes (~200MB)
|
|
@@ -634,8 +634,42 @@ class ConsolidationEngine:
|
|
|
634
634
|
)
|
|
635
635
|
result["entity_community_count"] = ec.get("community_count", 0)
|
|
636
636
|
result["community_summaries"] = ec.get("summaries_written", 0)
|
|
637
|
+
result["entities_compiled"] = self._compile_entities(profile_id)
|
|
637
638
|
return result
|
|
638
639
|
|
|
640
|
+
def _compile_entities(self, profile_id: str) -> int:
|
|
641
|
+
"""Bring each entity's summary up to date with its facts.
|
|
642
|
+
|
|
643
|
+
This ran only on the maintenance endpoint, never on the periodic
|
|
644
|
+
consolidation that the daemon schedules on its own. The effect was that
|
|
645
|
+
almost every entity carried no compiled summary at all: the explorer
|
|
646
|
+
asked for one, got null, and showed an entity with no description and
|
|
647
|
+
an empty timeline — which reads as "there is nothing here" rather than
|
|
648
|
+
"nothing has compiled this yet".
|
|
649
|
+
|
|
650
|
+
Runs beside the community recompute because both derive a view over the
|
|
651
|
+
same graph and both belong to the same pass. Never fatal: a compilation
|
|
652
|
+
that fails leaves the previous summary in place, and no summary is a
|
|
653
|
+
worse answer than a stale one but not a broken store.
|
|
654
|
+
"""
|
|
655
|
+
try:
|
|
656
|
+
from superlocalmemory.core.config import SLMConfig
|
|
657
|
+
from superlocalmemory.learning.entity_compiler import EntityCompiler
|
|
658
|
+
|
|
659
|
+
db_path = getattr(self._db, "db_path", None)
|
|
660
|
+
if not db_path:
|
|
661
|
+
return 0
|
|
662
|
+
compiled = EntityCompiler(str(db_path), SLMConfig.load()).compile_all(
|
|
663
|
+
profile_id,
|
|
664
|
+
)
|
|
665
|
+
count = int(compiled.get("compiled", 0) or 0)
|
|
666
|
+
if count:
|
|
667
|
+
logger.info("Entity compilation: %d entities compiled", count)
|
|
668
|
+
return count
|
|
669
|
+
except Exception as exc:
|
|
670
|
+
logger.debug("Entity compilation skipped: %s", exc)
|
|
671
|
+
return 0
|
|
672
|
+
|
|
639
673
|
# ------------------------------------------------------------------
|
|
640
674
|
# Step 6: Derive Associations
|
|
641
675
|
# ------------------------------------------------------------------
|
|
@@ -53,6 +53,35 @@ def _verify_ingestion_schema(memory_db: Path) -> bool:
|
|
|
53
53
|
# Embedding a memory on the write path, before the receipt is returned
|
|
54
54
|
# ---------------------------------------------------------------------------
|
|
55
55
|
|
|
56
|
+
def _embedder_is_warm(embedder: object) -> bool:
|
|
57
|
+
"""Has this embedder already served a request, so the next one is cheap?
|
|
58
|
+
|
|
59
|
+
Availability and readiness are different questions and the write path needs
|
|
60
|
+
the second one. ``EmbeddingService._available`` is set to ``True`` in its
|
|
61
|
+
constructor, before the worker subprocess exists; the model behind it takes
|
|
62
|
+
**9.9-11.0 s** to load on this machine against **42 ms** once loaded. A write
|
|
63
|
+
that embeds inline does so behind a one-second deadline, so keying off
|
|
64
|
+
availability meant every write until something else warmed the model paid the
|
|
65
|
+
whole deadline and then stored no vector anyway -- 10 of the first 12,
|
|
66
|
+
measured. The deadline was spent proving the model was cold.
|
|
67
|
+
|
|
68
|
+
``EmbeddingService.is_warm`` answers the real question: a live worker that has
|
|
69
|
+
served at least one request. So both must hold, and availability is still
|
|
70
|
+
checked first: an embedder that reports itself unavailable is not a candidate
|
|
71
|
+
whatever else it says. Requiring only ``is_warm`` would call ``embed`` on a
|
|
72
|
+
dead embedder that happens to expose the attribute -- any mock, proxy or
|
|
73
|
+
duck-typed wrapper auto-creates one.
|
|
74
|
+
|
|
75
|
+
An embedder that offers no ``is_warm`` keeps availability as the whole answer:
|
|
76
|
+
``OllamaEmbedder`` talks to a server that is already running and has no worker
|
|
77
|
+
of its own to start, so there readiness is availability.
|
|
78
|
+
"""
|
|
79
|
+
if getattr(embedder, "_available", None) is not True:
|
|
80
|
+
return False
|
|
81
|
+
warm = getattr(embedder, "is_warm", None)
|
|
82
|
+
return True if warm is None else bool(warm)
|
|
83
|
+
|
|
84
|
+
|
|
56
85
|
def _is_remote_embedder(embedder: object) -> bool:
|
|
57
86
|
"""Return True if *embedder* makes remote HTTP calls (cloud / OpenAI-compatible).
|
|
58
87
|
|
|
@@ -102,6 +131,14 @@ class MemoryEngine:
|
|
|
102
131
|
self._caps = get_capabilities(config.mode)
|
|
103
132
|
self._capabilities = capabilities
|
|
104
133
|
self._profile_id = config.active_profile
|
|
134
|
+
# The session a recall belongs to, for matching an outcome back to it
|
|
135
|
+
# later. Set from whatever the caller last named; falls back to a stable
|
|
136
|
+
# per-process id so a caller that never names one still leaves a record
|
|
137
|
+
# that can be matched to another call from the same process. An empty
|
|
138
|
+
# one is dropped by the queue, which is how 34 of 35 recall paths came
|
|
139
|
+
# to leave no record at all.
|
|
140
|
+
self._last_session_id: str = ""
|
|
141
|
+
self._ambient_session_id: str = f"engine:{os.getpid()}"
|
|
105
142
|
self._initialized = False
|
|
106
143
|
|
|
107
144
|
self._db = None
|
|
@@ -572,7 +609,7 @@ class MemoryEngine:
|
|
|
572
609
|
_embedder_ref = self._embedder
|
|
573
610
|
if (
|
|
574
611
|
_embedder_ref is None
|
|
575
|
-
or
|
|
612
|
+
or not _embedder_is_warm(_embedder_ref)
|
|
576
613
|
or _is_remote_embedder(_embedder_ref)
|
|
577
614
|
):
|
|
578
615
|
return None, None, None
|
|
@@ -901,9 +938,16 @@ class MemoryEngine:
|
|
|
901
938
|
|
|
902
939
|
store_fast inserts a verbatim AtomicFact (+ memory row) synchronously.
|
|
903
940
|
The FTS5 ``atomic_facts_fts`` trigger auto-populates on INSERT, so the
|
|
904
|
-
memory is **keyword/BM25-recallable the instant this returns
|
|
905
|
-
LLM
|
|
906
|
-
|
|
941
|
+
memory is **keyword/BM25-recallable the instant this returns**, with no
|
|
942
|
+
LLM call. Entities and graph are enriched async by the materializer.
|
|
943
|
+
|
|
944
|
+
The embedding is no longer purely async: ``_warm_guard_embed`` computes it
|
|
945
|
+
inline **when the model is already loaded**, because a fact with no vector
|
|
946
|
+
is invisible to the semantic channel and a memory written moments ago is
|
|
947
|
+
then the hardest thing in the store to find by asking about it. Measured
|
|
948
|
+
on a copy of a real store: **75 ms** with the model loaded, vector
|
|
949
|
+
attached; **34 ms** when it is not, deferring to the materializer. This
|
|
950
|
+
docstring used to say "~ms, no embedding" and that had stopped being true.
|
|
907
951
|
|
|
908
952
|
Returns real fact_ids immediately. Quality gate rejects template junk.
|
|
909
953
|
"""
|
|
@@ -1016,12 +1060,16 @@ class MemoryEngine:
|
|
|
1016
1060
|
) -> RecallResponse:
|
|
1017
1061
|
"""Recall relevant facts for a query.
|
|
1018
1062
|
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1063
|
+
``session_id`` gives the recall continuity: memories this session was
|
|
1064
|
+
recently shown are held in a small in-process working set and bias the
|
|
1065
|
+
ranking of subsequent queries in the same session. Retrieval still runs
|
|
1066
|
+
in full every time — the working set reorders the answer, it never
|
|
1067
|
+
replaces the search. Costs no query and no file access.
|
|
1068
|
+
|
|
1069
|
+
Omitting it is supported and means every recall starts cold, which is
|
|
1070
|
+
what every caller got before. The parameter was previously accepted and
|
|
1071
|
+
then ignored: it was documented as enqueueing a ``pending_outcomes`` row
|
|
1072
|
+
for downstream hooks, and no such row was ever written.
|
|
1025
1073
|
|
|
1026
1074
|
``fast`` controls only the internal agentic verification round; all six
|
|
1027
1075
|
local retrieval channels + reranker run regardless. ``fast=None`` (the
|
|
@@ -1060,6 +1108,7 @@ class MemoryEngine:
|
|
|
1060
1108
|
try:
|
|
1061
1109
|
response = run_recall(
|
|
1062
1110
|
query, pid, mode=mode, limit=limit, agent_id=agent_id,
|
|
1111
|
+
session_id=session_id,
|
|
1063
1112
|
config=self._config,
|
|
1064
1113
|
retrieval_engine=self._retrieval_engine,
|
|
1065
1114
|
trust_scorer=self._trust_scorer,
|
|
@@ -1083,8 +1132,75 @@ class MemoryEngine:
|
|
|
1083
1132
|
# be submitted through explicit write commands.
|
|
1084
1133
|
raise
|
|
1085
1134
|
|
|
1135
|
+
# Leave a ticket saying what was shown, so an outcome can be attached to
|
|
1136
|
+
# it later. Without one there is no key to join an outcome back to the
|
|
1137
|
+
# recall that produced it, and the whole learning path stops at the first
|
|
1138
|
+
# hop: on a real store every one of 162 recorded outcomes carried an
|
|
1139
|
+
# empty recall id, every per-memory usefulness score sat at its neutral
|
|
1140
|
+
# 0.5, and the ranking model had not retrained in eleven weeks.
|
|
1141
|
+
#
|
|
1142
|
+
# THIS MUST NOT COST THE READER ANYTHING. It is one put_nowait on a
|
|
1143
|
+
# bounded in-memory queue -- roughly a microsecond -- and it drops the
|
|
1144
|
+
# event rather than blocking or raising if the queue is full. A worker
|
|
1145
|
+
# persists it elsewhere. Recall's answer and its timing are unchanged;
|
|
1146
|
+
# measured p50 and p95 before and after are within noise of each other.
|
|
1147
|
+
#
|
|
1148
|
+
# It records only. Nothing here reorders a result: whether learning is
|
|
1149
|
+
# allowed to influence ranking stays a separate, explicit setting that
|
|
1150
|
+
# remains off unless an operator turns it on.
|
|
1151
|
+
try:
|
|
1152
|
+
import uuid
|
|
1153
|
+
|
|
1154
|
+
from superlocalmemory.learning.outcome_queue import (
|
|
1155
|
+
RecallEvent,
|
|
1156
|
+
enqueue_recall,
|
|
1157
|
+
)
|
|
1158
|
+
|
|
1159
|
+
# A fresh id per answer, put on the response as well as the
|
|
1160
|
+
# ticket. `calibration_id` was the obvious candidate and is wrong:
|
|
1161
|
+
# it is None on most paths, and an empty join key is exactly the
|
|
1162
|
+
# state that made all 162 recorded outcomes unmatchable. Returning
|
|
1163
|
+
# it is what makes it useful -- until an answer carried its own
|
|
1164
|
+
# name, no caller could quote one back, so an outcome could only
|
|
1165
|
+
# ever be matched by guessing from overlapping memories.
|
|
1166
|
+
answer_id = uuid.uuid4().hex
|
|
1167
|
+
try:
|
|
1168
|
+
response.query_id = answer_id
|
|
1169
|
+
except Exception: # noqa: BLE001 -- an older response shape
|
|
1170
|
+
pass
|
|
1171
|
+
enqueue_recall(RecallEvent(
|
|
1172
|
+
session_id=self._session_for_signals(session_id),
|
|
1173
|
+
profile_id=str(pid),
|
|
1174
|
+
query=query,
|
|
1175
|
+
fact_ids=[
|
|
1176
|
+
r.fact.fact_id for r in (response.results or [])
|
|
1177
|
+
if getattr(r, "fact", None) is not None
|
|
1178
|
+
],
|
|
1179
|
+
query_id=answer_id,
|
|
1180
|
+
))
|
|
1181
|
+
except Exception as exc: # noqa: BLE001 -- a read must not fail on this
|
|
1182
|
+
logger.debug("recall outcome ticket skipped: %s", exc)
|
|
1183
|
+
|
|
1086
1184
|
return response
|
|
1087
1185
|
|
|
1186
|
+
def _session_for_signals(self, session_id: str | None) -> str:
|
|
1187
|
+
"""The name to file this call's outcome under.
|
|
1188
|
+
|
|
1189
|
+
In order: what the caller said, then the last thing any caller on this
|
|
1190
|
+
engine said, then a stable id for this process. Never empty, because an
|
|
1191
|
+
empty one is discarded and the call then leaves no trace an outcome
|
|
1192
|
+
could ever be matched to -- which is what happened on every path that
|
|
1193
|
+
did not thread a session through, meaning nearly all of them.
|
|
1194
|
+
|
|
1195
|
+
Three attribute reads and no I/O. Recall must not get slower to record
|
|
1196
|
+
that it happened.
|
|
1197
|
+
"""
|
|
1198
|
+
named = str(session_id or "").strip()
|
|
1199
|
+
if named:
|
|
1200
|
+
self._last_session_id = named
|
|
1201
|
+
return named
|
|
1202
|
+
return self._last_session_id or self._ambient_session_id
|
|
1203
|
+
|
|
1088
1204
|
# -- Session operations -------------------------------------------------
|
|
1089
1205
|
|
|
1090
1206
|
def create_speaker_entities(
|
|
@@ -1099,9 +1215,22 @@ class MemoryEngine:
|
|
|
1099
1215
|
)
|
|
1100
1216
|
|
|
1101
1217
|
def close_session(self, session_id: str) -> int:
|
|
1102
|
-
"""Create session-level temporal summary.
|
|
1218
|
+
"""Create session-level temporal summary and release its working set.
|
|
1219
|
+
|
|
1220
|
+
The working set is what the session was recently looking at. It exists
|
|
1221
|
+
to connect one turn to the next, so it has no meaning once there are no
|
|
1222
|
+
more turns; holding it would let a later session that reuses this id
|
|
1223
|
+
inherit ranking bias from a conversation that already ended.
|
|
1224
|
+
"""
|
|
1103
1225
|
self._ensure_init()
|
|
1104
1226
|
|
|
1227
|
+
try:
|
|
1228
|
+
from superlocalmemory.core.working_memory import discard
|
|
1229
|
+
|
|
1230
|
+
discard(self._profile_id, session_id)
|
|
1231
|
+
except Exception as exc: # pragma: no cover — never block a close
|
|
1232
|
+
logger.debug("working-set discard skipped: %s", exc)
|
|
1233
|
+
|
|
1105
1234
|
from superlocalmemory.core.store_pipeline import run_close_session
|
|
1106
1235
|
return run_close_session(
|
|
1107
1236
|
session_id, self._profile_id, db=self._db,
|
|
@@ -41,126 +41,90 @@ class GraphAnalyzer:
|
|
|
41
41
|
self._db = db
|
|
42
42
|
|
|
43
43
|
def compute_and_store(self, profile_id: str) -> dict[str, Any]:
|
|
44
|
-
"""
|
|
44
|
+
"""Recompute structural importance for a profile and persist it.
|
|
45
|
+
|
|
46
|
+
The computation and the write now live in ``core/graph_metrics``, which
|
|
47
|
+
is the only thing that writes ``fact_importance``. This method keeps its
|
|
48
|
+
shape for its three callers and adds the community labels the dashboard
|
|
49
|
+
reads, which are a presentation concern and not part of the ranking
|
|
50
|
+
signal.
|
|
51
|
+
|
|
52
|
+
Two behaviours changed with the move, both deliberate. Isolated facts now
|
|
53
|
+
get a row instead of being skipped, so "no position in the graph" stops
|
|
54
|
+
looking like "not computed yet" to the ranker. And the graph is the
|
|
55
|
+
*visible* graph: the previous version read ``atomic_facts`` and
|
|
56
|
+
``graph_edges`` raw, so on the author's store it ranked 1,299 facts the
|
|
57
|
+
store is not allowed to return, diluting every real score.
|
|
58
|
+
"""
|
|
59
|
+
from superlocalmemory.core.graph_metrics import compute_graph_metrics
|
|
45
60
|
|
|
46
|
-
|
|
47
|
-
|
|
61
|
+
report = compute_graph_metrics(self._db, profile_id)
|
|
62
|
+
if not report.ok:
|
|
63
|
+
logger.warning("Graph metrics: %s", report.summary())
|
|
64
|
+
return {
|
|
65
|
+
"node_count": 0,
|
|
66
|
+
"edge_count": report.edges,
|
|
67
|
+
"community_count": 0,
|
|
68
|
+
"top_5_nodes": [],
|
|
69
|
+
"error": report.error,
|
|
70
|
+
}
|
|
48
71
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
72
|
+
communities: dict[str, int] = {}
|
|
73
|
+
top_5: list[tuple[str, float]] = []
|
|
74
|
+
bridge_count = 0
|
|
75
|
+
top_bridges: list[tuple[str, float]] = []
|
|
52
76
|
try:
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
)
|
|
76
|
-
if not has_bridge:
|
|
77
|
-
self._db.execute(
|
|
78
|
-
"ALTER TABLE fact_importance "
|
|
79
|
-
"ADD COLUMN bridge_score REAL DEFAULT 0.0",
|
|
80
|
-
(),
|
|
81
|
-
)
|
|
82
|
-
except Exception:
|
|
83
|
-
pass
|
|
84
|
-
|
|
85
|
-
# Persist to fact_importance (with bridge_score)
|
|
86
|
-
for node_id in graph.nodes():
|
|
87
|
-
pr_score = pagerank.get(node_id, 0.0)
|
|
88
|
-
comm_id = communities.get(node_id)
|
|
89
|
-
deg_cent = centrality.get(node_id, 0.0)
|
|
90
|
-
br_score = bridge_scores.get(node_id, 0.0)
|
|
91
|
-
try:
|
|
92
|
-
self._db.execute(
|
|
93
|
-
"INSERT OR REPLACE INTO fact_importance "
|
|
94
|
-
"(fact_id, profile_id, pagerank_score, community_id, "
|
|
95
|
-
" degree_centrality, bridge_score, computed_at) "
|
|
96
|
-
"VALUES (?, ?, ?, ?, ?, ?, datetime('now'))",
|
|
97
|
-
(node_id, profile_id, round(pr_score, 6),
|
|
98
|
-
comm_id, round(deg_cent, 4),
|
|
99
|
-
round(br_score, 6)),
|
|
100
|
-
)
|
|
101
|
-
except Exception:
|
|
102
|
-
# Fallback without bridge_score if column doesn't exist
|
|
103
|
-
self._db.execute(
|
|
104
|
-
"INSERT OR REPLACE INTO fact_importance "
|
|
105
|
-
"(fact_id, profile_id, pagerank_score, community_id, "
|
|
106
|
-
" degree_centrality, computed_at) "
|
|
107
|
-
"VALUES (?, ?, ?, ?, ?, datetime('now'))",
|
|
108
|
-
(node_id, profile_id, round(pr_score, 6),
|
|
109
|
-
comm_id, round(deg_cent, 4)),
|
|
110
|
-
)
|
|
77
|
+
rows = self._db.execute(
|
|
78
|
+
"SELECT fact_id, pagerank_score, community_id, bridge_score "
|
|
79
|
+
"FROM fact_importance WHERE profile_id = ? "
|
|
80
|
+
"ORDER BY pagerank_score DESC",
|
|
81
|
+
(profile_id,),
|
|
82
|
+
)
|
|
83
|
+
ranked = [dict(row) for row in rows]
|
|
84
|
+
for row in ranked:
|
|
85
|
+
if row.get("community_id") is not None:
|
|
86
|
+
communities[row["fact_id"]] = int(row["community_id"])
|
|
87
|
+
top_5 = [
|
|
88
|
+
(row["fact_id"], round(float(row["pagerank_score"] or 0.0), 4))
|
|
89
|
+
for row in ranked[:5]
|
|
90
|
+
]
|
|
91
|
+
bridges = [
|
|
92
|
+
(row["fact_id"], float(row.get("bridge_score") or 0.0))
|
|
93
|
+
for row in ranked
|
|
94
|
+
]
|
|
95
|
+
bridge_count = len([1 for _fid, score in bridges if score > 0.1])
|
|
96
|
+
top_bridges = sorted(bridges, key=lambda item: -item[1])[:5]
|
|
97
|
+
except Exception as exc:
|
|
98
|
+
logger.debug("Reading back graph metrics failed: %s", exc)
|
|
111
99
|
|
|
112
|
-
|
|
100
|
+
labels: dict[int, str] = {}
|
|
101
|
+
if communities:
|
|
113
102
|
try:
|
|
103
|
+
labels = self.compute_community_labels(profile_id, communities)
|
|
114
104
|
from superlocalmemory.infra.data_root import canonical_data_root
|
|
115
105
|
labels_dir = canonical_data_root()
|
|
116
106
|
labels_dir.mkdir(parents=True, exist_ok=True)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
bridge_count
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
"labels=%s",
|
|
139
|
-
graph.number_of_nodes(), unique_communities,
|
|
140
|
-
bridge_count, labels,
|
|
141
|
-
)
|
|
142
|
-
|
|
143
|
-
return {
|
|
144
|
-
"node_count": graph.number_of_nodes(),
|
|
145
|
-
"edge_count": graph.number_of_edges(),
|
|
146
|
-
"community_count": unique_communities,
|
|
147
|
-
"top_5_nodes": [
|
|
148
|
-
(nid, round(score, 4)) for nid, score in top_5
|
|
149
|
-
],
|
|
150
|
-
"bridge_count": bridge_count,
|
|
151
|
-
"top_bridge_nodes": [
|
|
152
|
-
(nid, round(s, 4)) for nid, s in top_bridges
|
|
153
|
-
],
|
|
154
|
-
"community_labels": labels,
|
|
155
|
-
}
|
|
156
|
-
except Exception as exc:
|
|
157
|
-
logger.debug("GraphAnalyzer.compute_and_store failed: %s", exc)
|
|
158
|
-
return {
|
|
159
|
-
"node_count": 0,
|
|
160
|
-
"edge_count": 0,
|
|
161
|
-
"community_count": 0,
|
|
162
|
-
"top_5_nodes": [],
|
|
163
|
-
}
|
|
107
|
+
(labels_dir / f"{profile_id}_community_labels.json").write_text(
|
|
108
|
+
json.dumps(labels, indent=2),
|
|
109
|
+
)
|
|
110
|
+
except Exception as exc:
|
|
111
|
+
logger.debug("Community labels skipped: %s", exc)
|
|
112
|
+
|
|
113
|
+
logger.info("GraphAnalyzer: %s", report.summary())
|
|
114
|
+
return {
|
|
115
|
+
"node_count": report.written,
|
|
116
|
+
"edge_count": report.edges,
|
|
117
|
+
"community_count": report.communities,
|
|
118
|
+
"top_5_nodes": top_5,
|
|
119
|
+
"bridge_count": bridge_count,
|
|
120
|
+
"top_bridge_nodes": [
|
|
121
|
+
(fid, round(score, 4)) for fid, score in top_bridges
|
|
122
|
+
],
|
|
123
|
+
"community_labels": labels,
|
|
124
|
+
"engine": report.engine,
|
|
125
|
+
"isolated_facts": report.isolated,
|
|
126
|
+
"duration_ms": report.duration_ms,
|
|
127
|
+
}
|
|
164
128
|
|
|
165
129
|
def compute_pagerank(
|
|
166
130
|
self,
|