superlocalmemory 4.0.7 → 4.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/README.md +3 -3
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/CLAUDE.md +3 -3
- package/plugin/agents/slm-governance-advisor.md +1 -1
- package/plugin/agents/slm-loop-runner.md +1 -1
- package/plugin/agents/slm-memory-advisor.md +1 -1
- package/plugin/agents/slm-optimize-advisor.md +1 -1
- package/plugin/requirements.txt +1 -1
- package/plugin/skills/slm-cache/SKILL.md +1 -1
- package/plugin/skills/slm-compress/SKILL.md +1 -1
- package/plugin/skills/slm-governance/SKILL.md +1 -1
- package/plugin/skills/slm-graph/SKILL.md +1 -1
- package/plugin/skills/slm-loop/SKILL.md +1 -1
- package/plugin/skills/slm-mesh/SKILL.md +1 -1
- package/plugin/skills/slm-profile/SKILL.md +1 -1
- package/plugin/skills/slm-recall/SKILL.md +1 -1
- package/plugin/skills/slm-remember/SKILL.md +1 -1
- package/plugin/skills/slm-scope/SKILL.md +1 -1
- package/plugin/skills/slm-session/SKILL.md +1 -1
- package/plugin/skills/slm-status/SKILL.md +1 -1
- package/plugin-src/rules/AGENTS.md +1 -1
- package/pyproject.toml +1 -1
- package/src/superlocalmemory/__init__.py +1 -1
- package/src/superlocalmemory/cli/summary_cmd.py +23 -3
- package/src/superlocalmemory/code_graph/bridge/maintenance.py +7 -1
- package/src/superlocalmemory/core/consolidation_engine.py +14 -15
- package/src/superlocalmemory/core/recall_worker.py +4 -0
- package/src/superlocalmemory/evolution/skill_evolver.py +16 -1
- package/src/superlocalmemory/hooks/hook_handlers.py +38 -11
- package/src/superlocalmemory/learning/pattern_miner.py +12 -7
- package/src/superlocalmemory/mcp/profiles.py +10 -3
- package/src/superlocalmemory/mcp/server.py +7 -0
- package/src/superlocalmemory/mcp/tools_summaries.py +147 -0
- package/src/superlocalmemory/server/consolidation_runner.py +140 -0
- package/src/superlocalmemory/server/recall_serializer.py +34 -2
- package/src/superlocalmemory/server/routes/agents.py +52 -8
- package/src/superlocalmemory/server/routes/brain.py +108 -0
- package/src/superlocalmemory/server/routes/memories.py +153 -0
- package/src/superlocalmemory/server/routes/v3_api.py +24 -46
- package/src/superlocalmemory/server/unified_daemon.py +107 -0
- package/src/superlocalmemory/summaries/base.py +159 -0
- package/src/superlocalmemory/summaries/daily_reflection.py +55 -8
- package/src/superlocalmemory/summaries/project_work_log.py +23 -7
- package/src/superlocalmemory/summaries/session_summary.py +9 -5
- package/src/superlocalmemory/ui/index.html +9 -3
- package/src/superlocalmemory/ui/js/od-boundedloops.js +324 -0
- package/src/superlocalmemory/ui/js/od-memories.js +337 -12
- package/src/superlocalmemory/ui/js/od-mesh.js +97 -5
- package/src/superlocalmemory/ui/js/od-operations.js +1 -150
- package/src/superlocalmemory/ui/js/od-optimize.js +36 -9
- package/src/superlocalmemory/ui/js/od-shell.js +10 -0
|
@@ -747,6 +747,159 @@ async def search_memories(request: Request, body: SearchRequest):
|
|
|
747
747
|
end_recall()
|
|
748
748
|
|
|
749
749
|
|
|
750
|
+
@router.get("/api/summary")
|
|
751
|
+
async def get_summary(request: Request, kind: str = "day", target: str = ""):
|
|
752
|
+
"""Readable summary of memories: a day, a project, or one session (#113).
|
|
753
|
+
|
|
754
|
+
The dashboard surface for the summary layer. 4.0.6 shipped the generators
|
|
755
|
+
with no caller, 4.0.7 added the CLI, 4.0.8 adds this and the MCP tool — the
|
|
756
|
+
"no command, tool or endpoint" gap, closed at the third point.
|
|
757
|
+
|
|
758
|
+
Always returns ``coverage`` and ``source_fact_ids``: a summary that hides how
|
|
759
|
+
much it covered is the opaque generic summary issue #113 warned against.
|
|
760
|
+
Reads memory.db directly; never runs during remember or recall.
|
|
761
|
+
"""
|
|
762
|
+
from datetime import date as _date, timedelta as _timedelta
|
|
763
|
+
|
|
764
|
+
kind = (kind or "day").strip().lower()
|
|
765
|
+
if kind not in ("day", "project", "session"):
|
|
766
|
+
raise HTTPException(status_code=400, detail=f"unknown summary kind '{kind}'")
|
|
767
|
+
|
|
768
|
+
profile = get_active_profile()
|
|
769
|
+
from superlocalmemory.infra.data_root import state_path
|
|
770
|
+
db_path = state_path("memory.db")
|
|
771
|
+
if not db_path.exists():
|
|
772
|
+
raise HTTPException(status_code=404, detail="no memory database")
|
|
773
|
+
|
|
774
|
+
# Pass the loaded config so Mode B/C write the summary. Omitting it silently
|
|
775
|
+
# forces the extractive path for every caller regardless of mode.
|
|
776
|
+
try:
|
|
777
|
+
from superlocalmemory.core.config import SLMConfig
|
|
778
|
+
cfg = SLMConfig.load()
|
|
779
|
+
except Exception:
|
|
780
|
+
cfg = None
|
|
781
|
+
|
|
782
|
+
try:
|
|
783
|
+
if kind == "day":
|
|
784
|
+
from superlocalmemory.summaries import generate_daily_reflection
|
|
785
|
+
day = (target or "").strip() or _date.today().isoformat()
|
|
786
|
+
if day == "today":
|
|
787
|
+
day = _date.today().isoformat()
|
|
788
|
+
elif day == "yesterday":
|
|
789
|
+
day = (_date.today() - _timedelta(days=1)).isoformat()
|
|
790
|
+
result = generate_daily_reflection(db_path, day, profile, cfg)
|
|
791
|
+
elif kind == "project":
|
|
792
|
+
if not (target or "").strip():
|
|
793
|
+
raise HTTPException(status_code=400, detail="project requires target")
|
|
794
|
+
from superlocalmemory.summaries import generate_project_work_log
|
|
795
|
+
result = generate_project_work_log(db_path, target.strip(), profile, cfg)
|
|
796
|
+
else:
|
|
797
|
+
if not (target or "").strip():
|
|
798
|
+
raise HTTPException(status_code=400, detail="session requires target")
|
|
799
|
+
from superlocalmemory.summaries import generate_session_summary
|
|
800
|
+
result = generate_session_summary(db_path, target.strip(), profile, cfg)
|
|
801
|
+
except HTTPException:
|
|
802
|
+
raise
|
|
803
|
+
except Exception:
|
|
804
|
+
raise _internal_error("Summary generation error")
|
|
805
|
+
|
|
806
|
+
return {
|
|
807
|
+
"kind": result.kind,
|
|
808
|
+
"profile_id": result.profile_id,
|
|
809
|
+
"summary": result.content,
|
|
810
|
+
"coverage": result.coverage,
|
|
811
|
+
"generated_by": result.generated_by,
|
|
812
|
+
"source_fact_ids": result.source_fact_ids,
|
|
813
|
+
"source_count": len(result.source_fact_ids),
|
|
814
|
+
"metadata": result.metadata,
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
@router.get("/api/summary/projects")
|
|
819
|
+
async def get_summary_projects(request: Request):
|
|
820
|
+
"""Projects SuperLocalMemory has actually observed, for the summary picker.
|
|
821
|
+
|
|
822
|
+
WHY THIS EXISTS
|
|
823
|
+
---------------
|
|
824
|
+
SLM is installed globally and the dashboard is a browser tab — it has no
|
|
825
|
+
working directory, so there is no such thing as "this project" from the
|
|
826
|
+
server's point of view. 4.0.8 shipped a "This project" button that sent an
|
|
827
|
+
empty target and produced "project requires target" every time. A button
|
|
828
|
+
that cannot know its own answer is the wrong control; a list of the projects
|
|
829
|
+
we have seen is the right one.
|
|
830
|
+
|
|
831
|
+
Scope comes from ``tool_events.project_path`` — the directory an agent was
|
|
832
|
+
working in when it called SLM. Deliberately NOT ``entity_profiles.
|
|
833
|
+
project_name``, which has exactly one distinct value on a real store (see
|
|
834
|
+
the note at the top of summaries/project_work_log.py).
|
|
835
|
+
|
|
836
|
+
``tool_events`` is a bounded ring buffer, so this lists recently active
|
|
837
|
+
projects rather than every project in history. ``truncated`` says so
|
|
838
|
+
honestly instead of implying the list is exhaustive.
|
|
839
|
+
"""
|
|
840
|
+
profile = get_active_profile()
|
|
841
|
+
try:
|
|
842
|
+
conn = get_db_connection()
|
|
843
|
+
# get_db_connection() hands back a SHARED read connection, and other
|
|
844
|
+
# handlers set row_factory on it. Never index these rows positionally —
|
|
845
|
+
# whichever handler ran last decides whether r[0] is a column or a
|
|
846
|
+
# KeyError. Name the columns and read them by name.
|
|
847
|
+
conn.row_factory = dict_factory
|
|
848
|
+
cursor = conn.cursor()
|
|
849
|
+
cursor.execute(
|
|
850
|
+
"""
|
|
851
|
+
SELECT project_path AS path, COUNT(*) AS events
|
|
852
|
+
FROM tool_events
|
|
853
|
+
WHERE project_path IS NOT NULL AND project_path != ''
|
|
854
|
+
AND profile_id = ?
|
|
855
|
+
GROUP BY project_path
|
|
856
|
+
ORDER BY events DESC
|
|
857
|
+
LIMIT 50
|
|
858
|
+
""",
|
|
859
|
+
(profile,),
|
|
860
|
+
)
|
|
861
|
+
rows = cursor.fetchall()
|
|
862
|
+
total = cursor.execute(
|
|
863
|
+
"SELECT COUNT(*) AS n FROM tool_events"
|
|
864
|
+
).fetchone()["n"]
|
|
865
|
+
except Exception:
|
|
866
|
+
raise _internal_error("Project list error")
|
|
867
|
+
|
|
868
|
+
projects = [
|
|
869
|
+
{
|
|
870
|
+
"path": r["path"],
|
|
871
|
+
"events": r["events"],
|
|
872
|
+
"label": _project_label(r["path"]),
|
|
873
|
+
}
|
|
874
|
+
for r in rows
|
|
875
|
+
]
|
|
876
|
+
return {
|
|
877
|
+
"projects": projects,
|
|
878
|
+
"profile_id": profile,
|
|
879
|
+
# Surfaced so the UI can explain an unexpectedly short list rather than
|
|
880
|
+
# leaving the user to assume their project was never recorded.
|
|
881
|
+
"truncated": total >= _TOOL_EVENT_RING_SIZE,
|
|
882
|
+
"event_rows": total,
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
|
|
886
|
+
#: tool_events is capped; at the cap the project list is a recent window, not history.
|
|
887
|
+
_TOOL_EVENT_RING_SIZE = 2000
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
def _project_label(path: str) -> str:
|
|
891
|
+
"""Short, human label for a project path.
|
|
892
|
+
|
|
893
|
+
Full paths are long and share prefixes ("/Users/x/Documents/official/..."),
|
|
894
|
+
so a dropdown of raw paths is unreadable. Last two segments keep sibling
|
|
895
|
+
projects distinguishable without the noise.
|
|
896
|
+
"""
|
|
897
|
+
parts = [p for p in str(path).replace("\\", "/").split("/") if p]
|
|
898
|
+
if not parts:
|
|
899
|
+
return str(path)
|
|
900
|
+
return "/".join(parts[-2:]) if len(parts) > 1 else parts[-1]
|
|
901
|
+
|
|
902
|
+
|
|
750
903
|
@router.get("/api/clusters")
|
|
751
904
|
async def get_clusters(request: Request):
|
|
752
905
|
"""Get cluster information with member counts and statistics."""
|
|
@@ -1932,55 +1932,33 @@ async def trigger_consolidation(request: Request):
|
|
|
1932
1932
|
profile_id=pid,
|
|
1933
1933
|
)
|
|
1934
1934
|
|
|
1935
|
-
#
|
|
1936
|
-
#
|
|
1937
|
-
#
|
|
1938
|
-
#
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
#
|
|
1943
|
-
# v3.4.64: ConsolidationEngine.consolidate() is CPU/IO bound (seconds
|
|
1944
|
-
# to minutes). Calling it directly in an async route blocks the ASGI
|
|
1945
|
-
# event loop. Moved into asyncio.to_thread() so the event loop stays
|
|
1946
|
-
# live. The runtime.operation() lease is acquired INSIDE the thread —
|
|
1947
|
-
# blocking a thread is fine; blocking the event loop is not.
|
|
1948
|
-
import asyncio as _asyncio
|
|
1949
|
-
from superlocalmemory.core.config import SLMConfig
|
|
1950
|
-
from superlocalmemory.storage.database import DatabaseManager
|
|
1951
|
-
from superlocalmemory.storage import schema as _schema
|
|
1952
|
-
from superlocalmemory.core.consolidation_engine import ConsolidationEngine
|
|
1953
|
-
from superlocalmemory.server.profile_runtime import get_profile_runtime
|
|
1954
|
-
|
|
1955
|
-
_app_state = request.app.state
|
|
1935
|
+
# 4.0.8: the body of this handler moved to server/consolidation_runner
|
|
1936
|
+
# so the periodic daemon trigger and this endpoint run the SAME code
|
|
1937
|
+
# under the SAME lock. Two copies would be two definitions of
|
|
1938
|
+
# "consolidated", and only one of them would get maintained.
|
|
1939
|
+
from superlocalmemory.server.consolidation_runner import (
|
|
1940
|
+
run_full_consolidation,
|
|
1941
|
+
)
|
|
1956
1942
|
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1943
|
+
# background=true returns as soon as the pass is scheduled. The
|
|
1944
|
+
# session-end hook needs this: a full consolidation runs for seconds to
|
|
1945
|
+
# minutes, and a hook that waits for it either blocks the user's shell
|
|
1946
|
+
# or times out and wrongly concludes the run failed.
|
|
1947
|
+
if body.get("background"):
|
|
1948
|
+
import asyncio as _asyncio
|
|
1949
|
+
|
|
1950
|
+
_app_state = request.app.state
|
|
1951
|
+
_asyncio.create_task(
|
|
1952
|
+
run_full_consolidation(
|
|
1953
|
+
_app_state, pid, lightweight=lightweight, trigger="hook",
|
|
1965
1954
|
)
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
from superlocalmemory.learning.consolidation_worker import (
|
|
1970
|
-
ConsolidationWorker,
|
|
1971
|
-
)
|
|
1972
|
-
learning_db = config.base_dir / "learning.db"
|
|
1973
|
-
cw = ConsolidationWorker(str(config.db_path), str(learning_db))
|
|
1974
|
-
pattern_count = cw._generate_patterns(pid, False)
|
|
1975
|
-
res["patterns_mined"] = pattern_count
|
|
1976
|
-
logger.info(
|
|
1977
|
-
"Auto-mined %d patterns after consolidation", pattern_count
|
|
1978
|
-
)
|
|
1979
|
-
except Exception as exc:
|
|
1980
|
-
logger.debug("Pattern mining after consolidation failed: %s", exc)
|
|
1981
|
-
return res
|
|
1955
|
+
)
|
|
1956
|
+
authorization.complete()
|
|
1957
|
+
return {"success": True, "started": True, "background": True}
|
|
1982
1958
|
|
|
1983
|
-
result = await
|
|
1959
|
+
result = await run_full_consolidation(
|
|
1960
|
+
request.app.state, pid, lightweight=lightweight, trigger="http",
|
|
1961
|
+
)
|
|
1984
1962
|
authorization.complete()
|
|
1985
1963
|
return {"success": True, **result}
|
|
1986
1964
|
except HTTPException:
|
|
@@ -418,6 +418,10 @@ class EngineRecallAdapter:
|
|
|
418
418
|
memory_map={k: _sanitize_json_text(v) for k, v in memory_map.items()},
|
|
419
419
|
per_fact_max=getattr(_rc, "recall_per_fact_max_chars", 2400),
|
|
420
420
|
total_max=getattr(_rc, "recall_total_max_chars", 12000),
|
|
421
|
+
# Option B: markers only on session-bearing recalls. A marker can
|
|
422
|
+
# only buy a learning signal when a pending_outcomes row exists
|
|
423
|
+
# to settle, and those exist only when session_id is present.
|
|
424
|
+
include_marker=bool(session_id),
|
|
421
425
|
)
|
|
422
426
|
for _r in results:
|
|
423
427
|
_r["content"] = _sanitize_json_text(_r.get("content", ""))
|
|
@@ -860,6 +864,86 @@ def _start_idle_watchdog(timeout_sec: int) -> None:
|
|
|
860
864
|
t.start()
|
|
861
865
|
|
|
862
866
|
|
|
867
|
+
# ---------------------------------------------------------------------------
|
|
868
|
+
# Periodic full consolidation (4.0.8)
|
|
869
|
+
# ---------------------------------------------------------------------------
|
|
870
|
+
#
|
|
871
|
+
# Steps 8-11 of ConsolidationEngine — behavioural assertions, soft prompts,
|
|
872
|
+
# skill performance, skill evolution — had no automatic trigger at all. The
|
|
873
|
+
# session-end hook is the fast path; this timer is the guarantee, because a hook
|
|
874
|
+
# that does not fire is indistinguishable from a feature that does not exist.
|
|
875
|
+
#
|
|
876
|
+
# Two constraints shape the schedule:
|
|
877
|
+
#
|
|
878
|
+
# 1. Remember and recall latency must not move. So the pass only starts when
|
|
879
|
+
# the daemon has served nothing for _CONSOLIDATION_IDLE_SEC. Consolidation
|
|
880
|
+
# is catch-up work; there is never a reason for it to compete with a live
|
|
881
|
+
# request. If the machine is busy every time we look, we simply skip and
|
|
882
|
+
# check again next tick.
|
|
883
|
+
# 2. It must not pile up. run_full_consolidation() holds a lock and skips
|
|
884
|
+
# rather than queues, so a slow pass cannot be overlapped by the next tick
|
|
885
|
+
# or by the session-end hook.
|
|
886
|
+
|
|
887
|
+
#: How often to consider running. Not how often it runs.
|
|
888
|
+
_CONSOLIDATION_CHECK_SEC = int(os.environ.get("SLM_CONSOLIDATION_CHECK_SEC", 900))
|
|
889
|
+
|
|
890
|
+
#: Minimum quiet period before a scheduled pass may start.
|
|
891
|
+
_CONSOLIDATION_IDLE_SEC = int(os.environ.get("SLM_CONSOLIDATION_IDLE_SEC", 300))
|
|
892
|
+
|
|
893
|
+
#: Minimum gap between two scheduled passes.
|
|
894
|
+
_CONSOLIDATION_MIN_GAP_SEC = int(
|
|
895
|
+
os.environ.get("SLM_CONSOLIDATION_MIN_GAP_SEC", 6 * 3600)
|
|
896
|
+
)
|
|
897
|
+
|
|
898
|
+
#: Delay before the first check, so daemon startup is never slowed by it.
|
|
899
|
+
_CONSOLIDATION_FIRST_DELAY_SEC = int(
|
|
900
|
+
os.environ.get("SLM_CONSOLIDATION_FIRST_DELAY_SEC", 120)
|
|
901
|
+
)
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
async def _consolidation_timer_loop(application: FastAPI) -> None:
|
|
905
|
+
"""Run full consolidation on a schedule, but only while the daemon is idle."""
|
|
906
|
+
from superlocalmemory.server.consolidation_runner import run_full_consolidation
|
|
907
|
+
|
|
908
|
+
await asyncio.sleep(_CONSOLIDATION_FIRST_DELAY_SEC)
|
|
909
|
+
last_run = 0.0
|
|
910
|
+
|
|
911
|
+
while True:
|
|
912
|
+
try:
|
|
913
|
+
await asyncio.sleep(_CONSOLIDATION_CHECK_SEC)
|
|
914
|
+
|
|
915
|
+
now = time.monotonic()
|
|
916
|
+
if last_run and (now - last_run) < _CONSOLIDATION_MIN_GAP_SEC:
|
|
917
|
+
continue
|
|
918
|
+
if (now - _last_activity) < _CONSOLIDATION_IDLE_SEC:
|
|
919
|
+
continue # busy — try again next tick
|
|
920
|
+
|
|
921
|
+
# get_engine_lazy, not state.engine directly: a mode switch nulls
|
|
922
|
+
# state.engine, and reading it raw would silently disable scheduled
|
|
923
|
+
# consolidation until the next daemon restart.
|
|
924
|
+
from superlocalmemory.server.routes.helpers import get_engine_lazy
|
|
925
|
+
|
|
926
|
+
engine = get_engine_lazy(application.state)
|
|
927
|
+
profile_id = getattr(engine, "profile_id", None) if engine else None
|
|
928
|
+
if not profile_id:
|
|
929
|
+
continue
|
|
930
|
+
|
|
931
|
+
result = await run_full_consolidation(
|
|
932
|
+
application.state, profile_id, trigger="timer",
|
|
933
|
+
)
|
|
934
|
+
# Only a pass that actually ran resets the clock; a skip must not
|
|
935
|
+
# buy another six hours of silence.
|
|
936
|
+
if not result.get("skipped"):
|
|
937
|
+
last_run = time.monotonic()
|
|
938
|
+
except asyncio.CancelledError:
|
|
939
|
+
raise
|
|
940
|
+
except Exception:
|
|
941
|
+
# Never let a bad pass kill the loop — that would silently disable
|
|
942
|
+
# consolidation for the life of the daemon, which is the failure
|
|
943
|
+
# mode this timer exists to end. Log loudly and try again.
|
|
944
|
+
logger.exception("scheduled consolidation failed; will retry")
|
|
945
|
+
|
|
946
|
+
|
|
863
947
|
# ---------------------------------------------------------------------------
|
|
864
948
|
# Legacy port TCP redirect (backward compat for port 8767)
|
|
865
949
|
# ---------------------------------------------------------------------------
|
|
@@ -2462,6 +2546,15 @@ async def lifespan(application: FastAPI):
|
|
|
2462
2546
|
except Exception as e:
|
|
2463
2547
|
logger.warning("optimize module not available: %s", e)
|
|
2464
2548
|
|
|
2549
|
+
# 4.0.8: periodic full consolidation. Idle-gated and lock-guarded — see
|
|
2550
|
+
# _consolidation_timer_loop. Started here rather than at import so a daemon
|
|
2551
|
+
# that never completes startup never schedules work.
|
|
2552
|
+
_consol_task = getattr(application.state, "_consolidation_task", None)
|
|
2553
|
+
if _consol_task is None or _consol_task.done():
|
|
2554
|
+
application.state._consolidation_task = asyncio.create_task(
|
|
2555
|
+
_consolidation_timer_loop(application)
|
|
2556
|
+
)
|
|
2557
|
+
|
|
2465
2558
|
# v3.6.7: Start MCP Streamable-HTTP session manager (GOTCHA #1).
|
|
2466
2559
|
# streamable_http_app() carries its own Starlette lifespan that initialises
|
|
2467
2560
|
# an anyio task group inside the session manager. Without entering that
|
|
@@ -2537,6 +2630,16 @@ async def lifespan(application: FastAPI):
|
|
|
2537
2630
|
except Exception: # pragma: no cover — defensive
|
|
2538
2631
|
pass
|
|
2539
2632
|
|
|
2633
|
+
# Cancel the periodic consolidation loop. Not awaited to completion — a pass
|
|
2634
|
+
# can take minutes and shutdown must not block on it; the lock and the
|
|
2635
|
+
# engine's own transaction boundaries make an interrupted pass safe to redo.
|
|
2636
|
+
try:
|
|
2637
|
+
_consol = getattr(application.state, "_consolidation_task", None)
|
|
2638
|
+
if _consol is not None and not _consol.done():
|
|
2639
|
+
_consol.cancel()
|
|
2640
|
+
except Exception: # pragma: no cover — defensive
|
|
2641
|
+
pass
|
|
2642
|
+
|
|
2540
2643
|
# Cancel optimize metrics flush loop + run final flush before shutdown
|
|
2541
2644
|
try:
|
|
2542
2645
|
_flush_task = getattr(application.state, "_optimize_flush_task", None)
|
|
@@ -3879,6 +3982,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
|
|
|
3879
3982
|
memory_map={k: _sanitize_json_text(v) for k, v in memory_map.items()},
|
|
3880
3983
|
per_fact_max=getattr(_rc, "recall_per_fact_max_chars", 2400),
|
|
3881
3984
|
total_max=getattr(_rc, "recall_total_max_chars", 12000),
|
|
3985
|
+
# Option B: markers only on session-bearing recalls. A marker can
|
|
3986
|
+
# only buy a learning signal when a pending_outcomes row exists
|
|
3987
|
+
# to settle, and those exist only when session_id is present.
|
|
3988
|
+
include_marker=bool(session_id),
|
|
3882
3989
|
full=full,
|
|
3883
3990
|
include_source=include_source,
|
|
3884
3991
|
)
|
|
@@ -98,6 +98,165 @@ GENERATED_BY_LLM_C = "llm_c"
|
|
|
98
98
|
"""Cloud LLM (Mode C). Falls back via llm_b to extractive."""
|
|
99
99
|
|
|
100
100
|
|
|
101
|
+
# ── highlight formatting ────────────────────────────────────────────────────
|
|
102
|
+
|
|
103
|
+
#: Display width for one bullet in a summary body.
|
|
104
|
+
#:
|
|
105
|
+
#: Chosen for a bullet, not for a paragraph. The generators originally truncated
|
|
106
|
+
#: at 300 characters and nothing else, which looks fine on a synthetic corpus of
|
|
107
|
+
#: one-line facts and falls apart on a real store: agent-written facts routinely
|
|
108
|
+
#: contain blank lines and markdown headings, so a 300-character slice rendered
|
|
109
|
+
#: as six or more display lines and the bullet list stopped being a list.
|
|
110
|
+
HIGHLIGHT_CHARS = 180
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def format_highlight(content: str, limit: int = HIGHLIGHT_CHARS) -> str:
|
|
114
|
+
"""Collapse *content* to a single readable line for a summary bullet.
|
|
115
|
+
|
|
116
|
+
Three things, in order:
|
|
117
|
+
|
|
118
|
+
1. **Flatten whitespace.** Newlines, blank lines and runs of spaces all
|
|
119
|
+
become one space. This is the fix that matters: character truncation
|
|
120
|
+
alone cannot keep a multi-paragraph fact on one line, and every
|
|
121
|
+
generator here writes into a bullet list.
|
|
122
|
+
2. **Prefer a whole first sentence** when there is one and it fits. A
|
|
123
|
+
complete sentence reads better than a slice of one, and the first
|
|
124
|
+
sentence of a report is usually its summary.
|
|
125
|
+
3. **Otherwise cut at a word boundary** and mark the cut with an ellipsis,
|
|
126
|
+
so it is visible that text was dropped rather than that a fact ended
|
|
127
|
+
mid-word.
|
|
128
|
+
|
|
129
|
+
Markdown heading markers are stripped because a flattened ``**Summary**``
|
|
130
|
+
mid-sentence reads as noise.
|
|
131
|
+
"""
|
|
132
|
+
import re
|
|
133
|
+
|
|
134
|
+
text = re.sub(r"\s+", " ", (content or "")).strip()
|
|
135
|
+
# Leading/inline markdown emphasis and heading marks, once flattened, add
|
|
136
|
+
# nothing but clutter to a one-line bullet.
|
|
137
|
+
text = re.sub(r"(?:^|\s)#{1,6}\s+", " ", text)
|
|
138
|
+
text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
|
|
139
|
+
text = re.sub(r"\s+", " ", text).strip()
|
|
140
|
+
|
|
141
|
+
if not text:
|
|
142
|
+
return ""
|
|
143
|
+
if len(text) <= limit:
|
|
144
|
+
return text
|
|
145
|
+
|
|
146
|
+
# A complete first sentence, if it fits comfortably.
|
|
147
|
+
match = re.match(r"(.+?[.!?])(?:\s|$)", text)
|
|
148
|
+
if match:
|
|
149
|
+
sentence = match.group(1).strip()
|
|
150
|
+
if len(sentence) <= limit:
|
|
151
|
+
return sentence
|
|
152
|
+
|
|
153
|
+
cut = text[:limit]
|
|
154
|
+
space = cut.rfind(" ")
|
|
155
|
+
if space > limit * 0.6: # don't cut a long unbroken token to a stub
|
|
156
|
+
cut = cut[:space]
|
|
157
|
+
return cut.rstrip(" ,;:—-") + "…"
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
# ── LLM output cleanup ──────────────────────────────────────────────────────
|
|
161
|
+
|
|
162
|
+
#: Sentences a chat-tuned model emits *around* the answer rather than as part of
|
|
163
|
+
#: it. Anchored to the start of a paragraph so they cannot match mid-content.
|
|
164
|
+
#:
|
|
165
|
+
#: Measured, not guessed: a Mode B summary on the author's own store opened with
|
|
166
|
+
#: "I apologize for the previous confusion. It seems that I misunderstood the
|
|
167
|
+
#: context of the texts provided.\n\nTo provide a concise summary paragraph, here
|
|
168
|
+
#: is a merge of all the key information:" — 180 characters of the model talking
|
|
169
|
+
#: to itself, shown to the user as their daily reflection.
|
|
170
|
+
_LLM_PREAMBLE = (
|
|
171
|
+
r"^(?:"
|
|
172
|
+
r"i\s+apologi[sz]e\b.*"
|
|
173
|
+
r"|i'?m\s+sorry\b.*"
|
|
174
|
+
r"|it\s+seems\s+that\s+i\b.*"
|
|
175
|
+
r"|sure[,!.]?\s*(?:thing)?\b.*"
|
|
176
|
+
r"|certainly[,!.]?\b.*"
|
|
177
|
+
r"|of\s+course[,!.]?\b.*"
|
|
178
|
+
r"|here(?:'s|\s+is|\s+are)\b[^.!?]*:"
|
|
179
|
+
r"|to\s+(?:provide|summari[sz]e|answer)\b[^.!?]*:"
|
|
180
|
+
r"|based\s+on\s+the\s+(?:facts|texts|information|data)\s+provided[,:]?"
|
|
181
|
+
r"|as\s+(?:an|a)\s+(?:ai|language\s+model)\b.*"
|
|
182
|
+
r")\s*$"
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
#: Closing pleasantries. Same anchoring rule.
|
|
186
|
+
_LLM_POSTAMBLE = (
|
|
187
|
+
r"^(?:"
|
|
188
|
+
r"(?:i\s+hope|hope)\s+(?:this|that)\s+helps\b.*"
|
|
189
|
+
r"|let\s+me\s+know\b.*"
|
|
190
|
+
r"|feel\s+free\s+to\b.*"
|
|
191
|
+
r"|would\s+you\s+like\s+me\s+to\b.*"
|
|
192
|
+
r")$"
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def clean_llm_summary(text: str) -> str:
|
|
197
|
+
"""Strip chat-assistant scaffolding from a model-written summary.
|
|
198
|
+
|
|
199
|
+
WHY THIS EXISTS
|
|
200
|
+
---------------
|
|
201
|
+
Mode B/C summaries are shown to the user as *their* memory, with no chat
|
|
202
|
+
framing around them. A chat-tuned model does not know that: it opens with an
|
|
203
|
+
apology or "Here is a concise summary:" and closes with "Let me know if you
|
|
204
|
+
want more detail". Both are addressed to a conversation that the reader
|
|
205
|
+
cannot see, and both make the product look broken.
|
|
206
|
+
|
|
207
|
+
The system prompt now asks for bare prose, which handles most of it. This is
|
|
208
|
+
the second line of defence, because instruction-following on a 3B local
|
|
209
|
+
model is not something to bet the displayed output on.
|
|
210
|
+
|
|
211
|
+
Conservative by construction: patterns are anchored to whole paragraphs or
|
|
212
|
+
whole leading sentences, so a summary that legitimately contains the word
|
|
213
|
+
"sure" mid-paragraph is untouched. If stripping would empty the text, the
|
|
214
|
+
original is returned — a scaffolded summary beats a blank one.
|
|
215
|
+
"""
|
|
216
|
+
import re
|
|
217
|
+
|
|
218
|
+
original = (text or "").strip()
|
|
219
|
+
if not original:
|
|
220
|
+
return ""
|
|
221
|
+
|
|
222
|
+
# Fenced code blocks wrapping the whole answer: keep the contents.
|
|
223
|
+
fenced = re.match(r"^```[a-zA-Z]*\n(.*?)\n?```$", original, re.DOTALL)
|
|
224
|
+
if fenced:
|
|
225
|
+
original = fenced.group(1).strip()
|
|
226
|
+
|
|
227
|
+
paras = [p.strip() for p in re.split(r"\n\s*\n", original) if p.strip()]
|
|
228
|
+
|
|
229
|
+
while paras and re.match(_LLM_PREAMBLE, paras[0], re.IGNORECASE | re.DOTALL):
|
|
230
|
+
paras.pop(0)
|
|
231
|
+
while paras and re.match(_LLM_POSTAMBLE, paras[-1], re.IGNORECASE | re.DOTALL):
|
|
232
|
+
paras.pop()
|
|
233
|
+
|
|
234
|
+
if not paras:
|
|
235
|
+
return original
|
|
236
|
+
|
|
237
|
+
# A preamble that shares a paragraph with real content: drop just the leading
|
|
238
|
+
# sentence, and only when what follows is substantial enough to stand alone.
|
|
239
|
+
lead = re.match(r"(.+?[.:!?])\s+(\S.*)$", paras[0], re.DOTALL)
|
|
240
|
+
if lead and re.match(_LLM_PREAMBLE, lead.group(1), re.IGNORECASE | re.DOTALL):
|
|
241
|
+
if len(lead.group(2).strip()) > 40:
|
|
242
|
+
paras[0] = lead.group(2).strip()
|
|
243
|
+
|
|
244
|
+
cleaned = "\n\n".join(paras).strip()
|
|
245
|
+
return cleaned or original
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
#: System prompt for every summary generator, Mode B and Mode C alike.
|
|
249
|
+
#:
|
|
250
|
+
#: Mode C always sent one; Mode B sent none at all, which is why local-model
|
|
251
|
+
#: output arrived wrapped in chat scaffolding while cloud output did not.
|
|
252
|
+
SUMMARY_SYSTEM_PROMPT = (
|
|
253
|
+
"You summarise a person's own saved notes for them. "
|
|
254
|
+
"Reply with the summary text only — no preamble, no apologies, no sign-off, "
|
|
255
|
+
"no markdown headings, and never refer to yourself or to these instructions. "
|
|
256
|
+
"Write plain prose in the third person about the work described."
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
|
|
101
260
|
def get_mode_str(config: object | None) -> str:
|
|
102
261
|
"""Extract the operating mode string ('a', 'b', or 'c') from a config."""
|
|
103
262
|
if config is None:
|
|
@@ -34,6 +34,9 @@ from datetime import date
|
|
|
34
34
|
from pathlib import Path
|
|
35
35
|
|
|
36
36
|
from .base import (
|
|
37
|
+
clean_llm_summary,
|
|
38
|
+
format_highlight,
|
|
39
|
+
SUMMARY_SYSTEM_PROMPT,
|
|
37
40
|
COVERAGE_FULL,
|
|
38
41
|
COVERAGE_INSUFFICIENT,
|
|
39
42
|
COVERAGE_UNAVAILABLE,
|
|
@@ -128,7 +131,7 @@ def generate_daily_reflection(
|
|
|
128
131
|
coverage = COVERAGE_FULL
|
|
129
132
|
|
|
130
133
|
# ── extractive summary (deterministic, always available) ──────────────────
|
|
131
|
-
extractive_content = _build_extractive_content(date_str, facts, fact_count)
|
|
134
|
+
extractive_content = _build_extractive_content(date_str, facts, fact_count, db_path, profile_id)
|
|
132
135
|
|
|
133
136
|
# ── LLM enrichment (optional) ─────────────────────────────────────────────
|
|
134
137
|
mode = get_mode_str(config)
|
|
@@ -162,8 +165,14 @@ def _build_extractive_content(
|
|
|
162
165
|
date_str: str,
|
|
163
166
|
facts: list[dict],
|
|
164
167
|
fact_count: int,
|
|
168
|
+
db_path: str | Path,
|
|
169
|
+
profile_id: str,
|
|
165
170
|
) -> str:
|
|
166
|
-
"""Build a deterministic extractive daily reflection.
|
|
171
|
+
"""Build a deterministic extractive daily reflection.
|
|
172
|
+
|
|
173
|
+
``db_path`` / ``profile_id`` are needed only to turn entity IDs into names
|
|
174
|
+
for the "Active entities" line — the facts themselves are already loaded.
|
|
175
|
+
"""
|
|
167
176
|
import json
|
|
168
177
|
|
|
169
178
|
lines = [
|
|
@@ -174,8 +183,7 @@ def _build_extractive_content(
|
|
|
174
183
|
]
|
|
175
184
|
for f in facts[:_BODY_FACTS]:
|
|
176
185
|
content = f.get("content", "")
|
|
177
|
-
|
|
178
|
-
content = content[:_MAX_FACT_CHARS - 3] + "..."
|
|
186
|
+
content = format_highlight(content)
|
|
179
187
|
lines.append(f" - {content}")
|
|
180
188
|
if fact_count > _BODY_FACTS:
|
|
181
189
|
lines.append(f" ... and {fact_count - _BODY_FACTS} additional facts.")
|
|
@@ -193,13 +201,50 @@ def _build_extractive_content(
|
|
|
193
201
|
|
|
194
202
|
if entity_counts:
|
|
195
203
|
top_entities = sorted(entity_counts.items(), key=lambda x: x[1], reverse=True)[:5]
|
|
196
|
-
|
|
204
|
+
# Resolve to names. canonical_entities_json stores entity IDs, so this
|
|
205
|
+
# line previously read "Active entities: 1666abc512904473 (127),
|
|
206
|
+
# 84288f2dde994afe (124)" — five 16-hex identifiers, which tell a reader
|
|
207
|
+
# nothing. They resolve to 'Fixed', 'Gateway', 'REVISED' and so on. This
|
|
208
|
+
# is the same defect 4.0.6 fixed in the Living Brain, where source
|
|
209
|
+
# quality listed internal identifiers; it survived here because this
|
|
210
|
+
# generator was never reachable to be looked at.
|
|
211
|
+
names = _entity_names(db_path, [e for e, _ in top_entities], profile_id)
|
|
212
|
+
labelled = [
|
|
213
|
+
(names.get(eid) or eid, count) for eid, count in top_entities
|
|
214
|
+
]
|
|
215
|
+
entity_str = ", ".join(f"{name} ({c})" for name, c in labelled)
|
|
197
216
|
lines.append("")
|
|
198
217
|
lines.append(f"Active entities: {entity_str}")
|
|
199
218
|
|
|
200
219
|
return "\n".join(lines)
|
|
201
220
|
|
|
202
221
|
|
|
222
|
+
def _entity_names(
|
|
223
|
+
db_path: str | Path, entity_ids: list[str], profile_id: str
|
|
224
|
+
) -> dict[str, str]:
|
|
225
|
+
"""Map entity_id → canonical_name. Missing or unreadable rows are omitted.
|
|
226
|
+
|
|
227
|
+
Fail-open: a summary is still useful with a raw id in it, so a query problem
|
|
228
|
+
must not lose the whole line. The caller falls back to the id per entity.
|
|
229
|
+
"""
|
|
230
|
+
if not entity_ids:
|
|
231
|
+
return {}
|
|
232
|
+
try:
|
|
233
|
+
conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
|
234
|
+
try:
|
|
235
|
+
placeholders = ",".join("?" for _ in entity_ids)
|
|
236
|
+
rows = conn.execute(
|
|
237
|
+
f"SELECT entity_id, canonical_name FROM canonical_entities "
|
|
238
|
+
f"WHERE entity_id IN ({placeholders}) AND profile_id = ?",
|
|
239
|
+
(*entity_ids, profile_id),
|
|
240
|
+
).fetchall()
|
|
241
|
+
finally:
|
|
242
|
+
conn.close()
|
|
243
|
+
return {r[0]: r[1] for r in rows if r[1]}
|
|
244
|
+
except sqlite3.Error:
|
|
245
|
+
return {}
|
|
246
|
+
|
|
247
|
+
|
|
203
248
|
def _try_llm(
|
|
204
249
|
date_str: str,
|
|
205
250
|
facts: list[dict],
|
|
@@ -249,6 +294,7 @@ def _call_ollama(
|
|
|
249
294
|
payload = json.dumps({
|
|
250
295
|
"model": model,
|
|
251
296
|
"prompt": full_prompt,
|
|
297
|
+
"system": SUMMARY_SYSTEM_PROMPT,
|
|
252
298
|
"stream": False,
|
|
253
299
|
"options": {"num_predict": 300},
|
|
254
300
|
}).encode()
|
|
@@ -259,7 +305,7 @@ def _call_ollama(
|
|
|
259
305
|
)
|
|
260
306
|
resp = urllib.request.urlopen(req, timeout=timeout)
|
|
261
307
|
data = json.loads(resp.read().decode())
|
|
262
|
-
text = data.get("response", "")
|
|
308
|
+
text = clean_llm_summary(data.get("response", ""))
|
|
263
309
|
return text if text and len(text) > 20 else None
|
|
264
310
|
except Exception as exc:
|
|
265
311
|
logger.debug("Ollama daily reflection failed: %s", exc)
|
|
@@ -283,11 +329,12 @@ def _call_cloud_llm(
|
|
|
283
329
|
full_prompt = f"{prompt}\n\nFacts:\n{fact_texts}"
|
|
284
330
|
text = llm.generate(
|
|
285
331
|
prompt=full_prompt,
|
|
286
|
-
system=
|
|
332
|
+
system=SUMMARY_SYSTEM_PROMPT,
|
|
287
333
|
max_tokens=300,
|
|
288
334
|
temperature=0.1,
|
|
289
335
|
)
|
|
290
|
-
|
|
336
|
+
cleaned = clean_llm_summary(text or "")
|
|
337
|
+
return cleaned if len(cleaned) > 20 else None
|
|
291
338
|
except Exception as exc:
|
|
292
339
|
logger.debug("Cloud LLM daily reflection failed: %s", exc)
|
|
293
340
|
return None
|