superlocalmemory 4.0.7 → 4.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +73 -0
  2. package/README.md +3 -3
  3. package/package.json +1 -1
  4. package/plugin/.claude-plugin/plugin.json +1 -1
  5. package/plugin/CLAUDE.md +3 -3
  6. package/plugin/agents/slm-governance-advisor.md +1 -1
  7. package/plugin/agents/slm-loop-runner.md +1 -1
  8. package/plugin/agents/slm-memory-advisor.md +1 -1
  9. package/plugin/agents/slm-optimize-advisor.md +1 -1
  10. package/plugin/requirements.txt +1 -1
  11. package/plugin/skills/slm-cache/SKILL.md +1 -1
  12. package/plugin/skills/slm-compress/SKILL.md +1 -1
  13. package/plugin/skills/slm-governance/SKILL.md +1 -1
  14. package/plugin/skills/slm-graph/SKILL.md +1 -1
  15. package/plugin/skills/slm-loop/SKILL.md +1 -1
  16. package/plugin/skills/slm-mesh/SKILL.md +1 -1
  17. package/plugin/skills/slm-profile/SKILL.md +1 -1
  18. package/plugin/skills/slm-recall/SKILL.md +1 -1
  19. package/plugin/skills/slm-remember/SKILL.md +1 -1
  20. package/plugin/skills/slm-scope/SKILL.md +1 -1
  21. package/plugin/skills/slm-session/SKILL.md +1 -1
  22. package/plugin/skills/slm-status/SKILL.md +1 -1
  23. package/plugin-src/rules/AGENTS.md +1 -1
  24. package/pyproject.toml +1 -1
  25. package/src/superlocalmemory/__init__.py +1 -1
  26. package/src/superlocalmemory/cli/summary_cmd.py +23 -3
  27. package/src/superlocalmemory/code_graph/bridge/maintenance.py +7 -1
  28. package/src/superlocalmemory/core/consolidation_engine.py +14 -15
  29. package/src/superlocalmemory/core/recall_worker.py +4 -0
  30. package/src/superlocalmemory/evolution/skill_evolver.py +16 -1
  31. package/src/superlocalmemory/hooks/hook_handlers.py +38 -11
  32. package/src/superlocalmemory/learning/pattern_miner.py +12 -7
  33. package/src/superlocalmemory/mcp/profiles.py +10 -3
  34. package/src/superlocalmemory/mcp/server.py +7 -0
  35. package/src/superlocalmemory/mcp/tools_summaries.py +147 -0
  36. package/src/superlocalmemory/server/consolidation_runner.py +140 -0
  37. package/src/superlocalmemory/server/recall_serializer.py +34 -2
  38. package/src/superlocalmemory/server/routes/agents.py +52 -8
  39. package/src/superlocalmemory/server/routes/brain.py +108 -0
  40. package/src/superlocalmemory/server/routes/memories.py +153 -0
  41. package/src/superlocalmemory/server/routes/v3_api.py +24 -46
  42. package/src/superlocalmemory/server/unified_daemon.py +107 -0
  43. package/src/superlocalmemory/summaries/base.py +159 -0
  44. package/src/superlocalmemory/summaries/daily_reflection.py +55 -8
  45. package/src/superlocalmemory/summaries/project_work_log.py +23 -7
  46. package/src/superlocalmemory/summaries/session_summary.py +9 -5
  47. package/src/superlocalmemory/ui/index.html +9 -3
  48. package/src/superlocalmemory/ui/js/od-boundedloops.js +324 -0
  49. package/src/superlocalmemory/ui/js/od-memories.js +337 -12
  50. package/src/superlocalmemory/ui/js/od-mesh.js +97 -5
  51. package/src/superlocalmemory/ui/js/od-operations.js +1 -150
  52. package/src/superlocalmemory/ui/js/od-optimize.js +36 -9
  53. package/src/superlocalmemory/ui/js/od-shell.js +10 -0
@@ -747,6 +747,159 @@ async def search_memories(request: Request, body: SearchRequest):
747
747
  end_recall()
748
748
 
749
749
 
750
+ @router.get("/api/summary")
751
+ async def get_summary(request: Request, kind: str = "day", target: str = ""):
752
+ """Readable summary of memories: a day, a project, or one session (#113).
753
+
754
+ The dashboard surface for the summary layer. 4.0.6 shipped the generators
755
+ with no caller, 4.0.7 added the CLI, 4.0.8 adds this and the MCP tool — the
756
+ "no command, tool or endpoint" gap, closed at the third point.
757
+
758
+ Always returns ``coverage`` and ``source_fact_ids``: a summary that hides how
759
+ much it covered is the opaque generic summary issue #113 warned against.
760
+ Reads memory.db directly; never runs during remember or recall.
761
+ """
762
+ from datetime import date as _date, timedelta as _timedelta
763
+
764
+ kind = (kind or "day").strip().lower()
765
+ if kind not in ("day", "project", "session"):
766
+ raise HTTPException(status_code=400, detail=f"unknown summary kind '{kind}'")
767
+
768
+ profile = get_active_profile()
769
+ from superlocalmemory.infra.data_root import state_path
770
+ db_path = state_path("memory.db")
771
+ if not db_path.exists():
772
+ raise HTTPException(status_code=404, detail="no memory database")
773
+
774
+ # Pass the loaded config so Mode B/C write the summary. Omitting it silently
775
+ # forces the extractive path for every caller regardless of mode.
776
+ try:
777
+ from superlocalmemory.core.config import SLMConfig
778
+ cfg = SLMConfig.load()
779
+ except Exception:
780
+ cfg = None
781
+
782
+ try:
783
+ if kind == "day":
784
+ from superlocalmemory.summaries import generate_daily_reflection
785
+ day = (target or "").strip() or _date.today().isoformat()
786
+ if day == "today":
787
+ day = _date.today().isoformat()
788
+ elif day == "yesterday":
789
+ day = (_date.today() - _timedelta(days=1)).isoformat()
790
+ result = generate_daily_reflection(db_path, day, profile, cfg)
791
+ elif kind == "project":
792
+ if not (target or "").strip():
793
+ raise HTTPException(status_code=400, detail="project requires target")
794
+ from superlocalmemory.summaries import generate_project_work_log
795
+ result = generate_project_work_log(db_path, target.strip(), profile, cfg)
796
+ else:
797
+ if not (target or "").strip():
798
+ raise HTTPException(status_code=400, detail="session requires target")
799
+ from superlocalmemory.summaries import generate_session_summary
800
+ result = generate_session_summary(db_path, target.strip(), profile, cfg)
801
+ except HTTPException:
802
+ raise
803
+ except Exception:
804
+ raise _internal_error("Summary generation error")
805
+
806
+ return {
807
+ "kind": result.kind,
808
+ "profile_id": result.profile_id,
809
+ "summary": result.content,
810
+ "coverage": result.coverage,
811
+ "generated_by": result.generated_by,
812
+ "source_fact_ids": result.source_fact_ids,
813
+ "source_count": len(result.source_fact_ids),
814
+ "metadata": result.metadata,
815
+ }
816
+
817
+
818
+ @router.get("/api/summary/projects")
819
+ async def get_summary_projects(request: Request):
820
+ """Projects SuperLocalMemory has actually observed, for the summary picker.
821
+
822
+ WHY THIS EXISTS
823
+ ---------------
824
+ SLM is installed globally and the dashboard is a browser tab — it has no
825
+ working directory, so there is no such thing as "this project" from the
826
+ server's point of view. 4.0.8 shipped a "This project" button that sent an
827
+ empty target and produced "project requires target" every time. A button
828
+ that cannot know its own answer is the wrong control; a list of the projects
829
+ we have seen is the right one.
830
+
831
+ Scope comes from ``tool_events.project_path`` — the directory an agent was
832
+ working in when it called SLM. Deliberately NOT ``entity_profiles.
833
+ project_name``, which has exactly one distinct value on a real store (see
834
+ the note at the top of summaries/project_work_log.py).
835
+
836
+ ``tool_events`` is a bounded ring buffer, so this lists recently active
837
+ projects rather than every project in history. ``truncated`` says so
838
+ honestly instead of implying the list is exhaustive.
839
+ """
840
+ profile = get_active_profile()
841
+ try:
842
+ conn = get_db_connection()
843
+ # get_db_connection() hands back a SHARED read connection, and other
844
+ # handlers set row_factory on it. Never index these rows positionally —
845
+ # whichever handler ran last decides whether r[0] is a column or a
846
+ # KeyError. Name the columns and read them by name.
847
+ conn.row_factory = dict_factory
848
+ cursor = conn.cursor()
849
+ cursor.execute(
850
+ """
851
+ SELECT project_path AS path, COUNT(*) AS events
852
+ FROM tool_events
853
+ WHERE project_path IS NOT NULL AND project_path != ''
854
+ AND profile_id = ?
855
+ GROUP BY project_path
856
+ ORDER BY events DESC
857
+ LIMIT 50
858
+ """,
859
+ (profile,),
860
+ )
861
+ rows = cursor.fetchall()
862
+ total = cursor.execute(
863
+ "SELECT COUNT(*) AS n FROM tool_events"
864
+ ).fetchone()["n"]
865
+ except Exception:
866
+ raise _internal_error("Project list error")
867
+
868
+ projects = [
869
+ {
870
+ "path": r["path"],
871
+ "events": r["events"],
872
+ "label": _project_label(r["path"]),
873
+ }
874
+ for r in rows
875
+ ]
876
+ return {
877
+ "projects": projects,
878
+ "profile_id": profile,
879
+ # Surfaced so the UI can explain an unexpectedly short list rather than
880
+ # leaving the user to assume their project was never recorded.
881
+ "truncated": total >= _TOOL_EVENT_RING_SIZE,
882
+ "event_rows": total,
883
+ }
884
+
885
+
886
+ #: tool_events is capped; at the cap the project list is a recent window, not history.
887
+ _TOOL_EVENT_RING_SIZE = 2000
888
+
889
+
890
+ def _project_label(path: str) -> str:
891
+ """Short, human label for a project path.
892
+
893
+ Full paths are long and share prefixes ("/Users/x/Documents/official/..."),
894
+ so a dropdown of raw paths is unreadable. Last two segments keep sibling
895
+ projects distinguishable without the noise.
896
+ """
897
+ parts = [p for p in str(path).replace("\\", "/").split("/") if p]
898
+ if not parts:
899
+ return str(path)
900
+ return "/".join(parts[-2:]) if len(parts) > 1 else parts[-1]
901
+
902
+
750
903
  @router.get("/api/clusters")
751
904
  async def get_clusters(request: Request):
752
905
  """Get cluster information with member counts and statistics."""
@@ -1932,55 +1932,33 @@ async def trigger_consolidation(request: Request):
1932
1932
  profile_id=pid,
1933
1933
  )
1934
1934
 
1935
- # v3.7.8 SEC-M-01: the prior "WorkerPool" fast path called
1936
- # ``pool.send_command(...)``, a method that does not exist on
1937
- # WorkerPool every call raised AttributeError, was silently
1938
- # swallowed by the bare ``except``, and fell through to this direct
1939
- # path unconditionally. That dead branch is removed; consolidation
1940
- # always runs directly against a lease-protected DB connection so a
1941
- # concurrent profile switch cannot commit mid-consolidation.
1942
- #
1943
- # v3.4.64: ConsolidationEngine.consolidate() is CPU/IO bound (seconds
1944
- # to minutes). Calling it directly in an async route blocks the ASGI
1945
- # event loop. Moved into asyncio.to_thread() so the event loop stays
1946
- # live. The runtime.operation() lease is acquired INSIDE the thread —
1947
- # blocking a thread is fine; blocking the event loop is not.
1948
- import asyncio as _asyncio
1949
- from superlocalmemory.core.config import SLMConfig
1950
- from superlocalmemory.storage.database import DatabaseManager
1951
- from superlocalmemory.storage import schema as _schema
1952
- from superlocalmemory.core.consolidation_engine import ConsolidationEngine
1953
- from superlocalmemory.server.profile_runtime import get_profile_runtime
1954
-
1955
- _app_state = request.app.state
1935
+ # 4.0.8: the body of this handler moved to server/consolidation_runner
1936
+ # so the periodic daemon trigger and this endpoint run the SAME code
1937
+ # under the SAME lock. Two copies would be two definitions of
1938
+ # "consolidated", and only one of them would get maintained.
1939
+ from superlocalmemory.server.consolidation_runner import (
1940
+ run_full_consolidation,
1941
+ )
1956
1942
 
1957
- def _run_consolidation() -> dict:
1958
- runtime = get_profile_runtime(_app_state)
1959
- with runtime.operation():
1960
- config = SLMConfig.load()
1961
- db = DatabaseManager(config.db_path)
1962
- db.initialize(_schema)
1963
- engine = ConsolidationEngine(
1964
- db=db, config=config.consolidation, slm_config=config,
1943
+ # background=true returns as soon as the pass is scheduled. The
1944
+ # session-end hook needs this: a full consolidation runs for seconds to
1945
+ # minutes, and a hook that waits for it either blocks the user's shell
1946
+ # or times out and wrongly concludes the run failed.
1947
+ if body.get("background"):
1948
+ import asyncio as _asyncio
1949
+
1950
+ _app_state = request.app.state
1951
+ _asyncio.create_task(
1952
+ run_full_consolidation(
1953
+ _app_state, pid, lightweight=lightweight, trigger="hook",
1965
1954
  )
1966
- res = engine.consolidate(profile_id=pid, lightweight=lightweight)
1967
- # v3.4.1: Auto-trigger behavioral pattern mining after consolidation
1968
- try:
1969
- from superlocalmemory.learning.consolidation_worker import (
1970
- ConsolidationWorker,
1971
- )
1972
- learning_db = config.base_dir / "learning.db"
1973
- cw = ConsolidationWorker(str(config.db_path), str(learning_db))
1974
- pattern_count = cw._generate_patterns(pid, False)
1975
- res["patterns_mined"] = pattern_count
1976
- logger.info(
1977
- "Auto-mined %d patterns after consolidation", pattern_count
1978
- )
1979
- except Exception as exc:
1980
- logger.debug("Pattern mining after consolidation failed: %s", exc)
1981
- return res
1955
+ )
1956
+ authorization.complete()
1957
+ return {"success": True, "started": True, "background": True}
1982
1958
 
1983
- result = await _asyncio.to_thread(_run_consolidation)
1959
+ result = await run_full_consolidation(
1960
+ request.app.state, pid, lightweight=lightweight, trigger="http",
1961
+ )
1984
1962
  authorization.complete()
1985
1963
  return {"success": True, **result}
1986
1964
  except HTTPException:
@@ -418,6 +418,10 @@ class EngineRecallAdapter:
418
418
  memory_map={k: _sanitize_json_text(v) for k, v in memory_map.items()},
419
419
  per_fact_max=getattr(_rc, "recall_per_fact_max_chars", 2400),
420
420
  total_max=getattr(_rc, "recall_total_max_chars", 12000),
421
+ # Option B: markers only on session-bearing recalls. A marker can
422
+ # only buy a learning signal when a pending_outcomes row exists
423
+ # to settle, and those exist only when session_id is present.
424
+ include_marker=bool(session_id),
421
425
  )
422
426
  for _r in results:
423
427
  _r["content"] = _sanitize_json_text(_r.get("content", ""))
@@ -860,6 +864,86 @@ def _start_idle_watchdog(timeout_sec: int) -> None:
860
864
  t.start()
861
865
 
862
866
 
867
+ # ---------------------------------------------------------------------------
868
+ # Periodic full consolidation (4.0.8)
869
+ # ---------------------------------------------------------------------------
870
+ #
871
+ # Steps 8-11 of ConsolidationEngine — behavioural assertions, soft prompts,
872
+ # skill performance, skill evolution — had no automatic trigger at all. The
873
+ # session-end hook is the fast path; this timer is the guarantee, because a hook
874
+ # that does not fire is indistinguishable from a feature that does not exist.
875
+ #
876
+ # Two constraints shape the schedule:
877
+ #
878
+ # 1. Remember and recall latency must not move. So the pass only starts when
879
+ # the daemon has served nothing for _CONSOLIDATION_IDLE_SEC. Consolidation
880
+ # is catch-up work; there is never a reason for it to compete with a live
881
+ # request. If the machine is busy every time we look, we simply skip and
882
+ # check again next tick.
883
+ # 2. It must not pile up. run_full_consolidation() holds a lock and skips
884
+ # rather than queues, so a slow pass cannot be overlapped by the next tick
885
+ # or by the session-end hook.
886
+
887
+ #: How often to consider running. Not how often it runs.
888
+ _CONSOLIDATION_CHECK_SEC = int(os.environ.get("SLM_CONSOLIDATION_CHECK_SEC", 900))
889
+
890
+ #: Minimum quiet period before a scheduled pass may start.
891
+ _CONSOLIDATION_IDLE_SEC = int(os.environ.get("SLM_CONSOLIDATION_IDLE_SEC", 300))
892
+
893
+ #: Minimum gap between two scheduled passes.
894
+ _CONSOLIDATION_MIN_GAP_SEC = int(
895
+ os.environ.get("SLM_CONSOLIDATION_MIN_GAP_SEC", 6 * 3600)
896
+ )
897
+
898
+ #: Delay before the first check, so daemon startup is never slowed by it.
899
+ _CONSOLIDATION_FIRST_DELAY_SEC = int(
900
+ os.environ.get("SLM_CONSOLIDATION_FIRST_DELAY_SEC", 120)
901
+ )
902
+
903
+
904
+ async def _consolidation_timer_loop(application: FastAPI) -> None:
905
+ """Run full consolidation on a schedule, but only while the daemon is idle."""
906
+ from superlocalmemory.server.consolidation_runner import run_full_consolidation
907
+
908
+ await asyncio.sleep(_CONSOLIDATION_FIRST_DELAY_SEC)
909
+ last_run = 0.0
910
+
911
+ while True:
912
+ try:
913
+ await asyncio.sleep(_CONSOLIDATION_CHECK_SEC)
914
+
915
+ now = time.monotonic()
916
+ if last_run and (now - last_run) < _CONSOLIDATION_MIN_GAP_SEC:
917
+ continue
918
+ if (now - _last_activity) < _CONSOLIDATION_IDLE_SEC:
919
+ continue # busy — try again next tick
920
+
921
+ # get_engine_lazy, not state.engine directly: a mode switch nulls
922
+ # state.engine, and reading it raw would silently disable scheduled
923
+ # consolidation until the next daemon restart.
924
+ from superlocalmemory.server.routes.helpers import get_engine_lazy
925
+
926
+ engine = get_engine_lazy(application.state)
927
+ profile_id = getattr(engine, "profile_id", None) if engine else None
928
+ if not profile_id:
929
+ continue
930
+
931
+ result = await run_full_consolidation(
932
+ application.state, profile_id, trigger="timer",
933
+ )
934
+ # Only a pass that actually ran resets the clock; a skip must not
935
+ # buy another six hours of silence.
936
+ if not result.get("skipped"):
937
+ last_run = time.monotonic()
938
+ except asyncio.CancelledError:
939
+ raise
940
+ except Exception:
941
+ # Never let a bad pass kill the loop — that would silently disable
942
+ # consolidation for the life of the daemon, which is the failure
943
+ # mode this timer exists to end. Log loudly and try again.
944
+ logger.exception("scheduled consolidation failed; will retry")
945
+
946
+
863
947
  # ---------------------------------------------------------------------------
864
948
  # Legacy port TCP redirect (backward compat for port 8767)
865
949
  # ---------------------------------------------------------------------------
@@ -2462,6 +2546,15 @@ async def lifespan(application: FastAPI):
2462
2546
  except Exception as e:
2463
2547
  logger.warning("optimize module not available: %s", e)
2464
2548
 
2549
+ # 4.0.8: periodic full consolidation. Idle-gated and lock-guarded — see
2550
+ # _consolidation_timer_loop. Started here rather than at import so a daemon
2551
+ # that never completes startup never schedules work.
2552
+ _consol_task = getattr(application.state, "_consolidation_task", None)
2553
+ if _consol_task is None or _consol_task.done():
2554
+ application.state._consolidation_task = asyncio.create_task(
2555
+ _consolidation_timer_loop(application)
2556
+ )
2557
+
2465
2558
  # v3.6.7: Start MCP Streamable-HTTP session manager (GOTCHA #1).
2466
2559
  # streamable_http_app() carries its own Starlette lifespan that initialises
2467
2560
  # an anyio task group inside the session manager. Without entering that
@@ -2537,6 +2630,16 @@ async def lifespan(application: FastAPI):
2537
2630
  except Exception: # pragma: no cover — defensive
2538
2631
  pass
2539
2632
 
2633
+ # Cancel the periodic consolidation loop. Not awaited to completion — a pass
2634
+ # can take minutes and shutdown must not block on it; the lock and the
2635
+ # engine's own transaction boundaries make an interrupted pass safe to redo.
2636
+ try:
2637
+ _consol = getattr(application.state, "_consolidation_task", None)
2638
+ if _consol is not None and not _consol.done():
2639
+ _consol.cancel()
2640
+ except Exception: # pragma: no cover — defensive
2641
+ pass
2642
+
2540
2643
  # Cancel optimize metrics flush loop + run final flush before shutdown
2541
2644
  try:
2542
2645
  _flush_task = getattr(application.state, "_optimize_flush_task", None)
@@ -3879,6 +3982,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
3879
3982
  memory_map={k: _sanitize_json_text(v) for k, v in memory_map.items()},
3880
3983
  per_fact_max=getattr(_rc, "recall_per_fact_max_chars", 2400),
3881
3984
  total_max=getattr(_rc, "recall_total_max_chars", 12000),
3985
+ # Option B: markers only on session-bearing recalls. A marker can
3986
+ # only buy a learning signal when a pending_outcomes row exists
3987
+ # to settle, and those exist only when session_id is present.
3988
+ include_marker=bool(session_id),
3882
3989
  full=full,
3883
3990
  include_source=include_source,
3884
3991
  )
@@ -98,6 +98,165 @@ GENERATED_BY_LLM_C = "llm_c"
98
98
  """Cloud LLM (Mode C). Falls back via llm_b to extractive."""
99
99
 
100
100
 
101
+ # ── highlight formatting ────────────────────────────────────────────────────
102
+
103
+ #: Display width for one bullet in a summary body.
104
+ #:
105
+ #: Chosen for a bullet, not for a paragraph. The generators originally truncated
106
+ #: at 300 characters and nothing else, which looks fine on a synthetic corpus of
107
+ #: one-line facts and falls apart on a real store: agent-written facts routinely
108
+ #: contain blank lines and markdown headings, so a 300-character slice rendered
109
+ #: as six or more display lines and the bullet list stopped being a list.
110
+ HIGHLIGHT_CHARS = 180
111
+
112
+
113
+ def format_highlight(content: str, limit: int = HIGHLIGHT_CHARS) -> str:
114
+ """Collapse *content* to a single readable line for a summary bullet.
115
+
116
+ Three things, in order:
117
+
118
+ 1. **Flatten whitespace.** Newlines, blank lines and runs of spaces all
119
+ become one space. This is the fix that matters: character truncation
120
+ alone cannot keep a multi-paragraph fact on one line, and every
121
+ generator here writes into a bullet list.
122
+ 2. **Prefer a whole first sentence** when there is one and it fits. A
123
+ complete sentence reads better than a slice of one, and the first
124
+ sentence of a report is usually its summary.
125
+ 3. **Otherwise cut at a word boundary** and mark the cut with an ellipsis,
126
+ so it is visible that text was dropped rather than that a fact ended
127
+ mid-word.
128
+
129
+ Markdown heading markers are stripped because a flattened ``**Summary**``
130
+ mid-sentence reads as noise.
131
+ """
132
+ import re
133
+
134
+ text = re.sub(r"\s+", " ", (content or "")).strip()
135
+ # Leading/inline markdown emphasis and heading marks, once flattened, add
136
+ # nothing but clutter to a one-line bullet.
137
+ text = re.sub(r"(?:^|\s)#{1,6}\s+", " ", text)
138
+ text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
139
+ text = re.sub(r"\s+", " ", text).strip()
140
+
141
+ if not text:
142
+ return ""
143
+ if len(text) <= limit:
144
+ return text
145
+
146
+ # A complete first sentence, if it fits comfortably.
147
+ match = re.match(r"(.+?[.!?])(?:\s|$)", text)
148
+ if match:
149
+ sentence = match.group(1).strip()
150
+ if len(sentence) <= limit:
151
+ return sentence
152
+
153
+ cut = text[:limit]
154
+ space = cut.rfind(" ")
155
+ if space > limit * 0.6: # don't cut a long unbroken token to a stub
156
+ cut = cut[:space]
157
+ return cut.rstrip(" ,;:—-") + "…"
158
+
159
+
160
+ # ── LLM output cleanup ──────────────────────────────────────────────────────
161
+
162
+ #: Sentences a chat-tuned model emits *around* the answer rather than as part of
163
+ #: it. Anchored to the start of a paragraph so they cannot match mid-content.
164
+ #:
165
+ #: Measured, not guessed: a Mode B summary on the author's own store opened with
166
+ #: "I apologize for the previous confusion. It seems that I misunderstood the
167
+ #: context of the texts provided.\n\nTo provide a concise summary paragraph, here
168
+ #: is a merge of all the key information:" — 180 characters of the model talking
169
+ #: to itself, shown to the user as their daily reflection.
170
+ _LLM_PREAMBLE = (
171
+ r"^(?:"
172
+ r"i\s+apologi[sz]e\b.*"
173
+ r"|i'?m\s+sorry\b.*"
174
+ r"|it\s+seems\s+that\s+i\b.*"
175
+ r"|sure[,!.]?\s*(?:thing)?\b.*"
176
+ r"|certainly[,!.]?\b.*"
177
+ r"|of\s+course[,!.]?\b.*"
178
+ r"|here(?:'s|\s+is|\s+are)\b[^.!?]*:"
179
+ r"|to\s+(?:provide|summari[sz]e|answer)\b[^.!?]*:"
180
+ r"|based\s+on\s+the\s+(?:facts|texts|information|data)\s+provided[,:]?"
181
+ r"|as\s+(?:an|a)\s+(?:ai|language\s+model)\b.*"
182
+ r")\s*$"
183
+ )
184
+
185
+ #: Closing pleasantries. Same anchoring rule.
186
+ _LLM_POSTAMBLE = (
187
+ r"^(?:"
188
+ r"(?:i\s+hope|hope)\s+(?:this|that)\s+helps\b.*"
189
+ r"|let\s+me\s+know\b.*"
190
+ r"|feel\s+free\s+to\b.*"
191
+ r"|would\s+you\s+like\s+me\s+to\b.*"
192
+ r")$"
193
+ )
194
+
195
+
196
+ def clean_llm_summary(text: str) -> str:
197
+ """Strip chat-assistant scaffolding from a model-written summary.
198
+
199
+ WHY THIS EXISTS
200
+ ---------------
201
+ Mode B/C summaries are shown to the user as *their* memory, with no chat
202
+ framing around them. A chat-tuned model does not know that: it opens with an
203
+ apology or "Here is a concise summary:" and closes with "Let me know if you
204
+ want more detail". Both are addressed to a conversation that the reader
205
+ cannot see, and both make the product look broken.
206
+
207
+ The system prompt now asks for bare prose, which handles most of it. This is
208
+ the second line of defence, because instruction-following on a 3B local
209
+ model is not something to bet the displayed output on.
210
+
211
+ Conservative by construction: patterns are anchored to whole paragraphs or
212
+ whole leading sentences, so a summary that legitimately contains the word
213
+ "sure" mid-paragraph is untouched. If stripping would empty the text, the
214
+ original is returned — a scaffolded summary beats a blank one.
215
+ """
216
+ import re
217
+
218
+ original = (text or "").strip()
219
+ if not original:
220
+ return ""
221
+
222
+ # Fenced code blocks wrapping the whole answer: keep the contents.
223
+ fenced = re.match(r"^```[a-zA-Z]*\n(.*?)\n?```$", original, re.DOTALL)
224
+ if fenced:
225
+ original = fenced.group(1).strip()
226
+
227
+ paras = [p.strip() for p in re.split(r"\n\s*\n", original) if p.strip()]
228
+
229
+ while paras and re.match(_LLM_PREAMBLE, paras[0], re.IGNORECASE | re.DOTALL):
230
+ paras.pop(0)
231
+ while paras and re.match(_LLM_POSTAMBLE, paras[-1], re.IGNORECASE | re.DOTALL):
232
+ paras.pop()
233
+
234
+ if not paras:
235
+ return original
236
+
237
+ # A preamble that shares a paragraph with real content: drop just the leading
238
+ # sentence, and only when what follows is substantial enough to stand alone.
239
+ lead = re.match(r"(.+?[.:!?])\s+(\S.*)$", paras[0], re.DOTALL)
240
+ if lead and re.match(_LLM_PREAMBLE, lead.group(1), re.IGNORECASE | re.DOTALL):
241
+ if len(lead.group(2).strip()) > 40:
242
+ paras[0] = lead.group(2).strip()
243
+
244
+ cleaned = "\n\n".join(paras).strip()
245
+ return cleaned or original
246
+
247
+
248
+ #: System prompt for every summary generator, Mode B and Mode C alike.
249
+ #:
250
+ #: Mode C always sent one; Mode B sent none at all, which is why local-model
251
+ #: output arrived wrapped in chat scaffolding while cloud output did not.
252
+ SUMMARY_SYSTEM_PROMPT = (
253
+ "You summarise a person's own saved notes for them. "
254
+ "Reply with the summary text only — no preamble, no apologies, no sign-off, "
255
+ "no markdown headings, and never refer to yourself or to these instructions. "
256
+ "Write plain prose in the third person about the work described."
257
+ )
258
+
259
+
101
260
  def get_mode_str(config: object | None) -> str:
102
261
  """Extract the operating mode string ('a', 'b', or 'c') from a config."""
103
262
  if config is None:
@@ -34,6 +34,9 @@ from datetime import date
34
34
  from pathlib import Path
35
35
 
36
36
  from .base import (
37
+ clean_llm_summary,
38
+ format_highlight,
39
+ SUMMARY_SYSTEM_PROMPT,
37
40
  COVERAGE_FULL,
38
41
  COVERAGE_INSUFFICIENT,
39
42
  COVERAGE_UNAVAILABLE,
@@ -128,7 +131,7 @@ def generate_daily_reflection(
128
131
  coverage = COVERAGE_FULL
129
132
 
130
133
  # ── extractive summary (deterministic, always available) ──────────────────
131
- extractive_content = _build_extractive_content(date_str, facts, fact_count)
134
+ extractive_content = _build_extractive_content(date_str, facts, fact_count, db_path, profile_id)
132
135
 
133
136
  # ── LLM enrichment (optional) ─────────────────────────────────────────────
134
137
  mode = get_mode_str(config)
@@ -162,8 +165,14 @@ def _build_extractive_content(
162
165
  date_str: str,
163
166
  facts: list[dict],
164
167
  fact_count: int,
168
+ db_path: str | Path,
169
+ profile_id: str,
165
170
  ) -> str:
166
- """Build a deterministic extractive daily reflection."""
171
+ """Build a deterministic extractive daily reflection.
172
+
173
+ ``db_path`` / ``profile_id`` are needed only to turn entity IDs into names
174
+ for the "Active entities" line — the facts themselves are already loaded.
175
+ """
167
176
  import json
168
177
 
169
178
  lines = [
@@ -174,8 +183,7 @@ def _build_extractive_content(
174
183
  ]
175
184
  for f in facts[:_BODY_FACTS]:
176
185
  content = f.get("content", "")
177
- if len(content) > _MAX_FACT_CHARS:
178
- content = content[:_MAX_FACT_CHARS - 3] + "..."
186
+ content = format_highlight(content)
179
187
  lines.append(f" - {content}")
180
188
  if fact_count > _BODY_FACTS:
181
189
  lines.append(f" ... and {fact_count - _BODY_FACTS} additional facts.")
@@ -193,13 +201,50 @@ def _build_extractive_content(
193
201
 
194
202
  if entity_counts:
195
203
  top_entities = sorted(entity_counts.items(), key=lambda x: x[1], reverse=True)[:5]
196
- entity_str = ", ".join(f"{e} ({c})" for e, c in top_entities)
204
+ # Resolve to names. canonical_entities_json stores entity IDs, so this
205
+ # line previously read "Active entities: 1666abc512904473 (127),
206
+ # 84288f2dde994afe (124)" — five 16-hex identifiers, which tell a reader
207
+ # nothing. They resolve to 'Fixed', 'Gateway', 'REVISED' and so on. This
208
+ # is the same defect 4.0.6 fixed in the Living Brain, where source
209
+ # quality listed internal identifiers; it survived here because this
210
+ # generator was never reachable to be looked at.
211
+ names = _entity_names(db_path, [e for e, _ in top_entities], profile_id)
212
+ labelled = [
213
+ (names.get(eid) or eid, count) for eid, count in top_entities
214
+ ]
215
+ entity_str = ", ".join(f"{name} ({c})" for name, c in labelled)
197
216
  lines.append("")
198
217
  lines.append(f"Active entities: {entity_str}")
199
218
 
200
219
  return "\n".join(lines)
201
220
 
202
221
 
222
+ def _entity_names(
223
+ db_path: str | Path, entity_ids: list[str], profile_id: str
224
+ ) -> dict[str, str]:
225
+ """Map entity_id → canonical_name. Missing or unreadable rows are omitted.
226
+
227
+ Fail-open: a summary is still useful with a raw id in it, so a query problem
228
+ must not lose the whole line. The caller falls back to the id per entity.
229
+ """
230
+ if not entity_ids:
231
+ return {}
232
+ try:
233
+ conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
234
+ try:
235
+ placeholders = ",".join("?" for _ in entity_ids)
236
+ rows = conn.execute(
237
+ f"SELECT entity_id, canonical_name FROM canonical_entities "
238
+ f"WHERE entity_id IN ({placeholders}) AND profile_id = ?",
239
+ (*entity_ids, profile_id),
240
+ ).fetchall()
241
+ finally:
242
+ conn.close()
243
+ return {r[0]: r[1] for r in rows if r[1]}
244
+ except sqlite3.Error:
245
+ return {}
246
+
247
+
203
248
  def _try_llm(
204
249
  date_str: str,
205
250
  facts: list[dict],
@@ -249,6 +294,7 @@ def _call_ollama(
249
294
  payload = json.dumps({
250
295
  "model": model,
251
296
  "prompt": full_prompt,
297
+ "system": SUMMARY_SYSTEM_PROMPT,
252
298
  "stream": False,
253
299
  "options": {"num_predict": 300},
254
300
  }).encode()
@@ -259,7 +305,7 @@ def _call_ollama(
259
305
  )
260
306
  resp = urllib.request.urlopen(req, timeout=timeout)
261
307
  data = json.loads(resp.read().decode())
262
- text = data.get("response", "").strip()
308
+ text = clean_llm_summary(data.get("response", ""))
263
309
  return text if text and len(text) > 20 else None
264
310
  except Exception as exc:
265
311
  logger.debug("Ollama daily reflection failed: %s", exc)
@@ -283,11 +329,12 @@ def _call_cloud_llm(
283
329
  full_prompt = f"{prompt}\n\nFacts:\n{fact_texts}"
284
330
  text = llm.generate(
285
331
  prompt=full_prompt,
286
- system="You are a concise personal memory summariser.",
332
+ system=SUMMARY_SYSTEM_PROMPT,
287
333
  max_tokens=300,
288
334
  temperature=0.1,
289
335
  )
290
- return text.strip() if text and len(text.strip()) > 20 else None
336
+ cleaned = clean_llm_summary(text or "")
337
+ return cleaned if len(cleaned) > 20 else None
291
338
  except Exception as exc:
292
339
  logger.debug("Cloud LLM daily reflection failed: %s", exc)
293
340
  return None