hexcli 2.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,241 @@
1
+ #!/usr/bin/env python3
2
+ """hexcli.markdown_stream — markdown-lite to ANSI, one character at a time.
3
+
4
+ Answers arrive as a token stream and used to print raw: ``## Heading``,
5
+ ``**bold**``, backtick spans and ``` fences all showed their markers. This
6
+ turns the common subset into terminal styling without waiting for the line
7
+ to finish, so streaming keeps its word-by-word feel:
8
+
9
+ * ``# heading`` (any level) → bold, markers dropped
10
+ * ``- item`` / ``* item`` / ``+ item`` → ``• item`` (indentation kept)
11
+ * ``**bold**`` → bold
12
+ * `` `code` `` → cyan
13
+ * ``` ```lang ``` … ``` ``` ``` → a dim rule with the language, code left
14
+ exactly as written (no gutter, so it copies cleanly), a dim rule after
15
+
16
+ Markers are held only as long as they are ambiguous (at most the three
17
+ backticks of a fence, or one ``*``) and released literally when they turn
18
+ out not to be markup, so ``2 * 3`` and ``C# code`` come through untouched.
19
+ Feeding the same text whole or one character at a time yields identical
20
+ output; that property is what the tests pin down.
21
+
22
+ The styles come from ``ui.C`` and are empty strings when colour is off, in
23
+ which case only the bullet and fence substitutions remain.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ _FENCE_WIDTH = 40
28
+ _FENCE_LABEL_CHARS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_+#.-")
29
+ _FENCE_LABEL_MAX = 24 # "```" then more than a language name is not a fence line
30
+
31
+
32
+ class MarkdownStream:
33
+ def __init__(self) -> None:
34
+ from hexcli.ui import C # lazy: ui imports this module for render_result
35
+ self._c = C
36
+ self._bol = True # at the beginning of a line
37
+ self._pending = "" # marker characters not yet decided
38
+ self._fence = False # inside a ``` block
39
+ self._fence_line: str | None = None # collecting "```lang" up to its newline
40
+ self._bold = False
41
+ self._code = False
42
+ self._heading = False
43
+ self._restyle = False # re-emit the open styles after a soft line break
44
+
45
+ # -- public -------------------------------------------------------------
46
+
47
+ def feed(self, text: str) -> str:
48
+ return "".join(self._char(ch) for ch in text)
49
+
50
+ def finish(self) -> str:
51
+ """Release anything still held and close open styling."""
52
+ out = self._flush_pending()
53
+ if self._fence_line is not None:
54
+ out += self._fence_marker(self._fence_line)
55
+ self._fence_line = None
56
+ if self._restyle:
57
+ # Styles carried over a line break were already reset on screen
58
+ # and never re-opened: nothing to close.
59
+ self._bold = self._code = self._heading = False
60
+ self._restyle = False
61
+ return out + self._close_styles()
62
+
63
+ # -- styling ------------------------------------------------------------
64
+
65
+ def _style(self) -> str:
66
+ c = self._c
67
+ return (c.BOLD if (self._bold or self._heading) else "") + (c.BCYAN if self._code else "")
68
+
69
+ def _close_styles(self) -> str:
70
+ if not (self._bold or self._code or self._heading):
71
+ return ""
72
+ self._bold = self._code = self._heading = False
73
+ return self._c.RESET
74
+
75
+ def _toggle_bold(self) -> str:
76
+ if self._code:
77
+ return "**" # literal inside a code span
78
+ self._bold = not self._bold
79
+ return self._c.RESET + self._style() if not self._bold else self._c.BOLD
80
+
81
+ def _toggle_code(self) -> str:
82
+ self._code = not self._code
83
+ return (self._c.BCYAN if self._code else self._c.RESET + self._style())
84
+
85
+ def _bullet(self) -> str:
86
+ c = self._c
87
+ return f"{c.DIM}•{c.RESET} "
88
+
89
+ def _fence_marker(self, lang: str) -> str:
90
+ c = self._c
91
+ if self._fence:
92
+ self._fence = False
93
+ self._restyle = False
94
+ return f"{c.DIM}{'─' * _FENCE_WIDTH}{c.RESET}"
95
+ self._fence = True
96
+ label = f" {lang.strip()} " if lang.strip() else ""
97
+ return f"{c.DIM}{'─' * 4}{label}{'─' * max(2, _FENCE_WIDTH - 4 - len(label))}{c.RESET}"
98
+
99
+ def _flush_pending(self) -> str:
100
+ """Release held markers at a line break or the end of the text. A
101
+ one- or two-backtick run there is a span boundary (`x` at the end of
102
+ a line is the common case); everything else is literal."""
103
+ p, self._pending = self._pending, ""
104
+ if p and set(p) == {"`"} and len(p) < 3 and not self._fence:
105
+ return self._toggle_code()
106
+ return p
107
+
108
+ # -- the state machine ----------------------------------------------------
109
+
110
+ def _char(self, ch: str) -> str:
111
+ if self._fence_line is not None:
112
+ if ch == "\n":
113
+ label = self._fence_line.strip()
114
+ self._fence_line = None
115
+ self._bol = True
116
+ if not self._fence and " " in label:
117
+ return "```" + label + "\n" # "```this is prose": not a fence
118
+ return self._fence_marker(label) + "\n"
119
+ # A closing fence takes anything after the backticks (trailing
120
+ # spaces, a stray word); an opening one only a language name.
121
+ ok = self._fence or (ch in _FENCE_LABEL_CHARS and len(self._fence_line) < _FENCE_LABEL_MAX)
122
+ if ok or ch in " \r":
123
+ self._fence_line += ch
124
+ return ""
125
+ # Not a fence after all ("```" followed by prose, or by code the
126
+ # model failed to break onto its own line): release the backticks
127
+ # and the label literally and carry on with the line.
128
+ held, self._fence_line = "```" + self._fence_line, None
129
+ self._bol = False
130
+ return held + self._inline_char(ch)
131
+ if ch == "\n":
132
+ # Bold and code spans may continue on the next line (a soft
133
+ # break); a blank line ends them. Headings end with their line.
134
+ out = self._flush_pending()
135
+ carry = (self._bold or self._code) and not self._bol
136
+ already_reset = self._restyle # a carried span was reset at the previous break
137
+ open_style = (self._bold or self._code or self._heading) and not already_reset
138
+ out += (self._c.RESET if open_style else "") + "\n"
139
+ self._heading = False
140
+ if carry:
141
+ self._restyle = True
142
+ else:
143
+ self._bold = self._code = False
144
+ self._restyle = False
145
+ self._bol = True
146
+ return out
147
+ if self._restyle and ch != "\n":
148
+ self._restyle = False
149
+ return self._style() + (self._bol_char(ch) if self._bol else self._inline_char(ch))
150
+ if self._bol:
151
+ return self._bol_char(ch)
152
+ return self._inline_char(ch)
153
+
154
+ def _bol_char(self, ch: str) -> str:
155
+ p = self._pending
156
+ # A fence opener or closer: three backticks at the start of a line.
157
+ if ch == "`" and p in ("", "`", "``"):
158
+ p += "`"
159
+ if p == "```":
160
+ self._pending = ""
161
+ self._fence_line = ""
162
+ return ""
163
+ self._pending = p
164
+ return ""
165
+ if p and set(p) == {"`"}:
166
+ # One or two backticks then something else: inline code after all.
167
+ self._pending = ""
168
+ self._bol = False
169
+ return "".join(self._inline_char(b) for b in p) + self._inline_char(ch)
170
+ if self._fence:
171
+ self._bol = False
172
+ return ch
173
+ if ch == " " and p == "":
174
+ return " " # indentation: still at the start for bullet purposes
175
+ if ch == "#" and (p == "" or set(p) == {"#"}) and len(p) < 6:
176
+ self._pending = p + "#"
177
+ return ""
178
+ if p and set(p) == {"#"}:
179
+ self._pending = ""
180
+ self._bol = False
181
+ if ch == " ":
182
+ self._heading = True
183
+ return self._c.BOLD
184
+ return p + self._inline_char(ch)
185
+ if p == "" and ch in "-*+":
186
+ self._pending = ch
187
+ return ""
188
+ if p in ("-", "+", "*"):
189
+ self._pending = ""
190
+ self._bol = False
191
+ if ch == " ":
192
+ return self._bullet()
193
+ if p == "*" and ch == "*":
194
+ self._pending = "**" # decided by what follows (see _inline_char)
195
+ return ""
196
+ return p + self._inline_char(ch)
197
+ self._bol = False
198
+ return self._inline_char(ch)
199
+
200
+ def _inline_char(self, ch: str) -> str:
201
+ if self._fence:
202
+ return ch
203
+ p = self._pending
204
+ if p == "**":
205
+ # An opening "**" must be followed by something other than a
206
+ # space (CommonMark's left-flanking rule); "next** x" is literal.
207
+ self._pending = ""
208
+ if ch in " \t":
209
+ return "**" + self._inline_char(ch)
210
+ return self._toggle_bold() + self._inline_char(ch)
211
+ if p == "*":
212
+ self._pending = ""
213
+ if ch == "*":
214
+ if self._bold or self._code:
215
+ return self._toggle_bold() # closing (or literal inside code)
216
+ self._pending = "**"
217
+ return ""
218
+ return "*" + self._inline_char(ch)
219
+ if p and set(p) == {"`"}:
220
+ # A run of backticks resolves on the next character: one or two
221
+ # open/close a code span, three mid-line are literal (a fence
222
+ # that never got its own line).
223
+ if ch == "`" and len(p) < 3:
224
+ self._pending = p + "`"
225
+ return ""
226
+ self._pending = ""
227
+ head = "```" if len(p) == 3 else self._toggle_code()
228
+ return head + self._inline_char(ch)
229
+ if ch == "*" and not self._code:
230
+ self._pending = "*"
231
+ return ""
232
+ if ch == "`":
233
+ self._pending = "`"
234
+ return ""
235
+ return ch
236
+
237
+
238
+ def render_markdown(text: str) -> str:
239
+ """The whole-text form, for answers that did not stream."""
240
+ md = MarkdownStream()
241
+ return md.feed(text) + md.finish()
hexcli/memory.py ADDED
@@ -0,0 +1,416 @@
1
+ #!/usr/bin/env python3
2
+ """hexcli.memory — lightweight on-device semantic memory for Hex CLI.
3
+
4
+ Pure NumPy vector index over a local ONNX sentence-embedding model
5
+ (sentence-transformers/all-MiniLM-L6-v2, ARM64-quantized). No FAISS/
6
+ ChromaDB/LangChain. One-way dependency, mirroring hexcli.ui and
7
+ hexcli.telemetry: hexcli.agent imports this module, never the reverse.
8
+
9
+ Every public method swallows its own exceptions — an embedding/model
10
+ load failure (e.g. offline on first use) must degrade to a silent
11
+ no-op, never crash the agent loop or block a turn that doesn't touch
12
+ memory.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import io
17
+ import json
18
+ import threading
19
+ import time
20
+ import uuid
21
+ from collections.abc import Callable
22
+ from datetime import UTC, datetime
23
+ from pathlib import Path
24
+ from typing import Any
25
+
26
+ import numpy as np
27
+
28
+ _STORE_DIR_NAME = ".shellai/vector_store"
29
+ _EMBED_DIM = 384
30
+ _LOCAL_MODEL_PATH: Path | None = None
31
+
32
+
33
+ def set_local_model_path(path: Path) -> None:
34
+ """Tell the embedder where to find the ONNX model. Call once at startup from agent.py."""
35
+ global _LOCAL_MODEL_PATH
36
+ _LOCAL_MODEL_PATH = path
37
+ _MAX_ENTRIES = 500
38
+ _GLOBAL_MAX_ENTRIES = 1_000
39
+ _MIN_SIMILARITY = 0.15
40
+ _MAX_SEQ_LEN = 256
41
+ _MAX_RULES = 50
42
+
43
+ # Global store — cross-project, lives in the user's home dir.
44
+ _GLOBAL_STORE_DIR = Path.home() / ".shellai" / "global_vector_store"
45
+ _RULES_PATH = Path.home() / ".shellai" / "memory_rules.md"
46
+
47
+ # NPU inference lock — prevents the dreaming daemon from calling the LLM
48
+ # concurrently with the main agent loop. Acquired by call_llm in agent.py
49
+ # and by _consolidate here with a 5-second timeout.
50
+ _NPU_INFERENCE_LOCK: threading.Lock = threading.Lock()
51
+
52
+ # Idle timer for the dreaming daemon. touch_last_turn() resets it on each
53
+ # user input. _consolidate fires after _IDLE_TIMEOUT seconds of silence.
54
+ _last_turn_time: float = 0.0
55
+ _IDLE_TIMEOUT: float = 300.0 # 5 minutes
56
+
57
+ # Injected by start_dreaming() from agent.py — avoids a circular import.
58
+ _dream_config_fn: Callable[[], dict[str, Any]] | None = None
59
+ _dream_llm_fn: Callable[[dict[str, Any], str, str], str] | None = None
60
+
61
+ _DREAM_SYSTEM = (
62
+ "Extract 3–5 concise factual rules from these session notes. "
63
+ "Return only a Markdown bullet list (one rule per line, starting with '- '). "
64
+ "No preamble, no commentary, no numbering."
65
+ )
66
+
67
+
68
+ class _Embedder:
69
+ """Process-wide lazy singleton — the ONNX session/tokenizer load once
70
+ and are reused across every add()/search() call for the life of the
71
+ process, regardless of how many VectorStore instances are created."""
72
+
73
+ _instance: _Embedder | None = None
74
+
75
+ def __init__(self) -> None:
76
+ self._session: Any = None
77
+ self._tokenizer: Any = None
78
+ self._unavailable = False
79
+
80
+ @classmethod
81
+ def instance(cls) -> _Embedder:
82
+ if cls._instance is None:
83
+ cls._instance = cls()
84
+ return cls._instance
85
+
86
+ def _ensure_loaded(self) -> None:
87
+ if self._session is not None or self._unavailable:
88
+ return
89
+ if _LOCAL_MODEL_PATH is None or not _LOCAL_MODEL_PATH.exists():
90
+ self._unavailable = True
91
+ return
92
+ tok_path = _LOCAL_MODEL_PATH.parent / "tokenizer.json"
93
+ if not tok_path.exists():
94
+ self._unavailable = True
95
+ return
96
+ try:
97
+ import onnxruntime as ort
98
+ from tokenizers import Tokenizer
99
+ tokenizer = Tokenizer.from_file(str(tok_path))
100
+ tokenizer.enable_padding()
101
+ tokenizer.enable_truncation(max_length=_MAX_SEQ_LEN)
102
+ self._session = ort.InferenceSession(str(_LOCAL_MODEL_PATH), providers=["CPUExecutionProvider"])
103
+ self._tokenizer = tokenizer
104
+ except Exception:
105
+ self._unavailable = True
106
+
107
+ def embed(self, texts: list[str]) -> np.ndarray | None:
108
+ self._ensure_loaded()
109
+ if self._unavailable or not texts:
110
+ return None
111
+ try:
112
+ encodings = self._tokenizer.encode_batch(texts)
113
+ input_ids = np.array([e.ids for e in encodings], dtype=np.int64)
114
+ attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64)
115
+ token_type_ids = np.zeros_like(input_ids)
116
+
117
+ feed = {
118
+ "input_ids": input_ids,
119
+ "attention_mask": attention_mask,
120
+ "token_type_ids": token_type_ids,
121
+ }
122
+ token_embeddings = self._session.run(None, feed)[0]
123
+
124
+ mask = attention_mask[..., None].astype(np.float32)
125
+ summed = (token_embeddings * mask).sum(axis=1)
126
+ counts = np.clip(mask.sum(axis=1), 1e-9, None)
127
+ pooled = summed / counts
128
+
129
+ norms = np.clip(np.linalg.norm(pooled, axis=1, keepdims=True), 1e-9, None)
130
+ return (pooled / norms).astype(np.float32)
131
+ except Exception:
132
+ return None
133
+
134
+
135
+ class VectorStore:
136
+ """One store per (config, cwd) — cheap to construct; only the shared
137
+ _Embedder singleton carries real load cost, so creating a fresh
138
+ VectorStore per tool call is fine."""
139
+
140
+ def __init__(
141
+ self,
142
+ config: dict[str, Any] | None = None,
143
+ cwd: str | None = None,
144
+ *,
145
+ store_dir: Path | None = None,
146
+ max_entries: int = _MAX_ENTRIES,
147
+ ) -> None:
148
+ self.enabled = bool((config or {}).get("memory_enabled", True))
149
+ if store_dir is not None:
150
+ self._dir = store_dir
151
+ else:
152
+ base = Path(cwd) if cwd else Path.cwd()
153
+ self._dir = base / _STORE_DIR_NAME
154
+ self._vectors_path = self._dir / "vectors.npz"
155
+ self._meta_path = self._dir / "metadata.json"
156
+ self._vectors: np.ndarray = np.zeros((0, _EMBED_DIM), dtype=np.float32)
157
+ self._meta: list[dict[str, Any]] = []
158
+ self._loaded = False
159
+ self._max_entries = max_entries
160
+
161
+ def _load(self) -> None:
162
+ if self._loaded:
163
+ return
164
+ self._loaded = True
165
+ try:
166
+ if self._meta_path.exists():
167
+ self._meta = json.loads(self._meta_path.read_text(encoding="utf-8"))
168
+ if self._vectors_path.exists():
169
+ with np.load(self._vectors_path) as data:
170
+ self._vectors = data["vectors"].astype(np.float32)
171
+ except Exception:
172
+ self._vectors = np.zeros((0, _EMBED_DIM), dtype=np.float32)
173
+ self._meta = []
174
+
175
+ def add(self, text: str, metadata: dict[str, Any]) -> None:
176
+ if not self.enabled:
177
+ return
178
+ try:
179
+ self._load()
180
+ vec = _Embedder.instance().embed([text])
181
+ if vec is None:
182
+ return
183
+ self._vectors = np.vstack([self._vectors, vec])
184
+ entry = {
185
+ "id": str(uuid.uuid4()),
186
+ "created_at": datetime.now(UTC).isoformat(),
187
+ "text": text,
188
+ **metadata,
189
+ }
190
+ self._meta.append(entry)
191
+ if len(self._meta) > self._max_entries:
192
+ overflow = len(self._meta) - self._max_entries
193
+ self._meta = self._meta[overflow:]
194
+ self._vectors = self._vectors[overflow:]
195
+ self._save()
196
+ except Exception:
197
+ pass
198
+
199
+ def search(self, query: str, top_k: int = 3) -> list[dict[str, Any]]:
200
+ if not self.enabled:
201
+ return []
202
+ try:
203
+ self._load()
204
+ if self._vectors.shape[0] == 0:
205
+ return []
206
+ qvec = _Embedder.instance().embed([query])
207
+ if qvec is None:
208
+ return []
209
+ sims = self._vectors @ qvec[0]
210
+ order = np.argsort(-sims)[: max(top_k, 1)]
211
+ results: list[dict[str, Any]] = []
212
+ for idx in order:
213
+ score = float(sims[idx])
214
+ if score < _MIN_SIMILARITY:
215
+ continue
216
+ entry = dict(self._meta[idx])
217
+ entry["score"] = round(score, 3)
218
+ results.append(entry)
219
+ return results
220
+ except Exception:
221
+ return []
222
+
223
+ def _save(self) -> None:
224
+ self._dir.mkdir(parents=True, exist_ok=True)
225
+
226
+ buf = io.BytesIO()
227
+ np.savez(buf, vectors=self._vectors)
228
+ tmp_vectors = self._vectors_path.with_suffix(".tmp")
229
+ tmp_vectors.write_bytes(buf.getvalue())
230
+ tmp_vectors.replace(self._vectors_path)
231
+
232
+ tmp_meta = self._meta_path.with_suffix(".tmp")
233
+ tmp_meta.write_text(json.dumps(self._meta, indent=2), encoding="utf-8")
234
+ tmp_meta.replace(self._meta_path)
235
+
236
+
237
+ def maybe_index_turn(
238
+ config: dict[str, Any],
239
+ prompt: str,
240
+ tools_used: list[str],
241
+ key_paths: list[str],
242
+ outcome: str = "completed",
243
+ ) -> None:
244
+ """Auto-index a finished agentic turn. Silent no-op on any failure,
245
+ including memory_enabled=False, an empty tool sequence, or an
246
+ unreachable/unavailable embedding model."""
247
+ if not tools_used:
248
+ return
249
+ try:
250
+ # File-touching turns → project store (cwd-scoped).
251
+ # Non-file-touching turns (preferences, patterns) → global store.
252
+ if key_paths:
253
+ store = VectorStore(config)
254
+ else:
255
+ store = VectorStore(config, store_dir=_GLOBAL_STORE_DIR, max_entries=_GLOBAL_MAX_ENTRIES)
256
+ summary = prompt.strip()
257
+ if len(summary) > 200:
258
+ summary = summary[:200] + "..."
259
+ store.add(summary, {
260
+ "tool_sequence": tools_used,
261
+ "key_paths": sorted(set(key_paths)),
262
+ "outcome": outcome,
263
+ })
264
+ except Exception:
265
+ pass
266
+
267
+
268
+ def search_memory_tool(config: dict[str, Any], query: str, top_k: int = 3) -> str:
269
+ if not bool(config.get("memory_enabled", True)):
270
+ return "Memory search is disabled."
271
+
272
+ project_store = VectorStore(config)
273
+ global_store = VectorStore(config, store_dir=_GLOBAL_STORE_DIR, max_entries=_GLOBAL_MAX_ENTRIES)
274
+ combined = project_store.search(query, top_k=top_k) + global_store.search(query, top_k=top_k)
275
+
276
+ # Merge: deduplicate by content hash, rank by similarity score.
277
+ seen: set[int] = set()
278
+ merged: list[dict[str, Any]] = []
279
+ for r in sorted(combined, key=lambda x: x.get("score", 0.0), reverse=True):
280
+ h = hash(r.get("text", ""))
281
+ if h not in seen:
282
+ seen.add(h)
283
+ merged.append(r)
284
+
285
+ if not merged:
286
+ return "No relevant memory found."
287
+ lines = []
288
+ for r in merged[:top_k]:
289
+ tools = ", ".join(r.get("tool_sequence", []) or [])
290
+ paths = ", ".join(r.get("key_paths", []) or [])
291
+ lines.append(
292
+ f"- [{r.get('created_at', '?')}] (score {r.get('score')}) {r.get('text', '')} "
293
+ f"| tools used: {tools or '(none)'} | files: {paths or '(none)'}"
294
+ )
295
+ return "\n".join(lines)
296
+
297
+
298
+ # ---------------------------------------------------------------------------
299
+ # Idle timer
300
+ # ---------------------------------------------------------------------------
301
+
302
+ def touch_last_turn() -> None:
303
+ """Reset the idle timer. Called from the REPL on every user input."""
304
+ global _last_turn_time
305
+ _last_turn_time = time.monotonic()
306
+
307
+
308
+ # ---------------------------------------------------------------------------
309
+ # Memory rules (Feature 15 — rules injection)
310
+ # ---------------------------------------------------------------------------
311
+
312
+ def read_memory_rules(max_rules: int = 5) -> list[str]:
313
+ """Return the last max_rules bullet lines from ~/.shellai/memory_rules.md."""
314
+ try:
315
+ if not _RULES_PATH.exists():
316
+ return []
317
+ lines = _RULES_PATH.read_text(encoding="utf-8").splitlines()
318
+ rules = [ln.strip() for ln in lines if ln.strip().startswith("- ")]
319
+ return rules[-max_rules:]
320
+ except Exception:
321
+ return []
322
+
323
+
324
+ def _append_rules(new_rules: list[str]) -> None:
325
+ """Append rules to memory_rules.md, evicting oldest if count exceeds _MAX_RULES."""
326
+ try:
327
+ _RULES_PATH.parent.mkdir(parents=True, exist_ok=True)
328
+ existing: list[str] = []
329
+ if _RULES_PATH.exists():
330
+ existing = [
331
+ ln.strip()
332
+ for ln in _RULES_PATH.read_text(encoding="utf-8").splitlines()
333
+ if ln.strip().startswith("- ")
334
+ ]
335
+ ts = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M")
336
+ stamped = [f"- [{ts}] {r.lstrip('- ').strip()}" for r in new_rules if r.strip()]
337
+ combined = existing + stamped
338
+ if len(combined) > _MAX_RULES:
339
+ combined = combined[-_MAX_RULES:]
340
+ tmp = _RULES_PATH.with_suffix(".tmp")
341
+ tmp.write_text("\n".join(combined) + "\n", encoding="utf-8")
342
+ tmp.replace(_RULES_PATH)
343
+ except Exception:
344
+ pass
345
+
346
+
347
+ def prune_memory_rules() -> int:
348
+ """Keep only the newest _MAX_RULES rules. Returns the number removed."""
349
+ try:
350
+ if not _RULES_PATH.exists():
351
+ return 0
352
+ lines = [
353
+ ln.strip()
354
+ for ln in _RULES_PATH.read_text(encoding="utf-8").splitlines()
355
+ if ln.strip().startswith("- ")
356
+ ]
357
+ if len(lines) <= _MAX_RULES:
358
+ return 0
359
+ removed = len(lines) - _MAX_RULES
360
+ tmp = _RULES_PATH.with_suffix(".tmp")
361
+ tmp.write_text("\n".join(lines[-_MAX_RULES:]) + "\n", encoding="utf-8")
362
+ tmp.replace(_RULES_PATH)
363
+ return removed
364
+ except Exception:
365
+ return 0
366
+
367
+
368
+ # ---------------------------------------------------------------------------
369
+ # Dreaming daemon (Feature 14 — async consolidation)
370
+ # ---------------------------------------------------------------------------
371
+
372
+ def _consolidate() -> None:
373
+ """Pull recent global entries, generate rules via LLM, append to rules file."""
374
+ global _dream_config_fn, _dream_llm_fn
375
+ if _dream_config_fn is None or _dream_llm_fn is None:
376
+ return
377
+ try:
378
+ store = VectorStore(None, store_dir=_GLOBAL_STORE_DIR, max_entries=_GLOBAL_MAX_ENTRIES)
379
+ store._load()
380
+ if not store._meta:
381
+ return
382
+ notes = "\n".join(e.get("text", "") for e in store._meta[-20:])
383
+ if not notes.strip():
384
+ return
385
+
386
+ if not _NPU_INFERENCE_LOCK.acquire(timeout=5):
387
+ return # main loop is busy; skip this dreaming cycle
388
+ try:
389
+ config = _dream_config_fn()
390
+ raw = _dream_llm_fn(config, _DREAM_SYSTEM, notes)
391
+ finally:
392
+ _NPU_INFERENCE_LOCK.release()
393
+
394
+ new_rules = [ln.strip() for ln in raw.splitlines() if ln.strip().startswith("- ")]
395
+ if new_rules:
396
+ _append_rules(new_rules)
397
+ except Exception:
398
+ pass
399
+
400
+
401
+ def _dream_loop() -> None:
402
+ """Background daemon: check every 30 s; fire consolidation after idle timeout."""
403
+ while True:
404
+ time.sleep(30)
405
+ if _last_turn_time > 0 and (time.monotonic() - _last_turn_time) >= _IDLE_TIMEOUT:
406
+ _consolidate()
407
+ touch_last_turn() # reset so it doesn't re-fire immediately
408
+
409
+
410
+ def start_dreaming(config_fn: Callable[[], dict[str, Any]], llm_fn: Callable) -> None:
411
+ """Start the background consolidation daemon. Called once from run_repl."""
412
+ global _dream_config_fn, _dream_llm_fn
413
+ _dream_config_fn = config_fn
414
+ _dream_llm_fn = llm_fn
415
+ t = threading.Thread(target=_dream_loop, daemon=True)
416
+ t.start()