hexcli 2.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hexcli/__init__.py +6 -0
- hexcli/agent.py +1931 -0
- hexcli/assets/hexcli.ico +0 -0
- hexcli/assets/hexcli.png +0 -0
- hexcli/cancel.py +76 -0
- hexcli/chatlog.py +232 -0
- hexcli/commands.py +73 -0
- hexcli/compaction.py +309 -0
- hexcli/config.py +217 -0
- hexcli/diffview.py +106 -0
- hexcli/distribution.py +237 -0
- hexcli/doctor.py +265 -0
- hexcli/escalate.py +192 -0
- hexcli/http_client.py +156 -0
- hexcli/launcher.py +481 -0
- hexcli/lineedit.py +1110 -0
- hexcli/llm.py +599 -0
- hexcli/local_escalation.py +191 -0
- hexcli/lockfile.py +71 -0
- hexcli/loop_v2.py +393 -0
- hexcli/markdown_stream.py +241 -0
- hexcli/memory.py +416 -0
- hexcli/network.py +154 -0
- hexcli/parsing.py +215 -0
- hexcli/paths.py +127 -0
- hexcli/prompts.py +321 -0
- hexcli/protocol_v2.py +505 -0
- hexcli/repl.py +807 -0
- hexcli/safety.py +127 -0
- hexcli/sessions.py +226 -0
- hexcli/setup_wizard.py +144 -0
- hexcli/shell_session.py +186 -0
- hexcli/statusbar.py +894 -0
- hexcli/stream_render.py +250 -0
- hexcli/telemetry.py +131 -0
- hexcli/tools.py +775 -0
- hexcli/ui.py +1106 -0
- hexcli-2.8.0.dist-info/METADATA +394 -0
- hexcli-2.8.0.dist-info/RECORD +42 -0
- hexcli-2.8.0.dist-info/WHEEL +4 -0
- hexcli-2.8.0.dist-info/entry_points.txt +3 -0
- hexcli-2.8.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""hexcli.markdown_stream — markdown-lite to ANSI, one character at a time.
|
|
3
|
+
|
|
4
|
+
Answers arrive as a token stream and used to print raw: ``## Heading``,
|
|
5
|
+
``**bold**``, backtick spans and ``` fences all showed their markers. This
|
|
6
|
+
turns the common subset into terminal styling without waiting for the line
|
|
7
|
+
to finish, so streaming keeps its word-by-word feel:
|
|
8
|
+
|
|
9
|
+
* ``# heading`` (any level) → bold, markers dropped
|
|
10
|
+
* ``- item`` / ``* item`` / ``+ item`` → ``• item`` (indentation kept)
|
|
11
|
+
* ``**bold**`` → bold
|
|
12
|
+
* `` `code` `` → cyan
|
|
13
|
+
* ``` ```lang ``` … ``` ``` ``` → a dim rule with the language, code left
|
|
14
|
+
exactly as written (no gutter, so it copies cleanly), a dim rule after
|
|
15
|
+
|
|
16
|
+
Markers are held only as long as they are ambiguous (at most the three
|
|
17
|
+
backticks of a fence, or one ``*``) and released literally when they turn
|
|
18
|
+
out not to be markup, so ``2 * 3`` and ``C# code`` come through untouched.
|
|
19
|
+
Feeding the same text whole or one character at a time yields identical
|
|
20
|
+
output; that property is what the tests pin down.
|
|
21
|
+
|
|
22
|
+
The styles come from ``ui.C`` and are empty strings when colour is off, in
|
|
23
|
+
which case only the bullet and fence substitutions remain.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
_FENCE_WIDTH = 40
|
|
28
|
+
_FENCE_LABEL_CHARS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_+#.-")
|
|
29
|
+
_FENCE_LABEL_MAX = 24 # "```" then more than a language name is not a fence line
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class MarkdownStream:
|
|
33
|
+
def __init__(self) -> None:
|
|
34
|
+
from hexcli.ui import C # lazy: ui imports this module for render_result
|
|
35
|
+
self._c = C
|
|
36
|
+
self._bol = True # at the beginning of a line
|
|
37
|
+
self._pending = "" # marker characters not yet decided
|
|
38
|
+
self._fence = False # inside a ``` block
|
|
39
|
+
self._fence_line: str | None = None # collecting "```lang" up to its newline
|
|
40
|
+
self._bold = False
|
|
41
|
+
self._code = False
|
|
42
|
+
self._heading = False
|
|
43
|
+
self._restyle = False # re-emit the open styles after a soft line break
|
|
44
|
+
|
|
45
|
+
# -- public -------------------------------------------------------------
|
|
46
|
+
|
|
47
|
+
def feed(self, text: str) -> str:
|
|
48
|
+
return "".join(self._char(ch) for ch in text)
|
|
49
|
+
|
|
50
|
+
def finish(self) -> str:
|
|
51
|
+
"""Release anything still held and close open styling."""
|
|
52
|
+
out = self._flush_pending()
|
|
53
|
+
if self._fence_line is not None:
|
|
54
|
+
out += self._fence_marker(self._fence_line)
|
|
55
|
+
self._fence_line = None
|
|
56
|
+
if self._restyle:
|
|
57
|
+
# Styles carried over a line break were already reset on screen
|
|
58
|
+
# and never re-opened: nothing to close.
|
|
59
|
+
self._bold = self._code = self._heading = False
|
|
60
|
+
self._restyle = False
|
|
61
|
+
return out + self._close_styles()
|
|
62
|
+
|
|
63
|
+
# -- styling ------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
def _style(self) -> str:
|
|
66
|
+
c = self._c
|
|
67
|
+
return (c.BOLD if (self._bold or self._heading) else "") + (c.BCYAN if self._code else "")
|
|
68
|
+
|
|
69
|
+
def _close_styles(self) -> str:
|
|
70
|
+
if not (self._bold or self._code or self._heading):
|
|
71
|
+
return ""
|
|
72
|
+
self._bold = self._code = self._heading = False
|
|
73
|
+
return self._c.RESET
|
|
74
|
+
|
|
75
|
+
def _toggle_bold(self) -> str:
|
|
76
|
+
if self._code:
|
|
77
|
+
return "**" # literal inside a code span
|
|
78
|
+
self._bold = not self._bold
|
|
79
|
+
return self._c.RESET + self._style() if not self._bold else self._c.BOLD
|
|
80
|
+
|
|
81
|
+
def _toggle_code(self) -> str:
|
|
82
|
+
self._code = not self._code
|
|
83
|
+
return (self._c.BCYAN if self._code else self._c.RESET + self._style())
|
|
84
|
+
|
|
85
|
+
def _bullet(self) -> str:
|
|
86
|
+
c = self._c
|
|
87
|
+
return f"{c.DIM}•{c.RESET} "
|
|
88
|
+
|
|
89
|
+
def _fence_marker(self, lang: str) -> str:
|
|
90
|
+
c = self._c
|
|
91
|
+
if self._fence:
|
|
92
|
+
self._fence = False
|
|
93
|
+
self._restyle = False
|
|
94
|
+
return f"{c.DIM}{'─' * _FENCE_WIDTH}{c.RESET}"
|
|
95
|
+
self._fence = True
|
|
96
|
+
label = f" {lang.strip()} " if lang.strip() else ""
|
|
97
|
+
return f"{c.DIM}{'─' * 4}{label}{'─' * max(2, _FENCE_WIDTH - 4 - len(label))}{c.RESET}"
|
|
98
|
+
|
|
99
|
+
def _flush_pending(self) -> str:
|
|
100
|
+
"""Release held markers at a line break or the end of the text. A
|
|
101
|
+
one- or two-backtick run there is a span boundary (`x` at the end of
|
|
102
|
+
a line is the common case); everything else is literal."""
|
|
103
|
+
p, self._pending = self._pending, ""
|
|
104
|
+
if p and set(p) == {"`"} and len(p) < 3 and not self._fence:
|
|
105
|
+
return self._toggle_code()
|
|
106
|
+
return p
|
|
107
|
+
|
|
108
|
+
# -- the state machine ----------------------------------------------------
|
|
109
|
+
|
|
110
|
+
def _char(self, ch: str) -> str:
|
|
111
|
+
if self._fence_line is not None:
|
|
112
|
+
if ch == "\n":
|
|
113
|
+
label = self._fence_line.strip()
|
|
114
|
+
self._fence_line = None
|
|
115
|
+
self._bol = True
|
|
116
|
+
if not self._fence and " " in label:
|
|
117
|
+
return "```" + label + "\n" # "```this is prose": not a fence
|
|
118
|
+
return self._fence_marker(label) + "\n"
|
|
119
|
+
# A closing fence takes anything after the backticks (trailing
|
|
120
|
+
# spaces, a stray word); an opening one only a language name.
|
|
121
|
+
ok = self._fence or (ch in _FENCE_LABEL_CHARS and len(self._fence_line) < _FENCE_LABEL_MAX)
|
|
122
|
+
if ok or ch in " \r":
|
|
123
|
+
self._fence_line += ch
|
|
124
|
+
return ""
|
|
125
|
+
# Not a fence after all ("```" followed by prose, or by code the
|
|
126
|
+
# model failed to break onto its own line): release the backticks
|
|
127
|
+
# and the label literally and carry on with the line.
|
|
128
|
+
held, self._fence_line = "```" + self._fence_line, None
|
|
129
|
+
self._bol = False
|
|
130
|
+
return held + self._inline_char(ch)
|
|
131
|
+
if ch == "\n":
|
|
132
|
+
# Bold and code spans may continue on the next line (a soft
|
|
133
|
+
# break); a blank line ends them. Headings end with their line.
|
|
134
|
+
out = self._flush_pending()
|
|
135
|
+
carry = (self._bold or self._code) and not self._bol
|
|
136
|
+
already_reset = self._restyle # a carried span was reset at the previous break
|
|
137
|
+
open_style = (self._bold or self._code or self._heading) and not already_reset
|
|
138
|
+
out += (self._c.RESET if open_style else "") + "\n"
|
|
139
|
+
self._heading = False
|
|
140
|
+
if carry:
|
|
141
|
+
self._restyle = True
|
|
142
|
+
else:
|
|
143
|
+
self._bold = self._code = False
|
|
144
|
+
self._restyle = False
|
|
145
|
+
self._bol = True
|
|
146
|
+
return out
|
|
147
|
+
if self._restyle and ch != "\n":
|
|
148
|
+
self._restyle = False
|
|
149
|
+
return self._style() + (self._bol_char(ch) if self._bol else self._inline_char(ch))
|
|
150
|
+
if self._bol:
|
|
151
|
+
return self._bol_char(ch)
|
|
152
|
+
return self._inline_char(ch)
|
|
153
|
+
|
|
154
|
+
def _bol_char(self, ch: str) -> str:
|
|
155
|
+
p = self._pending
|
|
156
|
+
# A fence opener or closer: three backticks at the start of a line.
|
|
157
|
+
if ch == "`" and p in ("", "`", "``"):
|
|
158
|
+
p += "`"
|
|
159
|
+
if p == "```":
|
|
160
|
+
self._pending = ""
|
|
161
|
+
self._fence_line = ""
|
|
162
|
+
return ""
|
|
163
|
+
self._pending = p
|
|
164
|
+
return ""
|
|
165
|
+
if p and set(p) == {"`"}:
|
|
166
|
+
# One or two backticks then something else: inline code after all.
|
|
167
|
+
self._pending = ""
|
|
168
|
+
self._bol = False
|
|
169
|
+
return "".join(self._inline_char(b) for b in p) + self._inline_char(ch)
|
|
170
|
+
if self._fence:
|
|
171
|
+
self._bol = False
|
|
172
|
+
return ch
|
|
173
|
+
if ch == " " and p == "":
|
|
174
|
+
return " " # indentation: still at the start for bullet purposes
|
|
175
|
+
if ch == "#" and (p == "" or set(p) == {"#"}) and len(p) < 6:
|
|
176
|
+
self._pending = p + "#"
|
|
177
|
+
return ""
|
|
178
|
+
if p and set(p) == {"#"}:
|
|
179
|
+
self._pending = ""
|
|
180
|
+
self._bol = False
|
|
181
|
+
if ch == " ":
|
|
182
|
+
self._heading = True
|
|
183
|
+
return self._c.BOLD
|
|
184
|
+
return p + self._inline_char(ch)
|
|
185
|
+
if p == "" and ch in "-*+":
|
|
186
|
+
self._pending = ch
|
|
187
|
+
return ""
|
|
188
|
+
if p in ("-", "+", "*"):
|
|
189
|
+
self._pending = ""
|
|
190
|
+
self._bol = False
|
|
191
|
+
if ch == " ":
|
|
192
|
+
return self._bullet()
|
|
193
|
+
if p == "*" and ch == "*":
|
|
194
|
+
self._pending = "**" # decided by what follows (see _inline_char)
|
|
195
|
+
return ""
|
|
196
|
+
return p + self._inline_char(ch)
|
|
197
|
+
self._bol = False
|
|
198
|
+
return self._inline_char(ch)
|
|
199
|
+
|
|
200
|
+
def _inline_char(self, ch: str) -> str:
|
|
201
|
+
if self._fence:
|
|
202
|
+
return ch
|
|
203
|
+
p = self._pending
|
|
204
|
+
if p == "**":
|
|
205
|
+
# An opening "**" must be followed by something other than a
|
|
206
|
+
# space (CommonMark's left-flanking rule); "next** x" is literal.
|
|
207
|
+
self._pending = ""
|
|
208
|
+
if ch in " \t":
|
|
209
|
+
return "**" + self._inline_char(ch)
|
|
210
|
+
return self._toggle_bold() + self._inline_char(ch)
|
|
211
|
+
if p == "*":
|
|
212
|
+
self._pending = ""
|
|
213
|
+
if ch == "*":
|
|
214
|
+
if self._bold or self._code:
|
|
215
|
+
return self._toggle_bold() # closing (or literal inside code)
|
|
216
|
+
self._pending = "**"
|
|
217
|
+
return ""
|
|
218
|
+
return "*" + self._inline_char(ch)
|
|
219
|
+
if p and set(p) == {"`"}:
|
|
220
|
+
# A run of backticks resolves on the next character: one or two
|
|
221
|
+
# open/close a code span, three mid-line are literal (a fence
|
|
222
|
+
# that never got its own line).
|
|
223
|
+
if ch == "`" and len(p) < 3:
|
|
224
|
+
self._pending = p + "`"
|
|
225
|
+
return ""
|
|
226
|
+
self._pending = ""
|
|
227
|
+
head = "```" if len(p) == 3 else self._toggle_code()
|
|
228
|
+
return head + self._inline_char(ch)
|
|
229
|
+
if ch == "*" and not self._code:
|
|
230
|
+
self._pending = "*"
|
|
231
|
+
return ""
|
|
232
|
+
if ch == "`":
|
|
233
|
+
self._pending = "`"
|
|
234
|
+
return ""
|
|
235
|
+
return ch
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def render_markdown(text: str) -> str:
|
|
239
|
+
"""The whole-text form, for answers that did not stream."""
|
|
240
|
+
md = MarkdownStream()
|
|
241
|
+
return md.feed(text) + md.finish()
|
hexcli/memory.py
ADDED
|
@@ -0,0 +1,416 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""hexcli.memory — lightweight on-device semantic memory for Hex CLI.
|
|
3
|
+
|
|
4
|
+
Pure NumPy vector index over a local ONNX sentence-embedding model
|
|
5
|
+
(sentence-transformers/all-MiniLM-L6-v2, ARM64-quantized). No FAISS/
|
|
6
|
+
ChromaDB/LangChain. One-way dependency, mirroring hexcli.ui and
|
|
7
|
+
hexcli.telemetry: hexcli.agent imports this module, never the reverse.
|
|
8
|
+
|
|
9
|
+
Every public method swallows its own exceptions — an embedding/model
|
|
10
|
+
load failure (e.g. offline on first use) must degrade to a silent
|
|
11
|
+
no-op, never crash the agent loop or block a turn that doesn't touch
|
|
12
|
+
memory.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import io
|
|
17
|
+
import json
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
import uuid
|
|
21
|
+
from collections.abc import Callable
|
|
22
|
+
from datetime import UTC, datetime
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
import numpy as np
|
|
27
|
+
|
|
28
|
+
_STORE_DIR_NAME = ".shellai/vector_store"
|
|
29
|
+
_EMBED_DIM = 384
|
|
30
|
+
_LOCAL_MODEL_PATH: Path | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def set_local_model_path(path: Path) -> None:
|
|
34
|
+
"""Tell the embedder where to find the ONNX model. Call once at startup from agent.py."""
|
|
35
|
+
global _LOCAL_MODEL_PATH
|
|
36
|
+
_LOCAL_MODEL_PATH = path
|
|
37
|
+
_MAX_ENTRIES = 500
|
|
38
|
+
_GLOBAL_MAX_ENTRIES = 1_000
|
|
39
|
+
_MIN_SIMILARITY = 0.15
|
|
40
|
+
_MAX_SEQ_LEN = 256
|
|
41
|
+
_MAX_RULES = 50
|
|
42
|
+
|
|
43
|
+
# Global store — cross-project, lives in the user's home dir.
|
|
44
|
+
_GLOBAL_STORE_DIR = Path.home() / ".shellai" / "global_vector_store"
|
|
45
|
+
_RULES_PATH = Path.home() / ".shellai" / "memory_rules.md"
|
|
46
|
+
|
|
47
|
+
# NPU inference lock — prevents the dreaming daemon from calling the LLM
|
|
48
|
+
# concurrently with the main agent loop. Acquired by call_llm in agent.py
|
|
49
|
+
# and by _consolidate here with a 5-second timeout.
|
|
50
|
+
_NPU_INFERENCE_LOCK: threading.Lock = threading.Lock()
|
|
51
|
+
|
|
52
|
+
# Idle timer for the dreaming daemon. touch_last_turn() resets it on each
|
|
53
|
+
# user input. _consolidate fires after _IDLE_TIMEOUT seconds of silence.
|
|
54
|
+
_last_turn_time: float = 0.0
|
|
55
|
+
_IDLE_TIMEOUT: float = 300.0 # 5 minutes
|
|
56
|
+
|
|
57
|
+
# Injected by start_dreaming() from agent.py — avoids a circular import.
|
|
58
|
+
_dream_config_fn: Callable[[], dict[str, Any]] | None = None
|
|
59
|
+
_dream_llm_fn: Callable[[dict[str, Any], str, str], str] | None = None
|
|
60
|
+
|
|
61
|
+
_DREAM_SYSTEM = (
|
|
62
|
+
"Extract 3–5 concise factual rules from these session notes. "
|
|
63
|
+
"Return only a Markdown bullet list (one rule per line, starting with '- '). "
|
|
64
|
+
"No preamble, no commentary, no numbering."
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class _Embedder:
|
|
69
|
+
"""Process-wide lazy singleton — the ONNX session/tokenizer load once
|
|
70
|
+
and are reused across every add()/search() call for the life of the
|
|
71
|
+
process, regardless of how many VectorStore instances are created."""
|
|
72
|
+
|
|
73
|
+
_instance: _Embedder | None = None
|
|
74
|
+
|
|
75
|
+
def __init__(self) -> None:
|
|
76
|
+
self._session: Any = None
|
|
77
|
+
self._tokenizer: Any = None
|
|
78
|
+
self._unavailable = False
|
|
79
|
+
|
|
80
|
+
@classmethod
|
|
81
|
+
def instance(cls) -> _Embedder:
|
|
82
|
+
if cls._instance is None:
|
|
83
|
+
cls._instance = cls()
|
|
84
|
+
return cls._instance
|
|
85
|
+
|
|
86
|
+
def _ensure_loaded(self) -> None:
|
|
87
|
+
if self._session is not None or self._unavailable:
|
|
88
|
+
return
|
|
89
|
+
if _LOCAL_MODEL_PATH is None or not _LOCAL_MODEL_PATH.exists():
|
|
90
|
+
self._unavailable = True
|
|
91
|
+
return
|
|
92
|
+
tok_path = _LOCAL_MODEL_PATH.parent / "tokenizer.json"
|
|
93
|
+
if not tok_path.exists():
|
|
94
|
+
self._unavailable = True
|
|
95
|
+
return
|
|
96
|
+
try:
|
|
97
|
+
import onnxruntime as ort
|
|
98
|
+
from tokenizers import Tokenizer
|
|
99
|
+
tokenizer = Tokenizer.from_file(str(tok_path))
|
|
100
|
+
tokenizer.enable_padding()
|
|
101
|
+
tokenizer.enable_truncation(max_length=_MAX_SEQ_LEN)
|
|
102
|
+
self._session = ort.InferenceSession(str(_LOCAL_MODEL_PATH), providers=["CPUExecutionProvider"])
|
|
103
|
+
self._tokenizer = tokenizer
|
|
104
|
+
except Exception:
|
|
105
|
+
self._unavailable = True
|
|
106
|
+
|
|
107
|
+
def embed(self, texts: list[str]) -> np.ndarray | None:
|
|
108
|
+
self._ensure_loaded()
|
|
109
|
+
if self._unavailable or not texts:
|
|
110
|
+
return None
|
|
111
|
+
try:
|
|
112
|
+
encodings = self._tokenizer.encode_batch(texts)
|
|
113
|
+
input_ids = np.array([e.ids for e in encodings], dtype=np.int64)
|
|
114
|
+
attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64)
|
|
115
|
+
token_type_ids = np.zeros_like(input_ids)
|
|
116
|
+
|
|
117
|
+
feed = {
|
|
118
|
+
"input_ids": input_ids,
|
|
119
|
+
"attention_mask": attention_mask,
|
|
120
|
+
"token_type_ids": token_type_ids,
|
|
121
|
+
}
|
|
122
|
+
token_embeddings = self._session.run(None, feed)[0]
|
|
123
|
+
|
|
124
|
+
mask = attention_mask[..., None].astype(np.float32)
|
|
125
|
+
summed = (token_embeddings * mask).sum(axis=1)
|
|
126
|
+
counts = np.clip(mask.sum(axis=1), 1e-9, None)
|
|
127
|
+
pooled = summed / counts
|
|
128
|
+
|
|
129
|
+
norms = np.clip(np.linalg.norm(pooled, axis=1, keepdims=True), 1e-9, None)
|
|
130
|
+
return (pooled / norms).astype(np.float32)
|
|
131
|
+
except Exception:
|
|
132
|
+
return None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class VectorStore:
|
|
136
|
+
"""One store per (config, cwd) — cheap to construct; only the shared
|
|
137
|
+
_Embedder singleton carries real load cost, so creating a fresh
|
|
138
|
+
VectorStore per tool call is fine."""
|
|
139
|
+
|
|
140
|
+
def __init__(
|
|
141
|
+
self,
|
|
142
|
+
config: dict[str, Any] | None = None,
|
|
143
|
+
cwd: str | None = None,
|
|
144
|
+
*,
|
|
145
|
+
store_dir: Path | None = None,
|
|
146
|
+
max_entries: int = _MAX_ENTRIES,
|
|
147
|
+
) -> None:
|
|
148
|
+
self.enabled = bool((config or {}).get("memory_enabled", True))
|
|
149
|
+
if store_dir is not None:
|
|
150
|
+
self._dir = store_dir
|
|
151
|
+
else:
|
|
152
|
+
base = Path(cwd) if cwd else Path.cwd()
|
|
153
|
+
self._dir = base / _STORE_DIR_NAME
|
|
154
|
+
self._vectors_path = self._dir / "vectors.npz"
|
|
155
|
+
self._meta_path = self._dir / "metadata.json"
|
|
156
|
+
self._vectors: np.ndarray = np.zeros((0, _EMBED_DIM), dtype=np.float32)
|
|
157
|
+
self._meta: list[dict[str, Any]] = []
|
|
158
|
+
self._loaded = False
|
|
159
|
+
self._max_entries = max_entries
|
|
160
|
+
|
|
161
|
+
def _load(self) -> None:
|
|
162
|
+
if self._loaded:
|
|
163
|
+
return
|
|
164
|
+
self._loaded = True
|
|
165
|
+
try:
|
|
166
|
+
if self._meta_path.exists():
|
|
167
|
+
self._meta = json.loads(self._meta_path.read_text(encoding="utf-8"))
|
|
168
|
+
if self._vectors_path.exists():
|
|
169
|
+
with np.load(self._vectors_path) as data:
|
|
170
|
+
self._vectors = data["vectors"].astype(np.float32)
|
|
171
|
+
except Exception:
|
|
172
|
+
self._vectors = np.zeros((0, _EMBED_DIM), dtype=np.float32)
|
|
173
|
+
self._meta = []
|
|
174
|
+
|
|
175
|
+
def add(self, text: str, metadata: dict[str, Any]) -> None:
|
|
176
|
+
if not self.enabled:
|
|
177
|
+
return
|
|
178
|
+
try:
|
|
179
|
+
self._load()
|
|
180
|
+
vec = _Embedder.instance().embed([text])
|
|
181
|
+
if vec is None:
|
|
182
|
+
return
|
|
183
|
+
self._vectors = np.vstack([self._vectors, vec])
|
|
184
|
+
entry = {
|
|
185
|
+
"id": str(uuid.uuid4()),
|
|
186
|
+
"created_at": datetime.now(UTC).isoformat(),
|
|
187
|
+
"text": text,
|
|
188
|
+
**metadata,
|
|
189
|
+
}
|
|
190
|
+
self._meta.append(entry)
|
|
191
|
+
if len(self._meta) > self._max_entries:
|
|
192
|
+
overflow = len(self._meta) - self._max_entries
|
|
193
|
+
self._meta = self._meta[overflow:]
|
|
194
|
+
self._vectors = self._vectors[overflow:]
|
|
195
|
+
self._save()
|
|
196
|
+
except Exception:
|
|
197
|
+
pass
|
|
198
|
+
|
|
199
|
+
def search(self, query: str, top_k: int = 3) -> list[dict[str, Any]]:
|
|
200
|
+
if not self.enabled:
|
|
201
|
+
return []
|
|
202
|
+
try:
|
|
203
|
+
self._load()
|
|
204
|
+
if self._vectors.shape[0] == 0:
|
|
205
|
+
return []
|
|
206
|
+
qvec = _Embedder.instance().embed([query])
|
|
207
|
+
if qvec is None:
|
|
208
|
+
return []
|
|
209
|
+
sims = self._vectors @ qvec[0]
|
|
210
|
+
order = np.argsort(-sims)[: max(top_k, 1)]
|
|
211
|
+
results: list[dict[str, Any]] = []
|
|
212
|
+
for idx in order:
|
|
213
|
+
score = float(sims[idx])
|
|
214
|
+
if score < _MIN_SIMILARITY:
|
|
215
|
+
continue
|
|
216
|
+
entry = dict(self._meta[idx])
|
|
217
|
+
entry["score"] = round(score, 3)
|
|
218
|
+
results.append(entry)
|
|
219
|
+
return results
|
|
220
|
+
except Exception:
|
|
221
|
+
return []
|
|
222
|
+
|
|
223
|
+
def _save(self) -> None:
|
|
224
|
+
self._dir.mkdir(parents=True, exist_ok=True)
|
|
225
|
+
|
|
226
|
+
buf = io.BytesIO()
|
|
227
|
+
np.savez(buf, vectors=self._vectors)
|
|
228
|
+
tmp_vectors = self._vectors_path.with_suffix(".tmp")
|
|
229
|
+
tmp_vectors.write_bytes(buf.getvalue())
|
|
230
|
+
tmp_vectors.replace(self._vectors_path)
|
|
231
|
+
|
|
232
|
+
tmp_meta = self._meta_path.with_suffix(".tmp")
|
|
233
|
+
tmp_meta.write_text(json.dumps(self._meta, indent=2), encoding="utf-8")
|
|
234
|
+
tmp_meta.replace(self._meta_path)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def maybe_index_turn(
|
|
238
|
+
config: dict[str, Any],
|
|
239
|
+
prompt: str,
|
|
240
|
+
tools_used: list[str],
|
|
241
|
+
key_paths: list[str],
|
|
242
|
+
outcome: str = "completed",
|
|
243
|
+
) -> None:
|
|
244
|
+
"""Auto-index a finished agentic turn. Silent no-op on any failure,
|
|
245
|
+
including memory_enabled=False, an empty tool sequence, or an
|
|
246
|
+
unreachable/unavailable embedding model."""
|
|
247
|
+
if not tools_used:
|
|
248
|
+
return
|
|
249
|
+
try:
|
|
250
|
+
# File-touching turns → project store (cwd-scoped).
|
|
251
|
+
# Non-file-touching turns (preferences, patterns) → global store.
|
|
252
|
+
if key_paths:
|
|
253
|
+
store = VectorStore(config)
|
|
254
|
+
else:
|
|
255
|
+
store = VectorStore(config, store_dir=_GLOBAL_STORE_DIR, max_entries=_GLOBAL_MAX_ENTRIES)
|
|
256
|
+
summary = prompt.strip()
|
|
257
|
+
if len(summary) > 200:
|
|
258
|
+
summary = summary[:200] + "..."
|
|
259
|
+
store.add(summary, {
|
|
260
|
+
"tool_sequence": tools_used,
|
|
261
|
+
"key_paths": sorted(set(key_paths)),
|
|
262
|
+
"outcome": outcome,
|
|
263
|
+
})
|
|
264
|
+
except Exception:
|
|
265
|
+
pass
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def search_memory_tool(config: dict[str, Any], query: str, top_k: int = 3) -> str:
|
|
269
|
+
if not bool(config.get("memory_enabled", True)):
|
|
270
|
+
return "Memory search is disabled."
|
|
271
|
+
|
|
272
|
+
project_store = VectorStore(config)
|
|
273
|
+
global_store = VectorStore(config, store_dir=_GLOBAL_STORE_DIR, max_entries=_GLOBAL_MAX_ENTRIES)
|
|
274
|
+
combined = project_store.search(query, top_k=top_k) + global_store.search(query, top_k=top_k)
|
|
275
|
+
|
|
276
|
+
# Merge: deduplicate by content hash, rank by similarity score.
|
|
277
|
+
seen: set[int] = set()
|
|
278
|
+
merged: list[dict[str, Any]] = []
|
|
279
|
+
for r in sorted(combined, key=lambda x: x.get("score", 0.0), reverse=True):
|
|
280
|
+
h = hash(r.get("text", ""))
|
|
281
|
+
if h not in seen:
|
|
282
|
+
seen.add(h)
|
|
283
|
+
merged.append(r)
|
|
284
|
+
|
|
285
|
+
if not merged:
|
|
286
|
+
return "No relevant memory found."
|
|
287
|
+
lines = []
|
|
288
|
+
for r in merged[:top_k]:
|
|
289
|
+
tools = ", ".join(r.get("tool_sequence", []) or [])
|
|
290
|
+
paths = ", ".join(r.get("key_paths", []) or [])
|
|
291
|
+
lines.append(
|
|
292
|
+
f"- [{r.get('created_at', '?')}] (score {r.get('score')}) {r.get('text', '')} "
|
|
293
|
+
f"| tools used: {tools or '(none)'} | files: {paths or '(none)'}"
|
|
294
|
+
)
|
|
295
|
+
return "\n".join(lines)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
# ---------------------------------------------------------------------------
|
|
299
|
+
# Idle timer
|
|
300
|
+
# ---------------------------------------------------------------------------
|
|
301
|
+
|
|
302
|
+
def touch_last_turn() -> None:
|
|
303
|
+
"""Reset the idle timer. Called from the REPL on every user input."""
|
|
304
|
+
global _last_turn_time
|
|
305
|
+
_last_turn_time = time.monotonic()
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
# ---------------------------------------------------------------------------
|
|
309
|
+
# Memory rules (Feature 15 — rules injection)
|
|
310
|
+
# ---------------------------------------------------------------------------
|
|
311
|
+
|
|
312
|
+
def read_memory_rules(max_rules: int = 5) -> list[str]:
|
|
313
|
+
"""Return the last max_rules bullet lines from ~/.shellai/memory_rules.md."""
|
|
314
|
+
try:
|
|
315
|
+
if not _RULES_PATH.exists():
|
|
316
|
+
return []
|
|
317
|
+
lines = _RULES_PATH.read_text(encoding="utf-8").splitlines()
|
|
318
|
+
rules = [ln.strip() for ln in lines if ln.strip().startswith("- ")]
|
|
319
|
+
return rules[-max_rules:]
|
|
320
|
+
except Exception:
|
|
321
|
+
return []
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _append_rules(new_rules: list[str]) -> None:
|
|
325
|
+
"""Append rules to memory_rules.md, evicting oldest if count exceeds _MAX_RULES."""
|
|
326
|
+
try:
|
|
327
|
+
_RULES_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
328
|
+
existing: list[str] = []
|
|
329
|
+
if _RULES_PATH.exists():
|
|
330
|
+
existing = [
|
|
331
|
+
ln.strip()
|
|
332
|
+
for ln in _RULES_PATH.read_text(encoding="utf-8").splitlines()
|
|
333
|
+
if ln.strip().startswith("- ")
|
|
334
|
+
]
|
|
335
|
+
ts = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M")
|
|
336
|
+
stamped = [f"- [{ts}] {r.lstrip('- ').strip()}" for r in new_rules if r.strip()]
|
|
337
|
+
combined = existing + stamped
|
|
338
|
+
if len(combined) > _MAX_RULES:
|
|
339
|
+
combined = combined[-_MAX_RULES:]
|
|
340
|
+
tmp = _RULES_PATH.with_suffix(".tmp")
|
|
341
|
+
tmp.write_text("\n".join(combined) + "\n", encoding="utf-8")
|
|
342
|
+
tmp.replace(_RULES_PATH)
|
|
343
|
+
except Exception:
|
|
344
|
+
pass
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def prune_memory_rules() -> int:
|
|
348
|
+
"""Keep only the newest _MAX_RULES rules. Returns the number removed."""
|
|
349
|
+
try:
|
|
350
|
+
if not _RULES_PATH.exists():
|
|
351
|
+
return 0
|
|
352
|
+
lines = [
|
|
353
|
+
ln.strip()
|
|
354
|
+
for ln in _RULES_PATH.read_text(encoding="utf-8").splitlines()
|
|
355
|
+
if ln.strip().startswith("- ")
|
|
356
|
+
]
|
|
357
|
+
if len(lines) <= _MAX_RULES:
|
|
358
|
+
return 0
|
|
359
|
+
removed = len(lines) - _MAX_RULES
|
|
360
|
+
tmp = _RULES_PATH.with_suffix(".tmp")
|
|
361
|
+
tmp.write_text("\n".join(lines[-_MAX_RULES:]) + "\n", encoding="utf-8")
|
|
362
|
+
tmp.replace(_RULES_PATH)
|
|
363
|
+
return removed
|
|
364
|
+
except Exception:
|
|
365
|
+
return 0
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
# ---------------------------------------------------------------------------
|
|
369
|
+
# Dreaming daemon (Feature 14 — async consolidation)
|
|
370
|
+
# ---------------------------------------------------------------------------
|
|
371
|
+
|
|
372
|
+
def _consolidate() -> None:
|
|
373
|
+
"""Pull recent global entries, generate rules via LLM, append to rules file."""
|
|
374
|
+
global _dream_config_fn, _dream_llm_fn
|
|
375
|
+
if _dream_config_fn is None or _dream_llm_fn is None:
|
|
376
|
+
return
|
|
377
|
+
try:
|
|
378
|
+
store = VectorStore(None, store_dir=_GLOBAL_STORE_DIR, max_entries=_GLOBAL_MAX_ENTRIES)
|
|
379
|
+
store._load()
|
|
380
|
+
if not store._meta:
|
|
381
|
+
return
|
|
382
|
+
notes = "\n".join(e.get("text", "") for e in store._meta[-20:])
|
|
383
|
+
if not notes.strip():
|
|
384
|
+
return
|
|
385
|
+
|
|
386
|
+
if not _NPU_INFERENCE_LOCK.acquire(timeout=5):
|
|
387
|
+
return # main loop is busy; skip this dreaming cycle
|
|
388
|
+
try:
|
|
389
|
+
config = _dream_config_fn()
|
|
390
|
+
raw = _dream_llm_fn(config, _DREAM_SYSTEM, notes)
|
|
391
|
+
finally:
|
|
392
|
+
_NPU_INFERENCE_LOCK.release()
|
|
393
|
+
|
|
394
|
+
new_rules = [ln.strip() for ln in raw.splitlines() if ln.strip().startswith("- ")]
|
|
395
|
+
if new_rules:
|
|
396
|
+
_append_rules(new_rules)
|
|
397
|
+
except Exception:
|
|
398
|
+
pass
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _dream_loop() -> None:
|
|
402
|
+
"""Background daemon: check every 30 s; fire consolidation after idle timeout."""
|
|
403
|
+
while True:
|
|
404
|
+
time.sleep(30)
|
|
405
|
+
if _last_turn_time > 0 and (time.monotonic() - _last_turn_time) >= _IDLE_TIMEOUT:
|
|
406
|
+
_consolidate()
|
|
407
|
+
touch_last_turn() # reset so it doesn't re-fire immediately
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def start_dreaming(config_fn: Callable[[], dict[str, Any]], llm_fn: Callable) -> None:
|
|
411
|
+
"""Start the background consolidation daemon. Called once from run_repl."""
|
|
412
|
+
global _dream_config_fn, _dream_llm_fn
|
|
413
|
+
_dream_config_fn = config_fn
|
|
414
|
+
_dream_llm_fn = llm_fn
|
|
415
|
+
t = threading.Thread(target=_dream_loop, daemon=True)
|
|
416
|
+
t.start()
|