subcortex 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- subcortex/__init__.py +3 -0
- subcortex/__main__.py +3 -0
- subcortex/adapters/__init__.py +48 -0
- subcortex/adapters/base.py +230 -0
- subcortex/adapters/claude_family.py +133 -0
- subcortex/adapters/codex.py +87 -0
- subcortex/adapters/copilot.py +60 -0
- subcortex/adapters/cursor.py +36 -0
- subcortex/adapters/docker_agent.py +115 -0
- subcortex/adapters/gemini_family.py +60 -0
- subcortex/adapters/grok.py +98 -0
- subcortex/adapters/kimi_code.py +138 -0
- subcortex/adapters/letta_vibe.py +96 -0
- subcortex/adapters/openhands.py +153 -0
- subcortex/auth.py +59 -0
- subcortex/backends/__init__.py +23 -0
- subcortex/backends/base.py +22 -0
- subcortex/backends/jev.py +460 -0
- subcortex/backends/laya.py +149 -0
- subcortex/cli.py +809 -0
- subcortex/client.py +77 -0
- subcortex/config.py +263 -0
- subcortex/daemon.py +502 -0
- subcortex/evalset.py +241 -0
- subcortex/hook.py +254 -0
- subcortex/installers/__init__.py +62 -0
- subcortex/installers/amp.py +39 -0
- subcortex/installers/base.py +874 -0
- subcortex/installers/claude_family.py +229 -0
- subcortex/installers/codex.py +110 -0
- subcortex/installers/copilot.py +65 -0
- subcortex/installers/crush.py +36 -0
- subcortex/installers/cursor.py +79 -0
- subcortex/installers/gemini_family.py +83 -0
- subcortex/installers/goose.py +186 -0
- subcortex/installers/kimi_code.py +71 -0
- subcortex/installers/mcp_only.py +111 -0
- subcortex/installers/more_hooks.py +184 -0
- subcortex/installers/opencode.py +66 -0
- subcortex/installers/openhands.py +84 -0
- subcortex/installers/pi_cline.py +53 -0
- subcortex/ledger.py +92 -0
- subcortex/localhttp.py +59 -0
- subcortex/mcp_server.py +187 -0
- subcortex/metrics.py +56 -0
- subcortex/plugins/amp/subcortex.ts +258 -0
- subcortex/plugins/cline/subcortex.ts +340 -0
- subcortex/plugins/opencode/subcortex.ts +265 -0
- subcortex/plugins/pi/subcortex.ts +292 -0
- subcortex/policy.py +341 -0
- subcortex/presets.py +163 -0
- subcortex/provision.py +188 -0
- subcortex/service.py +149 -0
- subcortex/state.py +137 -0
- subcortex/transcript.py +211 -0
- subcortex/tuis.py +51 -0
- subcortex/ui.py +319 -0
- subcortex/verdicts.py +233 -0
- subcortex/wizard.py +474 -0
- subcortex-0.3.0.dist-info/METADATA +287 -0
- subcortex-0.3.0.dist-info/RECORD +64 -0
- subcortex-0.3.0.dist-info/WHEEL +5 -0
- subcortex-0.3.0.dist-info/entry_points.txt +3 -0
- subcortex-0.3.0.dist-info/top_level.txt +1 -0
subcortex/tuis.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Canonical TUI names and their aliases (single source for CLI, adapters, installers)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Dict, Optional
|
|
6
|
+
|
|
7
|
+
ALIASES: Dict[str, str] = {
|
|
8
|
+
"claude": "claude-code",
|
|
9
|
+
"claudecode": "claude-code",
|
|
10
|
+
"codex-cli": "codex",
|
|
11
|
+
"open-code": "opencode",
|
|
12
|
+
"kimi": "kimi-code",
|
|
13
|
+
"kimicode": "kimi-code",
|
|
14
|
+
"qodercli": "qoder",
|
|
15
|
+
"qoder-cli": "qoder",
|
|
16
|
+
"codebuddy-code": "codebuddy",
|
|
17
|
+
"factory": "droid",
|
|
18
|
+
"factory-droid": "droid",
|
|
19
|
+
"junie-cli": "junie",
|
|
20
|
+
"devin-cli": "devin",
|
|
21
|
+
"open-hands": "openhands",
|
|
22
|
+
"ampcode": "amp",
|
|
23
|
+
"gemini": "gemini-cli",
|
|
24
|
+
"qwen": "qwen-code",
|
|
25
|
+
"cursor-agent": "cursor",
|
|
26
|
+
"cursor-cli": "cursor",
|
|
27
|
+
"copilot-cli": "copilot",
|
|
28
|
+
"github-copilot": "copilot",
|
|
29
|
+
"interpreter": "open-interpreter",
|
|
30
|
+
"openinterpreter": "open-interpreter",
|
|
31
|
+
"kilocode": "kilo",
|
|
32
|
+
"kilo-code": "kilo",
|
|
33
|
+
"grok": "grok-build",
|
|
34
|
+
"cagent": "docker-agent",
|
|
35
|
+
"docker": "docker-agent",
|
|
36
|
+
"letta-code": "letta",
|
|
37
|
+
"mistral-vibe": "vibe",
|
|
38
|
+
"augment": "auggie",
|
|
39
|
+
"kiro-cli": "kiro",
|
|
40
|
+
"cline-cli": "cline",
|
|
41
|
+
"pi-coding-agent": "pi",
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def canonical(name: str, known) -> Optional[str]:
|
|
46
|
+
"""Canonical id for ``name`` if it (or its alias) is in ``known``."""
|
|
47
|
+
key = str(name or "").strip().lower().replace("_", "-")
|
|
48
|
+
if key in known:
|
|
49
|
+
return key
|
|
50
|
+
alias = ALIASES.get(key) or ALIASES.get(key.replace("-", ""))
|
|
51
|
+
return alias if alias in known else None
|
subcortex/ui.py
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
"""Terminal prompts for interactive setup (stdlib only).
|
|
2
|
+
|
|
3
|
+
In a real terminal, menus use the arrow keys (↑/↓ or j/k to move, space to
|
|
4
|
+
toggle, a to toggle all, enter to accept, esc/q to cancel). Anywhere else —
|
|
5
|
+
pipes, CI, tests — every prompt falls back to plain numbered line input, and
|
|
6
|
+
with no input at all each prompt returns its default. ``Cancelled`` is raised
|
|
7
|
+
when the user backs out (esc, q, ctrl-c).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
from typing import Any, Callable, Iterator, List, Optional, Sequence, Set, TextIO, Tuple
|
|
18
|
+
|
|
19
|
+
Option = Tuple[Any, str, str] # (value, label, hint)
|
|
20
|
+
|
|
21
|
+
UP, DOWN, SPACE, ENTER, CANCEL, TOGGLE_ALL = "up", "down", "space", "enter", "cancel", "all"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Cancelled(Exception):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _read_key(stream: TextIO) -> str:
|
|
29
|
+
"""One keypress from a terminal in cbreak mode, normalized."""
|
|
30
|
+
ch = stream.read(1)
|
|
31
|
+
if ch == "\x1b":
|
|
32
|
+
nxt = stream.read(1)
|
|
33
|
+
if nxt == "[":
|
|
34
|
+
code = stream.read(1)
|
|
35
|
+
return {"A": UP, "B": DOWN}.get(code, "")
|
|
36
|
+
return CANCEL
|
|
37
|
+
if ch in ("\r", "\n"):
|
|
38
|
+
return ENTER
|
|
39
|
+
if ch == " ":
|
|
40
|
+
return SPACE
|
|
41
|
+
if ch in ("k",):
|
|
42
|
+
return UP
|
|
43
|
+
if ch in ("j",):
|
|
44
|
+
return DOWN
|
|
45
|
+
if ch in ("a", "A"):
|
|
46
|
+
return TOGGLE_ALL
|
|
47
|
+
if ch in ("q", "Q", "\x03", "\x04"):
|
|
48
|
+
return CANCEL
|
|
49
|
+
return ""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class UI:
|
|
53
|
+
def __init__(self, stdin: Optional[TextIO] = None, stdout: Optional[TextIO] = None,
|
|
54
|
+
interactive: Optional[bool] = None, color: Optional[bool] = None,
|
|
55
|
+
keys: Optional[Iterator[str]] = None) -> None:
|
|
56
|
+
self.stdin = stdin or sys.stdin
|
|
57
|
+
self.stdout = stdout or sys.stdout
|
|
58
|
+
tty = _isatty(self.stdin) and _isatty(self.stdout) and os.environ.get("TERM") != "dumb"
|
|
59
|
+
self.interactive = tty if interactive is None else interactive
|
|
60
|
+
self.color = (tty and not os.environ.get("NO_COLOR")) if color is None else color
|
|
61
|
+
self._keys = keys # injected keypresses (tests)
|
|
62
|
+
|
|
63
|
+
# -- output -------------------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
def _c(self, code: str, text: str) -> str:
|
|
66
|
+
return f"\x1b[{code}m{text}\x1b[0m" if self.color else text
|
|
67
|
+
|
|
68
|
+
def write(self, text: str = "") -> None:
|
|
69
|
+
self.stdout.write(text + "\n")
|
|
70
|
+
self.stdout.flush()
|
|
71
|
+
|
|
72
|
+
def title(self, text: str) -> None:
|
|
73
|
+
self.write()
|
|
74
|
+
self.write(self._c("1", f"◆ {text}"))
|
|
75
|
+
|
|
76
|
+
def step(self, n: int, total: int, text: str) -> None:
|
|
77
|
+
self.write()
|
|
78
|
+
self.write(self._c("1;36", f"[{n}/{total}] {text}"))
|
|
79
|
+
|
|
80
|
+
def info(self, text: str) -> None:
|
|
81
|
+
self.write(f" {text}")
|
|
82
|
+
|
|
83
|
+
def dim(self, text: str) -> None:
|
|
84
|
+
self.write(self._c("2", f" {text}"))
|
|
85
|
+
|
|
86
|
+
def ok(self, text: str) -> None:
|
|
87
|
+
self.write(f" {self._c('32', '✓')} {text}")
|
|
88
|
+
|
|
89
|
+
def warn(self, text: str) -> None:
|
|
90
|
+
self.write(f" {self._c('33', '!')} {text}")
|
|
91
|
+
|
|
92
|
+
def error(self, text: str) -> None:
|
|
93
|
+
self.write(f" {self._c('31', '✗')} {text}")
|
|
94
|
+
|
|
95
|
+
# -- line input -----------------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
def _readline(self, prompt: str) -> Optional[str]:
|
|
98
|
+
self.stdout.write(prompt)
|
|
99
|
+
self.stdout.flush()
|
|
100
|
+
try:
|
|
101
|
+
line = self.stdin.readline()
|
|
102
|
+
except KeyboardInterrupt:
|
|
103
|
+
raise Cancelled()
|
|
104
|
+
if line == "": # EOF: no more input, use defaults
|
|
105
|
+
self.stdout.write("\n")
|
|
106
|
+
return None
|
|
107
|
+
return line.rstrip("\n")
|
|
108
|
+
|
|
109
|
+
def confirm(self, question: str, default: bool = True) -> bool:
|
|
110
|
+
suffix = "[Y/n]" if default else "[y/N]"
|
|
111
|
+
while True:
|
|
112
|
+
answer = self._readline(f" {question} {self._c('2', suffix)} ")
|
|
113
|
+
if answer is None or not answer.strip():
|
|
114
|
+
return default
|
|
115
|
+
if answer.strip().lower() in ("y", "yes"):
|
|
116
|
+
return True
|
|
117
|
+
if answer.strip().lower() in ("n", "no"):
|
|
118
|
+
return False
|
|
119
|
+
self.warn("please answer y or n")
|
|
120
|
+
|
|
121
|
+
def ask(self, question: str, default: str = "", secret: bool = False,
|
|
122
|
+
validate: Optional[Callable[[str], Optional[str]]] = None) -> str:
|
|
123
|
+
shown = f" {self._c('2', f'[{default}]')}" if default and not secret else ""
|
|
124
|
+
while True:
|
|
125
|
+
if secret and self.interactive:
|
|
126
|
+
import getpass
|
|
127
|
+
|
|
128
|
+
try:
|
|
129
|
+
answer: Optional[str] = getpass.getpass(f" {question}: ", stream=self.stdout)
|
|
130
|
+
except (EOFError, KeyboardInterrupt):
|
|
131
|
+
raise Cancelled()
|
|
132
|
+
else:
|
|
133
|
+
answer = self._readline(f" {question}{shown}: ")
|
|
134
|
+
value = default if answer is None or not answer.strip() else answer.strip()
|
|
135
|
+
problem = validate(value) if validate else None
|
|
136
|
+
if not problem:
|
|
137
|
+
return value
|
|
138
|
+
self.warn(problem)
|
|
139
|
+
if answer is None: # no more input: don't loop forever
|
|
140
|
+
raise Cancelled()
|
|
141
|
+
|
|
142
|
+
# -- menus ------------------------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
def choose(self, question: str, options: Sequence[Option], default: int = 0) -> Any:
|
|
145
|
+
"""Single choice; returns the chosen option's value."""
|
|
146
|
+
if not options:
|
|
147
|
+
raise ValueError("no options")
|
|
148
|
+
if self.interactive:
|
|
149
|
+
index = self._menu(question, options, cursor=default, multi=False, selected=set())
|
|
150
|
+
return options[index][0]
|
|
151
|
+
self.write(f" {question}")
|
|
152
|
+
for i, (_, label, hint) in enumerate(options, 1):
|
|
153
|
+
marker = "*" if i - 1 == default else " "
|
|
154
|
+
self.write(f" {marker}{i}) {label}" + (f" — {hint}" if hint else ""))
|
|
155
|
+
while True:
|
|
156
|
+
answer = self._readline(f" choice {self._c('2', f'[{default + 1}]')}: ")
|
|
157
|
+
if answer is None or not answer.strip():
|
|
158
|
+
return options[default][0]
|
|
159
|
+
if answer.strip().isdigit() and 1 <= int(answer) <= len(options):
|
|
160
|
+
return options[int(answer) - 1][0]
|
|
161
|
+
self.warn(f"enter a number from 1 to {len(options)}")
|
|
162
|
+
|
|
163
|
+
def checklist(self, question: str, options: Sequence[Option], selected: Set[Any]) -> List[Any]:
|
|
164
|
+
"""Multiple choice; returns the chosen values in option order."""
|
|
165
|
+
chosen = {i for i, (value, _, _) in enumerate(options) if value in selected}
|
|
166
|
+
if self.interactive:
|
|
167
|
+
self._menu(question, options, cursor=0, multi=True, selected=chosen)
|
|
168
|
+
return [options[i][0] for i in sorted(chosen)]
|
|
169
|
+
while True:
|
|
170
|
+
self.write(f" {question}")
|
|
171
|
+
for i, (_, label, hint) in enumerate(options, 1):
|
|
172
|
+
box = "x" if i - 1 in chosen else " "
|
|
173
|
+
self.write(f" [{box}] {i:>2}) {label}" + (f" — {hint}" if hint else ""))
|
|
174
|
+
answer = self._readline(" numbers to toggle (e.g. 1 3), 'all', 'none', or enter to accept: ")
|
|
175
|
+
if answer is None or not answer.strip():
|
|
176
|
+
return [options[i][0] for i in sorted(chosen)]
|
|
177
|
+
tokens = answer.replace(",", " ").split()
|
|
178
|
+
if tokens == ["all"]:
|
|
179
|
+
chosen = set(range(len(options)))
|
|
180
|
+
elif tokens == ["none"]:
|
|
181
|
+
chosen = set()
|
|
182
|
+
elif all(t.isdigit() and 1 <= int(t) <= len(options) for t in tokens):
|
|
183
|
+
chosen ^= {int(t) - 1 for t in tokens}
|
|
184
|
+
else:
|
|
185
|
+
self.warn(f"use numbers from 1 to {len(options)}")
|
|
186
|
+
|
|
187
|
+
def _next_key(self) -> str:
|
|
188
|
+
if self._keys is not None:
|
|
189
|
+
try:
|
|
190
|
+
return next(self._keys)
|
|
191
|
+
except StopIteration:
|
|
192
|
+
return ENTER
|
|
193
|
+
return _read_key(self.stdin)
|
|
194
|
+
|
|
195
|
+
def _menu(self, question: str, options: Sequence[Option], cursor: int, multi: bool,
|
|
196
|
+
selected: Set[int]) -> int:
|
|
197
|
+
help_text = ("↑/↓ move · space toggle · a all · enter accept · esc cancel" if multi
|
|
198
|
+
else "↑/↓ move · enter select · esc cancel")
|
|
199
|
+
visible = min(len(options), self._max_visible())
|
|
200
|
+
scrolling = visible < len(options)
|
|
201
|
+
lines = visible + (2 if scrolling else 0) # constant height keeps redraws aligned
|
|
202
|
+
top = 0
|
|
203
|
+
# cbreak before the question is shown: keys typed as soon as it appears
|
|
204
|
+
# must reach the menu, not get echoed and line-buffered by the terminal.
|
|
205
|
+
with self._raw_mode():
|
|
206
|
+
self.write(f" {question}")
|
|
207
|
+
self.dim(help_text)
|
|
208
|
+
self.stdout.write("\x1b[?25l") # hide cursor
|
|
209
|
+
try:
|
|
210
|
+
first = True
|
|
211
|
+
while True:
|
|
212
|
+
if not first:
|
|
213
|
+
self.stdout.write(f"\x1b[{lines}A")
|
|
214
|
+
first = False
|
|
215
|
+
top = min(max(top, cursor - visible + 1), cursor) # keep the cursor in view
|
|
216
|
+
window = range(top, top + visible)
|
|
217
|
+
if scrolling:
|
|
218
|
+
more = f"↑ {top} more" if top else ""
|
|
219
|
+
self.stdout.write(f"\x1b[2K {self._c('2', more)}\n")
|
|
220
|
+
for i in window:
|
|
221
|
+
_, label, hint = options[i]
|
|
222
|
+
pointer = self._c("36", "❯") if i == cursor else " "
|
|
223
|
+
box = ""
|
|
224
|
+
if multi:
|
|
225
|
+
box = (self._c("32", "◉") if i in selected else "○") + " "
|
|
226
|
+
text = f"{label}" + (self._c("2", f" {hint}") if hint else "")
|
|
227
|
+
if i == cursor:
|
|
228
|
+
text = self._c("1", label) + (self._c("2", f" {hint}") if hint else "")
|
|
229
|
+
self.stdout.write(f"\x1b[2K {pointer} {box}{text}\n")
|
|
230
|
+
if scrolling:
|
|
231
|
+
below = len(options) - top - visible
|
|
232
|
+
more = f"↓ {below} more" if below else ""
|
|
233
|
+
self.stdout.write(f"\x1b[2K {self._c('2', more)}\n")
|
|
234
|
+
self.stdout.flush()
|
|
235
|
+
key = self._next_key()
|
|
236
|
+
if key == UP:
|
|
237
|
+
cursor = (cursor - 1) % len(options)
|
|
238
|
+
elif key == DOWN:
|
|
239
|
+
cursor = (cursor + 1) % len(options)
|
|
240
|
+
elif key == SPACE and multi:
|
|
241
|
+
selected ^= {cursor}
|
|
242
|
+
elif key == TOGGLE_ALL and multi:
|
|
243
|
+
if len(selected) == len(options):
|
|
244
|
+
selected.clear()
|
|
245
|
+
else:
|
|
246
|
+
selected.update(range(len(options)))
|
|
247
|
+
elif key == ENTER:
|
|
248
|
+
return cursor
|
|
249
|
+
elif key == CANCEL:
|
|
250
|
+
raise Cancelled()
|
|
251
|
+
finally:
|
|
252
|
+
self.stdout.write("\x1b[?25h")
|
|
253
|
+
self.stdout.flush()
|
|
254
|
+
|
|
255
|
+
def _max_visible(self) -> int:
|
|
256
|
+
"""Menu rows that fit the terminal (header, help and margins excluded)."""
|
|
257
|
+
import shutil
|
|
258
|
+
|
|
259
|
+
rows = shutil.get_terminal_size(fallback=(80, 24)).lines
|
|
260
|
+
return max(5, rows - 8)
|
|
261
|
+
|
|
262
|
+
@contextmanager
|
|
263
|
+
def _raw_mode(self) -> Iterator[None]:
|
|
264
|
+
if self._keys is not None:
|
|
265
|
+
yield
|
|
266
|
+
return
|
|
267
|
+
try:
|
|
268
|
+
import termios
|
|
269
|
+
import tty
|
|
270
|
+
|
|
271
|
+
fd = self.stdin.fileno()
|
|
272
|
+
saved = termios.tcgetattr(fd)
|
|
273
|
+
except Exception:
|
|
274
|
+
yield
|
|
275
|
+
return
|
|
276
|
+
try:
|
|
277
|
+
# TCSADRAIN, not setcbreak's default TCSAFLUSH: that discards type-ahead.
|
|
278
|
+
tty.setcbreak(fd, termios.TCSADRAIN)
|
|
279
|
+
yield
|
|
280
|
+
finally:
|
|
281
|
+
termios.tcsetattr(fd, termios.TCSADRAIN, saved)
|
|
282
|
+
|
|
283
|
+
# -- progress ---------------------------------------------------------------------------
|
|
284
|
+
|
|
285
|
+
@contextmanager
|
|
286
|
+
def spinner(self, text: str) -> Iterator[None]:
|
|
287
|
+
"""Show elapsed time next to ``text`` while a slow step runs."""
|
|
288
|
+
if not self.interactive:
|
|
289
|
+
self.info(f"{text}…")
|
|
290
|
+
yield
|
|
291
|
+
return
|
|
292
|
+
done = threading.Event()
|
|
293
|
+
started = time.time()
|
|
294
|
+
|
|
295
|
+
def spin() -> None:
|
|
296
|
+
frames = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
|
|
297
|
+
i = 0
|
|
298
|
+
while not done.wait(0.1):
|
|
299
|
+
self.stdout.write(f"\r\x1b[2K {frames[i % len(frames)]} {text} "
|
|
300
|
+
f"{self._c('2', f'{time.time() - started:.0f}s')}")
|
|
301
|
+
self.stdout.flush()
|
|
302
|
+
i += 1
|
|
303
|
+
|
|
304
|
+
thread = threading.Thread(target=spin, daemon=True)
|
|
305
|
+
thread.start()
|
|
306
|
+
try:
|
|
307
|
+
yield
|
|
308
|
+
finally:
|
|
309
|
+
done.set()
|
|
310
|
+
thread.join()
|
|
311
|
+
self.stdout.write("\r\x1b[2K")
|
|
312
|
+
self.stdout.flush()
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _isatty(stream: Any) -> bool:
|
|
316
|
+
try:
|
|
317
|
+
return bool(stream.isatty())
|
|
318
|
+
except Exception:
|
|
319
|
+
return False
|
subcortex/verdicts.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
"""High-level System-1 verdicts built on typed-decision backends.
|
|
2
|
+
|
|
3
|
+
Both functions take an optional ``backend`` (defaults to the configured one),
|
|
4
|
+
and NEVER raise — they return ``None`` on any failure so callers fail open.
|
|
5
|
+
|
|
6
|
+
Questions and thresholds are calibrated per backend against the labeled
|
|
7
|
+
examples in ``subcortex.evalset`` (``subcortex eval`` reruns it; numbers in
|
|
8
|
+
docs/calibration.md). The same question gets very different probability scales
|
|
9
|
+
from Jev and from the local Laya model, so each backend has its own rule. A
|
|
10
|
+
rule is a primary signal plus a veto: every condition must hold. Each question
|
|
11
|
+
is atomic, names the state fields it reads (the model never sees question ids),
|
|
12
|
+
and the evidence sits first in the state (Laya truncates the end of it).
|
|
13
|
+
|
|
14
|
+
What leaves the machine for a hosted backend is bounded and redacted: the
|
|
15
|
+
request, the tool call and a head+tail excerpt of the output, with anything
|
|
16
|
+
that looks like a secret masked.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import re
|
|
22
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
23
|
+
|
|
24
|
+
from . import metrics
|
|
25
|
+
from .backends import get_backend
|
|
26
|
+
from .backends.base import DecisionBackend
|
|
27
|
+
from .config import load_config
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _noul(instructions: str) -> Dict[str, Any]:
|
|
31
|
+
return {"type": "noul", "instructions": instructions}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
PROMPT_QUESTIONS: Dict[str, Any] = {
|
|
35
|
+
"quick": _noul("The request in `prompt` can be fully handled with one fact, one command, "
|
|
36
|
+
"or an edit of a few lines."),
|
|
37
|
+
"multi_step": _noul("The request in `prompt` needs several dependent steps of work or reasoning."),
|
|
38
|
+
"multi_file": _noul("The request in `prompt` asks for code changes in more than one file."),
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
OUTPUT_QUESTIONS: Dict[str, Any] = {
|
|
42
|
+
"routine": _noul("`output` is routine progress or log output, such as downloads, compilation "
|
|
43
|
+
"steps, or install messages, with nothing specific to the request in `task`."),
|
|
44
|
+
# Compound on paper, but it is the best-separating veto on both backends.
|
|
45
|
+
"needed": _noul("Is the tool output in `output`, produced by the call in `tool_call`, still "
|
|
46
|
+
"needed to accomplish the user's request in `task` — i.e. would dropping it "
|
|
47
|
+
"lose information that cannot be cheaply re-derived by re-running the call?"),
|
|
48
|
+
"depends": _noul("The request in `task` depends on information that appears in `output`."),
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
# (question, inverted?, threshold). Signals are oriented so that high means
|
|
52
|
+
# "simple" (prompt rules) or "needed" (output rules). A prompt is simple when
|
|
53
|
+
# every oriented signal is >= its threshold; an output is disposable when every
|
|
54
|
+
# oriented signal is < its threshold. The first condition is the primary one:
|
|
55
|
+
# it is reported, and the config threshold overrides it.
|
|
56
|
+
Rule = List[Tuple[str, bool, float]]
|
|
57
|
+
PROMPT_RULES: Dict[str, Rule] = {
|
|
58
|
+
# evalset, jev-1.13.0: 16/20 simple prompts hinted, 0/20 complex.
|
|
59
|
+
"jev": [("quick", False, 0.8), ("multi_step", True, 0.5)],
|
|
60
|
+
# evalset, laya multilingual: 6/20 simple, 0/20 complex (weaker model, stricter rule).
|
|
61
|
+
"laya": [("multi_step", True, 0.8), ("multi_file", True, 0.8)],
|
|
62
|
+
}
|
|
63
|
+
OUTPUT_RULES: Dict[str, Rule] = {
|
|
64
|
+
# evalset, jev-1.13.0: 12/12 disposable outputs trimmed, 0/12 needed ones;
|
|
65
|
+
# held out: 6/6 and 0/6.
|
|
66
|
+
"jev": [("routine", True, 0.4), ("needed", False, 0.3)],
|
|
67
|
+
# evalset, laya multilingual: 3/12 and 0/12 — but held out it trimmed 2/6
|
|
68
|
+
# NEEDED outputs (npm audit, git log). Opt-in only (see TRIMS_BY_DEFAULT).
|
|
69
|
+
"laya": [("depends", False, 0.9), ("needed", False, 0.75)],
|
|
70
|
+
}
|
|
71
|
+
# Backends whose output rule held up on the held-out examples. The others only
|
|
72
|
+
# trim when the user sets thresholds.output_needed_threshold explicitly.
|
|
73
|
+
TRIMS_BY_DEFAULT = frozenset({"jev"})
|
|
74
|
+
DEFAULT_PROFILE = "laya" # the conservative rule, for backends nobody calibrated
|
|
75
|
+
|
|
76
|
+
TASK_CHARS = 600
|
|
77
|
+
TOOL_CALL_CHARS = 300
|
|
78
|
+
OUTPUT_EXCERPT_CHARS = 1500
|
|
79
|
+
PROMPT_CHARS = 2000
|
|
80
|
+
|
|
81
|
+
# -- redaction ------------------------------------------------------------------------------
|
|
82
|
+
# Linear-time patterns only (no nested quantifiers): they run on every request.
|
|
83
|
+
|
|
84
|
+
_SECRET_PATTERNS = [
|
|
85
|
+
re.compile(r"-----BEGIN [A-Z ]{0,40}PRIVATE KEY-----.*?(?:-----END [A-Z ]{0,40}PRIVATE KEY-----|\Z)", re.S),
|
|
86
|
+
re.compile(r"\b(?:sk|pk|rk)-[A-Za-z0-9_-]{16,}"), # OpenAI/Anthropic/Stripe-style
|
|
87
|
+
re.compile(r"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{30,}|\bgithub_pat_[A-Za-z0-9_]{30,}"),
|
|
88
|
+
re.compile(r"\bxox[abposr]-[A-Za-z0-9-]{10,}"), # Slack
|
|
89
|
+
re.compile(r"\bAKIA[0-9A-Z]{16}\b"), # AWS access key id
|
|
90
|
+
re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b"), # Google API key
|
|
91
|
+
re.compile(r"\bapikey_[0-9a-f]{16,}_[0-9a-f]{16,}\b"), # TypeSafe
|
|
92
|
+
re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"), # JWT
|
|
93
|
+
]
|
|
94
|
+
_ASSIGNMENT = re.compile(
|
|
95
|
+
r"(?i)\b([A-Z0-9_]{0,40}(?:api[_-]?key|secret|token|passw(?:or)?d|pwd|auth|credential)[A-Z0-9_]{0,40})"
|
|
96
|
+
r"([ \t]*[:=][ \t]*)([\"']?)[^\s\"']{4,}")
|
|
97
|
+
_URL_CREDENTIALS = re.compile(r"(\b[a-z][a-z0-9+.-]{1,20}://)[^/\s:@]{1,100}:[^/\s@]{1,200}@")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def redact(text: str) -> str:
|
|
101
|
+
"""Mask values that look like credentials; everything else is kept."""
|
|
102
|
+
if not isinstance(text, str) or not text:
|
|
103
|
+
return text
|
|
104
|
+
for pattern in _SECRET_PATTERNS:
|
|
105
|
+
text = pattern.sub("[redacted]", text)
|
|
106
|
+
text = _ASSIGNMENT.sub(lambda m: f"{m.group(1)}{m.group(2)}{m.group(3)}[redacted]", text)
|
|
107
|
+
return _URL_CREDENTIALS.sub(r"\1[redacted]@", text)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def excerpt(text: str, room: int = OUTPUT_EXCERPT_CHARS) -> str:
|
|
111
|
+
"""Head and tail of ``text`` within ``room`` characters (as calibrated)."""
|
|
112
|
+
if len(text) <= room:
|
|
113
|
+
return text
|
|
114
|
+
return (text[:room * 2 // 3] + f"\n…[{len(text) - room} chars omitted]…\n"
|
|
115
|
+
+ text[-(room // 3):])
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# -- rules ----------------------------------------------------------------------------------
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _profile(backend: Any) -> str:
|
|
122
|
+
name = getattr(backend, "name", "")
|
|
123
|
+
return name if name in PROMPT_RULES else DEFAULT_PROFILE
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _signals(result: Any, rule: Rule) -> Dict[str, float]:
|
|
127
|
+
answers = result["answers"]
|
|
128
|
+
raw = {}
|
|
129
|
+
for question, _, _ in rule:
|
|
130
|
+
value = float(answers[question]["noul"])
|
|
131
|
+
if not 0.0 <= value <= 1.0:
|
|
132
|
+
raise ValueError(f"{question}: not a probability")
|
|
133
|
+
raw[question] = value
|
|
134
|
+
return raw
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _oriented(raw: Dict[str, float], rule: Rule) -> List[float]:
|
|
138
|
+
return [1.0 - raw[q] if inverted else raw[q] for q, inverted, _ in rule]
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _override(rule: Rule, value: Any) -> Rule:
|
|
142
|
+
"""The user's threshold replaces the primary condition's, when set."""
|
|
143
|
+
try:
|
|
144
|
+
threshold = float(value)
|
|
145
|
+
except (TypeError, ValueError):
|
|
146
|
+
return rule
|
|
147
|
+
if not 0.0 <= threshold <= 1.0:
|
|
148
|
+
return rule
|
|
149
|
+
question, inverted, _ = rule[0]
|
|
150
|
+
return [(question, inverted, threshold)] + rule[1:]
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def canned_answers(questions: Dict[str, Any], simple: bool = True,
|
|
154
|
+
disposable: bool = True) -> Dict[str, Any]:
|
|
155
|
+
"""What a perfectly sure model would answer: for self-tests and stubs."""
|
|
156
|
+
yes = {"quick": simple, "multi_step": not simple, "multi_file": not simple,
|
|
157
|
+
"routine": disposable, "needed": not disposable, "depends": not disposable}
|
|
158
|
+
return {"answers": {q: {"type": "noul", "noul": (1.0 if yes[q] else 0.0) if q in yes else 0.5}
|
|
159
|
+
for q in questions}}
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# -- verdicts -------------------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def classify_prompt(
|
|
166
|
+
prompt: str,
|
|
167
|
+
backend: Optional[DecisionBackend] = None,
|
|
168
|
+
config: Optional[Dict[str, Any]] = None,
|
|
169
|
+
) -> Optional[Dict[str, Any]]:
|
|
170
|
+
"""Classify a prompt as simple or complex.
|
|
171
|
+
|
|
172
|
+
Returns ``{"label": "simple"|"complex", "confidence": p, "signals": {...}}``
|
|
173
|
+
(``confidence`` is the primary signal oriented toward the label) or None.
|
|
174
|
+
"""
|
|
175
|
+
try:
|
|
176
|
+
cfg = config or load_config()
|
|
177
|
+
be = backend or get_backend(cfg)
|
|
178
|
+
rule = _override(PROMPT_RULES[_profile(be)],
|
|
179
|
+
(cfg.get("thresholds") or {}).get("prompt_simple_confidence"))
|
|
180
|
+
questions = {q: PROMPT_QUESTIONS[q] for q, _, _ in rule}
|
|
181
|
+
raw = _signals(be.predict({"prompt": redact(str(prompt))[:PROMPT_CHARS]}, questions), rule)
|
|
182
|
+
oriented = _oriented(raw, rule)
|
|
183
|
+
metrics.METRICS.record("verdict_prompt")
|
|
184
|
+
simple = all(p >= threshold for p, (_, _, threshold) in zip(oriented, rule))
|
|
185
|
+
primary = oriented[0]
|
|
186
|
+
return {"label": "simple" if simple else "complex",
|
|
187
|
+
"confidence": round(primary if simple else 1.0 - primary, 4),
|
|
188
|
+
"signals": {q: round(v, 4) for q, v in raw.items()}}
|
|
189
|
+
except Exception:
|
|
190
|
+
return None
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def judge_output(
|
|
194
|
+
output: str,
|
|
195
|
+
context: str = "",
|
|
196
|
+
backend: Optional[DecisionBackend] = None,
|
|
197
|
+
config: Optional[Dict[str, Any]] = None,
|
|
198
|
+
task: str = "",
|
|
199
|
+
) -> Optional[Dict[str, Any]]:
|
|
200
|
+
"""Judge whether a tool output is still needed or can be trimmed.
|
|
201
|
+
|
|
202
|
+
``needed`` is False only when every condition of the backend's rule says
|
|
203
|
+
so. Without the user's request there is no evidence to judge by (and no
|
|
204
|
+
calibration), so the output is kept. Outputs shorter than
|
|
205
|
+
``min_output_chars`` are cheap to keep and are not judged.
|
|
206
|
+
Returns ``{"needed": bool, "p_needed": float|None, "signals": {...}}`` or None.
|
|
207
|
+
"""
|
|
208
|
+
try:
|
|
209
|
+
cfg = config or load_config()
|
|
210
|
+
min_chars = int(cfg["thresholds"]["min_output_chars"])
|
|
211
|
+
if len(output) < min_chars:
|
|
212
|
+
return {"needed": True, "p_needed": 1.0}
|
|
213
|
+
if not task or not task.strip():
|
|
214
|
+
return {"needed": True, "p_needed": None, "reason": "no request to judge against"}
|
|
215
|
+
be = backend or get_backend(cfg)
|
|
216
|
+
profile = _profile(be)
|
|
217
|
+
override = (cfg.get("thresholds") or {}).get("output_needed_threshold")
|
|
218
|
+
if profile not in TRIMS_BY_DEFAULT and override is None:
|
|
219
|
+
return {"needed": True, "p_needed": None,
|
|
220
|
+
"reason": f"{profile} does not trim by default (docs/calibration.md)"}
|
|
221
|
+
rule = _override(OUTPUT_RULES[profile], override)
|
|
222
|
+
questions = {q: OUTPUT_QUESTIONS[q] for q, _, _ in rule}
|
|
223
|
+
state = {"task": redact(task.strip())[:TASK_CHARS],
|
|
224
|
+
"tool_call": redact(str(context))[:TOOL_CALL_CHARS],
|
|
225
|
+
"output": redact(excerpt(output))}
|
|
226
|
+
raw = _signals(be.predict(state, questions), rule)
|
|
227
|
+
oriented = _oriented(raw, rule)
|
|
228
|
+
metrics.METRICS.record("verdict_output")
|
|
229
|
+
disposable = all(p < threshold for p, (_, _, threshold) in zip(oriented, rule))
|
|
230
|
+
return {"needed": not disposable, "p_needed": round(oriented[0], 4),
|
|
231
|
+
"signals": {q: round(v, 4) for q, v in raw.items()}}
|
|
232
|
+
except Exception:
|
|
233
|
+
return None
|