subcortex 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- subcortex/__init__.py +3 -0
- subcortex/__main__.py +3 -0
- subcortex/adapters/__init__.py +48 -0
- subcortex/adapters/base.py +230 -0
- subcortex/adapters/claude_family.py +133 -0
- subcortex/adapters/codex.py +87 -0
- subcortex/adapters/copilot.py +60 -0
- subcortex/adapters/cursor.py +36 -0
- subcortex/adapters/docker_agent.py +115 -0
- subcortex/adapters/gemini_family.py +60 -0
- subcortex/adapters/grok.py +98 -0
- subcortex/adapters/kimi_code.py +138 -0
- subcortex/adapters/letta_vibe.py +96 -0
- subcortex/adapters/openhands.py +153 -0
- subcortex/auth.py +59 -0
- subcortex/backends/__init__.py +23 -0
- subcortex/backends/base.py +22 -0
- subcortex/backends/jev.py +460 -0
- subcortex/backends/laya.py +149 -0
- subcortex/cli.py +809 -0
- subcortex/client.py +77 -0
- subcortex/config.py +263 -0
- subcortex/daemon.py +502 -0
- subcortex/evalset.py +241 -0
- subcortex/hook.py +254 -0
- subcortex/installers/__init__.py +62 -0
- subcortex/installers/amp.py +39 -0
- subcortex/installers/base.py +874 -0
- subcortex/installers/claude_family.py +229 -0
- subcortex/installers/codex.py +110 -0
- subcortex/installers/copilot.py +65 -0
- subcortex/installers/crush.py +36 -0
- subcortex/installers/cursor.py +79 -0
- subcortex/installers/gemini_family.py +83 -0
- subcortex/installers/goose.py +186 -0
- subcortex/installers/kimi_code.py +71 -0
- subcortex/installers/mcp_only.py +111 -0
- subcortex/installers/more_hooks.py +184 -0
- subcortex/installers/opencode.py +66 -0
- subcortex/installers/openhands.py +84 -0
- subcortex/installers/pi_cline.py +53 -0
- subcortex/ledger.py +92 -0
- subcortex/localhttp.py +59 -0
- subcortex/mcp_server.py +187 -0
- subcortex/metrics.py +56 -0
- subcortex/plugins/amp/subcortex.ts +258 -0
- subcortex/plugins/cline/subcortex.ts +340 -0
- subcortex/plugins/opencode/subcortex.ts +265 -0
- subcortex/plugins/pi/subcortex.ts +292 -0
- subcortex/policy.py +341 -0
- subcortex/presets.py +163 -0
- subcortex/provision.py +188 -0
- subcortex/service.py +149 -0
- subcortex/state.py +137 -0
- subcortex/transcript.py +211 -0
- subcortex/tuis.py +51 -0
- subcortex/ui.py +319 -0
- subcortex/verdicts.py +233 -0
- subcortex/wizard.py +474 -0
- subcortex-0.3.0.dist-info/METADATA +287 -0
- subcortex-0.3.0.dist-info/RECORD +64 -0
- subcortex-0.3.0.dist-info/WHEEL +5 -0
- subcortex-0.3.0.dist-info/entry_points.txt +3 -0
- subcortex-0.3.0.dist-info/top_level.txt +1 -0
subcortex/provision.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Backend environment provisioning for setup.
|
|
2
|
+
|
|
3
|
+
The laya backend needs a heavy package (``laya-mlx`` on Apple Silicon, ``laya``
|
|
4
|
+
elsewhere) next to subcortex itself. By default setup gives the daemon a
|
|
5
|
+
dedicated virtualenv at ``~/.local/share/subcortex/venv`` holding both, so the
|
|
6
|
+
user's own Python environments are never modified. Everything here reports
|
|
7
|
+
problems as return values; nothing raises on expected failures.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import platform
|
|
15
|
+
import shutil
|
|
16
|
+
import subprocess
|
|
17
|
+
import sys
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Callable, Dict, List, Optional, Tuple
|
|
20
|
+
|
|
21
|
+
from . import __version__
|
|
22
|
+
from .config import venv_dir, venv_python
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def apple_silicon() -> bool:
|
|
27
|
+
return sys.platform == "darwin" and platform.machine() == "arm64"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def laya_package() -> str:
|
|
31
|
+
return "laya-mlx" if apple_silicon() else "laya"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def laya_module() -> str:
|
|
35
|
+
return "laya_mlx" if apple_silicon() else "laya"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def daemon_python() -> str:
|
|
39
|
+
"""The interpreter the daemon runs under: ``daemon_python`` from config (set by
|
|
40
|
+
setup), else the dedicated venv if present, else this interpreter."""
|
|
41
|
+
from .config import load_config
|
|
42
|
+
|
|
43
|
+
chosen = str(load_config().get("daemon_python") or "").strip()
|
|
44
|
+
if chosen and os.access(chosen, os.X_OK):
|
|
45
|
+
return chosen
|
|
46
|
+
return str(venv_python()) if venv_python().exists() else sys.executable
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def daemon_argv(python: Optional[str] = None) -> List[str]:
|
|
50
|
+
"""How to start the daemon.
|
|
51
|
+
|
|
52
|
+
- Isolated (``-I``): PYTHONPATH and friends are ignored and the working
|
|
53
|
+
directory never lands on sys.path, so a project's own ``json.py`` can't
|
|
54
|
+
break the daemon or run inside it. Callers also start it from the data
|
|
55
|
+
dir, never the user's project.
|
|
56
|
+
- It runs *this* subcortex, whatever the interpreter: the backend venv may
|
|
57
|
+
hold an older copy of the package (a daemon of another version would
|
|
58
|
+
disagree with the hooks). The package is loaded from its own directory,
|
|
59
|
+
which then leaves sys.path again so the venv still provides the model.
|
|
60
|
+
"""
|
|
61
|
+
python = python or daemon_python()
|
|
62
|
+
root = str(Path(__file__).resolve().parent.parent)
|
|
63
|
+
boot = (f"import sys; sys.path.insert(0, {root!r}); import subcortex; sys.path.remove({root!r}); "
|
|
64
|
+
"from subcortex.cli import main; sys.exit(main(['serve', '--foreground']))")
|
|
65
|
+
return [python, "-I", "-c", boot]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def source_checkout() -> Optional[Path]:
|
|
69
|
+
"""Repo root when subcortex runs from a source checkout (dev / editable install)."""
|
|
70
|
+
root = Path(__file__).resolve().parents[2]
|
|
71
|
+
pyproject = root / "pyproject.toml"
|
|
72
|
+
try:
|
|
73
|
+
if pyproject.is_file() and 'name = "subcortex"' in pyproject.read_text():
|
|
74
|
+
return root
|
|
75
|
+
except OSError:
|
|
76
|
+
pass
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def subcortex_requirement() -> List[str]:
|
|
81
|
+
"""pip arguments that install *this* subcortex into another environment:
|
|
82
|
+
the same source checkout, git commit or archive it was installed from
|
|
83
|
+
(PEP 610 ``direct_url.json``), else this version from PyPI."""
|
|
84
|
+
root = source_checkout()
|
|
85
|
+
if root:
|
|
86
|
+
return ["-e", str(root)]
|
|
87
|
+
try:
|
|
88
|
+
from importlib.metadata import distribution
|
|
89
|
+
|
|
90
|
+
direct = json.loads(distribution("subcortex").read_text("direct_url.json") or "null")
|
|
91
|
+
except Exception:
|
|
92
|
+
direct = None
|
|
93
|
+
if isinstance(direct, dict) and isinstance(direct.get("url"), str):
|
|
94
|
+
url = direct["url"]
|
|
95
|
+
vcs = direct.get("vcs_info") or {}
|
|
96
|
+
if vcs.get("vcs") == "git":
|
|
97
|
+
return [f"git+{url}@{vcs.get('commit_id') or vcs.get('requested_revision') or 'HEAD'}"]
|
|
98
|
+
if (direct.get("dir_info") or {}).get("editable") and url.startswith("file://"):
|
|
99
|
+
return ["-e", url[len("file://"):]]
|
|
100
|
+
return [url]
|
|
101
|
+
return [f"subcortex=={__version__}"]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def probe(python: str) -> Dict[str, Optional[str]]:
|
|
105
|
+
"""What ``python`` can import: {"python": version, "subcortex": version|None, "<laya module>": "yes"|None}."""
|
|
106
|
+
code = (
|
|
107
|
+
"import importlib.util, json, sys\n"
|
|
108
|
+
"out = {'python': sys.version.split()[0]}\n"
|
|
109
|
+
"try:\n"
|
|
110
|
+
" import subcortex; out['subcortex'] = subcortex.__version__\n"
|
|
111
|
+
"except Exception:\n"
|
|
112
|
+
" out['subcortex'] = None\n"
|
|
113
|
+
f"out['{laya_module()}'] = 'yes' if importlib.util.find_spec('{laya_module()}') else None\n"
|
|
114
|
+
"print(json.dumps(out))\n"
|
|
115
|
+
)
|
|
116
|
+
env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"}
|
|
117
|
+
try:
|
|
118
|
+
proc = subprocess.run([python, "-c", code], capture_output=True, text=True, timeout=60, env=env)
|
|
119
|
+
return json.loads(proc.stdout.strip().splitlines()[-1])
|
|
120
|
+
except Exception:
|
|
121
|
+
return {"python": None, "subcortex": None, laya_module(): None}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def backend_ready(python: str) -> Tuple[bool, str]:
|
|
125
|
+
info = probe(python)
|
|
126
|
+
if not info.get("python"):
|
|
127
|
+
return False, f"{python} does not run"
|
|
128
|
+
if not info.get(laya_module()):
|
|
129
|
+
return False, f"{laya_package()} is not installed for {python}"
|
|
130
|
+
if info.get("subcortex") != __version__:
|
|
131
|
+
found = info.get("subcortex") or "not installed"
|
|
132
|
+
return False, f"subcortex there is {found}, this is {__version__}"
|
|
133
|
+
return True, f"{laya_package()} and subcortex {__version__} ready in {python}"
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def venv_is_foreign() -> Optional[str]:
|
|
137
|
+
"""If the venv path is a symlink (e.g. into another project's env), its target."""
|
|
138
|
+
if venv_dir().is_symlink():
|
|
139
|
+
return os.readlink(venv_dir())
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def create_venv(replace: bool = False) -> Tuple[bool, str]:
|
|
144
|
+
"""Create the dedicated venv. ``replace`` drops a symlink/broken env first —
|
|
145
|
+
a symlink is only unlinked, never followed."""
|
|
146
|
+
try:
|
|
147
|
+
if venv_dir().is_symlink():
|
|
148
|
+
if not replace:
|
|
149
|
+
return False, f"{venv_dir()} is a symlink to {os.readlink(venv_dir())}"
|
|
150
|
+
venv_dir().unlink()
|
|
151
|
+
elif venv_dir().exists() and replace:
|
|
152
|
+
shutil.rmtree(venv_dir())
|
|
153
|
+
if not venv_python().exists():
|
|
154
|
+
venv_dir().parent.mkdir(parents=True, exist_ok=True)
|
|
155
|
+
subprocess.run([sys.executable, "-m", "venv", str(venv_dir())], check=True,
|
|
156
|
+
capture_output=True, text=True, timeout=300)
|
|
157
|
+
return True, str(venv_dir())
|
|
158
|
+
except subprocess.CalledProcessError as exc:
|
|
159
|
+
return False, (exc.stderr or exc.stdout or str(exc)).strip()[-500:]
|
|
160
|
+
except Exception as exc:
|
|
161
|
+
return False, str(exc)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def pip_install(python: str, args: List[str],
|
|
165
|
+
on_line: Optional[Callable[[str], None]] = None) -> Tuple[bool, str]:
|
|
166
|
+
"""``python -m pip install <args>``; returns (ok, last lines of output)."""
|
|
167
|
+
env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"}
|
|
168
|
+
cmd = [python, "-m", "pip", "install", "--disable-pip-version-check", *args]
|
|
169
|
+
has_pip = subprocess.run([python, "-m", "pip", "--version"], capture_output=True, env=env).returncode == 0
|
|
170
|
+
if not has_pip and shutil.which("uv"): # uv-managed environments ship without pip
|
|
171
|
+
cmd = ["uv", "pip", "install", "--python", python, *args]
|
|
172
|
+
try:
|
|
173
|
+
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
174
|
+
text=True, env=env)
|
|
175
|
+
except OSError as exc:
|
|
176
|
+
return False, str(exc)
|
|
177
|
+
tail: List[str] = []
|
|
178
|
+
assert proc.stdout is not None
|
|
179
|
+
for line in proc.stdout:
|
|
180
|
+
tail = (tail + [line.rstrip()])[-15:]
|
|
181
|
+
if on_line:
|
|
182
|
+
on_line(line.rstrip())
|
|
183
|
+
return proc.wait() == 0, "\n".join(tail)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def install_laya(python: str, on_line: Optional[Callable[[str], None]] = None) -> Tuple[bool, str]:
|
|
187
|
+
"""Install the laya package and this subcortex into ``python``'s environment."""
|
|
188
|
+
return pip_install(python, [laya_package(), *subcortex_requirement()], on_line)
|
subcortex/service.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Start the daemon at login: a launchd agent (macOS) or a systemd user unit (Linux).
|
|
2
|
+
|
|
3
|
+
The service runs ``<daemon python> -m subcortex serve --foreground`` and is
|
|
4
|
+
restarted only if it crashes (a second instance exits 0 when the daemon is
|
|
5
|
+
already running, so there is no restart loop). All system commands go through
|
|
6
|
+
``runner`` so tests never touch launchctl/systemctl.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Callable, Dict, List, Optional, Sequence
|
|
16
|
+
from xml.sax.saxutils import escape
|
|
17
|
+
|
|
18
|
+
from .config import data_dir, log_path
|
|
19
|
+
from .provision import daemon_argv, daemon_python
|
|
20
|
+
|
|
21
|
+
LABEL = "ai.subcortex.daemon"
|
|
22
|
+
Runner = Callable[[Sequence[str]], subprocess.CompletedProcess]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _run(argv: Sequence[str]) -> subprocess.CompletedProcess:
|
|
26
|
+
return subprocess.run(list(argv), capture_output=True, text=True, timeout=30)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def platform_kind() -> Optional[str]:
|
|
30
|
+
if sys.platform == "darwin":
|
|
31
|
+
return "launchd"
|
|
32
|
+
if sys.platform.startswith("linux"):
|
|
33
|
+
return "systemd"
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def unit_path() -> Optional[Path]:
|
|
38
|
+
kind = platform_kind()
|
|
39
|
+
if kind == "launchd":
|
|
40
|
+
return Path.home() / "Library" / "LaunchAgents" / f"{LABEL}.plist"
|
|
41
|
+
if kind == "systemd":
|
|
42
|
+
xdg = os.environ.get("XDG_CONFIG_HOME", "").strip()
|
|
43
|
+
return (Path(xdg) if xdg else Path.home() / ".config") / "systemd" / "user" / "subcortex.service"
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _environment() -> Dict[str, str]:
|
|
48
|
+
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin")}
|
|
49
|
+
for key in ("SUBCORTEX_CONFIG", "SUBCORTEX_DATA_DIR"):
|
|
50
|
+
if os.environ.get(key):
|
|
51
|
+
env[key] = os.environ[key]
|
|
52
|
+
return env
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def render(python: Optional[str] = None) -> str:
|
|
56
|
+
argv = daemon_argv(python or daemon_python()) # isolated; see provision.daemon_argv
|
|
57
|
+
workdir = str(data_dir())
|
|
58
|
+
env = _environment()
|
|
59
|
+
if platform_kind() == "launchd":
|
|
60
|
+
args = "".join(f"\n <string>{escape(a)}</string>" for a in argv)
|
|
61
|
+
envs = "".join(f"\n <key>{escape(k)}</key><string>{escape(v)}</string>" for k, v in env.items())
|
|
62
|
+
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
|
63
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
64
|
+
<plist version="1.0">
|
|
65
|
+
<dict>
|
|
66
|
+
<key>Label</key><string>{LABEL}</string>
|
|
67
|
+
<key>ProgramArguments</key>
|
|
68
|
+
<array>{args}
|
|
69
|
+
</array>
|
|
70
|
+
<key>EnvironmentVariables</key>
|
|
71
|
+
<dict>{envs}
|
|
72
|
+
</dict>
|
|
73
|
+
<key>WorkingDirectory</key><string>{escape(workdir)}</string>
|
|
74
|
+
<key>RunAtLoad</key><true/>
|
|
75
|
+
<key>KeepAlive</key>
|
|
76
|
+
<dict><key>SuccessfulExit</key><false/></dict>
|
|
77
|
+
<key>ThrottleInterval</key><integer>30</integer>
|
|
78
|
+
<key>StandardOutPath</key><string>{escape(str(log_path()))}</string>
|
|
79
|
+
<key>StandardErrorPath</key><string>{escape(str(log_path()))}</string>
|
|
80
|
+
</dict>
|
|
81
|
+
</plist>
|
|
82
|
+
"""
|
|
83
|
+
exec_start = " ".join(_systemd_quote(a) for a in argv)
|
|
84
|
+
env_lines = "\n".join(f"Environment={_systemd_quote(f'{k}={v}')}" for k, v in env.items())
|
|
85
|
+
return f"""[Unit]
|
|
86
|
+
Description=subcortex decision daemon
|
|
87
|
+
|
|
88
|
+
[Service]
|
|
89
|
+
ExecStart={exec_start}
|
|
90
|
+
WorkingDirectory={_systemd_quote(workdir)}
|
|
91
|
+
{env_lines}
|
|
92
|
+
Restart=on-failure
|
|
93
|
+
RestartSec=30
|
|
94
|
+
|
|
95
|
+
[Install]
|
|
96
|
+
WantedBy=default.target
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _systemd_quote(value: str) -> str:
|
|
101
|
+
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def install(python: Optional[str] = None, runner: Runner = _run) -> List[str]:
|
|
105
|
+
"""Write and load the unit; returns human-readable notes (raises RuntimeError on failure)."""
|
|
106
|
+
path = unit_path()
|
|
107
|
+
if path is None:
|
|
108
|
+
raise RuntimeError(f"no login service support for {sys.platform}; "
|
|
109
|
+
"hooks still start the daemon on demand")
|
|
110
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
path.write_text(render(python))
|
|
112
|
+
notes = [f"wrote {path}"]
|
|
113
|
+
if platform_kind() == "launchd":
|
|
114
|
+
domain = f"gui/{os.getuid()}"
|
|
115
|
+
runner(["launchctl", "bootout", f"{domain}/{LABEL}"]) # reload if already loaded
|
|
116
|
+
result = runner(["launchctl", "bootstrap", domain, str(path)])
|
|
117
|
+
if result.returncode != 0:
|
|
118
|
+
result = runner(["launchctl", "load", "-w", str(path)])
|
|
119
|
+
if result.returncode != 0:
|
|
120
|
+
raise RuntimeError(f"launchctl could not load {path}: {(result.stderr or '').strip()}")
|
|
121
|
+
notes.append("loaded with launchctl (starts at every login)")
|
|
122
|
+
else:
|
|
123
|
+
for argv in (["systemctl", "--user", "daemon-reload"],
|
|
124
|
+
["systemctl", "--user", "enable", "--now", "subcortex.service"]):
|
|
125
|
+
result = runner(argv)
|
|
126
|
+
if result.returncode != 0:
|
|
127
|
+
raise RuntimeError(f"{' '.join(argv)} failed: {(result.stderr or '').strip()}")
|
|
128
|
+
notes.append("enabled with systemctl --user (starts at every login)")
|
|
129
|
+
return notes
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def uninstall(runner: Runner = _run) -> List[str]:
|
|
133
|
+
path = unit_path()
|
|
134
|
+
if path is None or not path.exists():
|
|
135
|
+
return ["no login service installed"]
|
|
136
|
+
if platform_kind() == "launchd":
|
|
137
|
+
runner(["launchctl", "bootout", f"gui/{os.getuid()}/{LABEL}"])
|
|
138
|
+
else:
|
|
139
|
+
runner(["systemctl", "--user", "disable", "--now", "subcortex.service"])
|
|
140
|
+
path.unlink()
|
|
141
|
+
if platform_kind() == "systemd":
|
|
142
|
+
runner(["systemctl", "--user", "daemon-reload"])
|
|
143
|
+
return [f"removed {path}"]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def status() -> Dict[str, object]:
|
|
147
|
+
path = unit_path()
|
|
148
|
+
return {"supported": path is not None, "installed": bool(path and path.exists()),
|
|
149
|
+
"path": str(path) if path else None}
|
subcortex/state.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Per-session state files, shared by hook processes and the daemon.
|
|
2
|
+
|
|
3
|
+
Many sessions of many TUIs run at once against one data dir, and a single
|
|
4
|
+
session's hooks can overlap (e.g. a restore on SessionStart and on the next
|
|
5
|
+
prompt). So every file here is:
|
|
6
|
+
|
|
7
|
+
- keyed by (TUI, session id): collision-free across TUIs and across ids that
|
|
8
|
+
sanitize alike, and never able to escape its directory;
|
|
9
|
+
- written atomically (temp file + rename), so readers never see a torn file;
|
|
10
|
+
- private (dirs 0700, files 0600), since they hold prompts and conversation text;
|
|
11
|
+
- changed under a per-session lock (``locked``), so read-modify-write and
|
|
12
|
+
consume-once operations don't race across threads or processes.
|
|
13
|
+
|
|
14
|
+
Everything fails open: errors surface as None/False/TimeoutError for the
|
|
15
|
+
caller to swallow, never as a changed TUI outcome.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import contextlib
|
|
21
|
+
import fcntl
|
|
22
|
+
import hashlib
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import time
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any, Dict, Iterator, Optional
|
|
28
|
+
|
|
29
|
+
from .config import data_dir
|
|
30
|
+
|
|
31
|
+
LOCK_WAIT_S = 1.0 # a hook has a few seconds in total; give up well before that
|
|
32
|
+
_LOCK_STRIPES = 64 # fixed lock files, never deleted (deleting a lock file breaks flock)
|
|
33
|
+
_PRUNE_EVERY_S = 600
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def key(tui: Any, session_id: Any) -> Optional[str]:
|
|
37
|
+
"""Filename-safe, collision-free key for one TUI session (None without a session id)."""
|
|
38
|
+
if not isinstance(session_id, str) or not session_id.strip():
|
|
39
|
+
return None
|
|
40
|
+
sid = session_id.strip()
|
|
41
|
+
ns = tui.strip() if isinstance(tui, str) and tui.strip() else "_"
|
|
42
|
+
digest = hashlib.sha256(f"{ns}\0{sid}".encode("utf-8", "replace")).hexdigest()[:24]
|
|
43
|
+
readable = "".join(c if c.isalnum() or c in "-_" else "_" for c in f"{ns}-{sid}")[:64]
|
|
44
|
+
return f"{readable}-{digest}"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _private(directory: Path) -> Path:
|
|
48
|
+
"""Create ``directory`` owner-only. Only writers call this. The data dir
|
|
49
|
+
itself is created 0700 but never re-permissioned: SUBCORTEX_DATA_DIR may
|
|
50
|
+
point at a directory the user shares on purpose. Our own subdirectories
|
|
51
|
+
(and the 0600 files in them) carry the privacy."""
|
|
52
|
+
root = data_dir()
|
|
53
|
+
if not root.exists():
|
|
54
|
+
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
55
|
+
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
56
|
+
try:
|
|
57
|
+
if directory != root and directory.stat().st_mode & 0o077:
|
|
58
|
+
os.chmod(directory, 0o700) # mkdir's mode is masked by umask
|
|
59
|
+
except OSError:
|
|
60
|
+
pass
|
|
61
|
+
return directory
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def path(kind: str, tui: Any, session_id: Any) -> Optional[Path]:
|
|
65
|
+
"""Where this session's ``kind`` file lives. Creates nothing."""
|
|
66
|
+
k = key(tui, session_id)
|
|
67
|
+
return data_dir() / kind / f"{k}.json" if k else None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def write_json(target: Path, data: Dict[str, Any]) -> None:
|
|
71
|
+
_private(target.parent)
|
|
72
|
+
tmp = target.parent / f".tmp-{os.getpid()}-{os.urandom(6).hex()}.json"
|
|
73
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
74
|
+
try:
|
|
75
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
76
|
+
json.dump(data, fh, ensure_ascii=False)
|
|
77
|
+
os.replace(tmp, target)
|
|
78
|
+
except BaseException:
|
|
79
|
+
with contextlib.suppress(OSError):
|
|
80
|
+
os.unlink(tmp)
|
|
81
|
+
raise
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def read_json(target: Path) -> Optional[Dict[str, Any]]:
|
|
85
|
+
try:
|
|
86
|
+
data = json.loads(target.read_text(encoding="utf-8"))
|
|
87
|
+
except (OSError, ValueError):
|
|
88
|
+
return None
|
|
89
|
+
return data if isinstance(data, dict) else None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@contextlib.contextmanager
|
|
93
|
+
def locked(k: str, wait_s: Optional[float] = None) -> Iterator[None]:
|
|
94
|
+
"""Exclusive per-session lock across threads and processes; TimeoutError if busy."""
|
|
95
|
+
wait_s = LOCK_WAIT_S if wait_s is None else wait_s
|
|
96
|
+
stripe = int(hashlib.sha256(k.encode()).hexdigest()[:8], 16) % _LOCK_STRIPES
|
|
97
|
+
lock_path = _private(data_dir() / "locks") / f"{stripe:02d}.lock"
|
|
98
|
+
fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600)
|
|
99
|
+
try:
|
|
100
|
+
deadline = time.monotonic() + wait_s
|
|
101
|
+
while True:
|
|
102
|
+
try:
|
|
103
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
104
|
+
break
|
|
105
|
+
except BlockingIOError:
|
|
106
|
+
if time.monotonic() >= deadline:
|
|
107
|
+
raise TimeoutError(f"state lock busy: {k}") from None
|
|
108
|
+
time.sleep(0.01)
|
|
109
|
+
try:
|
|
110
|
+
yield
|
|
111
|
+
finally:
|
|
112
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
113
|
+
finally:
|
|
114
|
+
os.close(fd)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def prune(kind: str, max_age_s: float) -> None:
|
|
118
|
+
"""Delete this kind's files older than ``max_age_s``; runs at most every few minutes."""
|
|
119
|
+
try:
|
|
120
|
+
directory = data_dir() / kind
|
|
121
|
+
if not directory.is_dir():
|
|
122
|
+
return
|
|
123
|
+
stamp = directory / ".pruned"
|
|
124
|
+
now = time.time()
|
|
125
|
+
if stamp.exists() and now - stamp.stat().st_mtime < _PRUNE_EVERY_S:
|
|
126
|
+
return
|
|
127
|
+
stamp.touch()
|
|
128
|
+
for old in [*directory.glob("*.json"), *directory.glob("*.claim")]:
|
|
129
|
+
with contextlib.suppress(OSError):
|
|
130
|
+
if now - old.stat().st_mtime > max_age_s:
|
|
131
|
+
old.unlink()
|
|
132
|
+
for tmp in directory.glob(".tmp-*"): # left behind by a killed writer
|
|
133
|
+
with contextlib.suppress(OSError):
|
|
134
|
+
if now - tmp.stat().st_mtime > 60:
|
|
135
|
+
tmp.unlink()
|
|
136
|
+
except OSError:
|
|
137
|
+
pass
|
subcortex/transcript.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""Shape-tolerant transcript reading for compaction snapshots.
|
|
2
|
+
|
|
3
|
+
Every TUI records conversations differently (Claude Code JSONL entries with a
|
|
4
|
+
nested ``message``, Codex rollout lines wrapping a ``payload``, Gemini-style
|
|
5
|
+
``parts``, whole-file JSON with a ``messages`` array, ...). ``last_messages``
|
|
6
|
+
extracts the final few user/assistant texts from any of them and never raises.
|
|
7
|
+
Only the tail of large files is read.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Dict, Iterable, List, Optional
|
|
16
|
+
|
|
17
|
+
TAIL_BYTES = 1_000_000
|
|
18
|
+
WHOLE_FILE_MAX_BYTES = 20_000_000
|
|
19
|
+
|
|
20
|
+
_USER_ROLES = {"user", "human"}
|
|
21
|
+
_ASSISTANT_ROLES = {"assistant", "model", "gemini", "ai", "agent"}
|
|
22
|
+
_TEXT_BLOCK_TYPES = {"text", "input_text", "output_text"}
|
|
23
|
+
# Harness-injected pseudo-messages such as <environment_context>…
|
|
24
|
+
_INJECTED_RE = re.compile(r"<[a-z][\w-]*(?:\s[^>]*)?>", re.IGNORECASE) # matched on stripped text
|
|
25
|
+
_INJECTED_PREFIXES = ("# AGENTS.md instructions",) # Codex
|
|
26
|
+
# ...except wrappers around the user's own words (Cursor: <user_query>,
|
|
27
|
+
# Cline: <user_input mode="act">).
|
|
28
|
+
_USER_WRAPPERS = ("user_query", "user_message", "user_input")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _unwrap_user(text: str) -> Optional[str]:
|
|
32
|
+
"""The words inside <user_query>…</user_query> (attributes allowed), else
|
|
33
|
+
None. Plain string operations: a regex here backtracked cubically on long
|
|
34
|
+
whitespace runs."""
|
|
35
|
+
for tag in _USER_WRAPPERS:
|
|
36
|
+
opening, closing = f"<{tag}", f"</{tag}>"
|
|
37
|
+
if not (text.startswith(opening) and text.endswith(closing)):
|
|
38
|
+
continue
|
|
39
|
+
rest = text[len(opening):]
|
|
40
|
+
if not rest or rest[0] not in "> \t\n":
|
|
41
|
+
continue # a different tag that merely starts the same way
|
|
42
|
+
end = rest.find(">")
|
|
43
|
+
if end == -1 or end > len(rest) - len(closing):
|
|
44
|
+
continue
|
|
45
|
+
return rest[end + 1:len(rest) - len(closing)].strip()
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _role(value: Any) -> Optional[str]:
|
|
50
|
+
if not isinstance(value, str):
|
|
51
|
+
return None
|
|
52
|
+
lowered = value.strip().lower()
|
|
53
|
+
if lowered in _USER_ROLES:
|
|
54
|
+
return "user"
|
|
55
|
+
if lowered in _ASSISTANT_ROLES:
|
|
56
|
+
return "assistant"
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _text(content: Any) -> str:
|
|
61
|
+
if isinstance(content, str):
|
|
62
|
+
return content
|
|
63
|
+
if isinstance(content, list):
|
|
64
|
+
parts = []
|
|
65
|
+
for block in content:
|
|
66
|
+
if isinstance(block, str):
|
|
67
|
+
parts.append(block)
|
|
68
|
+
elif isinstance(block, dict) and isinstance(block.get("text"), str):
|
|
69
|
+
if block.get("type") in (None, *_TEXT_BLOCK_TYPES):
|
|
70
|
+
parts.append(block["text"])
|
|
71
|
+
return "\n".join(p for p in parts if p)
|
|
72
|
+
if isinstance(content, dict) and isinstance(content.get("text"), str):
|
|
73
|
+
return content["text"]
|
|
74
|
+
return ""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _display_text(entry: Dict[str, Any]) -> str:
|
|
78
|
+
"""What the user typed, without the hook context the TUI appended to the
|
|
79
|
+
model-bound message (Gemini CLI ``displayContent``, Qwen Code
|
|
80
|
+
``systemPayload.displayText``)."""
|
|
81
|
+
if "displayContent" in entry:
|
|
82
|
+
return _text(entry["displayContent"])
|
|
83
|
+
payload = entry.get("systemPayload")
|
|
84
|
+
if isinstance(payload, dict) and isinstance(payload.get("displayText"), str):
|
|
85
|
+
return payload["displayText"]
|
|
86
|
+
return ""
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _without_our_context(text: str) -> str:
|
|
90
|
+
"""Drop what subcortex itself injected (hint lines, restored-context blocks),
|
|
91
|
+
so a snapshot never carries — or compounds — our own output."""
|
|
92
|
+
if "[subcortex]" not in text:
|
|
93
|
+
return text
|
|
94
|
+
kept, skipping = [], False
|
|
95
|
+
for line in text.split("\n"):
|
|
96
|
+
if line.lstrip().startswith("[subcortex] Recent conversation from before context compaction"):
|
|
97
|
+
skipping = True # a restore block runs to the next blank line
|
|
98
|
+
continue
|
|
99
|
+
if skipping:
|
|
100
|
+
if line.strip():
|
|
101
|
+
continue
|
|
102
|
+
skipping = False
|
|
103
|
+
if line.lstrip().startswith("[subcortex]"):
|
|
104
|
+
continue
|
|
105
|
+
kept.append(line)
|
|
106
|
+
return "\n".join(kept)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def message_from_entry(entry: Any) -> Optional[Dict[str, str]]:
|
|
110
|
+
"""``{"role", "text"}`` for one transcript record, or None if it isn't a
|
|
111
|
+
user/assistant message with text."""
|
|
112
|
+
if not isinstance(entry, dict):
|
|
113
|
+
return None
|
|
114
|
+
if entry.get("isMeta") or entry.get("isCompactSummary") or entry.get("isSidechain") \
|
|
115
|
+
or entry.get("synthetic_reason"):
|
|
116
|
+
return None # reminders, compaction summaries, subagent turns, injected items
|
|
117
|
+
payload = entry.get("payload")
|
|
118
|
+
if isinstance(payload, dict):
|
|
119
|
+
return message_from_entry(payload)
|
|
120
|
+
kind = entry.get("type")
|
|
121
|
+
if isinstance(kind, str) and kind.endswith(".message"): # Copilot session events
|
|
122
|
+
data = entry.get("data") if isinstance(entry.get("data"), dict) else entry
|
|
123
|
+
role = _role(kind[: -len(".message")])
|
|
124
|
+
text = _text(data.get("content", data.get("text", data.get("message")))).strip()
|
|
125
|
+
return {"role": role, "text": text} if role and text else None
|
|
126
|
+
message = entry.get("message")
|
|
127
|
+
if isinstance(message, dict):
|
|
128
|
+
role = _role(message.get("role")) or _role(entry.get("type")) or _role(entry.get("role"))
|
|
129
|
+
text = _text(message.get("content", message.get("parts")))
|
|
130
|
+
else:
|
|
131
|
+
role = _role(entry.get("role")) or _role(entry.get("type")) or _role(entry.get("author"))
|
|
132
|
+
text = _text(entry.get("content", entry.get("parts", entry.get("text"))))
|
|
133
|
+
if not text and isinstance(message, str):
|
|
134
|
+
text = message
|
|
135
|
+
text = _without_our_context(_display_text(entry) or text).strip()
|
|
136
|
+
wrapped = _unwrap_user(text)
|
|
137
|
+
if wrapped is not None:
|
|
138
|
+
text = wrapped
|
|
139
|
+
elif _INJECTED_RE.match(text) or text.startswith(_INJECTED_PREFIXES):
|
|
140
|
+
return None
|
|
141
|
+
if role is None or not text:
|
|
142
|
+
return None
|
|
143
|
+
return {"role": role, "text": text}
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def read_tail(path: Path) -> str:
|
|
147
|
+
size = path.stat().st_size
|
|
148
|
+
with open(path, "rb") as fh:
|
|
149
|
+
if size > TAIL_BYTES:
|
|
150
|
+
fh.seek(size - TAIL_BYTES)
|
|
151
|
+
fh.readline() # drop the partial first line
|
|
152
|
+
return fh.read().decode("utf-8", errors="replace")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _unwrap(whole: Any) -> List[Any]:
|
|
156
|
+
if isinstance(whole, list):
|
|
157
|
+
return whole
|
|
158
|
+
if isinstance(whole, dict):
|
|
159
|
+
for key in ("messages", "history", "items", "conversation"):
|
|
160
|
+
if isinstance(whole.get(key), list):
|
|
161
|
+
return whole[key]
|
|
162
|
+
return [whole]
|
|
163
|
+
return []
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _entries(path: Path) -> Iterable[Any]:
|
|
167
|
+
"""JSONL records from the file's tail, or the message list of a whole-JSON file."""
|
|
168
|
+
raw = read_tail(path)
|
|
169
|
+
entries, bad = [], 0
|
|
170
|
+
for line in raw.splitlines():
|
|
171
|
+
line = line.strip()
|
|
172
|
+
if not line:
|
|
173
|
+
continue
|
|
174
|
+
try:
|
|
175
|
+
entries.append(json.loads(line))
|
|
176
|
+
except ValueError:
|
|
177
|
+
bad += 1
|
|
178
|
+
if entries and bad <= len(entries): # JSONL (a stray partial line is fine)
|
|
179
|
+
return _unwrap(entries[0]) if len(entries) == 1 else entries
|
|
180
|
+
if path.stat().st_size > WHOLE_FILE_MAX_BYTES: # pretty-printed and huge: give up
|
|
181
|
+
return []
|
|
182
|
+
try:
|
|
183
|
+
return _unwrap(json.loads(path.read_text(encoding="utf-8", errors="replace")))
|
|
184
|
+
except ValueError:
|
|
185
|
+
return []
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def messages_from_entries(entries: Iterable[Any], limit: int, max_chars: int) -> List[Dict[str, str]]:
|
|
189
|
+
out: List[Dict[str, str]] = []
|
|
190
|
+
for entry in entries:
|
|
191
|
+
msg = message_from_entry(entry)
|
|
192
|
+
if msg is None:
|
|
193
|
+
continue
|
|
194
|
+
msg["text"] = msg["text"][:max_chars]
|
|
195
|
+
if out and out[-1] == msg: # rollouts often log the same message twice
|
|
196
|
+
continue
|
|
197
|
+
out.append(msg)
|
|
198
|
+
return out[-limit:] if limit > 0 else []
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def last_messages(path: Any, limit: int = 5, max_chars: int = 500) -> List[Dict[str, str]]:
|
|
202
|
+
"""Last ``limit`` user/assistant messages from a transcript file; [] on any problem."""
|
|
203
|
+
try:
|
|
204
|
+
if not isinstance(path, str) or not path:
|
|
205
|
+
return []
|
|
206
|
+
p = Path(path).expanduser()
|
|
207
|
+
if not p.is_file():
|
|
208
|
+
return []
|
|
209
|
+
return messages_from_entries(_entries(p), limit, max_chars)
|
|
210
|
+
except Exception:
|
|
211
|
+
return []
|