code2okf 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code2okf/SPEC.md +1006 -0
- code2okf/__init__.py +8 -0
- code2okf/cli.py +234 -0
- code2okf/clis/inspectmd/pyproject.toml +40 -0
- code2okf/clis/inspectmd/src/inspectmd/__init__.py +8 -0
- code2okf/clis/inspectmd/src/inspectmd/__main__.py +5 -0
- code2okf/clis/inspectmd/src/inspectmd/cli.py +159 -0
- code2okf/clis/inspectmd/src/inspectmd/parse.py +212 -0
- code2okf/clis/inspectokf/pyproject.toml +40 -0
- code2okf/clis/inspectokf/src/inspectokf/__init__.py +8 -0
- code2okf/clis/inspectokf/src/inspectokf/__main__.py +5 -0
- code2okf/clis/inspectokf/src/inspectokf/cli.py +104 -0
- code2okf/clis/merkleokf/pyproject.toml +40 -0
- code2okf/clis/merkleokf/src/merkleokf/__init__.py +8 -0
- code2okf/clis/merkleokf/src/merkleokf/__main__.py +5 -0
- code2okf/clis/merkleokf/src/merkleokf/cli.py +121 -0
- code2okf/clis/merkleokf/src/merkleokf/merkle.py +145 -0
- code2okf/clis/sizeokf/pyproject.toml +40 -0
- code2okf/clis/sizeokf/src/sizeokf/__init__.py +8 -0
- code2okf/clis/sizeokf/src/sizeokf/__main__.py +5 -0
- code2okf/clis/sizeokf/src/sizeokf/cli.py +93 -0
- code2okf/clis/sizeokf/src/sizeokf/sizes.py +155 -0
- code2okf/compile.py +267 -0
- code2okf/events.py +86 -0
- code2okf/kit/README.md +128 -0
- code2okf/kit/files/home/.local/lib/code2okf/mount-state.sh +48 -0
- code2okf/kit/files/home/.pi/agent/AGENTS.md +185 -0
- code2okf/kit/files/home/.pi/agent/models.json +84 -0
- code2okf/kit/files/home/.pi/agent/settings.json +7 -0
- code2okf/kit/files/home/.pi/agent/skills/compile-okf/SKILL.md +142 -0
- code2okf/kit/files/home/.pi/agent/skills/compile-okf/scripts/check-okf.sh +155 -0
- code2okf/kit/files/home/.pi/agent/skills/compile-okf/scripts/frontmatter-guard.py +289 -0
- code2okf/kit/files/home/.pi/agent/skills/curate-okf/SKILL.md +68 -0
- code2okf/kit/files/home/.pi/agent/skills/inspect-md/SKILL.md +52 -0
- code2okf/kit/files/home/.pi/agent/skills/inspect-okf/SKILL.md +47 -0
- code2okf/kit/files/home/.pi/agent/skills/merkle-okf/SKILL.md +59 -0
- code2okf/kit/files/home/.pi/agent/skills/size-okf/SKILL.md +52 -0
- code2okf/kit/spec.yaml +312 -0
- code2okf/resources.py +74 -0
- code2okf/sandbox.py +266 -0
- code2okf/workbench.py +572 -0
- code2okf-0.1.0.dist-info/METADATA +391 -0
- code2okf-0.1.0.dist-info/RECORD +48 -0
- code2okf-0.1.0.dist-info/WHEEL +4 -0
- code2okf-0.1.0.dist-info/entry_points.txt +2 -0
- code2okf-0.1.0.dist-info/licenses/LICENSE +21 -0
- code2okf-0.1.0.dist-info/licenses/LICENSE-OKF-SPEC.txt +203 -0
- code2okf-0.1.0.dist-info/licenses/NOTICE-OKF-SPEC.md +37 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Measure Markdown content size, excluding YAML frontmatter.
|
|
2
|
+
|
|
3
|
+
Only ``*.md`` files count. A file's size is the number of whitespace-split words
|
|
4
|
+
after its leading frontmatter block is removed. A directory's size is the sum
|
|
5
|
+
over every Markdown file beneath it, recursively.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def strip_frontmatter(text: str) -> str:
|
|
16
|
+
"""Return ``text`` without its leading YAML frontmatter block.
|
|
17
|
+
|
|
18
|
+
The block must open on the very first line with exactly ``---`` and close on
|
|
19
|
+
a later line that is also exactly ``---``; the body is everything after the
|
|
20
|
+
closing line, so a blank line following it still counts. Text with no such
|
|
21
|
+
block — including an *unterminated* one — is returned unchanged, which is
|
|
22
|
+
the same rule ``inspectmd`` applies when mapping headings.
|
|
23
|
+
"""
|
|
24
|
+
if text.startswith(""):
|
|
25
|
+
text = text[1:]
|
|
26
|
+
|
|
27
|
+
if not text.startswith("---"):
|
|
28
|
+
return text
|
|
29
|
+
|
|
30
|
+
lines = text.splitlines(keepends=True)
|
|
31
|
+
if not lines or lines[0].strip() != "---":
|
|
32
|
+
return text
|
|
33
|
+
|
|
34
|
+
for i in range(1, len(lines)):
|
|
35
|
+
if lines[i].strip() == "---":
|
|
36
|
+
return "".join(lines[i + 1 :])
|
|
37
|
+
return text
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def count_words(text: str) -> int:
|
|
41
|
+
"""Count whitespace-separated tokens in ``text``."""
|
|
42
|
+
return len(text.split())
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class Entry:
|
|
47
|
+
"""One listed file or directory, with its content size."""
|
|
48
|
+
|
|
49
|
+
path: str
|
|
50
|
+
"""Display path, rooted at the walk target's name. Directories end in ``/``."""
|
|
51
|
+
|
|
52
|
+
is_dir: bool
|
|
53
|
+
words: int
|
|
54
|
+
"""Whitespace-split words of Markdown content, frontmatter excluded. Recursive for directories."""
|
|
55
|
+
|
|
56
|
+
files: int
|
|
57
|
+
"""Number of Markdown files counted. Always 1 for a file."""
|
|
58
|
+
|
|
59
|
+
depth: int
|
|
60
|
+
"""1 for entries directly inside the root; 0 for the root itself."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _measure_file(path: Path) -> int:
|
|
64
|
+
"""Words of content in one Markdown file, frontmatter excluded.
|
|
65
|
+
|
|
66
|
+
A file that cannot be read is reported on stderr and counted as zero rather
|
|
67
|
+
than aborting the walk — one unreadable page must not cost the rest.
|
|
68
|
+
"""
|
|
69
|
+
try:
|
|
70
|
+
text = path.read_text(encoding="utf-8")
|
|
71
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
72
|
+
print(f"sizeokf: skipping {path}: {exc}", file=sys.stderr)
|
|
73
|
+
return 0
|
|
74
|
+
return count_words(strip_frontmatter(text))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def collect(
|
|
78
|
+
root: Path, *, max_level: int | None = None, nolog: bool = False
|
|
79
|
+
) -> tuple[list[Entry], Entry]:
|
|
80
|
+
"""Walk ``root``, returning ``(listed_entries, root_total)``.
|
|
81
|
+
|
|
82
|
+
``listed_entries`` always includes ``root_total``. Every directory total is
|
|
83
|
+
recursive regardless of ``max_level``; the level only decides which non-root
|
|
84
|
+
entries get listed. ``max_level=0`` lists only the walk root; ``max_level=1``
|
|
85
|
+
lists the entries directly inside ``root``, matching ``inspectokf -L 1``.
|
|
86
|
+
|
|
87
|
+
When ``nolog`` is true, ``okf/log.md`` is omitted entirely (not listed and
|
|
88
|
+
not counted). Nested ``log.md`` files and ``log.md`` under other roots are
|
|
89
|
+
still measured.
|
|
90
|
+
"""
|
|
91
|
+
entries: list[Entry] = []
|
|
92
|
+
prefix = f"{root.name}/"
|
|
93
|
+
|
|
94
|
+
def walk(directory: Path, depth: int) -> tuple[int, int]:
|
|
95
|
+
"""Return ``(words, files)`` for ``directory``, recording listed entries."""
|
|
96
|
+
words = files = 0
|
|
97
|
+
try:
|
|
98
|
+
children = sorted(directory.iterdir(), key=lambda p: p.name)
|
|
99
|
+
except OSError as exc:
|
|
100
|
+
# Same skip policy as an unreadable file: warn and contribute zero.
|
|
101
|
+
print(f"sizeokf: skipping {directory}: {exc}", file=sys.stderr)
|
|
102
|
+
return 0, 0
|
|
103
|
+
|
|
104
|
+
for child in children:
|
|
105
|
+
try:
|
|
106
|
+
# is_dir() follows symlinks; skip them so cycles and links
|
|
107
|
+
# outside root cannot be scanned.
|
|
108
|
+
if child.is_symlink():
|
|
109
|
+
continue
|
|
110
|
+
is_directory = child.is_dir()
|
|
111
|
+
except OSError as exc:
|
|
112
|
+
print(f"sizeokf: skipping {child}: {exc}", file=sys.stderr)
|
|
113
|
+
continue
|
|
114
|
+
|
|
115
|
+
if is_directory:
|
|
116
|
+
child_words, child_files = walk(child, depth + 1)
|
|
117
|
+
words += child_words
|
|
118
|
+
files += child_files
|
|
119
|
+
if max_level is None or depth <= max_level:
|
|
120
|
+
entries.append(
|
|
121
|
+
Entry(
|
|
122
|
+
path=f"{prefix}{child.relative_to(root)}/",
|
|
123
|
+
is_dir=True,
|
|
124
|
+
words=child_words,
|
|
125
|
+
files=child_files,
|
|
126
|
+
depth=depth,
|
|
127
|
+
)
|
|
128
|
+
)
|
|
129
|
+
elif child.suffix == ".md":
|
|
130
|
+
# Anchored to the walk root, not to any directory named "okf":
|
|
131
|
+
# a nested okf/ keeps its own log.md, as the docstring promises.
|
|
132
|
+
if nolog and child.name == "log.md" and root.name == "okf" and child.parent == root:
|
|
133
|
+
continue
|
|
134
|
+
child_words = _measure_file(child)
|
|
135
|
+
words += child_words
|
|
136
|
+
files += 1
|
|
137
|
+
if max_level is None or depth <= max_level:
|
|
138
|
+
entries.append(
|
|
139
|
+
Entry(
|
|
140
|
+
path=f"{prefix}{child.relative_to(root)}",
|
|
141
|
+
is_dir=False,
|
|
142
|
+
words=child_words,
|
|
143
|
+
files=1,
|
|
144
|
+
depth=depth,
|
|
145
|
+
)
|
|
146
|
+
)
|
|
147
|
+
return words, files
|
|
148
|
+
|
|
149
|
+
total_words, total_files = walk(root, 1)
|
|
150
|
+
total = Entry(path=prefix, is_dir=True, words=total_words, files=total_files, depth=0)
|
|
151
|
+
entries.append(total)
|
|
152
|
+
|
|
153
|
+
# Largest first; ties broken by path so repeated runs are byte-identical.
|
|
154
|
+
entries.sort(key=lambda e: (-e.words, e.path))
|
|
155
|
+
return entries, total
|
code2okf/compile.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
"""Document resolution and the Ralph loop.
|
|
2
|
+
|
|
3
|
+
The prompts, the loop, the cap and `merkleokf --nolog -L 0` as the convergence
|
|
4
|
+
check are carried over from the retired scripts/compile-okf.sh (see the git
|
|
5
|
+
history for the original, and .claude/plans/interface-plan.md for why each
|
|
6
|
+
rule is what it is).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
import sys
|
|
13
|
+
from collections.abc import Callable, Iterable
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from code2okf import events, sandbox, workbench
|
|
18
|
+
|
|
19
|
+
DEFAULT_MAX_ITERATIONS = 10
|
|
20
|
+
|
|
21
|
+
COMPILE_PROMPT = (
|
|
22
|
+
"Load the compile-okf skill: read ~/.pi/agent/skills/compile-okf/SKILL.md, "
|
|
23
|
+
"then follow it to compile {document} directly into the workspace root. "
|
|
24
|
+
"You are already in the OKF wiki; never create an okf/ child directory."
|
|
25
|
+
)
|
|
26
|
+
# Appended on Ralph loop iterations after the first, so Pi knows it may be
|
|
27
|
+
# resuming unfinished work rather than starting the document over.
|
|
28
|
+
CONTINUATION_PROMPT = (
|
|
29
|
+
"This is a follow-up pass on this document: the wiki may already hold "
|
|
30
|
+
"partial work from a previous pass. Compare the source against what is "
|
|
31
|
+
"on disk and continue at the first gap -- do not start over."
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
_HASH_RE = re.compile(r"^[0-9a-f]+$")
|
|
35
|
+
|
|
36
|
+
# How much of pi's raw output to fold into a CompileError on a non-zero exit.
|
|
37
|
+
# Bounded so one runaway session can't blow up an error message, generous
|
|
38
|
+
# enough that the actual cause (a traceback, sbx's own diagnostic text) is
|
|
39
|
+
# almost always still in view.
|
|
40
|
+
_DIAGNOSTIC_TAIL_LINES = 20
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _diagnostic_tail(raw_lines: list[str]) -> str:
|
|
44
|
+
"""The last few non-JSON lines, for folding into a failure message.
|
|
45
|
+
|
|
46
|
+
Pi's own protocol events are JSON objects and say nothing useful about
|
|
47
|
+
why a run died; what does is whatever arrived on stderr in plain text
|
|
48
|
+
(a traceback, an sbx diagnostic), merged into the same stream. Keeping
|
|
49
|
+
only those makes the message the cause rather than a wall of envelopes.
|
|
50
|
+
"""
|
|
51
|
+
meaningful = [line for line in raw_lines if line.strip() and not line.lstrip().startswith("{")]
|
|
52
|
+
return "\n".join(meaningful[-_DIAGNOSTIC_TAIL_LINES:])
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class UsageError(Exception):
|
|
56
|
+
"""A problem decided before any work starts -- exit 2."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class CompileError(Exception):
|
|
60
|
+
"""A run failure once compiling has started -- exit 1."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, message: str, *, document: str | None = None) -> None:
|
|
63
|
+
"""Fold `document` into the message itself, so naming it is not optional.
|
|
64
|
+
|
|
65
|
+
str(exc) is what every caller actually prints; a document recorded
|
|
66
|
+
only on a separate attribute is a document that silently never
|
|
67
|
+
reaches the user, which is exactly what the plan's "exit 1 naming
|
|
68
|
+
the document" contract requires.
|
|
69
|
+
"""
|
|
70
|
+
self.document = document
|
|
71
|
+
super().__init__(f"{message} ({document})" if document else message)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class Document:
|
|
76
|
+
"""One document to compile, resolved from a CLI positional argument."""
|
|
77
|
+
|
|
78
|
+
display: str
|
|
79
|
+
"""Printed verbatim in TSV rows -- the path as supplied, or "-" for stdin."""
|
|
80
|
+
|
|
81
|
+
basename: str
|
|
82
|
+
"""The name it is staged under in work/md."""
|
|
83
|
+
|
|
84
|
+
stdin: bool = False
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass(frozen=True)
|
|
88
|
+
class Row:
|
|
89
|
+
"""One TSV output row: path, iterations, hash-before, hash-after."""
|
|
90
|
+
|
|
91
|
+
path: str
|
|
92
|
+
iterations: int
|
|
93
|
+
hash_before: str
|
|
94
|
+
hash_after: str
|
|
95
|
+
|
|
96
|
+
def as_tsv(self) -> str:
|
|
97
|
+
"""Render as the one TSV line this document contributes to stdout."""
|
|
98
|
+
return f"{self.path}\t{self.iterations}\t{self.hash_before}\t{self.hash_after}"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _check_display_path(display: str) -> None:
|
|
102
|
+
"""A tab or newline in a TSV row's path column would corrupt the row."""
|
|
103
|
+
if "\t" in display or "\n" in display:
|
|
104
|
+
raise UsageError(f"path contains a tab or newline: {display!r}")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def resolve_documents(
|
|
108
|
+
paths: list[str], *, stdin_is_tty: Callable[[], bool] = lambda: sys.stdin.isatty()
|
|
109
|
+
) -> list[Document]:
|
|
110
|
+
"""Resolve CLI positionals into documents, per the input contract.
|
|
111
|
+
|
|
112
|
+
"-" or no arguments means stdin (refused on a TTY, via UsageError); a
|
|
113
|
+
directory means its *.md, sorted, non-recursive; duplicate arguments are
|
|
114
|
+
kept, in the order given, never de-duplicated; two different sources that
|
|
115
|
+
would stage under the same basename is a UsageError.
|
|
116
|
+
"""
|
|
117
|
+
raw_paths = paths or ["-"]
|
|
118
|
+
if raw_paths.count("-") > 1:
|
|
119
|
+
raise UsageError("only one '-' (stdin) may be given")
|
|
120
|
+
|
|
121
|
+
docs: list[Document] = []
|
|
122
|
+
for raw in raw_paths:
|
|
123
|
+
if raw == "-":
|
|
124
|
+
if stdin_is_tty():
|
|
125
|
+
raise UsageError("refusing to read from a terminal; pipe input or name a file")
|
|
126
|
+
docs.append(Document(display="-", basename="stdin.md", stdin=True))
|
|
127
|
+
continue
|
|
128
|
+
_check_display_path(raw)
|
|
129
|
+
path = Path(raw)
|
|
130
|
+
workbench.reject_if_unsafe(path, what="an input path")
|
|
131
|
+
if path.is_dir():
|
|
132
|
+
for md in sorted(path.glob("*.md")):
|
|
133
|
+
# Checked individually, same as the directory argument above:
|
|
134
|
+
# a symlinked or tab/newline-carrying *.md discovered inside
|
|
135
|
+
# it must be rejected up front (exit 2), not surface later as
|
|
136
|
+
# a staging failure or a corrupted TSV row.
|
|
137
|
+
workbench.reject_if_unsafe(md, what="an input path")
|
|
138
|
+
display = str(md)
|
|
139
|
+
_check_display_path(display)
|
|
140
|
+
docs.append(Document(display=display, basename=md.name))
|
|
141
|
+
elif path.is_file():
|
|
142
|
+
docs.append(Document(display=raw, basename=path.name))
|
|
143
|
+
else:
|
|
144
|
+
raise UsageError(f"not a file or directory: {raw}")
|
|
145
|
+
|
|
146
|
+
seen: dict[str, str] = {}
|
|
147
|
+
for doc in docs:
|
|
148
|
+
if doc.basename in seen and seen[doc.basename] != doc.display:
|
|
149
|
+
raise UsageError(
|
|
150
|
+
f"two different inputs would both stage as {doc.basename!r}: "
|
|
151
|
+
f"{seen[doc.basename]!r} and {doc.display!r}"
|
|
152
|
+
)
|
|
153
|
+
seen[doc.basename] = doc.display
|
|
154
|
+
|
|
155
|
+
if not docs:
|
|
156
|
+
raise UsageError("no documents to compile")
|
|
157
|
+
return docs
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def stage_items(documents: Iterable[Document]) -> list[tuple[str, Path | bytes]]:
|
|
161
|
+
"""(basename, source) pairs for workbench.stage_inputs, reading stdin now."""
|
|
162
|
+
items: list[tuple[str, Path | bytes]] = []
|
|
163
|
+
for doc in documents:
|
|
164
|
+
items.append((doc.basename, sys.stdin.buffer.read()) if doc.stdin else (doc.basename, Path(doc.display)))
|
|
165
|
+
return items
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _parse_hash(output: str) -> str:
|
|
169
|
+
lines = output.splitlines()
|
|
170
|
+
if len(lines) < 3 or not lines[2].split():
|
|
171
|
+
raise CompileError(f"merkleokf produced no root row: {output!r}")
|
|
172
|
+
digest = lines[2].split()[0]
|
|
173
|
+
if not _HASH_RE.match(digest):
|
|
174
|
+
raise CompileError(f"merkleokf produced a malformed hash: {digest!r}")
|
|
175
|
+
return digest
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def wiki_root_hash(name: str, work_okf: Path) -> str:
|
|
179
|
+
"""The wiki root's hash, validated -- a missing or malformed hash is never convergence."""
|
|
180
|
+
result = sandbox.exec_capture(name, ["merkleokf", "--nolog", "-L", "0", str(work_okf)])
|
|
181
|
+
if result.returncode != 0:
|
|
182
|
+
raise CompileError(f"merkleokf failed: {result.stderr.strip()}")
|
|
183
|
+
return _parse_hash(result.stdout)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def compile_document(
|
|
187
|
+
name: str,
|
|
188
|
+
doc: Document,
|
|
189
|
+
wb: workbench.Workbench,
|
|
190
|
+
output_dir: Path,
|
|
191
|
+
max_iterations: int = DEFAULT_MAX_ITERATIONS,
|
|
192
|
+
*,
|
|
193
|
+
on_progress: Callable[[str], None] | None = None,
|
|
194
|
+
on_event: Callable[[str], None] | None = None,
|
|
195
|
+
) -> Row:
|
|
196
|
+
"""Run the Ralph loop for one document.
|
|
197
|
+
|
|
198
|
+
Re-runs Pi on the same document until merkleokf --nolog -L 0 reports an
|
|
199
|
+
unchanged wiki root hash, capped at max_iterations. Mirrors work_okf out
|
|
200
|
+
to output_dir after every iteration, so an interruption leaves the last
|
|
201
|
+
completed pass on disk. A hash-stable first pass is convergence, not
|
|
202
|
+
failure -- iterations=1, equal hashes.
|
|
203
|
+
"""
|
|
204
|
+
document_path = wb.work_md / doc.basename
|
|
205
|
+
hash_before = wiki_root_hash(name, wb.work_okf)
|
|
206
|
+
had_markdown_before = workbench.has_markdown(wb.work_okf)
|
|
207
|
+
|
|
208
|
+
iteration = 0
|
|
209
|
+
current_hash = hash_before
|
|
210
|
+
while True:
|
|
211
|
+
iteration += 1
|
|
212
|
+
if iteration > max_iterations:
|
|
213
|
+
raise CompileError(f"hit {max_iterations} iterations without converging", document=doc.display)
|
|
214
|
+
if on_progress is not None:
|
|
215
|
+
on_progress(f"Compiling document {doc.display} (iteration {iteration})")
|
|
216
|
+
|
|
217
|
+
prompt = COMPILE_PROMPT.format(document=document_path)
|
|
218
|
+
if iteration > 1:
|
|
219
|
+
prompt = f"{prompt} {CONTINUATION_PROMPT}"
|
|
220
|
+
|
|
221
|
+
stream = sandbox.exec_stream(name, ["pi", "--mode", "json", prompt])
|
|
222
|
+
tool_calls = 0
|
|
223
|
+
raw_lines: list[str] = []
|
|
224
|
+
try:
|
|
225
|
+
for line, display, is_tool_call in events.process(stream):
|
|
226
|
+
raw_lines.append(line)
|
|
227
|
+
if is_tool_call:
|
|
228
|
+
tool_calls += 1
|
|
229
|
+
# events.process() has already decided what is worth showing:
|
|
230
|
+
# rendered tool calls and assistant prose, plus any non-JSON
|
|
231
|
+
# diagnostic. Protocol events we do not render come back as
|
|
232
|
+
# None and are dropped here.
|
|
233
|
+
if on_event is not None and display and display.strip():
|
|
234
|
+
on_event(display)
|
|
235
|
+
finally:
|
|
236
|
+
# Reached on Ctrl-C too, so an interrupted run does not leave the
|
|
237
|
+
# local `sbx exec` conduit behind. mirror_out() is below this
|
|
238
|
+
# point, which is what keeps a half-finished iteration from ever
|
|
239
|
+
# reaching -o DIR.
|
|
240
|
+
stream.close()
|
|
241
|
+
if stream.returncode != 0:
|
|
242
|
+
tail = _diagnostic_tail(raw_lines)
|
|
243
|
+
detail = f": {tail}" if tail else ""
|
|
244
|
+
raise CompileError(f"pi exited {stream.returncode}{detail}", document=doc.display)
|
|
245
|
+
if tool_calls == 0:
|
|
246
|
+
raise CompileError("pi session made no tool calls -- it did not follow the skill", document=doc.display)
|
|
247
|
+
|
|
248
|
+
try:
|
|
249
|
+
workbench.mirror_out(wb.work_okf, output_dir)
|
|
250
|
+
except workbench.WorkbenchError as exc:
|
|
251
|
+
# Re-raised as a CompileError, not left as a WorkbenchError: the
|
|
252
|
+
# caller's per-document loop only catches CompileError, and a
|
|
253
|
+
# failed mirror-out is a failed run for *this* document (exit 1),
|
|
254
|
+
# not a setup problem decided before any work started.
|
|
255
|
+
raise CompileError(str(exc), document=doc.display) from exc
|
|
256
|
+
|
|
257
|
+
next_hash = wiki_root_hash(name, wb.work_okf)
|
|
258
|
+
if on_progress is not None:
|
|
259
|
+
on_progress(f"{current_hash} -> {next_hash}")
|
|
260
|
+
if next_hash == current_hash:
|
|
261
|
+
break
|
|
262
|
+
current_hash = next_hash
|
|
263
|
+
|
|
264
|
+
if not had_markdown_before and not workbench.has_markdown(wb.work_okf):
|
|
265
|
+
raise CompileError("the wiki is still empty after compiling", document=doc.display)
|
|
266
|
+
|
|
267
|
+
return Row(path=doc.display, iterations=iteration, hash_before=hash_before, hash_after=current_hash)
|
code2okf/events.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Turn Pi's `--mode json` event stream into the host-side progress view.
|
|
2
|
+
|
|
3
|
+
Replaces the jq filter in the old shell driver
|
|
4
|
+
(the retired `scripts/compile-okf.sh`): the same three cases, the same 120-character
|
|
5
|
+
cut on tool calls.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Iterable, Iterator
|
|
12
|
+
|
|
13
|
+
DISPLAY_WIDTH = 120
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _load(line: str) -> dict | None:
|
|
17
|
+
try:
|
|
18
|
+
event = json.loads(line)
|
|
19
|
+
except (json.JSONDecodeError, ValueError):
|
|
20
|
+
return None
|
|
21
|
+
return event if isinstance(event, dict) else None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _translate_event(event: dict) -> str | None:
|
|
25
|
+
event_type = event.get("type")
|
|
26
|
+
if event_type == "tool_execution_start":
|
|
27
|
+
tool_name = event.get("toolName", "")
|
|
28
|
+
args = event.get("args", "")
|
|
29
|
+
return f"{tool_name} {args}"[:DISPLAY_WIDTH]
|
|
30
|
+
|
|
31
|
+
if event_type == "message_end":
|
|
32
|
+
message = event.get("message") or {}
|
|
33
|
+
if message.get("role") != "assistant":
|
|
34
|
+
return None
|
|
35
|
+
parts = []
|
|
36
|
+
for block in message.get("content") or []:
|
|
37
|
+
if block.get("type") == "thinking":
|
|
38
|
+
parts.append(f"[thinking]\n{block.get('thinking', '')}")
|
|
39
|
+
elif block.get("type") == "text":
|
|
40
|
+
parts.append(block.get("text", ""))
|
|
41
|
+
return "\n\n".join(parts) or None
|
|
42
|
+
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def translate(line: str) -> str | None:
|
|
47
|
+
"""One raw `pi --mode json` line -> the text worth showing, or None.
|
|
48
|
+
|
|
49
|
+
A tool call becomes "toolName args" (cut to DISPLAY_WIDTH); an assistant
|
|
50
|
+
`message_end` becomes its joined text/thinking parts; anything else,
|
|
51
|
+
including a line that is not JSON, produces nothing.
|
|
52
|
+
"""
|
|
53
|
+
event = _load(line)
|
|
54
|
+
return _translate_event(event) if event is not None else None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def is_tool_call(line: str) -> bool:
|
|
58
|
+
"""Whether a raw line is a `tool_execution_start` event.
|
|
59
|
+
|
|
60
|
+
A session with zero of these did not follow the compile-okf skill (its
|
|
61
|
+
first two steps are tool calls) -- see compile.py's "did nothing" check.
|
|
62
|
+
"""
|
|
63
|
+
event = _load(line)
|
|
64
|
+
return event is not None and event.get("type") == "tool_execution_start"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def process(lines: Iterable[str]) -> Iterator[tuple[str, str | None, bool]]:
|
|
68
|
+
"""Yield (raw line, display-or-None, is_tool_call) for each line, in order.
|
|
69
|
+
|
|
70
|
+
`display` is what a human should actually see, and the distinction it
|
|
71
|
+
draws matters in both directions:
|
|
72
|
+
|
|
73
|
+
- A line that is **not JSON at all** -- plain stderr text, a traceback
|
|
74
|
+
merged in from stderr -- is shown verbatim. Dropping these is what
|
|
75
|
+
left a failing run reporting only "pi exited 1" with no cause.
|
|
76
|
+
- A line that **is** a Pi protocol event we do not render is dropped.
|
|
77
|
+
Pi emits a `message_update` envelope per *token*, so showing these
|
|
78
|
+
floods the terminal with thousands of empty-delta JSON objects and
|
|
79
|
+
buries the tool calls -v exists to reveal.
|
|
80
|
+
"""
|
|
81
|
+
for line in lines:
|
|
82
|
+
event = _load(line)
|
|
83
|
+
if event is None:
|
|
84
|
+
yield line, line, False # not JSON: a diagnostic, worth showing
|
|
85
|
+
continue
|
|
86
|
+
yield line, _translate_event(event), event.get("type") == "tool_execution_start"
|
code2okf/kit/README.md
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# kits/code2okf
|
|
2
|
+
|
|
3
|
+
The Docker Sandbox kit that runs Pi. `spec.yaml` declares the image, the network
|
|
4
|
+
allowlist, the credentials and the pinned installs — Pi itself, the linters, and
|
|
5
|
+
the `mq` and `okfctl` release binaries. Everything under
|
|
6
|
+
`files/home/` is copied into the sandbox at `~/`, so `files/home/.pi/agent/`
|
|
7
|
+
becomes Pi's config directory: `AGENTS.md`, `settings.json`, `models.json` and
|
|
8
|
+
`skills/`.
|
|
9
|
+
|
|
10
|
+
The `files/` level is fixed by the Sandbox Kit schema. It cannot be renamed or
|
|
11
|
+
moved.
|
|
12
|
+
|
|
13
|
+
Config is copied in when the kit is built, not mounted, so an edit reaches Pi on
|
|
14
|
+
the next fresh sandbox — which `code2okf` builds when the kit changes, or on
|
|
15
|
+
`--fresh`.
|
|
16
|
+
|
|
17
|
+
## Model configuration
|
|
18
|
+
|
|
19
|
+
The `openrouter` entry in `files/home/.pi/agent/models.json` names the provider
|
|
20
|
+
and stops there. It carries no `models` array, which is deliberate, and JSON has
|
|
21
|
+
nowhere to put the reason. So here it is. (The `litellm` entry alongside it does
|
|
22
|
+
carry one, for the mirror-image reason — see
|
|
23
|
+
[Using another provider](#using-another-provider).)
|
|
24
|
+
|
|
25
|
+
Pi merges a custom `models` entry by `id`, and that entry replaces the built-in
|
|
26
|
+
catalogue entry it matches. Name a model Pi already knows, such as `{"id":
|
|
27
|
+
"deepseek/deepseek-v4-pro", "name": "…"}`, and you throw its real metadata away.
|
|
28
|
+
What you get instead are Pi's defaults: 128K context, 16,384 max output tokens,
|
|
29
|
+
no reasoning. That output cap cuts a long `write` call off mid-argument and
|
|
30
|
+
takes the run down with it. Leave the array out and the catalogue values apply:
|
|
31
|
+
1M context, 384K max output.
|
|
32
|
+
|
|
33
|
+
Check after any change:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
sbx exec code2okf -- pi --list-models deepseek
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
To change a single field of a catalogue model, use `modelOverrides`, not a
|
|
40
|
+
`models` entry. And avoid `deepseek/deepseek-v4-flash` for this work: the
|
|
41
|
+
catalogue caps its output at 4.1K.
|
|
42
|
+
|
|
43
|
+
## Using another provider
|
|
44
|
+
|
|
45
|
+
`models.json` carries a second provider, `litellm`, switched off by default. It
|
|
46
|
+
points at a LiteLLM gateway, or at anything else that speaks the OpenAI
|
|
47
|
+
protocol: `api: "openai-completions"` is the wire format OpenRouter uses too.
|
|
48
|
+
The example model is `gemini-3.1-pro-preview`.
|
|
49
|
+
|
|
50
|
+
Switching to it takes four steps.
|
|
51
|
+
|
|
52
|
+
1. **Name the gateway.** In `files/home/.pi/agent/models.json`, put your
|
|
53
|
+
gateway's URL in `baseUrl` in place of the `litellm.example.com` placeholder,
|
|
54
|
+
and the model you want in `models`.
|
|
55
|
+
2. **Choose it.** In `files/home/.pi/agent/settings.json`, set
|
|
56
|
+
`"defaultProvider": "litellm"` and `"defaultModel"` to the model id. Pi needs
|
|
57
|
+
both, and both must match an entry in `models.json`.
|
|
58
|
+
3. **Open the road.** Add the gateway's host to `permissions.network.allow` in
|
|
59
|
+
`spec.yaml`, and give it a `credentials` entry like the OpenRouter one:
|
|
60
|
+
`header: Authorization`, `format: "Bearer %s"`, which is how LiteLLM
|
|
61
|
+
authenticates too.
|
|
62
|
+
4. **Hand over the key**, then run `code2okf md/`, which builds the kit and
|
|
63
|
+
copies the config in.
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
sbx secret set-custom --sandbox code2okf --host <your-gateway-host> \
|
|
67
|
+
--env LITELLM_API_KEY --value "$LITELLM_API_KEY"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Only `set-custom` here. Plain `set` knows a fixed list of built-in services;
|
|
71
|
+
OpenRouter is on it, a gateway is not. sbx still marks `set-custom`
|
|
72
|
+
experimental. It has no stdin form either, so the key is visible to anything
|
|
73
|
+
that can list processes for as long as the command runs. Reading it from an
|
|
74
|
+
exported variable, as above, at least keeps it out of your shell history.
|
|
75
|
+
|
|
76
|
+
## Context7 docs extension
|
|
77
|
+
|
|
78
|
+
The kit installs `@upstash/context7-pi` (pinned in `setup.install` and listed
|
|
79
|
+
under `packages` in `settings.json`). That is a native Pi package — not MCP —
|
|
80
|
+
and registers `resolve-library-id`, `query-docs`, and the `context7-docs` skill.
|
|
81
|
+
Egress to `context7.com` is allowlisted in `spec.yaml`.
|
|
82
|
+
|
|
83
|
+
### Why the litellm entry spells out its numbers
|
|
84
|
+
|
|
85
|
+
Pi ships a catalogue of 33 providers. OpenRouter is one of them; LiteLLM is not.
|
|
86
|
+
The advice above about the `models` array therefore turns on its head here.
|
|
87
|
+
Leave the array out for `openrouter` and the catalogue fills in real figures. Do
|
|
88
|
+
the same for `litellm` and there is nothing to fill in, so Pi falls back to 128K
|
|
89
|
+
of context, 16,384 output tokens and no reasoning.
|
|
90
|
+
|
|
91
|
+
That output cap is shared, which makes it tighter than it looks. Thinking tokens
|
|
92
|
+
come out of the same budget as the answer, so a small cap can buy a lot of
|
|
93
|
+
thought and no words at all: the gateway returns 200 and an empty `choices`
|
|
94
|
+
array. Hence the explicit numbers in the entry. `modelOverrides` is no help,
|
|
95
|
+
because it patches ids Pi already knows and drops the rest without a word.
|
|
96
|
+
|
|
97
|
+
Two of those numbers are guesses. `cost` feeds Pi's own usage tracking and says
|
|
98
|
+
nothing about what your gateway charges. `thinkingLevelMap` folds Pi's seven
|
|
99
|
+
thinking levels onto `low`, `medium` and `high`, which a gateway fronting Gemini
|
|
100
|
+
takes as `reasoning_effort`. If yours rejects a level, change the map, or set
|
|
101
|
+
`"reasoning": false` and Pi will stop sending one.
|
|
102
|
+
|
|
103
|
+
### Two things to check before you debug the wrong one
|
|
104
|
+
|
|
105
|
+
First, that the provider is there at all:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
sbx exec code2okf -- pi --list-models litellm
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The filter matches the provider name, so this lists your models and nothing
|
|
112
|
+
else. An empty list means the key never reached Pi, which drops a provider whose
|
|
113
|
+
`apiKey` resolves to nothing without an error or a warning. Beware that the
|
|
114
|
+
listing and the lookup disagree: Pi will still choose the model when it runs,
|
|
115
|
+
because the lookup that resolves `settings.json` pays no attention to keys. So a
|
|
116
|
+
missing key shows up not here but at the first request.
|
|
117
|
+
|
|
118
|
+
Second, that the gateway is reachable. A host that `permissions.network` allows
|
|
119
|
+
may still be out of reach, because lifting sbx's own policy does not give the
|
|
120
|
+
microVM a route to it. Ask from inside, not from your shell:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
sbx exec code2okf -- curl -sS -o /dev/null -w '%{http_code}\n' \
|
|
124
|
+
https://<your-gateway-host>/v1/models
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
`200` or `401` means the path works, `401` being a credential problem rather
|
|
128
|
+
than a routing one. A hang or a DNS failure means it does not.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Relocate an agent's trace folder onto this sandbox's state mount.
|
|
3
|
+
# Usage: sh mount-state.sh LINK SUBDIR
|
|
4
|
+
#
|
|
5
|
+
# A bind mount keeps LINK as a real directory while making its writes land in
|
|
6
|
+
# host-backed state. It must be recreated after every sandbox start.
|
|
7
|
+
set -eu
|
|
8
|
+
[ -n "${CODE2OKF_STATE_DIR:-}" ] || exit 0
|
|
9
|
+
link="$1"
|
|
10
|
+
target="${CODE2OKF_STATE_DIR}/$2"
|
|
11
|
+
|
|
12
|
+
same_fs() {
|
|
13
|
+
one="$(stat -c '%d:%i' "$1" 2>/dev/null || stat -f '%d:%i' "$1" 2>/dev/null)"
|
|
14
|
+
two="$(stat -c '%d:%i' "$2" 2>/dev/null || stat -f '%d:%i' "$2" 2>/dev/null)"
|
|
15
|
+
[ -n "${one}" ] && [ "${one}" = "${two}" ]
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
mkdir -p "${target}" || {
|
|
19
|
+
echo "mount-state: could not create ${target}" >&2
|
|
20
|
+
exit 1
|
|
21
|
+
}
|
|
22
|
+
mkdir -p "${link}" || {
|
|
23
|
+
echo "mount-state: could not create ${link}" >&2
|
|
24
|
+
exit 1
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
# shellcheck disable=SC2310 # false normally means "not bound yet"
|
|
28
|
+
if same_fs "${link}" "${target}"; then
|
|
29
|
+
exit 0
|
|
30
|
+
fi
|
|
31
|
+
|
|
32
|
+
# Host state is authoritative. Merge only names absent from it, including
|
|
33
|
+
# dotfiles, before covering the stock directory with the bind mount.
|
|
34
|
+
for entry in "${link}"/.[!.]* "${link}"/..?* "${link}"/*; do
|
|
35
|
+
[ -e "${entry}" ] || continue
|
|
36
|
+
name="$(basename "${entry}")"
|
|
37
|
+
[ "${name}" = "lost+found" ] && continue
|
|
38
|
+
[ -e "${target}/${name}" ] && continue
|
|
39
|
+
if ! cp -R "${entry}" "${target}/"; then
|
|
40
|
+
echo "mount-state: could not copy ${link}/${name} into ${target}" >&2
|
|
41
|
+
exit 1
|
|
42
|
+
fi
|
|
43
|
+
done
|
|
44
|
+
|
|
45
|
+
if ! sudo -n mount --bind "${target}" "${link}"; then
|
|
46
|
+
echo "mount-state: could not bind-mount ${target} onto ${link}" >&2
|
|
47
|
+
exit 1
|
|
48
|
+
fi
|