teich 0.2.2__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {teich-0.2.2 → teich-0.2.3}/DOCS.md +1 -0
- {teich-0.2.2 → teich-0.2.3}/PKG-INFO +7 -1
- {teich-0.2.2 → teich-0.2.3}/README.md +9 -3
- teich-0.2.3/docs/cli.md +111 -0
- {teich-0.2.2 → teich-0.2.3}/docs/data-format.md +6 -0
- {teich-0.2.2 → teich-0.2.3}/docs/generation.md +2 -0
- {teich-0.2.2 → teich-0.2.3}/docs/index.md +7 -0
- {teich-0.2.2 → teich-0.2.3}/docs/pipeline.md +4 -0
- {teich-0.2.2 → teich-0.2.3}/pyproject.toml +1 -1
- {teich-0.2.2 → teich-0.2.3}/src/teich/__init__.py +1 -1
- {teich-0.2.2 → teich-0.2.3}/src/teich/cli.py +178 -56
- {teich-0.2.2 → teich-0.2.3}/tests/test_extract_anonymize_cli.py +26 -2
- {teich-0.2.2 → teich-0.2.3}/uv.lock +1 -1
- {teich-0.2.2 → teich-0.2.3}/.github/workflows/test.yml +0 -0
- {teich-0.2.2 → teich-0.2.3}/.gitignore +0 -0
- {teich-0.2.2 → teich-0.2.3}/LICENSE +0 -0
- {teich-0.2.2 → teich-0.2.3}/ROADMAP.md +0 -0
- {teich-0.2.2 → teich-0.2.3}/config.example.yaml +0 -0
- {teich-0.2.2 → teich-0.2.3}/docker/codex-runtime.Dockerfile +0 -0
- {teich-0.2.2 → teich-0.2.3}/docs/prepare-data.md +0 -0
- {teich-0.2.2 → teich-0.2.3}/docs/python-api.md +0 -0
- {teich-0.2.2 → teich-0.2.3}/docs/studio.md +0 -0
- {teich-0.2.2 → teich-0.2.3}/docs/training.md +0 -0
- {teich-0.2.2 → teich-0.2.3}/gemma-template.jinja +0 -0
- {teich-0.2.2 → teich-0.2.3}/gemma4_example.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/__main__.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/anonymize.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/audit.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/config.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/converter.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/extract.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/formatter.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/loader.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/prepare.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/runner.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/__init__.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/events.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/generation.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/interactive.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/project.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/server.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/app.js +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/index.html +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/style.css +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/teich.svg +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/vendor/xterm-addon-fit.js +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/vendor/xterm.css +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/vendor/xterm.js +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/tool_schema.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/src/teich/trace_readme.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/teich_example.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/conftest.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_audit.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_cli.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_config.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_converter.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_formatter.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_integration.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_loader.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_prepare.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_runner.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_studio.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_tokenizer_smoke.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_tool_schema.py +0 -0
- {teich-0.2.2 → teich-0.2.3}/tests/test_trace_readme.py +0 -0
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
Teich documentation now lives in focused pages under [`docs/`](docs/index.md).
|
|
4
4
|
|
|
5
5
|
- [Generation](docs/generation.md): create new Codex, Pi, Claude Code, Hermes, or chat datasets, or extract existing local sessions with `teich extract`.
|
|
6
|
+
- [CLI Reference](docs/cli.md): command usage for `init`, `generate`, `extract`, `convert`, `anonymize`, `studio`, and `pool`.
|
|
6
7
|
- [Teich Studio](docs/studio.md): configure projects, manage prompts, run batches, and save interactive sessions from a browser.
|
|
7
8
|
- [Preparing Data](docs/prepare-data.md): load local files, folders, Hugging Face datasets, `datasets.Dataset` objects, and source mixes.
|
|
8
9
|
- [Training](docs/training.md): use Teich with TRL / Unsloth and apply response-only labels with `mask_data()`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: teich
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Turn coding agent traces into auditable supervised fine-tuning data
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -151,8 +151,13 @@ If the agent store is somewhere other than the default home-directory location,
|
|
|
151
151
|
|
|
152
152
|
```bash
|
|
153
153
|
teich extract claude --sessions-dir /path/to/.claude --out data
|
|
154
|
+
teich extract claude --sessions-dir /path/to/.claude/projects --out data
|
|
155
|
+
teich extract codex --sessions-dir /path/to/.codex --out data
|
|
154
156
|
teich extract codex --sessions-dir /path/to/.codex/sessions --out data
|
|
155
157
|
teich extract pi --sessions-dir /path/to/.pi --out data
|
|
158
|
+
teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
|
|
159
|
+
teich extract pi --sessions-dir /path/to/.pi/sessions --out data
|
|
160
|
+
teich extract hermes --sessions-dir /path/to/.hermes --out data
|
|
156
161
|
teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
|
|
157
162
|
```
|
|
158
163
|
|
|
@@ -168,6 +173,7 @@ teich convert data --out teich-training.jsonl
|
|
|
168
173
|
|
|
169
174
|
| Use case | Start here |
|
|
170
175
|
| --- | --- |
|
|
176
|
+
| Find command examples and options | [CLI Reference](docs/cli.md) |
|
|
171
177
|
| Configure and steer runs in a browser | [Teich Studio](docs/studio.md) |
|
|
172
178
|
| Generate Codex, Pi, Claude Code, Hermes, or chat data | [Generation](docs/generation.md) |
|
|
173
179
|
| Load local files, folders, Hugging Face datasets, or `datasets.Dataset` objects | [Preparing Data](docs/prepare-data.md) |
|
|
@@ -118,8 +118,13 @@ If the agent store is somewhere other than the default home-directory location,
|
|
|
118
118
|
|
|
119
119
|
```bash
|
|
120
120
|
teich extract claude --sessions-dir /path/to/.claude --out data
|
|
121
|
+
teich extract claude --sessions-dir /path/to/.claude/projects --out data
|
|
122
|
+
teich extract codex --sessions-dir /path/to/.codex --out data
|
|
121
123
|
teich extract codex --sessions-dir /path/to/.codex/sessions --out data
|
|
122
124
|
teich extract pi --sessions-dir /path/to/.pi --out data
|
|
125
|
+
teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
|
|
126
|
+
teich extract pi --sessions-dir /path/to/.pi/sessions --out data
|
|
127
|
+
teich extract hermes --sessions-dir /path/to/.hermes --out data
|
|
123
128
|
teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
|
|
124
129
|
```
|
|
125
130
|
|
|
@@ -133,9 +138,10 @@ teich convert data --out teich-training.jsonl
|
|
|
133
138
|
|
|
134
139
|
## What Teich Supports
|
|
135
140
|
|
|
136
|
-
| Use case | Start here |
|
|
137
|
-
| --- | --- |
|
|
138
|
-
|
|
|
141
|
+
| Use case | Start here |
|
|
142
|
+
| --- | --- |
|
|
143
|
+
| Find command examples and options | [CLI Reference](docs/cli.md) |
|
|
144
|
+
| Configure and steer runs in a browser | [Teich Studio](docs/studio.md) |
|
|
139
145
|
| Generate Codex, Pi, Claude Code, Hermes, or chat data | [Generation](docs/generation.md) |
|
|
140
146
|
| Load local files, folders, Hugging Face datasets, or `datasets.Dataset` objects | [Preparing Data](docs/prepare-data.md) |
|
|
141
147
|
| Train with TRL / Unsloth while keeping response-only labels correct | [Training](docs/training.md) |
|
teich-0.2.3/docs/cli.md
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# CLI Reference
|
|
2
|
+
|
|
3
|
+
Use `teich --help` for the command list and `teich COMMAND --help` for command-specific options.
|
|
4
|
+
|
|
5
|
+
## Project Setup
|
|
6
|
+
|
|
7
|
+
Create a starter project:
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
teich init my-project
|
|
11
|
+
cd my-project
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
This writes `config.yaml` and `prompts.jsonl`. Edit both, set an API key such as `TEICH_API_KEY`, `OPENROUTER_API_KEY`, or `OPENAI_API_KEY`, then run generation.
|
|
15
|
+
|
|
16
|
+
## Generate
|
|
17
|
+
|
|
18
|
+
Run prompts from a config file:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
teich generate -c config.yaml
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Useful options:
|
|
25
|
+
|
|
26
|
+
- `--config`, `-c`: config path, default `config.yaml`
|
|
27
|
+
- `--output`, `-o`: override `output.traces_dir`
|
|
28
|
+
- `--concurrency`, `-j`: number of prompts to run in parallel
|
|
29
|
+
- `--resume`: skip prompts that already have completed outputs
|
|
30
|
+
|
|
31
|
+
Generated runs write raw traces, converted JSONL rows, and a dataset `README.md` under `output/` by default. Agent providers also write workspace snapshots under `sandbox/`; failed or interrupted traces go under `failures/`.
|
|
32
|
+
|
|
33
|
+
## Extract
|
|
34
|
+
|
|
35
|
+
Extract existing local sessions:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
teich extract claude --model fable-5 --out data
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The first argument is the provider: `claude`, `codex`, `pi`, or `hermes`.
|
|
42
|
+
|
|
43
|
+
Useful options:
|
|
44
|
+
|
|
45
|
+
- `--sessions-dir`, `-s`: explicit agent root, native session folder, `state.db`, or JSONL file. Can be passed more than once.
|
|
46
|
+
- `--model`, `-m`: substring match against model metadata. For example, `fable-5` matches `claude-fable-5`.
|
|
47
|
+
- `--output`, `--out`, `-o`: staged dataset directory, default `data`
|
|
48
|
+
- `--no-anon`, `--no-anonymize`: skip automatic anonymization
|
|
49
|
+
- `--private`: if you choose the Hugging Face upload prompt, create the dataset as private
|
|
50
|
+
|
|
51
|
+
Default stores:
|
|
52
|
+
|
|
53
|
+
```text
|
|
54
|
+
claude CLAUDE_CONFIG_DIR/projects, CLAUDE_HOME/projects, ~/.claude/projects
|
|
55
|
+
codex CODEX_HOME/sessions, ~/.codex/sessions
|
|
56
|
+
pi PI_SESSION_DIR, PI_CODING_AGENT_DIR/sessions, ~/.pi/agent/sessions, ~/.pi/sessions
|
|
57
|
+
hermes HERMES_STATE_DB, HERMES_HOME/state.db, ~/.hermes/state.db
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Explicit paths can point at either the agent root or the native store:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
teich extract claude --sessions-dir /path/to/.claude --out data
|
|
64
|
+
teich extract claude --sessions-dir /path/to/.claude/projects --out data
|
|
65
|
+
teich extract codex --sessions-dir /path/to/.codex --out data
|
|
66
|
+
teich extract codex --sessions-dir /path/to/.codex/sessions --out data
|
|
67
|
+
teich extract pi --sessions-dir /path/to/.pi --out data
|
|
68
|
+
teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
|
|
69
|
+
teich extract pi --sessions-dir /path/to/.pi/sessions --out data
|
|
70
|
+
teich extract hermes --sessions-dir /path/to/.hermes --out data
|
|
71
|
+
teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Extraction anonymizes staged traces by default. Review the staged files before upload; `--no-anon` keeps raw values unchanged.
|
|
75
|
+
|
|
76
|
+
## Convert
|
|
77
|
+
|
|
78
|
+
Convert raw or extracted traces into normalized Teich JSONL:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
teich convert data --out teich-training.jsonl
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Each output line contains `prompt`, `messages`, `tools`, and `metadata`. Use this when your training stack can consume OpenAI-style message rows without importing Teich. Use `prepare_data()` and `mask_data()` when you want tokenizer-specific rendering and exact response-only labels.
|
|
85
|
+
|
|
86
|
+
## Anonymize
|
|
87
|
+
|
|
88
|
+
Scrub common secrets and local usernames:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
teich anonymize output --output output_anonymized
|
|
92
|
+
teich anonymize data --in-place
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Anonymization replaces API keys, email addresses, and home-directory usernames with deterministic dummy values while preserving embedded base64 media payloads. It is a best-effort pass; review data before publishing.
|
|
96
|
+
|
|
97
|
+
## Studio
|
|
98
|
+
|
|
99
|
+
Launch the browser UI:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
teich studio
|
|
103
|
+
teich studio ./my-project
|
|
104
|
+
teich studio --host 127.0.0.1 --port 8420 --no-open
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Studio edits the same `config.yaml` and `prompts.jsonl` files used by `teich generate`.
|
|
108
|
+
|
|
109
|
+
## Pool
|
|
110
|
+
|
|
111
|
+
`teich pool upload` is reserved for a future Teich community pool backend. For Hugging Face dataset uploads today, use `teich generate` or `teich extract` and confirm the upload prompt.
|
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
Teich normalizes supported sources into structured training examples.
|
|
4
4
|
|
|
5
|
+
To write this normalized format as standalone JSONL from raw or extracted traces, run:
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
teich convert data --out teich-training.jsonl
|
|
9
|
+
```
|
|
10
|
+
|
|
5
11
|
Core fields:
|
|
6
12
|
|
|
7
13
|
- `prompt`: initial task description
|
|
@@ -63,6 +63,8 @@ teich extract claude --sessions-dir /path/to/.claude/projects --out data
|
|
|
63
63
|
teich extract codex --sessions-dir /path/to/.codex --out data
|
|
64
64
|
teich extract codex --sessions-dir /path/to/.codex/sessions --out data
|
|
65
65
|
teich extract pi --sessions-dir /path/to/.pi --out data
|
|
66
|
+
teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
|
|
67
|
+
teich extract pi --sessions-dir /path/to/.pi/sessions --out data
|
|
66
68
|
teich extract hermes --sessions-dir /path/to/.hermes --out data
|
|
67
69
|
teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
|
|
68
70
|
```
|
|
@@ -5,6 +5,7 @@ Start with the README for a quick overview, then use these pages for implementat
|
|
|
5
5
|
## Guides
|
|
6
6
|
|
|
7
7
|
- [Teich Studio](studio.md): configure projects, manage prompts, run batches, and save interactive sessions from a browser.
|
|
8
|
+
- [CLI Reference](cli.md): command usage for `init`, `generate`, `extract`, `convert`, `anonymize`, `studio`, and `pool`.
|
|
8
9
|
- [Generation](generation.md): create new Codex, Pi, Claude Code, Hermes, or chat datasets.
|
|
9
10
|
- [Preparing Data](prepare-data.md): load local files, folders, Hugging Face datasets, `datasets.Dataset` objects, and source mixes.
|
|
10
11
|
- [Training](training.md): use Teich with TRL / Unsloth and apply response-only labels with `mask_data()`.
|
|
@@ -32,6 +33,12 @@ Already have local agent sessions:
|
|
|
32
33
|
teich extract claude --model fable-5 -> optional HF upload -> prepare_data()
|
|
33
34
|
```
|
|
34
35
|
|
|
36
|
+
Want normalized JSONL for another trainer:
|
|
37
|
+
|
|
38
|
+
```text
|
|
39
|
+
teich extract claude --out data -> teich convert data --out teich-training.jsonl
|
|
40
|
+
```
|
|
41
|
+
|
|
35
42
|
Prefer a browser:
|
|
36
43
|
|
|
37
44
|
```text
|
|
@@ -9,6 +9,7 @@ flowchart TD
|
|
|
9
9
|
A["Raw source<br/>HF repo, local traces, Dataset, source mix, or prompts"] --> B{"Need generation?"}
|
|
10
10
|
B -->|"yes"| C["teich generate"]
|
|
11
11
|
B -->|"existing local sessions"| X["teich extract"]
|
|
12
|
+
B -->|"want standalone JSONL"| Y["teich convert"]
|
|
12
13
|
B -->|"no"| D["prepare_data"]
|
|
13
14
|
C --> C1["Run provider<br/>codex, pi, claude-code, hermes, or chat"]
|
|
14
15
|
C1 --> C2["Write raw/native traces or structured chat rows"]
|
|
@@ -21,6 +22,8 @@ flowchart TD
|
|
|
21
22
|
X4 -->|"yes"| X5["Upload JSONL + README"]
|
|
22
23
|
X4 -->|"no"| D
|
|
23
24
|
X5 --> D
|
|
25
|
+
X3 --> Y
|
|
26
|
+
Y --> Y1["Write normalized Teich JSONL<br/>prompt, messages, tools, metadata"]
|
|
24
27
|
D --> E["Resolve and load sources"]
|
|
25
28
|
E --> F["Convert traces to messages + tools"]
|
|
26
29
|
F --> G["Render tokenizer chat template"]
|
|
@@ -163,6 +166,7 @@ flowchart TD
|
|
|
163
166
|
```text
|
|
164
167
|
prepare_data keeps human-readable text plus typed span metadata.
|
|
165
168
|
mask_data converts the selected spans into exact token-level labels after trainer tokenization.
|
|
169
|
+
teich convert writes normalized message JSONL without tokenizer rendering or token labels.
|
|
166
170
|
```
|
|
167
171
|
|
|
168
172
|
This lets Teich stay compatible with TRL / Unsloth trainer flows while still controlling exactly what the model learns.
|
|
@@ -15,6 +15,7 @@ from rich.console import Console, Group
|
|
|
15
15
|
from rich.live import Live
|
|
16
16
|
from rich.panel import Panel
|
|
17
17
|
from rich.table import Table
|
|
18
|
+
from typer.core import TyperCommand, TyperGroup
|
|
18
19
|
|
|
19
20
|
from .anonymize import anonymize_path
|
|
20
21
|
from .config import Config
|
|
@@ -36,19 +37,141 @@ from .trace_readme import write_traces_readme
|
|
|
36
37
|
from .tool_schema import snapshot_configured_tools
|
|
37
38
|
|
|
38
39
|
console = Console()
|
|
40
|
+
ROOT_EXTRA_HELP = """\
|
|
41
|
+
Common workflows:
|
|
42
|
+
teich init my-project
|
|
43
|
+
teich generate -c config.yaml
|
|
44
|
+
teich extract claude --model fable-5 --out data
|
|
45
|
+
teich convert data --out teich-training.jsonl
|
|
46
|
+
teich studio
|
|
47
|
+
|
|
48
|
+
For command-specific options and examples:
|
|
49
|
+
teich generate --help
|
|
50
|
+
teich extract --help
|
|
51
|
+
teich convert --help
|
|
52
|
+
teich studio --help
|
|
53
|
+
"""
|
|
54
|
+
EXTRACT_HELP = "Extract existing local agent sessions into a staged Teich dataset folder."
|
|
55
|
+
EXTRACT_EXTRA_HELP = """\
|
|
56
|
+
Common examples:
|
|
57
|
+
teich extract claude --model fable-5 --out data
|
|
58
|
+
teich extract claude --sessions-dir /path/to/.claude --out data
|
|
59
|
+
teich extract claude --sessions-dir /path/to/.claude/projects --out data
|
|
60
|
+
teich extract codex --sessions-dir /path/to/.codex --out data
|
|
61
|
+
teich extract codex --sessions-dir /path/to/.codex/sessions --out data
|
|
62
|
+
teich extract pi --sessions-dir /path/to/.pi --out data
|
|
63
|
+
teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
|
|
64
|
+
teich extract pi --sessions-dir /path/to/.pi/sessions --out data
|
|
65
|
+
teich extract hermes --sessions-dir /path/to/.hermes --out data
|
|
66
|
+
teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
|
|
67
|
+
|
|
68
|
+
Default stores:
|
|
69
|
+
claude CLAUDE_CONFIG_DIR/projects, CLAUDE_HOME/projects, ~/.claude/projects
|
|
70
|
+
codex CODEX_HOME/sessions, ~/.codex/sessions
|
|
71
|
+
pi PI_SESSION_DIR, PI_CODING_AGENT_DIR/sessions, ~/.pi/agent/sessions, ~/.pi/sessions
|
|
72
|
+
hermes HERMES_STATE_DB, HERMES_HOME/state.db, ~/.hermes/state.db
|
|
73
|
+
|
|
74
|
+
After extraction:
|
|
75
|
+
teich convert data --out teich-training.jsonl
|
|
76
|
+
This writes normalized Teich JSONL rows with prompt, messages, tools, and metadata for trainers that do not import Teich.
|
|
77
|
+
"""
|
|
78
|
+
GENERATE_EXTRA_HELP = """\
|
|
79
|
+
Typical generated project flow:
|
|
80
|
+
teich init my-project
|
|
81
|
+
cd my-project
|
|
82
|
+
edit prompts.jsonl and config.yaml
|
|
83
|
+
teich generate -c config.yaml
|
|
84
|
+
teich generate -c config.yaml --resume
|
|
85
|
+
|
|
86
|
+
Outputs:
|
|
87
|
+
output/ raw traces, converted JSONL rows, and README.md
|
|
88
|
+
sandbox/ workspace snapshots for agent providers
|
|
89
|
+
failures/ failed or interrupted traces excluded from resume/upload
|
|
90
|
+
|
|
91
|
+
Use teich extract instead when you already have local Claude, Codex, Pi, or Hermes sessions.
|
|
92
|
+
"""
|
|
93
|
+
CONVERT_EXTRA_HELP = """\
|
|
94
|
+
Examples:
|
|
95
|
+
teich convert data --out teich-training.jsonl
|
|
96
|
+
teich convert output/session.jsonl --out session.training.jsonl
|
|
97
|
+
|
|
98
|
+
Output format:
|
|
99
|
+
Newline-delimited JSON rows with prompt, messages, tools, and metadata.
|
|
100
|
+
This is useful when your trainer can consume OpenAI-style message rows without importing Teich.
|
|
101
|
+
|
|
102
|
+
Use prepare_data() and mask_data() when you want tokenizer-specific rendering and exact response-only labels.
|
|
103
|
+
"""
|
|
104
|
+
ANONYMIZE_EXTRA_HELP = """\
|
|
105
|
+
Examples:
|
|
106
|
+
teich anonymize output --output output_anonymized
|
|
107
|
+
teich anonymize data --in-place
|
|
108
|
+
|
|
109
|
+
What is scrubbed:
|
|
110
|
+
API keys, email addresses, and home-directory usernames.
|
|
111
|
+
Embedded base64 media payloads are preserved for conversation context.
|
|
112
|
+
|
|
113
|
+
This is a best-effort safety pass. Review data before publishing or uploading.
|
|
114
|
+
"""
|
|
115
|
+
INIT_EXTRA_HELP = """\
|
|
116
|
+
Creates:
|
|
117
|
+
config.yaml generation settings
|
|
118
|
+
prompts.jsonl starter prompt rows
|
|
119
|
+
|
|
120
|
+
Next:
|
|
121
|
+
edit config.yaml and prompts.jsonl
|
|
122
|
+
set TEICH_API_KEY, OPENROUTER_API_KEY, or OPENAI_API_KEY
|
|
123
|
+
teich generate -c config.yaml
|
|
124
|
+
"""
|
|
125
|
+
STUDIO_EXTRA_HELP = """\
|
|
126
|
+
Examples:
|
|
127
|
+
teich studio
|
|
128
|
+
teich studio ./my-project
|
|
129
|
+
teich studio --host 127.0.0.1 --port 8420 --no-open
|
|
130
|
+
|
|
131
|
+
Studio edits the same config.yaml and prompts.jsonl files used by teich generate.
|
|
132
|
+
"""
|
|
133
|
+
POOL_EXTRA_HELP = """\
|
|
134
|
+
This namespace is reserved for a future Teich community pool upload backend.
|
|
135
|
+
For Hugging Face dataset uploads today, use teich generate or teich extract and confirm the upload prompt.
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class ExtraHelpCommand(TyperCommand):
|
|
140
|
+
extra_help = ""
|
|
141
|
+
|
|
142
|
+
def format_help(self, ctx: typer.Context, formatter: typer.HelpFormatter) -> None:
|
|
143
|
+
super().format_help(ctx, formatter)
|
|
144
|
+
if self.extra_help:
|
|
145
|
+
formatter.write("\n" + self.extra_help)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class ExtraHelpGroup(TyperGroup):
|
|
149
|
+
extra_help = ""
|
|
150
|
+
|
|
151
|
+
def format_help(self, ctx: typer.Context, formatter: typer.HelpFormatter) -> None:
|
|
152
|
+
super().format_help(ctx, formatter)
|
|
153
|
+
if self.extra_help:
|
|
154
|
+
formatter.write("\n" + self.extra_help)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _help_command(name: str, extra_help: str) -> type[ExtraHelpCommand]:
|
|
158
|
+
return type(name, (ExtraHelpCommand,), {"extra_help": extra_help})
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _help_group(name: str, extra_help: str) -> type[ExtraHelpGroup]:
|
|
162
|
+
return type(name, (ExtraHelpGroup,), {"extra_help": extra_help})
|
|
163
|
+
|
|
164
|
+
|
|
39
165
|
app = typer.Typer(
|
|
40
166
|
name="teich",
|
|
41
|
-
help="Generate
|
|
167
|
+
help="Generate, extract, convert, prepare, and inspect agent training data.",
|
|
42
168
|
no_args_is_help=True,
|
|
169
|
+
cls=_help_group("TeichHelpGroup", ROOT_EXTRA_HELP),
|
|
43
170
|
)
|
|
44
|
-
|
|
45
|
-
help=
|
|
46
|
-
|
|
47
|
-
"--sessions-dir PATH to read a .claude, .codex, .pi, or .hermes folder outside the default location."
|
|
48
|
-
)
|
|
171
|
+
pool_app = typer.Typer(
|
|
172
|
+
help="Reserved Teich community pool commands.",
|
|
173
|
+
cls=_help_group("PoolHelpGroup", POOL_EXTRA_HELP),
|
|
49
174
|
)
|
|
50
|
-
pool_app = typer.Typer(help="Upload anonymized traces to the Teich community pool.")
|
|
51
|
-
app.add_typer(extract_app, name="extract")
|
|
52
175
|
app.add_typer(pool_app, name="pool")
|
|
53
176
|
NON_DATA_TRACE_DIR_NAMES = {"partials", "failures"}
|
|
54
177
|
UPLOAD_IGNORE_PATTERNS = ["partials/**", "failures/**"]
|
|
@@ -356,8 +479,9 @@ def _extract_sources_option() -> list[Path] | None:
|
|
|
356
479
|
"--sessions-dir",
|
|
357
480
|
"-s",
|
|
358
481
|
help=(
|
|
359
|
-
"Explicit agent
|
|
360
|
-
".
|
|
482
|
+
"Explicit agent root, native session folder, state.db file, or JSONL file. "
|
|
483
|
+
"Examples: .claude, .claude/projects, .codex, .codex/sessions, .pi, "
|
|
484
|
+
".pi/agent/sessions, .pi/sessions, .hermes, .hermes/state.db. Can be passed more than once."
|
|
361
485
|
),
|
|
362
486
|
)
|
|
363
487
|
|
|
@@ -374,55 +498,33 @@ def _extract_no_anonymize_option() -> bool:
|
|
|
374
498
|
return typer.Option(False, "--no-anon", "--no-anonymize", help="Skip automatic anonymization of extracted traces.")
|
|
375
499
|
|
|
376
500
|
|
|
377
|
-
@
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
@extract_app.command()
|
|
390
|
-
def claude(
|
|
391
|
-
output: Path = _extract_output_option(),
|
|
392
|
-
sessions_dir: list[Path] | None = _extract_sources_option(),
|
|
393
|
-
model: str | None = _extract_model_option(),
|
|
394
|
-
no_anon: bool = _extract_no_anonymize_option(),
|
|
395
|
-
private: bool = typer.Option(False, "--private", help="Create the Hugging Face dataset as private if upload is confirmed."),
|
|
396
|
-
) -> None:
|
|
397
|
-
"""Extract local Claude Code sessions."""
|
|
398
|
-
_run_extract_command("claude", output, sessions_dir, model_filter=model, skip_anonymize=no_anon, private=private)
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
@extract_app.command()
|
|
402
|
-
def pi(
|
|
403
|
-
output: Path = _extract_output_option(),
|
|
404
|
-
sessions_dir: list[Path] | None = _extract_sources_option(),
|
|
405
|
-
model: str | None = _extract_model_option(),
|
|
406
|
-
no_anon: bool = _extract_no_anonymize_option(),
|
|
407
|
-
private: bool = typer.Option(False, "--private", help="Create the Hugging Face dataset as private if upload is confirmed."),
|
|
408
|
-
) -> None:
|
|
409
|
-
"""Extract local Pi sessions."""
|
|
410
|
-
_run_extract_command("pi", output, sessions_dir, model_filter=model, skip_anonymize=no_anon, private=private)
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
@extract_app.command()
|
|
414
|
-
def hermes(
|
|
501
|
+
@app.command(
|
|
502
|
+
help=EXTRACT_HELP,
|
|
503
|
+
short_help="Extract existing local agent sessions.",
|
|
504
|
+
no_args_is_help=True,
|
|
505
|
+
cls=_help_command("ExtractHelpCommand", EXTRACT_EXTRA_HELP),
|
|
506
|
+
)
|
|
507
|
+
def extract(
|
|
508
|
+
provider: ExtractProvider = typer.Argument(
|
|
509
|
+
...,
|
|
510
|
+
metavar="PROVIDER",
|
|
511
|
+
help="Provider to extract: claude, codex, pi, or hermes.",
|
|
512
|
+
),
|
|
415
513
|
output: Path = _extract_output_option(),
|
|
416
514
|
sessions_dir: list[Path] | None = _extract_sources_option(),
|
|
417
515
|
model: str | None = _extract_model_option(),
|
|
418
516
|
no_anon: bool = _extract_no_anonymize_option(),
|
|
419
517
|
private: bool = typer.Option(False, "--private", help="Create the Hugging Face dataset as private if upload is confirmed."),
|
|
420
518
|
) -> None:
|
|
421
|
-
"""Extract local
|
|
422
|
-
_run_extract_command(
|
|
519
|
+
"""Extract local sessions for one provider."""
|
|
520
|
+
_run_extract_command(provider, output, sessions_dir, model_filter=model, skip_anonymize=no_anon, private=private)
|
|
423
521
|
|
|
424
522
|
|
|
425
|
-
@app.command(
|
|
523
|
+
@app.command(
|
|
524
|
+
help="Copy trace files and scrub common secrets/user identifiers.",
|
|
525
|
+
short_help="Anonymize trace files.",
|
|
526
|
+
cls=_help_command("AnonymizeHelpCommand", ANONYMIZE_EXTRA_HELP),
|
|
527
|
+
)
|
|
426
528
|
def anonymize(
|
|
427
529
|
input_path: Path = typer.Argument(Path("output"), help="Trace file or directory to anonymize"),
|
|
428
530
|
output: Path = typer.Option(
|
|
@@ -452,7 +554,11 @@ def anonymize(
|
|
|
452
554
|
console.print("[yellow]No emails, usernames, or API keys were detected.[/yellow]")
|
|
453
555
|
|
|
454
556
|
|
|
455
|
-
@app.command(
|
|
557
|
+
@app.command(
|
|
558
|
+
help="Convert raw or extracted traces into normalized Teich JSONL rows.",
|
|
559
|
+
short_help="Convert traces to Teich JSONL.",
|
|
560
|
+
cls=_help_command("ConvertHelpCommand", CONVERT_EXTRA_HELP),
|
|
561
|
+
)
|
|
456
562
|
def convert(
|
|
457
563
|
input_path: Path = typer.Argument(..., help="Raw trace JSONL file or folder to convert"),
|
|
458
564
|
output: Path = typer.Option(
|
|
@@ -485,7 +591,11 @@ def convert(
|
|
|
485
591
|
console.print("[cyan]Output format:[/cyan] Teich JSONL with prompt, messages, tools, and metadata.")
|
|
486
592
|
|
|
487
593
|
|
|
488
|
-
@pool_app.command(
|
|
594
|
+
@pool_app.command(
|
|
595
|
+
"upload",
|
|
596
|
+
help="Reserved placeholder for future Teich pool uploads.",
|
|
597
|
+
short_help="Reserved pool upload placeholder.",
|
|
598
|
+
)
|
|
489
599
|
def pool_upload(
|
|
490
600
|
path: Path = typer.Argument(Path("output_anonymized"), help="Anonymized trace directory to upload later"),
|
|
491
601
|
) -> None:
|
|
@@ -496,7 +606,11 @@ def pool_upload(
|
|
|
496
606
|
raise typer.Exit(1)
|
|
497
607
|
|
|
498
608
|
|
|
499
|
-
@app.command(
|
|
609
|
+
@app.command(
|
|
610
|
+
help="Run a configured prompt batch and write trace/data artifacts.",
|
|
611
|
+
short_help="Generate traces from config.",
|
|
612
|
+
cls=_help_command("GenerateHelpCommand", GENERATE_EXTRA_HELP),
|
|
613
|
+
)
|
|
500
614
|
def generate(
|
|
501
615
|
config: Path = typer.Option(
|
|
502
616
|
Path("config.yaml"),
|
|
@@ -835,7 +949,11 @@ class BatchProgressReporter:
|
|
|
835
949
|
return Group(Panel.fit(summary, title="Batch Summary"), table)
|
|
836
950
|
|
|
837
951
|
|
|
838
|
-
@app.command(
|
|
952
|
+
@app.command(
|
|
953
|
+
help="Create a starter config.yaml and prompts.jsonl project.",
|
|
954
|
+
short_help="Initialize a Teich project.",
|
|
955
|
+
cls=_help_command("InitHelpCommand", INIT_EXTRA_HELP),
|
|
956
|
+
)
|
|
839
957
|
def init(
|
|
840
958
|
path: Path = typer.Argument(Path("."), help="Directory to initialize"),
|
|
841
959
|
) -> None:
|
|
@@ -1016,7 +1134,11 @@ def _configure_studio_event_loop_policy(
|
|
|
1016
1134
|
return True
|
|
1017
1135
|
|
|
1018
1136
|
|
|
1019
|
-
@app.command(
|
|
1137
|
+
@app.command(
|
|
1138
|
+
help="Launch the local Teich Studio browser UI.",
|
|
1139
|
+
short_help="Launch Teich Studio.",
|
|
1140
|
+
cls=_help_command("StudioHelpCommand", STUDIO_EXTRA_HELP),
|
|
1141
|
+
)
|
|
1020
1142
|
def studio(
|
|
1021
1143
|
path: Path = typer.Argument(Path("."), help="Project directory (created/initialized if needed)"),
|
|
1022
1144
|
host: str = typer.Option("127.0.0.1", "--host", help="Host to bind the studio server to"),
|
|
@@ -37,16 +37,40 @@ def _write_minimal_hermes_state_db(state_db: Path, *, session_id: str = "session
|
|
|
37
37
|
connection.close()
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
def
|
|
40
|
+
def test_extract_help_covers_providers_options_defaults_and_examples():
|
|
41
|
+
root_help = runner.invoke(app, ["--help"])
|
|
42
|
+
|
|
43
|
+
assert root_help.exit_code == 0
|
|
44
|
+
root_output = _plain_cli_output(root_help.output)
|
|
45
|
+
assert "Extract existing local agent sessions" in root_output
|
|
46
|
+
assert "teich extract claude --model fable-5 --out data" in root_output
|
|
47
|
+
|
|
41
48
|
group_help = runner.invoke(app, ["extract", "--help"])
|
|
42
49
|
|
|
43
50
|
assert group_help.exit_code == 0
|
|
44
51
|
group_output = _plain_cli_output(group_help.output)
|
|
45
|
-
assert "
|
|
52
|
+
assert "PROVIDER" in group_output
|
|
53
|
+
for provider in ("claude", "codex", "pi", "hermes"):
|
|
54
|
+
assert provider in group_output
|
|
55
|
+
assert "--sessions-dir" in group_output
|
|
56
|
+
assert "--output,--out" in group_output
|
|
57
|
+
assert "--model" in group_output
|
|
58
|
+
assert "--no-anon" in group_output
|
|
59
|
+
assert "--no-anonymize" in group_output
|
|
46
60
|
assert ".claude" in group_output
|
|
61
|
+
assert ".claude/projects" in group_output
|
|
47
62
|
assert ".codex" in group_output
|
|
63
|
+
assert ".codex/sessions" in group_output
|
|
48
64
|
assert ".pi" in group_output
|
|
65
|
+
assert ".pi/agent/sessions" in group_output
|
|
66
|
+
assert ".pi/sessions" in group_output
|
|
49
67
|
assert ".hermes" in group_output
|
|
68
|
+
assert ".hermes/state.db" in group_output
|
|
69
|
+
assert "CLAUDE_CONFIG_DIR/projects" in group_output
|
|
70
|
+
assert "CODEX_HOME/sessions" in group_output
|
|
71
|
+
assert "HERMES_STATE_DB" in group_output
|
|
72
|
+
assert "fable-5" in group_output
|
|
73
|
+
assert "teich convert data --out teich-training.jsonl" in group_output
|
|
50
74
|
|
|
51
75
|
for provider in ("claude", "codex", "hermes", "pi"):
|
|
52
76
|
result = runner.invoke(app, ["extract", provider, "--help"])
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|