teich 0.2.2__tar.gz → 0.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. {teich-0.2.2 → teich-0.2.3}/DOCS.md +1 -0
  2. {teich-0.2.2 → teich-0.2.3}/PKG-INFO +7 -1
  3. {teich-0.2.2 → teich-0.2.3}/README.md +9 -3
  4. teich-0.2.3/docs/cli.md +111 -0
  5. {teich-0.2.2 → teich-0.2.3}/docs/data-format.md +6 -0
  6. {teich-0.2.2 → teich-0.2.3}/docs/generation.md +2 -0
  7. {teich-0.2.2 → teich-0.2.3}/docs/index.md +7 -0
  8. {teich-0.2.2 → teich-0.2.3}/docs/pipeline.md +4 -0
  9. {teich-0.2.2 → teich-0.2.3}/pyproject.toml +1 -1
  10. {teich-0.2.2 → teich-0.2.3}/src/teich/__init__.py +1 -1
  11. {teich-0.2.2 → teich-0.2.3}/src/teich/cli.py +178 -56
  12. {teich-0.2.2 → teich-0.2.3}/tests/test_extract_anonymize_cli.py +26 -2
  13. {teich-0.2.2 → teich-0.2.3}/uv.lock +1 -1
  14. {teich-0.2.2 → teich-0.2.3}/.github/workflows/test.yml +0 -0
  15. {teich-0.2.2 → teich-0.2.3}/.gitignore +0 -0
  16. {teich-0.2.2 → teich-0.2.3}/LICENSE +0 -0
  17. {teich-0.2.2 → teich-0.2.3}/ROADMAP.md +0 -0
  18. {teich-0.2.2 → teich-0.2.3}/config.example.yaml +0 -0
  19. {teich-0.2.2 → teich-0.2.3}/docker/codex-runtime.Dockerfile +0 -0
  20. {teich-0.2.2 → teich-0.2.3}/docs/prepare-data.md +0 -0
  21. {teich-0.2.2 → teich-0.2.3}/docs/python-api.md +0 -0
  22. {teich-0.2.2 → teich-0.2.3}/docs/studio.md +0 -0
  23. {teich-0.2.2 → teich-0.2.3}/docs/training.md +0 -0
  24. {teich-0.2.2 → teich-0.2.3}/gemma-template.jinja +0 -0
  25. {teich-0.2.2 → teich-0.2.3}/gemma4_example.py +0 -0
  26. {teich-0.2.2 → teich-0.2.3}/src/teich/__main__.py +0 -0
  27. {teich-0.2.2 → teich-0.2.3}/src/teich/anonymize.py +0 -0
  28. {teich-0.2.2 → teich-0.2.3}/src/teich/audit.py +0 -0
  29. {teich-0.2.2 → teich-0.2.3}/src/teich/config.py +0 -0
  30. {teich-0.2.2 → teich-0.2.3}/src/teich/converter.py +0 -0
  31. {teich-0.2.2 → teich-0.2.3}/src/teich/extract.py +0 -0
  32. {teich-0.2.2 → teich-0.2.3}/src/teich/formatter.py +0 -0
  33. {teich-0.2.2 → teich-0.2.3}/src/teich/loader.py +0 -0
  34. {teich-0.2.2 → teich-0.2.3}/src/teich/prepare.py +0 -0
  35. {teich-0.2.2 → teich-0.2.3}/src/teich/runner.py +0 -0
  36. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/__init__.py +0 -0
  37. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/events.py +0 -0
  38. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/generation.py +0 -0
  39. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/interactive.py +0 -0
  40. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/project.py +0 -0
  41. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/server.py +0 -0
  42. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/app.js +0 -0
  43. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/index.html +0 -0
  44. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/style.css +0 -0
  45. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/teich.svg +0 -0
  46. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/vendor/xterm-addon-fit.js +0 -0
  47. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/vendor/xterm.css +0 -0
  48. {teich-0.2.2 → teich-0.2.3}/src/teich/studio/static/vendor/xterm.js +0 -0
  49. {teich-0.2.2 → teich-0.2.3}/src/teich/tool_schema.py +0 -0
  50. {teich-0.2.2 → teich-0.2.3}/src/teich/trace_readme.py +0 -0
  51. {teich-0.2.2 → teich-0.2.3}/teich_example.py +0 -0
  52. {teich-0.2.2 → teich-0.2.3}/tests/conftest.py +0 -0
  53. {teich-0.2.2 → teich-0.2.3}/tests/test_audit.py +0 -0
  54. {teich-0.2.2 → teich-0.2.3}/tests/test_cli.py +0 -0
  55. {teich-0.2.2 → teich-0.2.3}/tests/test_config.py +0 -0
  56. {teich-0.2.2 → teich-0.2.3}/tests/test_converter.py +0 -0
  57. {teich-0.2.2 → teich-0.2.3}/tests/test_formatter.py +0 -0
  58. {teich-0.2.2 → teich-0.2.3}/tests/test_integration.py +0 -0
  59. {teich-0.2.2 → teich-0.2.3}/tests/test_loader.py +0 -0
  60. {teich-0.2.2 → teich-0.2.3}/tests/test_prepare.py +0 -0
  61. {teich-0.2.2 → teich-0.2.3}/tests/test_runner.py +0 -0
  62. {teich-0.2.2 → teich-0.2.3}/tests/test_studio.py +0 -0
  63. {teich-0.2.2 → teich-0.2.3}/tests/test_tokenizer_smoke.py +0 -0
  64. {teich-0.2.2 → teich-0.2.3}/tests/test_tool_schema.py +0 -0
  65. {teich-0.2.2 → teich-0.2.3}/tests/test_trace_readme.py +0 -0
@@ -3,6 +3,7 @@
3
3
  Teich documentation now lives in focused pages under [`docs/`](docs/index.md).
4
4
 
5
5
  - [Generation](docs/generation.md): create new Codex, Pi, Claude Code, Hermes, or chat datasets, or extract existing local sessions with `teich extract`.
6
+ - [CLI Reference](docs/cli.md): command usage for `init`, `generate`, `extract`, `convert`, `anonymize`, `studio`, and `pool`.
6
7
  - [Teich Studio](docs/studio.md): configure projects, manage prompts, run batches, and save interactive sessions from a browser.
7
8
  - [Preparing Data](docs/prepare-data.md): load local files, folders, Hugging Face datasets, `datasets.Dataset` objects, and source mixes.
8
9
  - [Training](docs/training.md): use Teich with TRL / Unsloth and apply response-only labels with `mask_data()`.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: teich
3
- Version: 0.2.2
3
+ Version: 0.2.3
4
4
  Summary: Turn coding agent traces into auditable supervised fine-tuning data
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -151,8 +151,13 @@ If the agent store is somewhere other than the default home-directory location,
151
151
 
152
152
  ```bash
153
153
  teich extract claude --sessions-dir /path/to/.claude --out data
154
+ teich extract claude --sessions-dir /path/to/.claude/projects --out data
155
+ teich extract codex --sessions-dir /path/to/.codex --out data
154
156
  teich extract codex --sessions-dir /path/to/.codex/sessions --out data
155
157
  teich extract pi --sessions-dir /path/to/.pi --out data
158
+ teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
159
+ teich extract pi --sessions-dir /path/to/.pi/sessions --out data
160
+ teich extract hermes --sessions-dir /path/to/.hermes --out data
156
161
  teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
157
162
  ```
158
163
 
@@ -168,6 +173,7 @@ teich convert data --out teich-training.jsonl
168
173
 
169
174
  | Use case | Start here |
170
175
  | --- | --- |
176
+ | Find command examples and options | [CLI Reference](docs/cli.md) |
171
177
  | Configure and steer runs in a browser | [Teich Studio](docs/studio.md) |
172
178
  | Generate Codex, Pi, Claude Code, Hermes, or chat data | [Generation](docs/generation.md) |
173
179
  | Load local files, folders, Hugging Face datasets, or `datasets.Dataset` objects | [Preparing Data](docs/prepare-data.md) |
@@ -118,8 +118,13 @@ If the agent store is somewhere other than the default home-directory location,
118
118
 
119
119
  ```bash
120
120
  teich extract claude --sessions-dir /path/to/.claude --out data
121
+ teich extract claude --sessions-dir /path/to/.claude/projects --out data
122
+ teich extract codex --sessions-dir /path/to/.codex --out data
121
123
  teich extract codex --sessions-dir /path/to/.codex/sessions --out data
122
124
  teich extract pi --sessions-dir /path/to/.pi --out data
125
+ teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
126
+ teich extract pi --sessions-dir /path/to/.pi/sessions --out data
127
+ teich extract hermes --sessions-dir /path/to/.hermes --out data
123
128
  teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
124
129
  ```
125
130
 
@@ -133,9 +138,10 @@ teich convert data --out teich-training.jsonl
133
138
 
134
139
  ## What Teich Supports
135
140
 
136
- | Use case | Start here |
137
- | --- | --- |
138
- | Configure and steer runs in a browser | [Teich Studio](docs/studio.md) |
141
+ | Use case | Start here |
142
+ | --- | --- |
143
+ | Find command examples and options | [CLI Reference](docs/cli.md) |
144
+ | Configure and steer runs in a browser | [Teich Studio](docs/studio.md) |
139
145
  | Generate Codex, Pi, Claude Code, Hermes, or chat data | [Generation](docs/generation.md) |
140
146
  | Load local files, folders, Hugging Face datasets, or `datasets.Dataset` objects | [Preparing Data](docs/prepare-data.md) |
141
147
  | Train with TRL / Unsloth while keeping response-only labels correct | [Training](docs/training.md) |
@@ -0,0 +1,111 @@
1
+ # CLI Reference
2
+
3
+ Use `teich --help` for the command list and `teich COMMAND --help` for command-specific options.
4
+
5
+ ## Project Setup
6
+
7
+ Create a starter project:
8
+
9
+ ```bash
10
+ teich init my-project
11
+ cd my-project
12
+ ```
13
+
14
+ This writes `config.yaml` and `prompts.jsonl`. Edit both, set an API key such as `TEICH_API_KEY`, `OPENROUTER_API_KEY`, or `OPENAI_API_KEY`, then run generation.
15
+
16
+ ## Generate
17
+
18
+ Run prompts from a config file:
19
+
20
+ ```bash
21
+ teich generate -c config.yaml
22
+ ```
23
+
24
+ Useful options:
25
+
26
+ - `--config`, `-c`: config path, default `config.yaml`
27
+ - `--output`, `-o`: override `output.traces_dir`
28
+ - `--concurrency`, `-j`: number of prompts to run in parallel
29
+ - `--resume`: skip prompts that already have completed outputs
30
+
31
+ Generated runs write raw traces, converted JSONL rows, and a dataset `README.md` under `output/` by default. Agent providers also write workspace snapshots under `sandbox/`; failed or interrupted traces go under `failures/`.
32
+
33
+ ## Extract
34
+
35
+ Extract existing local sessions:
36
+
37
+ ```bash
38
+ teich extract claude --model fable-5 --out data
39
+ ```
40
+
41
+ The first argument is the provider: `claude`, `codex`, `pi`, or `hermes`.
42
+
43
+ Useful options:
44
+
45
+ - `--sessions-dir`, `-s`: explicit agent root, native session folder, `state.db`, or JSONL file. Can be passed more than once.
46
+ - `--model`, `-m`: substring match against model metadata. For example, `fable-5` matches `claude-fable-5`.
47
+ - `--output`, `--out`, `-o`: staged dataset directory, default `data`
48
+ - `--no-anon`, `--no-anonymize`: skip automatic anonymization
49
+ - `--private`: if you choose the Hugging Face upload prompt, create the dataset as private
50
+
51
+ Default stores:
52
+
53
+ ```text
54
+ claude CLAUDE_CONFIG_DIR/projects, CLAUDE_HOME/projects, ~/.claude/projects
55
+ codex CODEX_HOME/sessions, ~/.codex/sessions
56
+ pi PI_SESSION_DIR, PI_CODING_AGENT_DIR/sessions, ~/.pi/agent/sessions, ~/.pi/sessions
57
+ hermes HERMES_STATE_DB, HERMES_HOME/state.db, ~/.hermes/state.db
58
+ ```
59
+
60
+ Explicit paths can point at either the agent root or the native store:
61
+
62
+ ```bash
63
+ teich extract claude --sessions-dir /path/to/.claude --out data
64
+ teich extract claude --sessions-dir /path/to/.claude/projects --out data
65
+ teich extract codex --sessions-dir /path/to/.codex --out data
66
+ teich extract codex --sessions-dir /path/to/.codex/sessions --out data
67
+ teich extract pi --sessions-dir /path/to/.pi --out data
68
+ teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
69
+ teich extract pi --sessions-dir /path/to/.pi/sessions --out data
70
+ teich extract hermes --sessions-dir /path/to/.hermes --out data
71
+ teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
72
+ ```
73
+
74
+ Extraction anonymizes staged traces by default. Review the staged files before upload; `--no-anon` keeps raw values unchanged.
75
+
76
+ ## Convert
77
+
78
+ Convert raw or extracted traces into normalized Teich JSONL:
79
+
80
+ ```bash
81
+ teich convert data --out teich-training.jsonl
82
+ ```
83
+
84
+ Each output line contains `prompt`, `messages`, `tools`, and `metadata`. Use this when your training stack can consume OpenAI-style message rows without importing Teich. Use `prepare_data()` and `mask_data()` when you want tokenizer-specific rendering and exact response-only labels.
85
+
86
+ ## Anonymize
87
+
88
+ Scrub common secrets and local usernames:
89
+
90
+ ```bash
91
+ teich anonymize output --output output_anonymized
92
+ teich anonymize data --in-place
93
+ ```
94
+
95
+ Anonymization replaces API keys, email addresses, and home-directory usernames with deterministic dummy values while preserving embedded base64 media payloads. It is a best-effort pass; review data before publishing.
96
+
97
+ ## Studio
98
+
99
+ Launch the browser UI:
100
+
101
+ ```bash
102
+ teich studio
103
+ teich studio ./my-project
104
+ teich studio --host 127.0.0.1 --port 8420 --no-open
105
+ ```
106
+
107
+ Studio edits the same `config.yaml` and `prompts.jsonl` files used by `teich generate`.
108
+
109
+ ## Pool
110
+
111
+ `teich pool upload` is reserved for a future Teich community pool backend. For Hugging Face dataset uploads today, use `teich generate` or `teich extract` and confirm the upload prompt.
@@ -2,6 +2,12 @@
2
2
 
3
3
  Teich normalizes supported sources into structured training examples.
4
4
 
5
+ To write this normalized format as standalone JSONL from raw or extracted traces, run:
6
+
7
+ ```bash
8
+ teich convert data --out teich-training.jsonl
9
+ ```
10
+
5
11
  Core fields:
6
12
 
7
13
  - `prompt`: initial task description
@@ -63,6 +63,8 @@ teich extract claude --sessions-dir /path/to/.claude/projects --out data
63
63
  teich extract codex --sessions-dir /path/to/.codex --out data
64
64
  teich extract codex --sessions-dir /path/to/.codex/sessions --out data
65
65
  teich extract pi --sessions-dir /path/to/.pi --out data
66
+ teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
67
+ teich extract pi --sessions-dir /path/to/.pi/sessions --out data
66
68
  teich extract hermes --sessions-dir /path/to/.hermes --out data
67
69
  teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
68
70
  ```
@@ -5,6 +5,7 @@ Start with the README for a quick overview, then use these pages for implementat
5
5
  ## Guides
6
6
 
7
7
  - [Teich Studio](studio.md): configure projects, manage prompts, run batches, and save interactive sessions from a browser.
8
+ - [CLI Reference](cli.md): command usage for `init`, `generate`, `extract`, `convert`, `anonymize`, `studio`, and `pool`.
8
9
  - [Generation](generation.md): create new Codex, Pi, Claude Code, Hermes, or chat datasets.
9
10
  - [Preparing Data](prepare-data.md): load local files, folders, Hugging Face datasets, `datasets.Dataset` objects, and source mixes.
10
11
  - [Training](training.md): use Teich with TRL / Unsloth and apply response-only labels with `mask_data()`.
@@ -32,6 +33,12 @@ Already have local agent sessions:
32
33
  teich extract claude --model fable-5 -> optional HF upload -> prepare_data()
33
34
  ```
34
35
 
36
+ Want normalized JSONL for another trainer:
37
+
38
+ ```text
39
+ teich extract claude --out data -> teich convert data --out teich-training.jsonl
40
+ ```
41
+
35
42
  Prefer a browser:
36
43
 
37
44
  ```text
@@ -9,6 +9,7 @@ flowchart TD
9
9
  A["Raw source<br/>HF repo, local traces, Dataset, source mix, or prompts"] --> B{"Need generation?"}
10
10
  B -->|"yes"| C["teich generate"]
11
11
  B -->|"existing local sessions"| X["teich extract"]
12
+ B -->|"want standalone JSONL"| Y["teich convert"]
12
13
  B -->|"no"| D["prepare_data"]
13
14
  C --> C1["Run provider<br/>codex, pi, claude-code, hermes, or chat"]
14
15
  C1 --> C2["Write raw/native traces or structured chat rows"]
@@ -21,6 +22,8 @@ flowchart TD
21
22
  X4 -->|"yes"| X5["Upload JSONL + README"]
22
23
  X4 -->|"no"| D
23
24
  X5 --> D
25
+ X3 --> Y
26
+ Y --> Y1["Write normalized Teich JSONL<br/>prompt, messages, tools, metadata"]
24
27
  D --> E["Resolve and load sources"]
25
28
  E --> F["Convert traces to messages + tools"]
26
29
  F --> G["Render tokenizer chat template"]
@@ -163,6 +166,7 @@ flowchart TD
163
166
  ```text
164
167
  prepare_data keeps human-readable text plus typed span metadata.
165
168
  mask_data converts the selected spans into exact token-level labels after trainer tokenization.
169
+ teich convert writes normalized message JSONL without tokenizer rendering or token labels.
166
170
  ```
167
171
 
168
172
  This lets Teich stay compatible with TRL / Unsloth trainer flows while still controlling exactly what the model learns.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "teich"
3
- version = "0.2.2"
3
+ version = "0.2.3"
4
4
  description = "Turn coding agent traces into auditable supervised fine-tuning data"
5
5
  readme = "README.md"
6
6
  license = {text = "Apache-2.0"}
@@ -1,6 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
- __version__ = "0.2.2"
3
+ __version__ = "0.2.3"
4
4
 
5
5
  from .audit import SFTAuditReport, audit_sft_dataset
6
6
  from .config import Config, load_config
@@ -15,6 +15,7 @@ from rich.console import Console, Group
15
15
  from rich.live import Live
16
16
  from rich.panel import Panel
17
17
  from rich.table import Table
18
+ from typer.core import TyperCommand, TyperGroup
18
19
 
19
20
  from .anonymize import anonymize_path
20
21
  from .config import Config
@@ -36,19 +37,141 @@ from .trace_readme import write_traces_readme
36
37
  from .tool_schema import snapshot_configured_tools
37
38
 
38
39
  console = Console()
40
+ ROOT_EXTRA_HELP = """\
41
+ Common workflows:
42
+ teich init my-project
43
+ teich generate -c config.yaml
44
+ teich extract claude --model fable-5 --out data
45
+ teich convert data --out teich-training.jsonl
46
+ teich studio
47
+
48
+ For command-specific options and examples:
49
+ teich generate --help
50
+ teich extract --help
51
+ teich convert --help
52
+ teich studio --help
53
+ """
54
+ EXTRACT_HELP = "Extract existing local agent sessions into a staged Teich dataset folder."
55
+ EXTRACT_EXTRA_HELP = """\
56
+ Common examples:
57
+ teich extract claude --model fable-5 --out data
58
+ teich extract claude --sessions-dir /path/to/.claude --out data
59
+ teich extract claude --sessions-dir /path/to/.claude/projects --out data
60
+ teich extract codex --sessions-dir /path/to/.codex --out data
61
+ teich extract codex --sessions-dir /path/to/.codex/sessions --out data
62
+ teich extract pi --sessions-dir /path/to/.pi --out data
63
+ teich extract pi --sessions-dir /path/to/.pi/agent/sessions --out data
64
+ teich extract pi --sessions-dir /path/to/.pi/sessions --out data
65
+ teich extract hermes --sessions-dir /path/to/.hermes --out data
66
+ teich extract hermes --sessions-dir /path/to/.hermes/state.db --out data
67
+
68
+ Default stores:
69
+ claude CLAUDE_CONFIG_DIR/projects, CLAUDE_HOME/projects, ~/.claude/projects
70
+ codex CODEX_HOME/sessions, ~/.codex/sessions
71
+ pi PI_SESSION_DIR, PI_CODING_AGENT_DIR/sessions, ~/.pi/agent/sessions, ~/.pi/sessions
72
+ hermes HERMES_STATE_DB, HERMES_HOME/state.db, ~/.hermes/state.db
73
+
74
+ After extraction:
75
+ teich convert data --out teich-training.jsonl
76
+ This writes normalized Teich JSONL rows with prompt, messages, tools, and metadata for trainers that do not import Teich.
77
+ """
78
+ GENERATE_EXTRA_HELP = """\
79
+ Typical generated project flow:
80
+ teich init my-project
81
+ cd my-project
82
+ edit prompts.jsonl and config.yaml
83
+ teich generate -c config.yaml
84
+ teich generate -c config.yaml --resume
85
+
86
+ Outputs:
87
+ output/ raw traces, converted JSONL rows, and README.md
88
+ sandbox/ workspace snapshots for agent providers
89
+ failures/ failed or interrupted traces excluded from resume/upload
90
+
91
+ Use teich extract instead when you already have local Claude, Codex, Pi, or Hermes sessions.
92
+ """
93
+ CONVERT_EXTRA_HELP = """\
94
+ Examples:
95
+ teich convert data --out teich-training.jsonl
96
+ teich convert output/session.jsonl --out session.training.jsonl
97
+
98
+ Output format:
99
+ Newline-delimited JSON rows with prompt, messages, tools, and metadata.
100
+ This is useful when your trainer can consume OpenAI-style message rows without importing Teich.
101
+
102
+ Use prepare_data() and mask_data() when you want tokenizer-specific rendering and exact response-only labels.
103
+ """
104
+ ANONYMIZE_EXTRA_HELP = """\
105
+ Examples:
106
+ teich anonymize output --output output_anonymized
107
+ teich anonymize data --in-place
108
+
109
+ What is scrubbed:
110
+ API keys, email addresses, and home-directory usernames.
111
+ Embedded base64 media payloads are preserved for conversation context.
112
+
113
+ This is a best-effort safety pass. Review data before publishing or uploading.
114
+ """
115
+ INIT_EXTRA_HELP = """\
116
+ Creates:
117
+ config.yaml generation settings
118
+ prompts.jsonl starter prompt rows
119
+
120
+ Next:
121
+ edit config.yaml and prompts.jsonl
122
+ set TEICH_API_KEY, OPENROUTER_API_KEY, or OPENAI_API_KEY
123
+ teich generate -c config.yaml
124
+ """
125
+ STUDIO_EXTRA_HELP = """\
126
+ Examples:
127
+ teich studio
128
+ teich studio ./my-project
129
+ teich studio --host 127.0.0.1 --port 8420 --no-open
130
+
131
+ Studio edits the same config.yaml and prompts.jsonl files used by teich generate.
132
+ """
133
+ POOL_EXTRA_HELP = """\
134
+ This namespace is reserved for a future Teich community pool upload backend.
135
+ For Hugging Face dataset uploads today, use teich generate or teich extract and confirm the upload prompt.
136
+ """
137
+
138
+
139
+ class ExtraHelpCommand(TyperCommand):
140
+ extra_help = ""
141
+
142
+ def format_help(self, ctx: typer.Context, formatter: typer.HelpFormatter) -> None:
143
+ super().format_help(ctx, formatter)
144
+ if self.extra_help:
145
+ formatter.write("\n" + self.extra_help)
146
+
147
+
148
+ class ExtraHelpGroup(TyperGroup):
149
+ extra_help = ""
150
+
151
+ def format_help(self, ctx: typer.Context, formatter: typer.HelpFormatter) -> None:
152
+ super().format_help(ctx, formatter)
153
+ if self.extra_help:
154
+ formatter.write("\n" + self.extra_help)
155
+
156
+
157
+ def _help_command(name: str, extra_help: str) -> type[ExtraHelpCommand]:
158
+ return type(name, (ExtraHelpCommand,), {"extra_help": extra_help})
159
+
160
+
161
+ def _help_group(name: str, extra_help: str) -> type[ExtraHelpGroup]:
162
+ return type(name, (ExtraHelpGroup,), {"extra_help": extra_help})
163
+
164
+
39
165
  app = typer.Typer(
40
166
  name="teich",
41
- help="Generate agent training data using Codex, Pi, Claude Code, Hermes, or chat",
167
+ help="Generate, extract, convert, prepare, and inspect agent training data.",
42
168
  no_args_is_help=True,
169
+ cls=_help_group("TeichHelpGroup", ROOT_EXTRA_HELP),
43
170
  )
44
- extract_app = typer.Typer(
45
- help=(
46
- "Extract local agent sessions into Teich output folders. Use a provider subcommand with "
47
- "--sessions-dir PATH to read a .claude, .codex, .pi, or .hermes folder outside the default location."
48
- )
171
+ pool_app = typer.Typer(
172
+ help="Reserved Teich community pool commands.",
173
+ cls=_help_group("PoolHelpGroup", POOL_EXTRA_HELP),
49
174
  )
50
- pool_app = typer.Typer(help="Upload anonymized traces to the Teich community pool.")
51
- app.add_typer(extract_app, name="extract")
52
175
  app.add_typer(pool_app, name="pool")
53
176
  NON_DATA_TRACE_DIR_NAMES = {"partials", "failures"}
54
177
  UPLOAD_IGNORE_PATTERNS = ["partials/**", "failures/**"]
@@ -356,8 +479,9 @@ def _extract_sources_option() -> list[Path] | None:
356
479
  "--sessions-dir",
357
480
  "-s",
358
481
  help=(
359
- "Explicit agent data folder or file. Examples: .claude, .claude/projects, "
360
- ".codex, .codex/sessions, .pi, .hermes, or .hermes/state.db. Can be passed more than once."
482
+ "Explicit agent root, native session folder, state.db file, or JSONL file. "
483
+ "Examples: .claude, .claude/projects, .codex, .codex/sessions, .pi, "
484
+ ".pi/agent/sessions, .pi/sessions, .hermes, .hermes/state.db. Can be passed more than once."
361
485
  ),
362
486
  )
363
487
 
@@ -374,55 +498,33 @@ def _extract_no_anonymize_option() -> bool:
374
498
  return typer.Option(False, "--no-anon", "--no-anonymize", help="Skip automatic anonymization of extracted traces.")
375
499
 
376
500
 
377
- @extract_app.command()
378
- def codex(
379
- output: Path = _extract_output_option(),
380
- sessions_dir: list[Path] | None = _extract_sources_option(),
381
- model: str | None = _extract_model_option(),
382
- no_anon: bool = _extract_no_anonymize_option(),
383
- private: bool = typer.Option(False, "--private", help="Create the Hugging Face dataset as private if upload is confirmed."),
384
- ) -> None:
385
- """Extract local Codex sessions."""
386
- _run_extract_command("codex", output, sessions_dir, model_filter=model, skip_anonymize=no_anon, private=private)
387
-
388
-
389
- @extract_app.command()
390
- def claude(
391
- output: Path = _extract_output_option(),
392
- sessions_dir: list[Path] | None = _extract_sources_option(),
393
- model: str | None = _extract_model_option(),
394
- no_anon: bool = _extract_no_anonymize_option(),
395
- private: bool = typer.Option(False, "--private", help="Create the Hugging Face dataset as private if upload is confirmed."),
396
- ) -> None:
397
- """Extract local Claude Code sessions."""
398
- _run_extract_command("claude", output, sessions_dir, model_filter=model, skip_anonymize=no_anon, private=private)
399
-
400
-
401
- @extract_app.command()
402
- def pi(
403
- output: Path = _extract_output_option(),
404
- sessions_dir: list[Path] | None = _extract_sources_option(),
405
- model: str | None = _extract_model_option(),
406
- no_anon: bool = _extract_no_anonymize_option(),
407
- private: bool = typer.Option(False, "--private", help="Create the Hugging Face dataset as private if upload is confirmed."),
408
- ) -> None:
409
- """Extract local Pi sessions."""
410
- _run_extract_command("pi", output, sessions_dir, model_filter=model, skip_anonymize=no_anon, private=private)
411
-
412
-
413
- @extract_app.command()
414
- def hermes(
501
+ @app.command(
502
+ help=EXTRACT_HELP,
503
+ short_help="Extract existing local agent sessions.",
504
+ no_args_is_help=True,
505
+ cls=_help_command("ExtractHelpCommand", EXTRACT_EXTRA_HELP),
506
+ )
507
+ def extract(
508
+ provider: ExtractProvider = typer.Argument(
509
+ ...,
510
+ metavar="PROVIDER",
511
+ help="Provider to extract: claude, codex, pi, or hermes.",
512
+ ),
415
513
  output: Path = _extract_output_option(),
416
514
  sessions_dir: list[Path] | None = _extract_sources_option(),
417
515
  model: str | None = _extract_model_option(),
418
516
  no_anon: bool = _extract_no_anonymize_option(),
419
517
  private: bool = typer.Option(False, "--private", help="Create the Hugging Face dataset as private if upload is confirmed."),
420
518
  ) -> None:
421
- """Extract local Hermes Agent state sessions."""
422
- _run_extract_command("hermes", output, sessions_dir, model_filter=model, skip_anonymize=no_anon, private=private)
519
+ """Extract local sessions for one provider."""
520
+ _run_extract_command(provider, output, sessions_dir, model_filter=model, skip_anonymize=no_anon, private=private)
423
521
 
424
522
 
425
- @app.command()
523
+ @app.command(
524
+ help="Copy trace files and scrub common secrets/user identifiers.",
525
+ short_help="Anonymize trace files.",
526
+ cls=_help_command("AnonymizeHelpCommand", ANONYMIZE_EXTRA_HELP),
527
+ )
426
528
  def anonymize(
427
529
  input_path: Path = typer.Argument(Path("output"), help="Trace file or directory to anonymize"),
428
530
  output: Path = typer.Option(
@@ -452,7 +554,11 @@ def anonymize(
452
554
  console.print("[yellow]No emails, usernames, or API keys were detected.[/yellow]")
453
555
 
454
556
 
455
- @app.command()
557
+ @app.command(
558
+ help="Convert raw or extracted traces into normalized Teich JSONL rows.",
559
+ short_help="Convert traces to Teich JSONL.",
560
+ cls=_help_command("ConvertHelpCommand", CONVERT_EXTRA_HELP),
561
+ )
456
562
  def convert(
457
563
  input_path: Path = typer.Argument(..., help="Raw trace JSONL file or folder to convert"),
458
564
  output: Path = typer.Option(
@@ -485,7 +591,11 @@ def convert(
485
591
  console.print("[cyan]Output format:[/cyan] Teich JSONL with prompt, messages, tools, and metadata.")
486
592
 
487
593
 
488
- @pool_app.command("upload")
594
+ @pool_app.command(
595
+ "upload",
596
+ help="Reserved placeholder for future Teich pool uploads.",
597
+ short_help="Reserved pool upload placeholder.",
598
+ )
489
599
  def pool_upload(
490
600
  path: Path = typer.Argument(Path("output_anonymized"), help="Anonymized trace directory to upload later"),
491
601
  ) -> None:
@@ -496,7 +606,11 @@ def pool_upload(
496
606
  raise typer.Exit(1)
497
607
 
498
608
 
499
- @app.command()
609
+ @app.command(
610
+ help="Run a configured prompt batch and write trace/data artifacts.",
611
+ short_help="Generate traces from config.",
612
+ cls=_help_command("GenerateHelpCommand", GENERATE_EXTRA_HELP),
613
+ )
500
614
  def generate(
501
615
  config: Path = typer.Option(
502
616
  Path("config.yaml"),
@@ -835,7 +949,11 @@ class BatchProgressReporter:
835
949
  return Group(Panel.fit(summary, title="Batch Summary"), table)
836
950
 
837
951
 
838
- @app.command()
952
+ @app.command(
953
+ help="Create a starter config.yaml and prompts.jsonl project.",
954
+ short_help="Initialize a Teich project.",
955
+ cls=_help_command("InitHelpCommand", INIT_EXTRA_HELP),
956
+ )
839
957
  def init(
840
958
  path: Path = typer.Argument(Path("."), help="Directory to initialize"),
841
959
  ) -> None:
@@ -1016,7 +1134,11 @@ def _configure_studio_event_loop_policy(
1016
1134
  return True
1017
1135
 
1018
1136
 
1019
- @app.command()
1137
+ @app.command(
1138
+ help="Launch the local Teich Studio browser UI.",
1139
+ short_help="Launch Teich Studio.",
1140
+ cls=_help_command("StudioHelpCommand", STUDIO_EXTRA_HELP),
1141
+ )
1020
1142
  def studio(
1021
1143
  path: Path = typer.Argument(Path("."), help="Project directory (created/initialized if needed)"),
1022
1144
  host: str = typer.Option("127.0.0.1", "--host", help="Host to bind the studio server to"),
@@ -37,16 +37,40 @@ def _write_minimal_hermes_state_db(state_db: Path, *, session_id: str = "session
37
37
  connection.close()
38
38
 
39
39
 
40
- def test_extract_subcommands_exist():
40
+ def test_extract_help_covers_providers_options_defaults_and_examples():
41
+ root_help = runner.invoke(app, ["--help"])
42
+
43
+ assert root_help.exit_code == 0
44
+ root_output = _plain_cli_output(root_help.output)
45
+ assert "Extract existing local agent sessions" in root_output
46
+ assert "teich extract claude --model fable-5 --out data" in root_output
47
+
41
48
  group_help = runner.invoke(app, ["extract", "--help"])
42
49
 
43
50
  assert group_help.exit_code == 0
44
51
  group_output = _plain_cli_output(group_help.output)
45
- assert "--sessions-dir PATH" in group_output
52
+ assert "PROVIDER" in group_output
53
+ for provider in ("claude", "codex", "pi", "hermes"):
54
+ assert provider in group_output
55
+ assert "--sessions-dir" in group_output
56
+ assert "--output,--out" in group_output
57
+ assert "--model" in group_output
58
+ assert "--no-anon" in group_output
59
+ assert "--no-anonymize" in group_output
46
60
  assert ".claude" in group_output
61
+ assert ".claude/projects" in group_output
47
62
  assert ".codex" in group_output
63
+ assert ".codex/sessions" in group_output
48
64
  assert ".pi" in group_output
65
+ assert ".pi/agent/sessions" in group_output
66
+ assert ".pi/sessions" in group_output
49
67
  assert ".hermes" in group_output
68
+ assert ".hermes/state.db" in group_output
69
+ assert "CLAUDE_CONFIG_DIR/projects" in group_output
70
+ assert "CODEX_HOME/sessions" in group_output
71
+ assert "HERMES_STATE_DB" in group_output
72
+ assert "fable-5" in group_output
73
+ assert "teich convert data --out teich-training.jsonl" in group_output
50
74
 
51
75
  for provider in ("claude", "codex", "hermes", "pi"):
52
76
  result = runner.invoke(app, ["extract", provider, "--help"])
@@ -1734,7 +1734,7 @@ wheels = [
1734
1734
 
1735
1735
  [[package]]
1736
1736
  name = "teich"
1737
- version = "0.2.2"
1737
+ version = "0.2.3"
1738
1738
  source = { editable = "." }
1739
1739
  dependencies = [
1740
1740
  { name = "datasets" },
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes