teich 0.3.2__tar.gz → 0.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {teich-0.3.2 → teich-0.3.4}/PKG-INFO +5 -4
  2. {teich-0.3.2 → teich-0.3.4}/README.md +2 -2
  3. {teich-0.3.2 → teich-0.3.4}/config.example.yaml +11 -1
  4. {teich-0.3.2 → teich-0.3.4}/docs/cli.md +2 -2
  5. {teich-0.3.2 → teich-0.3.4}/docs/generation.md +27 -4
  6. {teich-0.3.2 → teich-0.3.4}/docs/training.md +27 -0
  7. {teich-0.3.2 → teich-0.3.4}/examples/config.claude-code-thinking.yaml +6 -0
  8. {teich-0.3.2 → teich-0.3.4}/examples/config.codex-reasoning.yaml +6 -0
  9. {teich-0.3.2 → teich-0.3.4}/gemma4_example.py +34 -23
  10. {teich-0.3.2 → teich-0.3.4}/pyproject.toml +2 -1
  11. {teich-0.3.2 → teich-0.3.4}/src/teich/__init__.py +1 -1
  12. {teich-0.3.2 → teich-0.3.4}/src/teich/anonymize.py +274 -101
  13. {teich-0.3.2 → teich-0.3.4}/src/teich/cli.py +120 -37
  14. {teich-0.3.2 → teich-0.3.4}/src/teich/config.py +24 -7
  15. {teich-0.3.2 → teich-0.3.4}/src/teich/converter.py +124 -44
  16. {teich-0.3.2 → teich-0.3.4}/src/teich/formatter.py +42 -4
  17. teich-0.3.4/src/teich/harness_capture.py +520 -0
  18. {teich-0.3.2 → teich-0.3.4}/src/teich/runner.py +297 -51
  19. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/extraction.py +7 -7
  20. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/static/app.js +1 -1
  21. {teich-0.3.2 → teich-0.3.4}/src/teich/trace_readme.py +1 -1
  22. {teich-0.3.2 → teich-0.3.4}/tests/test_audit.py +10 -0
  23. {teich-0.3.2 → teich-0.3.4}/tests/test_config.py +4 -1
  24. {teich-0.3.2 → teich-0.3.4}/tests/test_converter.py +73 -14
  25. {teich-0.3.2 → teich-0.3.4}/tests/test_extract_anonymize_cli.py +177 -37
  26. {teich-0.3.2 → teich-0.3.4}/tests/test_formatter.py +93 -3
  27. teich-0.3.4/tests/test_harness_capture.py +127 -0
  28. {teich-0.3.2 → teich-0.3.4}/tests/test_runner.py +51 -1
  29. {teich-0.3.2 → teich-0.3.4}/tests/test_tokenizer_smoke.py +19 -1
  30. {teich-0.3.2 → teich-0.3.4}/tests/test_trace_readme.py +4 -1
  31. {teich-0.3.2 → teich-0.3.4}/uv.lock +12 -1
  32. {teich-0.3.2 → teich-0.3.4}/.github/workflows/test.yml +0 -0
  33. {teich-0.3.2 → teich-0.3.4}/.gitignore +0 -0
  34. {teich-0.3.2 → teich-0.3.4}/DOCS.md +0 -0
  35. {teich-0.3.2 → teich-0.3.4}/LICENSE +0 -0
  36. {teich-0.3.2 → teich-0.3.4}/docker/codex-runtime.Dockerfile +0 -0
  37. {teich-0.3.2 → teich-0.3.4}/docs/data-format.md +0 -0
  38. {teich-0.3.2 → teich-0.3.4}/docs/index.md +0 -0
  39. {teich-0.3.2 → teich-0.3.4}/docs/pipeline.md +0 -0
  40. {teich-0.3.2 → teich-0.3.4}/docs/prepare-data.md +0 -0
  41. {teich-0.3.2 → teich-0.3.4}/docs/python-api.md +0 -0
  42. {teich-0.3.2 → teich-0.3.4}/docs/studio.md +0 -0
  43. {teich-0.3.2 → teich-0.3.4}/examples/example_claude_code.jsonl +0 -0
  44. {teich-0.3.2 → teich-0.3.4}/examples/example_codex_session.jsonl +0 -0
  45. {teich-0.3.2 → teich-0.3.4}/examples/example_cursor_session.jsonl +0 -0
  46. {teich-0.3.2 → teich-0.3.4}/examples/example_droid_session.jsonl +0 -0
  47. {teich-0.3.2 → teich-0.3.4}/examples/example_hermes_session.jsonl +0 -0
  48. {teich-0.3.2 → teich-0.3.4}/examples/example_openclaw_session.jsonl +0 -0
  49. {teich-0.3.2 → teich-0.3.4}/examples/example_pi_session.jsonl +0 -0
  50. {teich-0.3.2 → teich-0.3.4}/examples/prompts.jsonl +0 -0
  51. {teich-0.3.2 → teich-0.3.4}/gemma-4-chat-template.jinja +0 -0
  52. {teich-0.3.2 → teich-0.3.4}/gemma-template.jinja +0 -0
  53. {teich-0.3.2 → teich-0.3.4}/new_gemma_4_template.jinja +0 -0
  54. {teich-0.3.2 → teich-0.3.4}/qwen3.5-chat-template.jinja +0 -0
  55. {teich-0.3.2 → teich-0.3.4}/qwen3.6-chat-template.jinja +0 -0
  56. {teich-0.3.2 → teich-0.3.4}/qwen3.6_chat_template.jinja +0 -0
  57. {teich-0.3.2 → teich-0.3.4}/src/teich/__main__.py +0 -0
  58. {teich-0.3.2 → teich-0.3.4}/src/teich/audit.py +0 -0
  59. {teich-0.3.2 → teich-0.3.4}/src/teich/codex_auth_broker.py +0 -0
  60. {teich-0.3.2 → teich-0.3.4}/src/teich/extract.py +0 -0
  61. {teich-0.3.2 → teich-0.3.4}/src/teich/loader.py +0 -0
  62. {teich-0.3.2 → teich-0.3.4}/src/teich/prepare.py +0 -0
  63. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/__init__.py +0 -0
  64. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/dataset_preview.py +0 -0
  65. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/events.py +0 -0
  66. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/generation.py +0 -0
  67. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/interactive.py +0 -0
  68. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/project.py +0 -0
  69. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/server.py +0 -0
  70. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/static/index.html +0 -0
  71. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/static/style.css +0 -0
  72. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/static/teich.svg +0 -0
  73. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/static/vendor/xterm-addon-fit.js +0 -0
  74. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/static/vendor/xterm.css +0 -0
  75. {teich-0.3.2 → teich-0.3.4}/src/teich/studio/static/vendor/xterm.js +0 -0
  76. {teich-0.3.2 → teich-0.3.4}/src/teich/tool_schema.py +0 -0
  77. {teich-0.3.2 → teich-0.3.4}/teich_example.py +0 -0
  78. {teich-0.3.2 → teich-0.3.4}/tests/conftest.py +0 -0
  79. {teich-0.3.2 → teich-0.3.4}/tests/test_cli.py +0 -0
  80. {teich-0.3.2 → teich-0.3.4}/tests/test_codex_auth_broker.py +0 -0
  81. {teich-0.3.2 → teich-0.3.4}/tests/test_codex_langfuse.py +0 -0
  82. {teich-0.3.2 → teich-0.3.4}/tests/test_integration.py +0 -0
  83. {teich-0.3.2 → teich-0.3.4}/tests/test_langfuse_agents.py +0 -0
  84. {teich-0.3.2 → teich-0.3.4}/tests/test_loader.py +0 -0
  85. {teich-0.3.2 → teich-0.3.4}/tests/test_prepare.py +0 -0
  86. {teich-0.3.2 → teich-0.3.4}/tests/test_studio.py +0 -0
  87. {teich-0.3.2 → teich-0.3.4}/tests/test_tool_schema.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: teich
3
- Version: 0.3.2
3
+ Version: 0.3.4
4
4
  Summary: Turn coding agent traces into auditable supervised fine-tuning data
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -28,6 +28,7 @@ Requires-Dist: mypy>=1.10; extra == 'dev'
28
28
  Requires-Dist: pytest>=8.0; extra == 'dev'
29
29
  Requires-Dist: ruff>=0.4; extra == 'dev'
30
30
  Requires-Dist: transformers>=5.0; extra == 'dev'
31
+ Requires-Dist: types-pyyaml>=6.0; extra == 'dev'
31
32
  Provides-Extra: studio
32
33
  Requires-Dist: fastapi>=0.110; extra == 'studio'
33
34
  Requires-Dist: uvicorn>=0.29; extra == 'studio'
@@ -174,7 +175,7 @@ To convert raw or extracted traces into standalone OpenAI-style JSONL rows that
174
175
  teich convert data --out teich-training.jsonl
175
176
  ```
176
177
 
177
- This writes standalone OpenAI-style rows with `prompt`, `messages`, `tools`, and `metadata`. Use `prepare_data()` and `mask_data()` when you want Teich to handle tokenizer-specific formatting and response-only labels.
178
+ This writes standalone OpenAI-style rows with `prompt`, `messages`, `tools`, `metadata`, and an optional captured `system` field. Use `prepare_data()` and `mask_data()` when you want Teich to handle tokenizer-specific formatting and response-only labels.
178
179
 
179
180
  ## What Teich Supports
180
181
 
@@ -264,7 +265,7 @@ publish:
264
265
 
265
266
  To run Codex on your ChatGPT subscription instead of an API key, set `agent.codex.use_host_auth: true` (Teich shares your host `codex login` across containers), and enable Codex fast mode with `model.service_tier: fast`. See [Generation](docs/generation.md#using-your-chatgpt-subscription-host-auth).
266
267
 
267
- To run Claude Code on your Claude subscription (Pro/Max), export a `claude setup-token` token as `CLAUDE_CODE_OAUTH_TOKEN` (or set `agent.claude.oauth_token`) — it activates automatically and bills your plan's rate limits, not API credits. Subscription request starts are paced 45 seconds apart by default; configure `agent.claude.subscription_request_delay_seconds` or set it to `0` to disable pacing. Batch and Studio Claude Code sessions use a real interactive PTY so readable thinking summaries can reach Claude's native trace; `agent.claude.always_thinking` and `agent.claude.show_thinking_summaries` default to `true` and can be disabled explicitly. Claude Code batch runs also support Teich-managed `agent.claude.fallback_model` retries, while `model.reasoning_effort` (`--effort`) and `agent.claude.max_thinking_tokens` apply to batch and Studio sessions. Codex supports `model.reasoning_summary: detailed` plus `model.reasoning_summaries_enabled: true` for its richest readable summaries. See [Generation](docs/generation.md#providers) and the runnable [`Codex`](examples/config.codex-reasoning.yaml) / [`Claude Code`](examples/config.claude-code-thinking.yaml) configs.
268
+ To run Claude Code on your Claude subscription (Pro/Max), export a `claude setup-token` token as `CLAUDE_CODE_OAUTH_TOKEN` (or set `agent.claude.oauth_token`) — it activates automatically and bills your plan's rate limits, not API credits. Subscription request starts are paced 45 seconds apart by default; configure `agent.claude.subscription_request_delay_seconds` or set it to `0` to disable pacing. Batch and Studio Claude Code sessions use a real interactive PTY so readable thinking summaries can reach Claude's native trace; `agent.claude.always_thinking` and `agent.claude.show_thinking_summaries` default to `true` and can be disabled explicitly. Claude Code batch runs also support Teich-managed `agent.claude.fallback_model` retries, while `model.reasoning_effort` (`--effort`) and `agent.claude.max_thinking_tokens` apply to batch and Studio sessions. Codex supports `model.reasoning_summary: detailed` plus `model.reasoning_summaries_enabled: true` for its richest readable summaries. Set `capture_harness_context.enabled: true` to make one local fake-provider preflight that records the client-visible Codex/Claude system instructions and tool schemas without using provider quota; `teich capture-context` can save the same context directly. See [Generation](docs/generation.md#providers) and the runnable [`Codex`](examples/config.codex-reasoning.yaml) / [`Claude Code`](examples/config.claude-code-thinking.yaml) configs.
268
269
 
269
270
  ## Python Entry Points
270
271
 
@@ -138,7 +138,7 @@ To convert raw or extracted traces into standalone OpenAI-style JSONL rows that
138
138
  teich convert data --out teich-training.jsonl
139
139
  ```
140
140
 
141
- This writes standalone OpenAI-style rows with `prompt`, `messages`, `tools`, and `metadata`. Use `prepare_data()` and `mask_data()` when you want Teich to handle tokenizer-specific formatting and response-only labels.
141
+ This writes standalone OpenAI-style rows with `prompt`, `messages`, `tools`, `metadata`, and an optional captured `system` field. Use `prepare_data()` and `mask_data()` when you want Teich to handle tokenizer-specific formatting and response-only labels.
142
142
 
143
143
  ## What Teich Supports
144
144
 
@@ -228,7 +228,7 @@ publish:
228
228
 
229
229
  To run Codex on your ChatGPT subscription instead of an API key, set `agent.codex.use_host_auth: true` (Teich shares your host `codex login` across containers), and enable Codex fast mode with `model.service_tier: fast`. See [Generation](docs/generation.md#using-your-chatgpt-subscription-host-auth).
230
230
 
231
- To run Claude Code on your Claude subscription (Pro/Max), export a `claude setup-token` token as `CLAUDE_CODE_OAUTH_TOKEN` (or set `agent.claude.oauth_token`) — it activates automatically and bills your plan's rate limits, not API credits. Subscription request starts are paced 45 seconds apart by default; configure `agent.claude.subscription_request_delay_seconds` or set it to `0` to disable pacing. Batch and Studio Claude Code sessions use a real interactive PTY so readable thinking summaries can reach Claude's native trace; `agent.claude.always_thinking` and `agent.claude.show_thinking_summaries` default to `true` and can be disabled explicitly. Claude Code batch runs also support Teich-managed `agent.claude.fallback_model` retries, while `model.reasoning_effort` (`--effort`) and `agent.claude.max_thinking_tokens` apply to batch and Studio sessions. Codex supports `model.reasoning_summary: detailed` plus `model.reasoning_summaries_enabled: true` for its richest readable summaries. See [Generation](docs/generation.md#providers) and the runnable [`Codex`](examples/config.codex-reasoning.yaml) / [`Claude Code`](examples/config.claude-code-thinking.yaml) configs.
231
+ To run Claude Code on your Claude subscription (Pro/Max), export a `claude setup-token` token as `CLAUDE_CODE_OAUTH_TOKEN` (or set `agent.claude.oauth_token`) — it activates automatically and bills your plan's rate limits, not API credits. Subscription request starts are paced 45 seconds apart by default; configure `agent.claude.subscription_request_delay_seconds` or set it to `0` to disable pacing. Batch and Studio Claude Code sessions use a real interactive PTY so readable thinking summaries can reach Claude's native trace; `agent.claude.always_thinking` and `agent.claude.show_thinking_summaries` default to `true` and can be disabled explicitly. Claude Code batch runs also support Teich-managed `agent.claude.fallback_model` retries, while `model.reasoning_effort` (`--effort`) and `agent.claude.max_thinking_tokens` apply to batch and Studio sessions. Codex supports `model.reasoning_summary: detailed` plus `model.reasoning_summaries_enabled: true` for its richest readable summaries. Set `capture_harness_context.enabled: true` to make one local fake-provider preflight that records the client-visible Codex/Claude system instructions and tool schemas without using provider quota; `teich capture-context` can save the same context directly. See [Generation](docs/generation.md#providers) and the runnable [`Codex`](examples/config.codex-reasoning.yaml) / [`Claude Code`](examples/config.claude-code-thinking.yaml) configs.
232
232
 
233
233
  ## Python Entry Points
234
234
 
@@ -209,4 +209,14 @@ openai_api_key: null
209
209
  # Think out loud so your problem-solving process is visible. Before each tool
210
210
  # call or edit, briefly explain what you're doing and why; after a command or
211
211
  # test runs, state what you concluded before the next step.
212
- developer_instructions: null
212
+ developer_instructions: null
213
+
214
+ # Codex/Claude Code only. Simulate one request against Teich's local fake
215
+ # provider before generation and append the client-visible harness instructions
216
+ # and tool schemas to each raw trace. This never contacts the real provider and
217
+ # never stores authentication headers or user messages. The converter exposes
218
+ # the captured instructions as a top-level `system` column.
219
+ capture_harness_context:
220
+ enabled: false
221
+ required: true
222
+ timeout_seconds: 45
@@ -87,7 +87,7 @@ Convert raw or extracted traces into standalone OpenAI-style training JSONL:
87
87
  teich convert data --out teich-training.jsonl
88
88
  ```
89
89
 
90
- Each output line contains `prompt`, `messages`, `tools`, and `metadata`. Use this when your training stack can consume standalone OpenAI-style message rows without importing Teich. Use `prepare_data()` and `mask_data()` when you want tokenizer-specific rendering and exact response-only labels.
90
+ Each output line contains `prompt`, `messages`, `tools`, `metadata`, and an optional captured `system` field. Use this when your training stack can consume standalone OpenAI-style message rows without importing Teich. Use `prepare_data()` and `mask_data()` when you want tokenizer-specific rendering and exact response-only labels.
91
91
 
92
92
  ## Anonymize
93
93
 
@@ -98,7 +98,7 @@ teich anonymize output --output output_anonymized
98
98
  teich anonymize data --in-place
99
99
  ```
100
100
 
101
- Anonymization replaces API keys, email addresses, and home-directory usernames with deterministic dummy values while preserving embedded base64 media payloads. It is a best-effort pass; review data before publishing.
101
+ Anonymization replaces known credential formats, high-confidence secret assignments, personal email addresses, and home-directory usernames with deterministic dummy values while preserving embedded base64 media payloads. Reserved example-domain addresses, known public bot addresses, provider thinking signatures, placeholders, and common public IDs are preserved to avoid corrupting training data. Assistant-authored prose, reasoning, and code examples are also preserved from credential/email heuristics; local usernames in paths are still anonymized, and structured tool inputs continue through the full scanner. Reported totals are replacement occurrences rather than estimates of unique secrets. It is a best-effort pass; review data before publishing.
102
102
 
103
103
  ## Studio
104
104
 
@@ -97,7 +97,7 @@ To turn raw or extracted traces into standalone OpenAI-style JSONL rows that do
97
97
  teich convert data --out teich-training.jsonl
98
98
  ```
99
99
 
100
- The output file is newline-delimited JSON with `prompt`, `messages`, `tools`, and `metadata` fields. Use this when another trainer already knows how to consume standalone OpenAI-style message rows. Use `prepare_data()` and `mask_data()` when you want Teich to render a specific tokenizer chat template and create exact response-only labels.
100
+ The output file is newline-delimited JSON with `prompt`, `messages`, `tools`, `metadata`, and an optional captured `system` field. Use this when another trainer already knows how to consume standalone OpenAI-style message rows. Use `prepare_data()` and `mask_data()` when you want Teich to render a specific tokenizer chat template and create exact response-only labels.
101
101
 
102
102
  ## Browser UI
103
103
 
@@ -163,9 +163,32 @@ Generated-run dataset tags are generated from provider and model. Extraction dat
163
163
  - `codex`, `pi`, `claude-code`, `hermes`, `cursor`: `agent-traces`, `format:agent-traces`, provider, model, `distillation`, `teich`
164
164
  - `chat`: `conversational`, model, `distillation`, `teich`
165
165
 
166
- If `publish.hf_token` is omitted, Teich also accepts `HF_TOKEN`, `HUGGINGFACE_HUB_TOKEN`, or `TEICH_HF_TOKEN`.
167
-
168
- ## Outputs
166
+ If `publish.hf_token` is omitted, Teich also accepts `HF_TOKEN`, `HUGGINGFACE_HUB_TOKEN`, or `TEICH_HF_TOKEN`.
167
+
168
+ ### Capture harness context without provider traffic
169
+
170
+ Codex and Claude Code add substantial client-side instructions and tool schemas before a request reaches the model. Teich can capture that client-visible context with one preflight request to a local fake provider:
171
+
172
+ ```yaml
173
+ capture_harness_context:
174
+ enabled: true
175
+ required: true
176
+ timeout_seconds: 45
177
+ ```
178
+
179
+ The preflight uses the same installed harness, model, reasoning settings, developer instructions, MCP configuration, permission mode, and an empty isolated workspace. It supports Codex's Responses wire protocol and Claude Code's Anthropic Messages protocol. It sends dummy credentials only to a random authenticated local endpoint, does not contact the real provider, and does not consume API or subscription quota. Authentication headers, user messages, and arbitrary request metadata are never retained.
180
+
181
+ Teich appends the normalized capture and its deterministic hash to every raw trace. `teich convert` exposes captured instructions as a top-level `system` field and adds request-visible tool declarations that were missing from the native trace without replacing trace-native schemas. Per-session context such as repository instructions, skills, hooks, MCP state, and recaps remains in the converted `messages` where the native trace recorded it.
182
+
183
+ To inspect a capture without running a generation batch:
184
+
185
+ ```bash
186
+ teich capture-context -c config.yaml -o harness-context.json
187
+ ```
188
+
189
+ This captures only what the harness sends over the client-provider boundary. It cannot recover provider-side instructions, server-added policy, or hidden raw chain-of-thought that never reaches the client.
190
+
191
+ ## Outputs
169
192
 
170
193
  Provider outputs:
171
194
 
@@ -47,6 +47,33 @@ trainer.train()
47
47
 
48
48
  Keep `packing=False` for this flow because packed datasets merge row boundaries before masking.
49
49
 
50
+ ## Live Gemma 4 Models
51
+
52
+ For the live Google Gemma 4 instruction checkpoints, keep the chat template
53
+ loaded from the selected model repository. The supported smoke-tested targets
54
+ are:
55
+
56
+ - `google/gemma-4-E4B-it`
57
+ - `google/gemma-4-26B-A4B-it`
58
+ - `google/gemma-4-31B-it`
59
+
60
+ Pass `chat_template_kwargs={"enable_thinking": True, "preserve_thinking": True}`
61
+ to `prepare_data()`. Do not replace `tokenizer.chat_template` unless you are
62
+ intentionally testing a maintained fork. Teich supervises the closing
63
+ `<turn|>` token for completed Gemma responses while keeping system, user, and
64
+ tool-response context masked.
65
+
66
+ `gemma4_example.py` uses the live remote template by default. Set
67
+ `CHAT_TEMPLATE_PATH` only to opt into a local custom template, and set
68
+ `MODEL_REVISION` when a reproducible non-`main` revision is required. Set
69
+ `HF_TOKEN` to an account that has accepted the Gemma repository terms when the
70
+ checkpoint is not already available through the local Hugging Face login.
71
+
72
+ Run the example in a dedicated, internally consistent Unsloth training
73
+ environment and check it with `python -m pip check` before a long run. Teich's
74
+ core environment does not pin the CUDA, PyTorch, Unsloth, and TRL stack because
75
+ those versions depend on the host GPU and CUDA runtime.
76
+
50
77
  ## What `mask_data()` Does
51
78
 
52
79
  Before `mask_data()`, the trainer dataset usually contains:
@@ -24,6 +24,12 @@ model:
24
24
  sandbox: danger-full-access
25
25
  reasoning_effort: max
26
26
 
27
+ # One local fake Anthropic Messages request records Claude Code's client-visible
28
+ # system prompt and tool declarations without consuming subscription quota.
29
+ capture_harness_context:
30
+ enabled: true
31
+ required: true
32
+
27
33
  prompts_file: prompts.jsonl
28
34
 
29
35
  output:
@@ -23,6 +23,12 @@ model:
23
23
  reasoning_summary: detailed
24
24
  reasoning_summaries_enabled: true
25
25
 
26
+ # One local fake Responses request records the client-visible Codex developer
27
+ # instructions and tool declarations without consuming provider quota.
28
+ capture_harness_context:
29
+ enabled: true
30
+ required: true
31
+
26
32
  prompts_file: prompts.jsonl
27
33
 
28
34
  output:
@@ -1,25 +1,36 @@
1
- # -*- coding: utf-8 -*-
2
- import os
3
- from unsloth import FastModel
4
- from trl import SFTConfig, SFTTrainer
5
- from teich import mask_data, prepare_data
1
+ # -*- coding: utf-8 -*-
2
+ import os
3
+
4
+ # Gemma 4's fused-loss path has had silent zero-gradient failures with
5
+ # response-only labels. Prefer the correctness path unless the caller has
6
+ # explicitly validated and enabled the fused implementation.
7
+ os.environ.setdefault("UNSLOTH_RETURN_LOGITS", "1")
8
+
9
+ from unsloth import FastModel
10
+ from trl import SFTConfig, SFTTrainer
11
+ from teich import mask_data, prepare_data
6
12
 
7
13
  MAX_SEQ_LEN = 16384
8
- MODEL_NAME = os.environ.get("MODEL_NAME", "google/gemma-4-26B-A4B-it")
9
- OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "outputs/gemma-tool-sft")
10
- HUB_REPO_ID = os.environ.get("HUB_REPO_ID") or ""
11
- HF_TOKEN = os.environ.get("HF_TOKEN", "")
12
- CHAT_TEMPLATE_PATH = os.environ.get("CHAT_TEMPLATE_PATH") or "gemma-template.jinja"
14
+ MODEL_NAME = os.environ.get("MODEL_NAME", "google/gemma-4-26B-A4B-it")
15
+ MODEL_REVISION = os.environ.get("MODEL_REVISION", "main")
16
+ OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "outputs/gemma-tool-sft")
17
+ HUB_REPO_ID = os.environ.get("HUB_REPO_ID") or ""
18
+ HF_TOKEN = os.environ.get("HF_TOKEN", "")
19
+ CHAT_TEMPLATE_PATH = os.environ.get("CHAT_TEMPLATE_PATH")
13
20
 
14
21
  model, tokenizer = FastModel.from_pretrained(
15
22
  model_name=MODEL_NAME,
16
23
  max_seq_length=MAX_SEQ_LEN,
17
24
  load_in_4bit=False,
18
- load_in_8bit=False,
19
- full_finetuning=False,
20
- )
21
-
22
- if CHAT_TEMPLATE_PATH:
25
+ load_in_8bit=False,
26
+ full_finetuning=False,
27
+ revision=MODEL_REVISION,
28
+ token=HF_TOKEN or None,
29
+ )
30
+
31
+ # By default, retain the template shipped by the selected live model revision.
32
+ # CHAT_TEMPLATE_PATH is an explicit escape hatch for controlled experiments.
33
+ if CHAT_TEMPLATE_PATH:
23
34
  with open(CHAT_TEMPLATE_PATH, "r", encoding="utf-8") as f:
24
35
  custom_chat_template = f.read()
25
36
  tokenizer.chat_template = custom_chat_template
@@ -55,16 +66,16 @@ train_dataset = prepare_data(
55
66
  tokenizer,
56
67
  split="train",
57
68
  hf_token=HF_TOKEN,
58
- chat_template_kwargs={"enable_thinking": True, "preserve_thinking": True},
59
- max_length=MAX_SEQ_LEN,
60
- drop_oversized_examples=False,
61
- tokenize=True,
62
- strict=True,
63
- )
69
+ chat_template_kwargs={"enable_thinking": True, "preserve_thinking": True},
70
+ max_length=MAX_SEQ_LEN,
71
+ oversized_policy="trim_followups",
72
+ tokenize=True,
73
+ strict=True,
74
+ )
64
75
 
65
76
  trainer = SFTTrainer(
66
- model=model,
67
- tokenizer=tokenizer,
77
+ model=model,
78
+ processing_class=tokenizer,
68
79
  train_dataset=train_dataset,
69
80
  eval_dataset=None,
70
81
  args=SFTConfig(
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "teich"
3
- version = "0.3.2"
3
+ version = "0.3.4"
4
4
  description = "Turn coding agent traces into auditable supervised fine-tuning data"
5
5
  readme = "README.md"
6
6
  license = {text = "Apache-2.0"}
@@ -32,6 +32,7 @@ dev = [
32
32
  "pytest>=8.0",
33
33
  "ruff>=0.4",
34
34
  "mypy>=1.10",
35
+ "types-PyYAML>=6.0",
35
36
  "jinja2>=3.1",
36
37
  "httpx2>=2.9",
37
38
  "transformers>=5.0",
@@ -1,6 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
- __version__ = "0.3.2"
3
+ __version__ = "0.3.4"
4
4
 
5
5
  from .audit import SFTAuditReport, audit_sft_dataset
6
6
  from .config import Config, load_config