teich 0.3.3__tar.gz → 0.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {teich-0.3.3 → teich-0.3.4}/PKG-INFO +2 -2
  2. {teich-0.3.3 → teich-0.3.4}/docs/training.md +27 -0
  3. {teich-0.3.3 → teich-0.3.4}/gemma4_example.py +34 -23
  4. {teich-0.3.3 → teich-0.3.4}/pyproject.toml +1 -1
  5. {teich-0.3.3 → teich-0.3.4}/src/teich/__init__.py +1 -1
  6. {teich-0.3.3 → teich-0.3.4}/src/teich/formatter.py +38 -2
  7. {teich-0.3.3 → teich-0.3.4}/src/teich/runner.py +2 -1
  8. {teich-0.3.3 → teich-0.3.4}/tests/test_audit.py +10 -0
  9. {teich-0.3.3 → teich-0.3.4}/tests/test_formatter.py +93 -3
  10. {teich-0.3.3 → teich-0.3.4}/tests/test_runner.py +8 -0
  11. {teich-0.3.3 → teich-0.3.4}/tests/test_tokenizer_smoke.py +19 -1
  12. {teich-0.3.3 → teich-0.3.4}/uv.lock +1 -1
  13. {teich-0.3.3 → teich-0.3.4}/.github/workflows/test.yml +0 -0
  14. {teich-0.3.3 → teich-0.3.4}/.gitignore +0 -0
  15. {teich-0.3.3 → teich-0.3.4}/DOCS.md +0 -0
  16. {teich-0.3.3 → teich-0.3.4}/LICENSE +0 -0
  17. {teich-0.3.3 → teich-0.3.4}/README.md +0 -0
  18. {teich-0.3.3 → teich-0.3.4}/config.example.yaml +0 -0
  19. {teich-0.3.3 → teich-0.3.4}/docker/codex-runtime.Dockerfile +0 -0
  20. {teich-0.3.3 → teich-0.3.4}/docs/cli.md +0 -0
  21. {teich-0.3.3 → teich-0.3.4}/docs/data-format.md +0 -0
  22. {teich-0.3.3 → teich-0.3.4}/docs/generation.md +0 -0
  23. {teich-0.3.3 → teich-0.3.4}/docs/index.md +0 -0
  24. {teich-0.3.3 → teich-0.3.4}/docs/pipeline.md +0 -0
  25. {teich-0.3.3 → teich-0.3.4}/docs/prepare-data.md +0 -0
  26. {teich-0.3.3 → teich-0.3.4}/docs/python-api.md +0 -0
  27. {teich-0.3.3 → teich-0.3.4}/docs/studio.md +0 -0
  28. {teich-0.3.3 → teich-0.3.4}/examples/config.claude-code-thinking.yaml +0 -0
  29. {teich-0.3.3 → teich-0.3.4}/examples/config.codex-reasoning.yaml +0 -0
  30. {teich-0.3.3 → teich-0.3.4}/examples/example_claude_code.jsonl +0 -0
  31. {teich-0.3.3 → teich-0.3.4}/examples/example_codex_session.jsonl +0 -0
  32. {teich-0.3.3 → teich-0.3.4}/examples/example_cursor_session.jsonl +0 -0
  33. {teich-0.3.3 → teich-0.3.4}/examples/example_droid_session.jsonl +0 -0
  34. {teich-0.3.3 → teich-0.3.4}/examples/example_hermes_session.jsonl +0 -0
  35. {teich-0.3.3 → teich-0.3.4}/examples/example_openclaw_session.jsonl +0 -0
  36. {teich-0.3.3 → teich-0.3.4}/examples/example_pi_session.jsonl +0 -0
  37. {teich-0.3.3 → teich-0.3.4}/examples/prompts.jsonl +0 -0
  38. {teich-0.3.3 → teich-0.3.4}/gemma-4-chat-template.jinja +0 -0
  39. {teich-0.3.3 → teich-0.3.4}/gemma-template.jinja +0 -0
  40. {teich-0.3.3 → teich-0.3.4}/new_gemma_4_template.jinja +0 -0
  41. {teich-0.3.3 → teich-0.3.4}/qwen3.5-chat-template.jinja +0 -0
  42. {teich-0.3.3 → teich-0.3.4}/qwen3.6-chat-template.jinja +0 -0
  43. {teich-0.3.3 → teich-0.3.4}/qwen3.6_chat_template.jinja +0 -0
  44. {teich-0.3.3 → teich-0.3.4}/src/teich/__main__.py +0 -0
  45. {teich-0.3.3 → teich-0.3.4}/src/teich/anonymize.py +0 -0
  46. {teich-0.3.3 → teich-0.3.4}/src/teich/audit.py +0 -0
  47. {teich-0.3.3 → teich-0.3.4}/src/teich/cli.py +0 -0
  48. {teich-0.3.3 → teich-0.3.4}/src/teich/codex_auth_broker.py +0 -0
  49. {teich-0.3.3 → teich-0.3.4}/src/teich/config.py +0 -0
  50. {teich-0.3.3 → teich-0.3.4}/src/teich/converter.py +0 -0
  51. {teich-0.3.3 → teich-0.3.4}/src/teich/extract.py +0 -0
  52. {teich-0.3.3 → teich-0.3.4}/src/teich/harness_capture.py +0 -0
  53. {teich-0.3.3 → teich-0.3.4}/src/teich/loader.py +0 -0
  54. {teich-0.3.3 → teich-0.3.4}/src/teich/prepare.py +0 -0
  55. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/__init__.py +0 -0
  56. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/dataset_preview.py +0 -0
  57. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/events.py +0 -0
  58. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/extraction.py +0 -0
  59. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/generation.py +0 -0
  60. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/interactive.py +0 -0
  61. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/project.py +0 -0
  62. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/server.py +0 -0
  63. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/app.js +0 -0
  64. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/index.html +0 -0
  65. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/style.css +0 -0
  66. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/teich.svg +0 -0
  67. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/vendor/xterm-addon-fit.js +0 -0
  68. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/vendor/xterm.css +0 -0
  69. {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/vendor/xterm.js +0 -0
  70. {teich-0.3.3 → teich-0.3.4}/src/teich/tool_schema.py +0 -0
  71. {teich-0.3.3 → teich-0.3.4}/src/teich/trace_readme.py +0 -0
  72. {teich-0.3.3 → teich-0.3.4}/teich_example.py +0 -0
  73. {teich-0.3.3 → teich-0.3.4}/tests/conftest.py +0 -0
  74. {teich-0.3.3 → teich-0.3.4}/tests/test_cli.py +0 -0
  75. {teich-0.3.3 → teich-0.3.4}/tests/test_codex_auth_broker.py +0 -0
  76. {teich-0.3.3 → teich-0.3.4}/tests/test_codex_langfuse.py +0 -0
  77. {teich-0.3.3 → teich-0.3.4}/tests/test_config.py +0 -0
  78. {teich-0.3.3 → teich-0.3.4}/tests/test_converter.py +0 -0
  79. {teich-0.3.3 → teich-0.3.4}/tests/test_extract_anonymize_cli.py +0 -0
  80. {teich-0.3.3 → teich-0.3.4}/tests/test_harness_capture.py +0 -0
  81. {teich-0.3.3 → teich-0.3.4}/tests/test_integration.py +0 -0
  82. {teich-0.3.3 → teich-0.3.4}/tests/test_langfuse_agents.py +0 -0
  83. {teich-0.3.3 → teich-0.3.4}/tests/test_loader.py +0 -0
  84. {teich-0.3.3 → teich-0.3.4}/tests/test_prepare.py +0 -0
  85. {teich-0.3.3 → teich-0.3.4}/tests/test_studio.py +0 -0
  86. {teich-0.3.3 → teich-0.3.4}/tests/test_tool_schema.py +0 -0
  87. {teich-0.3.3 → teich-0.3.4}/tests/test_trace_readme.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: teich
3
- Version: 0.3.3
3
+ Version: 0.3.4
4
4
  Summary: Turn coding agent traces into auditable supervised fine-tuning data
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -47,6 +47,33 @@ trainer.train()
47
47
 
48
48
  Keep `packing=False` for this flow because packed datasets merge row boundaries before masking.
49
49
 
50
+ ## Live Gemma 4 Models
51
+
52
+ For the live Google Gemma 4 instruction checkpoints, keep the chat template
53
+ loaded from the selected model repository. The supported smoke-tested targets
54
+ are:
55
+
56
+ - `google/gemma-4-E4B-it`
57
+ - `google/gemma-4-26B-A4B-it`
58
+ - `google/gemma-4-31B-it`
59
+
60
+ Pass `chat_template_kwargs={"enable_thinking": True, "preserve_thinking": True}`
61
+ to `prepare_data()`. Do not replace `tokenizer.chat_template` unless you are
62
+ intentionally testing a maintained fork. Teich supervises the closing
63
+ `<turn|>` token for completed Gemma responses while keeping system, user, and
64
+ tool-response context masked.
65
+
66
+ `gemma4_example.py` uses the live remote template by default. Set
67
+ `CHAT_TEMPLATE_PATH` only to opt into a local custom template, and set
68
+ `MODEL_REVISION` when a reproducible non-`main` revision is required. Set
69
+ `HF_TOKEN` to an account that has accepted the Gemma repository terms when the
70
+ checkpoint is not already available through the local Hugging Face login.
71
+
72
+ Run the example in a dedicated, internally consistent Unsloth training
73
+ environment and check it with `python -m pip check` before a long run. Teich's
74
+ core environment does not pin the CUDA, PyTorch, Unsloth, and TRL stack because
75
+ those versions depend on the host GPU and CUDA runtime.
76
+
50
77
  ## What `mask_data()` Does
51
78
 
52
79
  Before `mask_data()`, the trainer dataset usually contains:
@@ -1,25 +1,36 @@
1
- # -*- coding: utf-8 -*-
2
- import os
3
- from unsloth import FastModel
4
- from trl import SFTConfig, SFTTrainer
5
- from teich import mask_data, prepare_data
1
+ # -*- coding: utf-8 -*-
2
+ import os
3
+
4
+ # Gemma 4's fused-loss path has had silent zero-gradient failures with
5
+ # response-only labels. Prefer the correctness path unless the caller has
6
+ # explicitly validated and enabled the fused implementation.
7
+ os.environ.setdefault("UNSLOTH_RETURN_LOGITS", "1")
8
+
9
+ from unsloth import FastModel
10
+ from trl import SFTConfig, SFTTrainer
11
+ from teich import mask_data, prepare_data
6
12
 
7
13
  MAX_SEQ_LEN = 16384
8
- MODEL_NAME = os.environ.get("MODEL_NAME", "google/gemma-4-26B-A4B-it")
9
- OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "outputs/gemma-tool-sft")
10
- HUB_REPO_ID = os.environ.get("HUB_REPO_ID") or ""
11
- HF_TOKEN = os.environ.get("HF_TOKEN", "")
12
- CHAT_TEMPLATE_PATH = os.environ.get("CHAT_TEMPLATE_PATH") or "gemma-template.jinja"
14
+ MODEL_NAME = os.environ.get("MODEL_NAME", "google/gemma-4-26B-A4B-it")
15
+ MODEL_REVISION = os.environ.get("MODEL_REVISION", "main")
16
+ OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "outputs/gemma-tool-sft")
17
+ HUB_REPO_ID = os.environ.get("HUB_REPO_ID") or ""
18
+ HF_TOKEN = os.environ.get("HF_TOKEN", "")
19
+ CHAT_TEMPLATE_PATH = os.environ.get("CHAT_TEMPLATE_PATH")
13
20
 
14
21
  model, tokenizer = FastModel.from_pretrained(
15
22
  model_name=MODEL_NAME,
16
23
  max_seq_length=MAX_SEQ_LEN,
17
24
  load_in_4bit=False,
18
- load_in_8bit=False,
19
- full_finetuning=False,
20
- )
21
-
22
- if CHAT_TEMPLATE_PATH:
25
+ load_in_8bit=False,
26
+ full_finetuning=False,
27
+ revision=MODEL_REVISION,
28
+ token=HF_TOKEN or None,
29
+ )
30
+
31
+ # By default, retain the template shipped by the selected live model revision.
32
+ # CHAT_TEMPLATE_PATH is an explicit escape hatch for controlled experiments.
33
+ if CHAT_TEMPLATE_PATH:
23
34
  with open(CHAT_TEMPLATE_PATH, "r", encoding="utf-8") as f:
24
35
  custom_chat_template = f.read()
25
36
  tokenizer.chat_template = custom_chat_template
@@ -55,16 +66,16 @@ train_dataset = prepare_data(
55
66
  tokenizer,
56
67
  split="train",
57
68
  hf_token=HF_TOKEN,
58
- chat_template_kwargs={"enable_thinking": True, "preserve_thinking": True},
59
- max_length=MAX_SEQ_LEN,
60
- drop_oversized_examples=False,
61
- tokenize=True,
62
- strict=True,
63
- )
69
+ chat_template_kwargs={"enable_thinking": True, "preserve_thinking": True},
70
+ max_length=MAX_SEQ_LEN,
71
+ oversized_policy="trim_followups",
72
+ tokenize=True,
73
+ strict=True,
74
+ )
64
75
 
65
76
  trainer = SFTTrainer(
66
- model=model,
67
- tokenizer=tokenizer,
77
+ model=model,
78
+ processing_class=tokenizer,
68
79
  train_dataset=train_dataset,
69
80
  eval_dataset=None,
70
81
  args=SFTConfig(
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "teich"
3
- version = "0.3.3"
3
+ version = "0.3.4"
4
4
  description = "Turn coding agent traces into auditable supervised fine-tuning data"
5
5
  readme = "README.md"
6
6
  license = {text = "Apache-2.0"}
@@ -1,6 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
- __version__ = "0.3.3"
3
+ __version__ = "0.3.4"
4
4
 
5
5
  from .audit import SFTAuditReport, audit_sft_dataset
6
6
  from .config import Config, load_config
@@ -585,7 +585,10 @@ def _gemma_like_supervised_spans(text: str) -> list[tuple[int, int]]:
585
585
  block_end = turn_matches[index + 1].start() if index + 1 < len(turn_matches) else len(text)
586
586
  turn_end = text.find(_GEMMA_TURN_END, block_start, block_end)
587
587
  if turn_end >= 0:
588
- block_end = turn_end
588
+ # The model emits <turn|> to terminate a completed response. Keep
589
+ # that token in the target so response-only SFT still teaches the
590
+ # live Gemma 4 stopping contract.
591
+ block_end = turn_end + len(_GEMMA_TURN_END)
589
592
  supervised_start = block_start + len(_GEMMA_ASSISTANT_TURN_PREFIX)
590
593
  if supervised_start < block_end:
591
594
  supervised_spans.append((supervised_start, block_end))
@@ -1387,7 +1390,40 @@ def _select_supervised_spans(
1387
1390
  selected = _subtract_spans(selected, _source_spans_for_kind(spans, _SPAN_KIND_SYSTEM))
1388
1391
  if not train_on_developer:
1389
1392
  selected = _subtract_spans(selected, _source_spans_for_kind(spans, _SPAN_KIND_DEVELOPER))
1390
- return _merge_spans(selected)
1393
+ selected = _merge_spans(selected)
1394
+
1395
+ # A Gemma turn terminator is useful exactly when the same model turn still
1396
+ # contains an enabled training target. It is initially part of the inferred
1397
+ # final-answer span, so restore it when final answers are disabled but an
1398
+ # enabled reasoning or tool-call target remains. Conversely, do not keep a
1399
+ # reasoning-only row alive solely by labeling <turn|> after reasoning was
1400
+ # excluded.
1401
+ orphaned_turn_ends: list[tuple[int, int]] = []
1402
+ retained_turn_ends: list[tuple[int, int]] = []
1403
+ turn_matches = list(_GEMMA_TURN_START_PATTERN.finditer(text))
1404
+ for index, match in enumerate(turn_matches):
1405
+ if match.group(1) != "model":
1406
+ continue
1407
+ block_end = turn_matches[index + 1].start() if index + 1 < len(turn_matches) else len(text)
1408
+ turn_end = text.find(_GEMMA_TURN_END, match.end(), block_end)
1409
+ if turn_end < 0:
1410
+ continue
1411
+ turn_end_span = (turn_end, turn_end + len(_GEMMA_TURN_END))
1412
+ turn_end_was_supervised = any(
1413
+ span["start"] <= turn_end and span["end"] >= turn_end_span[1]
1414
+ for span in spans
1415
+ )
1416
+ has_selected_content = any(
1417
+ text[max(start, match.end()) : min(end, turn_end)].strip()
1418
+ for start, end in selected
1419
+ if start < turn_end and end > match.end()
1420
+ )
1421
+ if has_selected_content and turn_end_was_supervised:
1422
+ retained_turn_ends.append(turn_end_span)
1423
+ elif not has_selected_content:
1424
+ orphaned_turn_ends.append(turn_end_span)
1425
+ selected = _merge_spans(selected + retained_turn_ends)
1426
+ return _subtract_spans(selected, orphaned_turn_ends)
1391
1427
 
1392
1428
 
1393
1429
  def _labels_from_offsets(
@@ -144,11 +144,12 @@ TEXT_SUBPROCESS_KWARGS: dict[str, Any] = {
144
144
  "encoding": "utf-8",
145
145
  "errors": "replace",
146
146
  }
147
+ SUBPROCESS_CREATE_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0)
147
148
  if os.name == "nt":
148
149
  # Teich owns these background CLI processes and consumes their output via
149
150
  # pipes. They do not need a host console, and suppressing one prevents
150
151
  # conhost.exe accumulation in long Windows generation runs.
151
- TEXT_SUBPROCESS_KWARGS["creationflags"] = subprocess.CREATE_NO_WINDOW
152
+ TEXT_SUBPROCESS_KWARGS["creationflags"] = SUBPROCESS_CREATE_NO_WINDOW
152
153
 
153
154
 
154
155
  def _terminate_process_tree(
@@ -115,3 +115,13 @@ def test_teich_example_has_single_safe_training_flow():
115
115
  assert sum(isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "train" for node in ast.walk(tree)) == 1
116
116
 
117
117
 
118
+ def test_gemma4_example_uses_live_remote_template_and_safe_masks():
119
+ source = Path("gemma4_example.py").read_text(encoding="utf-8")
120
+
121
+ assert 'os.environ.setdefault("UNSLOTH_RETURN_LOGITS", "1")' in source
122
+ assert 'MODEL_REVISION = os.environ.get("MODEL_REVISION", "main")' in source
123
+ assert 'CHAT_TEMPLATE_PATH = os.environ.get("CHAT_TEMPLATE_PATH")' in source
124
+ assert 'or "gemma-template.jinja"' not in source
125
+ assert "token=HF_TOKEN or None" in source
126
+ assert 'oversized_policy="trim_followups"' in source
127
+ assert "processing_class=tokenizer" in source
@@ -27,6 +27,8 @@ def prepare_and_mask_for_test(
27
27
  tools_column="tools",
28
28
  chat_template_kwargs=None,
29
29
  train_on_reasoning=True,
30
+ train_on_final_answers=True,
31
+ train_on_tools=True,
30
32
  max_length=None,
31
33
  include_debug_columns=False,
32
34
  drop_oversized_examples=True,
@@ -51,7 +53,15 @@ def prepare_and_mask_for_test(
51
53
  args=SimpleNamespace(dataset_text_field="text", max_length=max_length if drop_oversized_examples else None, packing=False),
52
54
  tokenizer=tokenizer,
53
55
  )
54
- trainer = mask_data(trainer, tokenizer=tokenizer, train_on_reasoning=train_on_reasoning, audit=False, verbose=False)
56
+ trainer = mask_data(
57
+ trainer,
58
+ tokenizer=tokenizer,
59
+ train_on_reasoning=train_on_reasoning,
60
+ train_on_final_answers=train_on_final_answers,
61
+ train_on_tools=train_on_tools,
62
+ audit=False,
63
+ verbose=False,
64
+ )
55
65
  training_data = trainer.train_dataset
56
66
  if include_debug_columns:
57
67
  rows = []
@@ -1101,11 +1111,91 @@ def test_prepare_and_mask_can_exclude_reasoning_from_gemma_style_supervision():
1101
1111
  row = training_data[0]
1102
1112
  assert row["text"] == "<bos><|turn>user\nhello<turn|>\n<|turn>model\n<|channel>thought\nthink\n<channel|>world<turn|>\n"
1103
1113
  supervised_text = tokenizer.decode([token for token in row["labels"] if token != -100])
1104
- assert supervised_text == "world"
1114
+ assert supervised_text == "world<turn|>"
1105
1115
  masked_text = tokenizer.decode(
1106
1116
  [token_id for token_id, label in zip(row["input_ids"], row["labels"]) if label == -100]
1107
1117
  )
1108
1118
  assert "<|channel>thought\nthink\n<channel|>" in masked_text
1119
+ assert "<turn|>" not in masked_text.rsplit("<|turn>model\n", 1)[-1]
1120
+
1121
+
1122
+ def test_gemma_turn_end_remains_supervised_when_only_reasoning_is_enabled():
1123
+ tokenizer = GemmaLikeOffsetTokenizer()
1124
+ dataset = Dataset.from_list(
1125
+ [
1126
+ {
1127
+ "messages": [
1128
+ {"role": "user", "content": "hello"},
1129
+ {"role": "assistant", "content": "world", "reasoning_content": "think"},
1130
+ ],
1131
+ "tools": [],
1132
+ }
1133
+ ]
1134
+ )
1135
+
1136
+ training_data = prepare_and_mask_for_test(
1137
+ dataset,
1138
+ tokenizer,
1139
+ train_on_reasoning=True,
1140
+ train_on_final_answers=False,
1141
+ )
1142
+
1143
+ row = training_data[0]
1144
+ supervised_text = tokenizer.decode([token for token in row["labels"] if token != -100])
1145
+ assert "think" in supervised_text
1146
+ assert "world" not in supervised_text
1147
+ assert supervised_text.endswith("<turn|>")
1148
+
1149
+
1150
+ def test_gemma_turn_end_remains_supervised_when_only_tool_call_is_enabled():
1151
+ class ToolTurnClosingGemmaTokenizer(GemmaLikeOffsetTokenizer):
1152
+ def apply_chat_template(self, *args, **kwargs):
1153
+ rendered = super().apply_chat_template(*args, **kwargs)
1154
+ if isinstance(rendered, str):
1155
+ rendered = rendered.replace("<tool_call|>", "<tool_call|><turn|>\n")
1156
+ return rendered
1157
+
1158
+ tokenizer = ToolTurnClosingGemmaTokenizer()
1159
+ dataset = Dataset.from_list(
1160
+ [
1161
+ {
1162
+ "messages": [
1163
+ {"role": "user", "content": "list files"},
1164
+ {
1165
+ "role": "assistant",
1166
+ "content": "",
1167
+ "tool_calls": [
1168
+ {
1169
+ "id": "call_1",
1170
+ "type": "function",
1171
+ "function": {"name": "bash", "arguments": {"command": "ls"}},
1172
+ }
1173
+ ],
1174
+ },
1175
+ ],
1176
+ "tools": [
1177
+ {
1178
+ "type": "function",
1179
+ "function": {"name": "bash", "parameters": {"type": "object"}},
1180
+ }
1181
+ ],
1182
+ }
1183
+ ]
1184
+ )
1185
+
1186
+ training_data = prepare_and_mask_for_test(
1187
+ dataset,
1188
+ tokenizer,
1189
+ train_on_reasoning=False,
1190
+ train_on_final_answers=False,
1191
+ train_on_tools=True,
1192
+ )
1193
+
1194
+ row = training_data[0]
1195
+ supervised_text = tokenizer.decode([token for token in row["labels"] if token != -100])
1196
+ assert "<|tool_call>call:bash" in supervised_text
1197
+ assert "ls" in supervised_text
1198
+ assert supervised_text.endswith("<turn|>")
1109
1199
 
1110
1200
 
1111
1201
  def test_prepare_and_mask_falls_back_when_gemma_drops_marker_boundaries_with_thinking_disabled():
@@ -1139,7 +1229,7 @@ def test_prepare_and_mask_falls_back_when_gemma_drops_marker_boundaries_with_thi
1139
1229
 
1140
1230
  row = training_data[0]
1141
1231
  supervised_text = tokenizer.decode([token for token in row["labels"] if token != -100])
1142
- assert supervised_text == "world"
1232
+ assert supervised_text == "world<turn|>"
1143
1233
 
1144
1234
 
1145
1235
  def test_prepare_data_rejects_reserved_chat_template_kwargs():
@@ -72,6 +72,14 @@ def test_windows_owned_process_cleanup_kills_entire_tree():
72
72
  process.wait.assert_called_once_with(timeout=5)
73
73
 
74
74
 
75
+ def test_subprocess_creation_flags_are_platform_safe():
76
+ assert runner_module.SUBPROCESS_CREATE_NO_WINDOW == getattr(subprocess, "CREATE_NO_WINDOW", 0)
77
+ if os.name == "nt":
78
+ assert runner_module.TEXT_SUBPROCESS_KWARGS["creationflags"] == runner_module.SUBPROCESS_CREATE_NO_WINDOW
79
+ else:
80
+ assert "creationflags" not in runner_module.TEXT_SUBPROCESS_KWARGS
81
+
82
+
75
83
  def _filesystem_preserves_chmod(tmp_path: Path) -> bool:
76
84
  probe = tmp_path / "chmod-probe"
77
85
  probe.write_text("", encoding="utf-8")
@@ -11,6 +11,8 @@ from teich import mask_data, prepare_data
11
11
 
12
12
  TOKENIZER_SMOKE_MODELS = [
13
13
  pytest.param("unsloth/Qwen3.5-0.8B", {"enable_thinking": True}, id="unsloth-qwen3.5"),
14
+ pytest.param("google/gemma-4-E4B-it", {"enable_thinking": True}, id="gemma-4-e4b-it"),
15
+ pytest.param("google/gemma-4-26B-A4B-it", {"enable_thinking": True}, id="gemma-4-26b-a4b-it"),
14
16
  pytest.param("google/gemma-4-31B-it", {"enable_thinking": True}, id="gemma-4-31b-it"),
15
17
  ]
16
18
 
@@ -71,7 +73,7 @@ def test_real_tokenizer_prepare_and_mask_tool_dataset(model_id: str, chat_templa
71
73
  try:
72
74
  tokenizer = transformers.AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
73
75
  except Exception as exc:
74
- pytest.skip(f"Could not load tokenizer for {model_id}: {exc}")
76
+ pytest.fail(f"Could not load tokenizer for {model_id}: {exc}")
75
77
 
76
78
  prepared = prepare_data(
77
79
  _tool_call_dataset(),
@@ -107,3 +109,19 @@ def test_real_tokenizer_prepare_and_mask_tool_dataset(model_id: str, chat_templa
107
109
  assert "Found project files." in supervised_text
108
110
  assert "SECRET_TOOL_OUTPUT" not in supervised_text
109
111
  assert "SECRET_TOOL_OUTPUT" in masked_text
112
+
113
+ if model_id.startswith("google/gemma-4-"):
114
+ eot_token_id = tokenizer.convert_tokens_to_ids("<turn|>")
115
+ assert eot_token_id in supervised_ids
116
+ assert supervised_text.endswith("<turn|>")
117
+
118
+ source_row = _tool_call_dataset()[0]
119
+ generation_messages = source_row["messages"][:-1]
120
+ generation_prompt = tokenizer.apply_chat_template(
121
+ generation_messages,
122
+ tools=source_row["tools"],
123
+ tokenize=False,
124
+ add_generation_prompt=True,
125
+ **chat_template_kwargs,
126
+ )
127
+ assert generation_prompt.endswith("<|channel>thought\n")
@@ -2123,7 +2123,7 @@ wheels = [
2123
2123
 
2124
2124
  [[package]]
2125
2125
  name = "teich"
2126
- version = "0.3.3"
2126
+ version = "0.3.4"
2127
2127
  source = { editable = "." }
2128
2128
  dependencies = [
2129
2129
  { name = "datasets" },
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes