teich 0.3.3__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {teich-0.3.3 → teich-0.3.4}/PKG-INFO +2 -2
- {teich-0.3.3 → teich-0.3.4}/docs/training.md +27 -0
- {teich-0.3.3 → teich-0.3.4}/gemma4_example.py +34 -23
- {teich-0.3.3 → teich-0.3.4}/pyproject.toml +1 -1
- {teich-0.3.3 → teich-0.3.4}/src/teich/__init__.py +1 -1
- {teich-0.3.3 → teich-0.3.4}/src/teich/formatter.py +38 -2
- {teich-0.3.3 → teich-0.3.4}/src/teich/runner.py +2 -1
- {teich-0.3.3 → teich-0.3.4}/tests/test_audit.py +10 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_formatter.py +93 -3
- {teich-0.3.3 → teich-0.3.4}/tests/test_runner.py +8 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_tokenizer_smoke.py +19 -1
- {teich-0.3.3 → teich-0.3.4}/uv.lock +1 -1
- {teich-0.3.3 → teich-0.3.4}/.github/workflows/test.yml +0 -0
- {teich-0.3.3 → teich-0.3.4}/.gitignore +0 -0
- {teich-0.3.3 → teich-0.3.4}/DOCS.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/LICENSE +0 -0
- {teich-0.3.3 → teich-0.3.4}/README.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/config.example.yaml +0 -0
- {teich-0.3.3 → teich-0.3.4}/docker/codex-runtime.Dockerfile +0 -0
- {teich-0.3.3 → teich-0.3.4}/docs/cli.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/docs/data-format.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/docs/generation.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/docs/index.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/docs/pipeline.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/docs/prepare-data.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/docs/python-api.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/docs/studio.md +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/config.claude-code-thinking.yaml +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/config.codex-reasoning.yaml +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/example_claude_code.jsonl +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/example_codex_session.jsonl +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/example_cursor_session.jsonl +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/example_droid_session.jsonl +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/example_hermes_session.jsonl +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/example_openclaw_session.jsonl +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/example_pi_session.jsonl +0 -0
- {teich-0.3.3 → teich-0.3.4}/examples/prompts.jsonl +0 -0
- {teich-0.3.3 → teich-0.3.4}/gemma-4-chat-template.jinja +0 -0
- {teich-0.3.3 → teich-0.3.4}/gemma-template.jinja +0 -0
- {teich-0.3.3 → teich-0.3.4}/new_gemma_4_template.jinja +0 -0
- {teich-0.3.3 → teich-0.3.4}/qwen3.5-chat-template.jinja +0 -0
- {teich-0.3.3 → teich-0.3.4}/qwen3.6-chat-template.jinja +0 -0
- {teich-0.3.3 → teich-0.3.4}/qwen3.6_chat_template.jinja +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/__main__.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/anonymize.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/audit.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/cli.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/codex_auth_broker.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/config.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/converter.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/extract.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/harness_capture.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/loader.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/prepare.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/__init__.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/dataset_preview.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/events.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/extraction.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/generation.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/interactive.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/project.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/server.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/app.js +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/index.html +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/style.css +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/teich.svg +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/vendor/xterm-addon-fit.js +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/vendor/xterm.css +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/studio/static/vendor/xterm.js +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/tool_schema.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/src/teich/trace_readme.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/teich_example.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/conftest.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_cli.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_codex_auth_broker.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_codex_langfuse.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_config.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_converter.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_extract_anonymize_cli.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_harness_capture.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_integration.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_langfuse_agents.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_loader.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_prepare.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_studio.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_tool_schema.py +0 -0
- {teich-0.3.3 → teich-0.3.4}/tests/test_trace_readme.py +0 -0
|
@@ -47,6 +47,33 @@ trainer.train()
|
|
|
47
47
|
|
|
48
48
|
Keep `packing=False` for this flow because packed datasets merge row boundaries before masking.
|
|
49
49
|
|
|
50
|
+
## Live Gemma 4 Models
|
|
51
|
+
|
|
52
|
+
For the live Google Gemma 4 instruction checkpoints, keep the chat template
|
|
53
|
+
loaded from the selected model repository. The supported smoke-tested targets
|
|
54
|
+
are:
|
|
55
|
+
|
|
56
|
+
- `google/gemma-4-E4B-it`
|
|
57
|
+
- `google/gemma-4-26B-A4B-it`
|
|
58
|
+
- `google/gemma-4-31B-it`
|
|
59
|
+
|
|
60
|
+
Pass `chat_template_kwargs={"enable_thinking": True, "preserve_thinking": True}`
|
|
61
|
+
to `prepare_data()`. Do not replace `tokenizer.chat_template` unless you are
|
|
62
|
+
intentionally testing a maintained fork. Teich supervises the closing
|
|
63
|
+
`<turn|>` token for completed Gemma responses while keeping system, user, and
|
|
64
|
+
tool-response context masked.
|
|
65
|
+
|
|
66
|
+
`gemma4_example.py` uses the live remote template by default. Set
|
|
67
|
+
`CHAT_TEMPLATE_PATH` only to opt into a local custom template, and set
|
|
68
|
+
`MODEL_REVISION` when a reproducible non-`main` revision is required. Set
|
|
69
|
+
`HF_TOKEN` to an account that has accepted the Gemma repository terms when the
|
|
70
|
+
checkpoint is not already available through the local Hugging Face login.
|
|
71
|
+
|
|
72
|
+
Run the example in a dedicated, internally consistent Unsloth training
|
|
73
|
+
environment and check it with `python -m pip check` before a long run. Teich's
|
|
74
|
+
core environment does not pin the CUDA, PyTorch, Unsloth, and TRL stack because
|
|
75
|
+
those versions depend on the host GPU and CUDA runtime.
|
|
76
|
+
|
|
50
77
|
## What `mask_data()` Does
|
|
51
78
|
|
|
52
79
|
Before `mask_data()`, the trainer dataset usually contains:
|
|
@@ -1,25 +1,36 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
import os
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
# Gemma 4's fused-loss path has had silent zero-gradient failures with
|
|
5
|
+
# response-only labels. Prefer the correctness path unless the caller has
|
|
6
|
+
# explicitly validated and enabled the fused implementation.
|
|
7
|
+
os.environ.setdefault("UNSLOTH_RETURN_LOGITS", "1")
|
|
8
|
+
|
|
9
|
+
from unsloth import FastModel
|
|
10
|
+
from trl import SFTConfig, SFTTrainer
|
|
11
|
+
from teich import mask_data, prepare_data
|
|
6
12
|
|
|
7
13
|
MAX_SEQ_LEN = 16384
|
|
8
|
-
MODEL_NAME = os.environ.get("MODEL_NAME", "google/gemma-4-26B-A4B-it")
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
14
|
+
MODEL_NAME = os.environ.get("MODEL_NAME", "google/gemma-4-26B-A4B-it")
|
|
15
|
+
MODEL_REVISION = os.environ.get("MODEL_REVISION", "main")
|
|
16
|
+
OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "outputs/gemma-tool-sft")
|
|
17
|
+
HUB_REPO_ID = os.environ.get("HUB_REPO_ID") or ""
|
|
18
|
+
HF_TOKEN = os.environ.get("HF_TOKEN", "")
|
|
19
|
+
CHAT_TEMPLATE_PATH = os.environ.get("CHAT_TEMPLATE_PATH")
|
|
13
20
|
|
|
14
21
|
model, tokenizer = FastModel.from_pretrained(
|
|
15
22
|
model_name=MODEL_NAME,
|
|
16
23
|
max_seq_length=MAX_SEQ_LEN,
|
|
17
24
|
load_in_4bit=False,
|
|
18
|
-
load_in_8bit=False,
|
|
19
|
-
full_finetuning=False,
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
25
|
+
load_in_8bit=False,
|
|
26
|
+
full_finetuning=False,
|
|
27
|
+
revision=MODEL_REVISION,
|
|
28
|
+
token=HF_TOKEN or None,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# By default, retain the template shipped by the selected live model revision.
|
|
32
|
+
# CHAT_TEMPLATE_PATH is an explicit escape hatch for controlled experiments.
|
|
33
|
+
if CHAT_TEMPLATE_PATH:
|
|
23
34
|
with open(CHAT_TEMPLATE_PATH, "r", encoding="utf-8") as f:
|
|
24
35
|
custom_chat_template = f.read()
|
|
25
36
|
tokenizer.chat_template = custom_chat_template
|
|
@@ -55,16 +66,16 @@ train_dataset = prepare_data(
|
|
|
55
66
|
tokenizer,
|
|
56
67
|
split="train",
|
|
57
68
|
hf_token=HF_TOKEN,
|
|
58
|
-
chat_template_kwargs={"enable_thinking": True, "preserve_thinking": True},
|
|
59
|
-
max_length=MAX_SEQ_LEN,
|
|
60
|
-
|
|
61
|
-
tokenize=True,
|
|
62
|
-
strict=True,
|
|
63
|
-
)
|
|
69
|
+
chat_template_kwargs={"enable_thinking": True, "preserve_thinking": True},
|
|
70
|
+
max_length=MAX_SEQ_LEN,
|
|
71
|
+
oversized_policy="trim_followups",
|
|
72
|
+
tokenize=True,
|
|
73
|
+
strict=True,
|
|
74
|
+
)
|
|
64
75
|
|
|
65
76
|
trainer = SFTTrainer(
|
|
66
|
-
model=model,
|
|
67
|
-
|
|
77
|
+
model=model,
|
|
78
|
+
processing_class=tokenizer,
|
|
68
79
|
train_dataset=train_dataset,
|
|
69
80
|
eval_dataset=None,
|
|
70
81
|
args=SFTConfig(
|
|
@@ -585,7 +585,10 @@ def _gemma_like_supervised_spans(text: str) -> list[tuple[int, int]]:
|
|
|
585
585
|
block_end = turn_matches[index + 1].start() if index + 1 < len(turn_matches) else len(text)
|
|
586
586
|
turn_end = text.find(_GEMMA_TURN_END, block_start, block_end)
|
|
587
587
|
if turn_end >= 0:
|
|
588
|
-
|
|
588
|
+
# The model emits <turn|> to terminate a completed response. Keep
|
|
589
|
+
# that token in the target so response-only SFT still teaches the
|
|
590
|
+
# live Gemma 4 stopping contract.
|
|
591
|
+
block_end = turn_end + len(_GEMMA_TURN_END)
|
|
589
592
|
supervised_start = block_start + len(_GEMMA_ASSISTANT_TURN_PREFIX)
|
|
590
593
|
if supervised_start < block_end:
|
|
591
594
|
supervised_spans.append((supervised_start, block_end))
|
|
@@ -1387,7 +1390,40 @@ def _select_supervised_spans(
|
|
|
1387
1390
|
selected = _subtract_spans(selected, _source_spans_for_kind(spans, _SPAN_KIND_SYSTEM))
|
|
1388
1391
|
if not train_on_developer:
|
|
1389
1392
|
selected = _subtract_spans(selected, _source_spans_for_kind(spans, _SPAN_KIND_DEVELOPER))
|
|
1390
|
-
|
|
1393
|
+
selected = _merge_spans(selected)
|
|
1394
|
+
|
|
1395
|
+
# A Gemma turn terminator is useful exactly when the same model turn still
|
|
1396
|
+
# contains an enabled training target. It is initially part of the inferred
|
|
1397
|
+
# final-answer span, so restore it when final answers are disabled but an
|
|
1398
|
+
# enabled reasoning or tool-call target remains. Conversely, do not keep a
|
|
1399
|
+
# reasoning-only row alive solely by labeling <turn|> after reasoning was
|
|
1400
|
+
# excluded.
|
|
1401
|
+
orphaned_turn_ends: list[tuple[int, int]] = []
|
|
1402
|
+
retained_turn_ends: list[tuple[int, int]] = []
|
|
1403
|
+
turn_matches = list(_GEMMA_TURN_START_PATTERN.finditer(text))
|
|
1404
|
+
for index, match in enumerate(turn_matches):
|
|
1405
|
+
if match.group(1) != "model":
|
|
1406
|
+
continue
|
|
1407
|
+
block_end = turn_matches[index + 1].start() if index + 1 < len(turn_matches) else len(text)
|
|
1408
|
+
turn_end = text.find(_GEMMA_TURN_END, match.end(), block_end)
|
|
1409
|
+
if turn_end < 0:
|
|
1410
|
+
continue
|
|
1411
|
+
turn_end_span = (turn_end, turn_end + len(_GEMMA_TURN_END))
|
|
1412
|
+
turn_end_was_supervised = any(
|
|
1413
|
+
span["start"] <= turn_end and span["end"] >= turn_end_span[1]
|
|
1414
|
+
for span in spans
|
|
1415
|
+
)
|
|
1416
|
+
has_selected_content = any(
|
|
1417
|
+
text[max(start, match.end()) : min(end, turn_end)].strip()
|
|
1418
|
+
for start, end in selected
|
|
1419
|
+
if start < turn_end and end > match.end()
|
|
1420
|
+
)
|
|
1421
|
+
if has_selected_content and turn_end_was_supervised:
|
|
1422
|
+
retained_turn_ends.append(turn_end_span)
|
|
1423
|
+
elif not has_selected_content:
|
|
1424
|
+
orphaned_turn_ends.append(turn_end_span)
|
|
1425
|
+
selected = _merge_spans(selected + retained_turn_ends)
|
|
1426
|
+
return _subtract_spans(selected, orphaned_turn_ends)
|
|
1391
1427
|
|
|
1392
1428
|
|
|
1393
1429
|
def _labels_from_offsets(
|
|
@@ -144,11 +144,12 @@ TEXT_SUBPROCESS_KWARGS: dict[str, Any] = {
|
|
|
144
144
|
"encoding": "utf-8",
|
|
145
145
|
"errors": "replace",
|
|
146
146
|
}
|
|
147
|
+
SUBPROCESS_CREATE_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0)
|
|
147
148
|
if os.name == "nt":
|
|
148
149
|
# Teich owns these background CLI processes and consumes their output via
|
|
149
150
|
# pipes. They do not need a host console, and suppressing one prevents
|
|
150
151
|
# conhost.exe accumulation in long Windows generation runs.
|
|
151
|
-
TEXT_SUBPROCESS_KWARGS["creationflags"] =
|
|
152
|
+
TEXT_SUBPROCESS_KWARGS["creationflags"] = SUBPROCESS_CREATE_NO_WINDOW
|
|
152
153
|
|
|
153
154
|
|
|
154
155
|
def _terminate_process_tree(
|
|
@@ -115,3 +115,13 @@ def test_teich_example_has_single_safe_training_flow():
|
|
|
115
115
|
assert sum(isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "train" for node in ast.walk(tree)) == 1
|
|
116
116
|
|
|
117
117
|
|
|
118
|
+
def test_gemma4_example_uses_live_remote_template_and_safe_masks():
|
|
119
|
+
source = Path("gemma4_example.py").read_text(encoding="utf-8")
|
|
120
|
+
|
|
121
|
+
assert 'os.environ.setdefault("UNSLOTH_RETURN_LOGITS", "1")' in source
|
|
122
|
+
assert 'MODEL_REVISION = os.environ.get("MODEL_REVISION", "main")' in source
|
|
123
|
+
assert 'CHAT_TEMPLATE_PATH = os.environ.get("CHAT_TEMPLATE_PATH")' in source
|
|
124
|
+
assert 'or "gemma-template.jinja"' not in source
|
|
125
|
+
assert "token=HF_TOKEN or None" in source
|
|
126
|
+
assert 'oversized_policy="trim_followups"' in source
|
|
127
|
+
assert "processing_class=tokenizer" in source
|
|
@@ -27,6 +27,8 @@ def prepare_and_mask_for_test(
|
|
|
27
27
|
tools_column="tools",
|
|
28
28
|
chat_template_kwargs=None,
|
|
29
29
|
train_on_reasoning=True,
|
|
30
|
+
train_on_final_answers=True,
|
|
31
|
+
train_on_tools=True,
|
|
30
32
|
max_length=None,
|
|
31
33
|
include_debug_columns=False,
|
|
32
34
|
drop_oversized_examples=True,
|
|
@@ -51,7 +53,15 @@ def prepare_and_mask_for_test(
|
|
|
51
53
|
args=SimpleNamespace(dataset_text_field="text", max_length=max_length if drop_oversized_examples else None, packing=False),
|
|
52
54
|
tokenizer=tokenizer,
|
|
53
55
|
)
|
|
54
|
-
trainer = mask_data(
|
|
56
|
+
trainer = mask_data(
|
|
57
|
+
trainer,
|
|
58
|
+
tokenizer=tokenizer,
|
|
59
|
+
train_on_reasoning=train_on_reasoning,
|
|
60
|
+
train_on_final_answers=train_on_final_answers,
|
|
61
|
+
train_on_tools=train_on_tools,
|
|
62
|
+
audit=False,
|
|
63
|
+
verbose=False,
|
|
64
|
+
)
|
|
55
65
|
training_data = trainer.train_dataset
|
|
56
66
|
if include_debug_columns:
|
|
57
67
|
rows = []
|
|
@@ -1101,11 +1111,91 @@ def test_prepare_and_mask_can_exclude_reasoning_from_gemma_style_supervision():
|
|
|
1101
1111
|
row = training_data[0]
|
|
1102
1112
|
assert row["text"] == "<bos><|turn>user\nhello<turn|>\n<|turn>model\n<|channel>thought\nthink\n<channel|>world<turn|>\n"
|
|
1103
1113
|
supervised_text = tokenizer.decode([token for token in row["labels"] if token != -100])
|
|
1104
|
-
assert supervised_text == "world"
|
|
1114
|
+
assert supervised_text == "world<turn|>"
|
|
1105
1115
|
masked_text = tokenizer.decode(
|
|
1106
1116
|
[token_id for token_id, label in zip(row["input_ids"], row["labels"]) if label == -100]
|
|
1107
1117
|
)
|
|
1108
1118
|
assert "<|channel>thought\nthink\n<channel|>" in masked_text
|
|
1119
|
+
assert "<turn|>" not in masked_text.rsplit("<|turn>model\n", 1)[-1]
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def test_gemma_turn_end_remains_supervised_when_only_reasoning_is_enabled():
|
|
1123
|
+
tokenizer = GemmaLikeOffsetTokenizer()
|
|
1124
|
+
dataset = Dataset.from_list(
|
|
1125
|
+
[
|
|
1126
|
+
{
|
|
1127
|
+
"messages": [
|
|
1128
|
+
{"role": "user", "content": "hello"},
|
|
1129
|
+
{"role": "assistant", "content": "world", "reasoning_content": "think"},
|
|
1130
|
+
],
|
|
1131
|
+
"tools": [],
|
|
1132
|
+
}
|
|
1133
|
+
]
|
|
1134
|
+
)
|
|
1135
|
+
|
|
1136
|
+
training_data = prepare_and_mask_for_test(
|
|
1137
|
+
dataset,
|
|
1138
|
+
tokenizer,
|
|
1139
|
+
train_on_reasoning=True,
|
|
1140
|
+
train_on_final_answers=False,
|
|
1141
|
+
)
|
|
1142
|
+
|
|
1143
|
+
row = training_data[0]
|
|
1144
|
+
supervised_text = tokenizer.decode([token for token in row["labels"] if token != -100])
|
|
1145
|
+
assert "think" in supervised_text
|
|
1146
|
+
assert "world" not in supervised_text
|
|
1147
|
+
assert supervised_text.endswith("<turn|>")
|
|
1148
|
+
|
|
1149
|
+
|
|
1150
|
+
def test_gemma_turn_end_remains_supervised_when_only_tool_call_is_enabled():
|
|
1151
|
+
class ToolTurnClosingGemmaTokenizer(GemmaLikeOffsetTokenizer):
|
|
1152
|
+
def apply_chat_template(self, *args, **kwargs):
|
|
1153
|
+
rendered = super().apply_chat_template(*args, **kwargs)
|
|
1154
|
+
if isinstance(rendered, str):
|
|
1155
|
+
rendered = rendered.replace("<tool_call|>", "<tool_call|><turn|>\n")
|
|
1156
|
+
return rendered
|
|
1157
|
+
|
|
1158
|
+
tokenizer = ToolTurnClosingGemmaTokenizer()
|
|
1159
|
+
dataset = Dataset.from_list(
|
|
1160
|
+
[
|
|
1161
|
+
{
|
|
1162
|
+
"messages": [
|
|
1163
|
+
{"role": "user", "content": "list files"},
|
|
1164
|
+
{
|
|
1165
|
+
"role": "assistant",
|
|
1166
|
+
"content": "",
|
|
1167
|
+
"tool_calls": [
|
|
1168
|
+
{
|
|
1169
|
+
"id": "call_1",
|
|
1170
|
+
"type": "function",
|
|
1171
|
+
"function": {"name": "bash", "arguments": {"command": "ls"}},
|
|
1172
|
+
}
|
|
1173
|
+
],
|
|
1174
|
+
},
|
|
1175
|
+
],
|
|
1176
|
+
"tools": [
|
|
1177
|
+
{
|
|
1178
|
+
"type": "function",
|
|
1179
|
+
"function": {"name": "bash", "parameters": {"type": "object"}},
|
|
1180
|
+
}
|
|
1181
|
+
],
|
|
1182
|
+
}
|
|
1183
|
+
]
|
|
1184
|
+
)
|
|
1185
|
+
|
|
1186
|
+
training_data = prepare_and_mask_for_test(
|
|
1187
|
+
dataset,
|
|
1188
|
+
tokenizer,
|
|
1189
|
+
train_on_reasoning=False,
|
|
1190
|
+
train_on_final_answers=False,
|
|
1191
|
+
train_on_tools=True,
|
|
1192
|
+
)
|
|
1193
|
+
|
|
1194
|
+
row = training_data[0]
|
|
1195
|
+
supervised_text = tokenizer.decode([token for token in row["labels"] if token != -100])
|
|
1196
|
+
assert "<|tool_call>call:bash" in supervised_text
|
|
1197
|
+
assert "ls" in supervised_text
|
|
1198
|
+
assert supervised_text.endswith("<turn|>")
|
|
1109
1199
|
|
|
1110
1200
|
|
|
1111
1201
|
def test_prepare_and_mask_falls_back_when_gemma_drops_marker_boundaries_with_thinking_disabled():
|
|
@@ -1139,7 +1229,7 @@ def test_prepare_and_mask_falls_back_when_gemma_drops_marker_boundaries_with_thi
|
|
|
1139
1229
|
|
|
1140
1230
|
row = training_data[0]
|
|
1141
1231
|
supervised_text = tokenizer.decode([token for token in row["labels"] if token != -100])
|
|
1142
|
-
assert supervised_text == "world"
|
|
1232
|
+
assert supervised_text == "world<turn|>"
|
|
1143
1233
|
|
|
1144
1234
|
|
|
1145
1235
|
def test_prepare_data_rejects_reserved_chat_template_kwargs():
|
|
@@ -72,6 +72,14 @@ def test_windows_owned_process_cleanup_kills_entire_tree():
|
|
|
72
72
|
process.wait.assert_called_once_with(timeout=5)
|
|
73
73
|
|
|
74
74
|
|
|
75
|
+
def test_subprocess_creation_flags_are_platform_safe():
|
|
76
|
+
assert runner_module.SUBPROCESS_CREATE_NO_WINDOW == getattr(subprocess, "CREATE_NO_WINDOW", 0)
|
|
77
|
+
if os.name == "nt":
|
|
78
|
+
assert runner_module.TEXT_SUBPROCESS_KWARGS["creationflags"] == runner_module.SUBPROCESS_CREATE_NO_WINDOW
|
|
79
|
+
else:
|
|
80
|
+
assert "creationflags" not in runner_module.TEXT_SUBPROCESS_KWARGS
|
|
81
|
+
|
|
82
|
+
|
|
75
83
|
def _filesystem_preserves_chmod(tmp_path: Path) -> bool:
|
|
76
84
|
probe = tmp_path / "chmod-probe"
|
|
77
85
|
probe.write_text("", encoding="utf-8")
|
|
@@ -11,6 +11,8 @@ from teich import mask_data, prepare_data
|
|
|
11
11
|
|
|
12
12
|
TOKENIZER_SMOKE_MODELS = [
|
|
13
13
|
pytest.param("unsloth/Qwen3.5-0.8B", {"enable_thinking": True}, id="unsloth-qwen3.5"),
|
|
14
|
+
pytest.param("google/gemma-4-E4B-it", {"enable_thinking": True}, id="gemma-4-e4b-it"),
|
|
15
|
+
pytest.param("google/gemma-4-26B-A4B-it", {"enable_thinking": True}, id="gemma-4-26b-a4b-it"),
|
|
14
16
|
pytest.param("google/gemma-4-31B-it", {"enable_thinking": True}, id="gemma-4-31b-it"),
|
|
15
17
|
]
|
|
16
18
|
|
|
@@ -71,7 +73,7 @@ def test_real_tokenizer_prepare_and_mask_tool_dataset(model_id: str, chat_templa
|
|
|
71
73
|
try:
|
|
72
74
|
tokenizer = transformers.AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
|
73
75
|
except Exception as exc:
|
|
74
|
-
pytest.
|
|
76
|
+
pytest.fail(f"Could not load tokenizer for {model_id}: {exc}")
|
|
75
77
|
|
|
76
78
|
prepared = prepare_data(
|
|
77
79
|
_tool_call_dataset(),
|
|
@@ -107,3 +109,19 @@ def test_real_tokenizer_prepare_and_mask_tool_dataset(model_id: str, chat_templa
|
|
|
107
109
|
assert "Found project files." in supervised_text
|
|
108
110
|
assert "SECRET_TOOL_OUTPUT" not in supervised_text
|
|
109
111
|
assert "SECRET_TOOL_OUTPUT" in masked_text
|
|
112
|
+
|
|
113
|
+
if model_id.startswith("google/gemma-4-"):
|
|
114
|
+
eot_token_id = tokenizer.convert_tokens_to_ids("<turn|>")
|
|
115
|
+
assert eot_token_id in supervised_ids
|
|
116
|
+
assert supervised_text.endswith("<turn|>")
|
|
117
|
+
|
|
118
|
+
source_row = _tool_call_dataset()[0]
|
|
119
|
+
generation_messages = source_row["messages"][:-1]
|
|
120
|
+
generation_prompt = tokenizer.apply_chat_template(
|
|
121
|
+
generation_messages,
|
|
122
|
+
tools=source_row["tools"],
|
|
123
|
+
tokenize=False,
|
|
124
|
+
add_generation_prompt=True,
|
|
125
|
+
**chat_template_kwargs,
|
|
126
|
+
)
|
|
127
|
+
assert generation_prompt.endswith("<|channel>thought\n")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|