tlmtc 0.2.2__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.github/workflows/release.yml +2 -2
- {tlmtc-0.2.2 → tlmtc-0.3.0}/PKG-INFO +7 -7
- {tlmtc-0.2.2 → tlmtc-0.3.0}/README.md +6 -6
- {tlmtc-0.2.2 → tlmtc-0.3.0}/pyproject.toml +1 -1
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/api.py +93 -34
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/cli.py +51 -16
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/data_pipeline.py +18 -8
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/data_preparation.py +22 -11
- tlmtc-0.3.0/src/tlmtc/distributed.py +149 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/finetune_pipeline.py +113 -115
- tlmtc-0.3.0/src/tlmtc/hpo.py +203 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/meta.py +2 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/paths.py +27 -17
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/prediction.py +4 -2
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/runtime_output.py +35 -9
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/settings.py +14 -8
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/training.py +147 -22
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/integration/test_predict_tlmtc.py +17 -12
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/integration/test_train_tlmtc.py +11 -8
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_api.py +260 -14
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_cli.py +87 -35
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_data_pipeline.py +68 -31
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_data_preparation.py +66 -6
- tlmtc-0.3.0/tests/unit/test_distributed.py +365 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_evaluation_pipeline.py +1 -1
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_finetune_pipeline.py +162 -127
- tlmtc-0.3.0/tests/unit/test_hpo.py +263 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_meta.py +3 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_paths.py +74 -51
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_prediction.py +3 -1
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_runtime_output.py +59 -7
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_settings.py +99 -40
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_training.py +107 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/uv.lock +147 -132
- tlmtc-0.2.2/src/tlmtc/hpo.py +0 -157
- tlmtc-0.2.2/tests/unit/test_hpo.py +0 -185
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.github/ISSUE_TEMPLATE/implementation_task.yml +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.github/dependabot.yml +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.github/labeler.yml +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.github/workflows/ci.yml +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.github/workflows/pr-labeler.yml +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/.gitignore +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/CONTRIBUTING.md +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/LICENSE.md +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/SECURITY.md +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/assets/tlmtc-bright.png +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/assets/tlmtc-dark.png +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/examples/paired_example.csv +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/examples/paired_example_unlabeled.csv +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/__init__.py +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/__main__.py +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/data_contracts.py +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/evaluation.py +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/evaluation_pipeline.py +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/src/tlmtc/reporting.py +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/data/tiny_tokenizer/config.json +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/data/tiny_tokenizer/vocab.txt +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_data_contracts.py +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_evaluation.py +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_public_api.py +0 -0
- {tlmtc-0.2.2 → tlmtc-0.3.0}/tests/unit/test_reporting.py +0 -0
|
@@ -110,7 +110,7 @@ jobs:
|
|
|
110
110
|
run: uvx --from twine twine check dist/*
|
|
111
111
|
|
|
112
112
|
- name: Upload package distributions
|
|
113
|
-
uses: actions/upload-artifact@
|
|
113
|
+
uses: actions/upload-artifact@v7
|
|
114
114
|
with:
|
|
115
115
|
name: python-package-distributions
|
|
116
116
|
path: dist/*
|
|
@@ -132,7 +132,7 @@ jobs:
|
|
|
132
132
|
|
|
133
133
|
steps:
|
|
134
134
|
- name: Download package distributions
|
|
135
|
-
uses: actions/download-artifact@
|
|
135
|
+
uses: actions/download-artifact@v8
|
|
136
136
|
with:
|
|
137
137
|
name: python-package-distributions
|
|
138
138
|
path: dist/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tlmtc
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Production Workflows for Transformer-based Multi-label Text Classification
|
|
5
5
|
Project-URL: Homepage, https://github.com/saschagobel/tlmtc
|
|
6
6
|
Project-URL: Repository, https://github.com/saschagobel/tlmtc
|
|
@@ -82,7 +82,7 @@ In applied settings, text classification is rarely a simple single-label task or
|
|
|
82
82
|
- End-to-end prediction workflow that reloads trained models, metadata, labels, and thresholds automatically
|
|
83
83
|
- Automatic persistence and reuse of data splits, Optuna studies, trained models, thresholds, and run metadata
|
|
84
84
|
- Highly configurable through a small workflow-oriented Python API and CLI
|
|
85
|
-
- CPU and multi-GPU training and prediction support
|
|
85
|
+
- CPU and DDP-safe multi-GPU training and prediction support
|
|
86
86
|
|
|
87
87
|
## Installation
|
|
88
88
|
|
|
@@ -177,7 +177,7 @@ Fine-tune a model:
|
|
|
177
177
|
from tlmtc import train_tlmtc
|
|
178
178
|
|
|
179
179
|
train_tlmtc(
|
|
180
|
-
"paired_example.csv",
|
|
180
|
+
labeled_data="paired_example.csv",
|
|
181
181
|
target_name="Requirements Evidence Alignment",
|
|
182
182
|
checkpoint="google/bert_uncased_L-2_H-128_A-2",
|
|
183
183
|
tuning_trials=5,
|
|
@@ -205,14 +205,14 @@ predict_tlmtc(
|
|
|
205
205
|
|
|
206
206
|
```bash
|
|
207
207
|
tlmtc train \
|
|
208
|
-
--
|
|
208
|
+
--labeled-data paired_example.csv \
|
|
209
209
|
--target-name "Requirements Evidence Alignment" \
|
|
210
210
|
--checkpoint google/bert_uncased_L-2_H-128_A-2 \
|
|
211
211
|
--tuning-trials 5 \
|
|
212
212
|
--use-cpu
|
|
213
213
|
|
|
214
214
|
tlmtc predict \
|
|
215
|
-
--
|
|
215
|
+
--unlabeled-data paired_example_unlabeled.csv \
|
|
216
216
|
--use-cpu
|
|
217
217
|
```
|
|
218
218
|
|
|
@@ -221,7 +221,7 @@ tlmtc predict \
|
|
|
221
221
|
<details>
|
|
222
222
|
<summary><strong>Try your own data</strong></summary>
|
|
223
223
|
|
|
224
|
-
Your
|
|
224
|
+
Your labeled data must include:
|
|
225
225
|
|
|
226
226
|
- a `text` column
|
|
227
227
|
- at least two binary `label_`-prefixed columns
|
|
@@ -230,7 +230,7 @@ Add a `text_pair` column for paired-input classification. **tlmtc** detects and
|
|
|
230
230
|
|
|
231
231
|
Add a `split_group` column when semantically related rows must not cross train, validation, and test splits.
|
|
232
232
|
|
|
233
|
-
For prediction, provide
|
|
233
|
+
For prediction, provide unlabeled data with the same input columns used during training. For a paired-text model, this means both `text` and `text_pair`.
|
|
234
234
|
|
|
235
235
|
</details>
|
|
236
236
|
|
|
@@ -37,7 +37,7 @@ In applied settings, text classification is rarely a simple single-label task or
|
|
|
37
37
|
- End-to-end prediction workflow that reloads trained models, metadata, labels, and thresholds automatically
|
|
38
38
|
- Automatic persistence and reuse of data splits, Optuna studies, trained models, thresholds, and run metadata
|
|
39
39
|
- Highly configurable through a small workflow-oriented Python API and CLI
|
|
40
|
-
- CPU and multi-GPU training and prediction support
|
|
40
|
+
- CPU and DDP-safe multi-GPU training and prediction support
|
|
41
41
|
|
|
42
42
|
## Installation
|
|
43
43
|
|
|
@@ -132,7 +132,7 @@ Fine-tune a model:
|
|
|
132
132
|
from tlmtc import train_tlmtc
|
|
133
133
|
|
|
134
134
|
train_tlmtc(
|
|
135
|
-
"paired_example.csv",
|
|
135
|
+
labeled_data="paired_example.csv",
|
|
136
136
|
target_name="Requirements Evidence Alignment",
|
|
137
137
|
checkpoint="google/bert_uncased_L-2_H-128_A-2",
|
|
138
138
|
tuning_trials=5,
|
|
@@ -160,14 +160,14 @@ predict_tlmtc(
|
|
|
160
160
|
|
|
161
161
|
```bash
|
|
162
162
|
tlmtc train \
|
|
163
|
-
--
|
|
163
|
+
--labeled-data paired_example.csv \
|
|
164
164
|
--target-name "Requirements Evidence Alignment" \
|
|
165
165
|
--checkpoint google/bert_uncased_L-2_H-128_A-2 \
|
|
166
166
|
--tuning-trials 5 \
|
|
167
167
|
--use-cpu
|
|
168
168
|
|
|
169
169
|
tlmtc predict \
|
|
170
|
-
--
|
|
170
|
+
--unlabeled-data paired_example_unlabeled.csv \
|
|
171
171
|
--use-cpu
|
|
172
172
|
```
|
|
173
173
|
|
|
@@ -176,7 +176,7 @@ tlmtc predict \
|
|
|
176
176
|
<details>
|
|
177
177
|
<summary><strong>Try your own data</strong></summary>
|
|
178
178
|
|
|
179
|
-
Your
|
|
179
|
+
Your labeled data must include:
|
|
180
180
|
|
|
181
181
|
- a `text` column
|
|
182
182
|
- at least two binary `label_`-prefixed columns
|
|
@@ -185,7 +185,7 @@ Add a `text_pair` column for paired-input classification. **tlmtc** detects and
|
|
|
185
185
|
|
|
186
186
|
Add a `split_group` column when semantically related rows must not cross train, validation, and test splits.
|
|
187
187
|
|
|
188
|
-
For prediction, provide
|
|
188
|
+
For prediction, provide unlabeled data with the same input columns used during training. For a paired-text model, this means both `text` and `text_pair`.
|
|
189
189
|
|
|
190
190
|
</details>
|
|
191
191
|
|
|
@@ -4,8 +4,11 @@ from dataclasses import dataclass
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
7
9
|
from tlmtc.data_pipeline import DataPipeline
|
|
8
|
-
from tlmtc.data_preparation import create_prediction_dataset,
|
|
10
|
+
from tlmtc.data_preparation import create_prediction_dataset, read_prediction_data, tokenize_prediction_dataset
|
|
11
|
+
from tlmtc.distributed import DistributedContext
|
|
9
12
|
from tlmtc.evaluation_pipeline import EvaluationPipeline
|
|
10
13
|
from tlmtc.finetune_pipeline import FinetunePipeline
|
|
11
14
|
from tlmtc.meta import TrainRunMeta, read_run_meta, write_run_meta
|
|
@@ -43,7 +46,7 @@ class PredictResult:
|
|
|
43
46
|
|
|
44
47
|
|
|
45
48
|
def train_tlmtc(
|
|
46
|
-
|
|
49
|
+
labeled_data: str | Path | pd.DataFrame,
|
|
47
50
|
*,
|
|
48
51
|
raw_test_csv: str | Path | Unset = UNSET,
|
|
49
52
|
work_dir: str | Path | Unset = UNSET,
|
|
@@ -62,6 +65,7 @@ def train_tlmtc(
|
|
|
62
65
|
proxy_checkpoint: str | Unset = UNSET,
|
|
63
66
|
checkpoint: str | Unset = UNSET,
|
|
64
67
|
sequence_length: int | Unset = UNSET,
|
|
68
|
+
trust_remote_code: bool | Unset = UNSET,
|
|
65
69
|
best_model_metric: str | Unset = UNSET,
|
|
66
70
|
batch_size: int | Unset = UNSET,
|
|
67
71
|
train_epochs: int | Unset = UNSET,
|
|
@@ -77,6 +81,7 @@ def train_tlmtc(
|
|
|
77
81
|
lora_bias: str | Unset = UNSET,
|
|
78
82
|
early_stopping_patience: int | Unset = UNSET,
|
|
79
83
|
use_cpu: bool | Unset = UNSET,
|
|
84
|
+
trainer_args: dict[str, Any] | Unset = UNSET,
|
|
80
85
|
verbosity: str | Unset = UNSET,
|
|
81
86
|
) -> TrainResult:
|
|
82
87
|
"""Run the full multi-label text classification training workflow.
|
|
@@ -86,10 +91,10 @@ def train_tlmtc(
|
|
|
86
91
|
workflow flags.
|
|
87
92
|
|
|
88
93
|
Args:
|
|
89
|
-
|
|
90
|
-
at least two binary `label_*` columns, and optionally a `text_pair` column.
|
|
94
|
+
labeled_data: Path to labeled multi-label training data, or an in-memory DataFrame. The data must contain
|
|
95
|
+
a `text` column, at least two binary `label_*` columns, and optionally a `text_pair` column.
|
|
91
96
|
raw_test_csv: Path to a separate raw test CSV. If omitted, a test split is created
|
|
92
|
-
from `
|
|
97
|
+
from `labeled_data` using `test_size`. Defaults to no separate test CSV.
|
|
93
98
|
work_dir: Base directory for resolving inputs and writing run artifacts. Defaults to the
|
|
94
99
|
current working directory.
|
|
95
100
|
config_path: Path to a YAML configuration file. Defaults to no configuration file.
|
|
@@ -119,13 +124,16 @@ def train_tlmtc(
|
|
|
119
124
|
proxy_checkpoint: Compatible encoder-only Hugging Face checkpoint identifier used during
|
|
120
125
|
hyperparameter tuning. Defaults to `"microsoft/deberta-v3-small"`. If `checkpoint`
|
|
121
126
|
is supplied and `proxy_checkpoint` is omitted, the proxy checkpoint defaults to the
|
|
122
|
-
selected `checkpoint`. Loaded with `trust_remote_code
|
|
123
|
-
|
|
127
|
+
selected `checkpoint`. Loaded with the resolved `trust_remote_code` setting. Keep `trust_remote_code=False`
|
|
128
|
+
unless you trust the checkpoint repository.
|
|
124
129
|
checkpoint: Compatible encoder-only Hugging Face checkpoint identifier or local path used for
|
|
125
|
-
final fine-tuning. Defaults to `"microsoft/deberta-v3-base"`.
|
|
126
|
-
|
|
127
|
-
|
|
130
|
+
final fine-tuning. Defaults to `"microsoft/deberta-v3-base"`. Prediction reloads the trained model or
|
|
131
|
+
adapter artifacts using the resolved prediction `trust_remote_code` setting. Keep it disabled unless you
|
|
132
|
+
trust the saved artifacts and checkpoint repository.
|
|
128
133
|
sequence_length: Maximum tokenized sequence length. Defaults to `128`.
|
|
134
|
+
trust_remote_code: Whether Hugging Face tokenizer, config, and model loading may execute
|
|
135
|
+
custom remote code. Defaults to `False`. Only enable this for model repositories or
|
|
136
|
+
local model artifacts you trust.
|
|
129
137
|
best_model_metric: Metric used to select the best model checkpoint. Supported values are
|
|
130
138
|
`"f1_micro"`, `"f1_macro"`, `"roc_auc_micro"`, and `"roc_auc_macro"`. Defaults to
|
|
131
139
|
`"roc_auc_macro"`.
|
|
@@ -183,6 +191,10 @@ def train_tlmtc(
|
|
|
183
191
|
early_stopping_patience: Early stopping patience in epochs without improvement. Defaults to
|
|
184
192
|
`10`.
|
|
185
193
|
use_cpu: Whether to force CPU execution. Defaults to `False`.
|
|
194
|
+
trainer_args: Additional Hugging Face TrainingArguments keyword arguments.
|
|
195
|
+
Keys already managed by tlmtc, such as batch size, epochs, learning rate,
|
|
196
|
+
output directory, model-selection settings, logging/reporting behavior,
|
|
197
|
+
and CPU selection, are rejected. Defaults to no additional arguments.
|
|
186
198
|
verbosity: Runtime output mode. Supported values are `"progress"` and `"quiet"`. Defaults to
|
|
187
199
|
`"progress"`.
|
|
188
200
|
|
|
@@ -193,7 +205,7 @@ def train_tlmtc(
|
|
|
193
205
|
config=load_config_file(config_path) if isinstance(config_path, (str, Path)) else None,
|
|
194
206
|
env=None,
|
|
195
207
|
overrides={
|
|
196
|
-
"
|
|
208
|
+
"labeled_data": labeled_data,
|
|
197
209
|
"raw_test_csv": raw_test_csv,
|
|
198
210
|
"work_dir": work_dir,
|
|
199
211
|
"run_id": run_id,
|
|
@@ -202,6 +214,7 @@ def train_tlmtc(
|
|
|
202
214
|
"proxy_checkpoint": proxy_checkpoint,
|
|
203
215
|
"checkpoint": checkpoint,
|
|
204
216
|
"sequence_length": sequence_length,
|
|
217
|
+
"trust_remote_code": trust_remote_code,
|
|
205
218
|
},
|
|
206
219
|
"split": {
|
|
207
220
|
"validation_size": validation_size,
|
|
@@ -223,6 +236,7 @@ def train_tlmtc(
|
|
|
223
236
|
"lr_scheduler": lr_scheduler,
|
|
224
237
|
"best_model_metric": best_model_metric,
|
|
225
238
|
"early_stopping_patience": early_stopping_patience,
|
|
239
|
+
"trainer_args": trainer_args,
|
|
226
240
|
},
|
|
227
241
|
"threshold": {
|
|
228
242
|
"threshold_type": threshold_type,
|
|
@@ -247,22 +261,41 @@ def train_tlmtc(
|
|
|
247
261
|
},
|
|
248
262
|
)
|
|
249
263
|
|
|
250
|
-
|
|
251
|
-
|
|
264
|
+
distributed = DistributedContext.create(use_cpu=settings.hardware.use_cpu)
|
|
265
|
+
configure_runtime_output(
|
|
266
|
+
settings.runtime.verbosity,
|
|
267
|
+
is_main_process=distributed.is_main_process,
|
|
268
|
+
)
|
|
269
|
+
distributed.warn_if_multi_gpu_without_launcher(use_cpu=settings.hardware.use_cpu)
|
|
270
|
+
|
|
271
|
+
if distributed.is_distributed and settings.workflow.hyperparameter_tuning:
|
|
272
|
+
raise RuntimeError(
|
|
273
|
+
"Hyperparameter tuning is not supported under distributed launch. "
|
|
274
|
+
"Run HPO in a single-process run, then rerun distributed final training "
|
|
275
|
+
"with hyperparameter_tuning=False and the selected training settings."
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
resolved_run_id = distributed.resolve_run_id(settings.run_id)
|
|
252
279
|
|
|
253
280
|
paths = resolve_paths(
|
|
254
|
-
|
|
281
|
+
labeled_data=settings.labeled_data if isinstance(settings.labeled_data, Path) else None,
|
|
255
282
|
raw_test_csv=settings.raw_test_csv,
|
|
256
283
|
work_dir=settings.work_dir,
|
|
257
|
-
run_id=
|
|
258
|
-
)
|
|
284
|
+
run_id=resolved_run_id,
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
distributed.run_on_main(paths.ensure_dirs, sync=True)
|
|
288
|
+
|
|
289
|
+
emit_progress("Starting training run")
|
|
259
290
|
|
|
260
291
|
data_pipeline = DataPipeline(
|
|
261
292
|
paths=paths,
|
|
262
293
|
split=settings.split,
|
|
263
294
|
model=settings.model,
|
|
295
|
+
labeled_data=settings.labeled_data if isinstance(settings.labeled_data, pd.DataFrame) else None,
|
|
264
296
|
)
|
|
265
|
-
|
|
297
|
+
with distributed.main_process_first():
|
|
298
|
+
data_pipeline.split_data()
|
|
266
299
|
data_pipeline.get_multi_hot_vectors()
|
|
267
300
|
data_pipeline.create_hf_dataset()
|
|
268
301
|
data_pipeline.tokenize_data()
|
|
@@ -279,7 +312,6 @@ def train_tlmtc(
|
|
|
279
312
|
hardware=settings.hardware,
|
|
280
313
|
)
|
|
281
314
|
finetune_pipeline.tune_hyperparameters()
|
|
282
|
-
finetune_pipeline.load_pretrained()
|
|
283
315
|
finetune_pipeline.fine_tune_pretrained()
|
|
284
316
|
finetune_pipeline.tune_thresholds()
|
|
285
317
|
finetune_pipeline.save_pretrained()
|
|
@@ -295,17 +327,19 @@ def train_tlmtc(
|
|
|
295
327
|
input_mode=data_pipeline.input_mode,
|
|
296
328
|
)
|
|
297
329
|
evaluation_pipeline.run_evaluation()
|
|
298
|
-
evaluation_pipeline.save_metrics
|
|
299
|
-
evaluation_pipeline.render_tables
|
|
300
|
-
evaluation_pipeline.render_figures
|
|
330
|
+
distributed.run_on_main(evaluation_pipeline.save_metrics)
|
|
331
|
+
distributed.run_on_main(evaluation_pipeline.render_tables)
|
|
332
|
+
distributed.run_on_main(evaluation_pipeline.render_figures)
|
|
301
333
|
|
|
302
|
-
|
|
334
|
+
distributed.run_on_main(
|
|
335
|
+
write_run_meta,
|
|
303
336
|
meta=TrainRunMeta(
|
|
304
|
-
run_id=
|
|
337
|
+
run_id=resolved_run_id,
|
|
305
338
|
target_name=settings.model.target_name,
|
|
306
339
|
checkpoint=settings.model.checkpoint,
|
|
307
340
|
proxy_checkpoint=settings.model.proxy_checkpoint,
|
|
308
341
|
sequence_length=settings.model.sequence_length,
|
|
342
|
+
trust_remote_code=settings.model.trust_remote_code,
|
|
309
343
|
input_mode=data_pipeline.input_mode,
|
|
310
344
|
label_names=evaluation_pipeline.label_names,
|
|
311
345
|
threshold_type=settings.threshold.threshold_type,
|
|
@@ -324,12 +358,13 @@ def train_tlmtc(
|
|
|
324
358
|
|
|
325
359
|
|
|
326
360
|
def predict_tlmtc(
|
|
327
|
-
|
|
361
|
+
unlabeled_data: str | Path | pd.DataFrame,
|
|
328
362
|
*,
|
|
329
363
|
work_dir: str | Path | Unset = UNSET,
|
|
330
364
|
config_path: str | Path | Unset = UNSET,
|
|
331
365
|
run_id: str | None | Unset = UNSET,
|
|
332
366
|
batch_size: int | Unset = UNSET,
|
|
367
|
+
trust_remote_code: bool | Unset = UNSET,
|
|
333
368
|
use_cpu: bool | Unset = UNSET,
|
|
334
369
|
verbosity: str | Unset = UNSET,
|
|
335
370
|
) -> PredictResult:
|
|
@@ -340,8 +375,8 @@ def predict_tlmtc(
|
|
|
340
375
|
and binary prediction artifacts.
|
|
341
376
|
|
|
342
377
|
Args:
|
|
343
|
-
|
|
344
|
-
column and, for models trained with paired-text inputs, a `text_pair` column.
|
|
378
|
+
unlabeled_data: Path to unlabeled prediction data, or an in-memory DataFrame. The data must contain
|
|
379
|
+
a `text` column and, for models trained with paired-text inputs, a `text_pair` column.
|
|
345
380
|
Prediction artifacts preserve input text columns unchanged.
|
|
346
381
|
work_dir: Base directory for resolving inputs, reading training artifacts, and writing
|
|
347
382
|
prediction artifacts. Defaults to the current working directory.
|
|
@@ -352,6 +387,9 @@ def predict_tlmtc(
|
|
|
352
387
|
artifacts that require custom remote code are not supported. Only use saved model
|
|
353
388
|
artifacts and adapters you trust.
|
|
354
389
|
batch_size: Prediction batch size used for batched inference. Defaults to `32`.
|
|
390
|
+
trust_remote_code: Whether Hugging Face tokenizer and model loading may execute custom
|
|
391
|
+
remote code during prediction. Defaults to `False`. Required for runs trained with
|
|
392
|
+
`trust_remote_code=True`; only enable it for artifacts and checkpoints you trust.
|
|
355
393
|
use_cpu: Whether to force CPU execution. Defaults to `False`.
|
|
356
394
|
verbosity: Runtime output mode. Supported values are `"progress"` and `"quiet"`. Defaults to
|
|
357
395
|
`"progress"`.
|
|
@@ -363,10 +401,11 @@ def predict_tlmtc(
|
|
|
363
401
|
config=load_config_file(config_path) if isinstance(config_path, (str, Path)) else None,
|
|
364
402
|
env=None,
|
|
365
403
|
overrides={
|
|
366
|
-
"
|
|
404
|
+
"unlabeled_data": unlabeled_data,
|
|
367
405
|
"work_dir": work_dir,
|
|
368
406
|
"run_id": run_id,
|
|
369
407
|
"batch_size": batch_size,
|
|
408
|
+
"trust_remote_code": trust_remote_code,
|
|
370
409
|
"hardware": {
|
|
371
410
|
"use_cpu": use_cpu,
|
|
372
411
|
},
|
|
@@ -376,14 +415,21 @@ def predict_tlmtc(
|
|
|
376
415
|
},
|
|
377
416
|
)
|
|
378
417
|
|
|
379
|
-
|
|
418
|
+
distributed = DistributedContext.create(use_cpu=settings.hardware.use_cpu)
|
|
419
|
+
configure_runtime_output(
|
|
420
|
+
settings.runtime.verbosity,
|
|
421
|
+
is_main_process=distributed.is_main_process,
|
|
422
|
+
)
|
|
423
|
+
|
|
380
424
|
emit_progress("Starting prediction run")
|
|
381
425
|
|
|
382
426
|
paths = resolve_prediction_paths(
|
|
383
|
-
|
|
427
|
+
unlabeled_data=settings.unlabeled_data if isinstance(settings.unlabeled_data, Path) else None,
|
|
384
428
|
work_dir=settings.work_dir,
|
|
385
429
|
run_id=settings.run_id,
|
|
386
|
-
)
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
distributed.run_on_main(paths.ensure_dirs, sync=True)
|
|
387
433
|
|
|
388
434
|
emit_progress("Reading training metadata")
|
|
389
435
|
|
|
@@ -395,6 +441,14 @@ def predict_tlmtc(
|
|
|
395
441
|
f"Run '{meta.run_id}' did not persist a fine-tuned prediction model."
|
|
396
442
|
)
|
|
397
443
|
|
|
444
|
+
if meta.trust_remote_code and not settings.trust_remote_code:
|
|
445
|
+
raise RuntimeError(
|
|
446
|
+
f"Prediction run '{meta.run_id}' was trained with trust_remote_code=True, "
|
|
447
|
+
"but prediction was started with trust_remote_code=False. "
|
|
448
|
+
"Re-run prediction with trust_remote_code=True or --trust-remote-code only if you trust "
|
|
449
|
+
f"the Hugging Face model repository or local model artifacts for '{meta.checkpoint}'."
|
|
450
|
+
)
|
|
451
|
+
|
|
398
452
|
input_mode = meta.input_mode
|
|
399
453
|
label_names = meta.label_names
|
|
400
454
|
|
|
@@ -403,8 +457,11 @@ def predict_tlmtc(
|
|
|
403
457
|
|
|
404
458
|
emit_progress("Reading prediction inputs")
|
|
405
459
|
|
|
406
|
-
|
|
407
|
-
|
|
460
|
+
assert isinstance(settings.unlabeled_data, pd.DataFrame) or paths.unlabeled_data_path is not None
|
|
461
|
+
input_df = read_prediction_data(
|
|
462
|
+
data=(
|
|
463
|
+
settings.unlabeled_data if isinstance(settings.unlabeled_data, pd.DataFrame) else paths.unlabeled_data_path
|
|
464
|
+
),
|
|
408
465
|
expected_input_mode=input_mode,
|
|
409
466
|
)
|
|
410
467
|
prediction_dataset = create_prediction_dataset(
|
|
@@ -417,6 +474,7 @@ def predict_tlmtc(
|
|
|
417
474
|
checkpoint=meta.checkpoint,
|
|
418
475
|
input_mode=input_mode,
|
|
419
476
|
sequence_length=meta.sequence_length,
|
|
477
|
+
trust_remote_code=settings.trust_remote_code,
|
|
420
478
|
)
|
|
421
479
|
emit_progress("Loading fine-tuned prediction model")
|
|
422
480
|
model = load_prediction_model(
|
|
@@ -424,6 +482,7 @@ def predict_tlmtc(
|
|
|
424
482
|
checkpoint=meta.checkpoint,
|
|
425
483
|
num_labels=len(label_names),
|
|
426
484
|
wrap_peft=meta.wrap_peft,
|
|
485
|
+
trust_remote_code=settings.trust_remote_code,
|
|
427
486
|
)
|
|
428
487
|
emit_progress("Running prediction")
|
|
429
488
|
probabilities = predict_probabilities(
|
|
@@ -448,8 +507,8 @@ def predict_tlmtc(
|
|
|
448
507
|
)
|
|
449
508
|
|
|
450
509
|
emit_progress("Writing prediction artifacts")
|
|
451
|
-
probability_df.to_csv
|
|
452
|
-
prediction_df.to_csv
|
|
510
|
+
distributed.run_on_main(probability_df.to_csv, paths.probabilities_path, index=False)
|
|
511
|
+
distributed.run_on_main(prediction_df.to_csv, paths.predictions_path, index=False)
|
|
453
512
|
|
|
454
513
|
emit_progress("Prediction run complete")
|
|
455
514
|
return PredictResult(paths=paths)
|
|
@@ -15,16 +15,21 @@ app = typer.Typer(
|
|
|
15
15
|
)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
def
|
|
18
|
+
def parse_json_object(
|
|
19
19
|
value: str | None,
|
|
20
|
+
*,
|
|
21
|
+
option_name: str,
|
|
22
|
+
example: str,
|
|
20
23
|
) -> dict[str, Any] | Unset:
|
|
21
|
-
"""Parse an optional
|
|
24
|
+
"""Parse an optional JSON object from JSON or an @file path.
|
|
22
25
|
|
|
23
26
|
Args:
|
|
24
27
|
value: JSON object string, @file path, or None when the CLI option is omitted.
|
|
28
|
+
option_name: CLI option name used in error messages.
|
|
29
|
+
example: Example JSON object shown when the parsed value is not an object.
|
|
25
30
|
|
|
26
31
|
Returns:
|
|
27
|
-
Parsed
|
|
32
|
+
Parsed JSON object, or UNSET for omitted CLI values.
|
|
28
33
|
|
|
29
34
|
Raises:
|
|
30
35
|
typer.BadParameter: If the value cannot be read, decoded, or parsed as a JSON object.
|
|
@@ -38,11 +43,12 @@ def parse_optuna_space(
|
|
|
38
43
|
else:
|
|
39
44
|
parsed = json.loads(value)
|
|
40
45
|
except OSError as exc:
|
|
41
|
-
raise typer.BadParameter(f"Could not read JSON file: {value}") from exc
|
|
46
|
+
raise typer.BadParameter(f"Could not read JSON file for {option_name}: {value}") from exc
|
|
42
47
|
except json.JSONDecodeError as exc:
|
|
43
|
-
raise typer.BadParameter("Expected a JSON object or @file.json.") from exc
|
|
48
|
+
raise typer.BadParameter(f"Expected a JSON object or @file.json for {option_name}.") from exc
|
|
49
|
+
|
|
44
50
|
if not isinstance(parsed, dict):
|
|
45
|
-
raise typer.BadParameter("Expected a JSON object
|
|
51
|
+
raise typer.BadParameter(f"Expected a JSON object for {option_name}, for example: {example}.")
|
|
46
52
|
|
|
47
53
|
return parsed
|
|
48
54
|
|
|
@@ -69,15 +75,15 @@ def main(
|
|
|
69
75
|
|
|
70
76
|
@app.command("train")
|
|
71
77
|
def train_command(
|
|
72
|
-
|
|
78
|
+
labeled_data: str = typer.Option(
|
|
73
79
|
...,
|
|
74
|
-
"--
|
|
75
|
-
help="Path to
|
|
80
|
+
"--labeled-data",
|
|
81
|
+
help="Path to labeled multi-label data with text, label_* columns, and optional text_pair.",
|
|
76
82
|
),
|
|
77
83
|
raw_test_csv: str | None = typer.Option(
|
|
78
84
|
None,
|
|
79
85
|
"--raw-test-csv",
|
|
80
|
-
help="Path to a separate raw test CSV. If omitted, a test split is created from --
|
|
86
|
+
help="Path to a separate raw test CSV. If omitted, a test split is created from --labeled-data.",
|
|
81
87
|
),
|
|
82
88
|
work_dir: str | None = typer.Option(
|
|
83
89
|
None,
|
|
@@ -159,6 +165,11 @@ def train_command(
|
|
|
159
165
|
"--sequence-length",
|
|
160
166
|
help="Maximum tokenized sequence length.",
|
|
161
167
|
),
|
|
168
|
+
trust_remote_code: bool | None = typer.Option(
|
|
169
|
+
None,
|
|
170
|
+
"--trust-remote-code/--no-trust-remote-code",
|
|
171
|
+
help="Allow Hugging Face model, tokenizer, and config loading to execute custom remote code.",
|
|
172
|
+
),
|
|
162
173
|
best_model_metric: str | None = typer.Option(
|
|
163
174
|
None,
|
|
164
175
|
"--best-model-metric",
|
|
@@ -240,6 +251,14 @@ def train_command(
|
|
|
240
251
|
"--use-cpu/--no-use-cpu",
|
|
241
252
|
help="Force CPU execution.",
|
|
242
253
|
),
|
|
254
|
+
trainer_args: str | None = typer.Option(
|
|
255
|
+
None,
|
|
256
|
+
"--trainer-args",
|
|
257
|
+
help=(
|
|
258
|
+
"Additional Hugging Face TrainingArguments keyword arguments as JSON or @file.json. "
|
|
259
|
+
"Keys already managed by tlmtc are rejected."
|
|
260
|
+
),
|
|
261
|
+
),
|
|
243
262
|
verbosity: str | None = typer.Option(
|
|
244
263
|
None,
|
|
245
264
|
"--verbosity",
|
|
@@ -250,7 +269,7 @@ def train_command(
|
|
|
250
269
|
from tlmtc.api import train_tlmtc
|
|
251
270
|
|
|
252
271
|
result = train_tlmtc(
|
|
253
|
-
|
|
272
|
+
labeled_data=labeled_data,
|
|
254
273
|
raw_test_csv=UNSET if raw_test_csv is None else raw_test_csv,
|
|
255
274
|
work_dir=UNSET if work_dir is None else work_dir,
|
|
256
275
|
config_path=UNSET if config_path is None else config_path,
|
|
@@ -268,6 +287,7 @@ def train_command(
|
|
|
268
287
|
proxy_checkpoint=UNSET if proxy_checkpoint is None else proxy_checkpoint,
|
|
269
288
|
checkpoint=UNSET if checkpoint is None else checkpoint,
|
|
270
289
|
sequence_length=UNSET if sequence_length is None else sequence_length,
|
|
290
|
+
trust_remote_code=UNSET if trust_remote_code is None else trust_remote_code,
|
|
271
291
|
best_model_metric=UNSET if best_model_metric is None else best_model_metric,
|
|
272
292
|
batch_size=UNSET if batch_size is None else batch_size,
|
|
273
293
|
train_epochs=UNSET if train_epochs is None else train_epochs,
|
|
@@ -276,13 +296,22 @@ def train_command(
|
|
|
276
296
|
lr_scheduler=UNSET if lr_scheduler is None else lr_scheduler,
|
|
277
297
|
best_threshold_metric=UNSET if best_threshold_metric is None else best_threshold_metric,
|
|
278
298
|
tuning_trials=UNSET if tuning_trials is None else tuning_trials,
|
|
279
|
-
optuna_space=
|
|
299
|
+
optuna_space=parse_json_object(
|
|
300
|
+
optuna_space,
|
|
301
|
+
option_name="--optuna-space",
|
|
302
|
+
example='{"lr_low": 1e-5}',
|
|
303
|
+
),
|
|
280
304
|
lora_r=UNSET if lora_r is None else lora_r,
|
|
281
305
|
lora_alpha=UNSET if lora_alpha is None else lora_alpha,
|
|
282
306
|
lora_dropout=UNSET if lora_dropout is None else lora_dropout,
|
|
283
307
|
lora_bias=UNSET if lora_bias is None else lora_bias,
|
|
284
308
|
early_stopping_patience=UNSET if early_stopping_patience is None else early_stopping_patience,
|
|
285
309
|
use_cpu=UNSET if use_cpu is None else use_cpu,
|
|
310
|
+
trainer_args=parse_json_object(
|
|
311
|
+
trainer_args,
|
|
312
|
+
option_name="--trainer-args",
|
|
313
|
+
example='{"gradient_accumulation_steps": 2}',
|
|
314
|
+
),
|
|
286
315
|
verbosity=UNSET if verbosity is None else verbosity,
|
|
287
316
|
)
|
|
288
317
|
|
|
@@ -291,10 +320,10 @@ def train_command(
|
|
|
291
320
|
|
|
292
321
|
@app.command("predict")
|
|
293
322
|
def predict_command(
|
|
294
|
-
|
|
323
|
+
unlabeled_data: str = typer.Option(
|
|
295
324
|
...,
|
|
296
|
-
"--
|
|
297
|
-
help="Path to
|
|
325
|
+
"--unlabeled-data",
|
|
326
|
+
help="Path to unlabeled prediction data with text and, if required by the training run, text_pair.",
|
|
298
327
|
),
|
|
299
328
|
work_dir: str | None = typer.Option(
|
|
300
329
|
None,
|
|
@@ -316,6 +345,11 @@ def predict_command(
|
|
|
316
345
|
"--batch-size",
|
|
317
346
|
help="Prediction batch size used for batched inference.",
|
|
318
347
|
),
|
|
348
|
+
trust_remote_code: bool | None = typer.Option(
|
|
349
|
+
None,
|
|
350
|
+
"--trust-remote-code/--no-trust-remote-code",
|
|
351
|
+
help="Allow Hugging Face tokenizer and model loading to execute custom remote code during prediction.",
|
|
352
|
+
),
|
|
319
353
|
use_cpu: bool | None = typer.Option(
|
|
320
354
|
None,
|
|
321
355
|
"--use-cpu/--no-use-cpu",
|
|
@@ -331,11 +365,12 @@ def predict_command(
|
|
|
331
365
|
from tlmtc.api import predict_tlmtc
|
|
332
366
|
|
|
333
367
|
result = predict_tlmtc(
|
|
334
|
-
|
|
368
|
+
unlabeled_data=unlabeled_data,
|
|
335
369
|
work_dir=UNSET if work_dir is None else work_dir,
|
|
336
370
|
config_path=UNSET if config_path is None else config_path,
|
|
337
371
|
run_id=UNSET if run_id is None else run_id,
|
|
338
372
|
batch_size=UNSET if batch_size is None else batch_size,
|
|
373
|
+
trust_remote_code=UNSET if trust_remote_code is None else trust_remote_code,
|
|
339
374
|
use_cpu=UNSET if use_cpu is None else use_cpu,
|
|
340
375
|
verbosity=UNSET if verbosity is None else verbosity,
|
|
341
376
|
)
|