bertuner 0.2.2__tar.gz → 0.2.3__tar.gz

{bertuner-0.2.2 → bertuner-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertuner
-Version: 0.2.2
+Version: 0.2.3
 Summary: Hyperparameter optimization and fine-tuning for BERT-style text classifiers (Optuna + MLflow), with long-context ModernBERT support
 Author-email: elemets <alafunnell@gmail.com>
 License: MIT
@@ -58,6 +58,20 @@ git clone https://github.com/elemets/bertuner && cd bertuner
 pip install -r requirements.txt
 ```
 
+BERTuner supports Transformers 4.48+ and 5.x. Compatibility tests cover the
+4.48 baseline and the latest available 4.x and 5.x releases, including final
+training, TensorBoard logging, warmup scheduling, and gradient accumulation.
+Future releases are checked by a weekly CI run rather than assumed compatible.
+
+When initializing a classifier from a base pretrained model, a Transformers 5
+load report may list `lm_head.*` weights as `UNEXPECTED` and classifier weights
+as `MISSING`. This is expected when replacing the language-model head with a
+classification head; the new head is learned during fine-tuning. Unexpected
+encoder weights or shape mismatches should still be investigated.
+
+Run the test suite with `python -m pytest tests -q`. Training regression tests
+use a tiny local checkpoint; cached-model predictor tests skip if unavailable.
+
 MLflow tracking works in two modes:
 
 ```bash

{bertuner-0.2.2 → bertuner-0.2.3}/README.md

@@ -17,6 +17,20 @@ git clone https://github.com/elemets/bertuner && cd bertuner
 pip install -r requirements.txt
 ```
 
+BERTuner supports Transformers 4.48+ and 5.x. Compatibility tests cover the
+4.48 baseline and the latest available 4.x and 5.x releases, including final
+training, TensorBoard logging, warmup scheduling, and gradient accumulation.
+Future releases are checked by a weekly CI run rather than assumed compatible.
+
+When initializing a classifier from a base pretrained model, a Transformers 5
+load report may list `lm_head.*` weights as `UNEXPECTED` and classifier weights
+as `MISSING`. This is expected when replacing the language-model head with a
+classification head; the new head is learned during fine-tuning. Unexpected
+encoder weights or shape mismatches should still be investigated.
+
+Run the test suite with `python -m pytest tests -q`. Training regression tests
+use a tiny local checkpoint; cached-model predictor tests skip if unavailable.
+
 MLflow tracking works in two modes:
 
 ```bash

{bertuner-0.2.2 → bertuner-0.2.3}/bertuner/BERTuner.py

@@ -15,7 +15,6 @@ from transformers import (
     AutoTokenizer,
     AutoConfig,
     AutoModelForSequenceClassification,
-    TrainingArguments,
     DataCollatorWithPadding,
     EarlyStoppingCallback,
     set_seed,
@@ -38,6 +37,7 @@ from sklearn.preprocessing import label_binarize
 from mlflow.tracking import MlflowClient
 
 from bertuner.CustomTrainer import CustomTrainer
+from bertuner.compat import TrainingArguments
 from bertuner.exceptions import NonFiniteTrainingError, NoStableTrialError
 from bertuner.TensorBoardCallback import (
     TensorBoardSyncCallback,
@@ -56,6 +56,7 @@ from bertuner.constants import (
     MODEL_DROPOUT_ATTRS,
     SEED,
 )
+import inspect
 
 
 class BERTuneClassifier:
@@ -746,7 +747,6 @@ class BERTuneClassifier:
             "gradient_checkpointing": self._use_gradient_checkpointing(max_length),
             "gradient_checkpointing_kwargs": {"use_reentrant": False},
             "weight_decay": params["weight_decay"],
-            "warmup_ratio": params["warmup_ratio"],
             "metric_for_best_model": f"eval_{self.optimize_metric}",
             "greater_is_better": self.greater_is_better,
             "eval_strategy": "epoch",
@@ -760,17 +760,20 @@ class BERTuneClassifier:
             "seed": self.seed,
             **self._precision_flags(precision),
         }
-        if final:
-            # Canonical final metrics are logged explicitly after restoring the
-            # best checkpoint; Trainer only sends loss curves to TensorBoard.
-            kwargs.update(report_to=["tensorboard"], logging_dir=logging_dir)
-        else:
-            kwargs.update(
-                lr_scheduler_type=params["scheduler"],
-                remove_unused_columns=True,
-                report_to=["none"],
-            )
-        return TrainingArguments(**kwargs)
+
+        _TA_PARAMS = inspect.signature(TrainingArguments.__init__).parameters
+
+        # v5 removed warmup_ratio and accepts fractions in warmup_steps.
+        warmup_key = "warmup_ratio" if "warmup_ratio" in _TA_PARAMS else "warmup_steps"
+        kwargs[warmup_key] = params["warmup_ratio"]
+        kwargs["lr_scheduler_type"] = params["scheduler"]
+        # Final training uses an explicit TensorBoard writer in _build_trainer;
+        # newer Transformers versions removed TrainingArguments.logging_dir.
+        kwargs.update(remove_unused_columns=True, report_to=["none"])
+        args = TrainingArguments(**kwargs)
+        if warmup_key == "warmup_steps":
+            args._bertuner_warmup_ratio = params["warmup_ratio"]
+        return args
 
     def _build_trainer(
         self,
@@ -792,8 +795,12 @@ class BERTuneClassifier:
             )
         ]
         if final:
+            from transformers.integrations import TensorBoardCallback
+            from torch.utils.tensorboard import SummaryWriter
+
             callbacks.extend(
                 [
+                    TensorBoardCallback(tb_writer=SummaryWriter(logging_dir)),
                     TensorBoardSyncCallback(logging_dir),
                     CleanupCheckpointsCallback,
                 ]
@@ -857,11 +864,15 @@ class BERTuneClassifier:
         try:
             trainer.train()
         except Exception:
+            from transformers.integrations import TensorBoardCallback
+
             # Trainer does not emit on_train_end after an exception. Close only
             # BERTuner-owned writers before the caller decides whether to retry.
             for callback in trainer.callback_handler.callbacks:
                 if isinstance(callback, TensorBoardSyncCallback):
                     callback.writer.close()
+                elif isinstance(callback, TensorBoardCallback) and callback.tb_writer is not None:
+                    callback.tb_writer.close()
             raise
         return trainer, model
 

{bertuner-0.2.2 → bertuner-0.2.3}/bertuner/CustomTrainer.py

@@ -69,6 +69,9 @@ class CustomTrainer(Trainer):
         callbacks = list(kwargs.pop("callbacks", None) or [])
         callbacks.append(NonFiniteGradientCallback(training_precision))
         super().__init__(callbacks=callbacks, **kwargs)
+        # compute_loss returns a microbatch mean and does not consume
+        # num_items_in_batch, even when the model's forward accepts **kwargs.
+        self.model_accepts_loss_kwargs = False
         self.loss_type = loss_type
         self.class_weights = class_weights
         self.training_precision = training_precision

{bertuner-0.2.2 → bertuner-0.2.3}/bertuner/__init__.py

@@ -1,6 +1,6 @@
 """bertuner: hyperparameter optimization and fine-tuning for BERT-style text classifiers."""
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 
 from bertuner.BERTuner import BERTuneClassifier
 from bertuner.Predictor import BERTunePredictor

bertuner-0.2.3/bertuner/compat.py

@@ -0,0 +1,15 @@
+"""Small adapters for Transformers APIs shared by supported 4.x and 5.x."""
+
+import math
+
+from transformers import TrainingArguments as HFTrainingArguments
+
+
+class TrainingArguments(HFTrainingArguments):
+    def get_warmup_steps(self, num_training_steps):
+        ratio = getattr(self, "_bertuner_warmup_ratio", None)
+        if ratio is not None:
+            # New warmup_steps treats 1.0 as one step, whereas the old
+            # warmup_ratio treats it as the entire training run.
+            return math.ceil(num_training_steps * ratio)
+        return super().get_warmup_steps(num_training_steps)

{bertuner-0.2.2 → bertuner-0.2.3}/bertuner.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertuner
-Version: 0.2.2
+Version: 0.2.3
 Summary: Hyperparameter optimization and fine-tuning for BERT-style text classifiers (Optuna + MLflow), with long-context ModernBERT support
 Author-email: elemets <alafunnell@gmail.com>
 License: MIT
@@ -58,6 +58,20 @@ git clone https://github.com/elemets/bertuner && cd bertuner
 pip install -r requirements.txt
 ```
 
+BERTuner supports Transformers 4.48+ and 5.x. Compatibility tests cover the
+4.48 baseline and the latest available 4.x and 5.x releases, including final
+training, TensorBoard logging, warmup scheduling, and gradient accumulation.
+Future releases are checked by a weekly CI run rather than assumed compatible.
+
+When initializing a classifier from a base pretrained model, a Transformers 5
+load report may list `lm_head.*` weights as `UNEXPECTED` and classifier weights
+as `MISSING`. This is expected when replacing the language-model head with a
+classification head; the new head is learned during fine-tuning. Unexpected
+encoder weights or shape mismatches should still be investigated.
+
+Run the test suite with `python -m pytest tests -q`. Training regression tests
+use a tiny local checkpoint; cached-model predictor tests skip if unavailable.
+
 MLflow tracking works in two modes:
 
 ```bash

{bertuner-0.2.2 → bertuner-0.2.3}/bertuner.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ bertuner/CustomTrainer.py
 bertuner/Predictor.py
 bertuner/TensorBoardCallback.py
 bertuner/__init__.py
+bertuner/compat.py
 bertuner/constants.py
 bertuner/exceptions.py
 bertuner/utils.py
@@ -17,4 +18,5 @@ bertuner.egg-info/top_level.txt
 tests/test_bertuner.py
 tests/test_numerical_stability.py
 tests/test_predictor.py
+tests/test_transformers_compat.py
 tests/test_utils.py

{bertuner-0.2.2 → bertuner-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "bertuner"
-version = "0.2.2"
+version = "0.2.3"
 description = "Hyperparameter optimization and fine-tuning for BERT-style text classifiers (Optuna + MLflow), with long-context ModernBERT support"
 readme = "README.md"
 license = { text = "MIT" }

{bertuner-0.2.2 → bertuner-0.2.3}/tests/test_bertuner.py

@@ -483,6 +483,7 @@ class TestLossCurveLogging:
                     ]
                 )
 
+            (tmp_path / "downloaded-artifacts").mkdir()
             artifact_path = MlflowClient().download_artifacts(
                 run.info.run_id,
                 "plots/training_vs_evaluation_loss.png",
@@ -785,7 +786,7 @@ class TestMulticlassWeightedLoss:
             loss = trainer._singlelabel_loss(logits, labels, torch.device("cpu"))
             assert torch.isfinite(loss)
 
-    def test_end_to_end_training_three_classes(self, tmp_path):
+    def test_end_to_end_training_three_classes(self, tmp_path, tiny_model_path):
         import torch
         from transformers import (
             AutoModelForSequenceClassification,
@@ -795,7 +796,7 @@ class TestMulticlassWeightedLoss:
         )
         from bertuner.CustomTrainer import CustomTrainer
 
-        model_path = "prajjwal1/bert-tiny"
+        model_path = tiny_model_path
         clf = make_classifier(tmp_path, dataframe=make_df(n=60, num_classes=3))
         assert clf.num_labels == 3
 

bertuner-0.2.3/tests/test_transformers_compat.py

@@ -0,0 +1,88 @@
+"""Run unchanged against the oldest supported Transformers and current releases."""
+import copy
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+import torch
+from datasets import Dataset
+from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
+from transformers import AutoTokenizer, TrainingArguments
+
+from bertuner.CustomTrainer import CustomTrainer
+from test_bertuner import make_classifier, make_df
+from test_numerical_stability import sampled_params
+
+
+@pytest.mark.parametrize("ratio", [0.0, 0.1, 1.0])
+@pytest.mark.parametrize("final", [False, True])
+def test_warmup_and_scheduler(tmp_path, ratio, final):
+    clf = make_classifier(tmp_path)
+    clf.optimize_metric = "avg_precision"
+    clf.greater_is_better = True
+    params = dict(sampled_params(), warmup_ratio=ratio, scheduler="cosine")
+    args = clf._build_training_arguments(
+        params, str(tmp_path / "output"), 32, "fp32", final=final,
+        logging_dir=str(tmp_path / "logs"),
+    )
+    assert args.get_warmup_steps(100) == int(100 * ratio)
+    assert args.lr_scheduler_type == "cosine"
+
+
+def test_final_training_logs_and_reloads(tmp_path, tiny_model_path):
+    clf = make_classifier(tmp_path, dataframe=make_df(n=30, num_classes=3))
+    clf.optimize_metric = "f1"
+    clf.greater_is_better = True
+    tokenizer = AutoTokenizer.from_pretrained(tiny_model_path)
+    train, val, _ = clf._prepare_datasets(tokenizer, None, max_length=32)
+    params = sampled_params()
+    log_dir = str(tmp_path / "logs")
+    args = clf._build_training_arguments(
+        params, str(tmp_path / "output"), 32, "fp32", final=True, logging_dir=log_dir,
+    )
+    args.num_train_epochs = 1
+    args.use_cpu = True
+    trainer = clf._build_trainer(
+        clf._load_model(tiny_model_path, 0.0), args, train, val, tokenizer,
+        params, clf._compute_class_weights(train), "fp32", final=True,
+        logging_dir=log_dir,
+    )
+    assert np.isfinite(trainer.train().training_loss)
+    assert trainer.state.best_model_checkpoint is not None
+    assert np.isfinite(trainer.evaluate()["eval_loss"])
+    events = EventAccumulator(log_dir).Reload()
+    assert "train/loss" in events.Tags()["scalars"]
+    assert "eval/loss" in events.Tags()["scalars"]
+
+
+class KwargsClassifier(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.linear = torch.nn.Linear(2, 2)
+        self.accepts_loss_kwargs = True
+
+    def forward(self, features, labels=None, **kwargs):
+        return SimpleNamespace(logits=self.linear(features))
+
+
+def test_accumulation_matches_full_batch_update(tmp_path):
+    torch.manual_seed(42)
+    initial = KwargsClassifier()
+    dataset = Dataset.from_dict({"features": [[1., 2.]] * 4, "labels": [1] * 4})
+    models = []
+    for batch, accumulation in [(4, 1), (2, 2)]:
+        model = copy.deepcopy(initial)
+        args = TrainingArguments(
+            output_dir=str(tmp_path / str(batch)), use_cpu=True,
+            per_device_train_batch_size=batch, gradient_accumulation_steps=accumulation,
+            max_steps=1, max_grad_norm=0., report_to="none", save_strategy="no",
+            lr_scheduler_type="constant", disable_tqdm=True,
+        )
+        trainer = CustomTrainer(
+            model=model, args=args, train_dataset=dataset, loss_type="plain",
+            optimizers=(torch.optim.SGD(model.parameters(), lr=0.1), None),
+        )
+        trainer.train()
+        models.append(model)
+    for full, accumulated in zip(models[0].parameters(), models[1].parameters()):
+        torch.testing.assert_close(full, accumulated)