transformers 5.0.0__py3-none-any.whl → 5.0.0rc0__py3-none-any.whl

transformers/trainer.py

@@ -642,16 +642,6 @@ class Trainer:
                 "You should subclass `Trainer` and override the `create_optimizer_and_scheduler` method."
             )
         default_callbacks = DEFAULT_CALLBACKS + get_reporting_integration_callbacks(self.args.report_to)
-
-        # Add JIT checkpoint callback if enabled
-        if self.args.enable_jit_checkpoint:
-            from .trainer_jit_checkpoint import JITCheckpointCallback
-
-            jit_callback = JITCheckpointCallback()
-            default_callbacks = default_callbacks + [jit_callback]
-            # Set trainer reference for JIT callback after initialization
-            jit_callback.set_trainer(self)
-
         callbacks = default_callbacks if callbacks is None else default_callbacks + callbacks
         self.callback_handler = CallbackHandler(
             callbacks, self.model, self.processing_class, self.optimizer, self.lr_scheduler
@@ -872,7 +862,7 @@ class Trainer:
 
         # 1 - Align EOS token. EOS is more complex than the others, as `generation_config` may hold more than one EOS
         # token.
-        tokenizer_has_new_eos = tokenizer.eos_token_id != getattr(self.model.config, "eos_token_id", None)
+        tokenizer_has_new_eos = tokenizer.eos_token_id != self.model.config.eos_token_id
         if model_has_generation_config:
             # `generation_config.eos_token_id` is None: direct comparison
             if self.model.generation_config.eos_token_id is None:
@@ -896,7 +886,7 @@ class Trainer:
                 self.model.generation_config.eos_token_id = [token for token in all_eos_tokens if token is not None]
 
         # 2 - Align BOS
-        tokenizer_has_new_bos = tokenizer.bos_token_id != getattr(self.model.config, "bos_token_id", None)
+        tokenizer_has_new_bos = tokenizer.bos_token_id != self.model.config.bos_token_id
         if model_has_generation_config:
             tokenizer_has_new_bos |= tokenizer.bos_token_id != self.model.generation_config.bos_token_id
 
@@ -907,7 +897,7 @@ class Trainer:
                 self.model.generation_config.bos_token_id = tokenizer.bos_token_id
 
         # 3 - Align PAD
-        tokenizer_has_new_pad = tokenizer.pad_token_id != getattr(self.model.config, "pad_token_id", None)
+        tokenizer_has_new_pad = tokenizer.pad_token_id != self.model.config.pad_token_id
         if model_has_generation_config:
             tokenizer_has_new_pad |= tokenizer.pad_token_id != self.model.generation_config.pad_token_id
 
@@ -1671,12 +1661,6 @@ class Trainer:
                 optimizer_cls = AdamW8bit
             else:
                 raise ValueError("Invalid optimizer")
-            optimizer_kwargs.update(
-                {
-                    "block_size": optim_args.get("block_size", 256),
-                    "bf16_stochastic_round": strtobool(optim_args.get("bf16_stochastic_round", "False")),
-                }
-            )
             optimizer_kwargs.update(adam_kwargs)
         elif args.optim in [
             OptimizerNames.SCHEDULE_FREE_RADAM,
@@ -2271,7 +2255,8 @@ class Trainer:
                 # nn.DataParallel(model) replicates the model, creating new variables and module
                 # references registered here no longer work on other gpus, breaking the module
                 raise ValueError(
-                    "Currently --debug underflow_overflow is not supported under DP. Please use DDP with torchrun"
+                    "Currently --debug underflow_overflow is not supported under DP. Please use DDP"
+                    " (torchrun or torch.distributed.launch (deprecated))."
                 )
             else:
                 DebugUnderflowOverflow(self.model)
@@ -2333,7 +2318,11 @@ class Trainer:
         if use_accelerator_prepare:
             self.model.train()
             if hasattr(self.lr_scheduler, "step"):
-                model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
+                # We should avoid accelerate preparing the model in TP case since we dont need it as it is handled by transformers from_pretrained and also it goes into DDP based preparation.
+                if self.is_tp_enabled:
+                    self.optimizer = self.accelerator.prepare(self.optimizer)
+                else:
+                    model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
             else:
                 # to handle cases wherein we pass "DummyScheduler" such as when it is specified in DeepSpeed config.
                 model, self.optimizer, self.lr_scheduler = self.accelerator.prepare(
@@ -2349,9 +2338,6 @@ class Trainer:
 
         if self.is_fsdp_enabled:
             self.model = self.model_wrapped = model
-            # Fix `got mixed torch.Tensor and DTensor` error in model.generate() for FSDP2 with LoRA
-            if hasattr(self.model, "generate"):
-                dist.fsdp.register_fsdp_forward_method(self.model, "generate")
 
         # for the rest of this function `model` is the outside model, whether it was wrapped or not
         if model is not self.model:
@@ -2442,6 +2428,8 @@ class Trainer:
 
         for epoch in range(epochs_trained, num_train_epochs):
             epoch_dataloader = train_dataloader
+            if hasattr(epoch_dataloader, "set_epoch"):
+                epoch_dataloader.set_epoch(epoch)
 
             steps_in_epoch = (
                 len(epoch_dataloader)
@@ -2462,9 +2450,6 @@ class Trainer:
                 elif steps_trained_in_current_epoch == 0:
                     self._load_rng_state(resume_from_checkpoint)
 
-            if hasattr(epoch_dataloader, "set_epoch"):
-                epoch_dataloader.set_epoch(epoch)
-
             epoch_iterator = iter(epoch_dataloader)
             # We chunkify the epoch iterator into gradient accumulation steps `n` batches
             remainder = steps_in_epoch % args.gradient_accumulation_steps
@@ -2803,7 +2788,7 @@ class Trainer:
                 )
             else:
                 # We load the model state dict on the CPU to avoid an OOM error.
-                if os.path.isfile(safe_weights_file):
+                if self.args.save_safetensors and os.path.isfile(safe_weights_file):
                     state_dict = safetensors.torch.load_file(safe_weights_file, device="cpu")
                 else:
                     check_torch_load_is_safe()
@@ -2843,7 +2828,9 @@ class Trainer:
                 logger.warning(f"Could not load adapter model, make sure to have PEFT >= {MIN_PEFT_VERSION} installed")
         else:
             # We load the sharded checkpoint
-            load_result = load_sharded_checkpoint(model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled())
+            load_result = load_sharded_checkpoint(
+                model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled(), prefer_safe=self.args.save_safetensors
+            )
             if not is_sagemaker_mp_enabled():
                 self._issue_warnings_after_load(load_result)
 
@@ -2926,7 +2913,7 @@ class Trainer:
                         has_been_loaded = False
                 else:
                     # We load the model state dict on the CPU to avoid an OOM error.
-                    if os.path.isfile(best_safe_model_path):
+                    if self.args.save_safetensors and os.path.isfile(best_safe_model_path):
                         state_dict = safetensors.torch.load_file(best_safe_model_path, device="cpu")
                     else:
                         check_torch_load_is_safe()
@@ -3051,8 +3038,7 @@ class Trainer:
                 return
 
         with safe_globals():
-            check_torch_load_is_safe()
-            checkpoint_rng_state = torch.load(rng_file, weights_only=True)
+            checkpoint_rng_state = torch.load(rng_file)
         random.setstate(checkpoint_rng_state["python"])
         np.random.set_state(checkpoint_rng_state["numpy"])
         torch.random.set_rng_state(checkpoint_rng_state["cpu"])
@@ -3946,9 +3932,6 @@ class Trainer:
         # Both standard transformer models and Liger-patched models handle shift_labels correctly,
         # so we can directly use the computed loss from the model output.
         # See: https://huggingface.co/docs/accelerate/en/concept_guides/sequence_parallelism
-        if "labels" not in inputs and "shift_labels" in inputs:
-            # DeepSpeed SP Dataloader removes "labels" but we need it, otherwise, we won't compute the loss.
-            inputs["labels"] = inputs["shift_labels"]
         outputs = model(**inputs)
         loss = outputs.loss
 
@@ -4008,6 +3991,15 @@ class Trainer:
             if self.args.should_save:
                 self._save(output_dir, state_dict=state_dict)
             Path(os.path.join(output_dir, "user_content.pt")).touch()
+        # We are in N-D parallelism if we have parallelism_config set, so we check accelerate if we're on a to_save rank
+        elif getattr(self.accelerator, "parallelism_config", None) is not None:
+            # DeepSpeed SP already handles checkpoint saving below, so skip manual save in that case
+            pc = getattr(self.accelerator, "parallelism_config")
+            if self.accelerator.should_save_model and not (pc.sp_enabled and pc.sp_backend == "deepspeed"):
+                self._save(output_dir)
+        # If we drop to here, we're in 1D parallelism, so all ranks need to go to `save_pretrained`
+        elif (tp_size := getattr(self.model, "_tp_size", 0)) is not None and tp_size > 1:
+            self._save(output_dir)
         elif self.is_fsdp_enabled:
             if "FULL_STATE_DICT" in str(self.accelerator.state.fsdp_plugin.state_dict_type):
                 state_dict = self.accelerator.get_state_dict(self.model)
@@ -4015,16 +4007,7 @@ class Trainer:
                     self._save(output_dir, state_dict=state_dict)
         elif self.is_deepspeed_enabled:
             try:
-                accept_exclude_frozen_parameters = "exclude_frozen_parameters" in set(
-                    inspect.signature(self.model_wrapped.save_checkpoint).parameters.keys()
-                )
-                zero3_sharding = self.deepspeed.config.get("zero_optimization", {}).get("stage", None) == 3
-                if accept_exclude_frozen_parameters and _is_peft_model(self.model) and zero3_sharding:
-                    # When using PEFT with DeepSpeed ZeRO Stage 3,
-                    # we do not need to load the frozen parameters
-                    state_dict = self.deepspeed._zero3_consolidated_16bit_state_dict(exclude_frozen_parameters=True)
-                else:
-                    state_dict = self.accelerator.get_state_dict(self.deepspeed)
+                state_dict = self.accelerator.get_state_dict(self.deepspeed)
                 if self.args.should_save:
                     self._save(output_dir, state_dict=state_dict)
             except ValueError:
@@ -4084,7 +4067,12 @@ class Trainer:
                 model = model.module.module
                 unwrapped_model = self.accelerator.unwrap_model(model)
                 if isinstance(unwrapped_model, supported_classes):
-                    unwrapped_model.save_pretrained(output_dir, state_dict=full_state_dict)
+                    unwrapped_model.save_pretrained(
+                        output_dir,
+                        state_dict=full_state_dict,
+                        save_function=xm.save,
+                        safe_serialization=self.args.save_safetensors,
+                    )
                 else:
                     logger.info("Trainer.model is not a `PreTrainedModel`, only saving its state dict.")
                     xm.save(full_state_dict, os.path.join(output_dir, WEIGHTS_NAME))
@@ -4094,6 +4082,8 @@ class Trainer:
                     output_dir,
                     is_main_process=self.args.should_save,
                     state_dict=xm._maybe_convert_to_cpu(model.state_dict()),
+                    save_function=xm.save,
+                    safe_serialization=self.args.save_safetensors,
                 )
             else:
                 logger.info("Trainer.model is not a `PreTrainedModel`, only saving its state dict.")
@@ -4103,6 +4093,8 @@ class Trainer:
             model.save_pretrained(
                 output_dir,
                 is_main_process=self.args.should_save,
+                save_function=xm.save,
+                safe_serialization=self.args.save_safetensors,
                 state_dict=xm._maybe_convert_to_cpu(model.state_dict()),
             )
         if self.processing_class is not None and self.args.should_save:
@@ -4123,15 +4115,20 @@ class Trainer:
 
             if isinstance(self.accelerator.unwrap_model(self.model, keep_torch_compile=False), supported_classes):
                 self.accelerator.unwrap_model(self.model, keep_torch_compile=False).save_pretrained(
-                    output_dir, state_dict=state_dict
+                    output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors
                 )
             else:
                 logger.info("Trainer.model is not a `PreTrainedModel`, only saving its state dict.")
-                safetensors.torch.save_file(
-                    state_dict, os.path.join(output_dir, SAFE_WEIGHTS_NAME), metadata={"format": "pt"}
-                )
+                if self.args.save_safetensors:
+                    safetensors.torch.save_file(
+                        state_dict, os.path.join(output_dir, SAFE_WEIGHTS_NAME), metadata={"format": "pt"}
+                    )
+                else:
+                    torch.save(state_dict, os.path.join(output_dir, WEIGHTS_NAME))
         else:
-            self.model.save_pretrained(output_dir, state_dict=state_dict)
+            self.model.save_pretrained(
+                output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors
+            )
 
         if self.processing_class is not None:
             self.processing_class.save_pretrained(output_dir)
@@ -4830,7 +4827,6 @@ class Trainer:
         if not self.args.hub_always_push and self.push_in_progress is not None and not self.push_in_progress.is_done():
             return
 
-        self.callback_handler.on_push_begin(self.args, self.state, self.control)
         output_dir = self.args.output_dir
         # To avoid a new synchronization of all model weights, we just copy the file from the checkpoint folder
         modeling_files = [CONFIG_NAME, GENERATION_CONFIG_NAME, WEIGHTS_NAME, SAFE_WEIGHTS_NAME]
@@ -4925,8 +4921,6 @@ class Trainer:
             The URL of the repository where the model was pushed if `blocking=False`, or a `Future` object tracking the
             progress of the commit if `blocking=True`.
         """
-        self.callback_handler.on_push_begin(self.args, self.state, self.control)
-
         model_name = kwargs.pop("model_name", None)
         if model_name is None and self.args.should_save:
             if self.args.hub_model_id is None:
@@ -5080,16 +5074,15 @@ class Trainer:
         self.is_tp_enabled = False
         if getattr(self.model, "tp_size", None) is not None and self.model.tp_size > 1:
             self.is_tp_enabled = True
-            if self.args.parallelism_config is None:
-                if is_accelerate_available("1.12.0"):
-                    if self.args.parallelism_config is None:
+            if self.args.parallelism_config is not None:
+                if is_accelerate_available("1.10.1"):
+                    if self.args.parallelism_config is not None:
                         from accelerate import ParallelismConfig
 
                         args["parallelism_config"] = ParallelismConfig(tp_size=self.model.tp_size)
                 else:
-                    raise ValueError("Requires accelerate>1.12.0 to use Tensor Parallelism.")
-            elif args["parallelism_config"].tp_size != self.model.tp_size:
-                args["parallelism_config"].tp_size = self.model.tp_size
+                    raise ValueError("Requires accelerate>1.10.1 to use Tensor Parallelism.")
+
         if is_accelerate_available("1.2.0"):
             # it we don't have the correct version, we will rely on env var instead that were set in TrainingArguments
             from accelerate.utils import TorchDynamoPlugin

transformers/trainer_callback.py

@@ -420,11 +420,6 @@ class TrainerCallback:
         Event called after a prediction step.
         """
 
-    def on_push_begin(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):
-        """
-        Event called before pushing the model to the hub, at the beginning of Trainer.push_to_hub and Trainer._push_from_checkpoint.
-        """
-
 
 class CallbackHandler(TrainerCallback):
     """Internal class that just calls the list of callbacks in order."""
@@ -537,9 +532,6 @@ class CallbackHandler(TrainerCallback):
     def on_prediction_step(self, args: TrainingArguments, state: TrainerState, control: TrainerControl):
         return self.call_event("on_prediction_step", args, state, control)
 
-    def on_push_begin(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):
-        return self.call_event("on_push_begin", args, state, control, **kwargs)
-
     def call_event(self, event, args, state, control, **kwargs):
         for callback in self.callbacks:
             result = getattr(callback, event)(

transformers/trainer_seq2seq.py

@@ -333,11 +333,7 @@ class Seq2SeqTrainer(Trainer):
             self.model.generation_config._from_model_config = False
 
         # Retrieves GenerationConfig from model.generation_config
-        # Update with defaults because earlier the generation config used ot be init
-        # with default values. Now we init it with `None` and keep defaults for BC
         gen_config = self.model.generation_config
-        default_gen_config = gen_config._get_default_generation_params()
-        gen_config.update(**default_gen_config, defaults_only=True)
         # in case the batch is shorter than max length, the output should be padded
         if generated_tokens.shape[-1] < gen_config.max_length:
             generated_tokens = self._pad_tensors_to_max_len(generated_tokens, gen_config.max_length)
@@ -378,7 +374,7 @@ class Seq2SeqTrainer(Trainer):
                 else self.processing_class.eos_token_id
             )
         else:
-            if getattr(self.model.config, "pad_token_id", None) is not None:
+            if self.model.config.pad_token_id is not None:
                 pad_token_id = self.model.config.pad_token_id
             else:
                 raise ValueError("Pad_token_id must be set in the configuration of the model, in order to pad tensors")

transformers/trainer_utils.py

@@ -924,7 +924,7 @@ def load_sharded_checkpoint(model, folder, strict=True, prefer_safe=True):
     shard_files = list(set(index["weight_map"].values()))
 
     # If strict=True, error before loading any of the state dicts.
-    # TODO: Here, update the weight map with the config.dynamic_weight_conversion
+    # TODO: Here, update the weigth map with the config.dynamic_weight_conversion
     loaded_keys = index["weight_map"].keys()
     model_keys = model.state_dict().keys()
     missing_keys = [key for key in model_keys if key not in loaded_keys]

transformers/training_args.py

@@ -340,17 +340,9 @@ class TrainingArguments:
             `save_total_limit=5` and `load_best_model_at_end`, the four last checkpoints will always be retained
             alongside the best model. When `save_total_limit=1` and `load_best_model_at_end`, it is possible that two
             checkpoints are saved: the last one and the best one (if they are different).
-        enable_jit_checkpoint (`bool`, *optional*, defaults to `False`):
-            Whether to enable Just-In-Time (JIT) checkpointing on SIGTERM signal. When enabled, training will
-            checkpoint upon receiving SIGTERM, allowing for graceful termination without losing
-            progress. This is particularly useful for shared clusters with preemptible workloads (e.g., Kueue).
-            **Important**: You must configure your orchestrator's graceful shutdown period to allow sufficient time
-            for checkpoint completion. For Kubernetes, set `terminationGracePeriodSeconds` in your job definition
-            (method varies by cloud-native trainer: Kubeflow, Ray, etc.). Note: the default is only 30 seconds,
-            which is typically insufficient. For Slurm, use `--signal=USR1@<seconds>` in your sbatch script to send
-            SIGTERM with adequate time before the job time limit. Calculate the required grace period as: longest
-            possible iteration time + checkpoint saving time. For example, if an iteration takes 2 minutes and
-            checkpoint saving takes 2 minutes, set at least 4 minutes (240 seconds) of grace time.
+        save_safetensors (`bool`, *optional*, defaults to `True`):
+            Use [safetensors](https://huggingface.co/docs/safetensors) saving and loading for state dicts instead of
+            default `torch.load` and `torch.save`.
         save_on_each_node (`bool`, *optional*, defaults to `False`):
             When doing multi-node distributed training, whether to save models and checkpoints on each node, or only on
             the main one.
@@ -593,9 +585,9 @@ class TrainingArguments:
             instance of `Dataset`.
         report_to (`str` or `list[str]`, *optional*, defaults to `"none"`):
             The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`,
-            `"clearml"`, `"codecarbon"`, `"comet_ml"`, `"dagshub"`, `"dvclive"`, `"flyte"`, `"mlflow"`, `"swanlab"`,
-            `"tensorboard"`, `"trackio"` and `"wandb"`. Use `"all"` to report to all integrations installed, `"none"`
-            for no integrations.
+            `"clearml"`, `"codecarbon"`, `"comet_ml"`, `"dagshub"`, `"dvclive"`, `"flyte"`, `"mlflow"`, `"neptune"`,
+            `"swanlab"`, `"tensorboard"`, `"trackio"` and `"wandb"`. Use `"all"` to report to all integrations
+            installed, `"none"` for no integrations.
         project (`str`, *optional*, defaults to `"huggingface"`):
             The name of the project to use for logging. Currently, only used by Trackio.
         trackio_space_id (`str` or `None`, *optional*, defaults to `"trackio"`):
@@ -860,7 +852,7 @@ class TrainingArguments:
     warmup_ratio: float | None = field(
         default=None,
         metadata={
-            "help": "This argument is deprecated and will be removed in v5.2. Use `warmup_steps` instead as it also works with float values."
+            "help": "This argument is deprecated and will be removed in v5. Use `warmup_steps` instead as it also works with float values."
         },
     )
 
@@ -937,24 +929,14 @@ class TrainingArguments:
                 " for `save_total_limit=5` and `load_best_model_at_end=True`, the four last checkpoints will always be"
                 " retained alongside the best model. When `save_total_limit=1` and `load_best_model_at_end=True`,"
                 " it is possible that two checkpoints are saved: the last one and the best one (if they are different)."
-                " Default is unlimited checkpoints."
+                " Default is unlimited checkpoints"
             )
         },
     )
-    enable_jit_checkpoint: bool = field(
-        default=False,
+    save_safetensors: bool = field(
+        default=True,
         metadata={
-            "help": (
-                "Whether to enable Just-In-Time (JIT) checkpointing on SIGTERM signal. "
-                "When enabled, training will checkpoint upon receiving SIGTERM, "
-                "allowing for graceful termination without losing progress. "
-                "This is particularly useful for shared clusters with preemptible workloads (Kueue). "
-                "IMPORTANT: You must configure your orchestrator's graceful shutdown period. "
-                "Kubernetes: set terminationGracePeriodSeconds (default 30s is insufficient!) in your job definition. "
-                "Slurm: use --signal=USR1@<seconds> in sbatch to send SIGTERM before time limit. "
-                "Calculate required grace period as: iteration time + checkpoint saving time. "
-                "Example: 2min iteration + 2min checkpoint = 240 seconds minimum."
-            )
+            "help": "Use safetensors saving and loading for state dicts instead of default torch.load and torch.save."
         },
     )
     save_on_each_node: bool = field(
@@ -1432,15 +1414,15 @@ class TrainingArguments:
             )
 
         # Parse in args that could be `dict` sent in from the CLI as a string
-        for valid_field in self._VALID_DICT_FIELDS:
-            passed_value = getattr(self, valid_field)
+        for field in self._VALID_DICT_FIELDS:
+            passed_value = getattr(self, field)
             # We only want to do this if the str starts with a bracket to indicate a `dict`
             # else its likely a filename if supported
             if isinstance(passed_value, str) and passed_value.startswith("{"):
                 loaded_dict = json.loads(passed_value)
                 # Convert str values to types if applicable
                 loaded_dict = _convert_str_dict(loaded_dict)
-                setattr(self, valid_field, loaded_dict)
+                setattr(self, field, loaded_dict)
 
         # expand paths, if not os.makedirs("~/bar") will make directory
         # in the current directory instead of the actual home
@@ -1522,6 +1504,14 @@ class TrainingArguments:
                         f"steps, but found {self.save_steps}, which is not a round multiple of {self.eval_steps}."
                     )
 
+        if not self.save_safetensors:
+            logger.info(
+                f"Found safetensors installation, but --save_safetensors={self.save_safetensors}. "
+                f"Safetensors should be a preferred weights saving format due to security and performance reasons. "
+                f"If your model cannot be saved by safetensors please feel free to open an issue at "
+                f"https://github.com/huggingface/safetensors!"
+            )
+
         if (
             self.load_best_model_at_end or self.lr_scheduler_type == SchedulerType.REDUCE_ON_PLATEAU
         ) and self.metric_for_best_model is None:
@@ -1530,14 +1520,16 @@ class TrainingArguments:
             self.greater_is_better = not self.metric_for_best_model.endswith("loss")
         if is_torch_available():
             if self.bf16 or self.bf16_full_eval:
-                if (
-                    not self.use_cpu and not is_torch_bf16_gpu_available() and not is_torch_xla_available()
-                ):  # added for tpu support
-                    error_message = "Your setup doesn't support bf16/gpu. You need to assign use_cpu if you want to train the model on CPU"
-                    if is_torch_cuda_available():
-                        error_message += " You need Ampere+ GPU with cuda>=11.0"
-                    # gpu
-                    raise ValueError(error_message)
+                if self.use_cpu and not is_torch_xla_available():
+                    # cpu
+                    raise ValueError("Your setup doesn't support bf16/(cpu, tpu, neuroncore). You need torch>=1.10")
+                elif not self.use_cpu:
+                    if not is_torch_bf16_gpu_available() and not is_torch_xla_available():  # added for tpu support
+                        error_message = "Your setup doesn't support bf16/gpu."
+                        if is_torch_cuda_available():
+                            error_message += " You need Ampere+ GPU with cuda>=11.0"
+                        # gpu
+                        raise ValueError(error_message)
 
         if self.fp16 and self.bf16:
             raise ValueError("At most one of fp16 and bf16 can be True, but not both")
@@ -2367,8 +2359,8 @@ class TrainingArguments:
             report_to (`str` or `list[str]`, *optional*, defaults to `"none"`):
                 The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`,
                 `"clearml"`, `"codecarbon"`, `"comet_ml"`, `"dagshub"`, `"dvclive"`, `"flyte"`, `"mlflow"`,
-                `"swanlab"`, `"tensorboard"`, `"trackio"` and `"wandb"`. Use `"all"` to report to all integrations
-                installed, `"none"` for no integrations.
+                `"neptune"`, `"swanlab"`, `"tensorboard"`, `"trackio"` and `"wandb"`. Use `"all"` to report to all
+                integrations installed, `"none"` for no integrations.
             first_step (`bool`, *optional*, defaults to `False`):
                 Whether to log and evaluate the first `global_step` or not.
             nan_inf_filter (`bool`, *optional*, defaults to `True`):
@@ -2573,7 +2565,7 @@ class TrainingArguments:
         ```
         """
         if warmup_ratio is not None:
-            logger.warning("warmup_ratio is deprecated and will be removed in v5.2 . Use `warmup_steps` instead.")
+            logger.warning("warmup_ratio is deprecated and will be removed in v5. Use `warmup_steps` instead.")
             warmup_steps = warmup_ratio
 
         self.lr_scheduler_type = SchedulerType(name)
@@ -2750,24 +2742,10 @@ class TrainingArguments:
                         fsdp_plugin_args["transformer_cls_names_to_wrap"] = ",".join(
                             self.fsdp_config["transformer_layer_cls_to_wrap"]
                         )
-            fsdp_version = int(self.fsdp_config.get("version", 1))
-            fsdp_plugin_args["fsdp_version"] = fsdp_version
+            fsdp_plugin_args["fsdp_version"] = self.fsdp_config.get("fsdp_version", 1)
             prefetch_policy = self.fsdp_config.get("backward_prefetch", "NO_PREFETCH")
-            if fsdp_version == 2:
-                fsdp_plugin_args["reshard_after_forward"] = str_to_bool(
-                    str(self.fsdp_config.get("reshard_after_forward", "false")).lower()
-                )
-            else:
-                fsdp_plugin_args["forward_prefetch"] = str_to_bool(
-                    str(self.fsdp_config.get("forward_prefetch", "false")).lower()
-                )
-                fsdp_plugin_args["backward_prefetch"] = prefetch_policy.upper()
-                fsdp_plugin_args["reshard_after_forward"] = str(
-                    self.fsdp_config.get("reshard_after_forward", "FULL_SHARD")
-                ).lower()
-                fsdp_plugin_args["use_orig_params"] = str_to_bool(
-                    str(self.fsdp_config.get("use_orig_params", "true")).lower()
-                )
+            fsdp_plugin_args["backward_prefetch"] = prefetch_policy.upper()
+            fsdp_plugin_args["forward_prefetch"] = str(self.fsdp_config.get("forward_prefetch", "false")).lower()
 
             sync_module_states = str(self.fsdp_config.get("sync_module_states", "true")).lower()
             cpu_ram_efficient_loading = str(self.fsdp_config.get("cpu_ram_efficient_loading", "false")).lower()
@@ -2777,10 +2755,11 @@ class TrainingArguments:
                 raise ValueError('`sync_module_states` must be `"True"` if `cpu_ram_efficient_loading` is `"True"`')
 
             # we need to set the env here as otherwise we get a warning in accelerate + we need to set it for transformers
-            fsdp_plugin_args["cpu_ram_efficient_loading"] = str_to_bool(cpu_ram_efficient_loading)
+            fsdp_plugin_args["cpu_ram_efficient_loading"] = cpu_ram_efficient_loading
             os.environ["FSDP_CPU_RAM_EFFICIENT_LOADING"] = cpu_ram_efficient_loading
 
-            fsdp_plugin_args["sync_module_states"] = str_to_bool(sync_module_states)
+            fsdp_plugin_args["sync_module_states"] = sync_module_states
+            fsdp_plugin_args["use_orig_params"] = str(self.fsdp_config.get("use_orig_params", "true")).lower()
 
         return fsdp_plugin_args
 
@@ -2792,18 +2771,3 @@ class ParallelMode(Enum):
     SAGEMAKER_MODEL_PARALLEL = "sagemaker_model_parallel"
     SAGEMAKER_DATA_PARALLEL = "sagemaker_data_parallel"
     TPU = "tpu"
-
-
-def str_to_bool(value, to_bool: bool = True) -> int | bool:
-    """
-    Converts a string representation of truth to `True` (1) or `False` (0).
-
-    True values are `y`, `yes`, `t`, `true`, `on`, and `1`; False value are `n`, `no`, `f`, `false`, `off`, and `0`;
-    """
-    value = value.lower()
-    if value in ("y", "yes", "t", "true", "on", "1"):
-        return 1 if not to_bool else True
-    elif value in ("n", "no", "f", "false", "off", "0"):
-        return 0 if not to_bool else False
-    else:
-        raise ValueError(f"invalid truth value {value}")

transformers/utils/__init__.py

@@ -49,7 +49,6 @@ from .generic import (
     PaddingStrategy,
     TensorType,
     TransformersKwargs,
-    _is_tensor_or_array_like,
     can_return_loss,
     can_return_tuple,
     expand_dims,
@@ -92,6 +91,7 @@ from .hub import (
     extract_commit_hash,
     has_file,
     http_user_agent,
+    is_offline_mode,
     list_repo_templates,
     try_to_load_from_cache,
 )
@@ -114,6 +114,8 @@ from .import_utils import (
     is_apex_available,
     is_apollo_torch_available,
     is_aqlm_available,
+    is_auto_awq_available,
+    is_auto_gptq_available,
     is_auto_round_available,
     is_av_available,
     is_bitsandbytes_available,
@@ -127,8 +129,7 @@ from .import_utils import (
     is_datasets_available,
     is_decord_available,
     is_detectron2_available,
-    is_env_variable_false,
-    is_env_variable_true,
+    is_eetq_available,
     is_essentia_available,
     is_faiss_available,
     is_fbgemm_gpu_available,
@@ -145,7 +146,6 @@ from .import_utils import (
     is_gguf_available,
     is_gptqmodel_available,
     is_grokadamw_available,
-    is_grouped_mm_available,
     is_habana_gaudi1,
     is_hadamard_available,
     is_hqq_available,
@@ -161,7 +161,6 @@ from .import_utils import (
     is_libcst_available,
     is_librosa_available,
     is_liger_kernel_available,
-    is_llm_awq_available,
     is_lomo_available,
     is_matplotlib_available,
     is_mistral_common_available,
@@ -170,7 +169,6 @@ from .import_utils import (
     is_ninja_available,
     is_nltk_available,
     is_num2words_available,
-    is_numba_available,
     is_onnx_available,
     is_openai_available,
     is_optimum_available,
@@ -185,7 +183,6 @@ from .import_utils import (
     is_pyctcdecode_available,
     is_pytesseract_available,
     is_pytest_available,
-    is_pytest_order_available,
     is_pytorch_quantization_available,
     is_quanto_greater,
     is_quark_available,
@@ -248,7 +245,6 @@ from .import_utils import (
     is_xlstm_available,
     is_yt_dlp_available,
     requires_backends,
-    torch_compilable_check,
     torch_only_method,
 )
 from .kernel_config import KernelConfig

transformers/utils/attention_visualizer.py

@@ -21,7 +21,7 @@ from ..models.auto.auto_factory import _get_model_class
 from ..models.auto.configuration_auto import AutoConfig
 from ..models.auto.modeling_auto import MODEL_FOR_PRETRAINING_MAPPING, MODEL_MAPPING
 from ..models.auto.processing_auto import PROCESSOR_MAPPING_NAMES, AutoProcessor
-from ..models.auto.tokenization_auto import AutoTokenizer
+from ..models.auto.tokenization_auto import TOKENIZER_MAPPING_NAMES, AutoTokenizer
 from .import_utils import is_torch_available
 
 
@@ -76,7 +76,7 @@ def generate_attention_matrix_from_mask(
         f"{YELLOW}{BLACK_SQUARE}{RESET}"
         if mask[0, j]
         else f"{GREEN}{BLACK_SQUARE}{RESET}"
-        if j == 0
+        if 0 == j
         else BLACK_SQUARE
         if mask[0, j]
         else WHITE_SQUARE
@@ -199,12 +199,12 @@ class AttentionMaskVisualizer:
             if "token_type_ids" in inputs:  # TODO inspect signature of update causal mask
                 kwargs["token_type_ids"] = inputs["token_type_ids"]
             tokens = processor.tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])
-        else:
+        elif self.config.model_type in TOKENIZER_MAPPING_NAMES:
             tokenizer = AutoTokenizer.from_pretrained(self.repo_id)
             tokens = tokenizer.tokenize(input_sentence)
             attention_mask = tokenizer(input_sentence, return_tensors="pt")["attention_mask"]
-            if attention_mask is None:
-                raise ValueError(f"Model type {self.config.model_type} does not support attention visualization")
+        else:
+            raise ValueError(f"Model type {model.config.model_type} does not support attention visualization")
 
         model.config._attn_implementation = "eager"
         model.train()