transformers 5.0.0rc1__py3-none-any.whl → 5.0.0rc3__py3-none-any.whl

transformers/utils/auto_docstring.py

@@ -1,4 +1,3 @@
-# coding=utf-8
 # Copyright 2025 HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +16,8 @@ import inspect
 import os
 import textwrap
 from pathlib import Path
-from typing import get_args
+from types import UnionType
+from typing import Union, get_args, get_origin
 
 import regex as re
 
@@ -45,6 +45,7 @@ AUTODOC_FILES = [
 
 PLACEHOLDER_TO_AUTO_MODULE = {
     "image_processor_class": ("image_processing_auto", "IMAGE_PROCESSOR_MAPPING_NAMES"),
+    "tokenizer_class": ("tokenization_auto", "TOKENIZER_MAPPING_NAMES"),
     "video_processor_class": ("video_processing_auto", "VIDEO_PROCESSOR_MAPPING_NAMES"),
     "feature_extractor_class": ("feature_extraction_auto", "FEATURE_EXTRACTOR_MAPPING_NAMES"),
     "processor_class": ("processing_auto", "PROCESSOR_MAPPING_NAMES"),
@@ -53,10 +54,12 @@ PLACEHOLDER_TO_AUTO_MODULE = {
 
 UNROLL_KWARGS_METHODS = {
     "preprocess",
+    "__call__",
 }
 
 UNROLL_KWARGS_CLASSES = {
     "ImageProcessorFast",
+    "ProcessorMixin",
 }
 
 HARDCODED_CONFIG_FOR_MODELS = {
@@ -68,6 +71,7 @@ HARDCODED_CONFIG_FOR_MODELS = {
     "esmfold": "EsmConfig",
     "parakeet": "ParakeetCTCConfig",
     "lasr": "LasrCTCConfig",
+    "wav2vec2-with-lm": "Wav2Vec2Config",
 }
 
 _re_checkpoint = re.compile(r"\[(.+?)\]\((https://huggingface\.co/.+?)\)")
@@ -250,6 +254,270 @@ class ImageProcessorArgs:
     }
 
 
+class ProcessorArgs:
+    # __init__ arguments
+    image_processor = {
+        "description": """
+    The image processor is a required input.
+    """,
+        "type": "{image_processor_class}",
+    }
+
+    tokenizer = {
+        "description": """
+    The tokenizer is a required input.
+    """,
+        "type": "{tokenizer_class}",
+    }
+
+    video_processor = {
+        "description": """
+    The video processor is a required input.
+    """,
+        "type": "{video_processor_class}",
+    }
+
+    audio_processor = {
+        "description": """
+    The audio processor is a required input.
+    """,
+        "type": "{audio_processor_class}",
+    }
+
+    feature_extractor = {
+        "description": """
+    The feature extractor is a required input.
+    """,
+        "type": "{feature_extractor_class}",
+    }
+
+    chat_template = {
+        "description": """
+    A Jinja template to convert lists of messages in a chat into a tokenizable string.
+    """,
+        "type": "str",
+    }
+
+    # __call__ arguments
+    text = {
+        "description": """
+    The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
+    (pretokenized string). If you pass a pretokenized input, set `is_split_into_words=True` to avoid ambiguity with batched inputs.
+    """,
+    }
+
+    audio = {
+        "description": """
+    The audio or batch of audios to be prepared. Each audio can be a NumPy array or PyTorch tensor.
+    In case of a NumPy array/PyTorch tensor, each audio should be of shape (C, T), where C is a number of channels,
+    and T is the sample length of the audio.
+    """,
+    }
+
+    audios = {
+        "description": """
+    The audio or batch of audios to be prepared. Each audio can be a NumPy array or PyTorch tensor.
+    In case of a NumPy array/PyTorch tensor, each audio should be of shape (C, T), where C is a number of channels,
+    and T is the sample length of the audio.
+    """,
+    }
+
+    return_tensors = {
+        "description": """
+    If set, will return tensors of a particular framework. Acceptable values are:
+
+    - `'pt'`: Return PyTorch `torch.Tensor` objects.
+    - `'np'`: Return NumPy `np.ndarray` objects.
+    """,
+        "shape": None,
+    }
+
+    # Standard tokenizer arguments
+    add_special_tokens = {
+        "description": """
+    Whether or not to add special tokens when encoding the sequences. This will use the underlying
+    [`PretrainedTokenizerBase.build_inputs_with_special_tokens`] function, which defines which tokens are
+    automatically added to the input ids. This is useful if you want to add `bos` or `eos` tokens
+    automatically.
+    """,
+        "type": "bool",
+    }
+
+    padding = {
+        "description": """
+    Activates and controls padding. Accepts the following values:
+
+    - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+      sequence is provided).
+    - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+      acceptable input length for the model if that argument is not provided.
+    - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+      lengths).
+    """,
+        "type": "bool, str or [`~utils.PaddingStrategy`]",
+    }
+
+    truncation = {
+        "description": """
+    Activates and controls truncation. Accepts the following values:
+
+    - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or
+      to the maximum acceptable input length for the model if that argument is not provided. This will
+      truncate token by token, removing a token from the longest sequence in the pair if a pair of
+      sequences (or a batch of pairs) is provided.
+    - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+      maximum acceptable input length for the model if that argument is not provided. This will only
+      truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
+    - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the
+      maximum acceptable input length for the model if that argument is not provided. This will only
+      truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
+    - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
+      greater than the model maximum admissible input size).
+    """,
+        "type": "bool, str or [`~tokenization_utils_base.TruncationStrategy`]",
+    }
+
+    max_length = {
+        "description": """
+    Controls the maximum length to use by one of the truncation/padding parameters.
+
+    If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
+    is required by one of the truncation/padding parameters. If the model has no specific maximum input
+    length (like XLNet) truncation/padding to a maximum length will be deactivated.
+    """,
+        "type": "int",
+    }
+
+    stride = {
+        "description": """
+    If set to a number along with `max_length`, the overflowing tokens returned when
+    `return_overflowing_tokens=True` will contain some tokens from the end of the truncated sequence
+    returned to provide some overlap between truncated and overflowing sequences. The value of this
+    argument defines the number of overlapping tokens.
+    """,
+        "type": "int",
+    }
+
+    pad_to_multiple_of = {
+        "description": """
+    If set will pad the sequence to a multiple of the provided value. Requires `padding` to be activated.
+    This is especially useful to enable using Tensor Cores on NVIDIA hardware with compute capability
+    `>= 7.5` (Volta).
+    """,
+        "type": "int",
+    }
+
+    return_token_type_ids = {
+        "description": """
+    Whether to return token type IDs. If left to the default, will return the token type IDs according to
+    the specific tokenizer's default, defined by the `return_outputs` attribute.
+
+    [What are token type IDs?](../glossary#token-type-ids)
+    """,
+        "type": "bool",
+    }
+
+    return_attention_mask = {
+        "description": """
+    Whether to return the attention mask. If left to the default, will return the attention mask according
+    to the specific tokenizer's default, defined by the `return_outputs` attribute.
+
+    [What are attention masks?](../glossary#attention-mask)
+    """,
+        "type": "bool",
+    }
+
+    return_overflowing_tokens = {
+        "description": """
+    Whether or not to return overflowing token sequences. If a pair of sequences of input ids (or a batch
+    of pairs) is provided with `truncation_strategy = longest_first` or `True`, an error is raised instead
+    of returning overflowing tokens.
+    """,
+        "type": "bool",
+    }
+
+    return_special_tokens_mask = {
+        "description": """
+    Whether or not to return special tokens mask information.
+    """,
+        "type": "bool",
+    }
+
+    return_offsets_mapping = {
+        "description": """
+    Whether or not to return `(char_start, char_end)` for each token.
+
+    This is only available on fast tokenizers inheriting from [`PreTrainedTokenizerFast`], if using
+    Python's tokenizer, this method will raise `NotImplementedError`.
+    """,
+        "type": "bool",
+    }
+
+    return_length = {
+        "description": """
+    Whether or not to return the lengths of the encoded inputs.
+    """,
+        "type": "bool",
+    }
+
+    verbose = {
+        "description": """
+    Whether or not to print more information and warnings.
+    """,
+        "type": "bool",
+    }
+
+    text_pair = {
+        "description": """
+    Optional second sequence to be encoded. This can be a string, a list of strings (tokenized string using
+    the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
+    method).
+    """,
+        "type": "str, list[str] or list[int]",
+    }
+
+    text_target = {
+        "description": """
+    The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
+    list of strings (pretokenized string). If you pass pretokenized input, set `is_split_into_words=True`
+    to avoid ambiguity with batched inputs.
+    """,
+        "type": "str, list[str] or list[list[str]]",
+    }
+
+    text_pair_target = {
+        "description": """
+    The sequence or batch of sequences to be encoded as target texts. Each sequence can be a string or a
+    list of strings (pretokenized string). If you pass pretokenized input, set `is_split_into_words=True`
+    to avoid ambiguity with batched inputs.
+    """,
+        "type": "str, list[str] or list[list[str]]",
+    }
+
+    is_split_into_words = {
+        "description": """
+    Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the
+    tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace)
+    which it will tokenize. This is useful for NER or token classification.
+    """,
+        "type": "bool",
+    }
+
+    boxes = {
+        "description": """
+    Word-level bounding boxes. Each bounding box should be normalized to be on a 0-1000 scale.
+    """,
+        "type": "list[list[int]] or list[list[list[int]]]",
+    }
+
+    word_labels = {
+        "description": """
+    Word-level integer labels (for token classification tasks such as FUNSD, CORD).
+    """,
+        "type": "list[int] or list[list[int]]",
+    }
+
+
 class ModelArgs:
     labels = {
         "description": """
@@ -1131,6 +1399,63 @@ def get_model_name(obj):
     return "model"
 
 
+def generate_processor_intro(cls) -> str:
+    """
+    Generate the intro docstring for a processor class based on its attributes.
+
+    Args:
+        cls: Processor class to generate intro for
+
+    Returns:
+        str: Generated intro text
+    """
+    class_name = cls.__name__
+
+    # Get attributes and their corresponding class names
+    attributes = cls.get_attributes()
+    if not attributes:
+        return ""
+
+    # Build list of component names and their classes
+    components = []
+    component_classes = []
+
+    for attr in attributes:
+        # Get the class name for this attribute
+        class_attr = f"{attr}_class"
+        # Format attribute name for display
+        attr_display = attr.replace("_", " ")
+        components.append(attr_display)
+        component_classes.append(f"[`{{{class_attr}}}`]")
+    if not components:
+        return ""
+
+    # Generate the intro text
+    if len(components) == 1:
+        components_text = f"a {components[0]}"
+        classes_text = component_classes[0]
+        classes_text_short = component_classes[0].replace("[`", "[`~")
+    elif len(components) == 2:
+        components_text = f"a {components[0]} and a {components[1]}"
+        classes_text = f"{component_classes[0]} and {component_classes[1]}"
+        classes_text_short = (
+            f"{component_classes[0].replace('[`', '[`~')} and {component_classes[1].replace('[`', '[`~')}"
+        )
+    else:
+        components_text = ", ".join(f"a {c}" for c in components[:-1]) + f", and a {components[-1]}"
+        classes_text = ", ".join(component_classes[:-1]) + f", and {component_classes[-1]}"
+        classes_short = [c.replace("[`", "[`~") for c in component_classes]
+        classes_text_short = ", ".join(classes_short[:-1]) + f", and {classes_short[-1]}"
+
+    intro = f"""Constructs a {class_name} which wraps {components_text} into a single processor.
+
+[`{class_name}`] offers all the functionalities of {classes_text}. See the
+{classes_text_short} for more information.
+"""
+
+    return intro
+
+
 def get_placeholders_dict(placeholders: list, model_name: str) -> dict:
     """
     Get the dictionary of placeholders for the given model name.
@@ -1152,7 +1477,9 @@ def get_placeholders_dict(placeholders: list, model_name: str) -> dict:
                 place_holder_value = None
             if place_holder_value is not None:
                 if isinstance(place_holder_value, (list, tuple)):
-                    place_holder_value = place_holder_value[0]
+                    place_holder_value = (
+                        place_holder_value[-1] if place_holder_value[-1] is not None else place_holder_value[0]
+                    )
                 placeholders_dict[placeholder] = place_holder_value if place_holder_value is not None else placeholder
             else:
                 placeholders_dict[placeholder] = placeholder
@@ -1280,38 +1607,46 @@ def _get_model_info(func, parent_class):
     return model_name_lowercase, class_name, config_class
 
 
-def _process_parameter_type(param, param_name, func):
+def _process_parameter_type(param):
     """
     Process and format a parameter's type annotation.
 
     Args:
         param (`inspect.Parameter`): The parameter from the function signature
-        param_name (`str`): The name of the parameter
-        func (`function`): The function the parameter belongs to
     """
     optional = False
-    if param.annotation != inspect.Parameter.empty:
-        param_type = param.annotation
-        if "typing" in str(param_type):
-            param_type = "".join(str(param_type).split("typing.")).replace("transformers.", "~")
-        elif hasattr(param_type, "__module__"):
-            param_type = f"{param_type.__module__.replace('transformers.', '~').replace('builtins', '')}.{param.annotation.__name__}"
-            if param_type[0] == ".":
-                param_type = param_type[1:]
-        else:
-            if False:
-                print(
-                    f"[ERROR] {param_type} for {param_name} of {func.__qualname__} in file {func.__code__.co_filename} has an invalid type"
-                )
-        if "ForwardRef" in param_type:
-            param_type = re.sub(r"ForwardRef\('([\w.]+)'\)", r"\1", param_type)
-        if "Optional" in param_type:
-            param_type = re.sub(r"Optional\[(.*?)\]", r"\1", param_type)
+    if param.annotation == inspect.Parameter.empty:
+        return "", False
+    elif param.annotation is None:
+        return "None", True
+    # This is, astonishingly, the right way to do it: https://docs.python.org/3/library/typing.html#typing.Union
+    elif get_origin(param.annotation) is Union or get_origin(param.annotation) is UnionType:
+        subtypes = get_args(param.annotation)
+    else:
+        subtypes = [param.annotation]  # Just pretend it's a single-element union so we don't need two code paths
+    out_str = []
+    for subtype in subtypes:
+        if subtype is type(None):
             optional = True
+            continue
+        if hasattr(subtype, "__module__") and hasattr(subtype, "__name__"):
+            subtype = f"{subtype.__module__.replace('transformers.', '~').replace('builtins', '').replace('typing.', '')}.{subtype.__name__}".removeprefix(
+                "."
+            )
+        else:
+            subtype = str(subtype)  # Just give up
+        if "ForwardRef" in subtype:
+            subtype = re.sub(r"ForwardRef\('([\w.]+)'\)", r"\1", subtype)
+        out_str.append(subtype)
+
+    if param.default is not inspect.Parameter.empty:
+        optional = True
+    if not out_str:
+        return "", optional
+    elif len(out_str) == 1:
+        return out_str[0], optional
     else:
-        param_type = ""
-
-    return param_type, optional
+        return f"Union[{', '.join(out_str)}]", optional
 
 
 def _get_parameter_info(param_name, documented_params, source_args_dict, param_type, optional):
@@ -1343,13 +1678,14 @@ def _get_parameter_info(param_name, documented_params, source_args_dict, param_t
         ):
             param_type = documented_params[param_name]["type"]
         optional = documented_params[param_name]["optional"]
-        shape = documented_params[param_name]["shape"]
+        shape = documented_params[param_name].get("shape", None)
         shape_string = shape if shape else ""
         additional_info = documented_params[param_name]["additional_info"] or ""
         description = f"{documented_params[param_name]['description']}\n"
     elif param_name in source_args_dict:
         # Parameter is documented in ModelArgs or ImageProcessorArgs
-        shape = source_args_dict[param_name]["shape"]
+        param_type = source_args_dict[param_name].get("type", param_type)
+        shape = source_args_dict[param_name].get("shape", None)
         shape_string = " " + shape if shape else ""
         description = source_args_dict[param_name]["description"]
         additional_info = source_args_dict[param_name].get("additional_info", None)
@@ -1377,9 +1713,16 @@ def _process_regular_parameters(
         undocumented_parameters (`list`): List to append undocumented parameters to
     """
     docstring = ""
-    source_args_dict = (
-        get_args_doc_from_source([ModelArgs, ImageProcessorArgs]) if source_args_dict is None else source_args_dict
-    )
+    # Check if this is a processor by inspecting class hierarchy
+    is_processor = _is_processor_class(func, parent_class)
+
+    # Use appropriate args source based on whether it's a processor or not
+    if source_args_dict is None:
+        if is_processor:
+            source_args_dict = get_args_doc_from_source([ModelArgs, ImageProcessorArgs, ProcessorArgs])
+        else:
+            source_args_dict = get_args_doc_from_source([ModelArgs, ImageProcessorArgs])
+
     missing_args = {}
 
     for param_name, param in sig.parameters.items():
@@ -1392,7 +1735,7 @@ def _process_regular_parameters(
             continue
 
         # Process parameter type and optional status
-        param_type, optional = _process_parameter_type(param, param_name, func)
+        param_type, optional = _process_parameter_type(param)
 
         # Check for default value
         param_default = ""
@@ -1455,6 +1798,47 @@ def find_sig_line(lines, line_end):
     return sig_line_end
 
 
+def _is_processor_class(func, parent_class):
+    """
+    Check if a function belongs to a ProcessorMixin class.
+
+    Uses two methods:
+    1. Check parent_class inheritance (if provided)
+    2. Check if the source file is named processing_*.py (multimodal processors)
+       vs image_processing_*.py, video_processing_*.py, etc. (single-modality processors)
+
+    Args:
+        func: The function to check
+        parent_class: Optional parent class (if available)
+
+    Returns:
+        bool: True if this is a multimodal processor (inherits from ProcessorMixin), False otherwise
+    """
+    # First, check if parent_class is provided and use it
+    if parent_class is not None:
+        return "ProcessorMixin" in parent_class.__name__ or any(
+            "ProcessorMixin" in base.__name__ for base in parent_class.__mro__
+        )
+
+    # If parent_class is None, check the filename
+    # Multimodal processors are in files named "processing_*.py"
+    # Single-modality processors are in "image_processing_*.py", "video_processing_*.py", etc.
+    try:
+        source_file = inspect.getsourcefile(func)
+        if source_file:
+            filename = os.path.basename(source_file)
+            # Check if it's a processing file (processing_*.py) but NOT a single-modality processor
+            # Single-modality processors: image_processing_*.py, video_processing_*.py, feature_extraction_*.py
+            if filename.startswith("processing_") and filename.endswith(".py"):
+                # This is a multimodal processor file
+                return True
+    except Exception:
+        pass
+
+    # Default to False (conservative approach)
+    return False
+
+
 def _process_kwargs_parameters(sig, func, parent_class, documented_kwargs, indent_level, undocumented_parameters):
     """
     Process **kwargs parameters if needed.
@@ -1468,7 +1852,15 @@ def _process_kwargs_parameters(sig, func, parent_class, documented_kwargs, inden
         undocumented_parameters (`list`): List to append undocumented parameters to
     """
     docstring = ""
-    source_args_dict = get_args_doc_from_source(ImageProcessorArgs)
+
+    # Check if this is a processor by inspecting class hierarchy
+    is_processor = _is_processor_class(func, parent_class)
+
+    # Use appropriate args source based on whether it's a processor or not
+    if is_processor:
+        source_args_dict = get_args_doc_from_source([ImageProcessorArgs, ProcessorArgs])
+    else:
+        source_args_dict = get_args_doc_from_source(ImageProcessorArgs)
 
     # Check if we need to add typed kwargs description to the docstring
     unroll_kwargs = func.__name__ in UNROLL_KWARGS_METHODS
@@ -1477,7 +1869,6 @@ def _process_kwargs_parameters(sig, func, parent_class, documented_kwargs, inden
         unroll_kwargs = any(
             unroll_kwargs_class in parent_class.__name__ for unroll_kwargs_class in UNROLL_KWARGS_CLASSES
         )
-
     if unroll_kwargs:
         # get all unpackable "kwargs" parameters
         kwargs_parameters = [
@@ -1497,6 +1888,117 @@ def _process_kwargs_parameters(sig, func, parent_class, documented_kwargs, inden
 
             # Process each kwarg parameter
             for param_name, param_type_annotation in kwarg_param.annotation.__args__[0].__annotations__.items():
+                # Handle nested kwargs structures for processors
+                if is_processor and param_name.endswith("_kwargs"):
+                    # Check if this is a basic kwargs type that should be skipped
+                    # Basic kwargs types are generic containers that shouldn't be documented as individual params
+                    basic_kwargs_types = ["TextKwargs", "ImagesKwargs", "VideosKwargs", "AudioKwargs"]
+
+                    # Get the actual type (unwrap Optional if needed)
+                    actual_type = param_type_annotation
+                    type_name = getattr(param_type_annotation, "__name__", None)
+                    if type_name is None and hasattr(param_type_annotation, "__origin__"):
+                        # Handle Optional[Type] or Union cases
+                        args = getattr(param_type_annotation, "__args__", ())
+                        for arg in args:
+                            if arg is not type(None):
+                                actual_type = arg
+                                type_name = getattr(arg, "__name__", None)
+                                break
+
+                    # Skip only if it's one of the basic kwargs types
+                    if type_name in basic_kwargs_types:
+                        continue
+
+                    # Otherwise, unroll the custom typed kwargs
+                    # Get the nested TypedDict's annotations
+                    if hasattr(actual_type, "__annotations__"):
+                        nested_kwargs_doc = getattr(actual_type, "__doc__", None)
+                        documented_nested_kwargs = {}
+                        if nested_kwargs_doc:
+                            documented_nested_kwargs = parse_docstring(nested_kwargs_doc)[0]
+
+                        # Only process fields that are documented in the custom kwargs class's own docstring
+                        # This prevents showing too many inherited parameters
+                        if not documented_nested_kwargs:
+                            # No documentation in the custom kwargs class, skip unrolling
+                            continue
+
+                        # Process each field in the custom typed kwargs
+                        for nested_param_name, nested_param_type in actual_type.__annotations__.items():
+                            # Only document parameters that are explicitly documented in the TypedDict's docstring
+                            if nested_param_name not in documented_nested_kwargs:
+                                continue
+                            nested_param_type_str = str(nested_param_type)
+                            nested_optional = False
+
+                            # Process parameter type
+                            if "typing" in nested_param_type_str:
+                                nested_param_type_str = "".join(nested_param_type_str.split("typing.")).replace(
+                                    "transformers.", "~"
+                                )
+                            else:
+                                nested_param_type_str = f"{nested_param_type_str.replace('transformers.', '~').replace('builtins', '')}.{nested_param_name}"
+                            if "ForwardRef" in nested_param_type_str:
+                                nested_param_type_str = re.sub(
+                                    r"ForwardRef\('([\w.]+)'\)", r"\1", nested_param_type_str
+                                )
+                            if "Optional" in nested_param_type_str:
+                                nested_param_type_str = re.sub(r"Optional\[(.*?)\]", r"\1", nested_param_type_str)
+                                nested_optional = True
+
+                            # Check for default value
+                            nested_param_default = ""
+                            if parent_class is not None:
+                                nested_param_default = str(getattr(parent_class, nested_param_name, ""))
+                                nested_param_default = (
+                                    f", defaults to `{nested_param_default}`" if nested_param_default != "" else ""
+                                )
+
+                            # Only use the TypedDict's own docstring, not source_args_dict
+                            # This prevents pulling in too many inherited parameters
+                            (
+                                nested_param_type_str,
+                                nested_optional_string,
+                                nested_shape_string,
+                                nested_additional_info,
+                                nested_description,
+                                nested_is_documented,
+                            ) = _get_parameter_info(
+                                nested_param_name,
+                                documented_nested_kwargs,
+                                {},  # Empty dict - only use TypedDict's own docstring
+                                nested_param_type_str,
+                                nested_optional,
+                            )
+
+                            # nested_is_documented should always be True here since we filter for it above
+                            # Check if type is missing
+                            if nested_param_type_str == "":
+                                print(
+                                    f"🚨 {nested_param_name} for {type_name} in file {func.__code__.co_filename} has no type"
+                                )
+                            nested_param_type_str = (
+                                nested_param_type_str if "`" in nested_param_type_str else f"`{nested_param_type_str}`"
+                            )
+                            # Format the parameter docstring
+                            if nested_additional_info:
+                                docstring += set_min_indent(
+                                    f"{nested_param_name} ({nested_param_type_str}{nested_additional_info}):{nested_description}",
+                                    indent_level + 8,
+                                )
+                            else:
+                                docstring += set_min_indent(
+                                    f"{nested_param_name} ({nested_param_type_str}{nested_shape_string}{nested_optional_string}{nested_param_default}):{nested_description}",
+                                    indent_level + 8,
+                                )
+
+                        # Skip processing the _kwargs parameter itself since we've processed its contents
+                        continue
+                    else:
+                        # If we can't get annotations, skip this parameter
+                        continue
+
                 param_type = str(param_type_annotation)
                 optional = False
 
@@ -1547,6 +2049,43 @@ def _process_kwargs_parameters(sig, func, parent_class, documented_kwargs, inden
     return docstring
 
 
+def _add_return_tensors_for_processor_call(func, parent_class, docstring, indent_level):
+    """
+    Add return_tensors parameter documentation for processor __call__ methods if not already present.
+
+    Args:
+        func (`function`): Function being processed
+        parent_class (`class`): Parent class of the function
+        docstring (`str`): Current docstring being built
+        indent_level (`int`): Indentation level
+
+    Returns:
+        str: Updated docstring with return_tensors if applicable
+    """
+    # Check if this is a processor __call__ method
+    is_processor_call = False
+    if func.__name__ == "__call__":
+        # Check if this is a processor by inspecting class hierarchy
+        is_processor_call = _is_processor_class(func, parent_class)
+
+    # If it's a processor __call__ method and return_tensors is not already documented
+    if is_processor_call and "return_tensors" not in docstring:
+        # Get the return_tensors documentation from ImageProcessorArgs
+        source_args_dict = get_args_doc_from_source(ProcessorArgs)
+        return_tensors_info = source_args_dict["return_tensors"]
+        param_type = return_tensors_info.get("type", "`str` or [`~utils.TensorType`]")
+        description = return_tensors_info["description"]
+
+        # Format the parameter type
+        param_type = param_type if "`" in param_type else f"`{param_type}`"
+
+        # Format the parameter docstring
+        param_docstring = f"return_tensors ({param_type}, *optional*):{description}"
+        docstring += set_min_indent(param_docstring, indent_level + 8)
+
+    return docstring
+
+
 def _process_parameters_section(
     func_documentation, sig, func, class_name, model_name_lowercase, parent_class, indent_level, source_args_dict
 ):
@@ -1584,6 +2123,9 @@ def _process_parameters_section(
     )
     docstring += kwargs_docstring
 
+    # Add return_tensors for processor __call__ methods if not already present
+    docstring = _add_return_tensors_for_processor_call(func, parent_class, docstring, indent_level)
+
     # Report undocumented parameters
     if len(undocumented_parameters) > 0:
         print("\n".join(undocumented_parameters))
@@ -1648,10 +2190,13 @@ def _process_example_section(
     example_docstring = ""
 
     # Use existing example section if available
-
     if func_documentation is not None and (match := re.search(r"(?m)^([ \t]*)(?=Example)", func_documentation)):
         example_docstring = func_documentation[match.start() :]
         example_docstring = "\n" + set_min_indent(example_docstring, indent_level + 4)
+    # Skip examples for processors
+    elif _is_processor_class(func, parent_class):
+        # Processors don't get auto-generated examples
+        return example_docstring
     # No examples for __init__ methods or if the class is not a model
     elif parent_class is None and model_name_lowercase is not None:
         task = rf"({'|'.join(PT_SAMPLE_DOCSTRINGS.keys())})"
@@ -1782,12 +2327,22 @@ def auto_class_docstring(cls, custom_intro=None, custom_args=None, checkpoint=No
     from transformers.models import auto as auto_module
 
     is_dataclass = False
+    is_processor = False
     docstring_init = ""
     docstring_args = ""
     if "PreTrainedModel" in (x.__name__ for x in cls.__mro__):
         docstring_init = auto_method_docstring(
             cls.__init__, parent_class=cls, custom_args=custom_args, checkpoint=checkpoint
         ).__doc__.replace("Args:", "Parameters:")
+    elif "ProcessorMixin" in (x.__name__ for x in cls.__mro__):
+        is_processor = True
+        docstring_init = auto_method_docstring(
+            cls.__init__,
+            parent_class=cls,
+            custom_args=custom_args,
+            checkpoint=checkpoint,
+            source_args_dict=get_args_doc_from_source([ModelArgs, ImageProcessorArgs, ProcessorArgs]),
+        ).__doc__.replace("Args:", "Parameters:")
     elif "ModelOutput" in (x.__name__ for x in cls.__mro__):
         # We have a data class
         is_dataclass = True
@@ -1811,17 +2366,23 @@ def auto_class_docstring(cls, custom_intro=None, custom_args=None, checkpoint=No
         model_name_lowercase = model_name_lowercase.replace("_", "-")
 
     name = re.findall(rf"({'|'.join(ClassDocstring.__dict__.keys())})$", cls.__name__)
-    if name == [] and custom_intro is None and not is_dataclass:
+    if name == [] and custom_intro is None and not is_dataclass and not is_processor:
         raise ValueError(
             f"`{cls.__name__}` is not registered in the auto doc. Here are the available classes: {ClassDocstring.__dict__.keys()}.\n"
             "Add a `custom_intro` to the decorator if you want to use `auto_docstring` on a class not registered in the auto doc."
         )
-    if name != [] or custom_intro is not None or is_dataclass:
+    if name != [] or custom_intro is not None or is_dataclass or is_processor:
         name = name[0] if name else None
         if custom_intro is not None:
             pre_block = equalize_indent(custom_intro, indent_level)
             if not pre_block.endswith("\n"):
                 pre_block += "\n"
+        elif is_processor:
+            # Generate processor intro dynamically
+            pre_block = generate_processor_intro(cls)
+            if pre_block:
+                pre_block = equalize_indent(pre_block, indent_level)
+                pre_block = format_args_docstring(pre_block, model_name_lowercase)
         elif model_name_title is None or name is None:
             pre_block = ""
         else: