paddlex 3.0.0rc0__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

paddlex/inference/models/doc_vlm/predictor.py

@@ -0,0 +1,198 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import os
+from typing import List
+
+from ....modules.doc_vlm.model_list import MODELS
+from ....utils.device import TemporaryDeviceChanger
+from ....utils.env import get_device_type
+from ...common.batch_sampler import DocVLMBatchSampler
+from ..base import BasePredictor
+from .result import DocVLMResult
+
+
+class DocVLMPredictor(BasePredictor):
+
+    entities = MODELS
+
+    def __init__(self, *args, **kwargs):
+        """Initializes DocVLMPredictor.
+        Args:
+            *args: Arbitrary positional arguments passed to the superclass.
+            **kwargs: Arbitrary keyword arguments passed to the superclass.
+        """
+        super().__init__(*args, **kwargs)
+        self.device = kwargs.get("device", None)
+        self.infer, self.processor = self._build(**kwargs)
+
+    def _build_batch_sampler(self):
+        """Builds and returns an DocVLMBatchSampler instance.
+
+        Returns:
+            DocVLMBatchSampler: An instance of DocVLMBatchSampler.
+        """
+        return DocVLMBatchSampler()
+
+    def _get_result_class(self):
+        """Returns the result class, DocVLMResult.
+
+        Returns:
+            type: The DocVLMResult class.
+        """
+        return DocVLMResult
+
+    def _build(self, **kwargs):
+        """Build the model, and correspounding processor on the configuration.
+
+        Returns:
+            model: An instance of Paddle model, could be either a dynamic model or a static model.
+            processor: The correspounding processor for the model.
+        """
+        import paddle
+
+        from .modeling import PPDocBeeInference
+
+        # build model
+        if "PP-DocBee" in self.model_name:
+            if kwargs.get("use_hpip", False):
+                raise ValueError(
+                    f"PP-DocBee series do not support `use_hpip=True` for now."
+                )
+            dtype = (
+                "bfloat16"
+                if ("npu" in get_device_type() or paddle.amp.is_bfloat16_supported())
+                else "float32"
+            )
+            with TemporaryDeviceChanger(self.device):
+                model = PPDocBeeInference.from_pretrained(self.model_dir, dtype=dtype)
+        else:
+            raise NotImplementedError(f"Model {self.model_name} is not supported.")
+
+        # build processor
+        processor = self.build_processor()
+        return model, processor
+
+    def process(self, data: List[dict], **kwargs):
+        """
+        Process a batch of data through the preprocessing, inference, and postprocessing.
+
+        Args:
+            data (List[dict]): A batch of input data, must be a dict (e.g. {"image": /path/to/image, "query": some question}).
+            kwargs (Optional[dict]): Arbitrary keyword arguments passed to model.generate.
+
+        Returns:
+            dict: A dictionary containing the raw sample information and prediction results for every instance of the batch.
+        """
+        assert (
+            isinstance(data, List) and len(data) == 1
+        ), "data must be a list of length 1"
+        assert isinstance(data[0], dict)
+
+        data = data[0]
+        src_data = copy.copy(data)
+        # preprocess
+        data = self.processor.preprocess(**data)
+        data = self._switch_inputs_to_device(data)
+
+        # do infer
+        with TemporaryDeviceChanger(self.device):
+            preds = self.infer.generate(data, **kwargs)
+
+        # postprocess
+        preds = self.processor.postprocess(preds)
+
+        result_dict = self._format_result_dict(preds, src_data)
+        return result_dict
+
+    def build_processor(self, **kwargs):
+        from ..common.tokenizer import MIXQwen2Tokenizer
+        from .processors import PPDocBeeProcessor, Qwen2VLImageProcessor
+
+        if "PP-DocBee" in self.model_name:
+            image_processor = Qwen2VLImageProcessor()
+            tokenizer = MIXQwen2Tokenizer.from_pretrained(self.model_dir)
+            return PPDocBeeProcessor(
+                image_processor=image_processor, tokenizer=tokenizer
+            )
+        else:
+            raise NotImplementedError
+
+    def _format_result_dict(self, model_preds, src_data):
+        if not isinstance(model_preds, list):
+            model_preds = [model_preds]
+        if not isinstance(src_data, list):
+            src_data = [src_data]
+        if len(model_preds) != len(src_data):
+            raise ValueError(
+                f"Model predicts {len(model_preds)} results while src data has {len(src_data)} samples."
+            )
+
+        rst_format_dict = {k: [] for k in src_data[0].keys()}
+        rst_format_dict["result"] = []
+
+        for data_sample, model_pred in zip(src_data, model_preds):
+            for k in data_sample.keys():
+                rst_format_dict[k].append(data_sample[k])
+            rst_format_dict["result"].append(model_pred)
+
+        return rst_format_dict
+
+    def _infer_dynamic_forward_device(self, device):
+        """infer the forward device for dynamic graph model"""
+        import GPUtil
+
+        from ....utils.device import parse_device
+
+        if device is None:
+            return None
+        if "cpu" in device.lower():
+            return "cpu"
+        device_type, device_ids = parse_device(device)
+
+        cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None)
+        if cuda_visible_devices is None:
+            env_gpu_num = len(GPUtil.getGPUs())
+            cuda_visible_devices = ",".join([str(i) for i in range(env_gpu_num)])
+        env_device_ids = cuda_visible_devices.split(",")
+        for env_device_id in env_device_ids:
+            if not env_device_id.isdigit():
+                raise ValueError(
+                    f"CUDA_VISIBLE_DEVICES ID must be an integer. Invalid device ID: {env_device_id}"
+                )
+
+        if max(device_ids) >= len(env_device_ids):
+            raise ValueError(
+                f"Required gpu ids {device_ids} even larger than the number of visible devices {cuda_visible_devices}."
+            )
+
+        rst_global_gpu_ids = [env_device_ids[idx] for idx in device_ids]
+        return device_type + ":" + ",".join(rst_global_gpu_ids)
+
+    def _switch_inputs_to_device(self, input_dict):
+        """Switch the input to the specified device"""
+        import paddle
+
+        if self.device is None:
+            return input_dict
+        rst_dict = {
+            k: (
+                paddle.to_tensor(input_dict[k], place=self.device)
+                if isinstance(input_dict[k], paddle.Tensor)
+                else input_dict[k]
+            )
+            for k in input_dict
+        }
+        return rst_dict

paddlex/inference/models/doc_vlm/processors/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .qwen2_vl import PPDocBeeProcessor, Qwen2VLImageProcessor

paddlex/inference/models/doc_vlm/processors/common.py

@@ -0,0 +1,372 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import UserDict
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import paddle
+import PIL.Image
+from packaging import version
+
+from ...common.tokenizer.tokenizer_utils_base import ExplicitEnum
+
+
+def is_paddle_tensor(tensor):
+    return paddle.is_tensor(tensor)
+
+
+def to_numpy(obj):
+    """
+    Convert a TensorFlow tensor, PyTorch tensor, Numpy array or python list to a Numpy array.
+    """
+    if isinstance(obj, (dict, UserDict)):
+        return {k: to_numpy(v) for k, v in obj.items()}
+    elif isinstance(obj, (list, tuple)):
+        return np.array(obj)
+    elif is_paddle_tensor(obj):
+        return obj.detach().cpu().numpy()
+    else:
+        return obj
+
+
+if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"):
+    PILImageResampling = PIL.Image.Resampling
+else:
+    PILImageResampling = PIL.Image
+
+
+ImageInput = Union[
+    "PIL.Image.Image",
+    np.ndarray,
+    "paddle.Tensor",
+    List["PIL.Image.Image"],
+    List[np.ndarray],
+    List["paddle.Tensor"],
+]  # noqa
+
+TextInput = str
+
+
+class ChannelDimension(ExplicitEnum):
+    FIRST = "channels_first"
+    LAST = "channels_last"
+
+
+class TensorType(ExplicitEnum):
+    """
+    Possible values for the `return_tensors` argument in [`PretrainedTokenizerBase.__call__`]. Useful for
+    tab-completion in an IDE.
+    """
+
+    PADDLE = "pd"
+    NUMPY = "np"
+
+
+def is_valid_image(img):
+    return (
+        isinstance(img, PIL.Image.Image)
+        or isinstance(img, np.ndarray)
+        or is_paddle_tensor(img)
+    )
+
+
+def valid_images(imgs):
+    # If we have an list of images, make sure every image is valid
+    if isinstance(imgs, (list, tuple)):
+        for img in imgs:
+            if not valid_images(img):
+                return False
+    # If not a list of tuple, we have been given a single image or batched tensor of images
+    elif not is_valid_image(imgs):
+        return False
+    return True
+
+
+def is_batched(img):
+    if isinstance(img, (list, tuple)):
+        return is_valid_image(img[0])
+    return False
+
+
+def make_list_of_images(images, expected_ndims: int = 3) -> List[ImageInput]:
+    """
+    Ensure that the input is a list of images. If the input is a single image, it is converted to a list of length 1.
+    If the input is a batch of images, it is converted to a list of images.
+    Args:
+        images (`ImageInput`):
+            Image of images to turn into a list of images.
+        expected_ndims (`int`, *optional*, defaults to 3):
+            Expected number of dimensions for a single input image. If the input image has a different number of
+            dimensions, an error is raised.
+    """
+    if is_batched(images):
+        return images
+
+    # Either the input is a single image, in which case we create a list of length 1
+    if isinstance(images, PIL.Image.Image):
+        # PIL images are never batched
+        return [images]
+
+    if is_valid_image(images):
+        if images.ndim == expected_ndims + 1:
+            # Batch of images
+            images = list(images)
+        elif images.ndim == expected_ndims:
+            # Single image
+            images = [images]
+        else:
+            raise ValueError(
+                f"Invalid image shape. Expected either {expected_ndims + 1} or {expected_ndims} dimensions, but got"
+                f" {images.ndim} dimensions."
+            )
+        return images
+    raise ValueError(
+        "Invalid image type. Expected either PIL.Image.Image, numpy.ndarray, paddle.Tensor "
+        f"but got {type(images)}."
+    )
+
+
+def to_numpy_array(img) -> np.ndarray:
+    if not is_valid_image(img):
+        raise ValueError(f"Invalid image type: {type(img)}")
+
+    if isinstance(img, PIL.Image.Image):
+        return np.array(img)
+    return to_numpy(img)
+
+
+def infer_channel_dimension_format(image: np.ndarray) -> ChannelDimension:
+    """
+    Infers the channel dimension format of `image`.
+
+    Args:
+        image (`np.ndarray`):
+            The image to infer the channel dimension of.
+
+    Returns:
+        The channel dimension of the image.
+    """
+    if image.ndim == 3:
+        first_dim, last_dim = 0, 2
+    elif image.ndim == 4:
+        first_dim, last_dim = 1, 3
+    else:
+        raise ValueError(f"Unsupported number of image dimensions: {image.ndim}")
+
+    if image.shape[first_dim] in (1, 3):
+        return ChannelDimension.FIRST
+    elif image.shape[last_dim] in (1, 3):
+        return ChannelDimension.LAST
+    raise ValueError("Unable to infer channel dimension format")
+
+
+def get_channel_dimension_axis(image: np.ndarray) -> int:
+    """
+    Returns the channel dimension axis of the image.
+
+    Args:
+        image (`np.ndarray`):
+            The image to get the channel dimension axis of.
+
+    Returns:
+        The channel dimension axis of the image.
+    """
+    channel_dim = infer_channel_dimension_format(image)
+    if channel_dim == ChannelDimension.FIRST:
+        return image.ndim - 3
+    elif channel_dim == ChannelDimension.LAST:
+        return image.ndim - 1
+    raise ValueError(f"Unsupported data format: {channel_dim}")
+
+
+def get_image_size(
+    image: np.ndarray, channel_dim: ChannelDimension = None
+) -> Tuple[int, int]:
+    """
+    Returns the (height, width) dimensions of the image.
+
+    Args:
+        image (`np.ndarray`):
+            The image to get the dimensions of.
+        channel_dim (`ChannelDimension`, *optional*):
+            Which dimension the channel dimension is in. If `None`, will infer the channel dimension from the image.
+
+    Returns:
+        A tuple of the image's height and width.
+    """
+    if channel_dim is None:
+        channel_dim = infer_channel_dimension_format(image)
+
+    if channel_dim == ChannelDimension.FIRST:
+        return image.shape[-2], image.shape[-1]
+    elif channel_dim == ChannelDimension.LAST:
+        return image.shape[-3], image.shape[-2]
+    else:
+        raise ValueError(f"Unsupported data format: {channel_dim}")
+
+
+def convert_to_rgb(image: ImageInput) -> ImageInput:
+    """
+    Converts an image to RGB format. Only converts if the image is of type PIL.Image.Image, otherwise returns the image
+    as is.
+
+    Args:
+        image (Image):
+            The image to convert.
+    """
+
+    if not isinstance(image, PIL.Image.Image):
+        return image
+
+    image = image.convert("RGB")
+    return image
+
+
+def to_channel_dimension_format(
+    image: np.ndarray,
+    channel_dim: Union[ChannelDimension, str],
+    input_channel_dim: Optional[Union[ChannelDimension, str]] = None,
+) -> np.ndarray:
+    """
+    Converts `image` to the channel dimension format specified by `channel_dim`.
+
+    Args:
+        image (`numpy.ndarray`):
+            The image to have its channel dimension set.
+        channel_dim (`ChannelDimension`):
+            The channel dimension format to use.
+
+    Returns:
+        `np.ndarray`: The image with the channel dimension set to `channel_dim`.
+    """
+    if not isinstance(image, np.ndarray):
+        raise ValueError(f"Input image must be of type np.ndarray, got {type(image)}")
+
+    if input_channel_dim is None:
+        input_channel_dim = infer_channel_dimension_format(image)
+
+    target_channel_dim = ChannelDimension(channel_dim)
+    if input_channel_dim == target_channel_dim:
+        return image
+
+    if target_channel_dim == ChannelDimension.FIRST:
+        image = image.transpose((2, 0, 1))
+    elif target_channel_dim == ChannelDimension.LAST:
+        image = image.transpose((1, 2, 0))
+    else:
+        raise ValueError("Unsupported channel dimension format: {}".format(channel_dim))
+
+    return image
+
+
+class BatchFeature(UserDict):
+    r"""
+    Holds the feature extractor specific `__call__` methods.
+    This class is derived from a python dictionary and can be used as a dictionary.
+    Args:
+        data (`dict`):
+            Dictionary of lists/arrays/tensors returned by the __call__/pad methods ('input_values', 'attention_mask',
+            etc.).
+        tensor_type (`Union[None, str, TensorType]`, *optional*):
+            You can give a tensor_type here to convert the lists of integers in Paddle/Numpy Tensors at
+            initialization.
+    """
+
+    def __init__(
+        self,
+        data: Optional[Dict[str, Any]] = None,
+        tensor_type: Union[None, str, TensorType] = None,
+    ):
+        super().__init__(data)
+        self.convert_to_tensors(tensor_type=tensor_type)
+
+    def __getitem__(self, item: str):
+        """
+        If the key is a string, returns the value of the dict associated to `key` ('input_values', 'attention_mask',
+        etc.).
+        """
+        if isinstance(item, str):
+            return self.data[item]
+        else:
+            raise KeyError(
+                "Indexing with integers is not available when using Python based feature extractors"
+            )
+
+    def __getattr__(self, item: str):
+        try:
+            return self.data[item]
+        except KeyError:
+            raise AttributeError
+
+    def __getstate__(self):
+        return {"data": self.data}
+
+    def __setstate__(self, state):
+        if "data" in state:
+            self.data = state["data"]
+
+    def keys(self):
+        return self.data.keys()
+
+    def values(self):
+        return self.data.values()
+
+    def items(self):
+        return self.data.items()
+
+    def convert_to_tensors(self, tensor_type: Optional[Union[str, TensorType]] = None):
+        """
+        Convert the inner content to tensors.
+        Args:
+            tensor_type (`str` or [`TensorType`], *optional*):
+                The type of tensors to use. If `str`, should be one of the values of the enum [`TensorType`]. If
+                `None`, no modification is done.
+        """
+        if tensor_type is None:
+            return self
+
+        # Convert to TensorType
+        if not isinstance(tensor_type, TensorType):
+            tensor_type = TensorType(tensor_type)
+
+        # Get a function reference for the correct framework
+        if tensor_type == TensorType.PADDLE:
+            as_tensor = paddle.to_tensor
+            is_tensor = paddle.is_tensor
+        else:
+            as_tensor = np.asarray
+
+            def is_tensor(x):
+                return isinstance(x, np.ndarray)
+
+        # Do the tensor conversion in batch
+        for key, value in self.items():
+            try:
+                if not is_tensor(value):
+                    tensor = as_tensor(value)
+
+                    self[key] = tensor
+            except:  # noqa E722
+                if key == "overflowing_tokens":
+                    raise ValueError(
+                        "Unable to create tensor returning overflowing tokens of different lengths. "
+                        "Please see if a fast version of this tokenizer is available to have this feature available."
+                    )
+                raise ValueError(
+                    "Unable to create tensor, you should probably activate truncation and/or padding "
+                    "with 'padding=True' 'truncation=True' to have batched tensors with the same length."
+                )
+
+        return self