paddlex 3.0.0b2__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

paddlex/inference/models/common/vision/processors.py

@@ -0,0 +1,285 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import numpy as np
+from PIL import Image
+
+from .....utils.deps import class_requires_deps, is_dep_available
+from ....utils.benchmark import benchmark
+from . import funcs as F
+
+if is_dep_available("opencv-contrib-python"):
+    import cv2
+
+
+@class_requires_deps("opencv-contrib-python")
+class _BaseResize:
+    def __init__(self, size_divisor, interp, backend="cv2"):
+        _CV2_INTERP_DICT = {
+            "NEAREST": cv2.INTER_NEAREST,
+            "LINEAR": cv2.INTER_LINEAR,
+            "BICUBIC": cv2.INTER_CUBIC,
+            "AREA": cv2.INTER_AREA,
+            "LANCZOS4": cv2.INTER_LANCZOS4,
+        }
+        _PIL_INTERP_DICT = {
+            "NEAREST": Image.NEAREST,
+            "BILINEAR": Image.BILINEAR,
+            "BICUBIC": Image.BICUBIC,
+            "BOX": Image.BOX,
+            "LANCZOS4": Image.LANCZOS,
+        }
+
+        super().__init__()
+
+        if size_divisor is not None:
+            assert isinstance(
+                size_divisor, int
+            ), "`size_divisor` should be None or int."
+        self.size_divisor = size_divisor
+
+        try:
+            interp = interp.upper()
+            if backend == "cv2":
+                interp = _CV2_INTERP_DICT[interp]
+            elif backend == "pil":
+                interp = _PIL_INTERP_DICT[interp]
+            else:
+                raise ValueError("backend must be `cv2` or `pil`")
+        except KeyError:
+            raise ValueError(
+                "For backend '{}', `interp` should be one of {}. Please ensure the interpolation method matches the selected backend.".format(
+                    backend,
+                    (
+                        _CV2_INTERP_DICT.keys()
+                        if backend == "cv2"
+                        else _PIL_INTERP_DICT.keys()
+                    ),
+                )
+            )
+        self.interp = interp
+        self.backend = backend
+
+    @staticmethod
+    def _rescale_size(img_size, target_size):
+        """rescale size"""
+        scale = min(max(target_size) / max(img_size), min(target_size) / min(img_size))
+        rescaled_size = [round(i * scale) for i in img_size]
+        return rescaled_size, scale
+
+
+@benchmark.timeit
+class Resize(_BaseResize):
+    """Resize the image."""
+
+    def __init__(
+        self,
+        target_size,
+        keep_ratio=False,
+        size_divisor=None,
+        interp="LINEAR",
+        backend="cv2",
+    ):
+        """
+        Initialize the instance.
+
+        Args:
+            target_size (list|tuple|int): Target width and height.
+            keep_ratio (bool, optional): Whether to keep the aspect ratio of resized
+                image. Default: False.
+            size_divisor (int|None, optional): Divisor of resized image size.
+                Default: None.
+            interp (str, optional): Interpolation method. Choices are 'NEAREST',
+                'LINEAR', 'CUBIC', 'AREA', and 'LANCZOS4'. Default: 'LINEAR'.
+        """
+        super().__init__(size_divisor=size_divisor, interp=interp, backend=backend)
+
+        if isinstance(target_size, int):
+            target_size = [target_size, target_size]
+        F.check_image_size(target_size)
+        self.target_size = target_size
+
+        self.keep_ratio = keep_ratio
+
+    def __call__(self, imgs):
+        """apply"""
+        return [self.resize(img) for img in imgs]
+
+    def resize(self, img):
+        target_size = self.target_size
+        original_size = img.shape[:2][::-1]
+
+        if self.keep_ratio:
+            h, w = img.shape[0:2]
+            target_size, _ = self._rescale_size((w, h), self.target_size)
+
+        if self.size_divisor:
+            target_size = [
+                math.ceil(i / self.size_divisor) * self.size_divisor
+                for i in target_size
+            ]
+        img = F.resize(img, target_size, interp=self.interp, backend=self.backend)
+        return img
+
+
+@benchmark.timeit
+class ResizeByLong(_BaseResize):
+    """
+    Proportionally resize the image by specifying the target length of the
+    longest side.
+    """
+
+    def __init__(
+        self, target_long_edge, size_divisor=None, interp="LINEAR", backend="cv2"
+    ):
+        """
+        Initialize the instance.
+
+        Args:
+            target_long_edge (int): Target length of the longest side of image.
+            size_divisor (int|None, optional): Divisor of resized image size.
+                Default: None.
+            interp (str, optional): Interpolation method. Choices are 'NEAREST',
+                'LINEAR', 'CUBIC', 'AREA', and 'LANCZOS4'. Default: 'LINEAR'.
+        """
+        super().__init__(size_divisor=size_divisor, interp=interp, backend=backend)
+        self.target_long_edge = target_long_edge
+
+    def __call__(self, imgs):
+        """apply"""
+        return [self.resize(img) for img in imgs]
+
+    def resize(self, img):
+        h, w = img.shape[:2]
+        scale = self.target_long_edge / max(h, w)
+        h_resize = round(h * scale)
+        w_resize = round(w * scale)
+        if self.size_divisor is not None:
+            h_resize = math.ceil(h_resize / self.size_divisor) * self.size_divisor
+            w_resize = math.ceil(w_resize / self.size_divisor) * self.size_divisor
+
+        img = F.resize(
+            img, (w_resize, h_resize), interp=self.interp, backend=self.backend
+        )
+        return img
+
+
+@benchmark.timeit
+class ResizeByShort(_BaseResize):
+    """
+    Proportionally resize the image by specifying the target length of the
+    shortest side.
+    """
+
+    def __init__(
+        self, target_short_edge, size_divisor=None, interp="LINEAR", backend="cv2"
+    ):
+        """
+        Initialize the instance.
+
+        Args:
+            target_short_edge (int): Target length of the shortest side of image.
+            size_divisor (int|None, optional): Divisor of resized image size.
+                Default: None.
+            interp (str, optional): Interpolation method. Choices are 'NEAREST',
+                'LINEAR', 'CUBIC', 'AREA', and 'LANCZOS4'. Default: 'LINEAR'.
+        """
+        super().__init__(size_divisor=size_divisor, interp=interp, backend=backend)
+        self.target_short_edge = target_short_edge
+
+    def __call__(self, imgs):
+        """apply"""
+        return [self.resize(img) for img in imgs]
+
+    def resize(self, img):
+        h, w = img.shape[:2]
+        scale = self.target_short_edge / min(h, w)
+        h_resize = round(h * scale)
+        w_resize = round(w * scale)
+        if self.size_divisor is not None:
+            h_resize = math.ceil(h_resize / self.size_divisor) * self.size_divisor
+            w_resize = math.ceil(w_resize / self.size_divisor) * self.size_divisor
+
+        img = F.resize(
+            img, (w_resize, h_resize), interp=self.interp, backend=self.backend
+        )
+        return img
+
+
+@benchmark.timeit
+@class_requires_deps("opencv-contrib-python")
+class Normalize:
+    """Normalize the three-channel image."""
+
+    def __init__(self, scale=1.0 / 255, mean=0.5, std=0.5):
+        """
+        Initialize the instance.
+
+        Args:
+            scale (float, optional): Scaling factor to apply to the image before
+                applying normalization. Default: 1/255.
+            mean (float|tuple|list, optional): Means for each channel of the image.
+                Default: 0.5.
+            std (float|tuple|list|np.ndarray, optional): Standard deviations for each channel
+                of the image. Default: 0.5.
+        """
+        super().__init__()
+
+        if isinstance(mean, float):
+            mean = [mean] * 3
+        elif len(mean) != 3:
+            raise ValueError(
+                f"Expected `mean` to be a tuple or list of length 3, but got {len(mean)} elements."
+            )
+        if isinstance(std, float):
+            std = [std] * 3
+        elif len(std) != 3:
+            raise ValueError(
+                f"Expected `std` to be a tuple or list of length 3, but got {len(std)} elements."
+            )
+
+        self.alpha = [scale / std[i] for i in range(len(std))]
+        self.beta = [-mean[i] / std[i] for i in range(len(std))]
+
+    def norm(self, img):
+        split_im = list(cv2.split(img))
+
+        for c in range(img.shape[2]):
+            split_im[c] = split_im[c].astype(np.float32)
+            split_im[c] *= self.alpha[c]
+            split_im[c] += self.beta[c]
+
+        res = cv2.merge(split_im)
+        return res
+
+    def __call__(self, imgs):
+        """apply"""
+        return [self.norm(img) for img in imgs]
+
+
+@benchmark.timeit
+class ToCHWImage:
+    """Reorder the dimensions of the image from HWC to CHW."""
+
+    def __call__(self, imgs):
+        """apply"""
+        return [img.transpose((2, 0, 1)) for img in imgs]
+
+
+@benchmark.timeit
+class ToBatch:
+    def __call__(self, imgs):
+        return [np.stack(imgs, axis=0).astype(dtype=np.float32, copy=False)]

paddlex/inference/models/common/vlm/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

paddlex/inference/models/common/vlm/activations.py

@@ -0,0 +1,189 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from collections import OrderedDict
+
+import paddle
+import paddle.nn.functional as F
+from paddle import Tensor, nn
+
+
+class NewGELUActivation(nn.Layer):
+    """
+    Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Also see
+    the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
+    """
+
+    def forward(self, input: Tensor) -> Tensor:
+        return (
+            0.5
+            * input
+            * (
+                1.0
+                + paddle.tanh(
+                    math.sqrt(2.0 / math.pi)
+                    * (input + 0.044715 * paddle.pow(input, 3.0))
+                )
+            )
+        )
+
+
+class GELUActivation(nn.Layer):
+    """
+    Original Implementation of the GELU activation function in Google BERT repo when initially created. For
+    information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 +
+    torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional
+    Also see the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
+    """
+
+    def __init__(self, use_gelu_python: bool = False):
+        super().__init__()
+        if use_gelu_python:
+            self.act = self._gelu_python
+        else:
+            self.act = nn.functional.gelu
+
+    def _gelu_python(self, input: Tensor) -> Tensor:
+        return input * 0.5 * (1.0 + paddle.erf(input / math.sqrt(2.0)))
+
+    def forward(self, input: Tensor) -> Tensor:
+        return self.act(input)
+
+
+class FastGELUActivation(nn.Layer):
+    """
+    Applies GELU approximation that is slower than QuickGELU but more accurate. See: https://github.com/hendrycks/GELUs
+    """
+
+    def forward(self, input: Tensor) -> Tensor:
+        return (
+            0.5
+            * input
+            * (
+                1.0
+                + paddle.tanh(input * 0.7978845608 * (1.0 + 0.044715 * input * input))
+            )
+        )
+
+
+class QuickGELUActivation(nn.Layer):
+    """
+    Applies GELU approximation that is fast but somewhat inaccurate. See: https://github.com/hendrycks/GELUs
+    """
+
+    def forward(self, input: Tensor) -> Tensor:
+        return input * F.sigmoid(1.702 * input)
+
+
+class ClippedGELUActivation(nn.Layer):
+    """
+    Clip the range of possible GeLU outputs between [min, max]. This is especially useful for quantization purpose, as
+    it allows mapping negatives values in the GeLU spectrum. For more information on this trick, please refer to
+    https://arxiv.org/abs/2004.09602.
+
+    Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when
+    initially created.
+
+    For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 +
+    torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))). See https://arxiv.org/abs/1606.08415
+    """
+
+    def __init__(self, min: float, max: float):
+        if min > max:
+            raise ValueError(f"min should be < max (got min: {min}, max: {max})")
+
+        super().__init__()
+        self.min = min
+        self.max = max
+
+    def forward(self, x: Tensor) -> Tensor:
+        return paddle.clip(gelu(x), self.min, self.max)
+
+
+class SiLUActivation(nn.Layer):
+    """
+    See Gaussian Error Linear Units (Hendrycks et al., https://arxiv.org/abs/1606.08415) where the SiLU (Sigmoid Linear
+    Unit) was originally introduced and coined, and see Sigmoid-Weighted Linear Units for Neural Network Function
+    Approximation in Reinforcement Learning (Elfwing et al., https://arxiv.org/abs/1702.03118) and Swish: a Self-Gated
+    Activation Function (Ramachandran et al., https://arxiv.org/abs/1710.05941v1) where the SiLU was experimented with
+    later.
+    """
+
+    def forward(self, input: Tensor) -> Tensor:
+        return F.silu(input)
+
+
+class MishActivation(nn.Layer):
+    """
+    See Mish: A Self-Regularized Non-Monotonic Activation Function (Misra., https://arxiv.org/abs/1908.08681). Also
+    visit the official repository for the paper: https://github.com/digantamisra98/Mish
+    """
+
+    def forward(self, input: Tensor) -> Tensor:
+        return F.mish(input)
+
+
+class LinearActivation(nn.Layer):
+    """
+    Applies the linear activation function, i.e. forwarding input directly to output.
+    """
+
+    def forward(self, input: Tensor) -> Tensor:
+        return input
+
+
+class ClassInstantier(OrderedDict):
+    def __getitem__(self, key):
+        content = super().__getitem__(key)
+        cls, kwargs = content if isinstance(content, tuple) else (content, {})
+        return cls(**kwargs)
+
+
+ACT2CLS = {
+    "gelu": GELUActivation,
+    "gelu_10": (ClippedGELUActivation, {"min": -10, "max": 10}),
+    "gelu_fast": FastGELUActivation,
+    "gelu_new": NewGELUActivation,
+    "gelu_python": (GELUActivation, {"use_gelu_python": True}),
+    "linear": LinearActivation,
+    "mish": MishActivation,
+    "quick_gelu": QuickGELUActivation,
+    "relu": nn.ReLU,
+    "relu6": nn.ReLU6,
+    "sigmoid": nn.Sigmoid,
+    "silu": SiLUActivation,
+    "swish": SiLUActivation,
+    "tanh": nn.Tanh,
+}
+ACT2FN = ClassInstantier(ACT2CLS)
+
+
+def get_activation(activation_string):
+    if activation_string in ACT2FN:
+        return ACT2FN[activation_string]
+    else:
+        raise KeyError(
+            f"function {activation_string} not found in ACT2FN mapping {list(ACT2FN.keys())}"
+        )
+
+
+gelu_python = get_activation("gelu_python")
+gelu_new = get_activation("gelu_new")
+gelu = get_activation("gelu")
+gelu_fast = get_activation("gelu_fast")
+quick_gelu = get_activation("quick_gelu")
+silu = get_activation("silu")
+mish = get_activation("mish")
+linear_act = get_activation("linear")

paddlex/inference/models/common/vlm/bert_padding.py

@@ -0,0 +1,127 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import operator
+from functools import reduce
+
+import paddle
+import paddle.nn.functional as F
+
+
+class IndexFirstAxis(paddle.autograd.PyLayer):
+
+    @staticmethod
+    def forward(ctx, input, indices):
+        from einops import rearrange, repeat
+
+        ctx.save_for_backward(indices)
+        assert input.ndim >= 2
+        ctx.first_axis_dim, other_shape = input.shape[0], input.shape[1:]
+        second_dim = reduce(operator.mul, other_shape, 1)
+        return paddle.take_along_axis(
+            arr=rearrange(input, "b ... -> b (...)"),
+            axis=0,
+            indices=repeat(indices, "z -> z d", d=second_dim),
+        ).reshape([-1, *other_shape])
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        """Class Attribute: torch.autograd.function.FunctionCtx.saved_tensors, can not convert, please check whether it is torch.Tensor.*/torch.autograd.function.FunctionCtx.*/torch.distributions.Distribution.* and convert manually"""
+        from einops import rearrange, repeat
+
+        (indices,) = ctx.saved_tensor()
+        assert grad_output.ndim >= 2
+        other_shape = grad_output.shape[1:]
+        grad_output = rearrange(grad_output, "b ... -> b (...)")
+        grad_input = paddle.zeros(
+            shape=[ctx.first_axis_dim, tuple(grad_output.shape)[1]],
+            dtype=grad_output.dtype,
+        )
+
+        grad_input.put_along_axis_(
+            axis=0,
+            indices=repeat(indices, "z -> z d", d=tuple(grad_output.shape)[1]),
+            values=grad_output,
+        )
+        return grad_input.reshape([ctx.first_axis_dim, *other_shape]), None
+
+
+index_first_axis = IndexFirstAxis.apply
+
+
+class IndexPutFirstAxis(paddle.autograd.PyLayer):
+
+    @staticmethod
+    def forward(ctx, values, indices, first_axis_dim):
+        ctx.save_for_backward(indices)
+        assert indices.ndim == 1
+        assert values.ndim >= 2
+        output = paddle.zeros(
+            shape=[first_axis_dim, *tuple(values.shape)[1:]], dtype=values.dtype
+        )
+        output[indices] = values
+        return output
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        """Class Attribute: torch.autograd.function.FunctionCtx.saved_tensors, can not convert, please check whether it is torch.Tensor.*/torch.autograd.function.FunctionCtx.*/torch.distributions.Distribution.* and convert manually"""
+        (indices,) = ctx.saved_tensor()
+        grad_values = grad_output[indices]
+        return grad_values, None
+
+
+index_put_first_axis = IndexPutFirstAxis.apply
+
+
+def unpad_input(hidden_states, attention_mask):
+    """
+    Arguments:
+        hidden_states: (batch, seqlen, ...)
+        attention_mask: (batch, seqlen), bool / int, 1 means valid and 0 means not valid.
+    Return:
+        hidden_states: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask.
+        indices: (total_nnz), the indices of non-masked tokens from the flattened input sequence.
+        cu_seqlens: (batch + 1), the cumulative sequence lengths, used to index into hidden_states.
+        max_seqlen_in_batch: int
+    """
+    from einops import rearrange
+
+    seqlens_in_batch = paddle.sum(attention_mask, axis=-1, dtype="int32")
+    indices = paddle.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = paddle.max(seqlens_in_batch).item()
+    cu_seqlens = F.pad(paddle.cumsum(seqlens_in_batch, axis=0), [1, 0])
+
+    return (
+        index_first_axis(rearrange(hidden_states, "b s ... -> (b s) ..."), indices),
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+def pad_input(hidden_states, indices, batch, seqlen):
+    """
+    Arguments:
+        hidden_states: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask.
+        indices: (total_nnz), the indices that represent the non-masked tokens of the original padded input sequence.
+        batch: int, batch size for the padded sequence.
+        seqlen: int, maximum sequence length for the padded sequence.
+    Return:
+        hidden_states: (batch, seqlen, ...)
+    """
+    from einops import rearrange
+
+    output = index_put_first_axis(hidden_states, indices, batch * seqlen)
+    return rearrange(output, "(b s) ... -> b s ...", b=batch)