kaiko-eva 0.3.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

eva/multimodal/models/networks/api/anthropic.py

@@ -0,0 +1,34 @@
+"""Models from Anthropic."""
+
+import os
+
+from eva.multimodal.models import wrappers
+from eva.multimodal.models.networks.registry import model_registry
+
+
+class _Claude(wrappers.LiteLLMModel):
+    """Base class for Claude models."""
+
+    def __init__(self, model_name: str, system_prompt: str | None = None):
+        if not os.getenv("ANTHROPIC_API_KEY"):
+            raise ValueError("ANTHROPIC_API_KEY env variable must be set.")
+
+        super().__init__(model_name=model_name, system_prompt=system_prompt)
+
+
+@model_registry.register("anthropic/claude-3-5-sonnet-20240620")
+class Claude35Sonnet20240620(_Claude):
+    """Claude 3.5 Sonnet (June 2024) model."""
+
+    def __init__(self, system_prompt: str | None = None):
+        """Initialize the model."""
+        super().__init__(model_name="claude-3-5-sonnet-20240620", system_prompt=system_prompt)
+
+
+@model_registry.register("anthropic/claude-3-7-sonnet-20250219")
+class Claude37Sonnet20250219(_Claude):
+    """Claude 3.7 Sonnet (February 2025) model."""
+
+    def __init__(self, system_prompt: str | None = None):
+        """Initialize the model."""
+        super().__init__(model_name="claude-3-7-sonnet-20250219", system_prompt=system_prompt)

eva/multimodal/models/networks/others.py

@@ -0,0 +1,47 @@
+"""Models from other providers (non-major entities)."""
+
+import os
+
+import torch
+
+from eva.core.utils import requirements
+from eva.multimodal.models import wrappers
+from eva.multimodal.models.networks.registry import model_registry
+
+
+@model_registry.register("others/wenchuanzhang_patho-r1-3b")
+class PathoR13b(wrappers.HuggingFaceModel):
+    """Patho-R1-3B model by Wenchuan Zhang."""
+
+    def __init__(
+        self,
+        system_prompt: str | None = None,
+        cache_dir: str | None = None,
+        attn_implementation: str = "flash_attention_2",
+    ):
+        """Initialize the Patho-R1-3B model."""
+        requirements.check_dependencies(requirements={"torch": "2.5.1", "torchvision": "0.20.1"})
+
+        if not os.getenv("HF_TOKEN"):
+            raise ValueError("HF_TOKEN env variable must be set.")
+
+        super().__init__(
+            model_name_or_path="WenchuanZhang/Patho-R1-3B",
+            model_class="Qwen2_5_VLForConditionalGeneration",
+            model_kwargs={
+                "torch_dtype": torch.float16,
+                "trust_remote_code": True,
+                "cache_dir": cache_dir,
+                "attn_implementation": attn_implementation,
+            },
+            generation_kwargs={
+                "max_new_tokens": 512,
+                "do_sample": False,
+            },
+            processor_kwargs={
+                "padding": True,
+                "padding_side": "left",
+                "max_pixels": 451584,  # 672*672
+            },
+            system_prompt=system_prompt,
+        )

eva/multimodal/models/networks/registry.py

@@ -0,0 +1,5 @@
+"""Multimodal Model Registry."""
+
+from eva.core.utils.registry import Registry
+
+model_registry = Registry()

eva/multimodal/models/typings.py

@@ -0,0 +1,27 @@
+"""Type definitions for multimodal models."""
+
+from typing import Any, Dict, Generic, List, TypeVar
+
+from torchvision import tv_tensors
+from typing_extensions import NamedTuple
+
+from eva.language.data.messages import MessageSeries
+
+TargetType = TypeVar("TargetType")
+"""The target data type."""
+
+
+class TextImageBatch(NamedTuple, Generic[TargetType]):
+    """Text and image sample with target and metadata."""
+
+    text: List[MessageSeries]
+    """A batch of conversations with one or multiple messages each."""
+
+    image: List[tv_tensors.Image]
+    """Image tensor."""
+
+    target: TargetType | None
+    """Target data."""
+
+    metadata: Dict[str, Any] | None
+    """Additional metadata."""

eva/multimodal/models/wrappers/__init__.py

@@ -0,0 +1,13 @@
+"""Multimodal Wrapper API."""
+
+from eva.multimodal.models.wrappers.base import VisionLanguageModel
+from eva.multimodal.models.wrappers.from_registry import ModelFromRegistry
+from eva.multimodal.models.wrappers.huggingface import HuggingFaceModel
+from eva.multimodal.models.wrappers.litellm import LiteLLMModel
+
+__all__ = [
+    "HuggingFaceModel",
+    "LiteLLMModel",
+    "ModelFromRegistry",
+    "VisionLanguageModel",
+]

eva/multimodal/models/wrappers/base.py

@@ -0,0 +1,47 @@
+"""Base class for vision language model wrappers."""
+
+import abc
+from typing import Any, Callable, List
+
+from typing_extensions import override
+
+from eva.core.models.wrappers import base
+from eva.language.data.messages import ModelSystemMessage
+from eva.multimodal.models.typings import TextImageBatch
+
+
+class VisionLanguageModel(base.BaseModel[TextImageBatch, List[str]]):
+    """Base class for multimodal models.
+
+    Classes that inherit from this should implement the following methods:
+    - `load_model`: Loads & instantiates the model.
+    - `model_forward`: Implements the forward pass of the model. For API models,
+        this can be an API call.
+    - `format_inputs`: Preprocesses and converts the input batch into the format
+        expected by the `model_forward` method.
+    """
+
+    def __init__(
+        self, system_prompt: str | None, output_transforms: Callable | None = None
+    ) -> None:
+        """Creates a new model instance.
+
+        Args:
+            system_prompt: The system prompt to use for the model (optional).
+            output_transforms: Optional transforms to apply to the output of
+                the model's forward pass.
+        """
+        super().__init__(transforms=output_transforms)
+
+        self.system_message = ModelSystemMessage(content=system_prompt) if system_prompt else None
+
+    @override
+    def forward(self, batch: TextImageBatch) -> List[str]:
+        """Forward pass of the model."""
+        inputs = self.format_inputs(batch)
+        return super().forward(inputs)
+
+    @abc.abstractmethod
+    def format_inputs(self, batch: TextImageBatch) -> Any:
+        """Converts the inputs into the format expected by the model."""
+        raise NotImplementedError

eva/multimodal/models/wrappers/from_registry.py

@@ -0,0 +1,54 @@
+"""Vision backbone helper class."""
+
+from typing import Any, Callable, Dict, List
+
+from torch import nn
+from typing_extensions import override
+
+from eva.core.models.wrappers import base
+from eva.core.utils import factory
+from eva.multimodal.models.networks.registry import model_registry
+from eva.multimodal.models.typings import TextImageBatch
+
+
+class ModelFromRegistry(base.BaseModel[TextImageBatch, List[str]]):
+    """Wrapper class for vision backbone models.
+
+    This class can be used by load backbones available in eva's
+    model registry by name. New backbones can be registered by using
+    the `@backbone_registry.register(model_name)` decorator.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        model_kwargs: Dict[str, Any] | None = None,
+        model_extra_kwargs: Dict[str, Any] | None = None,
+        transforms: Callable | None = None,
+    ) -> None:
+        """Initializes the model.
+
+        Args:
+            model_name: The name of the model to load.
+            model_kwargs: The arguments used for instantiating the model.
+            model_extra_kwargs: Extra arguments used for instantiating the model.
+            transforms: The transforms to apply to the output tensor
+                produced by the model.
+        """
+        super().__init__(transforms=transforms)
+
+        self._model_name = model_name
+        self._model_kwargs = model_kwargs or {}
+        self._model_extra_kwargs = model_extra_kwargs or {}
+
+        self.model = self.load_model()
+
+    @override
+    def load_model(self) -> nn.Module:
+        ModelFromRegistry.__name__ = self._model_name
+
+        return factory.ModuleFactory(
+            registry=model_registry,
+            name=self._model_name,
+            init_args=self._model_kwargs | self._model_extra_kwargs,
+        )

eva/multimodal/models/wrappers/huggingface.py

@@ -0,0 +1,180 @@
+"""HuggingFace Vision-Language Model Wrapper."""
+
+import functools
+from typing import Any, Callable, Dict, List
+
+import torch
+import transformers
+from loguru import logger
+from torch import nn
+from typing_extensions import override
+
+from eva.language.models.typings import TextBatch
+from eva.language.utils.text import messages as language_message_utils
+from eva.multimodal.models.typings import TextImageBatch
+from eva.multimodal.models.wrappers import base
+from eva.multimodal.utils.text import messages as message_utils
+
+
+class HuggingFaceModel(base.VisionLanguageModel):
+    """Lightweight wrapper for Huggingface VLMs.
+
+    Args:
+        model_name_or_path: The name of the model to use.
+        model_class: The class of the model to use.
+        model_kwargs: Additional model arguments.
+        processor_kwargs: Additional processor arguments.
+        generation_kwargs: Additional generation arguments.
+    """
+
+    def __init__(
+        self,
+        model_name_or_path: str,
+        model_class: str,
+        model_kwargs: Dict[str, Any] | None = None,
+        system_prompt: str | None = None,
+        processor_kwargs: Dict[str, Any] | None = None,
+        generation_kwargs: Dict[str, Any] | None = None,
+    ):
+        """Initialize the HuggingFace model wrapper.
+
+        Args:
+            model_name_or_path: The name or path of the model to use.
+            model_class: The class of the model to use.
+            model_kwargs: Additional model arguments.
+            system_prompt: System prompt to use.
+            processor_kwargs: Additional processor arguments.
+            generation_kwargs: Additional generation arguments.
+        """
+        super().__init__(system_prompt=system_prompt)
+
+        self.model_name_or_path = model_name_or_path
+        self.model_kwargs = model_kwargs or {}
+        self.base_model_class = model_class
+        self.processor_kwargs = processor_kwargs or {}
+        self.generation_kwargs = generation_kwargs or {}
+
+        self.processor = self.load_processor()
+        self.model = self.load_model()
+
+    @override
+    def format_inputs(self, batch: TextImageBatch | TextBatch) -> Dict[str, torch.Tensor]:
+        """Formats inputs for HuggingFace models.
+
+        Args:
+            batch: A batch of text and image inputs.
+
+        Returns:
+            A dictionary produced by the provided processor following a format like:
+            {
+                "input_ids": ...,
+                "attention_mask": ...,
+                "pixel_values": ...
+            }
+        """
+        message_batch, image_batch, _, _ = self._unpack_batch(batch)
+        with_images = image_batch is not None
+
+        message_batch = language_message_utils.batch_insert_system_message(
+            message_batch, self.system_message
+        )
+        message_batch = list(map(language_message_utils.combine_system_messages, message_batch))
+
+        if self.processor.chat_template is not None:  # type: ignore
+            templated_text = [
+                self.processor.apply_chat_template(  # type: ignore
+                    message,
+                    add_generation_prompt=True,
+                    tokenize=False,
+                )
+                for message in map(
+                    functools.partial(
+                        message_utils.format_huggingface_message,
+                        with_images=with_images,
+                    ),
+                    message_batch,
+                )
+            ]
+        else:
+            raise NotImplementedError("Currently only chat models are supported.")
+
+        processor_inputs = {
+            "text": templated_text,
+            "return_tensors": "pt",
+            **self.processor_kwargs,
+        }
+
+        if with_images:
+            processor_inputs["image"] = [[image] for image in image_batch]
+
+        return self.processor(**processor_inputs).to(self.model.device)  # type: ignore
+
+    @override
+    def model_forward(self, batch: Dict[str, torch.Tensor]) -> List[str]:
+        """Generates text output from the model. Is called by the `generate` method.
+
+        Args:
+            batch: A dictionary containing the input data, which may include:
+                - "text": List of messages formatted for the model.
+                - "image": List of image tensors.
+
+        Returns:
+            A dictionary containing the processed input and the model's output.
+        """
+        output = self.model.generate(**batch, **self.generation_kwargs)  # type: ignore
+        return self._decode_output(output, batch["input_ids"].shape[-1])
+
+    @override
+    def load_model(self) -> nn.Module:
+        """Setting up the model. Used for delayed model initialization.
+
+        Raises:
+            ValueError: If the model class is not found in transformers or if the model
+                does not support gradient checkpointing but it is enabled.
+        """
+        logger.info(f"Configuring model: {self.model_name_or_path}")
+        if hasattr(transformers, self.base_model_class):
+            model_class = getattr(transformers, self.base_model_class)
+        else:
+            raise ValueError(f"Model class {self.base_model_class} not found in transformers")
+
+        model = model_class.from_pretrained(self.model_name_or_path, **self.model_kwargs)
+
+        if not hasattr(model, "generate"):
+            raise ValueError(f"Model {self.model_name_or_path} does not support generation. ")
+
+        return model
+
+    def load_processor(self) -> Callable:
+        """Initialize the processor."""
+        return transformers.AutoProcessor.from_pretrained(
+            self.model_name_or_path,
+            **self.processor_kwargs,
+        )
+
+    def _unpack_batch(self, batch: TextImageBatch | TextBatch) -> tuple:
+        if isinstance(batch, TextImageBatch):
+            return batch.text, batch.image, batch.target, batch.metadata
+        return batch.text, None, batch.target, batch.metadata
+
+    def _decode_output(self, output: torch.Tensor, instruction_length: int) -> List[str]:
+        """Decode the model's batch output to text.
+
+        Args:
+            output: The raw output from the model.
+            instruction_length: The length of the instruction in the input.
+
+        Returns:
+            A list of decoded text responses.
+        """
+        decoded_input = self.processor.batch_decode(  # type: ignore
+            output[:, :instruction_length], skip_special_tokens=True
+        )
+        decoded_output = self.processor.batch_decode(  # type: ignore
+            output[:, instruction_length:], skip_special_tokens=True
+        )
+
+        logger.debug(f"Decoded input: {decoded_input}")
+        logger.debug(f"Decoded output: {decoded_output}")
+
+        return decoded_output

eva/multimodal/models/wrappers/litellm.py

@@ -0,0 +1,56 @@
+"""LiteLLM vision-language model wrapper."""
+
+import logging
+from typing import Any, Dict, List
+
+from typing_extensions import override
+
+from eva.language.models import wrappers as language_wrappers
+from eva.language.utils.text import messages as language_message_utils
+from eva.multimodal.models.typings import TextImageBatch
+from eva.multimodal.models.wrappers import base
+from eva.multimodal.utils.text import messages as message_utils
+
+
+class LiteLLMModel(base.VisionLanguageModel):
+    """Wrapper class for LiteLLM vision-language models."""
+
+    def __init__(
+        self,
+        model_name: str,
+        model_kwargs: Dict[str, Any] | None = None,
+        system_prompt: str | None = None,
+        log_level: int | None = logging.INFO,
+    ):
+        """Initialize the LiteLLM Wrapper.
+
+        Args:
+            model_name: The name of the model to use.
+            model_kwargs: Additional keyword arguments to pass during
+                generation (e.g., `temperature`, `max_tokens`).
+            system_prompt: The system prompt to use (optional).
+            log_level: Optional logging level for LiteLLM. Defaults to WARNING.
+        """
+        super().__init__(system_prompt=system_prompt)
+
+        self.language_model = language_wrappers.LiteLLMModel(
+            model_name=model_name,
+            model_kwargs=model_kwargs,
+            system_prompt=system_prompt,
+            log_level=log_level,
+        )
+
+    @override
+    def format_inputs(self, batch: TextImageBatch) -> List[List[Dict[str, Any]]]:
+        message_batch, image_batch, _, _ = TextImageBatch(*batch)
+
+        message_batch = language_message_utils.batch_insert_system_message(
+            message_batch, self.system_message
+        )
+        message_batch = list(map(language_message_utils.combine_system_messages, message_batch))
+
+        return list(map(message_utils.format_litellm_message, message_batch, image_batch))
+
+    @override
+    def model_forward(self, batch: List[List[Dict[str, Any]]]) -> List[str]:
+        return self.language_model.model_forward(batch)

eva/multimodal/utils/__init__.py

@@ -0,0 +1 @@
+"""Multimodal utilities API."""

eva/multimodal/utils/image/__init__.py

@@ -0,0 +1,5 @@
+"""Multimodal image utilities API."""
+
+from eva.multimodal.utils.image.encode import encode_image
+
+__all__ = ["encode_image"]

eva/multimodal/utils/image/encode.py

@@ -0,0 +1,28 @@
+"""Image encoding utilities."""
+
+import base64
+import io
+from typing import Literal
+
+from torchvision import tv_tensors
+from torchvision.transforms.v2 import functional as F
+
+
+def encode_image(image: tv_tensors.Image, encoding: Literal["base64"]) -> str:
+    """Encodes an image tensor into a string format.
+
+    Args:
+        image: The image tensor to encode.
+        encoding: The encoding format to use. Currently only supports "base64".
+
+    Returns:
+        An encoded string representation of the image.
+    """
+    match encoding:
+        case "base64":
+            image_bytes = io.BytesIO()
+            F.to_pil_image(image).save(image_bytes, format="PNG", optimize=True)
+            image_bytes.seek(0)
+            return base64.b64encode(image_bytes.getvalue()).decode("utf-8")
+        case _:
+            raise ValueError(f"Unsupported encoding type: {encoding}. Supported: 'base64'")

eva/multimodal/utils/text/__init__.py

@@ -0,0 +1 @@
+"""Multimodal text utilities API."""

eva/multimodal/utils/text/messages.py

@@ -0,0 +1,79 @@
+"""Message formatting utilities for multimodal models."""
+
+from typing import Any, Dict, List
+
+from torchvision import tv_tensors
+
+from eva.language import utils as language_utils
+from eva.language.data.messages import MessageSeries, Role
+from eva.multimodal.utils import image as image_utils
+
+
+def format_huggingface_message(
+    message: MessageSeries, with_images: bool = False
+) -> List[Dict[str, Any]]:
+    """Formats a message series into a format suitable for Huggingface models."""
+    if not with_images:
+        return language_utils.format_chat_message(message)
+
+    formatted_message = []
+    for item in message:
+        if item.role == Role.SYSTEM:
+            formatted_message += language_utils.format_chat_message([item])
+        else:
+            formatted_message.append(
+                {
+                    "role": item.role,
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": str(item.content),
+                        },
+                        {"type": "image"},
+                    ],
+                }
+            )
+    return formatted_message
+
+
+def format_litellm_message(
+    message: MessageSeries, image: tv_tensors.Image | None
+) -> List[Dict[str, Any]]:
+    """Format a message series for LiteLLM API.
+
+    Args:
+        message: The message series to format.
+        image: Optional image to include in the message.
+
+    Returns:
+        A list of formatted message dictionaries.
+    """
+    if image is None:
+        return language_utils.format_chat_message(message)
+
+    formatted_message = []
+    for item in message:
+        if item.role == Role.SYSTEM:
+            formatted_message += language_utils.format_chat_message([item])
+        else:
+            formatted_message.append(
+                {
+                    "role": item.role,
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": str(item.content),
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": (
+                                    f"data:image/png;base64,"
+                                    f"{image_utils.encode_image(image, encoding='base64')}"
+                                )
+                            },
+                        },
+                    ],
+                }
+            )
+    return formatted_message

eva/vision/data/datasets/classification/patch_camelyon.py

@@ -61,6 +61,13 @@ class PatchCamelyon(vision.VisionDataset[tv_tensors.Image, torch.Tensor]):
     ]
     """Test resources."""
 
+    _expected_length = {
+        "train": 262144,
+        "val": 32768,
+        "test": 32768,
+    }
+    """Expected dataset length for each split."""
+
     _license: str = (
         "Creative Commons Zero v1.0 Universal (https://choosealicense.com/licenses/cc0-1.0/)"
     )
@@ -113,14 +120,9 @@ class PatchCamelyon(vision.VisionDataset[tv_tensors.Image, torch.Tensor]):
 
     @override
     def validate(self) -> None:
-        expected_length = {
-            "train": 262144,
-            "val": 32768,
-            "test": 32768,
-        }
         _validators.check_dataset_integrity(
             self,
-            length=expected_length.get(self._split, 0),
+            length=self._expected_length.get(self._split, 0),
             n_classes=2,
             first_and_last_labels=("no_tumor", "tumor"),
         )

eva/vision/data/transforms/__init__.py

@@ -13,10 +13,11 @@ from eva.vision.data.transforms.intensity import (
     RandShiftIntensity,
     ScaleIntensityRange,
 )
-from eva.vision.data.transforms.spatial import RandFlip, RandRotate90, Spacing
+from eva.vision.data.transforms.spatial import RandFlip, RandRotate90, Resize, Spacing
 from eva.vision.data.transforms.utility import EnsureChannelFirst
 
 __all__ = [
+    "Resize",
     "ResizeAndCrop",
     "Squeeze",
     "CropForeground",

eva/vision/data/transforms/spatial/__init__.py

@@ -1,7 +1,8 @@
 """Transforms for spatial operations."""
 
 from eva.vision.data.transforms.spatial.flip import RandFlip
+from eva.vision.data.transforms.spatial.resize import Resize
 from eva.vision.data.transforms.spatial.rotate import RandRotate90
 from eva.vision.data.transforms.spatial.spacing import Spacing
 
-__all__ = ["Spacing", "RandFlip", "RandRotate90"]
+__all__ = ["Spacing", "RandFlip", "RandRotate90", "Resize"]

eva/vision/data/transforms/spatial/functional/__init__.py

@@ -0,0 +1,5 @@
+"""Functional API for spatial transforms."""
+
+from eva.vision.data.transforms.spatial.functional.resize import resize_to_max_bytes
+
+__all__ = ["resize_to_max_bytes"]