deltpy 1.4.0__py3-none-any.whl

delt/__init__.py

@@ -0,0 +1,31 @@
+"""Main package of delt."""
+
+from .encoders import (
+    ClapEncoder,
+    ClipEncoder,
+    SentenceTransformerEncoder,
+    SiglipEncoder,
+    XclipEncoder,
+    get_encoder,
+)
+from .pipelines import (
+    AudioPipeline,
+    ImagePipeline,
+    TextPipeline,
+    VideoPipeline,
+)
+from .teachers import LLMTextTeacher
+
+__all__ = [
+    "SentenceTransformerEncoder",
+    "ClipEncoder",
+    "SiglipEncoder",
+    "ClapEncoder",
+    "XclipEncoder",
+    "get_encoder",
+    "TextPipeline",
+    "ImagePipeline",
+    "AudioPipeline",
+    "VideoPipeline",
+    "LLMTextTeacher",
+]

delt/common/__init__.py

@@ -0,0 +1,21 @@
+"""Subpackage for common utils."""
+
+from .experimental import unimodal_kfold
+from .io import base64_encode
+from .iterables import batchify, batchify_tensor, dict_cartesian_product
+from .llm import generate_completion, generate_completions
+from .logging import get_logger
+from .prompting import format_prompt, make_output_model
+
+__all__ = [
+    "get_logger",
+    "format_prompt",
+    "unimodal_kfold",
+    "dict_cartesian_product",
+    "batchify",
+    "batchify_tensor",
+    "generate_completion",
+    "generate_completions",
+    "base64_encode",
+    "make_output_model",
+]

delt/common/experimental.py

@@ -0,0 +1,46 @@
+"""Module for experimental utils."""
+
+from typing import Any
+
+import torch
+from sklearn.model_selection import StratifiedKFold
+
+
+def unimodal_kfold(
+    data: list[Any],
+    labels: list[str],
+    n_splits: int = 4,
+    random_state: int = 42,
+) -> tuple[list[dict], list[dict]]:
+    """
+    Create k folds for unimodal settings.
+
+    Args:
+        data (list[Any]): A list of input data samples.
+        labels (list[str]): A list of labels corresponding to the input data samples.
+        n_splits (int): The number of folds for cross-validation.
+        random_state (int): The random seed for reproducibility.
+
+    Returns:
+        tuple[list[dict], list[dict]]: A tuple containing two lists of dictionaries for training and validation splits.
+
+    """
+    kfold = StratifiedKFold(
+        n_splits=n_splits, shuffle=True, random_state=random_state
+    )
+    splits = kfold.split(data, labels)
+    train_splits, val_splits = [], []
+    for train_idxs, val_idxs in splits:
+        if isinstance(data, torch.Tensor):
+            train_data = data[train_idxs]
+            val_data = data[val_idxs]
+        else:
+            train_data = [data[i] for i in train_idxs]
+            val_data = [data[i] for i in val_idxs]
+
+        train_labels = [labels[i] for i in train_idxs]
+        val_labels = [labels[i] for i in val_idxs]
+
+        train_splits.append({"x": train_data, "y": train_labels})
+        val_splits.append({"x": val_data, "y": val_labels})
+    return train_splits, val_splits

delt/common/io.py

@@ -0,0 +1,24 @@
+"""Module for io utils."""
+
+from base64 import b64encode
+from typing import Optional
+
+
+def base64_encode(data: bytes, mime_type: Optional[str] = None) -> str:
+    """
+    Encode a `bytes` object into base64.
+
+    If `mime_type` is provided, a Data URI is returned with the mime type in the header.
+
+    Args:
+        data (bytes): bytes to encode.
+        mime_type (Optional[str]): mime type of the data.
+
+    Returns:
+        str: a base64 stream in Data URI format if `mime_type` is provided.
+
+    """
+    b64 = b64encode(data).decode("utf-8")
+    if mime_type:
+        return f"data:{mime_type};base64,{b64}"
+    return b64

delt/common/iterables.py

@@ -0,0 +1,67 @@
+"""Module for iterables."""
+
+from collections.abc import Iterable, Iterator
+from itertools import islice, product
+from typing import TypeVar
+
+import torch
+
+T = TypeVar("T")
+
+
+def batchify(iterable: Iterable[T], batch_size: int) -> Iterator[list[T]]:
+    """
+    Create batches of `batch_size` from a generic `iterable`.
+
+    Args:
+        iterable (Iterable[T]): generic iterable to get batches from.
+        batch_size (int): the batch size.
+
+    Yields:
+        list[T]: a batch of elements as a list.
+
+    """
+    if batch_size <= 0:
+        raise ValueError("batch_size must be positive")
+
+    iterator = iter(iterable)
+
+    while batch := list(islice(iterator, batch_size)):
+        yield batch
+
+
+def batchify_tensor(
+    x: torch.Tensor, batch_size: int
+) -> Iterator[torch.Tensor]:
+    """
+    Create batches of `batch_size` from a torch tensor with shape (N, ...).
+
+    Args:
+        x (torch.Tensor): the torch tensor.
+        batch_size (int): the batch size.
+
+    Yields:
+        torch.Tensor: a batch of elements as a torch tensor with shape (`batch_size`, ...)
+
+    """
+    if batch_size <= 0:
+        raise ValueError("batch_size must be positive")
+
+    for i in range(0, len(x), batch_size):
+        yield x[i : i + batch_size]
+
+
+def dict_cartesian_product(d: dict) -> list[dict]:
+    """
+    Get the cartesian product over the fields in a dictionary.
+
+    Args:
+        d (dict): a dictionary.
+
+    Returns:
+        list[dict]: list of dictionaries representing the cartesian product.
+
+    """
+    keys = d.keys()
+    values = d.values()
+    return [dict(zip(keys, combination)) for combination in product(*values)]

delt/common/llm.py

@@ -0,0 +1,94 @@
+"""Module for litellm utils."""
+
+import json
+from concurrent.futures import ThreadPoolExecutor
+
+import litellm
+from pydantic import BaseModel
+from tqdm import tqdm
+
+from .logging import get_logger
+
+_logger = get_logger(__name__)
+
+
+def generate_completion(
+    model_name: str,
+    conversation: list[dict],
+    response_format: BaseModel,
+    decoding_args: dict = {},
+    extra_body: dict = {},
+) -> BaseModel:
+    """
+    Generate a completion using the specified model for a conversation.
+
+    Args:
+        model_name (str): name of the model.
+        conversation (list[dict]): message thread.
+        response_format (BaseModel): pydantic model for response formatting.
+        decoding_args (dict): additional arguments for decoding.
+        extra_body (dict): extra body to send to the model.
+
+    Returns:
+        BaseModel: a completion output.
+
+    """
+    response = litellm.completion(
+        model=model_name,
+        messages=conversation,
+        response_format=response_format,
+        extra_body=extra_body,
+        **decoding_args,
+    )
+
+    if response.choices[0].finish_reason != "stop":
+        raise RuntimeError(
+            f"Completion did not finish properly: "
+            f"{response.choices[0].finish_reason}"
+        )
+
+    json_response = json.loads(response.choices[0].message.content)
+    return response_format(**json_response)
+
+
+def generate_completions(
+    model_name: str,
+    conversations: list[list[dict]],
+    response_format: BaseModel,
+    decoding_args: dict = {},
+    batch_size: int = 4,
+    extra_body: list[dict] = [],
+) -> list[BaseModel]:
+    """
+    Generate completions using the specified model and conversations.
+
+    Args:
+        model_name (str): name of the model.
+        conversations (list[list[dict]]): a list of conversations.
+        response_format (BaseModel): pydantic model for response formatting.
+        decoding_args (dict): additional arguments for decoding.
+        batch_size (int): number of concurrent requests.
+        extra_body (list[dict]): extra bodies to send to the model.
+
+    Returns:
+        list[BaseModel]: completion outputs.
+
+    """
+    completions, responses = [], []
+    with ThreadPoolExecutor(
+        max_workers=min(batch_size, len(conversations))
+    ) as thread_pool:
+        for i, message in enumerate(conversations):
+            responses.append(
+                thread_pool.submit(
+                    generate_completion,
+                    model_name,
+                    message,
+                    response_format,
+                    decoding_args,
+                    extra_body[i] if extra_body else None,
+                )
+            )
+        completions = [response.result() for response in tqdm(responses)]
+
+    return completions

delt/common/logging.py

@@ -0,0 +1,81 @@
+"""Module for logging utils."""
+
+import logging
+import sys
+from datetime import datetime
+from pathlib import Path
+
+_time = datetime.now()
+
+COLORS = {
+    "grey": "\x1b[38;20m",
+    "yellow": "\x1b[33;20m",
+    "bold_yellow": "\x1b[33;1m",
+    "red": "\x1b[31;20m",
+    "bold_red": "\x1b[31;1m",
+    "green": "\x1b[32;20m",
+    "bold_green": "\x1b[32;1m",
+    "blue": "\x1b[34;20m",
+    "bold_blue": "\x1b[34;1m",
+    "reset": "\x1b[0m",
+}
+
+
+def color_log(text: str, color: str) -> str:
+    """
+    Add color to a log text.
+
+    Args:
+        text (str): a text.
+        color (str): a color in `COLORS`.
+
+    Returns:
+        str: a text with color codes added.
+
+    """
+    return COLORS[color] + text + COLORS["reset"]
+
+
+def get_logger(
+    module_name: str,
+) -> logging.Logger:
+    """
+    Return the logger used across modules.
+
+    Args:
+        module_name (str): name of the module.
+
+    Returns:
+        logging.Logger: the logger.
+
+    """
+    logger = logging.getLogger(module_name)
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter(
+        "[%(asctime)s] - %(levelname)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    logfile = (
+        Path("logs")
+        .joinpath(
+            _time.strftime("%Y_%m_%d"),
+            _time.strftime("%H_%M_%S"),
+            "project.log",
+        )
+        .absolute()
+    )
+
+    logfile.parent.mkdir(parents=True, exist_ok=True)
+
+    fh = logging.FileHandler(logfile)
+    fh.setFormatter(formatter)
+
+    sh = logging.StreamHandler(sys.stdout)
+    sh.setFormatter(formatter)
+
+    logger.addHandler(sh)
+    logger.addHandler(fh)
+
+    return logger

delt/common/prompting.py

@@ -0,0 +1,41 @@
+"""Module for prompting utils."""
+
+from typing import Literal
+
+from pydantic import BaseModel, create_model
+
+
+def format_prompt(
+    label_verbalizations: dict[str, str], prompt_template: str
+) -> list[str]:
+    """
+    Format a prompt template with label verbalizations.
+
+    Args:
+        label_verbalizations (dict[str, str]): verbalizations of the labels, e.g. {"positive": "very cool!", "negative": "horrible"}
+        prompt_template (str): template to format label verbalizations, e.g., "This text is {}" being instantiated as "This text is very cool!".
+
+    Returns:
+        list[str]: instantiated prompts, one for each verbalization in `label_verbalizations`.
+
+    """
+    return [
+        prompt_template.format(verbalization)
+        for verbalization in label_verbalizations.values()
+    ]
+
+
+def make_output_model(label_set: list[str]) -> type[BaseModel]:
+    """
+    Create a pydantic model dynamically to be used as structured output in classification tasks.
+
+    Args:
+        label_set (list[str]): list of label names.
+
+    Returns:
+        type[BaseModel]: a pydantic model.
+
+    """
+    return create_model(
+        "Output", label=(Literal.__getitem__(tuple(label_set)), ...)
+    )

delt/common/utils.py

@@ -0,0 +1 @@
+"""Module for common utils."""

delt/encoders/__init__.py

@@ -0,0 +1,52 @@
+"""Package for encoders."""
+
+from typing import Any
+
+from .audio import ClapEncoder, GlapEncoder
+from .base import Encoder
+from .image import ClipEncoder, SiglipEncoder
+from .text import SentenceTransformerEncoder
+from .video import XclipEncoder
+
+REGISTRY: dict[str, type[Encoder]] = {
+    "glap": GlapEncoder,
+    "clap": ClapEncoder,
+    "clip": ClipEncoder,
+    "siglip": SiglipEncoder,
+    "sentence-transformer": SentenceTransformerEncoder,
+    "xclip": XclipEncoder,
+}
+
+
+def get_encoder(class_name: str, *args: Any, **kwargs: Any) -> Encoder:
+    """
+    Instantiate an encoder by class name, given arbitrary arguments.
+
+    Args:
+        class_name (str): class name of the encoder. It should be available in `REGISTRY`.
+        *args: positional arguments passed directly to the encoder's constructor.
+        **kwargs: keyword arguments passed directly to the encoder's constructor.
+
+    Returns:
+        Encoder: an instantiated encoder.
+
+    """
+    if class_name not in REGISTRY:
+        raise KeyError(
+            f"Unknown encoder class: {class_name}. Available: {list(REGISTRY.keys())}"
+        )
+
+    encoder_cls = REGISTRY[class_name]
+    return encoder_cls(*args, **kwargs)
+
+
+__all__ = [
+    "Encoder",
+    "SentenceTransformerEncoder",
+    "ClipEncoder",
+    "GlapEncoder",
+    "SiglipEncoder",
+    "ClapEncoder",
+    "XclipEncoder",
+    "get_encoder",
+]

delt/encoders/audio/__init__.py

@@ -0,0 +1,6 @@
+"""Package for audio encoders."""
+
+from .clap import ClapEncoder
+from .glap import GlapEncoder
+
+__all__ = ["ClapEncoder", "GlapEncoder"]

delt/encoders/audio/base.py

@@ -0,0 +1,129 @@
+"""Module for the base audio encoder."""
+
+from abc import abstractmethod
+
+import torch
+from transformers import (
+    AutoProcessor,
+    PreTrainedModel,
+    PreTrainedTokenizerBase,
+    ProcessorMixin,
+)
+
+from delt.encoders.base import Encoder
+from delt.types import Audio, Image, Video
+
+
+class AudioEncoder(Encoder):
+    """Base audio encoder model."""
+
+    def __init__(
+        self,
+        encoder_name: str,
+        normalize_embeddings: bool = True,
+    ) -> None:
+        """
+        Initialize a encoder model.
+
+        Args:
+            encoder_name (str): name of sentence-transformer-compatible model.
+            normalize_embeddings (bool): whether to apply L2 normalization to all embeddings.
+
+        """
+        super().__init__(
+            encoder_name=encoder_name,
+            normalize_embeddings=normalize_embeddings,
+        )
+
+    @property
+    def processor(self) -> PreTrainedTokenizerBase | ProcessorMixin | None:
+        """
+        Processor tied to an audio encoder.
+
+        Returns:
+            PreTrainedTokenizerBase | ProcessorMixin | None: tokenizers for text
+                models, processor mixin for image/audio/video, and `None` for
+                models outside the HuggingFace ecosystem.
+
+        """
+        if self._processor is None:
+            self._processor = AutoProcessor.from_pretrained(self.encoder_name)
+        return self._processor
+
+    @property
+    @abstractmethod
+    def encoder(self) -> PreTrainedModel:
+        """
+        A pretrained audio encoder model.
+
+        Returns:
+            PreTrainedModel: the pretrained audio encoder.
+
+        """
+        ...
+
+    @abstractmethod
+    def get_text_embeddings(
+        self, texts: list[str], batch_size: int = 64
+    ) -> torch.Tensor:
+        """
+        Embed a list of texts using the base encoder model.
+
+        Args:
+            texts (list[str]): a list of texts.
+            batch_size (int): batch size for inference.
+
+        Returns:
+            torch.Tensor: The text embeddings.
+
+        """
+        ...
+
+    def get_image_embeddings(
+        self, images: list[Image], batch_size: int = 8
+    ) -> torch.Tensor:
+        """
+        Embed a list of images using the base encoder model.
+
+        Args:
+            images (list[Image): list of images.
+            batch_size (int): batch size for inference.
+
+        Returns:
+            torch.Tensor: The image embeddings.
+
+        """
+        ...
+
+    @abstractmethod
+    def get_audio_embeddings(
+        self, audios: list[Audio], batch_size: int = 16
+    ) -> torch.Tensor:
+        """
+        Embed a list of audios using the base encoder model.
+
+        Args:
+            audios (list[Audio]): list of audios.
+            batch_size (int): batch size for inference.
+
+        Returns:
+            torch.Tensor: The audio embeddings.
+
+        """
+        ...
+
+    def get_video_embeddings(
+        self, videos: list[Video], batch_size: int = 16
+    ) -> torch.Tensor:
+        """
+        Embed a list of videos using the base encoder model.
+
+        Args:
+            videos (list[Video]): list of videos.
+            batch_size (int): batch size for inference.
+
+        Returns:
+            torch.Tensor: The video embeddings.
+
+        """
+        ...