sinapsis-huggingface 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

sinapsis_huggingface_transformers/src/sinapsis_huggingface_transformers/templates/pali_gemma/pali_gemma_detection.py

@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+from dataclasses import dataclass
+
+from sinapsis_core.data_containers.annotations import ImageAnnotations
+from sinapsis_core.template_base.base_models import TemplateAttributeType
+from sinapsis_huggingface_transformers.templates.pali_gemma.pali_gemma_inference import (
+    PaliGemmaInference,
+    PaliGemmaInferenceAttributes,
+)
+from sinapsis_huggingface_transformers.thirdparty.helpers import (
+    get_matches,
+    parse_label,
+    parse_location_tokens,
+)
+
+
+@dataclass(frozen=True)
+class PaliGemmaDetectionKeys:
+    "Keys to use during detection"
+
+    detection_prompt: str = "detect {}"
+
+
+class PaliGemmaDetectionAttributes(PaliGemmaInferenceAttributes):
+    """Configuration attributes for PaliGemma object detection tasks.
+
+    This class extends the base inference attributes to handle object detection specific configurations.
+
+    Attributes:
+        objects_to_detect (str | list[str]): Target objects to detect, can be a single string or list of strings
+    """
+
+    objects_to_detect: str | list[str]
+
+
+class PaliGemmaDetection(PaliGemmaInference):
+    """Implementation of PaliGemma object detection pipeline.
+
+    The template inherits functionality from its base class, extending
+    the functionality to run inference on an image and to identify
+    the objects from the attributes.
+
+    Usage example:
+
+    agent:
+      name: my_test_agent
+    templates:
+    - template_name: InputTemplate
+      class_name: InputTemplate
+      attributes: {}
+    - template_name: PaliGemmaDetection
+      class_name: PaliGemmaDetection
+      template_input: InputTemplate
+      attributes:
+        model_path: '/path/to/paligemma/model'
+        processor_path: '`/path/to/processor'
+        model_cache_dir: /path/to/cache/dir
+        device: 'cuda'
+        max_new_tokens: 200
+        torch_dtype: float16
+        prompt: <image> caption en
+        objects_to_detect: 'object to detect'
+
+    """
+
+    AttributesBaseModel = PaliGemmaDetectionAttributes
+    KEYS = PaliGemmaDetectionKeys
+
+    def __init__(self, attributes: TemplateAttributeType) -> None:
+        super().__init__(attributes)
+
+        objects_str = self.initialize_objects_str()
+        self.prompt = self.KEYS.detection_prompt.format(objects_str)
+
+    def initialize_objects_str(self) -> str:
+        """
+        Initialize the objects to detect string according to the specified format.
+
+        Returns:
+            str: String enlisting the objects to be defined in the detection prompt.
+        """
+
+        if isinstance(self.attributes.objects_to_detect, str):
+            return self.attributes.objects_to_detect
+        return "; ".join(self.attributes.objects_to_detect)
+
+    def _format_text_for_prompt(self, text: str) -> str:
+        """Formats input text as a detection prompt.
+
+        Args:
+            text (str): Raw text content (expected to be objects to detect)
+
+        Returns:
+            str: Formatted detection prompt
+        """
+        return self.KEYS.detection_prompt.format(text)
+
+    def _create_annotation(
+        self, caption: str, confidence: float, image_shape: tuple[int, ...]
+    ) -> list[ImageAnnotations]:
+        """Creates structured annotations from detection model outputs.
+
+        Processes the model's output caption to extract bounding box coordinates
+        and object labels for each detected instance.
+
+        Args:
+            caption (str): Raw detection output from the model
+            confidence (float): Confidence score for the predictions
+            image_shape (tuple[int, ...]): Dimensions of the input image (height, width)
+
+        Returns:
+            list[ImageAnnotations]: List of annotations containing bounding boxes and labels
+                                  for each detected object
+        """
+        annotations = []
+        matches = get_matches(caption)
+
+        for match_coord in matches:
+            coords = parse_location_tokens(match_coord, image_shape)
+            label = parse_label(match_coord)
+            annotation = self.create_bbox_annotation(coords, label, confidence)
+            annotations.append(annotation)
+
+        return annotations

sinapsis_huggingface_transformers/src/sinapsis_huggingface_transformers/templates/pali_gemma/pali_gemma_inference.py

@@ -0,0 +1,260 @@
+# -*- coding: utf-8 -*-
+import numpy as np
+import torch
+from sinapsis_core.data_containers.annotations import BoundingBox, ImageAnnotations
+from sinapsis_core.data_containers.data_packet import DataContainer, ImagePacket
+from sinapsis_core.template_base.base_models import TemplateAttributeType
+from sinapsis_data_visualization.helpers.detection_utils import bbox_xyxy_to_xywh
+from sinapsis_huggingface_transformers.templates.pali_gemma.pali_gemma_base import (
+    PaliGemmaBase,
+    PaliGemmaBaseAttributes,
+)
+from transformers.generation.utils import GenerateOutput
+
+
+class PaliGemmaInferenceAttributes(PaliGemmaBaseAttributes):
+    """Configuration attributes for PaliGemma inference.
+
+    Attributes:
+        prompt (str): Prompt to run the inference (default: "<image>caption en")
+
+    The <image> token is essential as it serves as a marker that tells the model where to look at the image
+    when processing the input. This token enables the model to understand the relationship between the visual
+    and textual components during processing.
+
+    Example prompts:
+        - "<image>caption en" -> Generates a basic caption in English
+        - "<image>What objects can you see in this image?" -> Lists objects in the image
+    """
+
+    prompt: str = "<image>caption en"
+
+
+class PaliGemmaInference(PaliGemmaBase):
+    """Implementation of PaliGemma inference pipeline for image processing and caption generation.
+
+    This class handles the inference process for PaliGemma models, including image processing,
+    caption generation, and annotation creation. It supports both basic captioning and
+    detection/segmentation tasks.
+
+    Usage example:
+
+    agent:
+      name: my_test_agent
+    templates:
+    - template_name: InputTemplate
+      class_name: InputTemplate
+      attributes: {}
+    - template_name: PaliGemmaInference
+      class_name: PaliGemmaInference
+      template_input: InputTemplate
+      attributes:
+        model_path: '/path/to/paligemma/model'
+        processor_path: '`/path/to/processor'
+        model_cache_dir: /path/to/cache/dir
+        device: 'cuda'
+        max_new_tokens: 200
+        torch_dtype: float16
+        prompt: <image> caption en
+
+    """
+
+    AttributesBaseModel = PaliGemmaInferenceAttributes
+    INPUT_IDS = "input_ids"
+
+    def __init__(self, attributes: TemplateAttributeType) -> None:
+        super().__init__(attributes)
+        self.prompt = self.attributes.prompt
+
+    def _prepare_inputs(self, image_content: np.ndarray) -> dict:
+        """Prepares the input for model inference by processing the image and text prompt.
+
+        Args:
+            image_content (np.ndarray): Raw image content to be processed as a numpy array
+
+        Returns:
+            dict: Processed inputs containing:
+                - input_ids (torch.Tensor): Token IDs for the text prompt and image tokens
+                - attention_mask (torch.Tensor): Binary mask indicating valid input positions (1s)
+                - pixel_values (torch.Tensor): Processed image tensor with normalized pixel values
+                    in shape (batch_size, channels, height, width)
+
+        Note:
+            - The format of the returns it's because uses PyTorch tensors (return_tensors="pt")
+        """
+
+        return self.processor(
+            images=image_content,
+            text=self.prompt,
+            return_tensors="pt",
+        ).to(self.attributes.device)
+
+    def _generate_caption(self, inputs: dict) -> torch.Tensor:
+        """Generates caption using the model.
+
+        Args:
+            inputs (dict): Processed model inputs for the processor, including input IDs of the image and prompt
+
+        Returns:
+            GeneratedCaptionOutput: A structured output containing:
+                - sequences: tensor with token IDs of the generated sequence
+                - scores: tuple of tensors with token prediction scores for each generation step
+                - logits: optional tensor with raw logits (None in this configuration)
+                - attentions: optional attention weights (None in this configuration)
+                - hidden_states: optional hidden states (None in this configuration)
+                - past_key_values: tuple of tensors containing past keys/values for attention mechanism
+
+        Configuration parameters:
+            - max_new_tokens: Maximum number of new tokens to generate
+            - return_dict_in_generate: Returns output as a structured dictionary
+            - output_scores: Includes prediction scores in the output
+        """
+        with torch.no_grad():
+            return self.model.generate(
+                **inputs,
+                max_new_tokens=self.attributes.max_new_tokens,
+                return_dict_in_generate=True,
+                output_scores=True,
+            )
+
+    @staticmethod
+    def _calculate_confidence_score(outputs: GenerateOutput) -> float:
+        """Calculates the confidence score from model generation outputs.
+
+        The confidence score is computed as the mean of the highest probability
+        for each generated token in the sequence.
+
+        Args:
+            outputs (GenerateOutput): Model generation output containing scores
+                for each generated token
+
+        Returns:
+            float: Average confidence score across all generated tokens
+        """
+        scores = torch.stack(outputs.scores)
+        probs = torch.softmax(scores, dim=-1)
+        token_confidences = torch.max(probs, dim=-1).values
+        return float(torch.mean(token_confidences).cpu())
+
+    def _decode_caption(self, outputs: GenerateOutput, input_len: int) -> str:
+        """Decodes the model output sequences into readable caption text.
+
+        Args:
+            outputs (GenerateOutput): Model generation output containing the
+                generated token sequences
+            input_len (int): Length of the input sequence to skip initial tokens
+
+        Returns:
+            str: Decoded caption text with special tokens removed
+        """
+        return self.processor.decode(outputs.sequences[0][input_len:], skip_special_tokens=True)
+
+    def _create_annotation(
+        self, caption: str, confidence: float, image_shape: tuple[int, ...]
+    ) -> list[ImageAnnotations]:
+        """Creates image annotations from the generated caption.
+
+        Args:
+            caption (str): Generated caption text
+            confidence (float): Confidence score for the prediction
+            image_shape (tuple[int, ...]): Shape of the input image
+
+        Returns:
+            list[ImageAnnotations]: List containing annotation with caption information
+        """
+
+        _, _ = self, image_shape
+        return [ImageAnnotations(text=caption, confidence_score=confidence)]
+
+    def _process_single_image(self, image_packet: ImagePacket) -> None:
+        """Processes a single image through the inference pipeline.
+
+        Args:
+            image_packet (ImagePacket): Container with image data and metadata
+
+        Returns:
+            None: Modifies the image_packet in place by adding annotations
+        """
+        inputs = self._prepare_inputs(image_packet.content)
+        outputs = self._generate_caption(inputs)
+        input_len = inputs[self.INPUT_IDS].shape[-1]
+        caption = self._decode_caption(outputs, input_len)
+        confidence = self._calculate_confidence_score(outputs)
+        annotations = self._create_annotation(caption, confidence, image_packet.content.shape)
+        image_packet.annotations.extend(annotations)
+
+    def _format_text_for_prompt(self, text: str) -> str:
+        """Formats the incoming text appropriately for the current task type.
+            Base implementation returns the text as-is, subclasses may override
+        to apply task-specific formatting.
+            Args:
+            text (str): Raw text content
+            Returns:
+            str: Formatted prompt text
+        """
+        _ = self
+        return text
+
+    def process_from_text_packet(self, container: DataContainer) -> None:
+        """
+        Extract prompts from the received list of text packets and use them to perform inference in each received image
+        packet.
+
+        Args:
+            container (DataContainer): Data-container with text and image packets to be processed.
+        """
+        for text_packet in container.texts:
+            self.prompt = self._format_text_for_prompt(text_packet.content)
+            if container.images:
+                for image_packet in container.images:
+                    self._process_single_image(image_packet)
+
+    def process_from_prompt(self, container: DataContainer) -> None:
+        """
+        Perform inference in each received image packet using the prompt defined in template attributes.
+
+        Args:
+            container (DataContainer): Data-container with image packets to be processed.
+        """
+        if container.images:
+            for image_packet in container.images:
+                self._process_single_image(image_packet)
+
+    def execute(self, container: DataContainer) -> DataContainer:
+        """Executes the inference pipeline on a batch of images.
+
+        If text packets are present, uses each text as input for prompt formatting.
+        If no text packets exist, uses the default prompt from attributes.
+
+        Args:
+            container (DataContainer): Container with text and image packets
+
+        Returns:
+            DataContainer: Processed container with added annotations
+        """
+        if container.texts:
+            self.process_from_text_packet(container)
+        else:
+            self.process_from_prompt(container)
+
+        return container
+
+    @staticmethod
+    def create_bbox_annotation(coords: tuple[float, ...], label: str, confidence: float) -> ImageAnnotations:
+        """Creates bounding box annotation from coordinates and metadata.
+
+        Args:
+            coords (tuple[float, ...]): Coordinates (x0, y0, x1, y1)
+            label (str): Label for the detected object
+            confidence (float): Confidence score for the detection
+
+        Returns:
+            ImageAnnotations: Annotation object with bounding box information
+        """
+        x0, y0, x1, y1 = coords
+        x, y, w, h = bbox_xyxy_to_xywh([x0, y0, x1, y1])
+        return ImageAnnotations(
+            label_str=label,
+            confidence_score=confidence,
+            bbox=BoundingBox(x=x, y=y, w=w, h=h),
+        )

sinapsis_huggingface_transformers/src/sinapsis_huggingface_transformers/templates/speech_to_text_transformers.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 from sinapsis_core.data_containers.data_packet import DataContainer, TextPacket
-from sinapsis_core.template_base.template import TemplateAttributeType
+from sinapsis_core.template_base.base_models import TemplateAttributeType
 
 from sinapsis_huggingface_transformers.templates.base_transformers import TransformersBase
 
@@ -39,6 +39,7 @@ class SpeechToTextTransformers(TransformersBase):
     def __init__(self, attributes: TemplateAttributeType) -> None:
         super().__init__(attributes)
         self.task = "automatic-speech-recognition"
+        self.setup_pipeline()
 
     def transformation_method(self, container: DataContainer) -> DataContainer:
         """Speech recognition (speech-to-text) using a Transformers Pipeline.

sinapsis_huggingface_transformers/src/sinapsis_huggingface_transformers/templates/summarization_transformers.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from sinapsis_core.data_containers.data_packet import DataContainer
-from sinapsis_core.template_base import TemplateAttributeType
+from sinapsis_core.template_base.base_models import TemplateAttributeType
 
 from sinapsis_huggingface_transformers.templates.base_transformers import TransformersBase
 
@@ -38,6 +38,7 @@ class SummarizationTransformers(TransformersBase):
     def __init__(self, attributes: TemplateAttributeType) -> None:
         super().__init__(attributes)
         self.task = "summarization"
+        self.setup_pipeline()
 
     def transformation_method(self, container: DataContainer) -> DataContainer:
         """Summarize text using a Transformers Pipeline.

sinapsis_huggingface_transformers/src/sinapsis_huggingface_transformers/templates/text_to_speech_transformers.py

@@ -3,7 +3,7 @@
 import numpy as np
 import torch
 from sinapsis_core.data_containers.data_packet import AudioPacket, DataContainer, TextPacket
-from sinapsis_core.template_base import TemplateAttributeType
+from sinapsis_core.template_base.base_models import TemplateAttributeType
 
 from sinapsis_huggingface_transformers.helpers import sentences_to_n_words, split_text_into_sentences
 from sinapsis_huggingface_transformers.templates.base_transformers import (
@@ -64,8 +64,9 @@ class TextToSpeechTransformers(TransformersBase):
 
     def __init__(self, attributes: TemplateAttributeType) -> None:
         super().__init__(attributes)
-        self.sample_rate = self._get_sample_rate()
         self.task = "text-to-speech"
+        self.setup_pipeline()
+        self.sample_rate = self._get_sample_rate()
 
     def _get_sample_rate(self) -> int:
         """Retrieve the sample rate for the generated audio.

sinapsis_huggingface_transformers/src/sinapsis_huggingface_transformers/templates/translation_transformers.py

@@ -2,7 +2,7 @@
 
 
 from sinapsis_core.data_containers.data_packet import DataContainer
-from sinapsis_core.template_base import TemplateAttributeType
+from sinapsis_core.template_base.base_models import TemplateAttributeType
 
 from sinapsis_huggingface_transformers.templates.base_transformers import (
     TransformersBase,
@@ -56,6 +56,7 @@ class TranslationTransformers(TransformersBase):
     def __init__(self, attributes: TemplateAttributeType) -> None:
         super().__init__(attributes)
         self.task = f"translation_{self.attributes.source_language}_to_{self.attributes.target_language}"
+        self.setup_pipeline()
 
     def transformation_method(self, container: DataContainer) -> DataContainer:
         """Translate text using a Transformers Pipeline.

sinapsis_huggingface_transformers/src/sinapsis_huggingface_transformers/thirdparty/helpers.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+The constants and methods declared in this file are inspired in the following source:
+
+https://github.com/google/generative-ai-docs/blob/main/site/en/gemma/docs/paligemma/inference-with-keras.ipynb
+
+which is Licensed under the Apache License, Version 2.0.
+
+"""
+
+import numpy as np
+import regex as re
+
+COORDS_PATTERN: str = r"<loc(?P<y0>\d\d\d\d)><loc(?P<x0>\d\d\d\d)><loc(?P<y1>\d\d\d\d)><loc(?P<x1>\d\d\d\d)>"
+LABEL_PATTERN: str = r" (?P<label>.+?)( ;|$)"
+
+DETECTION_PATTERN: str = COORDS_PATTERN + LABEL_PATTERN
+
+LOCATION_KEYS: tuple[str, ...] = ("y0", "x0", "y1", "x1")
+LOCATION_SCALE: float = 1024.0
+
+
+def parse_location_tokens(match_coord: re.Match, image_shape: tuple[int, ...]) -> np.ndarray:
+    """Parses location tokens from model output into normalized coordinates.
+
+    Args:
+        match_coord (dict): Dictionary containing matched location tokens
+        image_shape (tuple[int, ...]): Shape of the input image
+
+    Returns:
+        np.ndarray: Normalized coordinates (x0, y0, x1, y1)
+    """
+    match_dict = match_coord.groupdict()
+    x0 = float(match_dict[LOCATION_KEYS[1]]) / LOCATION_SCALE * image_shape[1]
+    y0 = float(match_dict[LOCATION_KEYS[0]]) / LOCATION_SCALE * image_shape[0]
+    x1 = float(match_dict[LOCATION_KEYS[3]]) / LOCATION_SCALE * image_shape[1]
+    y1 = float(match_dict[LOCATION_KEYS[2]]) / LOCATION_SCALE * image_shape[0]
+    return np.array([x0, y0, x1, y1])
+
+
+def parse_label(match_coord: re.Match) -> str:
+    """
+    Retrieves detection label from a regex Match object.
+
+
+    Args:
+        match_coord (Match): The Match object containing the label information.
+
+    Returns:
+        str: The detection label.
+    """
+    label = match_coord.groupdict().get("label")
+    if label is None:
+        return ""
+    return label.strip()
+
+
+def get_matches(caption: str) -> re.Scanner:
+    """
+    Creates an iterable containing all the detection matches found in the
+    produced model caption.
+
+    Args:
+        caption (str): The caption produced by the paligemma model.
+
+    Returns:
+        Scanner: An iterable object containing all the regex matches.
+    """
+
+    return re.finditer(DETECTION_PATTERN, caption)