mediapipe-nightly 0.10.21.post20250114__cp311-cp311-manylinux_2_28_x86_64.whl

mediapipe/tasks/python/components/containers/audio_data.py

@@ -0,0 +1,137 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MediaPipe audio data."""
+
+import dataclasses
+from typing import Optional
+
+import numpy as np
+
+
+@dataclasses.dataclass
+class AudioDataFormat:
+  """Audio format metadata.
+
+  Attributes:
+    num_channels: the number of channels of the audio data.
+    sample_rate: the audio sample rate.
+  """
+  num_channels: int = 1
+  sample_rate: Optional[float] = None
+
+
+class AudioData(object):
+  """MediaPipe Tasks' audio container."""
+
+  def __init__(
+      self,
+      buffer_length: int,
+      audio_format: AudioDataFormat = AudioDataFormat()
+  ) -> None:
+    """Initializes the `AudioData` object.
+
+    Args:
+      buffer_length: the length of the audio buffer.
+      audio_format: the audio format metadata.
+    """
+    self._audio_format = audio_format
+    self._buffer = np.zeros([buffer_length, self._audio_format.num_channels],
+                            dtype=np.float32)
+
+  def clear(self):
+    """Clears the internal buffer and fill it with zeros."""
+    self._buffer.fill(0)
+
+  def load_from_array(self,
+                      src: np.ndarray,
+                      offset: int = 0,
+                      size: int = -1) -> None:
+    """Loads the audio data from a NumPy array.
+
+    Args:
+      src: A NumPy source array contains the input audio.
+      offset: An optional offset for loading a slice of the `src` array to the
+        buffer.
+      size: An optional size parameter denoting the number of samples to load
+        from the `src` array.
+
+    Raises:
+      ValueError: If the input array has an incorrect shape or if
+        `offset` + `size` exceeds the length of the `src` array.
+    """
+    if len(src.shape) == 1:
+      if self._audio_format.num_channels != 1:
+        raise ValueError(f"Input audio is mono, but the audio data is expected "
+                         f"to have {self._audio_format.num_channels} channels.")
+    elif src.shape[1] != self._audio_format.num_channels:
+      raise ValueError(f"Input audio contains an invalid number of channels. "
+                       f"Expect {self._audio_format.num_channels}.")
+
+    if size < 0:
+      size = len(src)
+
+    if offset + size > len(src):
+      raise ValueError(
+          f"Index out of range. offset {offset} + size {size} should be <= "
+          f"src's length: {len(src)}")
+
+    if len(src) >= len(self._buffer):
+      # If the internal buffer is shorter than the load target (src), copy
+      # values from the end of the src array to the internal buffer.
+      new_offset = offset + size - len(self._buffer)
+      new_size = len(self._buffer)
+      self._buffer = src[new_offset:new_offset + new_size].copy()
+    else:
+      # Shift the internal buffer backward and add the incoming data to the end
+      # of the buffer.
+      shift = size
+      self._buffer = np.roll(self._buffer, -shift, axis=0)
+      self._buffer[-shift:, :] = src[offset:offset + size].copy()
+
+  @classmethod
+  def create_from_array(cls,
+                        src: np.ndarray,
+                        sample_rate: Optional[float] = None) -> "AudioData":
+    """Creates an `AudioData` object from a NumPy array.
+
+    Args:
+      src: A NumPy source array contains the input audio.
+      sample_rate: the optional audio sample rate.
+
+    Returns:
+      An `AudioData` object that contains a copy of the NumPy source array as
+      the data.
+    """
+    obj = cls(
+        buffer_length=src.shape[0],
+        audio_format=AudioDataFormat(
+            num_channels=1 if len(src.shape) == 1 else src.shape[1],
+            sample_rate=sample_rate))
+    obj.load_from_array(src)
+    return obj
+
+  @property
+  def audio_format(self) -> AudioDataFormat:
+    """Gets the audio format of the audio."""
+    return self._audio_format
+
+  @property
+  def buffer_length(self) -> int:
+    """Gets the sample count of the audio."""
+    return self._buffer.shape[0]
+
+  @property
+  def buffer(self) -> np.ndarray:
+    """Gets the internal buffer."""
+    return self._buffer

mediapipe/tasks/python/components/containers/bounding_box.py

@@ -0,0 +1,73 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Bounding box data class."""
+
+import dataclasses
+from typing import Any
+
+from mediapipe.framework.formats import location_data_pb2
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_BoundingBoxProto = location_data_pb2.LocationData.BoundingBox
+
+
+@dataclasses.dataclass
+class BoundingBox:
+  """An integer bounding box, axis aligned.
+
+  Attributes:
+    origin_x: The X coordinate of the top-left corner, in pixels.
+    origin_y: The Y coordinate of the top-left corner, in pixels.
+    width: The width of the bounding box, in pixels.
+    height: The height of the bounding box, in pixels.
+  """
+
+  origin_x: int
+  origin_y: int
+  width: int
+  height: int
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _BoundingBoxProto:
+    """Generates a BoundingBox protobuf object."""
+    return _BoundingBoxProto(
+        xmin=self.origin_x,
+        ymin=self.origin_y,
+        width=self.width,
+        height=self.height,
+    )
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls, pb2_obj: _BoundingBoxProto) -> 'BoundingBox':
+    """Creates a `BoundingBox` object from the given protobuf object."""
+    return BoundingBox(
+        origin_x=pb2_obj.xmin,
+        origin_y=pb2_obj.ymin,
+        width=pb2_obj.width,
+        height=pb2_obj.height)
+
+  def __eq__(self, other: Any) -> bool:
+    """Checks if this object is equal to the given object.
+
+    Args:
+      other: The object to be compared with.
+
+    Returns:
+      True if the objects are equal.
+    """
+    if not isinstance(other, BoundingBox):
+      return False
+
+    return self.to_pb2().__eq__(other.to_pb2())

mediapipe/tasks/python/components/containers/category.py

@@ -0,0 +1,78 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Category data class."""
+
+import dataclasses
+from typing import Any, Optional
+
+from mediapipe.framework.formats import classification_pb2
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_ClassificationProto = classification_pb2.Classification
+
+
+@dataclasses.dataclass
+class Category:
+  """A classification category.
+
+  Category is a util class, contains a label, its display name, a float
+  value as score, and the index of the label in the corresponding label file.
+  Typically it's used as the result of classification tasks.
+
+  Attributes:
+    index: The index of the label in the corresponding label file.
+    score: The probability score of this label category.
+    display_name: The display name of the label, which may be translated for
+      different locales. For example, a label, "apple", may be translated into
+      Spanish for display purpose, so that the `display_name` is "manzana".
+    category_name: The label of this category object.
+  """
+
+  index: Optional[int] = None
+  score: Optional[float] = None
+  display_name: Optional[str] = None
+  category_name: Optional[str] = None
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _ClassificationProto:
+    """Generates a Category protobuf object."""
+    return _ClassificationProto(
+        index=self.index,
+        score=self.score,
+        label=self.category_name,
+        display_name=self.display_name)
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls, pb2_obj: _ClassificationProto) -> 'Category':
+    """Creates a `Category` object from the given protobuf object."""
+    return Category(
+        index=pb2_obj.index,
+        score=pb2_obj.score,
+        display_name=pb2_obj.display_name,
+        category_name=pb2_obj.label)
+
+  def __eq__(self, other: Any) -> bool:
+    """Checks if this object is equal to the given object.
+
+    Args:
+      other: The object to be compared with.
+
+    Returns:
+      True if the objects are equal.
+    """
+    if not isinstance(other, Category):
+      return False
+
+    return self.to_pb2().__eq__(other.to_pb2())

mediapipe/tasks/python/components/containers/classification_result.py

@@ -0,0 +1,111 @@
+# Copyright 2022 The TensorFlow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Classifications data class."""
+
+import dataclasses
+from typing import List, Optional
+
+from mediapipe.framework.formats import classification_pb2
+from mediapipe.tasks.cc.components.containers.proto import classifications_pb2
+from mediapipe.tasks.python.components.containers import category as category_module
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_ClassificationProto = classification_pb2.Classification
+_ClassificationListProto = classification_pb2.ClassificationList
+_ClassificationsProto = classifications_pb2.Classifications
+_ClassificationResultProto = classifications_pb2.ClassificationResult
+
+
+@dataclasses.dataclass
+class Classifications:
+  """Represents the classification results for a given classifier head.
+
+  Attributes:
+    categories: The array of predicted categories, usually sorted by descending
+      scores (e.g. from high to low probability).
+    head_index: The index of the classifier head these categories refer to. This
+      is useful for multi-head models.
+    head_name: The name of the classifier head, which is the corresponding
+      tensor metadata name.
+  """
+
+  categories: List[category_module.Category]
+  head_index: int
+  head_name: Optional[str] = None
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _ClassificationsProto:
+    """Generates a Classifications protobuf object."""
+    classification_list_proto = _ClassificationListProto()
+    for category in self.categories:
+      classification_proto = category.to_pb2()
+      classification_list_proto.classification.append(classification_proto)
+    return _ClassificationsProto(
+        classification_list=classification_list_proto,
+        head_index=self.head_index,
+        head_name=self.head_name)
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls, pb2_obj: _ClassificationsProto) -> 'Classifications':
+    """Creates a `Classifications` object from the given protobuf object."""
+    categories = []
+    for classification in pb2_obj.classification_list.classification:
+      categories.append(
+          category_module.Category.create_from_pb2(classification))
+    return Classifications(
+        categories=categories,
+        head_index=pb2_obj.head_index,
+        head_name=pb2_obj.head_name)
+
+
+@dataclasses.dataclass
+class ClassificationResult:
+  """Contains the classification results of a model.
+
+  Attributes:
+    classifications: A list of `Classifications` objects, each for a head of the
+      model.
+    timestamp_ms: The optional timestamp (in milliseconds) of the start of the
+      chunk of data corresponding to these results. This is only used for
+      classification on time series (e.g. audio classification). In these use
+      cases, the amount of data to process might exceed the maximum size that
+      the model can process: to solve this, the input data is split into
+      multiple chunks starting at different timestamps.
+  """
+
+  classifications: List[Classifications]
+  timestamp_ms: Optional[int] = None
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _ClassificationResultProto:
+    """Generates a ClassificationResult protobuf object."""
+    return _ClassificationResultProto(
+        classifications=[
+            classification.to_pb2() for classification in self.classifications
+        ],
+        timestamp_ms=self.timestamp_ms)
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(
+      cls, pb2_obj: _ClassificationResultProto) -> 'ClassificationResult':
+    """Creates a `ClassificationResult` object from the given protobuf object.
+    """
+    return ClassificationResult(
+        classifications=[
+            Classifications.create_from_pb2(classification)
+            for classification in pb2_obj.classifications
+        ],
+        timestamp_ms=pb2_obj.timestamp_ms)

mediapipe/tasks/python/components/containers/detections.py

@@ -0,0 +1,181 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Detections data class."""
+
+import dataclasses
+from typing import Any, List, Optional
+
+from mediapipe.framework.formats import detection_pb2
+from mediapipe.framework.formats import location_data_pb2
+from mediapipe.tasks.python.components.containers import bounding_box as bounding_box_module
+from mediapipe.tasks.python.components.containers import category as category_module
+from mediapipe.tasks.python.components.containers import keypoint as keypoint_module
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_DetectionListProto = detection_pb2.DetectionList
+_DetectionProto = detection_pb2.Detection
+_LocationDataProto = location_data_pb2.LocationData
+
+
+@dataclasses.dataclass
+class Detection:
+  """Represents one detected object in the object detector's results.
+
+  Attributes:
+    bounding_box: A BoundingBox object.
+    categories: A list of Category objects.
+    keypoints: A list of NormalizedKeypoint objects.
+  """
+
+  bounding_box: bounding_box_module.BoundingBox
+  categories: List[category_module.Category]
+  keypoints: Optional[List[keypoint_module.NormalizedKeypoint]] = None
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _DetectionProto:
+    """Generates a Detection protobuf object."""
+    labels = []
+    label_ids = []
+    scores = []
+    display_names = []
+    relative_keypoints = []
+
+    for category in self.categories:
+      scores.append(category.score)
+      if category.index:
+        label_ids.append(category.index)
+      if category.category_name:
+        labels.append(category.category_name)
+      if category.display_name:
+        display_names.append(category.display_name)
+
+    if self.keypoints:
+      for keypoint in self.keypoints:
+        relative_keypoint_proto = _LocationDataProto.RelativeKeypoint()
+        if keypoint.x:
+          relative_keypoint_proto.x = keypoint.x
+        if keypoint.y:
+          relative_keypoint_proto.y = keypoint.y
+        if keypoint.label:
+          relative_keypoint_proto.keypoint_label = keypoint.label
+        if keypoint.score:
+          relative_keypoint_proto.score = keypoint.score
+        relative_keypoints.append(relative_keypoint_proto)
+
+    return _DetectionProto(
+        label=labels,
+        label_id=label_ids,
+        score=scores,
+        display_name=display_names,
+        location_data=_LocationDataProto(
+            format=_LocationDataProto.Format.BOUNDING_BOX,
+            bounding_box=self.bounding_box.to_pb2(),
+            relative_keypoints=relative_keypoints,
+        ),
+    )
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls, pb2_obj: _DetectionProto) -> 'Detection':
+    """Creates a `Detection` object from the given protobuf object."""
+    categories = []
+    keypoints = []
+
+    for idx, score in enumerate(pb2_obj.score):
+      categories.append(
+          category_module.Category(
+              score=score,
+              index=pb2_obj.label_id[idx]
+              if idx < len(pb2_obj.label_id)
+              else None,
+              category_name=pb2_obj.label[idx]
+              if idx < len(pb2_obj.label)
+              else None,
+              display_name=pb2_obj.display_name[idx]
+              if idx < len(pb2_obj.display_name)
+              else None,
+          )
+      )
+
+    if pb2_obj.location_data.relative_keypoints:
+      for idx, elem in enumerate(pb2_obj.location_data.relative_keypoints):
+        keypoints.append(
+            keypoint_module.NormalizedKeypoint(
+                x=elem.x,
+                y=elem.y,
+                label=elem.keypoint_label,
+                score=elem.score,
+            )
+        )
+
+    return Detection(
+        bounding_box=bounding_box_module.BoundingBox.create_from_pb2(
+            pb2_obj.location_data.bounding_box
+        ),
+        categories=categories,
+        keypoints=keypoints,
+    )
+
+  def __eq__(self, other: Any) -> bool:
+    """Checks if this object is equal to the given object.
+
+    Args:
+      other: The object to be compared with.
+
+    Returns:
+      True if the objects are equal.
+    """
+    if not isinstance(other, Detection):
+      return False
+
+    return self.to_pb2().__eq__(other.to_pb2())
+
+
+@dataclasses.dataclass
+class DetectionResult:
+  """Represents the list of detected objects.
+
+  Attributes:
+    detections: A list of `Detection` objects.
+  """
+
+  detections: List[Detection]
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _DetectionListProto:
+    """Generates a DetectionList protobuf object."""
+    return _DetectionListProto(
+        detection=[detection.to_pb2() for detection in self.detections])
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls, pb2_obj: _DetectionListProto) -> 'DetectionResult':
+    """Creates a `DetectionResult` object from the given protobuf object."""
+    return DetectionResult(detections=[
+        Detection.create_from_pb2(detection) for detection in pb2_obj.detection
+    ])
+
+  def __eq__(self, other: Any) -> bool:
+    """Checks if this object is equal to the given object.
+
+    Args:
+      other: The object to be compared with.
+
+    Returns:
+      True if the objects are equal.
+    """
+    if not isinstance(other, DetectionResult):
+      return False
+
+    return self.to_pb2().__eq__(other.to_pb2())

mediapipe/tasks/python/components/containers/embedding_result.py

@@ -0,0 +1,89 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Embeddings data class."""
+
+import dataclasses
+from typing import Optional, List
+
+import numpy as np
+from mediapipe.tasks.cc.components.containers.proto import embeddings_pb2
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_FloatEmbeddingProto = embeddings_pb2.FloatEmbedding
+_QuantizedEmbeddingProto = embeddings_pb2.QuantizedEmbedding
+_EmbeddingProto = embeddings_pb2.Embedding
+_EmbeddingResultProto = embeddings_pb2.EmbeddingResult
+
+
+@dataclasses.dataclass
+class Embedding:
+  """Embedding result for a given embedder head.
+
+  Attributes:
+    embedding: The actual embedding, either floating-point or scalar-quantized.
+    head_index: The index of the embedder head that produced this embedding.
+      This is useful for multi-head models.
+    head_name: The name of the embedder head, which is the corresponding tensor
+      metadata name (if any). This is useful for multi-head models.
+  """
+
+  embedding: np.ndarray
+  head_index: Optional[int] = None
+  head_name: Optional[str] = None
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls, pb2_obj: _EmbeddingProto) -> 'Embedding':
+    """Creates a `Embedding` object from the given protobuf object."""
+
+    quantized_embedding = np.array(
+        bytearray(pb2_obj.quantized_embedding.values))
+    float_embedding = np.array(pb2_obj.float_embedding.values, dtype=float)
+
+    if not pb2_obj.quantized_embedding.values:
+      return Embedding(
+          embedding=float_embedding,
+          head_index=pb2_obj.head_index,
+          head_name=pb2_obj.head_name)
+    else:
+      return Embedding(
+          embedding=quantized_embedding,
+          head_index=pb2_obj.head_index,
+          head_name=pb2_obj.head_name)
+
+
+@dataclasses.dataclass
+class EmbeddingResult:
+  """Embedding results for a given embedder model.
+
+  Attributes:
+    embeddings: A list of `Embedding` objects.
+    timestamp_ms: The optional timestamp (in milliseconds) of the start of the
+      chunk of data corresponding to these results. This is only used for
+      embedding extraction on time series (e.g. audio embedding). In these use
+      cases, the amount of data to process might exceed the maximum size that
+      the model can process: to solve this, the input data is split into
+      multiple chunks starting at different timestamps.
+  """
+
+  embeddings: List[Embedding]
+  timestamp_ms: Optional[int] = None
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls, pb2_obj: _EmbeddingResultProto) -> 'EmbeddingResult':
+    """Creates a `EmbeddingResult` object from the given protobuf object."""
+    return EmbeddingResult(embeddings=[
+        Embedding.create_from_pb2(embedding) for embedding in pb2_obj.embeddings
+    ])