mediapipe-nightly 0.10.21.post20250114__cp311-cp311-manylinux_2_28_x86_64.whl

mediapipe/tasks/python/components/containers/keypoint.py

@@ -0,0 +1,77 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Keypoint data class."""
+
+import dataclasses
+from typing import Any, Optional
+
+from mediapipe.framework.formats import location_data_pb2
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_RelativeKeypointProto = location_data_pb2.LocationData.RelativeKeypoint
+
+
+@dataclasses.dataclass
+class NormalizedKeypoint:
+  """A normalized keypoint.
+
+  Normalized keypoint represents a point in 2D space with x, y coordinates.
+  x and y are normalized to [0.0, 1.0] by the image width and height
+  respectively.
+
+  Attributes:
+    x: The x coordinates of the normalized keypoint.
+    y: The y coordinates of the normalized keypoint.
+    label: The optional label of the keypoint.
+    score: The score of the keypoint.
+  """
+
+  x: Optional[float] = None
+  y: Optional[float] = None
+  label: Optional[str] = None
+  score: Optional[float] = None
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _RelativeKeypointProto:
+    """Generates a RelativeKeypoint protobuf object."""
+    return _RelativeKeypointProto(
+        x=self.x, y=self.y, keypoint_label=self.label, score=self.score
+    )
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(
+      cls, pb2_obj: _RelativeKeypointProto
+  ) -> 'NormalizedKeypoint':
+    """Creates a `NormalizedKeypoint` object from the given protobuf object."""
+    return NormalizedKeypoint(
+        x=pb2_obj.x,
+        y=pb2_obj.y,
+        label=pb2_obj.keypoint_label,
+        score=pb2_obj.score,
+    )
+
+  def __eq__(self, other: Any) -> bool:
+    """Checks if this object is equal to the given object.
+
+    Args:
+      other: The object to be compared with.
+
+    Returns:
+      True if the objects are equal.
+    """
+    if not isinstance(other, NormalizedKeypoint):
+      return False
+
+    return self.to_pb2().__eq__(other.to_pb2())

mediapipe/tasks/python/components/containers/landmark.py

@@ -0,0 +1,122 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Landmark data class."""
+
+import dataclasses
+from typing import Optional
+
+from mediapipe.framework.formats import landmark_pb2
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_LandmarkProto = landmark_pb2.Landmark
+_NormalizedLandmarkProto = landmark_pb2.NormalizedLandmark
+
+
+@dataclasses.dataclass
+class Landmark:
+  """A landmark that can have 1 to 3 dimensions.
+
+  Use x for 1D points, (x, y) for 2D points and (x, y, z) for 3D points.
+
+  Attributes:
+    x: The x coordinate.
+    y: The y coordinate.
+    z: The z coordinate.
+    visibility: Landmark visibility. Should stay unset if not supported. Float
+      score of whether landmark is visible or occluded by other objects.
+      Landmark considered as invisible also if it is not present on the screen
+      (out of scene bounds). Depending on the model, visibility value is either
+      a sigmoid or an argument of sigmoid.
+    presence: Landmark presence. Should stay unset if not supported. Float score
+      of whether landmark is present on the scene (located within scene bounds).
+      Depending on the model, presence value is either a result of sigmoid or an
+      argument of sigmoid function to get landmark presence probability.
+  """
+
+  x: Optional[float] = None
+  y: Optional[float] = None
+  z: Optional[float] = None
+  visibility: Optional[float] = None
+  presence: Optional[float] = None
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _LandmarkProto:
+    """Generates a Landmark protobuf object."""
+    return _LandmarkProto(
+        x=self.x,
+        y=self.y,
+        z=self.z,
+        visibility=self.visibility,
+        presence=self.presence)
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls, pb2_obj: _LandmarkProto) -> 'Landmark':
+    """Creates a `Landmark` object from the given protobuf object."""
+    return Landmark(
+        x=pb2_obj.x,
+        y=pb2_obj.y,
+        z=pb2_obj.z,
+        visibility=pb2_obj.visibility,
+        presence=pb2_obj.presence)
+
+
+@dataclasses.dataclass
+class NormalizedLandmark:
+  """A normalized version of above Landmark proto.
+
+  All coordinates should be within [0, 1].
+
+  Attributes:
+    x: The normalized x coordinate.
+    y: The normalized y coordinate.
+    z: The normalized z coordinate.
+    visibility: Landmark visibility. Should stay unset if not supported. Float
+      score of whether landmark is visible or occluded by other objects.
+      Landmark considered as invisible also if it is not present on the screen
+      (out of scene bounds). Depending on the model, visibility value is either
+      a sigmoid or an argument of sigmoid.
+    presence: Landmark presence. Should stay unset if not supported. Float score
+      of whether landmark is present on the scene (located within scene bounds).
+      Depending on the model, presence value is either a result of sigmoid or an
+      argument of sigmoid function to get landmark presence probability.
+  """
+
+  x: Optional[float] = None
+  y: Optional[float] = None
+  z: Optional[float] = None
+  visibility: Optional[float] = None
+  presence: Optional[float] = None
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _NormalizedLandmarkProto:
+    """Generates a NormalizedLandmark protobuf object."""
+    return _NormalizedLandmarkProto(
+        x=self.x,
+        y=self.y,
+        z=self.z,
+        visibility=self.visibility,
+        presence=self.presence)
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(
+      cls, pb2_obj: _NormalizedLandmarkProto) -> 'NormalizedLandmark':
+    """Creates a `NormalizedLandmark` object from the given protobuf object."""
+    return NormalizedLandmark(
+        x=pb2_obj.x,
+        y=pb2_obj.y,
+        z=pb2_obj.z,
+        visibility=pb2_obj.visibility,
+        presence=pb2_obj.presence)

mediapipe/tasks/python/components/containers/landmark_detection_result.py

@@ -0,0 +1,106 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Landmarks Detection Result data class."""
+
+import dataclasses
+from typing import Optional, List
+
+from mediapipe.framework.formats import classification_pb2
+from mediapipe.framework.formats import landmark_pb2
+from mediapipe.tasks.cc.components.containers.proto import landmarks_detection_result_pb2
+from mediapipe.tasks.python.components.containers import category as category_module
+from mediapipe.tasks.python.components.containers import landmark as landmark_module
+from mediapipe.tasks.python.components.containers import rect as rect_module
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_LandmarksDetectionResultProto = landmarks_detection_result_pb2.LandmarksDetectionResult
+_ClassificationProto = classification_pb2.Classification
+_ClassificationListProto = classification_pb2.ClassificationList
+_LandmarkListProto = landmark_pb2.LandmarkList
+_NormalizedLandmarkListProto = landmark_pb2.NormalizedLandmarkList
+_NormalizedRect = rect_module.NormalizedRect
+_Category = category_module.Category
+_NormalizedLandmark = landmark_module.NormalizedLandmark
+_Landmark = landmark_module.Landmark
+
+
+@dataclasses.dataclass
+class LandmarksDetectionResult:
+  """Represents the landmarks detection result.
+
+  Attributes:
+    landmarks: A list of `NormalizedLandmark` objects.
+    categories: A list of `Category` objects.
+    world_landmarks: A list of `Landmark` objects.
+    rect: A `NormalizedRect` object.
+  """
+
+  landmarks: Optional[List[_NormalizedLandmark]]
+  categories: Optional[List[_Category]]
+  world_landmarks: Optional[List[_Landmark]]
+  rect: _NormalizedRect
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _LandmarksDetectionResultProto:
+    """Generates a LandmarksDetectionResult protobuf object."""
+
+    landmarks = _NormalizedLandmarkListProto()
+    classifications = _ClassificationListProto()
+    world_landmarks = _LandmarkListProto()
+
+    for landmark in self.landmarks:
+      landmarks.landmark.append(landmark.to_pb2())
+
+    for category in self.categories:
+      classifications.classification.append(
+          _ClassificationProto(
+              index=category.index,
+              score=category.score,
+              label=category.category_name,
+              display_name=category.display_name))
+
+    return _LandmarksDetectionResultProto(
+        landmarks=landmarks,
+        classifications=classifications,
+        world_landmarks=world_landmarks,
+        rect=self.rect.to_pb2())
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(
+      cls,
+      pb2_obj: _LandmarksDetectionResultProto) -> 'LandmarksDetectionResult':
+    """Creates a `LandmarksDetectionResult` object from the given protobuf object."""
+    categories = []
+    landmarks = []
+    world_landmarks = []
+
+    for classification in pb2_obj.classifications.classification:
+      categories.append(
+          category_module.Category(
+              score=classification.score,
+              index=classification.index,
+              category_name=classification.label,
+              display_name=classification.display_name))
+
+    for landmark in pb2_obj.landmarks.landmark:
+      landmarks.append(_NormalizedLandmark.create_from_pb2(landmark))
+
+    for landmark in pb2_obj.world_landmarks.landmark:
+      world_landmarks.append(_Landmark.create_from_pb2(landmark))
+    return LandmarksDetectionResult(
+        landmarks=landmarks,
+        categories=categories,
+        world_landmarks=world_landmarks,
+        rect=_NormalizedRect.create_from_pb2(pb2_obj.rect))

mediapipe/tasks/python/components/containers/rect.py

@@ -0,0 +1,109 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Rect data class."""
+
+import dataclasses
+from typing import Any, Optional
+
+from mediapipe.framework.formats import rect_pb2
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_NormalizedRectProto = rect_pb2.NormalizedRect
+
+
+@dataclasses.dataclass
+class Rect:
+  """A rectangle, used as part of detection results or as input region-of-interest.
+
+  The coordinates are normalized wrt the image dimensions, i.e. generally in
+  [0,1] but they may exceed these bounds if describing a region overlapping the
+  image. The origin is on the top-left corner of the image.
+
+  Attributes:
+    left: The X coordinate of the left side of the rectangle.
+    top: The Y coordinate of the top of the rectangle.
+    right: The X coordinate of the right side of the rectangle.
+    bottom: The Y coordinate of the bottom of the rectangle.
+  """
+
+  left: float
+  top: float
+  right: float
+  bottom: float
+
+
+@dataclasses.dataclass
+class NormalizedRect:
+  """A rectangle with rotation in normalized coordinates.
+
+  Location of the center of the rectangle in image coordinates. The (0.0, 0.0)
+  point is at the (top, left) corner.
+
+  The values of box center location and size are within [0, 1].
+
+  Attributes:
+    x_center: The normalized X coordinate of the rectangle, in image
+      coordinates.
+    y_center: The normalized Y coordinate of the rectangle, in image
+      coordinates.
+    width: The width of the rectangle.
+    height: The height of the rectangle.
+    rotation: Rotation angle is clockwise in radians.
+    rect_id: Optional unique id to help associate different rectangles to each
+      other.
+  """
+
+  x_center: float
+  y_center: float
+  width: float
+  height: float
+  rotation: Optional[float] = 0.0
+  rect_id: Optional[int] = None
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _NormalizedRectProto:
+    """Generates a NormalizedRect protobuf object."""
+    return _NormalizedRectProto(
+        x_center=self.x_center,
+        y_center=self.y_center,
+        width=self.width,
+        height=self.height,
+        rotation=self.rotation,
+        rect_id=self.rect_id)
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls, pb2_obj: _NormalizedRectProto) -> 'NormalizedRect':
+    """Creates a `NormalizedRect` object from the given protobuf object."""
+    return NormalizedRect(
+        x_center=pb2_obj.x_center,
+        y_center=pb2_obj.y_center,
+        width=pb2_obj.width,
+        height=pb2_obj.height,
+        rotation=pb2_obj.rotation,
+        rect_id=pb2_obj.rect_id)
+
+  def __eq__(self, other: Any) -> bool:
+    """Checks if this object is equal to the given object.
+
+    Args:
+      other: The object to be compared with.
+
+    Returns:
+      True if the objects are equal.
+    """
+    if not isinstance(other, NormalizedRect):
+      return False
+
+    return self.to_pb2().__eq__(other.to_pb2())

mediapipe/tasks/python/components/processors/__init__.py

@@ -0,0 +1,23 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""MediaPipe Tasks Components Processors API."""
+
+import mediapipe.tasks.python.components.processors.classifier_options
+
+ClassifierOptions = classifier_options.ClassifierOptions
+
+# Remove unnecessary modules to avoid duplication in API docs.
+del classifier_options
+del mediapipe

mediapipe/tasks/python/components/processors/classifier_options.py

@@ -0,0 +1,86 @@
+# Copyright 2022 The TensorFlow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Classifier options data class."""
+
+import dataclasses
+from typing import Any, List, Optional
+
+from mediapipe.tasks.cc.components.processors.proto import classifier_options_pb2
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_ClassifierOptionsProto = classifier_options_pb2.ClassifierOptions
+
+
+@dataclasses.dataclass
+class ClassifierOptions:
+  """Options for classification processor.
+
+  Attributes:
+    display_names_locale: The locale to use for display names specified through
+      the TFLite Model Metadata.
+    max_results: The maximum number of top-scored classification results to
+      return.
+    score_threshold: Overrides the ones provided in the model metadata. Results
+      below this value are rejected.
+    category_allowlist: Allowlist of category names. If non-empty, detection
+      results whose category name is not in this set will be filtered out.
+      Duplicate or unknown category names are ignored. Mutually exclusive with
+      `category_denylist`.
+    category_denylist: Denylist of category names. If non-empty, detection
+      results whose category name is in this set will be filtered out. Duplicate
+      or unknown category names are ignored. Mutually exclusive with
+      `category_allowlist`.
+  """
+
+  display_names_locale: Optional[str] = None
+  max_results: Optional[int] = None
+  score_threshold: Optional[float] = None
+  category_allowlist: Optional[List[str]] = None
+  category_denylist: Optional[List[str]] = None
+
+  @doc_controls.do_not_generate_docs
+  def to_pb2(self) -> _ClassifierOptionsProto:
+    """Generates a ClassifierOptions protobuf object."""
+    return _ClassifierOptionsProto(
+        score_threshold=self.score_threshold,
+        category_allowlist=self.category_allowlist,
+        category_denylist=self.category_denylist,
+        display_names_locale=self.display_names_locale,
+        max_results=self.max_results)
+
+  @classmethod
+  @doc_controls.do_not_generate_docs
+  def create_from_pb2(cls,
+                      pb2_obj: _ClassifierOptionsProto) -> 'ClassifierOptions':
+    """Creates a `ClassifierOptions` object from the given protobuf object."""
+    return ClassifierOptions(
+        score_threshold=pb2_obj.score_threshold,
+        category_allowlist=[str(name) for name in pb2_obj.category_allowlist],
+        category_denylist=[str(name) for name in pb2_obj.category_denylist],
+        display_names_locale=pb2_obj.display_names_locale,
+        max_results=pb2_obj.max_results)
+
+  def __eq__(self, other: Any) -> bool:
+    """Checks if this object is equal to the given object.
+
+    Args:
+      other: The object to be compared with.
+
+    Returns:
+      True if the objects are equal.
+    """
+    if not isinstance(other, ClassifierOptions):
+      return False
+
+    return self.to_pb2().__eq__(other.to_pb2())

mediapipe/tasks/python/components/utils/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mediapipe/tasks/python/components/utils/cosine_similarity.py

@@ -0,0 +1,68 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Cosine similarity utilities."""
+
+import numpy as np
+
+from mediapipe.tasks.python.components.containers import embedding_result
+
+_Embedding = embedding_result.Embedding
+
+
+def _compute_cosine_similarity(u: np.ndarray, v: np.ndarray):
+  """Computes cosine similarity between two embeddings."""
+
+  if len(u) <= 0:
+    raise ValueError("Cannot compute cosing similarity on empty embeddings.")
+
+  norm_u = np.linalg.norm(u)
+  norm_v = np.linalg.norm(v)
+
+  if norm_u <= 0 or norm_v <= 0:
+    raise ValueError(
+        "Cannot compute cosine similarity on embedding with 0 norm.")
+
+  return u.dot(v) / (norm_u * norm_v)
+
+
+def cosine_similarity(u: _Embedding, v: _Embedding) -> float:
+  """Utility function to compute cosine similarity between two embedding.
+
+  May return an InvalidArgumentError if e.g. the feature vectors are of
+  different types (quantized vs. float), have different sizes, or have an
+  L2-norm of 0.
+
+  Args:
+    u: An embedding.
+    v: An embedding.
+
+  Returns:
+    Cosine similarity value.
+  """
+  if len(u.embedding) != len(v.embedding):
+    raise ValueError(f"Cannot compute cosine similarity between embeddings "
+                     f"of different sizes "
+                     f"({len(u.embedding)} vs. {len(v.embedding)}).")
+
+  if u.embedding.dtype == float and v.embedding.dtype == float:
+    return _compute_cosine_similarity(u.embedding, v.embedding)
+
+  if u.embedding.dtype == np.uint8 and v.embedding.dtype == np.uint8:
+    return _compute_cosine_similarity(
+        u.embedding.view("int8").astype("float"),
+        v.embedding.view("int8").astype("float"),
+    )
+
+  raise ValueError("Cannot compute cosine similarity between quantized and "
+                   "float embeddings.")

mediapipe/tasks/python/core/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.