mediapipe-nightly 0.10.21.post20250114__cp310-cp310-manylinux_2_28_x86_64.whl

mediapipe/tasks/python/audio/core/audio_record.py

@@ -0,0 +1,125 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A module to record audio in a streaming basis."""
+import threading
+import numpy as np
+
+try:
+  import sounddevice as sd
+except OSError as oe:
+  sd = None
+  sd_error = oe
+except ImportError as ie:
+  sd = None
+  sd_error = ie
+
+
+class AudioRecord(object):
+  """A class to record audio in a streaming basis."""
+
+  def __init__(
+      self, channels: int, sampling_rate: int, buffer_size: int
+  ) -> None:
+    """Creates an AudioRecord instance.
+
+    Args:
+      channels: Number of input channels.
+      sampling_rate: Sampling rate in Hertz.
+      buffer_size: Size of the ring buffer in number of samples.
+
+    Raises:
+      ValueError: if any of the arguments is non-positive.
+      ImportError: if failed to import `sounddevice`.
+      OSError: if failed to load `PortAudio`.
+    """
+    if sd is None:
+      raise sd_error
+
+    if channels <= 0:
+      raise ValueError('channels must be positive.')
+    if sampling_rate <= 0:
+      raise ValueError('sampling_rate must be positive.')
+    if buffer_size <= 0:
+      raise ValueError('buffer_size must be positive.')
+
+    self._audio_buffer = []
+    self._buffer_size = buffer_size
+    self._channels = channels
+    self._sampling_rate = sampling_rate
+
+    # Create a ring buffer to store the input audio.
+    self._buffer = np.zeros([buffer_size, channels], dtype=float)
+    self._lock = threading.Lock()
+
+    def audio_callback(data, *_):
+      """A callback to receive recorded audio data from sounddevice."""
+      self._lock.acquire()
+      shift = len(data)
+      if shift > buffer_size:
+        self._buffer = np.copy(data[:buffer_size])
+      else:
+        self._buffer = np.roll(self._buffer, -shift, axis=0)
+        self._buffer[-shift:, :] = np.copy(data)
+      self._lock.release()
+
+    # Create an input stream to continuously capture the audio data.
+    self._stream = sd.InputStream(
+        channels=channels,
+        samplerate=sampling_rate,
+        callback=audio_callback,
+    )
+
+  @property
+  def channels(self) -> int:
+    return self._channels
+
+  @property
+  def sampling_rate(self) -> int:
+    return self._sampling_rate
+
+  @property
+  def buffer_size(self) -> int:
+    return self._buffer_size
+
+  def start_recording(self) -> None:
+    """Starts the audio recording."""
+    # Clear the internal ring buffer.
+    self._buffer.fill(0)
+
+    # Start recording using sounddevice's InputStream.
+    self._stream.start()
+
+  def stop(self) -> None:
+    """Stops the audio recording."""
+    self._stream.stop()
+
+  def read(self, size: int) -> np.ndarray:
+    """Reads the latest audio data captured in the buffer.
+
+    Args:
+      size: Number of samples to read from the buffer.
+
+    Returns:
+      A NumPy array containing the audio data.
+
+    Raises:
+      ValueError: Raised if `size` is larger than the buffer size.
+    """
+    if size > self._buffer_size:
+      raise ValueError('Cannot read more samples than the size of the buffer.')
+    elif size <= 0:
+      raise ValueError('Size must be positive.')
+
+    start_index = self._buffer_size - size
+    return np.copy(self._buffer[start_index:])

mediapipe/tasks/python/audio/core/audio_task_running_mode.py

@@ -0,0 +1,29 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The running mode of MediaPipe Audio Tasks."""
+
+import enum
+
+
+class AudioTaskRunningMode(enum.Enum):
+  """MediaPipe audio task running mode.
+
+  Attributes:
+    AUDIO_CLIPS: The mode for running a mediapipe audio task on independent
+      audio clips.
+    AUDIO_STREAM: The mode for running a mediapipe audio task on an audio
+      stream, such as from microphone.
+  """
+  AUDIO_CLIPS = 'AUDIO_CLIPS'
+  AUDIO_STREAM = 'AUDIO_STREAM'

mediapipe/tasks/python/audio/core/base_audio_task_api.py

@@ -0,0 +1,181 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MediaPipe audio task base api."""
+
+from typing import Callable, Mapping, Optional
+
+from mediapipe.framework import calculator_pb2
+from mediapipe.python import packet_creator
+from mediapipe.python._framework_bindings import packet as packet_module
+from mediapipe.python._framework_bindings import task_runner as task_runner_module
+from mediapipe.python._framework_bindings import timestamp as timestamp_module
+from mediapipe.tasks.python.audio.core import audio_record
+from mediapipe.tasks.python.audio.core import audio_task_running_mode as running_mode_module
+from mediapipe.tasks.python.core.optional_dependencies import doc_controls
+
+_TaskRunner = task_runner_module.TaskRunner
+_Packet = packet_module.Packet
+_RunningMode = running_mode_module.AudioTaskRunningMode
+_Timestamp = timestamp_module.Timestamp
+
+
+@doc_controls.do_not_generate_docs
+class BaseAudioTaskApi(object):
+  """The base class of the user-facing mediapipe audio task api classes."""
+
+  def __init__(
+      self,
+      graph_config: calculator_pb2.CalculatorGraphConfig,
+      running_mode: _RunningMode,
+      packet_callback: Optional[Callable[[Mapping[str, packet_module.Packet]],
+                                         None]] = None
+  ) -> None:
+    """Initializes the `BaseAudioTaskApi` object.
+
+    Args:
+      graph_config: The mediapipe audio task graph config proto.
+      running_mode: The running mode of the mediapipe audio task.
+      packet_callback: The optional packet callback for getting results
+        asynchronously in the audio stream mode.
+
+    Raises:
+      ValueError: The packet callback is not properly set based on the task's
+      running mode.
+    """
+    if running_mode == _RunningMode.AUDIO_STREAM:
+      if packet_callback is None:
+        raise ValueError(
+            'The audio task is in audio stream mode, a user-defined result '
+            'callback must be provided.')
+    elif packet_callback:
+      raise ValueError(
+          'The audio task is in audio clips mode, a user-defined result '
+          'callback should not be provided.')
+    self._runner = _TaskRunner.create(graph_config, packet_callback)
+    self._running_mode = running_mode
+    self._default_sample_rate = None
+
+  def _process_audio_clip(
+      self, inputs: Mapping[str, _Packet]) -> Mapping[str, _Packet]:
+    """A synchronous method to process independent audio clips.
+
+    The call blocks the current thread until a failure status or a successful
+    result is returned.
+
+    Args:
+      inputs: A dict contains (input stream name, data packet) pairs.
+
+    Returns:
+      A dict contains (output stream name, data packet) pairs.
+
+    Raises:
+      ValueError: If the task's running mode is not set to audio clips mode.
+    """
+    if self._running_mode != _RunningMode.AUDIO_CLIPS:
+      raise ValueError(
+          'Task is not initialized with the audio clips mode. Current running'
+          ' mode:'
+          + self._running_mode.name
+      )
+    return self._runner.process(inputs)
+
+  def _set_sample_rate(
+      self, sample_rate_stream_name: str, sample_rate: float
+  ) -> None:
+    """An asynchronous method to set audio sample rate in the audio stream mode.
+
+    Args:
+      sample_rate_stream_name: The audio sample rate stream name.
+      sample_rate: The audio sample rate.
+
+    Raises:
+      ValueError: If the task's running mode is not set to the audio stream
+      mode.
+    """
+    if self._running_mode != _RunningMode.AUDIO_STREAM:
+      raise ValueError(
+          'Task is not initialized with the audio stream mode. Current running mode:'
+          + self._running_mode.name)
+    self._runner.send({
+        sample_rate_stream_name:
+            packet_creator.create_double(sample_rate).at(_Timestamp.PRESTREAM)
+    })
+
+  def _send_audio_stream_data(self, inputs: Mapping[str, _Packet]) -> None:
+    """An asynchronous method to send audio stream data to the runner.
+
+    The results will be available in the user-defined results callback.
+
+    Args:
+      inputs: A dict contains (input stream name, data packet) pairs.
+
+    Raises:
+      ValueError: If the task's running mode is not set to the audio stream
+      mode.
+    """
+    if self._running_mode != _RunningMode.AUDIO_STREAM:
+      raise ValueError(
+          'Task is not initialized with the audio stream mode. Current running'
+          ' mode:'
+          + self._running_mode.name
+      )
+    self._runner.send(inputs)
+
+  def create_audio_record(
+      self, num_channels: int, sample_rate: int, required_input_buffer_size: int
+  ) -> audio_record.AudioRecord:
+    """Creates an AudioRecord instance to record audio stream.
+
+    The returned AudioRecord instance is initialized and client needs to call
+    the appropriate method to start recording.
+
+    Note that MediaPipe Audio tasks will up/down sample automatically to fit the
+    sample rate required by the model. The default sample rate of the MediaPipe
+    pretrained audio model, Yamnet is 16kHz.
+
+    Args:
+      num_channels: The number of audio channels.
+      sample_rate: The audio sample rate.
+      required_input_buffer_size: The required input buffer size in number of
+        float elements.
+
+    Returns:
+      An AudioRecord instance.
+
+    Raises:
+      ValueError: If there's a problem creating the AudioRecord instance.
+    """
+    return audio_record.AudioRecord(
+        num_channels, sample_rate, required_input_buffer_size
+    )
+
+  def close(self) -> None:
+    """Shuts down the mediapipe audio task instance.
+
+    Raises:
+      RuntimeError: If the mediapipe audio task failed to close.
+    """
+    self._runner.close()
+
+  def __enter__(self):
+    """Return `self` upon entering the runtime context."""
+    return self
+
+  def __exit__(self, unused_exc_type, unused_exc_value, unused_traceback):
+    """Shuts down the mediapipe audio task instance on exit of the context manager.
+
+    Raises:
+      RuntimeError: If the mediapipe audio task failed to close.
+    """
+    self.close()

mediapipe/tasks/python/benchmark/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mediapipe/tasks/python/benchmark/benchmark_utils.py

@@ -0,0 +1,70 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Benchmark utils for MediaPipe Tasks."""
+
+import os
+import numpy as np
+
+
+def nth_percentile(inference_times, percentile):
+  """Calculate the nth percentile of the inference times."""
+  return np.percentile(inference_times, percentile)
+
+
+def average(inference_times):
+  """Calculate the average of the inference times."""
+  return np.mean(inference_times)
+
+
+def get_test_data_path(test_srcdir, file_or_dirname_path: str) -> str:
+  """Determine the test data path.
+
+  Args:
+      test_srcdir: The path to the test source directory.
+      file_or_dirname_path: The path to the file or directory.
+
+  Returns:
+      The full test data path.
+  """
+  for directory, subdirs, files in os.walk(test_srcdir):
+    for f in subdirs + files:
+      path = os.path.join(directory, f)
+      if path.endswith(file_or_dirname_path):
+        return path
+  raise ValueError(
+      "No %s in test directory: %s." % (file_or_dirname_path, test_srcdir)
+  )
+
+
+def get_model_path(custom_model, default_model_path):
+  """Determine the model path based on the existence of the custom model.
+
+  Args:
+      custom_model: The path to the custom model provided by the user.
+      default_model_path: The path to the default model.
+
+  Returns:
+      The path to the model to be used.
+  """
+  if custom_model is not None and os.path.exists(custom_model):
+    print(f"Using provided model: {custom_model}")
+    return custom_model
+  else:
+    if custom_model is not None:
+      print(
+          f"Warning: Provided model '{custom_model}' not found. "
+          "Using default model instead."
+      )
+    print(f"Using default model: {default_model_path}")
+    return default_model_path

mediapipe/tasks/python/benchmark/vision/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mediapipe/tasks/python/benchmark/vision/benchmark.py

@@ -0,0 +1,99 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MediaPipe vision benchmarker."""
+
+import argparse
+
+from mediapipe.tasks.python.benchmark import benchmark_utils as bu
+from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api
+from mediapipe.tasks.python.core import base_options
+
+
+def benchmarker(benchmark_function, default_model_name):
+  """Executes a benchmarking process using a specified function ann model.
+
+  Args:
+      benchmark_function: A callable function to be executed for benchmarking.
+        This function should contain the logic of the task to be benchmarked and
+        should be capable of utilizing a model specified by its name.
+      default_model_name: The name or path of the default model to be used in
+        the benchmarking process. This is useful when the benchmarking function
+        requires a model and no other model is explicitly specified.
+  """
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter
+  )
+
+  parser.add_argument(
+      '--mode',
+      help='Benchmarking mode (e.g., "nth_percentile").',
+      required=False,
+      default='nth_percentile',
+  )
+  parser.add_argument('--model', help='Path to the model.', default=None)
+  parser.add_argument(
+      '--iterations',
+      help='Number of iterations for benchmarking.',
+      type=int,
+      default=100,
+  )
+  parser.add_argument(
+      '--percentile',
+      help='Percentile for benchmarking statistics.',
+      type=float,
+      default=95.0,
+  )
+
+  args = parser.parse_args()
+
+  # Get the model path
+  default_model_path = bu.get_test_data_path(
+      base_vision_benchmark_api.VISION_TEST_DATA_DIR, default_model_name
+  )
+  model_path = bu.get_model_path(args.model, default_model_path)
+
+  # Define a mapping of modes to their respective function argument lists
+  mode_args_mapping = {
+      'nth_percentile': {'percentile': args.percentile},
+      'average': {},
+  }
+
+  # Check if the mode is supported and get the argument dictionary
+  if args.mode not in mode_args_mapping:
+    raise ValueError(f'Unsupported benchmarking mode: {args.mode}')
+
+  mode_args = mode_args_mapping[args.mode]
+
+  # Run the benchmark for both CPU and GPU and calculate results based on mode
+  results = {}
+  for delegate_type in [
+      base_options.BaseOptions.Delegate.CPU,
+      base_options.BaseOptions.Delegate.GPU,
+  ]:
+    inference_times = benchmark_function(
+        model_path, args.iterations, delegate_type
+    )
+
+    # Calculate the benchmark result based on the mode
+    if args.mode == 'nth_percentile':
+      results[delegate_type] = bu.nth_percentile(inference_times, **mode_args)
+    elif args.mode == 'average':
+      results[delegate_type] = bu.average(inference_times)
+
+  # Report benchmarking results
+  for delegate_type, result in results.items():
+    print(
+        f'Inference time {delegate_type} {mode_args_mapping[args.mode]}: '
+        f'{result:.6f} milliseconds'
+    )

mediapipe/tasks/python/benchmark/vision/core/__init__.py

@@ -0,0 +1,14 @@
+"""Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

mediapipe/tasks/python/benchmark/vision/core/base_vision_benchmark_api.py

@@ -0,0 +1,40 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MediaPipe vision benchmark base api."""
+import time
+
+VISION_TEST_DATA_DIR = 'mediapipe/tasks/testdata/vision'
+
+
+def benchmark_task(func, image, n_iterations):
+  """Collect inference times for a given task after benchmarking.
+
+  Args:
+      func: The task function used for benchmarking.
+      image: The input MediaPipe Image.
+      n_iterations: Number of iterations to run the benchmark.
+
+  Returns:
+      List of inference times in milliseconds.
+  """
+  inference_times = []
+
+  for _ in range(n_iterations):
+    start_time_ns = time.time_ns()
+    # Run the method for the task (e.g., classify)
+    func(image)
+    end_time_ns = time.time_ns()
+    inference_times.append((end_time_ns - start_time_ns) / 1_000_000)
+
+  return inference_times

mediapipe/tasks/python/components/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mediapipe/tasks/python/components/containers/__init__.py

@@ -0,0 +1,53 @@
+# Copyright 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""MediaPipe Tasks Components Containers API."""
+
+import mediapipe.tasks.python.components.containers.audio_data
+import mediapipe.tasks.python.components.containers.bounding_box
+import mediapipe.tasks.python.components.containers.category
+import mediapipe.tasks.python.components.containers.classification_result
+import mediapipe.tasks.python.components.containers.detections
+import mediapipe.tasks.python.components.containers.embedding_result
+import mediapipe.tasks.python.components.containers.landmark
+import mediapipe.tasks.python.components.containers.landmark_detection_result
+import mediapipe.tasks.python.components.containers.rect
+
+AudioDataFormat = audio_data.AudioDataFormat
+AudioData = audio_data.AudioData
+BoundingBox = bounding_box.BoundingBox
+Category = category.Category
+Classifications = classification_result.Classifications
+ClassificationResult = classification_result.ClassificationResult
+Detection = detections.Detection
+DetectionResult = detections.DetectionResult
+Embedding = embedding_result.Embedding
+EmbeddingResult = embedding_result.EmbeddingResult
+Landmark = landmark.Landmark
+NormalizedLandmark = landmark.NormalizedLandmark
+LandmarksDetectionResult = landmark_detection_result.LandmarksDetectionResult
+Rect = rect.Rect
+NormalizedRect = rect.NormalizedRect
+
+# Remove unnecessary modules to avoid duplication in API docs.
+del audio_data
+del bounding_box
+del category
+del classification_result
+del detections
+del embedding_result
+del landmark
+del landmark_detection_result
+del rect
+del mediapipe