huri-module-emo 0.1.0__py3-none-any.whl

huri_emo/aggregator.py

@@ -0,0 +1,75 @@
+from collections import defaultdict
+from typing import Dict, Optional
+
+from huri_events.emotion import Emotion
+from huri_events.question import PartialQuestion
+
+from huri_core.core.module import Module
+
+
+class EAG(Module):
+    """EAG Module
+
+    Aggregate all emotions and send when voice end.
+
+    input: huri.emotion,
+    output: huri.partial_question
+
+    :ema_alpha: if not None, the aggragation will use ema computation instead
+    of average. Recents emotion will have stronger impact on the final score.
+    Lower alpha will make impact lower, and higher alpha will make it higher. \
+    Default alpha would be ~0.3.
+    """
+
+    input_type = "huri.emotion"
+    output_type = "huri.partial_question"
+    input_event = Emotion
+    output_event = PartialQuestion
+
+    def __init__(self, ema_alpha: Optional[float] = None):
+        super().__init__()
+
+        self.scores: Dict[str, float] = defaultdict(float)
+        self.count: int = 0
+
+        self.ema_alpha = ema_alpha
+
+    def _finalize(self) -> Emotion:
+        avg_scores = (
+            {label: score / self.count for label, score in self.scores.items()}
+            if self.ema_alpha is None
+            else self.scores
+        )
+
+        best_label = max(avg_scores, key=lambda label: avg_scores[label])
+
+        result = Emotion(
+            label=best_label,
+            confidence=avg_scores[best_label],
+            scores=avg_scores,
+            end=True,
+        )
+
+        self.scores.clear()
+        self.count = 0
+
+        return result
+
+    async def process(self, emotion: Emotion) -> Optional[PartialQuestion]:
+        if self.ema_alpha is not None:
+            for label, score in emotion.scores.items():
+                self.scores[label] = (
+                    self.ema_alpha * score + (1 - self.ema_alpha) * self.scores[label]
+                )
+        else:
+            for label, score in emotion.scores.items():
+                self.scores[label] += score
+
+        self.count += 1
+
+        if emotion.end:
+            emotion = self._finalize()
+
+            return PartialQuestion(transcript=None, emotion=emotion)
+
+        return None

huri_emo/manifests/eag/huri_module.toml

@@ -0,0 +1,7 @@
+name = "eag"
+summary = "Aggregates emotion estimates over an utterance"
+input = { topic = "huri.emotion", event = "huri_events.emotion:Emotion" }
+output = { topic = "huri.partial_question", event = "huri_events.question:PartialQuestion" }
+events = ["huri-events>=0.1,<0.2"]
+requires = { gpu = false, actuation = false }
+huri_core = ">=0.4,<0.5"

huri_emo/manifests/emo/huri_module.toml

@@ -0,0 +1,7 @@
+name = "emo"
+summary = "Estimates the emotion of speech from its prosody"
+input = { topic = "huri.voice", event = "huri_events.speech:Voice" }
+output = { topic = "huri.emotion", event = "huri_events.emotion:Emotion" }
+events = ["huri-events>=0.1,<0.2"]
+requires = { gpu = false, actuation = false }
+huri_core = ">=0.4,<0.5"

huri_emo/module.py

@@ -0,0 +1,155 @@
+import asyncio
+import os
+from typing import Any, AsyncGenerator, Dict, List
+
+import numpy as np
+from huri_events.emotion import Emotion
+from huri_events.speech import Voice
+from ray import serve
+from ray.serve import handle
+
+from huri_core.core.module import ModuleWithHandle
+
+_MODEL_NAME = os.environ.get("HURI_EMO_MODEL", "superb/hubert-large-superb-er")
+
+
+@serve.deployment(name="EMOHandle", max_ongoing_requests=8)
+class EMOHandle:
+    """Prosody emotion model.
+
+    :model_name: name of the Emotion Analysis model.
+    :sample_rate: sample rate of the audio passed to predict().
+    """
+
+    def __init__(
+        self,
+        model_name: str = _MODEL_NAME,
+        sample_rate: int = 16000,
+    ):
+        from transformers import (
+            AutoModelForAudioClassification,
+            Wav2Vec2FeatureExtractor,
+        )
+
+        self.model = AutoModelForAudioClassification.from_pretrained(model_name)
+        self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_name)
+        self.sample_rate = sample_rate
+
+    async def predict(self, audio: np.ndarray) -> Dict[str, Any]:
+        # Blocking torch inference runs off the replica's event loop.
+        return await asyncio.to_thread(self._predict_sync, audio)
+
+    def _predict_sync(self, audio_np: np.ndarray) -> Dict[str, Any]:
+        import torch
+
+        inputs = self.feature_extractor(
+            audio_np, sampling_rate=self.sample_rate, return_tensors="pt", padding=True
+        )
+
+        with torch.no_grad():
+            logits = self.model(**inputs).logits
+            probs = torch.softmax(logits, dim=-1)[0]
+
+        predicted_id = int(torch.argmax(probs).item())
+
+        labels = self.model.config.id2label
+
+        return {
+            "label": labels[predicted_id],
+            "confidence": float(probs[predicted_id]),
+            "scores": {labels[i]: float(probs[i]) for i in range(len(labels))},
+        }
+
+
+class EMO(ModuleWithHandle):
+    """EMO Module
+
+    Prosody Analysis of user voice speech.
+
+    input: huri.voice,
+    output: huri.emotion
+
+    :sample_rate: size of received voice audio. Usually 8000, 16000 or 48000.
+    :block_duration: size of received voice audio (in s).
+    :analysis_window: duration of audio per analysis (in s).
+    """
+
+    _handle_cls = EMOHandle
+    input_type = "huri.voice"
+    output_type = "huri.emotion"
+    input_event = Voice
+    output_event = Emotion
+
+    def __init__(
+        self,
+        _handle: handle.DeploymentHandle,
+        sample_rate: int = 16000,
+        block_duration: float = 0.020,  # s
+        analysis_window: float = 4.0,  # s
+        **kwargs,
+    ):
+        super().__init__(_handle=_handle, **kwargs)
+
+        self.sample_rate = sample_rate
+        self.window_size = int(analysis_window / block_duration)
+
+        self.buffer: List[np.ndarray] = []
+
+        self.running = False
+        self.lock: asyncio.Lock = asyncio.Lock()
+
+        # Set when the end of the voice is received. Remembered if an analysis
+        # is in flight, so that call handles it once it finishes.
+        self._end_requested: bool = False
+
+    async def process(self, voice: Voice) -> AsyncGenerator[Emotion, None]:
+        async with self.lock:
+            if voice.data is None:
+                self._end_requested = True
+            else:
+                self.buffer.append(voice.data)
+            if self.running:
+                return
+            self.running = True
+
+        try:
+            async for emotion in self._analyze_pending():
+                yield emotion
+        finally:
+            self.running = False
+
+    async def _analyze_pending(self) -> AsyncGenerator[Emotion, None]:
+        """Run analyses until there is nothing left to do.
+
+        Loops so that an end-of-turn that arrives while a window is being
+        analyzed is handled right after it, in this same call. Each pass is
+        yielded as its own Emotion.
+        """
+
+        while True:
+            async with self.lock:
+                end = self._end_requested
+                if end:
+                    self._end_requested = False
+                    chunks = list(self.buffer)
+                else:
+                    if len(self.buffer) < self.window_size:
+                        return
+                    chunks = self.buffer[: self.window_size]
+
+            # On end, analyze the remaining buffer; fall back to a short zero
+            # buffer when it is empty so the model still gets valid input.
+            audio = (
+                np.concatenate(chunks, axis=0)
+                if chunks
+                else np.zeros(self.sample_rate // 10, dtype=np.float32)
+            )
+            result = await self._handle.predict.remote(audio)
+
+            async with self.lock:
+                # Frames received during the await are kept for the next pass.
+                self.buffer = self.buffer[len(chunks) :]
+
+            yield Emotion(result["label"], result["confidence"], result["scores"], end)
+            if not self._end_requested:
+                return

huri_module_emo-0.1.0.dist-info/METADATA

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: huri-module-emo
+Version: 0.1.0
+Summary: HuRI module: Speech emotion from prosody (wav2vec2 classifier), and its aggregator
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: huri-core[server]<0.5,>=0.4
+Requires-Dist: huri-events<0.2,>=0.1
+Requires-Dist: numpy<2,>=1.26
+Provides-Extra: model
+Requires-Dist: torch; extra == "model"
+Requires-Dist: transformers>=4.46; extra == "model"
+
+# huri-module-emo
+
+Speech emotion from prosody (wav2vec2 classifier), and its aggregator.
+
+| Module | Topics | What it does |
+|---|---|---|
+| `emo` | `huri.voice` → `huri.emotion` | Estimates the emotion of speech from its prosody |
+| `eag` | `huri.emotion` → `huri.partial_question` | Aggregates emotion estimates over an utterance |
+
+```sh
+pip install -e 'packages/huri-module-emo[model]'   # from a huri-modules checkout, next to huri-core and huri-events
+HURI_MODULES=emo,eag ...   # enable it on the server
+```
+
+`emo` needs the `model` extra (torch and transformers, for its wav2vec2
+classifier); `eag` only aggregates events and needs none of it, so a host that
+runs `eag` alone installs the package without the extra
+(`[tool.huri-modules.extras]` in `pyproject.toml` says which module needs which
+extra; `huri hub install` and `scripts/install_local.sh` add it). Hardware pins
+of `emo`: `requirements-<variant>.emo.txt` (`cpu`, `nvidia`, `amd`; install them
+before the package).
+
+A distribution of
+[huri-modules](https://github.com/Lucy-Robotics/huri-modules), versioned in its
+`pyproject.toml` (see RELEASING.md there). It depends on `huri-core` and
+`huri-events`, which are not on PyPI: without a checkout, install them from git
+in the same pip command, so pip never looks them up on an index:
+
+```sh
+ref=0.1.0   # a huri-modules release tag
+core=0.4.0  # HURI_CORE_REF in that release's versions.env
+pip install \
+  "huri-core[server] @ git+https://github.com/Lucy-Robotics/huri-core.git@$core" \
+  "huri-events @ git+https://github.com/Lucy-Robotics/huri-modules.git@$ref#subdirectory=packages/huri-events" \
+  "huri-module-emo[model] @ git+https://github.com/Lucy-Robotics/huri-modules.git@$ref#subdirectory=packages/huri-module-emo"
+```

huri_module_emo-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+huri_emo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+huri_emo/aggregator.py,sha256=0migoA8J-4LIJ4sjOixWxs4E5ytdAK37it4ksJ7OrdA,2123
+huri_emo/module.py,sha256=UEFkGKa8dJtFp5CPqVOWsp9Ey94Pgoni8AynuhXiOIc,5019
+huri_emo/manifests/eag/huri_module.toml,sha256=LZidbO9x9wkjS5cfWuHLDYmLq-jH9euOjDR_f9jEsVs,345
+huri_emo/manifests/emo/huri_module.toml,sha256=Grrk-z2gDZl62hWm2msRgI8epQovt6nyjv0DUJbQzxY,324
+huri_module_emo-0.1.0.dist-info/METADATA,sha256=uqlSCLm2epqWal8JQ8ZokGOpzfu42t1fyuw_7GEw2cw,2193
+huri_module_emo-0.1.0.dist-info/WHEEL,sha256=YLJXdYXQ2FQ0Uqn2J-6iEIC-3iOey8lH3xCtvFLkd8Q,91
+huri_module_emo-0.1.0.dist-info/entry_points.txt,sha256=Chzfw4drNh5-OmZiPonr7IRDD6WZeiIva9lhLvxscGU,71
+huri_module_emo-0.1.0.dist-info/top_level.txt,sha256=fZoBI9AbPjeCF5DOmu8R5hI-11cwU6aHDi9Dm6lXSvk,9
+huri_module_emo-0.1.0.dist-info/RECORD,,

huri_module_emo-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (81.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

huri_module_emo-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[huri.modules]
+eag = huri_emo.aggregator:EAG
+emo = huri_emo.module:EMO

huri_module_emo-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+huri_emo