huri-module-emo 0.1.0__tar.gz

huri_module_emo-0.1.0/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: huri-module-emo
+Version: 0.1.0
+Summary: HuRI module: Speech emotion from prosody (wav2vec2 classifier), and its aggregator
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: huri-core[server]<0.5,>=0.4
+Requires-Dist: huri-events<0.2,>=0.1
+Requires-Dist: numpy<2,>=1.26
+Provides-Extra: model
+Requires-Dist: torch; extra == "model"
+Requires-Dist: transformers>=4.46; extra == "model"
+
+# huri-module-emo
+
+Speech emotion from prosody (wav2vec2 classifier), and its aggregator.
+
+| Module | Topics | What it does |
+|---|---|---|
+| `emo` | `huri.voice` → `huri.emotion` | Estimates the emotion of speech from its prosody |
+| `eag` | `huri.emotion` → `huri.partial_question` | Aggregates emotion estimates over an utterance |
+
+```sh
+pip install -e 'packages/huri-module-emo[model]'   # from a huri-modules checkout, next to huri-core and huri-events
+HURI_MODULES=emo,eag ...   # enable it on the server
+```
+
+`emo` needs the `model` extra (torch and transformers, for its wav2vec2
+classifier); `eag` only aggregates events and needs none of it, so a host that
+runs `eag` alone installs the package without the extra
+(`[tool.huri-modules.extras]` in `pyproject.toml` says which module needs which
+extra; `huri hub install` and `scripts/install_local.sh` add it). Hardware pins
+of `emo`: `requirements-<variant>.emo.txt` (`cpu`, `nvidia`, `amd`; install them
+before the package).
+
+A distribution of
+[huri-modules](https://github.com/Lucy-Robotics/huri-modules), versioned in its
+`pyproject.toml` (see RELEASING.md there). It depends on `huri-core` and
+`huri-events`, which are not on PyPI: without a checkout, install them from git
+in the same pip command, so pip never looks them up on an index:
+
+```sh
+ref=0.1.0   # a huri-modules release tag
+core=0.4.0  # HURI_CORE_REF in that release's versions.env
+pip install \
+  "huri-core[server] @ git+https://github.com/Lucy-Robotics/huri-core.git@$core" \
+  "huri-events @ git+https://github.com/Lucy-Robotics/huri-modules.git@$ref#subdirectory=packages/huri-events" \
+  "huri-module-emo[model] @ git+https://github.com/Lucy-Robotics/huri-modules.git@$ref#subdirectory=packages/huri-module-emo"
+```

huri_module_emo-0.1.0/README.md

@@ -0,0 +1,36 @@
+# huri-module-emo
+
+Speech emotion from prosody (wav2vec2 classifier), and its aggregator.
+
+| Module | Topics | What it does |
+|---|---|---|
+| `emo` | `huri.voice` → `huri.emotion` | Estimates the emotion of speech from its prosody |
+| `eag` | `huri.emotion` → `huri.partial_question` | Aggregates emotion estimates over an utterance |
+
+```sh
+pip install -e 'packages/huri-module-emo[model]'   # from a huri-modules checkout, next to huri-core and huri-events
+HURI_MODULES=emo,eag ...   # enable it on the server
+```
+
+`emo` needs the `model` extra (torch and transformers, for its wav2vec2
+classifier); `eag` only aggregates events and needs none of it, so a host that
+runs `eag` alone installs the package without the extra
+(`[tool.huri-modules.extras]` in `pyproject.toml` says which module needs which
+extra; `huri hub install` and `scripts/install_local.sh` add it). Hardware pins
+of `emo`: `requirements-<variant>.emo.txt` (`cpu`, `nvidia`, `amd`; install them
+before the package).
+
+A distribution of
+[huri-modules](https://github.com/Lucy-Robotics/huri-modules), versioned in its
+`pyproject.toml` (see RELEASING.md there). It depends on `huri-core` and
+`huri-events`, which are not on PyPI: without a checkout, install them from git
+in the same pip command, so pip never looks them up on an index:
+
+```sh
+ref=0.1.0   # a huri-modules release tag
+core=0.4.0  # HURI_CORE_REF in that release's versions.env
+pip install \
+  "huri-core[server] @ git+https://github.com/Lucy-Robotics/huri-core.git@$core" \
+  "huri-events @ git+https://github.com/Lucy-Robotics/huri-modules.git@$ref#subdirectory=packages/huri-events" \
+  "huri-module-emo[model] @ git+https://github.com/Lucy-Robotics/huri-modules.git@$ref#subdirectory=packages/huri-module-emo"
+```

huri_module_emo-0.1.0/huri_emo/aggregator.py

@@ -0,0 +1,75 @@
+from collections import defaultdict
+from typing import Dict, Optional
+
+from huri_events.emotion import Emotion
+from huri_events.question import PartialQuestion
+
+from huri_core.core.module import Module
+
+
+class EAG(Module):
+    """EAG Module
+
+    Aggregate all emotions and send when voice end.
+
+    input: huri.emotion,
+    output: huri.partial_question
+
+    :ema_alpha: if not None, the aggragation will use ema computation instead
+    of average. Recents emotion will have stronger impact on the final score.
+    Lower alpha will make impact lower, and higher alpha will make it higher. \
+    Default alpha would be ~0.3.
+    """
+
+    input_type = "huri.emotion"
+    output_type = "huri.partial_question"
+    input_event = Emotion
+    output_event = PartialQuestion
+
+    def __init__(self, ema_alpha: Optional[float] = None):
+        super().__init__()
+
+        self.scores: Dict[str, float] = defaultdict(float)
+        self.count: int = 0
+
+        self.ema_alpha = ema_alpha
+
+    def _finalize(self) -> Emotion:
+        avg_scores = (
+            {label: score / self.count for label, score in self.scores.items()}
+            if self.ema_alpha is None
+            else self.scores
+        )
+
+        best_label = max(avg_scores, key=lambda label: avg_scores[label])
+
+        result = Emotion(
+            label=best_label,
+            confidence=avg_scores[best_label],
+            scores=avg_scores,
+            end=True,
+        )
+
+        self.scores.clear()
+        self.count = 0
+
+        return result
+
+    async def process(self, emotion: Emotion) -> Optional[PartialQuestion]:
+        if self.ema_alpha is not None:
+            for label, score in emotion.scores.items():
+                self.scores[label] = (
+                    self.ema_alpha * score + (1 - self.ema_alpha) * self.scores[label]
+                )
+        else:
+            for label, score in emotion.scores.items():
+                self.scores[label] += score
+
+        self.count += 1
+
+        if emotion.end:
+            emotion = self._finalize()
+
+            return PartialQuestion(transcript=None, emotion=emotion)
+
+        return None

huri_module_emo-0.1.0/huri_emo/manifests/eag/huri_module.toml

@@ -0,0 +1,7 @@
+name = "eag"
+summary = "Aggregates emotion estimates over an utterance"
+input = { topic = "huri.emotion", event = "huri_events.emotion:Emotion" }
+output = { topic = "huri.partial_question", event = "huri_events.question:PartialQuestion" }
+events = ["huri-events>=0.1,<0.2"]
+requires = { gpu = false, actuation = false }
+huri_core = ">=0.4,<0.5"

huri_module_emo-0.1.0/huri_emo/manifests/emo/huri_module.toml

@@ -0,0 +1,7 @@
+name = "emo"
+summary = "Estimates the emotion of speech from its prosody"
+input = { topic = "huri.voice", event = "huri_events.speech:Voice" }
+output = { topic = "huri.emotion", event = "huri_events.emotion:Emotion" }
+events = ["huri-events>=0.1,<0.2"]
+requires = { gpu = false, actuation = false }
+huri_core = ">=0.4,<0.5"

huri_module_emo-0.1.0/huri_emo/module.py

@@ -0,0 +1,155 @@
+import asyncio
+import os
+from typing import Any, AsyncGenerator, Dict, List
+
+import numpy as np
+from huri_events.emotion import Emotion
+from huri_events.speech import Voice
+from ray import serve
+from ray.serve import handle
+
+from huri_core.core.module import ModuleWithHandle
+
+_MODEL_NAME = os.environ.get("HURI_EMO_MODEL", "superb/hubert-large-superb-er")
+
+
+@serve.deployment(name="EMOHandle", max_ongoing_requests=8)
+class EMOHandle:
+    """Prosody emotion model.
+
+    :model_name: name of the Emotion Analysis model.
+    :sample_rate: sample rate of the audio passed to predict().
+    """
+
+    def __init__(
+        self,
+        model_name: str = _MODEL_NAME,
+        sample_rate: int = 16000,
+    ):
+        from transformers import (
+            AutoModelForAudioClassification,
+            Wav2Vec2FeatureExtractor,
+        )
+
+        self.model = AutoModelForAudioClassification.from_pretrained(model_name)
+        self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_name)
+        self.sample_rate = sample_rate
+
+    async def predict(self, audio: np.ndarray) -> Dict[str, Any]:
+        # Blocking torch inference runs off the replica's event loop.
+        return await asyncio.to_thread(self._predict_sync, audio)
+
+    def _predict_sync(self, audio_np: np.ndarray) -> Dict[str, Any]:
+        import torch
+
+        inputs = self.feature_extractor(
+            audio_np, sampling_rate=self.sample_rate, return_tensors="pt", padding=True
+        )
+
+        with torch.no_grad():
+            logits = self.model(**inputs).logits
+            probs = torch.softmax(logits, dim=-1)[0]
+
+        predicted_id = int(torch.argmax(probs).item())
+
+        labels = self.model.config.id2label
+
+        return {
+            "label": labels[predicted_id],
+            "confidence": float(probs[predicted_id]),
+            "scores": {labels[i]: float(probs[i]) for i in range(len(labels))},
+        }
+
+
+class EMO(ModuleWithHandle):
+    """EMO Module
+
+    Prosody Analysis of user voice speech.
+
+    input: huri.voice,
+    output: huri.emotion
+
+    :sample_rate: size of received voice audio. Usually 8000, 16000 or 48000.
+    :block_duration: size of received voice audio (in s).
+    :analysis_window: duration of audio per analysis (in s).
+    """
+
+    _handle_cls = EMOHandle
+    input_type = "huri.voice"
+    output_type = "huri.emotion"
+    input_event = Voice
+    output_event = Emotion
+
+    def __init__(
+        self,
+        _handle: handle.DeploymentHandle,
+        sample_rate: int = 16000,
+        block_duration: float = 0.020,  # s
+        analysis_window: float = 4.0,  # s
+        **kwargs,
+    ):
+        super().__init__(_handle=_handle, **kwargs)
+
+        self.sample_rate = sample_rate
+        self.window_size = int(analysis_window / block_duration)
+
+        self.buffer: List[np.ndarray] = []
+
+        self.running = False
+        self.lock: asyncio.Lock = asyncio.Lock()
+
+        # Set when the end of the voice is received. Remembered if an analysis
+        # is in flight, so that call handles it once it finishes.
+        self._end_requested: bool = False
+
+    async def process(self, voice: Voice) -> AsyncGenerator[Emotion, None]:
+        async with self.lock:
+            if voice.data is None:
+                self._end_requested = True
+            else:
+                self.buffer.append(voice.data)
+            if self.running:
+                return
+            self.running = True
+
+        try:
+            async for emotion in self._analyze_pending():
+                yield emotion
+        finally:
+            self.running = False
+
+    async def _analyze_pending(self) -> AsyncGenerator[Emotion, None]:
+        """Run analyses until there is nothing left to do.
+
+        Loops so that an end-of-turn that arrives while a window is being
+        analyzed is handled right after it, in this same call. Each pass is
+        yielded as its own Emotion.
+        """
+
+        while True:
+            async with self.lock:
+                end = self._end_requested
+                if end:
+                    self._end_requested = False
+                    chunks = list(self.buffer)
+                else:
+                    if len(self.buffer) < self.window_size:
+                        return
+                    chunks = self.buffer[: self.window_size]
+
+            # On end, analyze the remaining buffer; fall back to a short zero
+            # buffer when it is empty so the model still gets valid input.
+            audio = (
+                np.concatenate(chunks, axis=0)
+                if chunks
+                else np.zeros(self.sample_rate // 10, dtype=np.float32)
+            )
+            result = await self._handle.predict.remote(audio)
+
+            async with self.lock:
+                # Frames received during the await are kept for the next pass.
+                self.buffer = self.buffer[len(chunks) :]
+
+            yield Emotion(result["label"], result["confidence"], result["scores"], end)
+            if not self._end_requested:
+                return

huri_module_emo-0.1.0/huri_module_emo.egg-info/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: huri-module-emo
+Version: 0.1.0
+Summary: HuRI module: Speech emotion from prosody (wav2vec2 classifier), and its aggregator
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: huri-core[server]<0.5,>=0.4
+Requires-Dist: huri-events<0.2,>=0.1
+Requires-Dist: numpy<2,>=1.26
+Provides-Extra: model
+Requires-Dist: torch; extra == "model"
+Requires-Dist: transformers>=4.46; extra == "model"
+
+# huri-module-emo
+
+Speech emotion from prosody (wav2vec2 classifier), and its aggregator.
+
+| Module | Topics | What it does |
+|---|---|---|
+| `emo` | `huri.voice` → `huri.emotion` | Estimates the emotion of speech from its prosody |
+| `eag` | `huri.emotion` → `huri.partial_question` | Aggregates emotion estimates over an utterance |
+
+```sh
+pip install -e 'packages/huri-module-emo[model]'   # from a huri-modules checkout, next to huri-core and huri-events
+HURI_MODULES=emo,eag ...   # enable it on the server
+```
+
+`emo` needs the `model` extra (torch and transformers, for its wav2vec2
+classifier); `eag` only aggregates events and needs none of it, so a host that
+runs `eag` alone installs the package without the extra
+(`[tool.huri-modules.extras]` in `pyproject.toml` says which module needs which
+extra; `huri hub install` and `scripts/install_local.sh` add it). Hardware pins
+of `emo`: `requirements-<variant>.emo.txt` (`cpu`, `nvidia`, `amd`; install them
+before the package).
+
+A distribution of
+[huri-modules](https://github.com/Lucy-Robotics/huri-modules), versioned in its
+`pyproject.toml` (see RELEASING.md there). It depends on `huri-core` and
+`huri-events`, which are not on PyPI: without a checkout, install them from git
+in the same pip command, so pip never looks them up on an index:
+
+```sh
+ref=0.1.0   # a huri-modules release tag
+core=0.4.0  # HURI_CORE_REF in that release's versions.env
+pip install \
+  "huri-core[server] @ git+https://github.com/Lucy-Robotics/huri-core.git@$core" \
+  "huri-events @ git+https://github.com/Lucy-Robotics/huri-modules.git@$ref#subdirectory=packages/huri-events" \
+  "huri-module-emo[model] @ git+https://github.com/Lucy-Robotics/huri-modules.git@$ref#subdirectory=packages/huri-module-emo"
+```

huri_module_emo-0.1.0/huri_module_emo.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+pyproject.toml
+huri_emo/__init__.py
+huri_emo/aggregator.py
+huri_emo/module.py
+huri_emo/manifests/eag/huri_module.toml
+huri_emo/manifests/emo/huri_module.toml
+huri_module_emo.egg-info/PKG-INFO
+huri_module_emo.egg-info/SOURCES.txt
+huri_module_emo.egg-info/dependency_links.txt
+huri_module_emo.egg-info/entry_points.txt
+huri_module_emo.egg-info/requires.txt
+huri_module_emo.egg-info/top_level.txt
+tests/test_emo.py

huri_module_emo-0.1.0/huri_module_emo.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

huri_module_emo-0.1.0/huri_module_emo.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[huri.modules]
+eag = huri_emo.aggregator:EAG
+emo = huri_emo.module:EMO

huri_module_emo-0.1.0/huri_module_emo.egg-info/requires.txt

@@ -0,0 +1,7 @@
+huri-core[server]<0.5,>=0.4
+huri-events<0.2,>=0.1
+numpy<2,>=1.26
+
+[model]
+torch
+transformers>=4.46

huri_module_emo-0.1.0/huri_module_emo.egg-info/top_level.txt

@@ -0,0 +1 @@
+huri_emo

huri_module_emo-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=64,<82"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "huri-module-emo"
+version = "0.1.0"
+description = "HuRI module: Speech emotion from prosody (wav2vec2 classifier), and its aggregator"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+  "huri-core[server]>=0.4,<0.5",
+  "huri-events>=0.1,<0.2",
+  "numpy>=1.26,<2",
+]
+
+# The classifier of `emo` (wav2vec2). `eag` only aggregates its events, so a
+# host that plans eag alone installs neither torch nor transformers.
+[project.optional-dependencies]
+model = ["torch", "transformers>=4.46"]
+
+[project.entry-points."huri.modules"]
+emo = "huri_emo.module:EMO"
+eag = "huri_emo.aggregator:EAG"
+
+# The extra each module needs: read by huri-modules' tooling
+# (scripts/module_requirements.py, check_registry.py against registry.toml's
+# `extra`), not by huri-core.
+[tool.huri-modules.extras]
+emo = "model"
+
+[tool.setuptools.packages.find]
+include = ["huri_emo*"]
+
+[tool.setuptools.package-data]
+huri_emo = ["huri_module.toml", "manifests/*/huri_module.toml"]

huri_module_emo-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

huri_module_emo-0.1.0/tests/test_emo.py

@@ -0,0 +1,90 @@
+import asyncio
+
+import numpy as np
+from huri_events.speech import Voice
+
+from huri_core.testing import LocalHandle, ModuleHarness
+
+
+class FakeHandle:
+    """Stands in for EMOHandle: a fixed emotion per call; can block the first."""
+
+    def __init__(self, gate=None):
+        self.gate = gate
+        self.calls = []
+
+    async def predict(self, audio):
+        self.calls.append(len(audio))
+        if self.gate is not None and len(self.calls) == 1:
+            await self.gate.wait()
+        return {"label": "neu", "confidence": 0.9, "scores": {"neu": 0.9, "ang": 0.1}}
+
+
+def _emo(handle):
+    from huri_emo.module import EMO
+
+    # 20 ms blocks -> window of 10 frames (fast tests).
+    return ModuleHarness(EMO, handle=LocalHandle(handle), analysis_window=0.2)
+
+
+async def _collect(emo, voice):
+    return await emo.feed(voice)
+
+
+def _voice():
+    return Voice(np.zeros(4, dtype=np.float32))
+
+
+def test_interim_emotion_once_window_is_full():
+    asyncio.run(_test_interim_emotion_once_window_is_full_impl())
+
+
+async def _test_interim_emotion_once_window_is_full_impl():
+    handle = FakeHandle()
+    emo = _emo(handle)
+
+    results = []
+    for _ in range(10):
+        results += await _collect(emo, _voice())
+
+    assert [e.end for e in results] == [False]
+    assert handle.calls == [40]
+
+
+def test_end_marker_always_yields_final_emotion():
+    asyncio.run(_test_end_marker_always_yields_final_emotion_impl())
+
+
+async def _test_end_marker_always_yields_final_emotion_impl():
+    handle = FakeHandle()
+    emo = _emo(handle)
+
+    # Very short turn: no frames at all still produces Emotion(end=True).
+    results = await _collect(emo, Voice(None))
+
+    assert [e.end for e in results] == [True]
+    assert results[0].label == "neu"
+
+
+def test_end_marker_during_inflight_analysis_is_not_dropped():
+    asyncio.run(_test_end_marker_during_inflight_analysis_is_not_dropped_impl())
+
+
+async def _test_end_marker_during_inflight_analysis_is_not_dropped_impl():
+    gate = asyncio.Event()
+    handle = FakeHandle(gate)
+    emo = _emo(handle)
+
+    for _ in range(9):
+        await _collect(emo, _voice())
+
+    # 10th frame starts an analysis that blocks on the gate.
+    inflight = asyncio.create_task(_collect(emo, _voice()))
+    await asyncio.sleep(0)
+
+    # The single end marker lands mid-analysis: it must be remembered.
+    assert await _collect(emo, Voice(None)) == []
+    gate.set()
+
+    results = await inflight
+    assert [e.end for e in results] == [False, True]