bithuman 1.0.2__py3-none-any.whl

bithuman/video_graph/trigger.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import json
+from typing import List, Literal, Optional
+
+from loguru import logger
+from pydantic import BaseModel, Field
+
+
+class TriggerData(BaseModel):
+    """Data to be sent when a trigger is activated"""
+
+    target_video: Optional[str] = None
+    actions: List[str] | str = Field(default_factory=list)
+    description: str = ""
+
+
+class VideoActionTrigger(BaseModel):
+    """Base class for video action triggers"""
+
+    trigger_data: TriggerData = Field(
+        description="Data to be sent when trigger conditions are met"
+    )
+
+    def check_trigger(self, condition: any) -> Optional[TriggerData]:
+        """
+        Base method to check if trigger conditions are met
+        Args:
+            condition: The condition to check against (type varies by trigger type)
+        Returns:
+            TriggerData if triggered, None otherwise
+        """
+        return None
+
+    @classmethod
+    def from_json(cls, json_str: str) -> List["VideoActionTrigger"]:
+        """
+        Create KeywordTrigger instances from JSON string using Pydantic validation
+        Args:
+            json_str: JSON string containing trigger configurations
+        Returns:
+            List of validated KeywordTrigger instances
+        """
+        if not json_str:
+            return []
+        try:
+            triggers_data = json.loads(json_str)
+            return [
+                cls.model_validate_json(json.dumps(trigger))
+                for trigger in triggers_data
+            ]
+        except Exception as e:
+            logger.exception(f"Error parsing KeywordTrigger: {e}")
+            return []
+
+
+class KeywordTrigger(VideoActionTrigger):
+    """Trigger that activates when specific keywords are detected"""
+
+    keywords: List[str] = Field(
+        description="List of keywords that can trigger this action"
+    )
+    trigger_source: Literal["user", "agent", "both"] = Field(
+        default="both", description="Who can trigger this action - user, agent, or both"
+    )
+
+    def check_trigger(
+        self, text: str, source: Literal["user", "agent"]
+    ) -> Optional[TriggerData]:
+        """
+        Check if the given text and source triggers this keyword
+        Args:
+            text: The text to check
+            source: The source of the text - either "user" or "agent"
+        Returns:
+            TriggerData if triggered, None otherwise
+        """
+        if self.trigger_source != "both" and source != self.trigger_source:
+            return None
+
+        if any(keyword.lower() in text.lower() for keyword in self.keywords):
+            return self.trigger_data
+        return None
+
+
+if __name__ == "__main__":
+    triggers = KeywordTrigger.from_json(
+        '[{"keywords": ["goodbye", "bye"], "trigger_source": "user", "trigger_data": {"target_video": null, "actions": "wave", "description": "Wave animation"}}]',  # noqa: E501
+    )
+    print(triggers)

bithuman/video_graph/video_script.py

@@ -0,0 +1,344 @@
+"""Definition of the script for the video graph."""
+
+from __future__ import annotations
+
+import copy
+import json
+from collections import OrderedDict, defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import yaml
+from dataclasses_json import dataclass_json
+from loguru import logger
+
+from ..api import Emotion, EmotionPrediction
+from ..config import Settings
+from .driver_video import DriverVideo, LoopingVideo, SingleActionVideo
+
+VIDEO_EXTS = [".mp4", ".mov", ".avi"]
+
+
+@dataclass_json
+@dataclass
+class EmotionToVideo:
+    """Dataclass for emotion to video mapping.
+
+    Attributes:
+        emotion: The emotion.
+        video_name: The video name.
+        threshold: The threshold for the emotion.
+        actions: The actions before the video.
+    """
+
+    emotion: Emotion
+    video_name: str
+    threshold: float = 0.1
+    actions: List[str] = field(default_factory=list)
+
+
+@dataclass_json
+@dataclass
+class IdleAction:
+    """Dataclass for idle action.
+
+    Attributes:
+        actions: The actions for the idle video.
+        interval: Play the action every interval seconds
+            or randomly between the interval.
+    """
+
+    actions: List[str] = field(default_factory=list)
+    interval: float | Tuple[float, float] = 60
+
+    @property
+    def min_interval(self) -> float:
+        return self.interval if isinstance(self.interval, float) else self.interval[0]
+
+
+@dataclass_json
+@dataclass
+class VideoScript:
+    default_video: Optional[str] = None
+    action_hi_video: Optional[str] = None
+    emotions_map: List[EmotionToVideo] = field(default_factory=list)
+    idle_video: Optional[str] = None
+    idle_actions: List[IdleAction] = field(default_factory=list)
+    FPS: float = 25
+    BACK_TO_IDLE: float = 10  # back to idle after 10 seconds
+
+    def __post_init__(self):
+        self.last_nonidle_frame = 0
+        self.last_video_name = None
+
+        self.update_index()
+
+    def update_index(self):
+        # index the videos by emotion and group by threshold
+        emotions_map_index: Dict[Emotion, Dict[float, List[EmotionToVideo]]] = {}
+        for e2v in self.emotions_map:
+            emotions_map_index.setdefault(e2v.emotion, defaultdict(list))[
+                e2v.threshold
+            ].append(e2v)
+
+        # sort the videos by threshold in descending order
+        for emotion in emotions_map_index:
+            emotions_map_index[emotion] = OrderedDict(
+                sorted(
+                    emotions_map_index[emotion].items(),
+                    key=lambda x: x[0],
+                    reverse=True,
+                )
+            )
+        self.emotions_map_index = emotions_map_index
+
+        # Next idle action, randomly selected from the idle actions
+        self.set_next_idle_action()
+
+    def set_next_idle_action(self, next_idle_action: IdleAction | None = None):
+        """Set the next idle action.
+
+        Args:
+            next_idle_action: The next idle action. If None, randomly select one.
+        """
+        if not next_idle_action:
+            next_idle_action = (
+                np.random.choice(self.idle_actions) if self.idle_actions else None
+            )
+
+        self.next_idle_action = next_idle_action
+        if not self.next_idle_action:
+            self.next_idle_action_interval = None
+            return
+
+        if isinstance(self.next_idle_action.interval, (tuple, list)):
+            interval = np.random.uniform(*self.next_idle_action.interval)
+        else:
+            interval = self.next_idle_action.interval
+        self.next_idle_action_interval = interval
+
+    def get_video_and_actions(
+        self,
+        curr_frame_index: int,
+        emotions: List[EmotionPrediction] = None,
+        text: str = None,
+        is_idle: bool = False,
+        settings: Settings = None,
+    ) -> Tuple[str, List[str], bool]:
+        """Get the videos for the emotion."""
+        is_idle = is_idle and not emotions  # idle only when no emotion
+
+        reset_action = False
+        video_name, actions = None, []
+        if is_idle:
+            idle_time = (curr_frame_index - self.last_nonidle_frame) / self.FPS
+
+            # Back to the default video if idle for a long time
+            if idle_time >= self.BACK_TO_IDLE:
+                next_idle_video = (
+                    self.idle_video
+                    if settings and settings.LIVA_IDEL_VIDEO_ENABLED
+                    else None
+                )
+                # TODO: random select one idle video from a list
+                video_name = next_idle_video or self.default_video
+
+            # Play a random idle action if the idle time is long enough
+            if self.next_idle_action and idle_time >= self.next_idle_action_interval:
+                # Play the idle action, then reset the next idle action
+                actions = self.next_idle_action.actions
+                reset_action = True
+                self.set_next_idle_action()
+                self.last_nonidle_frame = curr_frame_index
+        else:
+            self.last_nonidle_frame = curr_frame_index
+
+            # Get the video for the emotion
+            if emotions:
+                # Ignore neutral if there are other emotions > 0.2
+                if len(emotions) > 1:
+                    if (
+                        emotions[0].emotion == Emotion.NEUTRAL
+                        and emotions[1].score > 0.2
+                    ):
+                        emotions = emotions[1:]
+
+                top_emotion = emotions[0]
+                for threshold, e2v_list in self.emotions_map_index.get(
+                    top_emotion.emotion, {}
+                ).items():
+                    if top_emotion.score >= threshold and e2v_list:
+                        # Randomly select one video from the list
+                        e2v: EmotionToVideo = np.random.choice(e2v_list)
+                        video_name, actions = e2v.video_name, e2v.actions
+                        break
+            if (
+                text
+                and self.action_hi_video
+                and settings
+                and settings.LIVA_AUTO_SAY_HI
+                and self.action_hi_video not in actions
+            ):
+                # Detect Hello, hi, Bye, Goodbye
+                say_hi = False
+                text = text.lower()
+                for word in ["hello", "hi ", "bye", "goodbye"]:
+                    if text.startswith(word):
+                        say_hi = True
+                        break
+                if say_hi:
+                    actions = [self.action_hi_video] + actions
+
+            if (
+                not video_name
+                and self.idle_video
+                and self.last_video_name == self.idle_video
+            ):
+                # Back to the default video if nonidle and no video is selected
+                video_name = self.default_video
+
+        self.last_video_name = video_name
+        return video_name, actions, reset_action
+
+
+@dataclass_json
+@dataclass
+class VideoConfig:
+    name: str
+    video_file: str
+    video_type: str = "LoopingVideo"
+    stride: int = 10
+    loop_between: Tuple[int, int] = (0, -1)
+    remove_nodes: Optional[List[int]] = None
+    transition_frames: Optional[List[int]] = None
+    action_frame: int = -1
+    single_direction: bool = False
+    adding_kwargs: Dict = field(default_factory=dict)
+    lip_sync_required: bool = True
+    stop_on_user_speech: bool = False
+    stop_on_agent_speech: bool = False
+
+    def load_video(self) -> DriverVideo:
+        if self.video_type == "LoopingVideo":
+            video = LoopingVideo(
+                name=self.name,
+                video_path=self.video_file,
+                stride=self.stride,
+                single_direction=self.single_direction,
+                stop_on_user_speech=self.stop_on_user_speech,
+                stop_on_agent_speech=self.stop_on_agent_speech,
+                loop_between=self.loop_between,
+                lip_sync_required=self.lip_sync_required,
+            )
+        elif self.video_type == "SingleActionVideo":
+            video = SingleActionVideo(
+                name=self.name,
+                video_path=self.video_file,
+                single_direction=self.single_direction,
+                stop_on_user_speech=self.stop_on_user_speech,
+                stop_on_agent_speech=self.stop_on_agent_speech,
+                transition_frames=self.transition_frames,
+                action_frame=self.action_frame,
+                lip_sync_required=self.lip_sync_required,
+            )
+        else:
+            raise ValueError(f"Unknown video type: {self.video_type}")
+
+        if self.lip_sync_required and not video.video_data_path:
+            raise ValueError(
+                f"Lip sync is required for video {self.name}, but no video data path is provided"
+            )
+
+        if self.remove_nodes:
+            video.remove_nodes(frame_indices=self.remove_nodes)
+        return video
+
+
+@dataclass_json
+@dataclass
+class VideoConfigs:
+    videos: List[VideoConfig]
+    videos_script: Optional[VideoScript] = field(default_factory=VideoScript)
+    talking_face_configs: Optional[Dict[str, Any]] = None
+
+    def load_videos(
+        self, video_root: str = None, verbose: bool = True
+    ) -> List["DriverVideo"]:
+        if video_root:
+            video_configs = copy.deepcopy(self.videos)
+            video_root = Path(video_root)
+            for video in video_configs:
+                video.video_file = str(video_root / video.video_file)
+        else:
+            video_configs = self.videos
+
+        return [video_config.load_video() for video_config in video_configs]
+
+    @classmethod
+    def from_videofolder(cls, video_folder: str) -> "VideoConfigs":
+        """Create a VideoConfigs object with a video folder."""
+        video_files = [
+            p
+            for p in Path(video_folder).iterdir()
+            if p.suffix.lower() in VIDEO_EXTS and not p.name.startswith(".")
+        ]
+        if len(video_files) == 0:
+            raise ValueError(f"No video files found in {video_folder}")
+        if len(video_files) > 1:
+            raise ValueError(
+                f"Multiple video files found in {video_folder}: {video_files}"
+            )
+        video_file = video_files[0]
+        return cls(
+            videos=[VideoConfig(name=video_file.stem, video_file=str(video_file))]
+        )
+
+    @classmethod
+    def from_videofile(
+        cls, video_file: str, inference_data_file: str = None
+    ) -> "VideoConfigs":
+        """Create a VideoConfigs object with a video file."""
+        return cls(
+            videos=[
+                VideoConfig(
+                    name=Path(video_file).stem,
+                    video_file=str(video_file),
+                    inference_data_file=inference_data_file,
+                )
+            ]
+        )
+
+    @classmethod
+    def from_yaml(cls, file_path: str) -> "VideoConfigs":
+        """Load the video configs from a YAML file."""
+        with open(file_path) as f:
+            data = yaml.safe_load(f)
+        return cls.from_dict(data)
+
+    def to_yaml(self, file_path: str) -> None:
+        """Save the video configs to a YAML file."""
+        with open(file_path, "w") as f:
+            data = json.loads(self.to_json())
+            yaml.dump(data, f, sort_keys=False)
+
+    def update_runtime_configs(self, settings: Settings):
+        """Update the runtime configs."""
+        if not settings.ALLOW_VIDEO_SCRIPT_UPDATE:
+            logger.info(
+                "Video script update is disabled, skip updating runtime configs."
+            )
+            return
+
+        configs = self.talking_face_configs or {}
+        fails, success = {}, {}
+        for k, v in configs.items():
+            if hasattr(settings, k):
+                setattr(settings, k, v)
+                success[k] = v
+            else:
+                fails[k] = v
+        logger.info(f"Updated runtime configs from model: {success}")
+        if fails:
+            logger.warning(f"Runtime configs not found in settings: {fails}")

bithuman-1.0.2.dist-info/METADATA

@@ -0,0 +1,37 @@
+Metadata-Version: 2.4
+Name: bithuman
+Version: 1.0.2
+Summary: Bithuman avatar runtime — pure Python engine
+Platform: Linux
+Platform: Mac OS X
+Platform: Windows
+Requires-Python: >=3.9
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: h5py~=3.13
+Requires-Dist: loguru~=0.7
+Requires-Dist: numba<1.0,>=0.60
+Requires-Dist: soxr~=0.5
+Requires-Dist: scipy<2.0,>=1.13
+Requires-Dist: moviepy~=1.0.3
+Requires-Dist: soundfile~=0.13
+Requires-Dist: pydantic~=2.10
+Requires-Dist: pydantic-settings~=2.8
+Requires-Dist: networkx<4.0,>=3.1
+Requires-Dist: tqdm~=4.67
+Requires-Dist: dataclasses-json~=0.6
+Requires-Dist: librosa~=0.10
+Requires-Dist: pyzmq~=26.2; python_version < "3.14"
+Requires-Dist: msgpack~=1.1
+Requires-Dist: PyYAML~=6.0
+Requires-Dist: aiohttp~=3.11
+Requires-Dist: onnxruntime>=1.18
+Requires-Dist: av>=12.0
+Requires-Dist: PyJWT>=2.8
+Requires-Dist: requests>=2.31
+Requires-Dist: lz4>=4.3
+Requires-Dist: PyTurboJPEG>=1.7
+Requires-Dist: opencv-python-headless>=4.8
+Provides-Extra: agent
+Requires-Dist: livekit-agents~=1.1; extra == "agent"
+Dynamic: platform
+Dynamic: requires-python

bithuman-1.0.2.dist-info/RECORD

@@ -0,0 +1,44 @@
+bithuman/__init__.py,sha256=MI3NNap09CT5durEAjHBRW-gv3bu-yHfHQQNeP63ZoQ,286
+bithuman/_version.py,sha256=Y3LSfRioSl2xch70pq_ULlvyECXyEtN3krVaWeGyaxk,22
+bithuman/api.py,sha256=yJZIkMODYXcvdRx7t9nYuNktr1gfCfqfcx8t-xjJbO4,4704
+bithuman/config.py,sha256=_x3O_HEpw05obPWB0bjzmRrh-IPZQ2-wBFq75p6-280,2646
+bithuman/runtime.py,sha256=h9AzmHJBejJ5kULZrh9efveL4oaF7qHX0DVVECc26RA,40122
+bithuman/runtime_async.py,sha256=2iQ-zKjAqmJj0ezJSw32hTJhGJBUeXpZ-pEHUHEbJmw,18953
+bithuman/audio/__init__.py,sha256=tSUqsLLmxLEeBZ-O0XRSGQIB2igsA9eWr7vzgxMtugg,358
+bithuman/audio/audio.py,sha256=Wfrm7UGTBAMgkzwwoicVf9SvvCF8KzW-mqcG-R5K-wI,10807
+bithuman/audio/hparams.py,sha256=ytgTtxvNgJvrm5rAmhq8jdF_rgt1IMmSxFYnjD-hpmk,4078
+bithuman/audio/utils.py,sha256=16B5klsdIbhknoNrbHz1BsvWaoVec1JQ4YswIWzwelc,8464
+bithuman/engine/__init__.py,sha256=fZXWUyaYxokaiUp62VtbXwlJD3t-1FAtUH0CQ1E2sLc,518
+bithuman/engine/auth.py,sha256=BcIQJvibohgJYlWFuKhFbkcIYWBMKSZyHkG_94jHLDU,11540
+bithuman/engine/compression.py,sha256=JPnVTL1oeGXWFjR2HdkDLdXiTkbJXX-D0KK82yE0jLM,8659
+bithuman/engine/enums.py,sha256=m4yAP4j4UD2E2qSOiY7fMP5y45WaQ7PDPhzKzTy9sRA,254
+bithuman/engine/image_ops.py,sha256=gUl8LQdgVj2NSAH9peLdiv2KN49f6Y-RHmkuk3fVhTQ,6672
+bithuman/engine/inference.py,sha256=eYGASWlTM3URbG-tUGs25MVCm75KTag2NqaRwDE69BA,4058
+bithuman/engine/knn.py,sha256=lWvGkgudYhp75la1FFZDoeW8PHmMQ_g_BSFbEYtv9Uk,1832
+bithuman/engine/video_data.py,sha256=BEWPdWu-qakM_bMr9pEdK_HDga37_f6Iq2yQudIxv90,14052
+bithuman/engine/video_reader.py,sha256=l40YUpzbotbq73XjypEru1KMqDASXd81vYgbuS5gcqA,5132
+bithuman/lib/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+bithuman/lib/audio_encoder.onnx,sha256=6r2eD03YdD_ONH_H8rAlA7ue0KQSv3T0JC__mhNRiM0,2840632
+bithuman/lib/generator.py,sha256=JJJ6G42qhnfJDVYgDPMRaYbEstQEb7PsCAQWCi-kjwk,28954
+bithuman/lib/pth2h5.py,sha256=mOdFbuZtQfydWbsJ-9U3IqRPH10Jtt5Z3bfwu2S0b3g,3004
+bithuman/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bithuman/plugins/stt.py,sha256=_i3ChycPjg7Woeq-EXQGNJq7bDowuiWmkaU8X3zSJSM,5802
+bithuman/service/__init__.py,sha256=YfXb20u8xkceD1LmqWbqrODa1NAeYluPY_WA-mF9MeM,208
+bithuman/service/client.py,sha256=66CQhPvk-zV331IcCWS7igFAvEPL5alFnfOjUB-FF60,28851
+bithuman/service/messages.py,sha256=iZ7bjwAV9M06xxE5z87FLzvRL2KsGUmtTK9HQlZPWbk,6075
+bithuman/service/server.py,sha256=cfbD2WfPUjEPgiGPtgJz1Tc6ACZ8tZnveJ1YAzO7OFI,28026
+bithuman/utils/__init__.py,sha256=HRszl5Vv_qjq0_NDlP6uFPc-dl0R-IrrcUeObAVHOnc,1245
+bithuman/utils/agent.py,sha256=2LvzXcaE5xuViTnPMntsnX1Mmj73pTfYE_2z1eQpF7Y,11955
+bithuman/utils/fps_controller.py,sha256=NvesSQeGG75aVBOfofujH-orCpbIutrh4c8Dt4fFoGg,3005
+bithuman/utils/image.py,sha256=Cu51bKvcfYdG4cvuLSRI0fl6cfN6eQdigMQR-szEaog,1173
+bithuman/utils/unzip.py,sha256=S5dKjEcQAGuDl9RM3o0SYBP4vOVoMcu7cViT96U9Kwo,1298
+bithuman/video_graph/__init__.py,sha256=qKqr5wjUr15YRuCB_ivEjnS34vkwZS5W3O3eFxlkvlk,400
+bithuman/video_graph/action_trigger.py,sha256=22it95VOHSL6zR1HdSGqtQgysQN5t9awGt1NgRGfioM,2636
+bithuman/video_graph/driver_video.py,sha256=TVBANOo2R25Q_sTf03pxUVosVHrU4MlYP_EkFrwMhv4,17076
+bithuman/video_graph/navigator.py,sha256=4UJpqBQHc4_aBbZcyyNuFOSuQZLqnzikKrZ57jaPWes,27475
+bithuman/video_graph/trigger.py,sha256=0I8p4JGzB2LVtB8y_OIz4IIX3qYc6YuHMU-00KDb4IQ,2910
+bithuman/video_graph/video_script.py,sha256=_I0aG7hAcnIU8GKghUNc4ZqyvkB_-4MaEbAdlnefJcY,12056
+bithuman-1.0.2.dist-info/METADATA,sha256=eplXy89F0kbptdedwg6paE5Zdf6QLSk5C27_Zp5BM24,1076
+bithuman-1.0.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+bithuman-1.0.2.dist-info/top_level.txt,sha256=NhjtjyoEtM5UOK1XHTJ8xzw72SanqkCwpeTKMMTykG8,9
+bithuman-1.0.2.dist-info/RECORD,,

bithuman-1.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

bithuman-1.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+bithuman