bithuman 1.0.2__py3-none-any.whl

bithuman/audio/utils.py

@@ -0,0 +1,255 @@
+"""Utilities for audio resample."""
+from __future__ import annotations
+
+import tempfile
+from functools import cached_property
+from pathlib import Path
+from typing import Iterable, Optional, Tuple
+
+import numpy as np
+import soundfile
+import soxr
+
+from ..api import AudioChunk
+
+INT16_MAX = 2**15 - 1  # 32767
+
+
+def load_audio(
+    audio_path: str, target_sr: Optional[int] = None
+) -> Tuple[np.ndarray, int]:
+    """
+    Load an audio file and resample it to the target sample rate.
+
+    Args:
+        audio_path (str): Path to the audio file.
+        target_sr (int): Target sample rate for resampling.
+
+    Returns:
+        np.array: Resampled audio buffer.
+    """
+    audio_np, sr = soundfile.read(audio_path, dtype=np.float32)
+    # convert multichannel to single channel
+    if len(audio_np.shape) > 1:
+        audio_np = np.mean(audio_np, axis=0)
+
+    if target_sr is not None and sr != target_sr:
+        audio_np = soxr.resample(audio_np, sr, target_sr)
+        sr = target_sr
+
+    return audio_np, sr
+
+
+def float32_to_int16(x: np.ndarray) -> np.ndarray:
+    """
+    Converts an array of float32 values to int16.
+
+    Args:
+        x (np.array): Array of float32 values.
+
+    Returns:
+        np.array: Array of int16 values.
+    """
+    return (x * INT16_MAX).astype(np.int16)
+
+
+def int16_to_float32(x: np.ndarray) -> np.ndarray:
+    """
+    Converts an array of int16 values to float32.
+
+    Args:
+        x (np.array): Array of int16 values.
+
+    Returns:
+        np.array: Array of float32 values.
+    """
+    return x.astype(np.float32) / INT16_MAX
+
+
+def resample(audio_buffer: np.ndarray, origin_sr: int, target_sr: int) -> np.ndarray:
+    """
+    Resamples an audio buffer.
+
+    Args:
+        audio_buffer (np.array): Array of audio samples, only support int16 and float32.
+        origin_sr (int): Original sample rate of the audio buffer.
+        target_sr (int): Target sample rate for resampling.
+
+    Returns:
+        np.array: Resampled audio buffer.
+    """
+
+    return soxr.resample(audio_buffer, origin_sr, target_sr)
+
+
+class AudioStreamBatcher:
+    """
+    A class to batch audio streams into chunks suitable for video processing.
+
+    This class takes in a stream of AudioFrame objects and outputs batched AudioFrame
+    objects that are suitable for synchronization with video frames.
+
+    Attributes:
+        pre_pad (int): Number of samples to pad before the audio data.
+        post_pad (int): Number of samples to pad after the audio data.
+        hop_size (int): Number of samples to hop between audio data.
+        min_video_frames (int): Minimum number of video frames to process.
+        expected_video_frames (int): Expected number of video frames to process.
+        fps (int): Frames per second of the video.
+        output_sample_rate (int): Sample rate of the output audio data.
+    """
+
+    def __init__(
+        self,
+        *,
+        pre_pad: int = 400,
+        post_pad: int = 200 * 13,
+        hop_size: int = 200,
+        min_video_frames: int = 2,
+        expected_video_frames: int = 10,
+        fps: int = 25,
+        output_sample_rate: int = 16000,
+    ) -> None:
+        """
+        Initialize the AudioStreamBatcher.
+
+        Args:
+            pre_pad (int): Number of samples to pad before the audio data.
+            post_pad (int): Number of samples to pad after the audio data.
+            hop_size (int): Number of samples to hop between audio data.
+            min_video_frames (int): Minimum number of video frames to process.
+            expected_video_frames (int): Expected number of video frames to process.
+            fps (int): Frames per second of the video.
+            output_sample_rate (int): Sample rate of the output audio data.
+        """
+        self.pre_pad = pre_pad
+        self.post_pad = post_pad
+        self.hop_size = hop_size
+        self.min_video_frames = min_video_frames
+        self.expected_video_frames = expected_video_frames
+        self.fps = fps
+        self.output_sample_rate = output_sample_rate
+
+        self.bytes_per_sample = 2  # int16
+        self.min_n_samples = int(
+            output_sample_rate / fps * min_video_frames + pre_pad + post_pad
+        )
+        self.expect_n_samples = int(
+            output_sample_rate / fps * expected_video_frames + pre_pad + post_pad
+        )
+        self.reset()
+
+    def reset(self) -> None:
+        """Reset the batcher to the initial state."""
+        self._buffer = bytearray(np.zeros(self.pre_pad, dtype=np.int16).tobytes())
+        self._target_length = self.min_n_samples * self.bytes_per_sample
+        self._resampler: soxr.ResampleStream | None = None
+
+    def push(self, data: AudioChunk | None) -> Iterable[AudioChunk]:
+        """
+        Process an incoming AudioChunk object and yield properly padded AudioChunk objects.
+        Args:
+            data (AudioChunk): The incoming audio data to process.
+
+        Yields:
+            AudioChunk: A padded AudioChunk.
+        """
+        # add the audio data to the buffer if provided
+        if data is not None:
+            audio_array = data.array
+            if data.sample_rate != self.output_sample_rate and self._resampler is None:
+                self._resampler = soxr.ResampleStream(
+                    data.sample_rate, self.output_sample_rate, 1, dtype="int16"
+                )
+
+            if self._resampler is not None:
+                audio_array = self._resampler.resample_chunk(
+                    audio_array, last=data.last_chunk
+                )
+            self._buffer.extend(audio_array.tobytes())
+
+        if data is None or data.last_chunk:
+            last_chunk = self.flush()
+            if last_chunk:
+                yield last_chunk
+            return
+
+        while len(self._buffer) >= self._target_length:
+            new_chunk = AudioChunk.from_bytes(
+                bytes(self._buffer[: self._target_length]),
+                self.output_sample_rate,
+                last_chunk=data.last_chunk,
+            )
+            yield new_chunk
+
+            rest = len(self._buffer) - self._target_length
+            keep_from = len(self._buffer) - rest - (self.pre_pad + self.post_pad) * self.bytes_per_sample
+            self._buffer = bytearray(self._buffer[keep_from:])
+            # increase the batch size after the first chunk until the next flush
+            self._target_length = self.expect_n_samples * self.bytes_per_sample
+
+    def flush(self) -> AudioChunk | None:
+        """Flush the audio buffer and yield the remaining audio data."""
+        n_samples = len(self._buffer) // self.bytes_per_sample
+        if n_samples <= self.pre_pad:
+            return None
+
+        # make sure the samples is n x hop_size
+        end_pad = (self.hop_size - n_samples % self.hop_size) % self.hop_size
+        self._buffer.extend(np.zeros(end_pad + self.post_pad, dtype=np.int16).tobytes())
+        chunk = AudioChunk.from_bytes(
+            bytes(self._buffer),
+            self.output_sample_rate,
+            last_chunk=True,
+        )
+        self.reset()
+        return chunk
+
+    def unpad(self, audio_array: np.ndarray) -> np.ndarray:
+        audio_array = audio_array[self.pre_pad : -self.post_pad]
+        return audio_array
+
+    @cached_property
+    def pre_pad_video_frames(self) -> int:
+        return self.pre_pad // 200
+
+
+def write_video_with_audio(
+    output_path: str | Path,
+    frames: list,
+    audio_np: np.ndarray,
+    sample_rate: int,
+    fps: int = 25,
+) -> None:
+    """Write frames and audio numpy array to a video file.
+
+    Args:
+        output_path: Path to save the video
+        frames: List of RGB frames as numpy arrays
+        audio_np: Audio as numpy array
+        sample_rate: Audio sample rate in Hz
+        fps: Video frame rate (default: 25)
+    """
+    from moviepy.editor import AudioFileClip, ImageSequenceClip
+
+    # Create video clip from frames
+    video_clip = ImageSequenceClip(frames, fps=fps)
+
+    # Write audio to temp file
+    with tempfile.NamedTemporaryFile(suffix=".wav") as temp_audio:
+        soundfile.write(temp_audio.name, audio_np, sample_rate)
+        # Create audio clip from temp file
+        audio_clip = AudioFileClip(temp_audio.name)
+        video_clip = video_clip.set_audio(audio_clip)
+
+        # Write output video
+        video_clip.write_videofile(
+            str(output_path),
+            codec="libx264",
+            audio_codec="aac",
+            fps=fps,
+        )
+
+        # Close clips
+        video_clip.close()
+        audio_clip.close()

bithuman/config.py

@@ -0,0 +1,88 @@
+"""Configuration settings for the bithuman runtime."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import ConfigDict, Field
+from pydantic_settings import BaseSettings
+
+_THIS_DIR = Path(__file__).parent
+
+
+class Settings(BaseSettings):
+    """Settings for the bithuman runtime."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    # Video settings
+    ALLOW_VIDEO_SCRIPT_UPDATE: bool = Field(
+        True, description="Whether to allow the video script to update the settings."
+    )
+    OUTPUT_WIDTH: int = Field(
+        1280,
+        description="The size of the output video for the longest side.",
+    )
+    FPS: int = Field(25, description="The frames per second for the video.")
+    COMPRESS_METHOD: Literal["NONE", "JPEG", "TEMP_FILE"] = Field(
+        "JPEG", description="The method to compress the image."
+    )
+    LOADING_MODE: Literal["SYNC", "ASYNC", "ON_DEMAND"] = Field(
+        "ASYNC", description="The mode to load the video."
+    )
+    INPUT_SAMPLE_RATE: int = Field(
+        16_000, frozen=True, description="The sample rate of the input audio."
+    )
+    AUDIO_ENCODER_PATH: Path = Field(
+        str(_THIS_DIR / "lib" / "audio_encoder.onnx"),
+        description="The path to the audio encoder model.",
+    )
+    EXTRACT_WORKSPACE_TO_LOCAL: bool = Field(
+        False, description="Whether to extract the workspace to local."
+    )
+    PROCESS_IDLE_VIDEO: bool = Field(True, description="Whether to process idle video.")
+
+    # LIVA
+    LIVA_IDEL_VIDEO_ENABLED: bool = Field(
+        True, description="Whether to enable idle video."
+    )
+    LIVA_AUTO_SAY_HI: bool = Field(
+        False, description="Whether to automatically say hi in the video."
+    )
+
+    # Video triggers
+    KEYWORD_VIDEO_TRIGGERS_ENABLED: Annotated[
+        bool, Field(description="Whether to enable keyword video triggers.")
+    ] = True
+    KEYWORD_VIDEO_TRIGGERS_JSON: Annotated[
+        str,
+        Field(
+            description="JSON string containing keyword trigger configurations",
+            default="[]",
+        ),
+    ] = "[]"
+
+
+_settings = None
+
+
+def load_settings(force_reload: bool = False) -> Settings:
+    """Load the settings for the bithuman runtime.
+
+    Args:
+        force_reload: Whether to force a reload of the settings. Defaults to False
+
+    Returns:
+        The settings for the bithuman runtime
+    """
+    global _settings
+
+    if force_reload:
+        _settings = None
+
+    if not _settings:
+        # level： environ > .env file > default
+        _settings = Settings(_env_file=".env", _env_file_encoding="utf-8")
+
+    return _settings

bithuman/engine/__init__.py

@@ -0,0 +1,15 @@
+"""Pure Python engine replacing the C++ _bithuman_py extension."""
+
+import gc as _gc
+
+from .enums import CompressionType, LoadingMode
+
+RUNTIME_VERSION = "1.0.1"
+
+# Tune GC to reduce pause frequency during frame processing.
+# Default thresholds are (700, 10, 10). Raising gen0 to 50000 means
+# gen0 collections happen ~70x less often, reducing tail latency spikes.
+# Gen1/gen2 ratios stay at 10 (unchanged from default).
+_gc.set_threshold(50000, 10, 10)
+
+__all__ = ["CompressionType", "LoadingMode", "RUNTIME_VERSION"]

bithuman/engine/auth.py

@@ -0,0 +1,335 @@
+"""JWT validation, hardware fingerprinting, token refresh — replaces
+jwt.cpp, fingerprint.cpp, token_request.cpp, hash.cpp.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import platform
+import socket
+import sys
+import threading
+import time
+import uuid
+from datetime import datetime
+from typing import Dict, List, Optional, Set
+
+from loguru import logger
+
+from . import RUNTIME_VERSION
+
+# ---------------------------------------------------------------------------
+# The JWT signing secret — matches getSecretKey() in generator.cpp.
+# The C++ code uses compile-time XOR obfuscation, but the XOR+un-XOR
+# in getSecretKey() is a round-trip that returns the original plaintext.
+# ---------------------------------------------------------------------------
+_SECRET_KEY = "myxVr8FBASSOyPet2Jf2VQBbe3Pbeliv"
+
+
+# ---------------------------------------------------------------------------
+# Hashing — replaces hash.cpp
+# ---------------------------------------------------------------------------
+def calculate_md5(data: str) -> str:
+    """MD5 hex digest of a string. Matches hash.cpp calculateMD5."""
+    return hashlib.md5(data.encode("utf-8")).hexdigest()
+
+
+def calculate_sha256(data: str) -> str:
+    """SHA-256 hex digest of a string. Matches hash.cpp calculateSHA256."""
+    return hashlib.sha256(data.encode("utf-8")).hexdigest()
+
+
+def calculate_file_md5(file_path: str) -> str:
+    """MD5 hex digest of file contents. Matches hash.cpp calculateFileMD5."""
+    md5 = hashlib.md5()
+    try:
+        with open(file_path, "rb") as f:
+            while True:
+                chunk = f.read(8192)
+                if not chunk:
+                    break
+                md5.update(chunk)
+        return md5.hexdigest()
+    except OSError:
+        return ""
+
+
+# ---------------------------------------------------------------------------
+# Hardware fingerprint — replaces fingerprint.cpp
+# ---------------------------------------------------------------------------
+def generate_hardware_fingerprint() -> str:
+    """Generate MD5 hash of machine-id + hostname + timestamp.
+
+    Matches fingerprint.cpp generateHardwareFingerprint (lines 22-108):
+    Linux: /etc/machine-id + "-" + hostname + "-" + YYYYMMDDHHMM → MD5
+    macOS: hostname + "-" + platform UUID + "-" + YYYYMMDDHHMM → MD5
+    Windows: computer name + "-" + volume serial + "-" + YYYYMMDDHHMM → MD5
+    """
+    parts: List[str] = []
+
+    if sys.platform == "win32":
+        parts.append(platform.node())
+        # Volume serial approximation
+        parts.append(str(uuid.getnode()))
+    elif sys.platform == "darwin":
+        parts.append(socket.gethostname())
+        # macOS hardware UUID via ioreg
+        try:
+            import subprocess
+
+            result = subprocess.run(
+                ["ioreg", "-rd1", "-c", "IOPlatformExpertDevice"],
+                capture_output=True,
+                text=True,
+            )
+            for line in result.stdout.split("\n"):
+                if "IOPlatformUUID" in line:
+                    hw_uuid = line.split('"')[-2]
+                    parts.append(hw_uuid)
+                    break
+        except Exception:
+            pass
+    else:
+        # Linux
+        try:
+            with open("/etc/machine-id") as f:
+                parts.append(f.read().strip())
+        except FileNotFoundError:
+            pass
+        parts.append(socket.gethostname())
+
+    # Add timestamp (YYYYMMDDHHMM) — matches C++ strftime "%Y%m%d%H%M"
+    now = datetime.now()
+    parts.append(now.strftime("%Y%m%d%H%M"))
+
+    fingerprint = "-".join(parts)
+    return calculate_md5(fingerprint)
+
+
+# ---------------------------------------------------------------------------
+# JWT Validator — replaces JWTValidator class in jwt.cpp
+# ---------------------------------------------------------------------------
+class JWTValidator:
+    """Validates JWT tokens with HS256 signing.
+
+    Matches jwt.cpp JWTValidator class: verifies signature, extracts claims
+    (iss, sub, aud, exp, nbf, videos, models, instance_id, model_hash_type,
+    ref.runtime_model_hash), manages allowed hash sets.
+    """
+
+    def __init__(self, secret_key: str = _SECRET_KEY) -> None:
+        self._secret_key = secret_key
+        self._is_valid = False
+        self._expiration_time: Optional[float] = None  # Unix timestamp
+        self._issuer = ""
+        self._subject = ""
+        self._audience = ""
+        self._not_before: Optional[int] = None
+        self._allowed_video_hashes: Set[str] = set()
+        self._allowed_model_hashes: Set[str] = set()
+        self._use_raw_model_hash = False
+        self._instance_id = ""
+        self._runtime_model_hash = ""
+
+    def validate_token(self, token: str, verbose: bool = True) -> bool:
+        """Validate JWT token. Matches jwt.cpp validateToken (lines 29-456).
+
+        Uses PyJWT for HS256 verification with 300s leeway.
+        """
+        try:
+            import jwt as pyjwt
+
+            payload = pyjwt.decode(
+                token,
+                self._secret_key,
+                algorithms=["HS256"],
+                options={
+                    "verify_exp": True,
+                    "verify_aud": False,  # C++ jwt-cpp doesn't verify aud
+                    "require": ["exp"],
+                },
+                leeway=300,  # 5 min clock skew — matches C++
+            )
+
+            # Extract standard claims
+            self._issuer = payload.get("iss", "")
+            self._subject = payload.get("sub", "")
+            self._audience = payload.get("aud", "")
+            self._not_before = payload.get("nbf")
+
+            # Expiration
+            exp = payload.get("exp")
+            if exp is not None:
+                self._expiration_time = float(exp)
+                if time.time() > self._expiration_time:
+                    if verbose:
+                        logger.error("Token has expired!")
+                    return False
+
+            # Video hashes
+            videos = payload.get("videos", [])
+            if isinstance(videos, list):
+                self._allowed_video_hashes = {h for h in videos if isinstance(h, str)}
+
+            # Model hashes
+            models = payload.get("models", [])
+            if isinstance(models, list):
+                self._allowed_model_hashes = {h for h in models if isinstance(h, str)}
+
+            # Instance ID
+            self._instance_id = payload.get("instance_id", "")
+
+            # Model hash type
+            hash_type = payload.get("model_hash_type", "")
+            self._use_raw_model_hash = hash_type == "md5sum"
+
+            # Runtime model hash from ref claim
+            ref = payload.get("ref", {})
+            if isinstance(ref, dict):
+                self._runtime_model_hash = ref.get("runtime_model_hash", "")
+
+            self._is_valid = True
+            return True
+
+        except Exception as e:
+            if verbose:
+                logger.error(f"Token validation failed: {e}")
+            return False
+
+    def is_validated(self) -> bool:
+        return self._is_valid
+
+    def has_expired(self) -> bool:
+        """Check if token has expired. Matches jwt.cpp hasExpired."""
+        if not self._is_valid:
+            return True
+        if self._expiration_time is None:
+            return False
+        return time.time() > self._expiration_time
+
+    def get_expiration_time(self) -> Optional[float]:
+        if not self._is_valid or self._expiration_time is None:
+            return None
+        return self._expiration_time
+
+    def get_issuer(self) -> str:
+        return self._issuer
+
+    def get_audience(self) -> str:
+        return self._audience
+
+    def get_not_before(self) -> Optional[int]:
+        return self._not_before
+
+    def get_runtime_model_hash(self) -> str:
+        return self._runtime_model_hash
+
+    def is_video_hash_allowed(self, video_hash: str) -> bool:
+        """Matches jwt.cpp isVideoHashAllowed (lines 458-470)."""
+        if not self._is_valid:
+            return False
+        if not self._allowed_video_hashes:
+            return True
+        return video_hash in self._allowed_video_hashes
+
+    def is_model_hash_allowed(
+        self, raw_model_hash: str, encrypted_model_hash: str = ""
+    ) -> bool:
+        """Matches jwt.cpp isModelHashAllowed (lines 472-498)."""
+        if not self._is_valid:
+            return False
+        if not self._allowed_model_hashes:
+            return True
+        if self._use_raw_model_hash:
+            return raw_model_hash in self._allowed_model_hashes
+        hash_to_check = (
+            encrypted_model_hash if encrypted_model_hash else self.encrypt_model_hash(raw_model_hash)
+        )
+        return hash_to_check in self._allowed_model_hashes
+
+    def encrypt_model_hash(self, model_hash: str) -> str:
+        """Matches jwt.cpp encryptModelHash (lines 500-510):
+        SHA256(model_hash + SHA256(secret_key))
+        """
+        secret_hash = calculate_sha256(self._secret_key)
+        combined = model_hash + secret_hash
+        return calculate_sha256(combined)
+
+
+# ---------------------------------------------------------------------------
+# Token request — replaces token_request.cpp
+# ---------------------------------------------------------------------------
+def request_token(
+    api_url: str,
+    api_secret: str,
+    fingerprint: str,
+    runtime_model_hash: Optional[str] = None,
+    tags: Optional[str] = None,
+    transaction_id: Optional[str] = None,
+    insecure: bool = False,
+    timeout: float = 30.0,
+) -> str:
+    """HTTP POST to auth API, returns JWT token string.
+
+    Matches token_request.cpp requestToken (lines 26-134):
+    POST with JSON body and api-secret header.
+    """
+    import requests
+
+    body: Dict[str, str] = {
+        "fingerprint": fingerprint,
+        "runtime_version": RUNTIME_VERSION,
+    }
+    if runtime_model_hash:
+        body["runtime_model_hash"] = runtime_model_hash
+    if tags:
+        body["tags"] = tags
+    if transaction_id:
+        body["transaction_id"] = transaction_id
+
+    headers = {
+        "Content-Type": "application/json",
+        "api-secret": api_secret,
+    }
+
+    resp = requests.post(
+        api_url,
+        json=body,
+        headers=headers,
+        timeout=timeout,
+        verify=not insecure,
+    )
+
+    if resp.status_code == 200:
+        data = resp.json()
+        # API wraps response in {"data": {"token": ...}, "status": ...}
+        token = data.get("token", "") or data.get("data", {}).get("token", "")
+        if not token:
+            raise RuntimeError("Failed to parse token from response")
+        return token
+    elif resp.status_code == 402:
+        raise RuntimeError(
+            "Payment Required (402): Your quota has been exceeded or "
+            "payment is required. Runtime terminated."
+        )
+    elif resp.status_code == 403:
+        raise RuntimeError("Forbidden (403): Access denied. Runtime terminated.")
+    elif resp.status_code == 400:
+        raise RuntimeError(
+            "Bad Request (400): Runtime version is too old or invalid. "
+            "Please upgrade. Runtime terminated."
+        )
+    else:
+        raise RuntimeError(
+            f"Token request failed with status code: {resp.status_code}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# UUID generation — replaces jwt.cpp generateUUID
+# ---------------------------------------------------------------------------
+def generate_uuid() -> str:
+    """Generate a UUID4 string. Matches jwt.cpp generateUUID."""
+    return str(uuid.uuid4())