openfacefx 0.18.0__py3-none-any.whl

openfacefx/__init__.py

@@ -0,0 +1,156 @@
+"""OpenFaceFX -- an open-source facial/lip-sync animation pipeline.
+
+audio + text  ->  time-stamped phonemes  ->  visemes  ->  coarticulated
+animation curves  ->  export for your rig.
+"""
+
+from .g2p import G2P
+from .pronounce import (Pronouncer, Dictionary, read_dictionary, PronouncerHook,
+                       Tokenizer)
+from .phonemes import (IPA_ALIASES, SAMPA_ALIASES, ALPHABETS, to_ipa, from_ipa,
+                      to_sampa, from_sampa, from_alphabet)
+from .alignment import PhonemeSegment, NaiveAligner, load_mfa_textgrid
+from .coarticulation import (build_viseme_curves, CoartParams, STYLE_PRESETS,
+                             style_params)
+from .coart_jali import RULE_IDS as JALI_RULE_IDS, load_rules as load_jali_rules
+from .curves import FaceTrack, Channel, Keyframe, reduce_to_track
+from .postprocess import smooth_matrix, time_shift
+from .pipeline import (generate_from_alignment, generate_naive,
+                       naive_segments, wav_duration, derive_events)
+from .streaming import StreamingGenerator, frames_to_track
+from .energy import energy_envelope, generate_from_energy
+from .prosody import (ProsodyParams, ProsodyTrack, pitch_track,
+                      prosody_features, prosody_events, detect_events)
+from .gestures import (GestureParams, generate_gestures, gestures_from_wav,
+                       add_gestures_to_track, GESTURE_CHANNELS)
+from .events import (Event, Alternative, VariantGroup, Variants, EVENT_TYPES,
+                     resolve, choose, add_event, attach_events, read_events,
+                     validate_events)
+from .io_export import to_dict, write_json, write_csv, from_dict, read_json
+from .qa import summarize, normalize_transcript, cue_flags
+from .texttags import (parse_tagged_transcript, Tag, has_tags,
+                       resolve_tagged_segments, curve_channels, tag_events,
+                       build_curve_channel)
+from .ssml import (parse_ssml, looks_like_ssml, EMPHASIS_STRENGTH,
+                   BREAK_STRENGTH)
+from .edits import (EditsDoc, diff_edits, apply_edits, load_edits, save_edits,
+                    sample)
+from .emotion import (EmotionEnvelope, bake_emotion, load_envelope,
+                     save_envelope, va_to_pose, VA_TABLE, VA_EMOTION_CHANNELS)
+from .importers import (import_cues, detect_format, build_cue_track,
+                       RHUBARB_TO_VISEME, PRESTON_BLAIR_TO_VISEME)
+from .importers_csv import read_csv
+from .inspect import (inspect_track, validate_asset, validate_file,
+                     detect_kind)
+from .transforms import (retime, retime_to_duration, mirror, trim, concat,
+                        MIRROR_PAIRS, MIRROR_NEGATE)
+from .lod import (generate_lods, make_lod, lod_metadata, switching_table,
+                 LOD_DEFAULT_RDP, LOD_DEFAULT_FPS)
+from .budget import (channel_energy, rank_channels, keep_channels,
+                    keep_top_weight, budget_channels, budget_metadata)
+from .layers import (Layer, build_layers, flatten_layers, layers_to_dict,
+                    layers_from_dict)
+from .batch_manifest import read_manifest, manifest_jobs, COLUMN_ALIASES
+from .vo_audit import audit_delivery, audit_report_text
+from .trackdiff import diff_tracks, render_diff
+from .export_unity import write_unity_anim, NAMING_PRESETS
+from .export_unreal_notifies import write_unreal_notifies, notifies_to_dict
+from .export_live2d import write_live2d_motion, lipsync_param_ids
+from .export_godot import write_godot_anim
+from .export_gltf import write_gltf, build_gltf
+from .export_lip import (write_lip, lip_bytes, skyrim_mapping,
+                        SKYRIM_SLOT_MAP)
+from .export_cues import (
+    dominant_cues, write_rhubarb_tsv, write_rhubarb_xml, write_rhubarb_json,
+    write_moho_dat, write_pgo,
+)
+from .export_captions import (
+    write_captions, write_srt, write_vtt, srt_text, vtt_text, build_cues,
+    word_timings, format_timestamp, CaptionCue, parse_vtt,
+)
+from .mapping import Mapping, Target
+from .ipa import IPA_MAPPING, is_ipa_vowel, ipa_unknown_symbols
+from .retarget import (retarget, apply_adjust, rename_only, PRESETS,
+                       PRESET_FALLBACKS)
+from .visemes import VISEMES, phoneme_to_viseme
+from .timing import (
+    TimingEvent, resolve_ends, to_segments, parse_pho, parse_piper_alignments,
+    parse_cartesia, parse_azure_visemes, parse_polly_marks,
+    viseme_events_to_segments, build_vendor_mapping,
+    AZURE_VISEME_TO_TARGET, POLLY_VISEME_TO_TARGET,
+)
+from .anchors import (
+    Anchor, anchored_segments, anchors_transcript, parse_srt,
+    parse_word_anchors, from_azure_word_boundaries, from_elevenlabs_alignment,
+    from_kokoro_tokens, google_ssml_with_marks, from_google_timepoints,
+)
+from .aligners import (from_whisper_json, from_whisperx, from_gentle,
+                      from_gentle_phones)
+
+__version__ = "0.18.0"
+
+__all__ = [
+    "G2P", "PhonemeSegment", "NaiveAligner", "load_mfa_textgrid",
+    "Pronouncer", "Dictionary", "read_dictionary", "PronouncerHook", "Tokenizer",
+    "IPA_ALIASES", "SAMPA_ALIASES", "ALPHABETS", "to_ipa", "from_ipa",
+    "to_sampa", "from_sampa", "from_alphabet",
+    "build_viseme_curves", "CoartParams", "STYLE_PRESETS", "style_params",
+    "JALI_RULE_IDS", "load_jali_rules",
+    "FaceTrack", "Channel", "Keyframe",
+    "reduce_to_track", "smooth_matrix", "time_shift",
+    "generate_from_alignment", "generate_naive",
+    "naive_segments", "wav_duration", "derive_events",
+    "StreamingGenerator", "frames_to_track",
+    "energy_envelope", "generate_from_energy",
+    "ProsodyParams", "ProsodyTrack", "pitch_track", "prosody_features",
+    "prosody_events", "detect_events",
+    "GestureParams", "generate_gestures", "gestures_from_wav",
+    "add_gestures_to_track", "GESTURE_CHANNELS",
+    "Event", "Alternative", "VariantGroup", "Variants", "EVENT_TYPES",
+    "resolve", "choose", "add_event", "attach_events", "read_events",
+    "validate_events",
+    "to_dict", "write_json", "write_csv", "from_dict", "read_json",
+    "summarize", "normalize_transcript", "cue_flags",
+    "parse_tagged_transcript", "Tag", "has_tags", "resolve_tagged_segments",
+    "curve_channels", "tag_events", "build_curve_channel",
+    "parse_ssml", "looks_like_ssml", "EMPHASIS_STRENGTH", "BREAK_STRENGTH",
+    "EditsDoc", "diff_edits", "apply_edits", "load_edits", "save_edits", "sample",
+    "EmotionEnvelope", "bake_emotion", "load_envelope", "save_envelope",
+    "va_to_pose", "VA_TABLE", "VA_EMOTION_CHANNELS",
+    "import_cues", "detect_format", "build_cue_track",
+    "RHUBARB_TO_VISEME", "PRESTON_BLAIR_TO_VISEME", "read_csv",
+    "inspect_track", "validate_asset", "validate_file", "detect_kind",
+    "retime", "retime_to_duration", "mirror", "trim", "concat",
+    "MIRROR_PAIRS", "MIRROR_NEGATE",
+    "generate_lods", "make_lod", "lod_metadata", "switching_table",
+    "LOD_DEFAULT_RDP", "LOD_DEFAULT_FPS",
+    "channel_energy", "rank_channels", "keep_channels", "keep_top_weight",
+    "budget_channels", "budget_metadata",
+    "Layer", "build_layers", "flatten_layers", "layers_to_dict",
+    "layers_from_dict",
+    "read_manifest", "manifest_jobs", "COLUMN_ALIASES",
+    "audit_delivery", "audit_report_text",
+    "diff_tracks", "render_diff",
+    "write_unity_anim", "NAMING_PRESETS",
+    "write_unreal_notifies", "notifies_to_dict",
+    "write_live2d_motion", "lipsync_param_ids", "write_godot_anim",
+    "write_gltf", "build_gltf",
+    "write_lip", "lip_bytes", "skyrim_mapping", "SKYRIM_SLOT_MAP",
+    "dominant_cues", "write_rhubarb_tsv", "write_rhubarb_xml",
+    "write_rhubarb_json", "write_moho_dat", "write_pgo",
+    "write_captions", "write_srt", "write_vtt", "srt_text", "vtt_text",
+    "build_cues", "word_timings", "format_timestamp", "CaptionCue", "parse_vtt",
+    "Mapping", "Target",
+    "IPA_MAPPING", "is_ipa_vowel", "ipa_unknown_symbols",
+    "retarget", "apply_adjust", "rename_only", "PRESETS", "PRESET_FALLBACKS",
+    "VISEMES", "phoneme_to_viseme",
+    "TimingEvent", "resolve_ends", "to_segments", "parse_pho",
+    "parse_piper_alignments", "parse_cartesia", "parse_azure_visemes",
+    "parse_polly_marks", "viseme_events_to_segments", "build_vendor_mapping",
+    "AZURE_VISEME_TO_TARGET", "POLLY_VISEME_TO_TARGET",
+    "Anchor", "anchored_segments", "anchors_transcript", "parse_srt",
+    "parse_word_anchors", "from_azure_word_boundaries",
+    "from_elevenlabs_alignment", "from_kokoro_tokens", "google_ssml_with_marks",
+    "from_google_timepoints",
+    "from_whisper_json", "from_whisperx", "from_gentle", "from_gentle_phones",
+]

openfacefx/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+import sys
+sys.exit(main())

openfacefx/aligners.py

@@ -0,0 +1,167 @@
+"""First-class adapters for the free, open-source aligners / ASR (issue #54).
+
+OpenFaceFX ships timing adapters for every commercial TTS source (Azure,
+ElevenLabs, Kokoro, Google, Piper, Cartesia, Polly) but used to punt the
+open-source tools — Whisper, WhisperX, Gentle — to the user with a "write a
+~15-line adapter" note in three places. These are those adapters: siblings of
+:func:`openfacefx.anchors.from_azure_word_boundaries`, stdlib ``json`` only,
+returning the normalized :class:`~openfacefx.anchors.Anchor` list (or, for
+Gentle's phone timings, :class:`~openfacefx.alignment.PhonemeSegment` s directly).
+
+  * :func:`from_whisper_json` — OpenAI Whisper ``verbose_json`` word timestamps
+    (``segments[].words[]`` or a top-level ``words[]``). **Tolerant** of the key
+    variance across openai-whisper / faster-whisper / whisper.cpp (``word`` vs
+    ``text``, ``probability`` vs ``score``).
+  * :func:`from_whisperx` — WhisperX ``segments[].words[]`` (``word``/``start``/
+    ``end``/``score``).
+  * :func:`from_gentle` — Gentle ``words[]`` (word-level anchors); and
+    :func:`from_gentle_phones` — Gentle's per-word ``phones[]`` (relative
+    ``duration`` s, ARPAbet with ``_B``/``_I``/``_E``/``_S`` position suffixes)
+    accumulated from the word start into phone-level segments, a path that skips
+    the naive spacer entirely.
+
+**Deterministic missing-timestamp rule:** aligners leave some words unaligned
+(Whisper omits timestamps, Gentle marks ``case != "success"``). Such a word is
+**dropped** — its neighbours' anchors still pin the timeline and the aligner
+spreads the gap. A phone/word symbol outside the ARPAbet inventory passes through
+and falls to ``sil`` at the viseme stage (documented), never a crash.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import List
+
+from .alignment import PhonemeSegment
+from .anchors import Anchor, _first, _json, _num, _obj_list
+from .phonemes import SILENCE
+
+_HAS_WORD_CHAR = re.compile(r"\w", re.UNICODE)      # skip pure-punctuation tokens
+
+
+def _word_anchors(words, who: str) -> List[Anchor]:
+    """Shared word-list -> ``Anchor`` core for Whisper / WhisperX. A word with a
+    missing/null ``start`` (the aligner left it unaligned) or that is
+    punctuation-only is dropped; ``end`` may be ``None`` (inferred from the next
+    anchor). Tolerant of ``word``/``text`` keys."""
+    out: List[Anchor] = []
+    for i, w in enumerate(words):
+        if not isinstance(w, dict):
+            raise ValueError(f"{who}: word {i} is not an object")
+        text = _first(w, ("word", "text"))
+        if not isinstance(text, str):
+            raise ValueError(f"{who}: word {i} needs a 'word'/'text' string")
+        text = text.strip()
+        start = _first(w, ("start", "start_time"))
+        if start is None or not _HAS_WORD_CHAR.search(text):
+            continue                              # unaligned / punctuation -> drop
+        end = _first(w, ("end", "end_time"))
+        out.append(Anchor(text, _num(start, f"{who} word {i} 'start'"),
+                          None if end is None else _num(end, f"{who} word {i} 'end'")))
+    if not out:
+        raise ValueError(f"{who}: no aligned words found")
+    return out
+
+
+def _segment_words(segments, who: str) -> list:
+    return [w for seg in segments if isinstance(seg, dict)
+            for w in (seg.get("words") or [])]
+
+
+def from_whisper_json(json_text: str) -> List[Anchor]:
+    """OpenAI Whisper ``verbose_json`` (``word_timestamps=True``) -> word anchors.
+
+    Accepts the ``{"segments": [{"words": [...]}]}`` shape and the flat
+    ``{"words": [...]}`` / bare-array shapes different wrappers emit, and the
+    ``word``/``text`` + ``probability``/``score`` key variance across
+    openai-whisper, faster-whisper and whisper.cpp. Words Whisper left without a
+    timestamp are dropped."""
+    d = _json(json_text, "whisper")
+    if isinstance(d, dict) and isinstance(d.get("segments"), list):
+        words = _segment_words(d["segments"], "whisper")
+    elif isinstance(d, dict) and isinstance(d.get("words"), list):
+        words = d["words"]
+    elif isinstance(d, list):
+        words = d
+    else:
+        raise ValueError("whisper: expected 'segments[].words[]', a 'words[]' "
+                         "array, or a bare word array")
+    return _word_anchors(words, "whisper")
+
+
+def from_whisperx(json_text: str) -> List[Anchor]:
+    """WhisperX alignment (``segments[].words[]`` with ``word``/``start``/``end``/
+    ``score``) -> word anchors. Words WhisperX could not align (no ``start``) are
+    dropped, the same deterministic rule as :func:`from_whisper_json`."""
+    d = _json(json_text, "whisperx")
+    if not (isinstance(d, dict) and isinstance(d.get("segments"), list)):
+        raise ValueError("whisperx: expected a 'segments' array")
+    return _word_anchors(_segment_words(d["segments"], "whisperx"), "whisperx")
+
+
+def from_gentle(json_text: str) -> List[Anchor]:
+    """Gentle forced-aligner JSON (``words[]``) -> word anchors. Only
+    ``case == "success"`` words are kept; ``not-found-in-audio`` /
+    ``not-found-in-transcript`` words fall out as gaps."""
+    items = _obj_list(_json(json_text, "gentle"), ("words",), "gentle")
+    out: List[Anchor] = []
+    for i, w in enumerate(items):
+        if not isinstance(w, dict):
+            raise ValueError(f"gentle: word {i} is not an object")
+        if w.get("case") != "success":
+            continue
+        text = _first(w, ("alignedWord", "word"))
+        start = _first(w, ("start",))
+        if not isinstance(text, str) or start is None:
+            continue
+        end = _first(w, ("end",))
+        out.append(Anchor(text, _num(start, f"gentle word {i} 'start'"),
+                          None if end is None else _num(end, f"gentle word {i} 'end'")))
+    if not out:
+        raise ValueError("gentle: no successfully-aligned words found")
+    return out
+
+
+_GENTLE_SUFFIX = re.compile(r"_[BIES]$")
+
+
+def _gentle_phone(symbol: str) -> str:
+    """Gentle phone token -> internal ARPAbet: strip the ``_B``/``_I``/``_E``/
+    ``_S`` position suffix and upper-case (``hh_B`` -> ``HH``, ``ow1_E`` ->
+    ``OW1``); Gentle's silence / out-of-vocab tokens map to ``sil``."""
+    ph = _GENTLE_SUFFIX.sub("", symbol).upper()
+    return SILENCE if ph in ("SIL", "OOV", "SP", "") else ph
+
+
+def from_gentle_phones(json_text: str) -> List[PhonemeSegment]:
+    """Gentle ``words[].phones[]`` -> phone-level :class:`PhonemeSegment` s.
+
+    Each successful word's phones carry a relative ``duration``; accumulating
+    them from the word's ``start`` gives absolute phone times (so the last phone
+    ends at the word span within float tolerance). Gaps between successful words
+    become silence. This is the accurate phone path — it skips the naive spacer
+    and feeds ``generate_from_alignment`` directly."""
+    items = _obj_list(_json(json_text, "gentle-phones"), ("words",), "gentle-phones")
+    segs: List[PhonemeSegment] = []
+    cursor = 0.0
+    for i, w in enumerate(items):
+        if not isinstance(w, dict) or w.get("case") != "success":
+            continue
+        start = _first(w, ("start",))
+        phones = w.get("phones")
+        if start is None or not isinstance(phones, list) or not phones:
+            continue
+        t = _num(start, f"gentle-phones word {i} 'start'")
+        if t > cursor + 1e-9:                     # silence gap before this word
+            segs.append(PhonemeSegment(SILENCE, cursor, t))
+        for j, p in enumerate(phones):
+            if not isinstance(p, dict):
+                raise ValueError(f"gentle-phones: word {i} phone {j} is not an object")
+            dur = _num(_first(p, ("duration",)), f"gentle-phones {i}.{j} 'duration'")
+            ph = _gentle_phone(_first(p, ("phone", "phoneme")) or "")
+            segs.append(PhonemeSegment(ph, t, t + dur))
+            t += dur
+        cursor = t
+    if not segs:
+        raise ValueError("gentle-phones: no aligned phones found")
+    return segs

openfacefx/alignment.py

@@ -0,0 +1,131 @@
+"""Forced alignment: producing time-stamped phonemes.
+
+This is the model-heavy stage of the pipeline. Rather than reinvent a speech
+recogniser, OpenFaceFX defines a small ``PhonemeSegment`` contract and provides:
+
+  * ``NaiveAligner`` -- no acoustic model. Given the phonemes for an utterance
+    and the utterance duration (or word timings), it distributes phonemes over
+    time using per-phoneme duration priors. Good enough to see the pipeline end
+    to end; not accurate lip-sync.
+
+  * ``load_mfa_textgrid`` -- parse the output of the Montreal Forced Aligner
+    (the recommended production aligner). MFA gives real acoustic alignment.
+
+Built-in adapters for the open-source aligners live in
+:mod:`openfacefx.aligners` (Whisper / WhisperX word timings, Gentle word **and**
+phone timings, issue #54); any other source just needs to produce a list of
+``PhonemeSegment`` and the rest of the pipeline is unchanged.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import List, Optional
+
+from .phonemes import strip_stress, is_vowel, SILENCE
+
+
+@dataclass
+class PhonemeSegment:
+    phoneme: str        # ARPAbet, may carry stress digit
+    start: float        # seconds
+    end: float          # seconds
+    confidence: Optional[float] = None   # aligner score in [0,1], if provided
+
+    @property
+    def dur(self) -> float:
+        return self.end - self.start
+
+
+def dump_segments(segments: List["PhonemeSegment"]) -> List[dict]:
+    """Serialise phoneme segments to the plain-JSON shape the HTML previewer
+    consumes (``tools/build_preview.py --segments``): a list of
+    ``{"phoneme", "start", "end"[, "confidence"]}``. ``confidence`` is emitted
+    only when the aligner supplied one."""
+    out = []
+    for s in segments:
+        d = {"phoneme": s.phoneme, "start": round(s.start, 4),
+             "end": round(s.end, 4)}
+        if s.confidence is not None:
+            d["confidence"] = round(s.confidence, 4)
+        out.append(d)
+    return out
+
+
+# Rough relative-duration priors (unitless). Vowels last longer than stops.
+_DUR_PRIOR = {
+    "AA": 1.6, "AE": 1.5, "AH": 1.0, "AO": 1.6, "AW": 1.8, "AY": 1.8,
+    "EH": 1.3, "ER": 1.5, "EY": 1.6, "IH": 1.1, "IY": 1.4, "OW": 1.6,
+    "OY": 1.8, "UH": 1.2, "UW": 1.5,
+    "CH": 1.1, "JH": 1.0, "SH": 1.1, "ZH": 1.0, "S": 1.1, "Z": 1.0,
+    "F": 1.0, "V": 0.9, "TH": 1.0, "DH": 0.8, "HH": 0.7,
+    "P": 0.6, "B": 0.6, "T": 0.6, "D": 0.6, "K": 0.7, "G": 0.7,
+    "M": 0.9, "N": 0.9, "NG": 0.9, "L": 0.9, "R": 1.0, "W": 0.8, "Y": 0.8,
+    SILENCE: 1.0,
+}
+
+
+def _prior(ph: str) -> float:
+    return _DUR_PRIOR.get(strip_stress(ph).upper(), 1.0)
+
+
+class NaiveAligner:
+    """Distribute phonemes across a time span using duration priors."""
+
+    def align(
+        self,
+        phonemes: List[str],
+        total_duration: float,
+        start: float = 0.0,
+    ) -> List[PhonemeSegment]:
+        if not phonemes:
+            return []
+        weights = [_prior(p) for p in phonemes]
+        wsum = sum(weights) or 1.0
+        segs: List[PhonemeSegment] = []
+        t = start
+        for p, w in zip(phonemes, weights):
+            d = total_duration * (w / wsum)
+            segs.append(PhonemeSegment(p, t, t + d))
+            t += d
+        return segs
+
+    def align_words(
+        self,
+        words: List[tuple],  # (phoneme_list, word_start, word_end)
+    ) -> List[PhonemeSegment]:
+        """When you already have word-level timings (common from ASR), align
+        phonemes within each word span. Much better than utterance-level."""
+        segs: List[PhonemeSegment] = []
+        for phones, ws, we in words:
+            segs.extend(self.align(phones, we - ws, start=ws))
+        return segs
+
+
+def load_mfa_textgrid(path: str, tier: str = "phones") -> List[PhonemeSegment]:
+    """Parse a Praat TextGrid produced by the Montreal Forced Aligner.
+
+    Only the interval tier named ``tier`` is read. Empty / silence intervals
+    become the ``sil`` phoneme so the mouth relaxes between words.
+    """
+    text = open(path, "r", encoding="utf-8").read()
+    # Find the requested interval tier block.
+    tier_pat = re.compile(
+        r'name = "%s".*?intervals: size = \d+(.*?)(?:item \[\d+\]:|$)' % re.escape(tier),
+        re.DOTALL,
+    )
+    m = tier_pat.search(text)
+    if not m:
+        raise ValueError(f"tier {tier!r} not found in TextGrid")
+    block = m.group(1)
+    seg_pat = re.compile(
+        r"xmin = ([\d.]+)\s*xmax = ([\d.]+)\s*text = \"([^\"]*)\"",
+        re.DOTALL,
+    )
+    segs: List[PhonemeSegment] = []
+    for xmin, xmax, label in seg_pat.findall(block):
+        label = label.strip()
+        ph = label if label else SILENCE
+        segs.append(PhonemeSegment(ph, float(xmin), float(xmax)))
+    return segs