loqalkit 1.6.2__py3-none-any.whl

loqalkit/__init__.py

@@ -0,0 +1,197 @@
+"""LoqalKit — unified language SDK.
+
+Detection, transcription, romanization, translation, vocalization,
+bilingual segmentation, and script metadata. Single source of truth
+for all Loqal language tech.
+"""
+
+from .accent import (
+    ACCENT_MAPS,
+    has_accent_map,
+    map_accent,
+    map_accent_or_default,
+)
+from .bilingual import (
+    BILINGUAL_TAG_RE,
+    has_bilingual_markers,
+    parse_bilingual_segments,
+    resolve_marker,
+    strip_bilingual_markers,
+    tag_bilingual_async,
+)
+from .canonical import (
+    canonical,
+    coerce_detected_code,
+    contains_cantonese,
+    same_language,
+)
+from .codes import ALL_BCP47_CODES, LANG_TO_BCP47, normalize_bcp47
+from .detect import (
+    detect_language_from_audio,
+    detect_language_from_text,
+    detect_language_from_text_async,
+)
+from .disambiguate import (
+    DISAMBIGUATION_FAMILIES,
+    disambiguate_within_family,
+    disambiguate_within_family_async,
+    family_for_code,
+)
+from .furigana import (
+    align_reading_to_surface,
+    annotate as furigana_annotate,
+    contains_kanji,
+)
+from .prompts import LANG_NAMES, language_prompt
+from .refine import (
+    Refiner,
+    is_refinable,
+    refine_segments,
+    refine_text,
+)
+from .registry import (
+    LANGUAGE_LABELS,
+    LANGUAGES,
+    get_language,
+    has_grammatical_gender,
+    is_rtl,
+    language_label,
+    uses_non_latin_script,
+)
+from .romanize import (
+    LANG_ROMANIZATION_INSTRUCTIONS,
+    japanese_readings,
+    romanize_batch,
+    romanize_batch_as_dict,
+    romanize_batch_async,
+    romanize_segments,
+    romanize_segments_async,
+)
+from .script_info import (
+    supports_romanization,
+    uses_non_latin_script,
+)
+from .localize_document import (
+    CONTENT_TYPES as LOCALIZE_CONTENT_TYPES,
+    LocalizeResult,
+    SUPPORTED_EXTENSIONS as LOCALIZE_SUPPORTED_EXTENSIONS,
+    content_type_for as localize_content_type_for,
+    localize_document,
+    localize_document_sync,
+)
+from .gender import (
+    DisambiguationResult,
+    GenderInference,
+    Replacement,
+    disambiguate as gender_disambiguate,
+    detect_gendered_language,
+    has_arabic,
+    has_hebrew,
+    infer_gender,
+    preprocess_for_tts,
+)
+from .segment_turns import segment_turns
+from .segmentation import split_long_segments
+from .speakers import extract_speaker_info
+from .transcribe import transcribe, transcribe_async
+from .transcribe_document import (
+    DocumentCancelled,
+    transcribe_document,
+    transcribe_document_async,
+    transliterate_name,
+    transliterate_name_for_reader,
+)
+from .translate import (
+    LANGUAGE_NAMES,
+    translate_json_content,
+    translate_strings_to_locales,
+    translate_text,
+)
+from .sample_text import generate_sample_text
+from .tts import synthesize as tts_synthesize, synthesize_async as tts_synthesize_async
+from .vocalize import vocalize_batch
+
+__all__ = [
+    "ACCENT_MAPS",
+    "ALL_BCP47_CODES",
+    "BILINGUAL_TAG_RE",
+    "DISAMBIGUATION_FAMILIES",
+    "LANG_NAMES",
+    "LANG_ROMANIZATION_INSTRUCTIONS",
+    "LANG_TO_BCP47",
+    "LANGUAGES",
+    "LANGUAGE_LABELS",
+    "LANGUAGE_NAMES",
+    "LOCALIZE_CONTENT_TYPES",
+    "LOCALIZE_SUPPORTED_EXTENSIONS",
+    "LocalizeResult",
+    "uses_non_latin_script",
+    "DisambiguationResult",
+    "GenderInference",
+    "Replacement",
+    "align_reading_to_surface",
+    "canonical",
+    "coerce_detected_code",
+    "contains_cantonese",
+    "contains_kanji",
+    "furigana_annotate",
+    "detect_gendered_language",
+    "detect_language_from_audio",
+    "detect_language_from_text",
+    "detect_language_from_text_async",
+    "disambiguate_within_family",
+    "disambiguate_within_family_async",
+    "has_accent_map",
+    "extract_speaker_info",
+    "gender_disambiguate",
+    "generate_sample_text",
+    "family_for_code",
+    "get_language",
+    "has_arabic",
+    "has_grammatical_gender",
+    "has_hebrew",
+    "is_refinable",
+    "is_rtl",
+    "japanese_readings",
+    "language_label",
+    "language_prompt",
+    "localize_content_type_for",
+    "localize_document",
+    "localize_document_sync",
+    "map_accent",
+    "map_accent_or_default",
+    "infer_gender",
+    "normalize_bcp47",
+    "has_bilingual_markers",
+    "parse_bilingual_segments",
+    "resolve_marker",
+    "Refiner",
+    "refine_segments",
+    "refine_text",
+    "romanize_batch",
+    "romanize_batch_as_dict",
+    "romanize_batch_async",
+    "romanize_segments",
+    "romanize_segments_async",
+    "same_language",
+    "segment_turns",
+    "split_long_segments",
+    "strip_bilingual_markers",
+    "supports_romanization",
+    "tag_bilingual_async",
+    "transcribe",
+    "transcribe_async",
+    "DocumentCancelled",
+    "transcribe_document",
+    "transcribe_document_async",
+    "transliterate_name",
+    "transliterate_name_for_reader",
+    "preprocess_for_tts",
+    "translate_json_content",
+    "translate_strings_to_locales",
+    "translate_text",
+    "tts_synthesize",
+    "tts_synthesize_async",
+    "uses_non_latin_script",
+    "vocalize_batch",
+]

loqalkit/_client.py

@@ -0,0 +1,61 @@
+"""Internal OpenAI client accessor for loqalkit modules.
+
+Two paths:
+  1. When imported inside ManabuDashboard (`shared/` on sys.path), we
+     reuse the Dashboard's `shared.manabu_openai.get_client` so cost
+     tracking and the shared secret discovery stay consistent.
+  2. When imported standalone (pip install loqalkit), we fall back to a
+     self-contained client that reads MANABU_APP_SECRET from the
+     environment and points at the Cloudflare proxy.
+
+Every loqalkit module (detect, transcribe, romanize, translate, etc.)
+uses `from ._client import get_client` and the runtime decides which
+variant to use.
+"""
+
+from __future__ import annotations
+
+import os
+
+
+_PROXY_BASE_URL = os.environ.get(
+    "MANABU_PROXY_BASE_URL",
+    "https://manabu-openai-proxy.aviah.workers.dev",
+)
+
+
+def _fallback_get_client(*, cost_tracking: bool = True):
+    """Standalone client — used when shared.manabu_openai isn't available."""
+    try:
+        from openai import OpenAI
+    except ImportError as e:  # pragma: no cover
+        raise ImportError(
+            "openai package is required; install loqalkit with "
+            "its base deps or install 'openai>=1.0.0' manually."
+        ) from e
+
+    secret = os.environ.get("MANABU_APP_SECRET")
+    if not secret:
+        raise RuntimeError(
+            "MANABU_APP_SECRET is not set — loqalkit cannot reach the "
+            "OpenAI proxy. Set the env var on the sidecar deployment."
+        )
+
+    return OpenAI(
+        base_url=f"{_PROXY_BASE_URL}/v1",
+        api_key="proxied",
+        default_headers={"X-App-Secret": secret},
+    )
+
+
+try:
+    # Prefer Dashboard's implementation when available — keeps cost
+    # tracking + secret discovery consistent with the rest of the repo.
+    from shared.manabu_openai import get_client as _shared_get_client
+
+    def get_client(*, cost_tracking: bool = True):
+        return _shared_get_client(cost_tracking=cost_tracking)
+
+except ImportError:
+    def get_client(*, cost_tracking: bool = True):  # type: ignore[no-redef]
+        return _fallback_get_client(cost_tracking=cost_tracking)

loqalkit/accent.py

@@ -0,0 +1,86 @@
+"""Accent-to-BCP47 mapping for audio-based accent detection.
+
+Maps accent classifier output labels to TTS-supported BCP-47 codes.
+Each language that has an accent classifier gets its own mapping table.
+Labels that don't map to a TTS-supported code are folded into the nearest
+supported variant (e.g. "scotland" → "en-GB").
+
+Adding a new language:
+  1. Train or find an accent classifier (see LoqalKitGPU/app/asr/accent_classifier.py).
+  2. Add a mapping table to ACCENT_MAPS below.
+  3. Done — detect.py picks it up automatically.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+# Maps (iso_language, accent_label) → BCP-47 code that has TTS support.
+#
+# Structure: ACCENT_MAPS[iso639_3] = {model_label: bcp47_code}
+#
+# Only codes listed in extra_languages.txt / languages.txt should appear
+# as values — these are the codes the TTS pipeline can actually serve.
+# Labels that map to the same code are intentional folds (e.g. Scotland
+# and Wales fold into en-GB because we don't have separate TTS for them).
+
+ACCENT_MAPS: dict[str, dict[str, str]] = {
+    "eng": {
+        "us": "en-US",
+        "england": "en-GB",
+        "australia": "en-AU",
+        "indian": "en-IN",
+        "canada": "en-CA",
+        "ireland": "en-IE",
+        "newzealand": "en-NZ",
+        "philippines": "en-PH",
+        "singapore": "en-SG",
+        "hongkong": "en-HK",
+        "african": "en-ZA",
+        "scotland": "en-GB",
+        "wales": "en-GB",
+        "bermuda": "en-US",
+        "southatlandtic": "en-GB",
+        "malaysia": "en-SG",
+    },
+}
+
+# Default code per language when accent detection fails or returns unknown.
+ACCENT_DEFAULTS: dict[str, str] = {
+    "eng": "en-US",
+}
+
+
+def map_accent(iso_language: str, accent_label: str) -> Optional[str]:
+    """Map a model's accent label to a TTS-supported BCP-47 code.
+
+    Returns None if no mapping exists for this language or label.
+    """
+    lang_map = ACCENT_MAPS.get(iso_language)
+    if not lang_map:
+        return None
+    return lang_map.get(accent_label)
+
+
+def map_accent_or_default(iso_language: str, accent_label: str) -> Optional[str]:
+    """Like map_accent but falls back to the language default."""
+    code = map_accent(iso_language, accent_label)
+    if code:
+        return code
+    return ACCENT_DEFAULTS.get(iso_language)
+
+
+def has_accent_map(iso_language: str) -> bool:
+    """Whether we have accent mappings for this language."""
+    return iso_language in ACCENT_MAPS
+
+
+def supported_languages() -> list[str]:
+    """ISO 639-3 codes that have accent mapping tables."""
+    return sorted(ACCENT_MAPS.keys())
+
+
+def codes_for_language(iso_language: str) -> list[str]:
+    """All distinct BCP-47 codes reachable from a language's accent map."""
+    lang_map = ACCENT_MAPS.get(iso_language, {})
+    return sorted(set(lang_map.values()))

loqalkit/audio_chunk.py

@@ -0,0 +1,190 @@
+"""Input chunking for long/large audio, inside loqalkit core.
+
+gpt-4o-transcribe rejects any single request over 25 MB OR longer than ~1400 s
+("…is the maximum for this model"), and a long diarized clip can also overrun an
+upstream proxy timeout. So a long recording must be split *before* it reaches
+the model — and "long" means either trigger: a low-bitrate mobile recording can
+sail under the size cap while still blowing the duration cap. This is the single
+place that decision lives; every caller (mobile, the token server, omnisite's
+BFF, any future client) forwards whole audio and gets the same treatment without
+re-implementing it.
+
+We transcode with ffmpeg to speech-grade 16 kHz mono 32 kbps MP3 (~8× smaller,
+strips video), cut into fixed-duration chunks, transcribe each independently,
+and stitch the segments back with offset timecodes.
+
+`merge_chunk_transcripts` is pure (no ffmpeg, no I/O) so the offset/concat logic
+is unit-tested in isolation.
+"""
+
+import logging
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Inputs at or below this size keep the single-shot path (already < the 25 MB cap).
+CHUNK_THRESHOLD_BYTES = 24 * 1024 * 1024
+
+# gpt-4o-transcribe also rejects any single request whose audio runs longer than
+# 1400 s ("…is the maximum for this model"), INDEPENDENT of byte size. A low-bitrate
+# recording — e.g. 31 min of 32 kbps mobile AAC ≈ 7 MB — slips under the size cap
+# above yet still blows that duration cap, so length is a second, independent reason
+# to chunk. Kept with a safe margin below the 1400 s hard cap.
+MAX_SINGLE_SHOT_SECONDS = 1200  # 20 min (model hard cap is 1400 s)
+
+# 3 min @ 32 kbps mono ≈ 0.7 MB — far under the 25 MB cap, and short enough that
+# diarized transcription (~2× real-time, ≈90 s) stays well under upstream timeouts.
+SEGMENT_SECONDS = 180
+
+# Refuse absurd inputs rather than spawning hundreds of requests. 60 × 3 min = 3 h.
+MAX_CHUNKS = 60
+
+
+def needs_chunking(audio_path: Path) -> bool:
+    """True when the file needs transcode + segmentation before the model.
+
+    Two independent triggers — either is enough:
+      - size   > CHUNK_THRESHOLD_BYTES   (gpt-4o-transcribe's 25 MB per-request cap)
+      - length > MAX_SINGLE_SHOT_SECONDS (its ~1400 s per-request duration cap)
+
+    Checking size alone misses the "small but long" case: a low-bitrate mobile
+    recording (e.g. 31 min of 32 kbps AAC ≈ 7 MB) slips under the size cap yet
+    still blows the duration cap, and the model 400s with "audio duration … is
+    longer than 1400 seconds which is the maximum for this model".
+    """
+    try:
+        if os.path.getsize(audio_path) > CHUNK_THRESHOLD_BYTES:
+            return True
+    except OSError:
+        return False
+    # Only reached when under the size cap. The duration probe is a cheap header
+    # read; an unknown duration (0.0 — no header / ffprobe unavailable) degrades to
+    # size-only, leaving MAX_CHUNKS to bound truly pathological inputs. No regression.
+    return _audio_duration_seconds(audio_path) > MAX_SINGLE_SHOT_SECONDS
+
+
+def _audio_duration_seconds(audio_path: Path) -> float:
+    """Audio duration in seconds via ffprobe, or 0.0 if it can't be determined.
+
+    A header read (no decode), so it's cheap even for a large file. 0.0 means
+    "unknown" — the caller treats that as "no duration trigger". Mirrors
+    detect._audio_duration so duration is read the same way across the package.
+    """
+    try:
+        out = subprocess.run(
+            [
+                "ffprobe", "-v", "error", "-show_entries", "format=duration",
+                "-of", "default=nokey=1:noprint_wrappers=1", str(audio_path),
+            ],
+            capture_output=True, text=True, check=True,
+        )
+        return float(out.stdout.strip())
+    except Exception:  # noqa: BLE001 - missing/odd ffprobe → fall back to size-only
+        return 0.0
+
+
+class AudioChunk:
+    __slots__ = ("path", "offset_seconds")
+
+    def __init__(self, path: Path, offset_seconds: float):
+        self.path = path
+        self.offset_seconds = offset_seconds
+
+
+def chunk_audio(audio_path: Path) -> tuple[list[AudioChunk], "Path"]:
+    """Transcode + segment an audio/video file into speech-grade MP3 chunks.
+
+    Returns (chunks, work_dir). The caller MUST remove work_dir when done
+    (shutil.rmtree). Raises if the input would exceed MAX_CHUNKS or ffmpeg fails.
+    """
+    work_dir = Path(tempfile.mkdtemp(prefix="loqalkit-chunk-"))
+    out_pattern = str(work_dir / "chunk_%04d.mp3")
+    try:
+        subprocess.run(
+            [
+                "ffmpeg", "-hide_banner", "-loglevel", "error",
+                "-i", str(audio_path),
+                "-vn", "-ac", "1", "-ar", "16000",
+                "-c:a", "libmp3lame", "-b:a", "32k",
+                "-f", "segment", "-segment_time", str(SEGMENT_SECONDS),
+                "-reset_timestamps", "1",
+                out_pattern,
+            ],
+            capture_output=True,
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        shutil.rmtree(work_dir, ignore_errors=True)
+        stderr = (e.stderr or b"").decode("utf-8", "replace")[-2000:]
+        raise RuntimeError(f"ffmpeg segmentation failed: {stderr.strip()}") from e
+
+    files = sorted(
+        f for f in os.listdir(work_dir)
+        if re.fullmatch(r"chunk_\d{4}\.mp3", f)
+    )
+    if not files:
+        shutil.rmtree(work_dir, ignore_errors=True)
+        raise RuntimeError("ffmpeg produced no audio chunks (unsupported or empty input)")
+    if len(files) > MAX_CHUNKS:
+        shutil.rmtree(work_dir, ignore_errors=True)
+        raise RuntimeError(
+            f"Recording too long: {len(files)} chunks exceeds the {MAX_CHUNKS}-chunk "
+            f"limit (~{MAX_CHUNKS * SEGMENT_SECONDS // 3600} h)."
+        )
+
+    chunks = [
+        AudioChunk(work_dir / name, i * SEGMENT_SECONDS)
+        for i, name in enumerate(files)
+    ]
+    return chunks, work_dir
+
+
+def merge_chunk_transcripts(parts: list[dict]) -> dict:
+    """Stitch per-chunk transcribe() dicts into one. Pure — no ffmpeg, no I/O.
+
+    Each part is a transcribe() result plus an "offset_seconds" key. Text is
+    joined in chunk order; every segment's start/end is shifted by its chunk's
+    offset; speaker_genders maps are merged (earlier chunks win on conflict).
+
+    Acoustic speaker IDs ("Speaker 1"…) are per-chunk and NOT guaranteed to mean
+    the same person across chunk boundaries — acceptable for the single/few-
+    speaker long-recording case this path targets.
+    """
+    ordered = sorted(parts, key=lambda p: p.get("offset_seconds", 0))
+
+    text = "\n".join(
+        (p.get("text") or "").strip()
+        for p in ordered
+        if (p.get("text") or "").strip()
+    )
+
+    segments: list[dict] = []
+    for part in ordered:
+        offset = part.get("offset_seconds", 0)
+        for seg in part.get("segments") or []:
+            new_seg = dict(seg)
+            if isinstance(seg.get("start"), (int, float)):
+                new_seg["start"] = round(seg["start"] + offset, 2)
+            if isinstance(seg.get("end"), (int, float)):
+                new_seg["end"] = round(seg["end"] + offset, 2)
+            segments.append(new_seg)
+
+    speaker_genders: dict = {}
+    for part in ordered:
+        pg = part.get("speaker_genders") or {}
+        for k, v in pg.items():
+            speaker_genders.setdefault(k, v)
+
+    language = next((p.get("language") for p in ordered if p.get("language")), None)
+
+    return {
+        "text": text,
+        "segments": segments,
+        "speaker_genders": speaker_genders,
+        "language": language,
+    }