utterplan 0.1.1__py3-none-any.whl

utterplan/format.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from .model import FORMAT, SCHEMA_VERSION, UtterancePlan
+
+SCHEMA_PATH = Path(__file__).resolve().parent.parent / "spec" / "utterplan.schema.json"
+PACKAGE_SCHEMA_PATH = Path(__file__).resolve().parent / "utterplan.schema.json"
+
+
+def validate_json(value: Any) -> UtterancePlan:
+    return UtterancePlan.from_dict(value)
+
+
+def validate_file(path: str | Path) -> UtterancePlan:
+    return UtterancePlan.load(path)
+
+
+def schema() -> dict[str, Any]:
+    path = SCHEMA_PATH if SCHEMA_PATH.exists() else PACKAGE_SCHEMA_PATH
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+__all__ = [
+    "FORMAT",
+    "SCHEMA_VERSION",
+    "SCHEMA_PATH",
+    "PACKAGE_SCHEMA_PATH",
+    "validate_json",
+    "validate_file",
+    "schema",
+]

utterplan/hashing.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+
+def canonical_json(value: Any) -> str:
+    return json.dumps(
+        value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False
+    )
+
+
+def semantic_hash(value: Any) -> str:
+    return "sha256:" + hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
+
+
+def unit_hash_payload(unit: Any) -> dict[str, Any]:
+    marker_values: Any = getattr(unit, "marker_values", None)
+    if marker_values is None:
+        marker_values = getattr(unit, "marker_ids", ())
+    markers = []
+    for marker in marker_values:
+        if hasattr(marker, "to_dict"):
+            value = marker.to_dict()
+            value.pop("id", None)
+            markers.append(value)
+        else:
+            markers.append(marker)
+    return {
+        "hash_schema": "utterplan-unit-v1",
+        "segments": [
+            {
+                "text": segment.text,
+                "language": segment.language,
+                "directives": segment.directives.to_dict(),
+                "pause_before": segment.pause_before.to_dict(),
+                "pause_after": segment.pause_after.to_dict(),
+            }
+            for segment in unit.segments
+        ],
+        "markers": markers,
+    }

utterplan/language.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+from .exceptions import LanguagePlanError
+
+
+@dataclass(frozen=True, slots=True)
+class LanguageRun:
+    id: str
+    spoken_start: int
+    spoken_end: int
+    language: str
+    source: str = "document-default"
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "id": self.id,
+            "spoken_start": self.spoken_start,
+            "spoken_end": self.spoken_end,
+            "language": self.language,
+            "source": self.source,
+        }
+
+
+def normalize_language(language: str, aliases: dict[str, str] | None = None) -> str:
+    if not isinstance(language, str):
+        raise LanguagePlanError("language must be a string")
+    value = language.strip().lower().replace("_", "-")
+    if not value:
+        raise LanguagePlanError("language must not be empty")
+    alias_map = {key.lower().replace("_", "-"): val for key, val in (aliases or {}).items()}
+    return alias_map.get(value, value).strip().lower().replace("_", "-")
+
+
+def build_language_runs(
+    text: str,
+    spans: Iterable[tuple[int, int, str, str]],
+    default_language: str,
+    aliases: dict[str, str] | None = None,
+) -> tuple[LanguageRun, ...]:
+    default = normalize_language(default_language, aliases)
+    explicit = [
+        (start, end, normalize_language(lang, aliases), source)
+        for start, end, lang, source in spans
+    ]
+    for start, end, _lang, _source in explicit:
+        if not (0 <= start < end <= len(text)):
+            raise LanguagePlanError(f"language span {start}:{end} is outside spoken text")
+    for index, left in enumerate(explicit):
+        for right in explicit[index + 1 :]:
+            if left[2] == right[2] or not (left[0] < right[1] and right[0] < left[1]):
+                continue
+            nested = (left[0] <= right[0] and right[1] <= left[1]) or (
+                right[0] <= left[0] and left[1] <= right[1]
+            )
+            if not nested:
+                raise LanguagePlanError(
+                    f"Conflicting crossing language spans: {left[0]}:{left[1]} and {right[0]}:{right[1]}"
+                )
+    positions = sorted({0, len(text), *(point for span in explicit for point in span[:2])})
+    runs: list[LanguageRun] = []
+    for start, end in zip(positions, positions[1:], strict=False):
+        if start == end:
+            continue
+        covering = [span for span in explicit if span[0] <= start and end <= span[1]]
+        chosen = min(covering, key=lambda span: (span[1] - span[0], -span[0])) if covering else None
+        language = chosen[2] if chosen else default
+        source = chosen[3] if chosen else "document-default"
+        if runs and runs[-1].language == language and runs[-1].spoken_end == start:
+            runs[-1] = LanguageRun(runs[-1].id, runs[-1].spoken_start, end, language, source)
+        else:
+            runs.append(LanguageRun(f"lang-{len(runs)}", start, end, language, source))
+    return tuple(runs)
+
+
+def spans_from_annotations(annotations: Sequence[Any]) -> list[tuple[int, int, str, str]]:
+    result: list[tuple[int, int, str, str]] = []
+    for item in annotations:
+        attrs = getattr(item, "attrs", {})
+        language = attrs.get("lang") or attrs.get("language")
+        if not language or str(attrs.get("scope", "")).lower() in {"pronunciation", "phoneme"}:
+            continue
+        start = getattr(item, "spoken_start", None)
+        end = getattr(item, "spoken_end", None)
+        if start is None or end is None:
+            start = getattr(item, "structural_start", None)
+            end = getattr(item, "structural_end", None)
+        if start is not None and end is not None:
+            result.append((int(start), int(end), str(language), "explicit-span"))
+    return result

utterplan/linguistics.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from .config import LinguisticsConfig
+from .language import LanguageRun
+from .model import TokenAnnotation
+from .spacy_models import resolve_spacy_model
+
+
+@dataclass(frozen=True, slots=True)
+class RunAnalysis:
+    """Request-local linguistic state. Never serialize this object."""
+
+    language: str
+    start: int
+    end: int
+    tokens: tuple[TokenAnnotation, ...]
+    provider_doc: object | None = None
+    model_name: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class LinguisticAnalysis:
+    language: str
+    text: str
+    tokens: tuple[TokenAnnotation, ...]
+    model_name: str | None = None
+    provider_doc: object | None = None
+
+
+class LinguisticResourcePool:
+    """Cache local spaCy pipelines without downloading a model."""
+
+    def __init__(self) -> None:
+        self._pipelines: dict[str, Any] = {}
+
+    def pipeline(self, model: str | None, *, require: bool = False) -> Any | None:
+        if not model:
+            return None
+        if model in self._pipelines:
+            return self._pipelines[model]
+        try:
+            import spacy
+
+            pipeline = spacy.load(model)
+        except (ImportError, OSError, ValueError) as exc:
+            if require:
+                raise RuntimeError(f"Requested spaCy model {model!r} is unavailable") from exc
+            return None
+        self._pipelines[model] = pipeline
+        return pipeline
+
+    def clear(self) -> None:
+        self._pipelines.clear()
+
+    def _select_model(self, language: str, config: LinguisticsConfig) -> str | None:
+        if config.spacy_model:
+            return config.spacy_model
+        if config.spacy_model_size:
+            return resolve_spacy_model(language, config.spacy_model_size)
+        if config.use_spacy is False:
+            return None
+        if config.use_spacy is not True:
+            return None
+        try:
+            import spacy
+
+            installed = set(spacy.util.get_installed_models())
+        except ImportError:
+            return None
+        base = language.split("-", 1)[0].lower()
+        families = (f"{base}_core_web_", f"{base}_core_news_")
+        for suffix in ("trf", "lg", "md", "sm"):
+            for family in families:
+                candidate = family + suffix
+                if candidate in installed:
+                    return candidate
+        return None
+
+    def analyze(self, text: str, run: LanguageRun, config: LinguisticsConfig) -> LinguisticAnalysis:
+        model = self._select_model(run.language, config)
+        use_spacy = config.use_spacy if config.use_spacy is not None else model is not None
+        if use_spacy:
+            pipeline = self.pipeline(model, require=config.require_spacy)
+            if pipeline is not None:
+                doc = pipeline(text)
+                return LinguisticAnalysis(
+                    run.language,
+                    text,
+                    tuple(
+                        TokenAnnotation(
+                            int(token.idx),
+                            int(token.idx + len(token.text)),
+                            str(token.text),
+                            getattr(token, "pos_", None) or None,
+                            getattr(token, "tag_", None) or None,
+                            getattr(token, "lemma_", None) or None,
+                            run.language,
+                            f"token-{i}",
+                        )
+                        for i, token in enumerate(doc)
+                        if token.text
+                    ),
+                    model,
+                    doc,
+                )
+            if config.require_spacy:
+                raise RuntimeError("spaCy is required but no local model is available")
+        return LinguisticAnalysis(
+            run.language, text, _fallback_tokens(text, run.language), None, None
+        )
+
+
+def _fallback_tokens(text: str, language: str) -> tuple[TokenAnnotation, ...]:
+    return tuple(
+        TokenAnnotation(
+            match.start(),
+            match.end(),
+            match.group(0),
+            None,
+            None,
+            match.group(0).lower(),
+            language,
+            f"token-{i}",
+        )
+        for i, match in enumerate(re.finditer(r"\S+", text))
+    )
+
+
+def analyze_run_analyses(
+    text: str,
+    runs: tuple[LanguageRun, ...],
+    config: LinguisticsConfig,
+    pool: LinguisticResourcePool,
+) -> tuple[RunAnalysis, ...]:
+    analyses: list[RunAnalysis] = []
+    for run in runs:
+        local = text[run.spoken_start : run.spoken_end]
+        analysis = pool.analyze(local, run, config)
+        tokens = tuple(
+            TokenAnnotation(
+                token.spoken_start + run.spoken_start,
+                token.spoken_end + run.spoken_start,
+                token.text,
+                token.pos,
+                token.tag,
+                token.lemma,
+                token.language,
+                f"token-{sum(len(item.tokens) for item in analyses) + i}",
+            )
+            for i, token in enumerate(analysis.tokens)
+        )
+        analyses.append(
+            RunAnalysis(
+                run.language,
+                run.spoken_start,
+                run.spoken_end,
+                tokens,
+                analysis.provider_doc,
+                analysis.model_name,
+            )
+        )
+    return tuple(analyses)
+
+
+def analyze_runs(
+    text: str,
+    runs: tuple[LanguageRun, ...],
+    config: LinguisticsConfig,
+    pool: LinguisticResourcePool,
+) -> tuple[TokenAnnotation, ...]:
+    return tuple(
+        token
+        for analysis in analyze_run_analyses(text, runs, config, pool)
+        for token in analysis.tokens
+    )