utterplan 0.1.1__py3-none-any.whl

utterplan/serialization.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from .model import UtterancePlan
+
+
+def to_dict(plan: UtterancePlan) -> dict[str, Any]:
+    return plan.to_dict()
+
+
+def from_dict(data: Mapping[str, Any]) -> UtterancePlan:
+    return UtterancePlan.from_dict(data)
+
+
+def to_json(plan: UtterancePlan, *, indent: int | None = 2) -> str:
+    return plan.to_json(indent=indent)
+
+
+def from_json(value: str) -> UtterancePlan:
+    return UtterancePlan.from_json(value)

utterplan/spacy_models.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from typing import Literal
+
+SpacyModelSize = Literal["sm", "md", "lg", "trf"]
+
+
+def normalize_spacy_language(language: str | None) -> str:
+    value = (language or "en").strip().lower().replace("_", "-")
+    return value.split("-", 1)[0] or "en"
+
+
+def resolve_spacy_model(language: str, size: SpacyModelSize = "sm") -> str:
+    if size not in {"sm", "md", "lg", "trf"}:
+        raise ValueError("spacy model size must be one of sm, md, lg, trf")
+    base = normalize_spacy_language(language)
+    family = "web" if base in {"en", "zh"} else "news"
+    return f"{base}_core_{family}_{size}"
+
+
+__all__ = ["SpacyModelSize", "normalize_spacy_language", "resolve_spacy_model"]

utterplan/spans.py

@@ -0,0 +1,3 @@
+from .model import AnnotationSpan, BoundaryEvent
+
+__all__ = ["AnnotationSpan", "BoundaryEvent"]

utterplan/stages/document/plain.py

@@ -0,0 +1,4 @@
+from ...parsers import PlainDocumentParser
+from .. import protocols
+
+__all__ = ["PlainDocumentParser", "protocols"]

utterplan/stages/document/ssmd.py

@@ -0,0 +1,3 @@
+from ...parsers import SSMDDocumentParser
+
+__all__ = ["SSMDDocumentParser"]

utterplan/stages/preparation/identity.py

@@ -0,0 +1,3 @@
+from ...preparation import IdentityTextPreparer
+
+__all__ = ["IdentityTextPreparer"]

utterplan/stages/preparation/spokenform.py

@@ -0,0 +1,3 @@
+from ...preparation import SpokenformTextPreparer
+
+__all__ = ["SpokenformTextPreparer"]

utterplan/stages/protocols.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Protocol
+
+from ..config import PlannerConfig
+from ..model import AnnotationSpan, BoundaryEvent
+from ..parsers import ParsedDocument
+
+
+class DocumentParser(Protocol):
+    def parse(self, text: str, config: PlannerConfig) -> ParsedDocument: ...
+
+
+class TextPreparer(Protocol):
+    def prepare(
+        self,
+        text: str,
+        language: str,
+        runs: tuple[object, ...],
+        annotations: tuple[AnnotationSpan, ...] = (),
+        boundaries: tuple[BoundaryEvent, ...] = (),
+    ) -> object: ...
+
+
+__all__ = ["DocumentParser", "TextPreparer"]

utterplan/stages/segmentation/phrasplit.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from ...config import PlannerConfig
+from ...model import PlanSegment
+
+
+def split_text(text: str, config: PlannerConfig) -> tuple[PlanSegment, ...]:
+    try:
+        import phrasplit
+
+        items = phrasplit.split_with_offsets(
+            text, mode="sentence", use_spacy=False, language=config.language
+        )
+    except (ImportError, OSError, TypeError, ValueError):
+        items = ()
+    result = []
+    for index, item in enumerate(items):
+        start, end = int(item.char_start), int(item.char_end)
+        result.append(
+            PlanSegment(
+                f"seg-{index:06d}",
+                text[start:end],
+                start,
+                end,
+                config.language,
+                int(getattr(item, "paragraph_idx", 0) or 0),
+                int(getattr(item, "sentence_idx", 0) or 0),
+            )
+        )
+    if not result and text:
+        result.append(PlanSegment("seg-000000", text, 0, len(text), config.language))
+    return tuple(result)
+
+
+__all__ = ["split_text"]

utterplan/units.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from dataclasses import replace
+
+from .hashing import semantic_hash, unit_hash_payload
+from .model import Marker, PlanSegment, PlanUnit
+
+
+def make_units(
+    segments: tuple[PlanSegment, ...], markers: tuple[Marker, ...], kind: str
+) -> tuple[PlanUnit, ...]:
+    if not segments:
+        return ()
+    groups: list[list[PlanSegment]] = []
+    current: list[PlanSegment] = []
+    current_key: tuple[int, int] | None = None
+    closed: set[tuple[int, int]] = set()
+    for segment in segments:
+        key = (segment.paragraph, segment.sentence if kind == "sentence" else -1)
+        if current and key != current_key:
+            groups.append(current)
+            if current_key is not None:
+                closed.add(current_key)
+            current = []
+        if key in closed:
+            raise ValueError(f"non-contiguous unit key {key}")
+        current.append(segment)
+        current_key = key
+    if current:
+        groups.append(current)
+
+    units: list[PlanUnit] = []
+    for index, group in enumerate(groups):
+        start, end = group[0].spoken_start, group[-1].spoken_end
+        marker_ids = tuple(
+            marker.id
+            for marker in markers
+            if _marker_belongs(marker.spoken_position, groups, index)
+        )
+        marker_values = tuple(marker for marker in markers if marker.id in marker_ids)
+        provisional = PlanUnit(
+            f"unit-{index:04d}",
+            index,
+            kind,
+            start,
+            end,
+            tuple(segment.id for segment in group),
+            marker_ids,
+        )
+        units.append(
+            replace(
+                provisional,
+                content_hash=semantic_hash(
+                    unit_hash_payload(_UnitView(group, marker_ids, marker_values))
+                ),
+            )
+        )
+    return tuple(units)
+
+
+def _marker_belongs(position: int, groups: list[list[PlanSegment]], index: int) -> bool:
+    group = groups[index]
+    start = group[0].spoken_start
+    end = group[-1].spoken_end
+    if index < len(groups) - 1 and end <= position <= groups[index + 1][0].spoken_start:
+        return False
+    if index > 0 and groups[index - 1][-1].spoken_end <= position <= start:
+        return True
+    return start <= position < end or (index == len(groups) - 1 and position == end)
+
+
+class _UnitView:
+    def __init__(
+        self,
+        segments: list[PlanSegment],
+        marker_ids: tuple[str, ...],
+        marker_values: tuple[Marker, ...],
+    ) -> None:
+        self.segments = tuple(segments)
+        self.marker_ids = marker_ids
+        self.marker_values = marker_values

utterplan/utterplan.schema.json

@@ -0,0 +1,260 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/buchwandler/utterplan/blob/main/spec/utterplan.schema.json",
+  "title": "UtterPlan schema",
+  "type": "object",
+  "required": [
+    "format",
+    "schema_version",
+    "producer",
+    "plan_id",
+    "source",
+    "config",
+    "texts",
+    "preparation",
+    "languages",
+    "annotations",
+    "boundaries",
+    "tokens",
+    "segments",
+    "units",
+    "markers",
+    "warnings",
+    "diagnostics"
+  ],
+  "properties": {
+    "format": { "const": "utterplan" },
+    "schema_version": { "const": 1 },
+    "producer": {
+      "type": "object",
+      "required": ["name", "version"],
+      "properties": {
+        "name": { "type": "string" },
+        "version": { "type": "string" }
+      },
+      "additionalProperties": true
+    },
+    "plan_id": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
+    "source": { "$ref": "#/$defs/source" },
+    "config": { "type": "object" },
+    "texts": {
+      "type": "object",
+      "required": ["structural", "spoken"],
+      "properties": {
+        "structural": { "type": "string" },
+        "spoken": { "type": "string" }
+      },
+      "additionalProperties": false
+    },
+    "preparation": { "$ref": "#/$defs/preparation" },
+    "languages": { "type": "array", "items": { "$ref": "#/$defs/language" } },
+    "annotations": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/annotation" }
+    },
+    "boundaries": { "type": "array", "items": { "$ref": "#/$defs/boundary" } },
+    "tokens": { "type": "array", "items": { "$ref": "#/$defs/token" } },
+    "segments": { "type": "array", "items": { "$ref": "#/$defs/segment" } },
+    "units": { "type": "array", "items": { "$ref": "#/$defs/unit" } },
+    "markers": { "type": "array", "items": { "$ref": "#/$defs/marker" } },
+    "document_metadata": { "type": "object" },
+    "warnings": { "type": "array", "items": { "type": "string" } },
+    "diagnostics": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/diagnostic" }
+    }
+  },
+  "additionalProperties": false,
+  "$defs": {
+    "source": {
+      "type": "object",
+      "required": ["format", "text"],
+      "properties": {
+        "format": { "enum": ["plain", "ssmd"] },
+        "text": { "type": "string" }
+      },
+      "additionalProperties": false
+    },
+    "preparation": {
+      "type": "object",
+      "required": ["backend", "languages", "replacements", "warnings"],
+      "properties": {
+        "backend": { "type": "string" },
+        "version": { "type": ["string", "null"] },
+        "languages": { "type": "array", "items": { "type": "string" } },
+        "replacements": { "type": "array", "items": { "type": "object" } },
+        "warnings": { "type": "array", "items": { "type": "string" } }
+      },
+      "additionalProperties": false
+    },
+    "range": { "type": "integer", "minimum": 0 },
+    "language": {
+      "type": "object",
+      "required": ["id", "spoken_start", "spoken_end", "language", "source"],
+      "properties": {
+        "id": { "type": "string" },
+        "spoken_start": { "$ref": "#/$defs/range" },
+        "spoken_end": { "$ref": "#/$defs/range" },
+        "language": { "type": "string" },
+        "source": { "type": "string" }
+      },
+      "additionalProperties": false
+    },
+    "annotation": {
+      "type": "object",
+      "required": [
+        "id",
+        "structural_start",
+        "structural_end",
+        "spoken_start",
+        "spoken_end",
+        "kind",
+        "attrs"
+      ],
+      "properties": {
+        "id": { "type": "string" },
+        "structural_start": { "$ref": "#/$defs/range" },
+        "structural_end": { "$ref": "#/$defs/range" },
+        "spoken_start": { "type": ["integer", "null"], "minimum": 0 },
+        "spoken_end": { "type": ["integer", "null"], "minimum": 0 },
+        "kind": { "type": "string" },
+        "attrs": { "type": "object" }
+      },
+      "additionalProperties": false
+    },
+    "boundary": {
+      "type": "object",
+      "required": ["id", "position", "kind", "seconds", "origin", "strength"],
+      "properties": {
+        "id": { "type": "string" },
+        "position": { "$ref": "#/$defs/range" },
+        "kind": { "type": "string" },
+        "seconds": { "type": ["number", "null"], "minimum": 0 },
+        "origin": { "type": "string" },
+        "strength": { "type": ["string", "null"] },
+        "attrs": { "type": "object" }
+      },
+      "additionalProperties": false
+    },
+    "token": {
+      "type": "object",
+      "required": ["spoken_start", "spoken_end", "text"],
+      "properties": {
+        "id": { "type": "string" },
+        "spoken_start": { "$ref": "#/$defs/range" },
+        "spoken_end": { "$ref": "#/$defs/range" },
+        "text": { "type": "string" },
+        "pos": { "type": ["string", "null"] },
+        "tag": { "type": ["string", "null"] },
+        "lemma": { "type": ["string", "null"] },
+        "language": { "type": ["string", "null"] }
+      },
+      "additionalProperties": false
+    },
+    "pause": {
+      "type": "object",
+      "required": ["seconds", "events"],
+      "properties": {
+        "seconds": { "type": "number", "minimum": 0 },
+        "events": { "type": "array", "items": { "type": "string" } }
+      },
+      "additionalProperties": false
+    },
+    "directives": {
+      "type": "object",
+      "properties": {
+        "voice": { "type": ["object", "null"] },
+        "pronunciation": { "type": ["object", "null"] },
+        "prosody": { "type": ["object", "null"] },
+        "emphasis": { "type": ["object", "null"] },
+        "audio": { "type": ["object", "null"] }
+      },
+      "additionalProperties": false
+    },
+    "segment": {
+      "type": "object",
+      "required": [
+        "id",
+        "text",
+        "spoken_start",
+        "spoken_end",
+        "language",
+        "paragraph",
+        "sentence",
+        "clause",
+        "pause_before",
+        "pause_after",
+        "directives",
+        "token_indices",
+        "annotation_ids"
+      ],
+      "properties": {
+        "id": { "type": "string" },
+        "text": { "type": "string" },
+        "spoken_start": { "$ref": "#/$defs/range" },
+        "spoken_end": { "$ref": "#/$defs/range" },
+        "language": { "type": "string" },
+        "paragraph": { "type": "integer", "minimum": 0 },
+        "sentence": { "type": "integer", "minimum": 0 },
+        "clause": { "type": "integer", "minimum": 0 },
+        "structural_start": { "type": ["integer", "null"], "minimum": 0 },
+        "structural_end": { "type": ["integer", "null"], "minimum": 0 },
+        "pause_before": { "$ref": "#/$defs/pause" },
+        "pause_after": { "$ref": "#/$defs/pause" },
+        "directives": { "$ref": "#/$defs/directives" },
+        "token_indices": {
+          "type": "array",
+          "items": { "type": "integer", "minimum": 0 }
+        },
+        "annotation_ids": { "type": "array", "items": { "type": "string" } }
+      },
+      "additionalProperties": false
+    },
+    "unit": {
+      "type": "object",
+      "required": [
+        "id",
+        "index",
+        "kind",
+        "spoken_start",
+        "spoken_end",
+        "segment_ids",
+        "marker_ids",
+        "content_hash"
+      ],
+      "properties": {
+        "id": { "type": "string" },
+        "index": { "type": "integer", "minimum": 0 },
+        "kind": { "enum": ["paragraph", "sentence"] },
+        "spoken_start": { "$ref": "#/$defs/range" },
+        "spoken_end": { "$ref": "#/$defs/range" },
+        "segment_ids": { "type": "array", "items": { "type": "string" } },
+        "marker_ids": { "type": "array", "items": { "type": "string" } },
+        "content_hash": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }
+      },
+      "additionalProperties": false
+    },
+    "marker": {
+      "type": "object",
+      "required": ["id", "name", "spoken_position"],
+      "properties": {
+        "id": { "type": "string" },
+        "name": { "type": "string" },
+        "spoken_position": { "$ref": "#/$defs/range" },
+        "attrs": { "type": "object" }
+      },
+      "additionalProperties": false
+    },
+    "diagnostic": {
+      "type": "object",
+      "required": ["code", "message", "severity", "path"],
+      "properties": {
+        "code": { "type": "string" },
+        "message": { "type": "string" },
+        "severity": { "type": "string" },
+        "path": { "type": ["string", "null"] }
+      },
+      "additionalProperties": false
+    }
+  }
+}