accentuk 0.1.0__py3-none-any.whl

accentuk/__init__.py

@@ -0,0 +1,23 @@
+"""accentuk: Offline Ukrainian stress marking and optional TTS verbalization."""
+
+from .engine import Accentor, StressResult
+from .errors import AccentukError, AssetError, NormalizationError, ServingError
+from .frontend import Frontend, FrontendResult
+from .lexicon import Lexicon
+from .text import StressToken
+
+__version__ = "0.1.0"
+__all__ = [
+    "Accentor",
+    "AccentukError",
+    "AssetError",
+    "Frontend",
+    "FrontendResult",
+    "Lexicon",
+    "NormalizationError",
+    "ServingError",
+    "StressResult",
+    "StressToken",
+    "__version__",
+]
+

accentuk/assets/NOTICE.md

@@ -0,0 +1,17 @@
+# Notices and Attributions
+
+`accentuk` incorporates, derives from, or provides access to the following works and datasets:
+
+## Models and Checkpoints
+
+- **uk-stress-crossencoder**: Developed by aloudreader / vacuum-in under Apache-2.0.
+- **uk-stress-token-classifier**: Developed by aloudreader / vacuum-in under Apache-2.0.
+- **uk-stress-combiner**: Developed by aloudreader / vacuum-in under Apache-2.0.
+- **marian-uk-verbalizer**: Developed by aloudreader / vacuum-in under Apache-2.0.
+- **uk_core_news_trf (3.8.0)**: Trained and published by Explosion AI under the MIT license, based on Ukrainian UD treebanks.
+
+## Lexicon and Dictionaries
+
+- **ukrainian-word-stress**: Copyright (c) 2018-2024 Bohdan Moseiko, licensed under the MIT License.
+- **Ukrainian Wiktionary**: Data derived from Wikimedia Foundation under CC BY-SA 3.0 / 4.0.
+- **VESUM / pymorphy3-dicts-uk**: Ukrainian morphological dictionary, licensed under GPL / MIT / CC BY-SA as published by the lang-uk project.

accentuk/assets/releases.json

@@ -0,0 +1,128 @@
+{
+  "release_version": "0.1.0",
+  "dataset_id": 2,
+  "inventory_hash": "759baefd94a6d8bc865f0a93ff378542b7a2b65dd58617c3194bdf85746c2769",
+  "normalization_version": "ukstress-nfd-v1",
+  "schema_version": 1,
+  "data_release": {
+    "url_base": "https://github.com/vacuum-in/accentuk-api/releases/download/data-v2",
+    "files": {
+      "lexicon-serving.tar": {
+        "sha256": "b94dcdf833bf40938a1fa4c4110964227995a7c4e3818918c2ea15c1dd22d891"
+      },
+      "lexicon-assets.tar.gz": {
+        "sha256": "e65def53d4fc95d95bbd8c18142bd1f58b7af0eba2db3eef79794907cdf18cdc"
+      }
+    }
+  },
+  "models": {
+    "crossencoder": {
+      "repo": "aloudreader/uk-stress-crossencoder",
+      "revision": "6760529e714ffac7f89f33b8f7001b885a5ebc8f",
+      "files": {
+        "checkpoint/model.safetensors": "35cb3190904f352680850a507f45540df0ea8544653821b4aee459c693ce23c5",
+        "checkpoint/config.json": "fa756aaec2d8c2fc215e1d22070554ed834a8fbda9789a6fcfd961635ec62b49",
+        "checkpoint/tokenizer.json": "013ee4d5884a2e49945b7181b918285367b7e2ebcba843106ce22e2635bb4a8a",
+        "checkpoint/tokenizer_config.json": "5f3587fd1486c98a01f9c67bff3748fb150d7a3862289769833cb1c6309d76ca",
+        "checkpoint/special_tokens_map.json": "38d989b0fdad0fec0c67c14b1f3c8b68184022cf6d4adc5444526ced8653f738",
+        "serving_manifest.json": "39593fc90af96c0a48f24586c280b74152a54847af8500d3f7543753874d3684"
+      }
+    },
+    "token_classifier": {
+      "repo": "aloudreader/uk-stress-token-classifier",
+      "revision": "d2795e0c32154ba284248729f35362de135b0d72",
+      "files": {
+        "model.safetensors": "28607d6d0c99fe184b54f7540ae6abe6d04e6923e06abfacf7d11681f6cea2b6",
+        "config.json": "5a8242cc875e21f364b29cedc5d10d6dea6c9adc023acfee64bd34ac50b00088",
+        "head.pt": "5dfbb68e4e0caca8a66e853b0b6ab5dc34f7106f3e958b5393febfaba201730c",
+        "resolver.json": "4599961ae8f299ed0458a665d8d009d961a35e8c2b0a6c8ed570030e582c7e5f",
+        "coverage.json": "a24de1bf1f815a402116fb328d171508676b89d227573d698429387ff657fe41"
+      }
+    },
+    "combiner": {
+      "repo": "aloudreader/uk-stress-combiner",
+      "revision": "f54a4f2f15615b201e26a284882d5ef65c7821c0",
+      "files": {
+        "model.pt": "ca630edde162a81afcb9d70c5d5a52203758887f0b24a264dc47bb7503a30840",
+        "audio_prior.json": "e14cf2c5be46c9d6d4cb7d3d89a9692c96cbe9968318e1ba0ea520221b837727",
+        "classifier_profile.json": "b311120d232858d6e93a2189e2e84830bd94d8c8d49db6d037027f9694409a43",
+        "readings.jsonl": "0ad426e929e56000c49a294091305e6b2e25bb696b076e92d9899f965cc04d7c"
+      }
+    },
+    "spacy_trf": {
+      "url": "https://github.com/explosion/spacy-models/releases/download/uk_core_news_trf-3.8.0/uk_core_news_trf-3.8.0-py3-none-any.whl",
+      "version": "3.8.0"
+    },
+    "verbalizer": {
+      "repo": "aloudreader/marian-uk-verbalizer",
+      "revision": "7726f30947caf3b2882c8e753805f930e67ec858",
+      "files": {
+        "pytorch_model.bin": "d221528b5a001bec01834ae053da5b9327d2812edb36497185ab56d12849b06c",
+        "config.json": "d83a1a7a284a4391fdf9d359be5191e01be04007461a54cea2321eca6921344b",
+        "source.spm": "de8d503a8c321617ea0ba65b82539e50fd255dda0e5e6e5fc57bed5a55dfbd58",
+        "target.spm": "de8d503a8c321617ea0ba65b82539e50fd255dda0e5e6e5fc57bed5a55dfbd58",
+        "vocab.json": "0f11e1d7404edd5176fe1aa82561207fcbc489adac41b7f5b957feecbc1c9cc2"
+      }
+    }
+  },
+  "lexicon_bundle": {
+    "url": "https://github.com/vacuum-in/accentuk/releases/download/lexicon-v2/accentuk-lexicon-v2.tar.gz",
+    "sha256": "73b068ab20d2884cbe81a479ec62a2a98d093b026bda5e4a61e3605816004220"
+  },
+  "hub_models": {
+    "crossencoder_ct2": {
+      "repo": "aloudreader/uk-stress-crossencoder-ct2-int8",
+      "revision": "cf9133781f0f74c142a7d589a7af8ace1d8e5573",
+      "target": "ct2_models/stress-model-ct2-int8",
+      "set": "light"
+    },
+    "token_classifier_ct2": {
+      "repo": "aloudreader/uk-stress-token-classifier-ct2-int8",
+      "revision": "0d85c73a7565a5b1c4e2a5593d529b50c1ea9a30",
+      "target": "ct2_models/tok-v5-ct2-int8",
+      "set": "light"
+    },
+    "spacy_sm": {
+      "repo": "spacy/uk_core_news_sm",
+      "revision": "ca2310baff047c262a04258e0d7f6a8721a06bd1",
+      "target": "spacy-uk_core_news_sm",
+      "set": "light"
+    },
+    "verbalizer_ct2": {
+      "repo": "aloudreader/marian-uk-verbalizer-ct2-int8",
+      "revision": "483bd81b6ad3bc1a9eaaf6d5835ced755774e95e",
+      "target": "ct2_models/verbalizer-ct2-int8",
+      "set": "light-tts"
+    },
+    "crossencoder": {
+      "repo": "aloudreader/uk-stress-crossencoder",
+      "revision": "6760529e714ffac7f89f33b8f7001b885a5ebc8f",
+      "target": "stress-model",
+      "set": "full"
+    },
+    "token_classifier": {
+      "repo": "aloudreader/uk-stress-token-classifier",
+      "revision": "d2795e0c32154ba284248729f35362de135b0d72",
+      "target": "tok-v5",
+      "set": "full"
+    },
+    "combiner": {
+      "repo": "aloudreader/uk-stress-combiner",
+      "revision": "f54a4f2f15615b201e26a284882d5ef65c7821c0",
+      "target": "combiner-v1",
+      "set": "full"
+    },
+    "spacy_trf": {
+      "repo": "spacy/uk_core_news_trf",
+      "revision": "695c843e091e2659b1ee747af0b564ce54c8aff5",
+      "target": "spacy-uk_core_news_trf",
+      "set": "full"
+    },
+    "verbalizer": {
+      "repo": "aloudreader/marian-uk-verbalizer",
+      "revision": "7726f30947caf3b2882c8e753805f930e67ec858",
+      "target": "verbalizer",
+      "set": "full-tts"
+    }
+  }
+}

accentuk/assets.py

@@ -0,0 +1,218 @@
+"""Asset bundle verification, atomic download, and cache management."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import shutil
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+from .errors import AssetError
+from .export import build_bundle, sha256_file
+
+log = logging.getLogger("accentuk.assets")
+
+DEFAULT_CACHE_DIR = Path.home() / ".cache" / "accentuk"
+
+
+def get_releases_descriptor() -> dict[str, Any]:
+    """Load bundled releases.json descriptor."""
+    desc_file = Path(__file__).parent / "assets" / "releases.json"
+    if not desc_file.is_file():
+        raise AssetError("releases.json descriptor is missing.")
+    return json.loads(desc_file.read_text(encoding="utf-8"))
+
+
+class AssetBundle:
+    """Represents an active, verified accentuk asset bundle."""
+
+    def __init__(self, path: Path, manifest: dict[str, Any]) -> None:
+        self.path = path
+        self.manifest = manifest
+        self.schema_version = manifest.get("schema_version", 1)
+        self.normalization_version = manifest.get("normalization_version", "ukstress-nfd-v1")
+        self.dataset_id = manifest.get("dataset_id", 2)
+        self.inventory_hash = manifest.get("inventory_hash", "")
+        self.model_version = manifest.get("model_version", "")
+
+    @property
+    def lexicon_path(self) -> Path:
+        return self.path / "lexicon.marisa"
+
+    @classmethod
+    def open(cls, directory: Path | str) -> AssetBundle:
+        """Open and validate an existing asset bundle directory."""
+        dir_path = Path(directory)
+        manifest_file = dir_path / "manifest.json"
+        if not manifest_file.is_file():
+            raise AssetError(
+                f"Asset bundle manifest missing at {manifest_file}.",
+                "Run 'accentuk download' to fetch or repair the bundle.",
+            )
+
+        try:
+            manifest = json.loads(manifest_file.read_text(encoding="utf-8"))
+        except Exception as err:
+            raise AssetError(
+                f"Corrupt bundle manifest at {manifest_file}: {err}",
+                "Run 'accentuk download' to re-download the bundle.",
+            ) from err
+
+        if manifest.get("schema_version") != 1:
+            raise AssetError(
+                f"Incompatible bundle schema version: {manifest.get('schema_version')}, expected 1.",
+                "Upgrade accentuk or download a compatible bundle.",
+            )
+        if manifest.get("normalization_version") != "ukstress-nfd-v1":
+            raise AssetError(
+                f"Incompatible normalization version: {manifest.get('normalization_version')}, expected 'ukstress-nfd-v1'.",
+                "Rebuild or re-download the bundle.",
+            )
+
+        # Verify bundled file checksums
+        files = manifest.get("files", {})
+        for filename, expected_sha in files.items():
+            fpath = dir_path / filename
+            if not fpath.is_file():
+                raise AssetError(
+                    f"Required bundle file missing: {fpath}.",
+                    "Run 'accentuk download' to repair the bundle.",
+                )
+            actual_sha = sha256_file(fpath)
+            if actual_sha.lower() != expected_sha.lower():
+                raise AssetError(
+                    f"Corrupted bundle file {filename}: expected {expected_sha}, got {actual_sha}.",
+                    "Run 'accentuk download' to re-download and repair.",
+                )
+
+        return cls(dir_path, manifest)
+
+
+def _download_file(url: str, dest: Path, expected_sha: str | None = None) -> None:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    temp_dest = dest.with_suffix(dest.suffix + ".part")
+    try:
+        req = urllib.request.Request(url, headers={"User-Agent": "accentuk/0.1.0"})
+        with urllib.request.urlopen(req) as resp, temp_dest.open("wb") as out:
+            shutil.copyfileobj(resp, out)
+
+        if expected_sha:
+            actual = sha256_file(temp_dest)
+            if actual.lower() != expected_sha.lower():
+                raise AssetError(
+                    f"Checksum mismatch for {dest.name}: expected {expected_sha}, got {actual}."
+                )
+        temp_dest.replace(dest)
+    finally:
+        if temp_dest.is_file():
+            temp_dest.unlink(missing_ok=True)
+
+
+def download_assets(
+    target_dir: Path | str | None = None,
+    *,
+    include_tts: bool = False,
+    allow_local_fixtures: bool = True,
+    light: bool = False,
+    full: bool = False,
+) -> Path:
+    """Atomically download or build and verify the stress assets bundle."""
+    target_path = Path(target_dir) if target_dir else DEFAULT_CACHE_DIR
+    target_path.mkdir(parents=True, exist_ok=True)
+
+    desc = get_releases_descriptor()
+    staging_dir = target_path.parent / f".staging-{os.getpid()}-{target_path.name}"
+    if staging_dir.is_dir():
+        shutil.rmtree(staging_dir)
+    staging_dir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        # Optional local sources instead of downloads (maintainers): ACCENTUK_LOCAL_DATA holds serving/ and
+        # lexicon-assets.tar.gz, ACCENTUK_LOCAL_MODELS the model directories, ACCENTUK_LOCAL_CT2 the CT2 ones.
+        local_data = Path(os.environ.get("ACCENTUK_LOCAL_DATA", "/nonexistent"))
+        local_archive = local_data / "serving"
+        local_assets_tar = local_data / "lexicon-assets.tar.gz"
+        local_models = Path(os.environ.get("ACCENTUK_LOCAL_MODELS", "/nonexistent"))
+
+        if allow_local_fixtures and local_archive.is_dir() and (local_models / "stress-model").is_dir():
+            log.info("Using local archive fixtures from %s...", local_archive)
+            build_bundle(
+                archive=local_archive,
+                model_manifest=local_models / "stress-model" / "serving_manifest.json",
+                output=staging_dir,
+                assets_dir=local_assets_tar if local_assets_tar.is_file() else local_models / "lexicon",
+            )
+            # Copy CT2 light models if present
+            ct2_source = Path(os.environ.get("ACCENTUK_LOCAL_CT2", "/nonexistent"))
+            if ct2_source.is_dir():
+                for ct2_name in ("stress-model-ct2-int8", "tok-v5-ct2-int8", "verbalizer-ct2-int8"):
+                    src_ct2 = ct2_source / ct2_name
+                    if src_ct2.is_dir():
+                        shutil.copytree(src_ct2, staging_dir / ct2_name, dirs_exist_ok=True)
+
+            # Copy full models unless light-only was explicitly requested
+            if not light or full:
+                for model_name in ("stress-model", "tok-v5", "combiner-v1", "spacy-uk_core_news_trf_380"):
+                    src_m = local_models / model_name
+                    if src_m.is_dir():
+                        shutil.copytree(src_m, staging_dir / model_name, dirs_exist_ok=True)
+
+                if include_tts:
+                    src_verb = local_models / "verbalizer"
+                    if src_verb.is_dir():
+                        shutil.copytree(src_verb, staging_dir / "verbalizer", dirs_exist_ok=True)
+        else:
+            # The lexicon comes prebuilt (building it needs ukrainian-word-stress for the readings),
+            # the models from the Hub at the commits pinned in releases.json.
+            lex = desc["lexicon_bundle"]
+            downloads_dir = staging_dir / "downloads"
+            downloads_dir.mkdir(exist_ok=True)
+            lex_tar = downloads_dir / "lexicon.tar.gz"
+            _download_file(lex["url"], lex_tar, lex["sha256"])
+            import tarfile
+            with tarfile.open(lex_tar) as tf:
+                tf.extractall(staging_dir, filter="data")
+            shutil.rmtree(downloads_dir)
+
+            from huggingface_hub import snapshot_download
+            sets = {"full"} if full and not light else {"light"}
+            if include_tts:
+                sets |= {f"{x}-tts" for x in sets}
+            for name, m in desc["hub_models"].items():
+                if m["set"] not in sets:
+                    continue
+                log.info("Downloading %s (%s@%s)...", name, m["repo"], m["revision"][:12])
+                snapshot_download(m["repo"], revision=m["revision"], local_dir=staging_dir / m["target"],
+                                  ignore_patterns=["*.whl", ".gitattributes", "README.md"])
+            # Both cross-encoders read the serving manifest from the bundle root.
+            for cand in ("ct2_models/stress-model-ct2-int8", "stress-model"):
+                if (staging_dir / cand / "serving_manifest.json").is_file():
+                    shutil.copy(staging_dir / cand / "serving_manifest.json", staging_dir / "serving_manifest.json")
+                    break
+
+        # Validate staging before activating
+        AssetBundle.open(staging_dir)
+
+        # If target_path already exists and we're just adding --tts:
+        if (target_path / "lexicon.marisa").is_file() and include_tts and (staging_dir / "verbalizer").is_dir():
+            shutil.copytree(staging_dir / "verbalizer", target_path / "verbalizer", dirs_exist_ok=True)
+        else:
+            # Atomic copy/replace
+            for item in staging_dir.iterdir():
+                dest = target_path / item.name
+                if item.is_dir():
+                    if dest.is_dir():
+                        shutil.rmtree(dest)
+                    shutil.copytree(item, dest)
+                else:
+                    shutil.copy2(item, dest)
+
+        log.info("Assets successfully installed and verified at %s", target_path)
+        return target_path
+    finally:
+        if staging_dir.is_dir():
+            shutil.rmtree(staging_dir, ignore_errors=True)

accentuk/cli.py

@@ -0,0 +1,104 @@
+"""Command-line interface for accentuk."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .assets import download_assets
+from .engine import Accentor
+from .errors import AccentukError
+from .export import build_bundle
+
+
+def main(args: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="accentuk",
+        description="Offline Ukrainian stress marking and optional TTS preparation",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # download
+    p_dl = subparsers.add_parser("download", help="Download and verify required stress models/lexicon")
+    p_dl.add_argument("--light", action="store_true", help="Download light bundle (CTranslate2 INT8 + Lexicon, ~450 MB)")
+    p_dl.add_argument("--full", action="store_true", help="Download full bundle (PyTorch weights + spaCy TRF, ~2.2 GB)")
+    p_dl.add_argument("--tts", action="store_true", help="Also download optional TTS verbalizer model")
+    p_dl.add_argument("--directory", type=Path, default=None, help="Target cache directory")
+
+    # export
+    p_exp = subparsers.add_parser("export", help="Export serving archive into MARISA bundle")
+    p_exp.add_argument("--archive", type=Path, required=True, help="Path to serving archive or directory")
+    p_exp.add_argument("--manifest", type=Path, required=True, help="Path to serving_manifest.json")
+    p_exp.add_argument("--output", type=Path, required=True, help="Output bundle directory")
+    p_exp.add_argument("--assets", type=Path, default=None, help="Path to lexicon-assets.tar.gz or dir")
+
+    # stress
+    p_stress = subparsers.add_parser("stress", help="Mark stress on Ukrainian text")
+    p_stress.add_argument("text", help="Ukrainian input text")
+    p_stress.add_argument("--mode", choices=["auto", "light", "full"], default="auto", help="Inference mode: light (CT2 INT8) or full (PyTorch/spaCy)")
+    p_stress.add_argument("--ambiguity", choices=["default", "preserve"], default="default", help="Handling of ambiguous words")
+    p_stress.add_argument("--combiner", action="store_true", help="Enable learned combiner")
+    p_stress.add_argument("--assets", type=Path, default=None, help="Path to asset bundle directory")
+
+    # prepare
+    p_prep = subparsers.add_parser("prepare", help="Prepare text for TTS (verbalize and stress mark)")
+    p_prep.add_argument("text", help="Input text with numbers/dates/symbols")
+    p_prep.add_argument("--mode", choices=["auto", "light", "full"], default="auto", help="Inference mode: light (CT2 INT8) or full (PyTorch/spaCy)")
+    p_prep.add_argument("--ambiguity", choices=["default", "preserve"], default="default", help="Handling of ambiguous words")
+    p_prep.add_argument("--combiner", action="store_true", help="Enable learned combiner")
+    p_prep.add_argument("--assets", type=Path, default=None, help="Path to asset bundle directory")
+
+    parsed = parser.parse_args(args)
+
+    try:
+        if parsed.command == "download":
+            download_assets(
+                target_dir=parsed.directory,
+                include_tts=parsed.tts,
+                light=parsed.light,
+                full=parsed.full,
+            )
+            print("Download completed successfully.")
+            return 0
+
+        elif parsed.command == "export":
+            build_bundle(
+                archive=parsed.archive,
+                model_manifest=parsed.manifest,
+                output=parsed.output,
+                assets_dir=parsed.assets,
+            )
+            print(f"Export completed successfully to {parsed.output}.")
+            return 0
+
+        elif parsed.command == "stress":
+            with Accentor(asset_dir=parsed.assets, mode=parsed.mode, enable_combiner=parsed.combiner) as engine:
+                res = engine.stress(parsed.text, on_ambiguity=parsed.ambiguity, combiner=parsed.combiner)
+                print(res.text)
+            return 0
+
+        elif parsed.command == "prepare":
+            try:
+                from .frontend import Frontend
+
+                with Frontend(asset_dir=parsed.assets, mode=parsed.mode) as frontend:
+                    res = frontend.prepare(parsed.text, on_ambiguity=parsed.ambiguity, combiner=parsed.combiner)
+                    print(res.text)
+                return 0
+            except ImportError as err:
+                print(f"TTS extra not available: {err}. Install with 'accentuk[tts]'.", file=sys.stderr)
+                return 1
+
+    except AccentukError as err:
+        print(f"Error: {err}", file=sys.stderr)
+        return 1
+    except Exception as err:
+        print(f"Unexpected error: {err}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())