turkic-translit 0.2.0__tar.gz

turkic_translit-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Austin Wagner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

turkic_translit-0.2.0/PKG-INFO

@@ -0,0 +1,253 @@
+Metadata-Version: 2.4
+Name: turkic-translit
+Version: 0.2.0
+Summary: Deterministic Latin and IPA transliteration for Kazakh, Kyrgyz, plus tokenizer/glue scripts.
+Author-email: Austin Wagner <austinwagner@msn.com>
+License-Expression: MIT
+Keywords: kazakh,kyrgyz,transliteration,ipa
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: epitran<1.27,>=1.0
+Requires-Dist: fasttext-wheel==0.9.2
+Requires-Dist: numpy<2
+Requires-Dist: packaging>=23.0
+Requires-Dist: panphon<0.22,>=0.20
+Requires-Dist: PyICU<2.16,>=2.13; sys_platform != "win32"
+Requires-Dist: pytest>=8.0
+Requires-Dist: rapidfuzz>=3.5
+Requires-Dist: rich>=13.7
+Requires-Dist: sentencepiece>=0.2.0
+Requires-Dist: click>=8.1
+Requires-Dist: types-requests>=2.31
+Requires-Dist: datasets<4.0.0,>=3.6.0
+Requires-Dist: evaluate>=0.4
+Requires-Dist: accelerate>=0.28
+Requires-Dist: transformers<5,>=4.41
+Requires-Dist: tqdm>=4.66
+Requires-Dist: scikit-learn>=1.4
+Requires-Dist: typing_extensions>=4.12
+Requires-Dist: wikipedia<2.0.0,>=1.4.0
+Requires-Dist: gradio>=4.0
+Requires-Dist: pandas
+Requires-Dist: matplotlib>=3.9
+Requires-Dist: pycountry>=23.12
+Provides-Extra: examples
+Requires-Dist: flask; extra == "examples"
+Requires-Dist: streamlit; extra == "examples"
+Requires-Dist: jupyterlab; extra == "examples"
+Provides-Extra: winlid
+Provides-Extra: corpus
+Requires-Dist: datasets>=3.0; extra == "corpus"
+Requires-Dist: pyarrow>=14.0; extra == "corpus"
+Requires-Dist: requests>=2.0; extra == "corpus"
+Requires-Dist: zstandard>=0.23; extra == "corpus"
+Provides-Extra: dev
+Requires-Dist: ruff>=0.2.0; extra == "dev"
+Requires-Dist: mypy>=1.0; extra == "dev"
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: build>=1.0; extra == "dev"
+Requires-Dist: twine>=4.0; extra == "dev"
+Requires-Dist: make>=0.1.6; extra == "dev"
+Requires-Dist: types-requests>=2.31; extra == "dev"
+Dynamic: license-file
+
+---
+title: Turkic Transliteration Demo
+emoji: 🌖
+colorFrom: green
+colorTo: green
+sdk: gradio
+sdk_version: 5.29.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+short_description: Transliteration of Kazakh & Kyrgyz into Latin and IPA
+---
+
+turkic\_transliterate
+Deterministic Latin and IPA transliteration for Kazakh and Kyrgyz, plus helper utilities for tokenizer training and Russian-token filtering.
+
+Quick install
+
+1. Install Miniconda or Anaconda (recommended).
+2. Clone the repo and create the environment:
+   conda env create -f env.yml
+3. Activate the environment:
+   conda activate turkic
+4. Run the verification tests:
+   python -m pytest      (all tests should pass)
+
+Python compatibility
+• Works on CPython 3.10 and 3.11.
+• CPython 3.12+ is supported everywhere except on Windows until official PyICU wheels are available; see “Windows & PyICU” below.
+
+Package names
+• Runtime import path:  turkic\_translit
+• Distributable name on PyPI:  turkic\_transliterate
+• Command-line entry point:  turkic-translit
+
+## Developer Setup
+
+For the simplest developer setup experience, run the setup script:
+
+```bash
+python scripts/setup_dev.py
+```
+
+This script will:
+1. Install the package with all development dependencies
+2. Set up PyICU on Windows automatically
+3. Verify that development tools are working properly
+
+### Manual Installation
+
+Alternatively, install with pip:
+
+```bash
+pip install -e .[dev,ui]        # add ,winlid on Windows if you need fasttext-wheel
+```
+
+### Development Tools
+
+#### Linux/macOS/Windows with GNU Make
+
+If you have GNU Make installed, you can use the Makefile for common tasks:
+
+```bash
+make lint       # Run linting (ruff, black, mypy)
+make format     # Auto-format code
+make test       # Run tests
+make web        # Launch the web UI
+make help       # Show all available commands
+```
+
+#### Windows
+
+**Option 1: Install GNU Make using Chocolatey (Recommended)**
+
+Install GNU Make using Chocolatey (requires admin privileges):
+
+```powershell
+# In an Admin PowerShell window
+choco install make
+```
+
+After installation, you can use the same `make` commands as on Linux/macOS.
+
+**Option 2: Use the PowerShell Script Alternative**
+
+If you prefer not to install Chocolatey or GNU Make, use the PowerShell script:
+
+```powershell
+./scripts/run.ps1 lint       # Run linting
+./scripts/run.ps1 format     # Auto-format code
+./scripts/run.ps1 test       # Run tests
+./scripts/run.ps1 web        # Launch the web UI
+./scripts/run.ps1 help       # Show all available commands
+```
+
+Optional extras
+dev   → black, ruff, pytest
+ui    → gradio web demo
+winlid (Windows only) → fasttext-wheel for language ID
+
+Windows & PyICU
+
+**Important:** Due to PyPI rules, the correct PyICU wheel for Windows cannot be installed automatically during pip install. After installing this package with pip, Windows users must run the helper script to install the appropriate PyICU wheel:
+
+    turkic-pyicu-install
+
+This script will download and install the correct PyICU wheel from Christoph Gohlke’s repository based on your Python version. See the script for details.
+
+Command-line usage
+turkic-translit --lang kk --in text.txt --out\_latin kk\_lat.txt --ipa --out\_ipa kk\_ipa.txt --arabic --log-level debug
+• --lang            kk or ky
+• --ipa             emit IPA alongside Latin
+• --arabic          also transliterate embedded Arabic script
+• --benchmark       print throughput statistics
+• --log-level       debug | info | warning | error | critical (default: info)
+
+Logging
+The central logging setup uses Rich for colour when available.
+Set TURKIC\_LOG\_LEVEL or pass --log-level to the CLI.
+Fallback to standard logging when Rich is absent.
+
+# Project Organization
+
+The project is organized into the following directories:
+
+- `src/turkic_translit/` - Core source code for the package
+- `examples/` - Example scripts showing how to use the package
+  - `examples/web/` - Web interface for demonstrating transliteration features
+- `data/` - Sample data files and language resources
+- `docs/` - Documentation and reference materials
+- `scripts/` - Utility scripts for development and release
+  - `scripts/release/` - Scripts for building and publishing packages
+- `vendor/pyicu/` - Pre-built PyICU wheels for Windows
+- `tests/` - Test suite for the package
+
+## FastText Language Identification Model
+
+This package uses the [FastText language identification model](https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin) (`lid.176.bin`) for Russian token filtering and language detection. **The model file is not included in the repository or pip package due to its large size.**
+
+**Automatic Download:**
+- When you use features that require language identification (such as Russian token filtering or the Gradio web demo), the package will automatically download `lid.176.bin` from the official Facebook AI public link if it is not already present.
+- The file will be saved in the package directory on first use.
+
+**No manual action is needed.** This ensures compatibility with pip installs, Hugging Face Spaces, and other cloud environments.
+
+If you need to download the model manually, you can do so from:
+https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin
+
+
+## Using the Examples
+
+Use the main entry point script to run examples:
+
+```bash
+python turkic_tools.py [command]
+```
+
+Available commands:
+- `web` - Launch the Gradio web interface for real-time transliteration
+- `demo` - Run the simple CLI demo
+- `full-demo` - Run the comprehensive demo with multiple languages
+- `help` - Display available commands
+
+Tokenizer training example
+turkic-build-spm --input corpora/kk\_lat.txt,corpora/ky\_lat.txt --model\_prefix spm/turkic12k --vocab\_size 12000
+
+Filtering Russian tokens from Uzbek
+cat uz\_raw\.txt | turkic-filter-russian --mode drop > uz\_clean.txt
+
+Developer checklist
+black .
+ruff check .
+pytest -q
+
+All code is UTF-8-only; on Windows a BOM is written when piping to files to avoid encoding issues.
+
+License
+Apache-2.0
+
+### Type-checking
+
+```bash
+pip install mypy
+mypy --strict .
+```
+
+The included mypy.ini restricts analysis to the src/ tree and skips
+build/, dist/, virtual-env and egg directories so duplicate-module
+errors do not occur even if you build wheels locally.

turkic_translit-0.2.0/README.md

@@ -0,0 +1,189 @@
+---
+title: Turkic Transliteration Demo
+emoji: 🌖
+colorFrom: green
+colorTo: green
+sdk: gradio
+sdk_version: 5.29.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+short_description: Transliteration of Kazakh & Kyrgyz into Latin and IPA
+---
+
+turkic\_transliterate
+Deterministic Latin and IPA transliteration for Kazakh and Kyrgyz, plus helper utilities for tokenizer training and Russian-token filtering.
+
+Quick install
+
+1. Install Miniconda or Anaconda (recommended).
+2. Clone the repo and create the environment:
+   conda env create -f env.yml
+3. Activate the environment:
+   conda activate turkic
+4. Run the verification tests:
+   python -m pytest      (all tests should pass)
+
+Python compatibility
+• Works on CPython 3.10 and 3.11.
+• CPython 3.12+ is supported everywhere except on Windows until official PyICU wheels are available; see “Windows & PyICU” below.
+
+Package names
+• Runtime import path:  turkic\_translit
+• Distributable name on PyPI:  turkic\_transliterate
+• Command-line entry point:  turkic-translit
+
+## Developer Setup
+
+For the simplest developer setup experience, run the setup script:
+
+```bash
+python scripts/setup_dev.py
+```
+
+This script will:
+1. Install the package with all development dependencies
+2. Set up PyICU on Windows automatically
+3. Verify that development tools are working properly
+
+### Manual Installation
+
+Alternatively, install with pip:
+
+```bash
+pip install -e .[dev,ui]        # add ,winlid on Windows if you need fasttext-wheel
+```
+
+### Development Tools
+
+#### Linux/macOS/Windows with GNU Make
+
+If you have GNU Make installed, you can use the Makefile for common tasks:
+
+```bash
+make lint       # Run linting (ruff, black, mypy)
+make format     # Auto-format code
+make test       # Run tests
+make web        # Launch the web UI
+make help       # Show all available commands
+```
+
+#### Windows
+
+**Option 1: Install GNU Make using Chocolatey (Recommended)**
+
+Install GNU Make using Chocolatey (requires admin privileges):
+
+```powershell
+# In an Admin PowerShell window
+choco install make
+```
+
+After installation, you can use the same `make` commands as on Linux/macOS.
+
+**Option 2: Use the PowerShell Script Alternative**
+
+If you prefer not to install Chocolatey or GNU Make, use the PowerShell script:
+
+```powershell
+./scripts/run.ps1 lint       # Run linting
+./scripts/run.ps1 format     # Auto-format code
+./scripts/run.ps1 test       # Run tests
+./scripts/run.ps1 web        # Launch the web UI
+./scripts/run.ps1 help       # Show all available commands
+```
+
+Optional extras
+dev   → black, ruff, pytest
+ui    → gradio web demo
+winlid (Windows only) → fasttext-wheel for language ID
+
+Windows & PyICU
+
+**Important:** Due to PyPI rules, the correct PyICU wheel for Windows cannot be installed automatically during pip install. After installing this package with pip, Windows users must run the helper script to install the appropriate PyICU wheel:
+
+    turkic-pyicu-install
+
+This script will download and install the correct PyICU wheel from Christoph Gohlke’s repository based on your Python version. See the script for details.
+
+Command-line usage
+turkic-translit --lang kk --in text.txt --out\_latin kk\_lat.txt --ipa --out\_ipa kk\_ipa.txt --arabic --log-level debug
+• --lang            kk or ky
+• --ipa             emit IPA alongside Latin
+• --arabic          also transliterate embedded Arabic script
+• --benchmark       print throughput statistics
+• --log-level       debug | info | warning | error | critical (default: info)
+
+Logging
+The central logging setup uses Rich for colour when available.
+Set TURKIC\_LOG\_LEVEL or pass --log-level to the CLI.
+Fallback to standard logging when Rich is absent.
+
+# Project Organization
+
+The project is organized into the following directories:
+
+- `src/turkic_translit/` - Core source code for the package
+- `examples/` - Example scripts showing how to use the package
+  - `examples/web/` - Web interface for demonstrating transliteration features
+- `data/` - Sample data files and language resources
+- `docs/` - Documentation and reference materials
+- `scripts/` - Utility scripts for development and release
+  - `scripts/release/` - Scripts for building and publishing packages
+- `vendor/pyicu/` - Pre-built PyICU wheels for Windows
+- `tests/` - Test suite for the package
+
+## FastText Language Identification Model
+
+This package uses the [FastText language identification model](https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin) (`lid.176.bin`) for Russian token filtering and language detection. **The model file is not included in the repository or pip package due to its large size.**
+
+**Automatic Download:**
+- When you use features that require language identification (such as Russian token filtering or the Gradio web demo), the package will automatically download `lid.176.bin` from the official Facebook AI public link if it is not already present.
+- The file will be saved in the package directory on first use.
+
+**No manual action is needed.** This ensures compatibility with pip installs, Hugging Face Spaces, and other cloud environments.
+
+If you need to download the model manually, you can do so from:
+https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin
+
+
+## Using the Examples
+
+Use the main entry point script to run examples:
+
+```bash
+python turkic_tools.py [command]
+```
+
+Available commands:
+- `web` - Launch the Gradio web interface for real-time transliteration
+- `demo` - Run the simple CLI demo
+- `full-demo` - Run the comprehensive demo with multiple languages
+- `help` - Display available commands
+
+Tokenizer training example
+turkic-build-spm --input corpora/kk\_lat.txt,corpora/ky\_lat.txt --model\_prefix spm/turkic12k --vocab\_size 12000
+
+Filtering Russian tokens from Uzbek
+cat uz\_raw\.txt | turkic-filter-russian --mode drop > uz\_clean.txt
+
+Developer checklist
+black .
+ruff check .
+pytest -q
+
+All code is UTF-8-only; on Windows a BOM is written when piping to files to avoid encoding issues.
+
+License
+Apache-2.0
+
+### Type-checking
+
+```bash
+pip install mypy
+mypy --strict .
+```
+
+The included mypy.ini restricts analysis to the src/ tree and skips
+build/, dist/, virtual-env and egg directories so duplicate-module
+errors do not occur even if you build wheels locally.

turkic_translit-0.2.0/pyproject.toml

@@ -0,0 +1,139 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+
+[project]
+name = "turkic-translit"
+version = "0.2.0"
+description = "Deterministic Latin and IPA transliteration for Kazakh, Kyrgyz, plus tokenizer/glue scripts."
+authors = [ {name="Austin Wagner", email="austinwagner@msn.com"} ]
+requires-python = ">=3.9"
+readme = "README.md"
+license = "MIT"
+keywords = ["kazakh", "kyrgyz", "transliteration", "ipa"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Operating System :: OS Independent",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Text Processing :: Linguistic"
+]
+
+dependencies = [
+    # Core dependencies (alphabetized)
+    "epitran>=1.0,<1.27",           # 1.26.0 is the latest on PyPI
+    # Universal fasttext-wheel for all platforms and Python 3.10-3.13
+    "fasttext-wheel==0.9.2",
+    "numpy<2",
+    "packaging>=23.0",              # Used in tests/test_fasttext.py
+    "panphon>=0.20,<0.22",          # 0.21.2 is the newest published wheel
+    # PyICU is required, but Windows users must install the correct wheel manually. See README and turkic-pyicu-install for help.
+    # PyICU wheels: pre-built for Windows versions, fallback to PyPI elsewhere
+
+    "PyICU>=2.13,<2.16; sys_platform != 'win32'",
+    "pytest>=8.0",                  # Test runner
+    "rapidfuzz>=3.5",
+    "rich>=13.7",                   # Color-aware logging and console output
+    "sentencepiece>=0.2.0",
+    "click>=8.1",
+    "types-requests>=2.31",
+    "datasets (>=3.6.0,<4.0.0)",
+    "evaluate>=0.4",             # PPL metric
+    "accelerate>=0.28",          # HF Trainer device dispatch
+    "transformers>=4.41,<5",        # mGPT / BLOOM causal LM
+    "tqdm>=4.66",                # progress bars for LM utilities
+    "scikit-learn>=1.4",        # cosine similarity utilities
+    "typing_extensions>=4.12",    # for Never on Python ≤3.11
+    "wikipedia (>=1.4.0,<2.0.0)",
+    # UI dependencies
+    "gradio>=4.0",
+    "pandas",
+    "matplotlib>=3.9",           # cosine-similarity heat-map
+    "pycountry>=23.12",          # ISO language names for pretty dropdown labels
+]
+
+[project.optional-dependencies]
+examples = [
+    "flask",
+    "streamlit",
+    "jupyterlab",
+]
+winlid = []
+corpus = [
+    "datasets>=3.0",
+    "pyarrow>=14.0",
+    "requests>=2.0",
+    "zstandard>=0.23",
+]
+
+# Development tools
+dev = [
+    "ruff>=0.2.0",         # Fast linter
+    "mypy>=1.0",           # Type checker
+    "pytest>=8.0",         # Testing framework
+    "build>=1.0",          # Package builder
+    "twine>=4.0",          # Package uploader
+    "make>=0.1.6",         # Make command for Windows
+    "types-requests>=2.31", # Type stubs for requests
+]
+
+[project.scripts]
+turkic-translit = "turkic_translit.cli:main"
+turkic-pyicu-install = "turkic_translit.cli.pyicu_install:main"
+turkic-filter-russian = "turkic_translit.cli.filter_russian:main"
+turkic-download-corpus = "turkic_translit.cli.download_corpus:cli"
+turkic-build-spm = "turkic_translit.cli.build_spm:main"
+turkic-leven = "turkic_translit.cli.run_leven:main"
+turkic-web = "turkic_translit.web.web_demo:main"
+turkic-train-lm = "turkic_translit.cli.train_lm:cli"
+turkic-eval-lm  = "turkic_translit.cli.eval_lm:cli"
+turkic-train-spm = "turkic_translit.cli.train_spm:main"
+
+
+[tool.setuptools.packages]
+find = { where = ["src"] }
+
+[tool.setuptools]
+include-package-data = true
+package-dir = {"" = "src"}
+
+[tool.setuptools.package-data]
+"turkic_translit" = ["rules/*.rules"]
+"turkic_translit.cli" = ["corpora.yaml"]
+
+[tool.ruff]
+line-length = 88
+target-version = "py39"
+src = ["src", "tests", "examples"]
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "F",   # pyflakes
+    "I",   # isort
+    "UP",  # pyupgrade
+    "N",   # pep8-naming
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "SIM", # flake8-simplify
+    "PT",  # flake8-pytest-style
+    "RET", # flake8-return
+    "PIE", # flake8-pie
+]
+ignore = [
+    "E501"  # line too long (handled by formatter)
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["turkic_translit"]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]  # unused imports
+"tests/*" = ["E731"]      # do not assign a lambda expression
+"examples/*" = ["E402"]   # module level import not at top of file

turkic_translit-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

turkic_translit-0.2.0/src/turkic_translit/__init__.py

@@ -0,0 +1,53 @@
+import platform
+import sys
+from importlib.metadata import version
+
+# Public sub-packages (optional heavy deps)
+try:
+    from . import lm as lm  # noqa: F401
+except Exception:  # pragma: no cover – missing optional deps
+    lm = None  # type: ignore
+
+from . import patches as _patches  # noqa: F401
+from .core import to_ipa, to_latin
+from .logging_config import setup as _log_setup
+from .web.web_utils import (
+    direct_transliterate,
+    mask_russian,
+    median_levenshtein,
+    pipeline_transliterate,
+    token_table_markdown,
+)
+
+_log_setup()
+
+__all__ = [
+    "to_latin",
+    "to_ipa",
+    "direct_transliterate",
+    "pipeline_transliterate",
+    "token_table_markdown",
+    "mask_russian",
+    "median_levenshtein",
+    "lang_filter",
+    *(["lm"] if lm is not None else []),
+]
+# Retrieve version from package metadata. Package is published as "turkic-translit" on PyPI.
+__version__ = version("turkic-translit")
+
+# Warn Windows users on Python >=3.12 if PyICU is not importable
+if platform.system() == "Windows" and sys.version_info >= (3, 12):
+    try:
+        import PyICU  # noqa: F401
+    except ImportError:
+        import sys as _sys
+
+        _sys.stderr.write(
+            "[turkic-transliterate] ERROR: PyICU is not installed and cannot be built automatically on Windows for Python 3.12+!\n"
+            "To use this package, please create a virtual environment with Python 3.11 and install as follows:\n\n"
+            "    py -3.11 -m venv turkic311\n"
+            "    turkic311\\Scripts\\activate\n"
+            "    pip install turkic-transliterate\n"
+            "    turkic-pyicu-install\n\n"
+            "See the README for more details.\n"
+        )