indic-language-utils 0.1.0__tar.gz

indic_language_utils-0.1.0/.env.example

@@ -0,0 +1,4 @@
+# Bhashini inference API credentials (required for live tests and translation)
+BHASHINI_API_KEY="your-bhashini-api-key-here"
+BHASHINI_ENDPOINT_URL="https://dhruva-api.bhashini.gov.in/services/inference/pipeline"
+BHASHINI_TRANSLATION_SERVICE_ID="ai4bharat/indictrans-v2-all-gpu--t4"

indic_language_utils-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: uv sync --locked --dev
+      - run: uv run ruff format --check .
+      - run: uv run ruff check .
+      - run: uv run mypy
+      - run: uv run pytest
+      - run: uv build
+      - name: Test the built wheel
+        run: |
+          wheel_venv="$(mktemp -d)/venv"
+          uv venv "$wheel_venv"
+          uv pip install --python "$wheel_venv/bin/python" dist/*.whl
+          cd /tmp
+          "$wheel_venv/bin/python" -c 'import indic_language_utils; assert indic_language_utils.__version__ == "0.1.0"'

indic_language_utils-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  pypi-publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/indic-language-utils
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.12"
+
+      - name: Build package distributions
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

indic_language_utils-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+.venv/
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+__pycache__/
+*.py[cod]
+dist/
+site/
+.env
+.cache/

indic_language_utils-0.1.0/.indic-language-utils.toml

@@ -0,0 +1,31 @@
+[cache]
+enabled = true
+backend = "sqlite"
+path = ".cache/translations.sqlite3"
+namespace = "language-utils"
+max_entries = 50000
+ttl_seconds = 86400
+
+[retry]
+max_attempts = 3
+base_delay_seconds = 0.5
+max_delay_seconds = 2.0
+
+[telemetry]
+logging_enabled = true
+metrics_enabled = true
+traces_enabled = true
+include_content = false
+
+[providers.bhashini]
+endpoint = "https://dhruva-api.bhashini.gov.in/services/inference/pipeline"
+translation_service_id = "ai4bharat/indictrans-v2-all-gpu--t4"
+timeout_seconds = 20.0
+max_concurrency = 8
+
+[providers.googletrans]
+timeout_seconds = 20.0
+max_concurrency = 4
+
+[routes]
+translation = ["bhashini", "googletrans"]

indic_language_utils-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+        args: [--unsafe]
+      - id: check-toml
+      - id: check-added-large-files
+        args: [--maxkb=5120]
+
+  - repo: local
+    hooks:
+      - id: ruff-format
+        name: ruff format
+        entry: uv run ruff format
+        language: system
+        types_or: [python, markdown]
+        require_serial: true
+
+      - id: ruff-check
+        name: ruff check
+        entry: uv run ruff check --fix
+        language: system
+        types: [python]
+        require_serial: true
+
+      - id: mypy
+        name: mypy
+        entry: uv run mypy
+        language: system
+        types: [python]
+        pass_filenames: false

indic_language_utils-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Harikesh Kushwaha contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

indic_language_utils-0.1.0/PKG-INFO

@@ -0,0 +1,88 @@
+Metadata-Version: 2.5
+Name: indic-language-utils
+Version: 0.1.0
+Summary: Provider-neutral foundations for Indian language applications
+Project-URL: Documentation, https://github.com/indic-language-utils/indic-language-utils
+Project-URL: Source, https://github.com/indic-language-utils/indic-language-utils
+Author: indic-language-utils contributors
+License: MIT License
+        
+        Copyright (c) 2026 Harikesh Kushwaha contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: indic,language,speech,translation
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Typing :: Typed
+Requires-Python: <3.13,>=3.11
+Requires-Dist: httpx<1,>=0.28
+Provides-Extra: googletrans
+Requires-Dist: googletrans>=4.0.2; extra == 'googletrans'
+Provides-Extra: local-tld
+Requires-Dist: fasttext-langdetect>=1.1.1; extra == 'local-tld'
+Description-Content-Type: text/markdown
+
+# indic-language-utils
+
+A Python 3.11 and 3.12 library for Indian-language translation, text language detection, transliteration, speech-to-text, and text-to-speech. The project uses `uv` and will be published on PyPI as `indic-language-utils`. Its Python import is `indic_language_utils`. The code is licensed under the MIT License.
+
+Phase 0 implements the shared library foundation, Phase 1 adds structured translation with Bhashini and Google Translate adapters, and Phase 2 adds text language detection with local FastText and cloud Bhashini adapters. Runtime caching can use bounded process memory or persistent SQLite storage. The architecture and ecosystem review is in [plans/architecture-proposal.md](plans/architecture-proposal.md).
+
+The intended shape is a small core with optional provider packages. Applications should be able to start with local or low-cost providers during a proof of concept, then change routing configuration for production without rewriting text and audio handling.
+
+Work will be delivered in this order:
+
+- shared library foundation
+- text translation
+- text language detection, abbreviated as TLD
+- transliteration
+- non-streaming speech-to-text, abbreviated as STT
+- non-streaming text-to-speech, abbreviated as TTS
+
+Provider work starts with Bhashini and then Sarvam. Unofficial Google or Microsoft adapters can be added later as opt-in extras or external plugins. Optional dependency groups will keep translation-only installations free of speech and audio dependencies.
+
+Cross-cutting requirements include structured-text preservation, bounded concurrency, retries, routing and fallback, versioned terminology, caching, logging, metrics, and explicit data-retention controls.
+
+## Development
+
+Install the locked development environment with `uv sync --dev`. The usual checks are:
+
+```console
+uv run pre-commit install
+uv run ruff format .
+uv run ruff check .
+uv run mypy
+uv run pytest
+uv build
+uv run mkdocs serve
+uv run mkdocs build
+```
+
+The runtime uses HTTPX for Bhashini's reusable asynchronous HTTP client. For usage and guides, see:
+
+- [docs/user-guide.md](docs/user-guide.md) for system capabilities and getting started
+- [docs/translation.md](docs/translation.md) for translation features and examples
+- [docs/detection.md](docs/detection.md) for text language and script detection
+- [docs/configuration.md](docs/configuration.md) for TOML and environment loading
+- [docs/processors.md](docs/processors.md) for processor composition and markdown protection
+- [docs/adapter-author-guide.md](docs/adapter-author-guide.md) for provider integration

indic_language_utils-0.1.0/README.md

@@ -0,0 +1,44 @@
+# indic-language-utils
+
+A Python 3.11 and 3.12 library for Indian-language translation, text language detection, transliteration, speech-to-text, and text-to-speech. The project uses `uv` and will be published on PyPI as `indic-language-utils`. Its Python import is `indic_language_utils`. The code is licensed under the MIT License.
+
+Phase 0 implements the shared library foundation, Phase 1 adds structured translation with Bhashini and Google Translate adapters, and Phase 2 adds text language detection with local FastText and cloud Bhashini adapters. Runtime caching can use bounded process memory or persistent SQLite storage. The architecture and ecosystem review is in [plans/architecture-proposal.md](plans/architecture-proposal.md).
+
+The intended shape is a small core with optional provider packages. Applications should be able to start with local or low-cost providers during a proof of concept, then change routing configuration for production without rewriting text and audio handling.
+
+Work will be delivered in this order:
+
+- shared library foundation
+- text translation
+- text language detection, abbreviated as TLD
+- transliteration
+- non-streaming speech-to-text, abbreviated as STT
+- non-streaming text-to-speech, abbreviated as TTS
+
+Provider work starts with Bhashini and then Sarvam. Unofficial Google or Microsoft adapters can be added later as opt-in extras or external plugins. Optional dependency groups will keep translation-only installations free of speech and audio dependencies.
+
+Cross-cutting requirements include structured-text preservation, bounded concurrency, retries, routing and fallback, versioned terminology, caching, logging, metrics, and explicit data-retention controls.
+
+## Development
+
+Install the locked development environment with `uv sync --dev`. The usual checks are:
+
+```console
+uv run pre-commit install
+uv run ruff format .
+uv run ruff check .
+uv run mypy
+uv run pytest
+uv build
+uv run mkdocs serve
+uv run mkdocs build
+```
+
+The runtime uses HTTPX for Bhashini's reusable asynchronous HTTP client. For usage and guides, see:
+
+- [docs/user-guide.md](docs/user-guide.md) for system capabilities and getting started
+- [docs/translation.md](docs/translation.md) for translation features and examples
+- [docs/detection.md](docs/detection.md) for text language and script detection
+- [docs/configuration.md](docs/configuration.md) for TOML and environment loading
+- [docs/processors.md](docs/processors.md) for processor composition and markdown protection
+- [docs/adapter-author-guide.md](docs/adapter-author-guide.md) for provider integration

indic_language_utils-0.1.0/docs/adapter-author-guide.md

@@ -0,0 +1,36 @@
+# Adapter author guide
+
+An adapter is an object with a `ProviderIdentity` and a tuple of `CapabilityDeclaration` values. It implements only its concrete capability protocol. There is no shared provider base class.
+
+Register an adapter explicitly with `ProviderRegistry.register`. Declare canonical `LanguageTag` values, supported pairs, required features, and named limits. Do not place provider-specific language codes in public models. Convert them inside the adapter.
+
+Adapters that own reusable clients implement `start()` and `close()`. `ResourceManager` starts resources in order, closes them in reverse order, and supports `async with`.
+
+Map provider failures to the matching exception in `indic_language_utils.errors`. Exception messages and structured fields must not contain request content, credentials, headers, or raw provider responses. Mark only rate limits, timeouts, and temporary provider failures as retryable.
+
+This illustrative provider declares a fake operation. It is not a supported language capability:
+
+```python
+from dataclasses import dataclass
+
+from indic_language_utils.models import ProviderIdentity
+from indic_language_utils.providers import CapabilityDeclaration, CapabilityId, ProviderRegistry
+
+
+@dataclass
+class ExampleProvider:
+    identity = ProviderIdentity("example")
+    capabilities = (CapabilityDeclaration(CapabilityId.TRANSLATION),)
+
+    async def start(self) -> None:
+        pass
+
+    async def close(self) -> None:
+        pass
+
+
+registry = ProviderRegistry()
+registry.register(ExampleProvider())
+```
+
+Real translation models and provider methods arrive in Phase 1.

indic_language_utils-0.1.0/docs/configuration.md

@@ -0,0 +1,187 @@
+# Configuration
+
+The library uses a tiered configuration system combining TOML project files, user configurations, environment variables, and programmatic overrides. Settings are validated strictly on load; unknown keys or invalid data types raise a `ConfigurationError`.
+
+## Configuration Precedence
+
+`Settings.load()` evaluates configuration sources in the following order, from lowest to highest precedence:
+
+- Library defaults
+- User configuration at `~/.config/indic-language-utils/config.toml` (or `$XDG_CONFIG_HOME/indic-language-utils/config.toml`)
+- The nearest `.indic-language-utils.toml` found by walking upward from the current working directory
+- An explicit configuration file specified via `Settings.load(path=...)` or the `ILU_CONFIG_FILE` environment variable
+- Environment variables (`ILU_*`, `BHASHINI_*`, and `TRANSLATION_SERVICE_PROVIDER`)
+- Programmatic overrides passed via `Settings.load(overrides=...)`
+
+Only existing discovered files are merged. If an explicit configuration path is provided, that file must exist.
+
+## Project Configuration File
+
+A project configuration file (`.indic-language-utils.toml`) can be placed at the root of a project repository. The following example illustrates all supported configuration sections:
+
+```toml
+[cache]
+enabled = true
+backend = "sqlite"
+path = ".cache/translations.sqlite3"
+namespace = "my-application"
+max_entries = 50000
+ttl_seconds = 86400
+
+[retry]
+max_attempts = 3
+base_delay_seconds = 0.25
+max_delay_seconds = 5.0
+
+[telemetry]
+logging_enabled = true
+metrics_enabled = true
+traces_enabled = true
+include_content = false
+
+[providers.bhashini]
+endpoint = "https://dhruva-api.bhashini.gov.in/services/inference/pipeline"
+translation_service_id = "default-translation-model-id"
+detection_service_id = "default-tld-model-id"
+timeout_seconds = 20.0
+max_concurrency = 8
+
+[providers.bhashini.translation_service_ids]
+"hi-IN" = "service-for-any-source-to-hindi"
+"en-IN>ta-IN" = "service-for-english-to-tamil"
+
+[providers.googletrans]
+timeout_seconds = 20.0
+max_concurrency = 4
+
+[routes]
+translation = ["bhashini", "googletrans"]
+text_language_detection = ["bhashini", "fasttext"]
+```
+
+Relative cache paths resolve relative to the current working directory of the process. In production containers or multi-directory environments, specify an absolute path.
+
+## Secrets and Environment Variables
+
+TOML files are strictly prohibited from storing credentials, passwords, tokens, or API keys. Attempting to define fields containing sensitive terms (such as `api_key`, `token`, `secret`, or `password`) raises a `ConfigurationError`.
+
+All sensitive values must be supplied via environment variables.
+
+### Provider Credentials and Overrides
+
+Bhashini credentials and service endpoints:
+
+```console
+# Required for Bhashini live API calls
+export BHASHINI_API_KEY="your-bhashini-api-key"
+
+# Optional overrides for endpoints and service IDs
+export BHASHINI_ENDPOINT_URL="https://dhruva-api.bhashini.gov.in/services/inference/pipeline"
+export BHASHINI_TRANSLATION_SERVICE_ID="your-translation-service-id"
+export BHASHINI_DETECTION_SERVICE_ID="your-tld-service-id"
+export BHASHINI_TIMEOUT_SECONDS="20"
+export BHASHINI_MAX_CONCURRENCY="8"
+```
+
+To quickly select an active translation provider during development without editing configuration files:
+
+```console
+export TRANSLATION_SERVICE_PROVIDER="googletrans"
+```
+
+### Shared System Settings
+
+Global cache, retry, telemetry, and routing settings use the `ILU_` prefix:
+
+```console
+# Caching settings
+export ILU_CACHE_ENABLED="true"
+export ILU_CACHE_BACKEND="sqlite"
+export ILU_CACHE_PATH="/var/lib/indic-language-utils/cache.sqlite3"
+export ILU_CACHE_NAMESPACE="my-app"
+export ILU_CACHE_MAX_ENTRIES="50000"
+export ILU_CACHE_TTL_SECONDS="86400"
+
+# Retry settings
+export ILU_RETRY_MAX_ATTEMPTS="3"
+export ILU_RETRY_BASE_DELAY_SECONDS="0.25"
+export ILU_RETRY_MAX_DELAY_SECONDS="5.0"
+
+# Telemetry settings
+export ILU_LOGGING_ENABLED="true"
+export ILU_METRICS_ENABLED="true"
+export ILU_TRACES_ENABLED="true"
+export ILU_TELEMETRY_INCLUDE_CONTENT="false"
+
+# Route overrides (comma-separated provider names in priority order)
+export ILU_ROUTE_TRANSLATION="bhashini,googletrans"
+export ILU_ROUTE_TEXT_LANGUAGE_DETECTION="bhashini,fasttext"
+```
+
+## Programmatic Loading and Overrides
+
+### Automatic Discovery
+
+The easiest way to initialize a client is via the built-in factories, which call `Settings.load()` automatically:
+
+```python
+from indic_language_utils import get_translation_client
+
+# Automatically reads .indic-language-utils.toml and environment variables
+client = get_translation_client()
+```
+
+### Explicit Settings Loading
+
+For granular control over settings loading:
+
+```python
+from pathlib import Path
+from indic_language_utils import Settings
+
+# Load settings from a specific TOML file
+settings = Settings.load(path=Path("/etc/indic-language-utils/production.toml"))
+
+# Load settings with test overrides
+test_settings = Settings.load(
+    overrides={
+        "cache": {"enabled": False},
+        "retry": {"max_attempts": 1},
+    }
+)
+```
+
+### Building Custom Clients from Settings
+
+Assemble a client pipeline explicitly using loaded settings:
+
+```python
+from indic_language_utils import (
+    BhashiniConfig,
+    BhashiniTranslationProvider,
+    CacheKeyBuilder,
+    CapabilityId,
+    GoogleTranslateProvider,
+    OrderedRouter,
+    ProviderRegistry,
+    Settings,
+    TranslationClient,
+    create_translation_cache,
+)
+
+settings = Settings.load()
+
+registry = ProviderRegistry()
+if "bhashini" in settings.providers:
+    registry.register(BhashiniTranslationProvider(BhashiniConfig.from_settings(settings)))
+registry.register(GoogleTranslateProvider())
+
+route = settings.routes.get("translation", ("bhashini", "googletrans"))
+router = OrderedRouter(registry, {CapabilityId.TRANSLATION: route})
+
+client = TranslationClient(
+    router=router,
+    cache=create_translation_cache(settings.cache),
+    cache_keys=CacheKeyBuilder(settings.cache.namespace),
+)
+```