dharma-transcribe 0.1.0__tar.gz

dharma_transcribe-0.1.0/.env.example

@@ -0,0 +1,44 @@
+# Dharma Transcription Pipeline — Environment Variables
+# Copy to .env and fill in your values. See .gitignore — .env is never committed.
+
+# --- Required for LLM correction (Stage 6) ---
+# Any OpenAI-compatible API endpoint. Examples:
+#   Synthetic.new: https://api.example.com/v1
+#   OpenAI:        https://api.openai.com/v1
+#   Ollama (local): http://localhost:11434/v1
+#   vLLM (local):  http://localhost:8000/v1
+#   LM Studio:     http://localhost:1234/v1
+DHARMA_LLM_API_URL=https://api.example.com/v1
+DHARMA_LLM_API_KEY=your-api-key-here
+DHARMA_LLM_MODEL=hf:openai/gpt-oss-120b
+
+# --- Required for diarization (Stage 4) ---
+# HuggingFace token for pyannote speaker-diarization-community-1 model.
+# Without this, diarization is skipped (transcription still completes).
+# Get one at: https://huggingface.co/settings/tokens
+HF_TOKEN=hf_your_token_here
+
+# --- Optional overrides ---
+# Compute device: "cuda" (GPU) or "cpu"
+# DHARMA_DEVICE=cuda
+
+# WhisperX model size: large-v3, large-v2, medium, small, etc.
+# DHARMA_WHISPER_MODEL=large-v3
+
+# Compute type: int8, float16, int8_float16, etc.
+# DHARMA_COMPUTE_TYPE=int8
+
+# Batch size for WhisperX transcription
+# DHARMA_BATCH_SIZE=4
+
+# Tibetan second-pass model (OpenPecha dharma-trained)
+# DHARMA_TIBETAN_MODEL=openpecha/op-whisper_small-ft-v2
+
+# Output directory (defaults to ./output)
+# DHARMA_OUTPUT_DIR=/path/to/output
+
+# Log directory (defaults to ./logs)
+# DHARMA_LOG_DIR=/path/to/logs
+
+# Confidence threshold for review queue (0.0-1.0)
+# DHARMA_CONFIDENCE_THRESHOLD=0.5

dharma_transcribe-0.1.0/.github/FUNDING.yml

@@ -0,0 +1,2 @@
+# Cryptocurrency donations — see README Support section for addresses
+custom: ["https://github.com/guan-tends#support"]

dharma_transcribe-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - run: pip install ruff
+      - run: ruff check src/ tests/
+      - run: ruff format --check src/ tests/
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - run: pip install mypy
+      - run: mypy src/dharma_transcribe/
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e ".[dev]"
+      - run: pytest tests/ --ignore=tests/test_e2e.py --cov=dharma_transcribe --cov-report=xml --cov-report=term-missing
+      - uses: actions/upload-artifact@v4
+        if: matrix.python-version == '3.10'
+        with:
+          name: coverage-report
+          path: coverage.xml

dharma_transcribe-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,38 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+# Trusted publishing via OIDC — no API tokens needed.
+# Configure at: https://pypi.org/manage/account/publishing/
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - run: pip install build
+      - run: python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # Required for trusted publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

dharma_transcribe-0.1.0/.gitignore

@@ -0,0 +1,19 @@
+venv/
+.venv/
+output/wav/
+output/json/
+output/srt/
+output/vtt/
+output/txt/
+output/review/
+output/corrections/
+output/manifest.json
+__pycache__/
+*.pyc
+.env
+models/
+logs/
+tests/test_e2e_llm.py
+output/
+tests/test_e2e_llm.py
+tests/test_e2e_llm.py

dharma_transcribe-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.10.0
+    hooks:
+      - id: mypy
+        additional_dependencies: []
+        args: [--ignore-missing-imports]

dharma_transcribe-0.1.0/CHANGELOG.md

@@ -0,0 +1,41 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- GitHub Actions CI pipeline (lint, typecheck, test on Python 3.10/3.11/3.12)
+- PyPI publishing workflow (trusted publishing via OIDC)
+- Pre-commit hooks configuration (ruff + mypy)
+- `.env.example` with all configurable environment variables
+- `CONTRIBUTING.md` with development setup and testing guide
+- `tests/fixtures/` directory with README for test audio samples
+
+## [0.1.0] - 2026-08-23
+
+### Added
+- 7-stage pipeline architecture: Ingest → WhisperX → Alignment → Diarization → Tibetan Second-Pass → LLM Correction → Output
+- WhisperX large-v3 primary transcription with automatic language detection
+- Per-language forced alignment (wav2vec2 for en, ja, bo, sa)
+- Speaker diarization via pyannote speaker-diarization-community-1
+- Tibetan second-pass transcription using OpenPecha op-whisper_small-ft-v2 (dharma-trained)
+- LLM post-correction via any OpenAI-compatible API (cloud or local)
+- Output formats: JSON, SRT, VTT, TXT, review queue
+- Corrections dictionary for deterministic pre-LLM fixes
+- Idempotent manifest for batch processing
+- VRAM management with model flushing between stages (6 GB consumer GPU support)
+- `--device cpu` flag for CPU-only mode
+- `--skip-llm` flag to skip LLM correction
+- Environment-variable-based configuration (no hardcoded secrets)
+- `src/` package layout with `pyproject.toml` (hatchling build backend)
+- 54 unit tests covering all modules
+- MIT license
+
+### Security
+- Git history scrubbed of all API keys, URLs, and private paths
+- All sensitive values read from environment variables
+- No credentials in source code or commit history

dharma_transcribe-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,85 @@
+# Contributing
+
+## Development Setup
+
+```bash
+git clone https://github.com/guan-tends/dharma-transcribe.git
+cd dharma-transcribe
+python3 -m venv venv
+source venv/bin/activate
+pip install -e ".[dev]"
+pre-commit install
+```
+
+## Workflow
+
+1. Create a branch: `git checkout -b feature/your-feature`
+2. Make changes. Commit early, commit often.
+3. Run checks before pushing:
+   ```bash
+   ruff check src tests
+   ruff format --check src tests
+   mypy src/dharma_transcribe
+   pytest
+   ```
+4. Push and open a PR.
+
+## Testing
+
+```bash
+# Run all tests
+pytest
+
+# Run without GPU tests
+pytest -m "not gpu"
+
+# Run with coverage
+pytest --cov=dharma_transcribe --cov-report=term-missing
+
+# Run a single module
+pytest tests/test_config.py
+```
+
+### Adding Test Fixtures
+
+Place small audio samples in `tests/fixtures/`. A fixture should be:
+- Short (5–30 seconds)
+- Royalty-free or your own recording
+- Named descriptively: `english_sample.wav`, `tibetan_sample.wav`
+
+Generate a test WAV with ffmpeg:
+```bash
+ffmpeg -f lavfi -i "sine=frequency=440:duration=5" -ar 16000 -ac 1 tests/fixtures/tone_5s.wav
+```
+
+## Code Style
+
+- **Linter**: ruff (replaces flake8 + isort + black)
+- **Formatter**: ruff format
+- **Type checking**: mypy (non-strict, `ignore_missing_imports = true`)
+- **Line length**: 100 characters
+- **Python**: 3.10+ (uses `str | None` union syntax)
+
+## Project Structure
+
+```
+src/dharma_transcribe/     # Package source
+  __init__.py              # Package metadata
+  cli.py                  # CLI entry point (argparse)
+  pipeline.py             # Stage orchestration
+  config.py               # Environment-based configuration
+  ingest.py               # Stage 1: audio extraction
+  transcribe.py           # Stage 2: WhisperX ASR
+  align.py                # Stage 3: forced alignment
+  diarize.py              # Stage 4: speaker diarization
+  tibetan_second_pass.py  # Stage 5: Tibetan re-transcription
+  llm_correct.py          # Stage 6: LLM post-correction
+  output.py               # Stage 7: multi-format output
+  gpu.py                  # GPU memory utilities
+tests/                    # Test suite
+  conftest.py             # Shared fixtures
+  test_*.py               # Unit and integration tests
+  fixtures/               # Test audio samples
+pyproject.toml            # Project metadata, tool config
+.env.example              # Environment variable template
+```

dharma_transcribe-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 David Newman & Guan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dharma_transcribe-0.1.0/PKG-INFO

@@ -0,0 +1,227 @@
+Metadata-Version: 2.5
+Name: dharma-transcribe
+Version: 0.1.0
+Summary: Headless multilingual transcription pipeline for Buddhist dharma teachings
+Project-URL: Homepage, https://github.com/guan-tends/dharma-transcribe
+Project-URL: Repository, https://github.com/guan-tends/dharma-transcribe
+Author: David Newman & Guan
+License-Expression: MIT
+License-File: LICENSE
+Keywords: asr,buddhist,dharma,multilingual,sanskrit,speech-to-text,tibetan,transcription,whisper,whisperx
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
+Classifier: Topic :: Religion
+Requires-Python: >=3.10
+Requires-Dist: openai>=1.0
+Requires-Dist: requests>=2.28
+Requires-Dist: tiktoken>=0.5
+Requires-Dist: torch>=2.0
+Requires-Dist: torchaudio>=2.0
+Requires-Dist: transformers>=4.40
+Requires-Dist: whisperx>=3.8.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pre-commit>=3.7; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest-mock>=3.12; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Dharma Transcription Pipeline
+
+A headless multilingual transcription pipeline designed for Buddhist dharma teachings. Handles mixed-language audio (Tibetan, Sanskrit, English, Japanese) with per-language forced alignment, speaker diarization, a Tibetan second-pass using dharma-trained models, and optional LLM post-correction.
+
+**Privacy-first**: all processing runs locally. Sacred content never touches a cloud unless you explicitly configure LLM correction with a cloud API.
+
+## Architecture — 7 Stages
+
+```
+Audio/Video → Ingest → WhisperX → Alignment → Diarization → Tibetan 2nd-Pass → LLM Correction → Output
+```
+
+| Stage | Purpose | Model | GPU Memory |
+|-------|---------|-------|-----------|
+| 1. Ingest | ffmpeg extract to 16kHz mono WAV, checksum idempotency | ffmpeg | — |
+| 2. Transcription | Primary ASR with auto language detection | WhisperX large-v3 (int8) | ~4 GB |
+| 3. Alignment | Word-level timestamps per language | wav2vec2 (per-language) | ~1–2 GB |
+| 4. Diarization | Speaker identification | pyannote diarization-community-1 | ~1 GB |
+| 5. Tibetan 2nd-Pass | Re-transcribe Tibetan segments with dharma-trained model | OpenPecha op-whisper_small-ft-v2 | ~1 GB |
+| 6. LLM Correction | Post-correction with dharma domain knowledge | Any OpenAI-compatible API | (cloud or local) |
+| 7. Output | JSON, SRT, VTT, TXT, review queue | — | — |
+
+Stages run serially with GPU memory flushing between each — designed for 6 GB consumer GPUs.
+
+## Setup
+
+### Prerequisites
+
+- Python 3.10+
+- ffmpeg + ffprobe (system install)
+- NVIDIA GPU with CUDA (optional — `--device cpu` works for all stages)
+- HuggingFace token (for diarization only — transcription works without it)
+
+### Installation
+
+```bash
+# Clone
+git clone https://github.com/guan-tends/dharma-transcribe.git
+cd dharma-transcribe
+
+# Create virtual environment
+python3 -m venv venv
+source venv/bin/activate
+
+# Install with pip
+pip install -e ".[dev]"
+
+# Or install from PyPI (when published)
+pip install dharma-transcribe[dev]
+```
+
+### GPU Setup (optional but recommended)
+
+```bash
+# Install PyTorch with CUDA support (adjust cuXXX for your CUDA version)
+pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu128
+```
+
+### HuggingFace Token (for diarization)
+
+The pipeline needs a HuggingFace token to download the pyannote diarization model. Without it, diarization is skipped — transcription still completes normally.
+
+```bash
+# Get a token: https://huggingface.co/settings/tokens
+# Also accept the model license: https://huggingface.co/pyannote/speaker-diarization-community-1
+
+export HF_TOKEN=hf_your_token_here
+```
+
+### LLM Correction Configuration (optional)
+
+Stage 6 uses any OpenAI-compatible API. Configure via environment variables:
+
+```bash
+# Cloud API example (Synthetic.new)
+export DHARMA_LLM_API_URL=https://api.example.com/v1
+export DHARMA_LLM_API_KEY=your-key
+export DHARMA_LLM_MODEL=hf:openai/gpt-oss-120b
+
+# Local model example (Ollama)
+ollama pull qwen3.5:9b
+export DHARMA_LLM_API_URL=http://localhost:11434/v1
+export DHARMA_LLM_API_KEY=ollama
+export DHARMA_LLM_MODEL=qwen3.5:9b
+
+# vLLM example (local GPU)
+# Start vLLM server: vllm serve Qwen/Qwen3.5-9B
+export DHARMA_LLM_API_URL=http://localhost:8000/v1
+export DHARMA_LLM_API_KEY=none
+export DHARMA_LLM_MODEL=Qwen/Qwen3.5-9B
+```
+
+Or skip LLM correction entirely: `--skip-llm`
+
+## Usage
+
+```bash
+# Single file
+dharma-transcribe /path/to/teaching.mp4
+
+# Directory (batch — finds all audio/video recursively)
+dharma-transcribe /path/to/recordings/
+
+# Skip LLM correction (ASR only, faster)
+dharma-transcribe /path/to/teaching.mp4 --skip-llm
+
+# CPU-only mode (no GPU required)
+dharma-transcribe /path/to/teaching.mp4 --device cpu --skip-llm
+
+# With explicit HF token
+dharma-transcribe /path/to/teaching.mp4 --hf-token $HF_TOKEN
+```
+
+### CLI Flags
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `input` (positional) | — | File or directory to process |
+| `--source-dir` | — | Default source directory |
+| `--skip-llm` | off | Skip LLM correction stage |
+| `--hf-token` | `$HF_TOKEN` | HuggingFace token for diarization |
+| `--device` | `cuda` | Compute device: `cuda` or `cpu` |
+
+### Environment Variables
+
+See `.env.example` for the full list. Key variables:
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `DHARMA_LLM_API_URL` | (empty) | OpenAI-compatible API endpoint |
+| `DHARMA_LLM_API_KEY` | (empty) | API key for LLM correction |
+| `DHARMA_LLM_MODEL` | (empty) | Model name for LLM correction |
+| `HF_TOKEN` | (empty) | HuggingFace token for diarization |
+| `DHARMA_DEVICE` | `cuda` | Compute device |
+| `DHARMA_WHISPER_MODEL` | `large-v3` | WhisperX model size |
+| `DHARMA_OUTPUT_DIR` | `./output` | Output directory |
+| `DHARMA_TIBETAN_MODEL` | `openpecha/op-whisper_small-ft-v2` | Tibetan second-pass model |
+
+## Output Formats
+
+Each processed file generates:
+
+- **JSON** — full structured transcript with word-level timestamps, speaker labels, confidence scores
+- **SRT** — subtitle file with speaker labels
+- **VTT** — WebVTT subtitles with speaker tags
+- **TXT** — plain text reading copy
+- **Review queue** — JSON of low-confidence segments for manual review
+
+Outputs are written to `output/{json,srt,vtt,txt,review}/`.
+
+## Corrections Dictionary
+
+`output/corrections/corrections.json` — case-insensitive string replacement applied before LLM correction. Grows from manual review.
+
+```json
+{
+  "corrections": [
+    {"pattern": "bodichita", "replacement": "bodhicitta"},
+    {"pattern": "shun yata", "replacement": "shunyata"}
+  ]
+}
+```
+
+## Idempotency
+
+The manifest (`output/manifest.json`) tracks processed files. Re-running the pipeline skips completed files. Delete the manifest to reprocess everything.
+
+## VRAM Management
+
+Designed for consumer GPUs (6 GB+). Stages run serially with `gc.collect()` + `torch.cuda.empty_cache()` between each. No CPU fallback needed — the GPU is flushed fully before loading the next model.
+
+## Why This Exists
+
+Most transcription tools handle single languages. Dharma teachings commonly mix Tibetan, Sanskrit, English, and sometimes Japanese in a single recording. This pipeline:
+
+1. **Detects language per segment** — not per file
+2. **Aligns each language separately** — wav2vec2 models for bo, sa, en, ja
+3. **Re-transcribes Tibetan** — OpenPecha's model (trained on Garchen Rinpoche's teachings) often outperforms WhisperX on Tibetan
+4. **Corrects with dharma knowledge** — LLM post-correction knows bodhicitta from bodichita
+
+## License
+
+MIT — see [LICENSE](LICENSE).
+
+## Support
+
+If this pipeline helps preserve dharma teachings, consider supporting its continued development:
+
+- **Solana**: `Eu8wQcW68TKMs1a6eqzZu8znzU52QLqQugAMG8uCD6y6`
+- **EVM** (Ethereum / Base / Arbitrum / Optimism / Polygon): `0x2733ff7c865C56d565a99BE1DC11B81cc76850A5`
+- **XRP Ledger**: `r4X6e7McAQj7e8vBCeued1RYu4mCJrREDG`