distill-align 0.1.1__tar.gz

distill_align-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Distill-Align Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

distill_align-0.1.1/PKG-INFO

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: distill-align
+Version: 0.1.1
+Summary: The Structured Reasoning Extraction Factory — Generate high-quality fine-tuning datasets from raw domain data
+License: MIT
+License-File: LICENSE
+Keywords: llm,fine-tuning,dataset,distillation,unsloth,synthesis
+Author: o69dn
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Provides-Extra: all
+Provides-Extra: hub
+Provides-Extra: parquet
+Requires-Dist: beautifulsoup4 (>=4.12.0)
+Requires-Dist: httpx (>=0.27.0)
+Requires-Dist: huggingface_hub (>=0.23.0,<0.24.0) ; extra == "hub" or extra == "all"
+Requires-Dist: jinja2 (>=3.1.0,<4.0.0)
+Requires-Dist: loguru (>=0.7.0,<0.8.0)
+Requires-Dist: pyarrow (>=14.0.0) ; extra == "parquet" or extra == "all"
+Requires-Dist: pydantic (>=2.7.0,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.2.0,<3.0.0)
+Requires-Dist: pypdf (>=4.2.0,<5.0.0)
+Requires-Dist: python-docx (>=1.1.0)
+Requires-Dist: pyyaml (>=6.0,<7.0)
+Requires-Dist: rich (>=13.7.0,<14.0.0)
+Requires-Dist: textual (>=0.58.0,<0.59.0)
+Requires-Dist: tiktoken (>=0.7.0)
+Requires-Dist: typer (>=0.13.0)
+Project-URL: Documentation, https://o69dn.github.io/Distill-Align/
+Project-URL: Homepage, https://github.com/o69dn/Distill-Align
+Project-URL: Repository, https://github.com/o69dn/Distill-Align
+Description-Content-Type: text/markdown
+
+# Distill-Align
+
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![CI](https://github.com/o69dn/Distill-Align/actions/workflows/ci.yml/badge.svg)](https://github.com/o69dn/Distill-Align/actions/workflows/ci.yml)
+[![PyPI version](https://img.shields.io/pypi/v/distill-align.svg)](https://pypi.org/project/distill-align/)
+[![Security](https://github.com/o69dn/Distill-Align/actions/workflows/security-scan.yml/badge.svg)](https://github.com/o69dn/Distill-Align/actions/workflows/security-scan.yml)
+
+> **Distill-Align: The Structured Reasoning Extraction Factory**
+>
+> A CLI/Python framework that automates the generation of high-quality, fine-tuning datasets from raw domain data. It utilizes high-context frontier reasoning models as teachers, captures their deep thinking traces, and filters/prunes those traces into highly structured instruction-following formats optimized for fine-tuning.
+
+🌐 **العربية**: [README.ar.md](README.ar.md) — الترجمة العربية لهذا الدليل.
+
+## Features
+
+- **Smart Ingestion**: Async chunking pipelines with semantic-aware splitting for Markdown and Code (PDF, DOCX, HTML, CSV, JSON, Jupyter notebooks, and web pages also supported).
+- **Multi-Provider Synthesis**: Supports **OpenAI**, **Ollama**, **vLLM**, **Anthropic Claude**, **Google Gemini**, and **Azure OpenAI** backends with async worker pools.
+- **Socratic Transformer**: Converts raw reasoning into structured, multi-turn conversational Q&A.
+- **Scaffold Action Pruner**: Strips conversational filler to extract pure tool-calling or structural output.
+- **LLM-as-Judge Evaluation** (optional): Automated quality scoring of generated conversations using a separate judge model, with confidence scores normalized 0–1.
+- **Preference (DPO) Generation**: Create preference pairs from judge-scored conversations for Direct Preference Optimization training.
+- **Multiple Export Formats**: ShareGPT, Alpaca, ChatML, HuggingFace messages (JSONL/JSON), streaming JSON Lines, and **Apache Parquet**.
+- **Streaming Export**: Export large datasets without loading them entirely into memory using iterative producers.
+- **Cost Tracking**: Pay-as-you-go cost estimation and tracking across all providers with per-request token accounting.
+- **Unsloth Integration**: Auto-generates optimized `train.py` scripts for Unsloth fine-tuning.
+- **Rich TUI**: Interactive terminal dashboard for monitoring pipeline execution.
+
+## Installation
+
+```bash
+pip install distill-align
+
+# With optional dependencies
+pip install distill-align[parquet]   # Parquet export support
+pip install distill-align[hub]       # HuggingFace Hub integration
+pip install distill-align[all]       # All extras
+```
+
+## Quick Start
+
+```bash
+# Ingest and process data
+distill-align ingest --source ./my-docs --output ./chunks.json
+
+# Synthesize conversations (with judge evaluation)
+distill-align synthesize \
+    --input ./chunks.json \
+    --output ./conversations.json \
+    --provider openai \
+    --model gpt-4o \
+    --judge \
+    --judge-model gpt-4o-mini
+
+# Export to training format
+distill-align export \
+    --input ./conversations.json \
+    --format hf_messages \
+    --output ./dataset
+
+# Generate preference pairs for DPO training
+distill-align export \
+    --input ./conversations.json \
+    --format preference \
+    --output ./dpo-pairs
+
+# Launch TUI
+distill-align tui
+```
+
+## Supported Providers
+
+| Provider    | SDK-Free | Structured Output | Auth                          |
+|-------------|----------|-------------------|-------------------------------|
+| OpenAI      | ✓        | ✓                 | API key                       |
+| Anthropic   | ✓        | ✓ (JSON mode)     | API key                       |
+| Google Gemini | ✓      | ✓ (MIME type)     | API key                       |
+| Azure OpenAI | ✓       | ✓                 | API key or Entra ID (OAuth2)  |
+| Ollama      | ✓        | —                 | None (local)                  |
+| vLLM        | ✓        | ✓ (OpenAI compat) | None / API key                |
+
+## Export Formats
+
+| Format             | Extension | Description                                    |
+|--------------------|-----------|------------------------------------------------|
+| `hf_messages`      | `.jsonl`  | HuggingFace messages format (JSONL recommended) |
+| `jsonl`            | `.jsonl`  | Generic JSON Lines (streaming-capable)         |
+| `parquet`          | `.parquet`| Columnar format (requires `pyarrow`)           |
+| `sharegpt`         | `.json`   | ShareGPT conversation format                   |
+| `alpaca`           | `.json`   | Alpaca instruction format                      |
+| `chatml`           | `.json`   | ChatML markup format                           |
+| `conversation`     | `.json`   | Raw conversation schema export                 |
+| `preference`       | `.json`   | DPO preference pairs (requires judge scores)   |
+
+## Project Structure
+
+This project follows a **Modular Monolith** architecture.
+
+```text
+distill-align/
+├── src/distill_align/    # Core application package
+│   ├── core/             # Config, schemas, logging, caching, checkpointing
+│   ├── ingestion/        # Data loaders and chunkers (PDF, DOCX, HTML, code, etc.)
+│   ├── synthesis/        # LLM clients, worker pool, prompts, judge, cost tracking
+│   │   └── models/       # Provider-specific clients (OpenAI, Anthropic, Gemini, Azure, Ollama, vLLM)
+│   ├── exporter/         # Formatters, validator, splitter, preference generator
+│   │   └── formatters/   # Output format converters (JSONL, Parquet, ShareGPT, Alpaca, etc.)
+│   ├── tui/              # Textual terminal UI
+│   └── cli/              # Typer CLI entry points
+├── tests/                # Pytest suite
+└── docs/                 # Documentation (MkDocs)
+```
+
+## Development
+
+1. Clone the repository
+2. Install dependencies with Poetry: `poetry install`
+3. Install dev dependencies: `poetry install --with dev`
+4. Run tests: `poetry run pytest`
+5. Run linting: `poetry run ruff check src/`
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.
+

distill_align-0.1.1/README.md

@@ -0,0 +1,124 @@
+# Distill-Align
+
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![CI](https://github.com/o69dn/Distill-Align/actions/workflows/ci.yml/badge.svg)](https://github.com/o69dn/Distill-Align/actions/workflows/ci.yml)
+[![PyPI version](https://img.shields.io/pypi/v/distill-align.svg)](https://pypi.org/project/distill-align/)
+[![Security](https://github.com/o69dn/Distill-Align/actions/workflows/security-scan.yml/badge.svg)](https://github.com/o69dn/Distill-Align/actions/workflows/security-scan.yml)
+
+> **Distill-Align: The Structured Reasoning Extraction Factory**
+>
+> A CLI/Python framework that automates the generation of high-quality, fine-tuning datasets from raw domain data. It utilizes high-context frontier reasoning models as teachers, captures their deep thinking traces, and filters/prunes those traces into highly structured instruction-following formats optimized for fine-tuning.
+
+🌐 **العربية**: [README.ar.md](README.ar.md) — الترجمة العربية لهذا الدليل.
+
+## Features
+
+- **Smart Ingestion**: Async chunking pipelines with semantic-aware splitting for Markdown and Code (PDF, DOCX, HTML, CSV, JSON, Jupyter notebooks, and web pages also supported).
+- **Multi-Provider Synthesis**: Supports **OpenAI**, **Ollama**, **vLLM**, **Anthropic Claude**, **Google Gemini**, and **Azure OpenAI** backends with async worker pools.
+- **Socratic Transformer**: Converts raw reasoning into structured, multi-turn conversational Q&A.
+- **Scaffold Action Pruner**: Strips conversational filler to extract pure tool-calling or structural output.
+- **LLM-as-Judge Evaluation** (optional): Automated quality scoring of generated conversations using a separate judge model, with confidence scores normalized 0–1.
+- **Preference (DPO) Generation**: Create preference pairs from judge-scored conversations for Direct Preference Optimization training.
+- **Multiple Export Formats**: ShareGPT, Alpaca, ChatML, HuggingFace messages (JSONL/JSON), streaming JSON Lines, and **Apache Parquet**.
+- **Streaming Export**: Export large datasets without loading them entirely into memory using iterative producers.
+- **Cost Tracking**: Pay-as-you-go cost estimation and tracking across all providers with per-request token accounting.
+- **Unsloth Integration**: Auto-generates optimized `train.py` scripts for Unsloth fine-tuning.
+- **Rich TUI**: Interactive terminal dashboard for monitoring pipeline execution.
+
+## Installation
+
+```bash
+pip install distill-align
+
+# With optional dependencies
+pip install distill-align[parquet]   # Parquet export support
+pip install distill-align[hub]       # HuggingFace Hub integration
+pip install distill-align[all]       # All extras
+```
+
+## Quick Start
+
+```bash
+# Ingest and process data
+distill-align ingest --source ./my-docs --output ./chunks.json
+
+# Synthesize conversations (with judge evaluation)
+distill-align synthesize \
+    --input ./chunks.json \
+    --output ./conversations.json \
+    --provider openai \
+    --model gpt-4o \
+    --judge \
+    --judge-model gpt-4o-mini
+
+# Export to training format
+distill-align export \
+    --input ./conversations.json \
+    --format hf_messages \
+    --output ./dataset
+
+# Generate preference pairs for DPO training
+distill-align export \
+    --input ./conversations.json \
+    --format preference \
+    --output ./dpo-pairs
+
+# Launch TUI
+distill-align tui
+```
+
+## Supported Providers
+
+| Provider    | SDK-Free | Structured Output | Auth                          |
+|-------------|----------|-------------------|-------------------------------|
+| OpenAI      | ✓        | ✓                 | API key                       |
+| Anthropic   | ✓        | ✓ (JSON mode)     | API key                       |
+| Google Gemini | ✓      | ✓ (MIME type)     | API key                       |
+| Azure OpenAI | ✓       | ✓                 | API key or Entra ID (OAuth2)  |
+| Ollama      | ✓        | —                 | None (local)                  |
+| vLLM        | ✓        | ✓ (OpenAI compat) | None / API key                |
+
+## Export Formats
+
+| Format             | Extension | Description                                    |
+|--------------------|-----------|------------------------------------------------|
+| `hf_messages`      | `.jsonl`  | HuggingFace messages format (JSONL recommended) |
+| `jsonl`            | `.jsonl`  | Generic JSON Lines (streaming-capable)         |
+| `parquet`          | `.parquet`| Columnar format (requires `pyarrow`)           |
+| `sharegpt`         | `.json`   | ShareGPT conversation format                   |
+| `alpaca`           | `.json`   | Alpaca instruction format                      |
+| `chatml`           | `.json`   | ChatML markup format                           |
+| `conversation`     | `.json`   | Raw conversation schema export                 |
+| `preference`       | `.json`   | DPO preference pairs (requires judge scores)   |
+
+## Project Structure
+
+This project follows a **Modular Monolith** architecture.
+
+```text
+distill-align/
+├── src/distill_align/    # Core application package
+│   ├── core/             # Config, schemas, logging, caching, checkpointing
+│   ├── ingestion/        # Data loaders and chunkers (PDF, DOCX, HTML, code, etc.)
+│   ├── synthesis/        # LLM clients, worker pool, prompts, judge, cost tracking
+│   │   └── models/       # Provider-specific clients (OpenAI, Anthropic, Gemini, Azure, Ollama, vLLM)
+│   ├── exporter/         # Formatters, validator, splitter, preference generator
+│   │   └── formatters/   # Output format converters (JSONL, Parquet, ShareGPT, Alpaca, etc.)
+│   ├── tui/              # Textual terminal UI
+│   └── cli/              # Typer CLI entry points
+├── tests/                # Pytest suite
+└── docs/                 # Documentation (MkDocs)
+```
+
+## Development
+
+1. Clone the repository
+2. Install dependencies with Poetry: `poetry install`
+3. Install dev dependencies: `poetry install --with dev`
+4. Run tests: `poetry run pytest`
+5. Run linting: `poetry run ruff check src/`
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.

distill_align-0.1.1/pyproject.toml

@@ -0,0 +1,132 @@
+[tool.poetry]
+name = "distill-align"
+version = "0.1.1"
+description = "The Structured Reasoning Extraction Factory — Generate high-quality fine-tuning datasets from raw domain data"
+authors = ["o69dn"]
+readme = "README.md"
+license = "MIT"
+homepage = "https://github.com/o69dn/Distill-Align"
+repository = "https://github.com/o69dn/Distill-Align"
+documentation = "https://o69dn.github.io/Distill-Align/"
+keywords = ["llm", "fine-tuning", "dataset", "distillation", "unsloth", "synthesis"]
+packages = [{include = "distill_align", from = "src"}]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+# CLI & TUI
+typer = ">=0.13.0"
+textual = "^0.58.0"
+rich = "^13.7.0"
+# Data Validation
+pydantic = "^2.7.0"
+pydantic-settings = "^2.2.0"
+# Async HTTP
+httpx = ">=0.27.0"
+# PDF Parsing
+pypdf = "^4.2.0"
+# Logging
+loguru = "^0.7.0"
+# Templates
+jinja2 = "^3.1.0"
+# Config file support
+pyyaml = "^6.0"
+# Token counting
+tiktoken = ">=0.7.0"
+# Loaders
+python-docx = ">=1.1.0"
+beautifulsoup4 = ">=4.12.0"
+# Optional: HuggingFace Hub support
+huggingface_hub = {version = "^0.23.0", optional = true}
+# Optional: Parquet export
+pyarrow = {version = ">=14.0.0", optional = true}
+
+[tool.poetry.extras]
+hub = ["huggingface_hub"]
+parquet = ["pyarrow"]
+all = ["huggingface_hub", "pyarrow"]
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^9.0.0"
+pytest-asyncio = ">=0.25.0"
+pytest-cov = "^5.0.0"
+pytest-httpx = "^0.36.0"
+pre-commit = "^3.7.0"
+mypy = "^1.10.0"
+ruff = "^0.9.0"
+mkdocs = "^1.6.0"
+mkdocs-material = "^9.5.0"
+
+[tool.poetry.group.benchmark.dependencies]
+pytest-benchmark = "^4.0"
+
+[tool.poetry.group.property.dependencies]
+hypothesis = "^6.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+distill-align = "distill_align.cli.main:entry_point"
+
+# Ruff Configuration
+[tool.ruff]
+line-length = 120
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # Pyflakes
+    "I",   # isort
+    "N",   # pep8-naming
+    "UP",  # pyupgrade
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "SIM", # flake8-simplify
+]
+ignore = ["E501"]  # Line too long (handled by formatter)
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+
+# Pytest Configuration
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = "-v --tb=short --cov=distill_align --cov-report=term-missing --cov-report=xml --cov-report=html --cov-fail-under=25"
+
+# Bandit Configuration
+[tool.bandit]
+exclude_dirs = ["tests", ".github"]
+skips = ["B110", "B112", "B311"]
+
+# MyPy Configuration
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = false
+check_untyped_defs = true
+no_implicit_optional = true
+show_error_codes = true
+plugins = "pydantic.mypy"

distill_align-0.1.1/src/distill_align/__init__.py

@@ -0,0 +1,9 @@
+"""
+Distill-Align: The Structured Reasoning Extraction Factory
+
+A CLI/Python framework for automating the generation of high-quality,
+fine-tuning datasets from raw domain data.
+"""
+
+__version__ = "0.1.1"
+__author__ = "Distill-Align Contributors"

distill_align-0.1.1/src/distill_align/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI module: Typer entry points for the Distill-Align framework."""