vit-curator 0.3.1__tar.gz

vit_curator-0.3.1/.gitignore

@@ -0,0 +1,52 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+dist/
+*.egg-info/
+*.egg
+.installed.cfg
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Test / tooling caches
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.pyright_cache/
+.coverage
+.coverage.*
+htmlcov/
+.tox/
+.cache
+
+# uv
+uv.lock.local
+
+# OS / editor
+.DS_Store
+Thumbs.db
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Build / hatch artifacts
+/src/*.egg-info/
+
+# Pipeline runtime data (created by CLI runs)
+var/
+index.duckdb
+*.duckdb
+*.duckdb.wal
+
+# Benchmark artifacts
+bench_output/

vit_curator-0.3.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jakob Charles
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vit_curator-0.3.1/PKG-INFO

@@ -0,0 +1,178 @@
+Metadata-Version: 2.4
+Name: vit-curator
+Version: 0.3.1
+Summary: End-to-end ML curation pipeline: image ingest, preprocess, VLM label, train/predict, and RAG-style chunk/embed/enrich — single CLI, single DuckDB.
+Author: Jakob Charles
+License: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: duckdb>=1.0.0
+Requires-Dist: fastai>=2.7.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: networkx>=3.2
+Requires-Dist: numpy>=1.26.0
+Requires-Dist: orjson>=3.10.0
+Requires-Dist: pillow>=10.0.0
+Requires-Dist: psutil>=7.2.1
+Requires-Dist: pydantic>=2.7.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: rich>=13.7.0
+Requires-Dist: torch>=2.0.0
+Requires-Dist: torchvision>=0.16.0
+Requires-Dist: typer>=0.12.0
+Requires-Dist: xxhash>=3.4.1
+Provides-Extra: dali
+Requires-Dist: nvidia-dali-cuda120; extra == 'dali'
+Provides-Extra: dev
+Requires-Dist: pyright>=1.1.380; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.6.0; extra == 'dev'
+Provides-Extra: embed
+Requires-Dist: sentence-transformers>=2.2.0; extra == 'embed'
+Provides-Extra: label
+Requires-Dist: nvidia-ml-py>=12.0.0; extra == 'label'
+Provides-Extra: langgraph
+Requires-Dist: langgraph-checkpoint-sqlite>=2.0.0; extra == 'langgraph'
+Requires-Dist: langgraph>=0.2.0; extra == 'langgraph'
+Provides-Extra: train
+Requires-Dist: fastai>=2.7.0; extra == 'train'
+Requires-Dist: ipython>=8.0.0; extra == 'train'
+Requires-Dist: onnx>=1.14.0; extra == 'train'
+Requires-Dist: scikit-learn>=1.3.0; extra == 'train'
+Requires-Dist: torch>=2.0.0; extra == 'train'
+Requires-Dist: torchvision>=0.16.0; extra == 'train'
+Provides-Extra: tui
+Requires-Dist: textual>=0.47.0; extra == 'tui'
+Provides-Extra: vips
+Requires-Dist: pyvips>=2.2.0; extra == 'vips'
+Description-Content-Type: text/markdown
+
+# ViT-Curator
+
+> End-to-end ML curation pipeline: image ingest → preprocess → VLM label →
+> train/predict → chunk/embed/enrich, plus a live TUI dashboard.
+> Single CLI, single DuckDB.
+
+[![CI](https://github.com/Veedubin/vit-curator/actions/workflows/ci.yml/badge.svg)](https://github.com/Veedubin/vit-curator/actions/workflows/ci.yml)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-261230.svg)](https://docs.astral.sh/ruff/)
+
+## What it does
+
+ViT-Curator is a single command-line tool that turns a giant pile of messy
+images into a deduplicated, labeled, searchable knowledge base — and lets
+you train a custom model to auto-label new images.
+
+```
+download/unzip
+    → scan → hash → dedupe → crop/deskew → resize to N presets
+        → VLM (Qwen3-VL) labels every image
+            → train a fastai ViT/ResNet on the labels
+                → predict on new images at low cost
+                    → chunk / embed / enrich the captions + text
+                        → Textual TUI watches it all run
+```
+
+Everything lives in **one DuckDB file** — `files`, derivatives, labels,
+runs, predictions, trained models, chunks, embeddings, and enrichments
+all share a single schema with an additive-migration framework.
+
+## Installation
+
+```bash
+uv sync                    # Core dependencies
+uv sync --extra train      # + FastAI training
+uv sync --extra label      # + nvidia-ml-py for GPU monitoring
+uv sync --extra tui        # + Textual dashboard
+uv sync --extra dali       # + NVIDIA DALI for GPU decode
+uv sync --extra embed      # + sentence-transformers for embeddings
+uv sync --extra dev        # + pytest, ruff, pyright
+```
+
+The CLI entry point is `vit-curator` (alias `up`).
+
+## Usage
+
+```bash
+# Run directly via uv (recommended — no venv activation needed)
+
+# 0. Ingest: download URLs, extract archives, sort into a directory layout
+vit-curator ingest --dest /data --download urls.txt
+
+# 1-2. Preprocess: scan, hash, dedupe, decode, generate per-preset derivatives
+vit-curator preprocess --src /data/sorted --out /data \
+    --presets "vit-train-256=256,thumb-64=64"
+
+# 1.5. Near-duplicate detection (phash)
+vit-curator perceptual-dedupe --src /data/sorted --threshold 8
+
+# 3. VLM labeling against a running vLLM server
+vit-curator label --db /data/index.duckdb --server-url http://localhost:8000
+
+# 4-6. Train, evaluate, predict, export
+vit-curator train      --db /data/index.duckdb --run-id <uuid>
+vit-curator evaluate   --db /data/index.duckdb --run-id <uuid> --model model.pkl
+vit-curator predict    --db /data/index.duckdb --model model.pkl --target-run-id <uuid>
+vit-curator export-model --model model.pkl --formats onnx,torchscript
+
+# 7. Post-processing (RAG)
+vit-curator chunk   --db /data/index.duckdb --source predictions --run-id <uuid>
+vit-curator embed   --db /data/index.duckdb --model sentence-transformers/all-MiniLM-L6-v2
+vit-curator enrich  --db /data/index.duckdb --server-url http://localhost:9001
+
+# Live Textual TUI dashboard
+vit-curator dashboard --db /data/index.duckdb
+
+# YAML config-driven full chain
+vit-curator run-all --config pipeline.yaml --dry-run
+vit-curator run-all --config pipeline.yaml --stages preprocess,train
+```
+
+## Real-world jobs it's good for
+
+- **E-commerce cataloging** at scale (dedupe, crop, deskew, auto-tag, train a custom classifier).
+- **Document / archive digitization** with skew correction + semantic search over the OCR'd text.
+- **Dataset curation** before a vision-model fine-tune.
+- **Reverse image search** / near-duplicate detection for a content platform.
+- **RAG from a visual corpus** — turn a museum or archive into a queryable knowledge base.
+
+See [CHANGELOG.md](CHANGELOG.md) for the full feature list.
+
+## Architecture
+
+```
+src/vit_curator/
+├── cli.py              # Typer entrypoint — every stage as a subcommand
+├── config.py           # Frozen dataclasses for all stage configs
+├── shared/             # DB schema, hashing, errors, progress, signals
+├── ingest/             # Stage 0: download / unarchive / sort
+├── preprocess/         # Stages 1-2: scan, hash, dedupe, decode, derivatives, crop/deskew, perceptual dedupe
+├── label/              # Stage 3: VLM dispatcher + metrics + scheduler
+├── train/              # Stages 4-6: train, evaluate, predict, export
+├── post/               # Stage 7: chunk, embed, enrich
+└── tui/                # Textual dashboard (5 screens, GPU meter, latency histogram)
+```
+
+## Development
+
+```bash
+uv sync --extra dev
+uv run ruff check .
+uv run pytest -m "not torch and not fastai and not dali and not nvidia and not slow"
+uv run python scripts/benchmark.py    # smoke benchmark of all 4 core stages
+```
+
+The default test deselection skips tests that need optional heavy stacks
+(torch, fastai, DALI, nvidia-ml-py). To run those locally:
+
+```bash
+VIT_CURATOR_TEST_TORCH=1   uv run pytest
+VIT_CURATOR_TEST_FASTAI=1  uv run pytest
+VIT_CURATOR_TEST_DALI=1    uv run pytest
+VIT_CURATOR_TEST_NVIDIA=1  uv run pytest
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).

vit_curator-0.3.1/README.md

@@ -0,0 +1,129 @@
+# ViT-Curator
+
+> End-to-end ML curation pipeline: image ingest → preprocess → VLM label →
+> train/predict → chunk/embed/enrich, plus a live TUI dashboard.
+> Single CLI, single DuckDB.
+
+[![CI](https://github.com/Veedubin/vit-curator/actions/workflows/ci.yml/badge.svg)](https://github.com/Veedubin/vit-curator/actions/workflows/ci.yml)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-261230.svg)](https://docs.astral.sh/ruff/)
+
+## What it does
+
+ViT-Curator is a single command-line tool that turns a giant pile of messy
+images into a deduplicated, labeled, searchable knowledge base — and lets
+you train a custom model to auto-label new images.
+
+```
+download/unzip
+    → scan → hash → dedupe → crop/deskew → resize to N presets
+        → VLM (Qwen3-VL) labels every image
+            → train a fastai ViT/ResNet on the labels
+                → predict on new images at low cost
+                    → chunk / embed / enrich the captions + text
+                        → Textual TUI watches it all run
+```
+
+Everything lives in **one DuckDB file** — `files`, derivatives, labels,
+runs, predictions, trained models, chunks, embeddings, and enrichments
+all share a single schema with an additive-migration framework.
+
+## Installation
+
+```bash
+uv sync                    # Core dependencies
+uv sync --extra train      # + FastAI training
+uv sync --extra label      # + nvidia-ml-py for GPU monitoring
+uv sync --extra tui        # + Textual dashboard
+uv sync --extra dali       # + NVIDIA DALI for GPU decode
+uv sync --extra embed      # + sentence-transformers for embeddings
+uv sync --extra dev        # + pytest, ruff, pyright
+```
+
+The CLI entry point is `vit-curator` (alias `up`).
+
+## Usage
+
+```bash
+# Run directly via uv (recommended — no venv activation needed)
+
+# 0. Ingest: download URLs, extract archives, sort into a directory layout
+vit-curator ingest --dest /data --download urls.txt
+
+# 1-2. Preprocess: scan, hash, dedupe, decode, generate per-preset derivatives
+vit-curator preprocess --src /data/sorted --out /data \
+    --presets "vit-train-256=256,thumb-64=64"
+
+# 1.5. Near-duplicate detection (phash)
+vit-curator perceptual-dedupe --src /data/sorted --threshold 8
+
+# 3. VLM labeling against a running vLLM server
+vit-curator label --db /data/index.duckdb --server-url http://localhost:8000
+
+# 4-6. Train, evaluate, predict, export
+vit-curator train      --db /data/index.duckdb --run-id <uuid>
+vit-curator evaluate   --db /data/index.duckdb --run-id <uuid> --model model.pkl
+vit-curator predict    --db /data/index.duckdb --model model.pkl --target-run-id <uuid>
+vit-curator export-model --model model.pkl --formats onnx,torchscript
+
+# 7. Post-processing (RAG)
+vit-curator chunk   --db /data/index.duckdb --source predictions --run-id <uuid>
+vit-curator embed   --db /data/index.duckdb --model sentence-transformers/all-MiniLM-L6-v2
+vit-curator enrich  --db /data/index.duckdb --server-url http://localhost:9001
+
+# Live Textual TUI dashboard
+vit-curator dashboard --db /data/index.duckdb
+
+# YAML config-driven full chain
+vit-curator run-all --config pipeline.yaml --dry-run
+vit-curator run-all --config pipeline.yaml --stages preprocess,train
+```
+
+## Real-world jobs it's good for
+
+- **E-commerce cataloging** at scale (dedupe, crop, deskew, auto-tag, train a custom classifier).
+- **Document / archive digitization** with skew correction + semantic search over the OCR'd text.
+- **Dataset curation** before a vision-model fine-tune.
+- **Reverse image search** / near-duplicate detection for a content platform.
+- **RAG from a visual corpus** — turn a museum or archive into a queryable knowledge base.
+
+See [CHANGELOG.md](CHANGELOG.md) for the full feature list.
+
+## Architecture
+
+```
+src/vit_curator/
+├── cli.py              # Typer entrypoint — every stage as a subcommand
+├── config.py           # Frozen dataclasses for all stage configs
+├── shared/             # DB schema, hashing, errors, progress, signals
+├── ingest/             # Stage 0: download / unarchive / sort
+├── preprocess/         # Stages 1-2: scan, hash, dedupe, decode, derivatives, crop/deskew, perceptual dedupe
+├── label/              # Stage 3: VLM dispatcher + metrics + scheduler
+├── train/              # Stages 4-6: train, evaluate, predict, export
+├── post/               # Stage 7: chunk, embed, enrich
+└── tui/                # Textual dashboard (5 screens, GPU meter, latency histogram)
+```
+
+## Development
+
+```bash
+uv sync --extra dev
+uv run ruff check .
+uv run pytest -m "not torch and not fastai and not dali and not nvidia and not slow"
+uv run python scripts/benchmark.py    # smoke benchmark of all 4 core stages
+```
+
+The default test deselection skips tests that need optional heavy stacks
+(torch, fastai, DALI, nvidia-ml-py). To run those locally:
+
+```bash
+VIT_CURATOR_TEST_TORCH=1   uv run pytest
+VIT_CURATOR_TEST_FASTAI=1  uv run pytest
+VIT_CURATOR_TEST_DALI=1    uv run pytest
+VIT_CURATOR_TEST_NVIDIA=1  uv run pytest
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).

vit_curator-0.3.1/pyproject.toml

@@ -0,0 +1,90 @@
+[project]
+name = "vit-curator"
+version = "0.3.1"
+description = "End-to-end ML curation pipeline: image ingest, preprocess, VLM label, train/predict, and RAG-style chunk/embed/enrich — single CLI, single DuckDB."
+readme = "README.md"
+requires-python = ">=3.11"
+license = {text = "MIT"}
+authors = [{name = "Jakob Charles"}]
+dependencies = [
+    "duckdb>=1.0.0",
+    "rich>=13.7.0",
+    "typer>=0.12.0",
+    "xxhash>=3.4.1",
+    "pillow>=10.0.0",
+    "requests>=2.31.0",
+    "httpx>=0.27.0",
+    "orjson>=3.10.0",
+    "pydantic>=2.7.0",
+    "psutil>=7.2.1",
+    "numpy>=1.26.0",
+    "torch>=2.0.0",
+    "torchvision>=0.16.0",
+    "fastai>=2.7.0",
+    "networkx>=3.2",
+]
+
+[project.optional-dependencies]
+dali = ["nvidia-dali-cuda120"]
+vips = ["pyvips>=2.2.0"]
+train = ["fastai>=2.7.0", "torch>=2.0.0", "torchvision>=0.16.0", "scikit-learn>=1.3.0", "onnx>=1.14.0", "ipython>=8.0.0"]
+label = ["nvidia-ml-py>=12.0.0"]
+tui = ["textual>=0.47.0"]
+embed = ["sentence-transformers>=2.2.0"]
+langgraph = ["langgraph>=0.2.0", "langgraph-checkpoint-sqlite>=2.0.0"]
+dev = ["pytest>=8.0.0", "ruff>=0.6.0", "pyright>=1.1.380"]
+
+[project.scripts]
+vit-curator = "vit_curator.cli:app"
+up = "vit_curator.cli:app"
+
+[build-system]
+requires = ["hatchling>=1.21.0"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vit_curator"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/vit_curator",
+    "configs",
+    "README.md",
+    "LICENSE",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "SIM", "PL", "RUF"]
+ignore = ["PLR0913", "PLR2004", "PLR0912", "PLR0915", "B008"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = ["PLC0415"]
+
+[tool.pyright]
+typeCheckingMode = "basic"
+pythonVersion = "3.11"
+venvPath = "."
+venv = ".venv"
+reportMissingImports = "warning"
+reportMissingTypeStubs = "warning"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "torch: tests requiring torch (deselect with '-m \"not torch\"')",
+    "fastai: tests requiring fastai (deselect with '-m \"not fastai\"')",
+    "dali: tests requiring nvidia-dali (deselect with '-m \"not dali\"')",
+    "vips: tests requiring pyvips/libvips (deselect with '-m \"not vips\"')",
+    "nvidia: tests requiring nvidia GPU (deselect with '-m \"not nvidia\"')",
+    "slow: slow-running tests (deselect with '-m \"not slow\"')",
+]
+
+[dependency-groups]
+dev = [
+    "pyright>=1.1.409",
+    "pytest>=9.0.3",
+]

vit_curator-0.3.1/src/vit_curator/__init__.py

@@ -0,0 +1,6 @@
+"""ViT-Curator — image ingest, preprocess, label, train, predict, RAG post-process, dashboard."""
+
+from __future__ import annotations
+
+__all__ = ["__version__"]
+__version__ = "0.3.0"