verifily 1.2.0__tar.gz

verifily-1.2.0/LICENSE

@@ -0,0 +1,76 @@
+Business Source License 1.1
+
+Parameters
+
+Licensor:             Verifily / Arsenis Papachristos
+Licensed Work:        Verifily 1.2.0
+                      The Licensed Work is (c) 2026 Verifily.
+Additional Use Grant: You may use the Licensed Work for any purpose
+                      except offering it as a commercial data quality
+                      or ML pipeline gating service to third parties.
+Change Date:          2030-02-16
+Change License:       Apache License, Version 2.0
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited
+production use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.
+
+MariaDB hereby grants you permission to use this License's text to license
+your works, and to refer to it using the trademark "Business Source License",
+as long as you comply with the Covenants of Licensor below.
+
+Covenants of Licensor
+
+In consideration of the right to use this License's text and the "Business
+Source License" name and trademark, Licensor covenants to MariaDB, and to all
+other recipients of the licensed work to be provided by Licensor:
+
+1. To specify as the Change License the GPL Version 2.0 or any later version,
+   or a license that is compatible with GPL Version 2.0 or a later version,
+   where "compatible" means that software provided under the Change License
+   can be included in a program with software provided under GPL Version 2.0
+   or a later version. Licensor may specify additional Change Licenses without
+   limitation.
+
+2. To either: (a) specify an additional grant of rights to use that does not
+   impose any additional restriction on the right granted in this License, as
+   the Additional Use Grant; or (b) insert the text "None" to specify a
+   Change Date. Not to modify this License in any other way.

verifily-1.2.0/MANIFEST.in

@@ -0,0 +1,2 @@
+exclude scripts/keygen.py
+prune scripts/.keys

verifily-1.2.0/PKG-INFO

@@ -0,0 +1,222 @@
+Metadata-Version: 2.4
+Name: verifily
+Version: 1.2.0
+Summary: ML data quality gate — ingest, validate, and ship datasets with confidence.
+Author: Verifily Team
+License: BSL-1.1
+Project-URL: Homepage, https://verifily.io
+Project-URL: Documentation, https://verifily.io/docs
+Project-URL: Repository, https://github.com/verifily/verifily
+Keywords: ml,data-quality,dataset,validation,ci,pipeline
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.9.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: cryptography>=41.0.0
+Provides-Extra: api
+Requires-Dist: fastapi>=0.100.0; extra == "api"
+Requires-Dist: uvicorn>=0.23.0; extra == "api"
+Requires-Dist: pydantic>=2.0.0; extra == "api"
+Provides-Extra: sdk
+Requires-Dist: httpx>=0.24.0; extra == "sdk"
+Requires-Dist: pydantic>=2.0.0; extra == "sdk"
+Provides-Extra: arrow
+Requires-Dist: pyarrow>=14.0.0; extra == "arrow"
+Provides-Extra: pii
+Requires-Dist: spacy>=3.5.0; extra == "pii"
+Provides-Extra: huggingface
+Requires-Dist: datasets>=2.14.0; extra == "huggingface"
+Provides-Extra: wandb
+Requires-Dist: wandb>=0.16.0; extra == "wandb"
+Provides-Extra: mlflow
+Requires-Dist: mlflow>=2.8.0; extra == "mlflow"
+Provides-Extra: integrations
+Requires-Dist: datasets>=2.14.0; extra == "integrations"
+Requires-Dist: wandb>=0.16.0; extra == "integrations"
+Requires-Dist: mlflow>=2.8.0; extra == "integrations"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: fastapi>=0.100.0; extra == "dev"
+Requires-Dist: uvicorn>=0.23.0; extra == "dev"
+Requires-Dist: pydantic>=2.0.0; extra == "dev"
+Requires-Dist: httpx>=0.24.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: fastapi>=0.100.0; extra == "all"
+Requires-Dist: uvicorn>=0.23.0; extra == "all"
+Requires-Dist: pydantic>=2.0.0; extra == "all"
+Requires-Dist: httpx>=0.24.0; extra == "all"
+Requires-Dist: pytest>=7.0; extra == "all"
+Requires-Dist: pyarrow>=14.0.0; extra == "all"
+Requires-Dist: spacy>=3.5.0; extra == "all"
+Requires-Dist: datasets>=2.14.0; extra == "all"
+Requires-Dist: wandb>=0.16.0; extra == "all"
+Requires-Dist: mlflow>=2.8.0; extra == "all"
+Dynamic: license-file
+Dynamic: requires-python
+
+# Verifily
+
+ML data quality gate. Ingest, validate, and ship datasets with confidence.
+
+Verifily catches contamination, PII leaks, SQL template leakage, contract violations, and metric regressions before they reach production. It runs locally — no network, no GPU, no external services.
+
+One command gates your CI pipeline. Exit 0 means ship.
+
+## Install
+
+```bash
+pip install -e .
+```
+
+For integrations (HuggingFace, W&B, MLflow) and API server:
+
+```bash
+pip install -e ".[all]"
+```
+
+## 60-Second Quick Start
+
+```bash
+# 1. Scaffold a project
+verifily quickstart my_project
+
+# 2. Ingest raw data (JSONL, CSV, Parquet, or HuggingFace)
+verifily ingest --in my_project/data/raw/sample.csv \
+                --out my_project/data/artifact \
+                --schema sft
+
+# 3. Run the CI gate
+verifily pipeline --config my_project/verifily.yaml --ci
+# Exit 0 = SHIP, 1 = DONT_SHIP, 2 = INVESTIGATE
+```
+
+Or run the full demo end-to-end:
+
+```bash
+bash scripts/demo_quickstart_ci.sh
+```
+
+## What Verifily Prevents
+
+| Risk | How Verifily catches it |
+|------|------------------------|
+| Train/eval data leakage | Exact-match + Jaccard contamination detection via MinHash LSH |
+| SQL template leakage | Three-tier NL2SQL gate: exact SQL, template fingerprint, question near-dup |
+| PII in training data | Regex-based PII scan with configurable thresholds and redaction |
+| Missing or corrupt artifacts | Run contract validation (hashes, configs, eval results) |
+| Metric regressions | Threshold checks against baselines with delta tracking |
+| Ambiguous ship decisions | Deterministic gate: blockers always block, no silent passes |
+| Dataset drift | Privacy-safe fingerprinting and diff without raw data exposure |
+
+## Supported Schemas
+
+8 canonical dataset types, auto-detected from field names:
+
+| Schema | Required fields | Use case |
+|--------|----------------|----------|
+| `sft` | instruction, output | Supervised fine-tuning |
+| `qa` | question, answer | Question answering |
+| `classification` | text, label | Text classification |
+| `chat` | messages | Multi-turn conversations |
+| `summarization` | document, summary | Summarization tasks |
+| `translation` | source, target | Translation pairs |
+| `rm_pairwise` | prompt, chosen, rejected | Reward model training |
+| `nl2sql` | question, sql, schema | Natural language to SQL |
+
+## CLI Commands
+
+| Command | Purpose |
+|---------|---------|
+| `verifily quickstart <path>` | Scaffold a working project |
+| `verifily ingest` | Normalize raw data to artifact format (JSONL, CSV, Parquet, hf://) |
+| `verifily pipeline --ci` | Run full quality gate (CI mode) |
+| `verifily report` | Dataset quality report with PII scan |
+| `verifily contamination` | Detect train/eval overlap |
+| `verifily contract-check` | Validate run artifacts |
+| `verifily fingerprint` | Privacy-safe dataset summary |
+| `verifily diff-datasets` | Compare two datasets |
+| `verifily ci-init` | Generate GitHub/GitLab CI config |
+| `verifily serve` | Start API server |
+| `verifily version` | Show version, Python, platform |
+
+### NL2SQL Commands
+
+| Command | Purpose |
+|---------|---------|
+| `verifily nl2sql validate` | Validate NL2SQL dataset structure |
+| `verifily nl2sql fingerprint` | SQL normalization + template fingerprinting |
+| `verifily nl2sql split` | Leakage-resistant train/eval splitting |
+| `verifily nl2sql gate` | Three-tier contamination gate for NL2SQL |
+
+## Integrations
+
+All opt-in with lazy imports. No hard dependencies.
+
+| Integration | What it does |
+|-------------|-------------|
+| **HuggingFace Datasets** | Load datasets via `hf://` URIs |
+| **Weights & Biases** | Log decisions, metrics, and artifacts |
+| **MLflow** | Track runs with model registry integration |
+| **GitHub Actions** | Pre-built action + CI workflow generator |
+
+```bash
+# HuggingFace
+verifily ingest --in "hf://squad" --out datasets/squad --schema qa
+
+# W&B + MLflow
+verifily pipeline --config pipeline.yaml --wandb --mlflow
+```
+
+## CI Exit Codes
+
+| Code | Label | Meaning |
+|------|-------|---------|
+| `0` | SHIP | All quality gates passed |
+| `1` | DONT_SHIP | One or more blockers failed |
+| `2` | INVESTIGATE | Risk flags present, no hard blockers |
+| `3` | CONTRACT_FAIL | Run contract invalid |
+| `4` | TOOL_ERROR | Invalid config or unexpected error |
+
+## Documentation
+
+- [Product Overview](docs/product-overview.md)
+- [Quick Install](docs/quick_install.md)
+- [3-Minute Quickstart](docs/3_minute_quickstart.md)
+- [Decision Gate](docs/decision_gate.md)
+- [Dataset Fingerprints](docs/fingerprints.md)
+- [CI Init](docs/ci/quick_ci_init.md)
+- [API & Jobs](docs/api_jobs.md)
+- [Monitor](docs/monitor.md)
+- [Versioning & Stability](VERSIONING.md)
+- [Changelog](CHANGELOG.md)
+
+## Versioning
+
+Verifily follows [Semantic Versioning](https://semver.org/). See [VERSIONING.md](VERSIONING.md).
+
+Current version: `1.2.0`
+
+## Stability Guarantees
+
+- **Deterministic outputs** — fixed seed produces identical results across runs
+- **Stable contracts** — `run_contract_v1` schema is frozen within the v1.x line
+- **Stable exit codes** — 0/1/2/3/4 semantics are frozen
+- **Backward compatibility** within MAJOR version — artifacts from any v1.x release are accepted
+- **1,300+ tests** — all deterministic, no network, no GPU
+
+## License
+
+Business Source License 1.1 (BSL-1.1). See [LICENSE](LICENSE) for details.
+
+You may use Verifily for any purpose except offering it as a commercial data quality or ML pipeline gating service to third parties. On 2030-02-16, the license converts to Apache 2.0.

verifily-1.2.0/README.md

@@ -0,0 +1,155 @@
+# Verifily
+
+ML data quality gate. Ingest, validate, and ship datasets with confidence.
+
+Verifily catches contamination, PII leaks, SQL template leakage, contract violations, and metric regressions before they reach production. It runs locally — no network, no GPU, no external services.
+
+One command gates your CI pipeline. Exit 0 means ship.
+
+## Install
+
+```bash
+pip install -e .
+```
+
+For integrations (HuggingFace, W&B, MLflow) and API server:
+
+```bash
+pip install -e ".[all]"
+```
+
+## 60-Second Quick Start
+
+```bash
+# 1. Scaffold a project
+verifily quickstart my_project
+
+# 2. Ingest raw data (JSONL, CSV, Parquet, or HuggingFace)
+verifily ingest --in my_project/data/raw/sample.csv \
+                --out my_project/data/artifact \
+                --schema sft
+
+# 3. Run the CI gate
+verifily pipeline --config my_project/verifily.yaml --ci
+# Exit 0 = SHIP, 1 = DONT_SHIP, 2 = INVESTIGATE
+```
+
+Or run the full demo end-to-end:
+
+```bash
+bash scripts/demo_quickstart_ci.sh
+```
+
+## What Verifily Prevents
+
+| Risk | How Verifily catches it |
+|------|------------------------|
+| Train/eval data leakage | Exact-match + Jaccard contamination detection via MinHash LSH |
+| SQL template leakage | Three-tier NL2SQL gate: exact SQL, template fingerprint, question near-dup |
+| PII in training data | Regex-based PII scan with configurable thresholds and redaction |
+| Missing or corrupt artifacts | Run contract validation (hashes, configs, eval results) |
+| Metric regressions | Threshold checks against baselines with delta tracking |
+| Ambiguous ship decisions | Deterministic gate: blockers always block, no silent passes |
+| Dataset drift | Privacy-safe fingerprinting and diff without raw data exposure |
+
+## Supported Schemas
+
+8 canonical dataset types, auto-detected from field names:
+
+| Schema | Required fields | Use case |
+|--------|----------------|----------|
+| `sft` | instruction, output | Supervised fine-tuning |
+| `qa` | question, answer | Question answering |
+| `classification` | text, label | Text classification |
+| `chat` | messages | Multi-turn conversations |
+| `summarization` | document, summary | Summarization tasks |
+| `translation` | source, target | Translation pairs |
+| `rm_pairwise` | prompt, chosen, rejected | Reward model training |
+| `nl2sql` | question, sql, schema | Natural language to SQL |
+
+## CLI Commands
+
+| Command | Purpose |
+|---------|---------|
+| `verifily quickstart <path>` | Scaffold a working project |
+| `verifily ingest` | Normalize raw data to artifact format (JSONL, CSV, Parquet, hf://) |
+| `verifily pipeline --ci` | Run full quality gate (CI mode) |
+| `verifily report` | Dataset quality report with PII scan |
+| `verifily contamination` | Detect train/eval overlap |
+| `verifily contract-check` | Validate run artifacts |
+| `verifily fingerprint` | Privacy-safe dataset summary |
+| `verifily diff-datasets` | Compare two datasets |
+| `verifily ci-init` | Generate GitHub/GitLab CI config |
+| `verifily serve` | Start API server |
+| `verifily version` | Show version, Python, platform |
+
+### NL2SQL Commands
+
+| Command | Purpose |
+|---------|---------|
+| `verifily nl2sql validate` | Validate NL2SQL dataset structure |
+| `verifily nl2sql fingerprint` | SQL normalization + template fingerprinting |
+| `verifily nl2sql split` | Leakage-resistant train/eval splitting |
+| `verifily nl2sql gate` | Three-tier contamination gate for NL2SQL |
+
+## Integrations
+
+All opt-in with lazy imports. No hard dependencies.
+
+| Integration | What it does |
+|-------------|-------------|
+| **HuggingFace Datasets** | Load datasets via `hf://` URIs |
+| **Weights & Biases** | Log decisions, metrics, and artifacts |
+| **MLflow** | Track runs with model registry integration |
+| **GitHub Actions** | Pre-built action + CI workflow generator |
+
+```bash
+# HuggingFace
+verifily ingest --in "hf://squad" --out datasets/squad --schema qa
+
+# W&B + MLflow
+verifily pipeline --config pipeline.yaml --wandb --mlflow
+```
+
+## CI Exit Codes
+
+| Code | Label | Meaning |
+|------|-------|---------|
+| `0` | SHIP | All quality gates passed |
+| `1` | DONT_SHIP | One or more blockers failed |
+| `2` | INVESTIGATE | Risk flags present, no hard blockers |
+| `3` | CONTRACT_FAIL | Run contract invalid |
+| `4` | TOOL_ERROR | Invalid config or unexpected error |
+
+## Documentation
+
+- [Product Overview](docs/product-overview.md)
+- [Quick Install](docs/quick_install.md)
+- [3-Minute Quickstart](docs/3_minute_quickstart.md)
+- [Decision Gate](docs/decision_gate.md)
+- [Dataset Fingerprints](docs/fingerprints.md)
+- [CI Init](docs/ci/quick_ci_init.md)
+- [API & Jobs](docs/api_jobs.md)
+- [Monitor](docs/monitor.md)
+- [Versioning & Stability](VERSIONING.md)
+- [Changelog](CHANGELOG.md)
+
+## Versioning
+
+Verifily follows [Semantic Versioning](https://semver.org/). See [VERSIONING.md](VERSIONING.md).
+
+Current version: `1.2.0`
+
+## Stability Guarantees
+
+- **Deterministic outputs** — fixed seed produces identical results across runs
+- **Stable contracts** — `run_contract_v1` schema is frozen within the v1.x line
+- **Stable exit codes** — 0/1/2/3/4 semantics are frozen
+- **Backward compatibility** within MAJOR version — artifacts from any v1.x release are accepted
+- **1,300+ tests** — all deterministic, no network, no GPU
+
+## License
+
+Business Source License 1.1 (BSL-1.1). See [LICENSE](LICENSE) for details.
+
+You may use Verifily for any purpose except offering it as a commercial data quality or ML pipeline gating service to third parties. On 2030-02-16, the license converts to Apache 2.0.

verifily-1.2.0/pyproject.toml

@@ -0,0 +1,97 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "verifily"
+version = "1.2.0"
+description = "ML data quality gate — ingest, validate, and ship datasets with confidence."
+readme = "README.md"
+license = {text = "BSL-1.1"}
+requires-python = ">=3.9"
+authors = [
+    {name = "Verifily Team"},
+]
+keywords = ["ml", "data-quality", "dataset", "validation", "ci", "pipeline"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+
+dependencies = [
+    "typer>=0.9.0",
+    "rich>=13.0.0",
+    "pyyaml>=6.0",
+    "cryptography>=41.0.0",
+]
+
+[project.urls]
+Homepage = "https://verifily.io"
+Documentation = "https://verifily.io/docs"
+Repository = "https://github.com/verifily/verifily"
+
+[project.optional-dependencies]
+api = [
+    "fastapi>=0.100.0",
+    "uvicorn>=0.23.0",
+    "pydantic>=2.0.0",
+]
+sdk = [
+    "httpx>=0.24.0",
+    "pydantic>=2.0.0",
+]
+arrow = [
+    "pyarrow>=14.0.0",
+]
+pii = [
+    "spacy>=3.5.0",
+]
+huggingface = [
+    "datasets>=2.14.0",
+]
+wandb = [
+    "wandb>=0.16.0",
+]
+mlflow = [
+    "mlflow>=2.8.0",
+]
+integrations = [
+    "datasets>=2.14.0",
+    "wandb>=0.16.0",
+    "mlflow>=2.8.0",
+]
+dev = [
+    "pytest>=7.0",
+    "fastapi>=0.100.0",
+    "uvicorn>=0.23.0",
+    "pydantic>=2.0.0",
+    "httpx>=0.24.0",
+]
+all = [
+    "fastapi>=0.100.0",
+    "uvicorn>=0.23.0",
+    "pydantic>=2.0.0",
+    "httpx>=0.24.0",
+    "pytest>=7.0",
+    "pyarrow>=14.0.0",
+    "spacy>=3.5.0",
+    "datasets>=2.14.0",
+    "wandb>=0.16.0",
+    "mlflow>=2.8.0",
+]
+
+[project.scripts]
+verifily = "verifily_cli_v1.cli:app"
+
+[tool.setuptools.packages.find]
+include = ["verifily_cli_v1*", "verifily_sdk*"]
+
+[tool.pytest.ini_options]
+testpaths = ["verifily_cli_v1/tests", "verifily_sdk/tests"]

verifily-1.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

verifily-1.2.0/setup.py

@@ -0,0 +1,36 @@
+"""Package setup for verifily-train."""
+
+from setuptools import setup, find_packages
+
+setup(
+    name="verifily-train",
+    version="1.1.0",
+    description="Dataset-aware fine-tuning in one command",
+    packages=["verifily_train"],
+    python_requires=">=3.9",
+    install_requires=[
+        "torch>=2.0.0",
+        "transformers>=4.35.0",
+        "datasets>=2.14.0",
+        "peft>=0.6.0",
+        "accelerate>=0.24.0",
+        "evaluate>=0.4.0",
+        "scikit-learn>=1.3.0",
+        "numpy>=1.24.0",
+        "tqdm>=4.65.0",
+        "pyyaml>=6.0",
+        "click>=8.0.0",
+    ],
+    extras_require={
+        "qlora": ["bitsandbytes>=0.42.0"],
+        "wandb": ["wandb>=0.16.0"],
+        "web": ["gradio>=4.0.0"],
+        "rouge": ["rouge-score>=0.1.2"],
+        "all": ["bitsandbytes>=0.42.0", "wandb>=0.16.0", "gradio>=4.0.0", "rouge-score>=0.1.2"],
+    },
+    entry_points={
+        "console_scripts": [
+            "verifily=verifily_train.cli:main",
+        ],
+    },
+)