dqflow 0.1.0__tar.gz

dqflow-0.1.0/.claude/settings.local.json

@@ -0,0 +1,10 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(pip install:*)",
+      "Bash(python:*)",
+      "Bash(dq --help:*)",
+      "Bash(dq show:*)"
+    ]
+  }
+}

dqflow-0.1.0/.gitignore

@@ -0,0 +1,44 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info/
+*.egg
+.eggs/
+
+# Virtual environments
+venv/
+.venv/
+env/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+
+# mypy
+.mypy_cache/
+
+# ruff
+.ruff_cache/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Local dev
+*.local
+.env
+.env.*

dqflow-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Quang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dqflow-0.1.0/PKG-INFO

@@ -0,0 +1,146 @@
+Metadata-Version: 2.4
+Name: dqflow
+Version: 0.1.0
+Summary: Lightweight, contract-first data quality engine for modern data pipelines
+Project-URL: Homepage, https://github.com/quang/dqflow
+Project-URL: Repository, https://github.com/quang/dqflow
+Project-URL: Issues, https://github.com/quang/dqflow/issues
+Author-email: Quang <nguyenvanquang247@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: data-contracts,data-quality,data-validation,etl,pandas
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: click>=8.0
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.0; extra == 'dev'
+Requires-Dist: pre-commit>=3.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# dqflow
+
+**dqflow** is a lightweight, contract-first data quality engine for modern data pipelines.
+
+Define explicit expectations for your data (schema, validity, freshness) and **fail fast** when data breaks — before bad data reaches downstream systems.
+
+---
+
+## Why dqflow?
+
+Data quality issues are inevitable — silent failures are not.
+
+Most teams rely on ad-hoc checks, fragile assertions, or heavyweight frameworks that are hard to maintain. dqflow takes a different approach:
+
+* **Contracts over checks** — expectations are explicit and versionable
+* **Pipeline-first** — designed for ETL, ELT, and streaming workflows
+* **Lightweight & Pythonic** — minimal API, easy to embed
+* **Fail fast** — break pipelines intentionally, not silently
+
+---
+
+## Quick example
+
+```python
+from dqflow import Contract, Column
+
+orders = Contract(
+    name="orders",
+    columns={
+        "order_id": Column(str, not_null=True),
+        "amount": Column(float, min=0),
+        "currency": Column(str, allowed=["USD", "EUR"]),
+        "created_at": Column("timestamp", freshness_minutes=60),
+    },
+    rules=[
+        "row_count > 1000",
+        "null_rate(amount) < 0.01",
+    ],
+)
+
+result = orders.validate(df)
+
+if not result.ok:
+    raise Exception(result.summary())
+```
+
+---
+
+## Features (v0.1 scope)
+
+* Contract-as-code (Python & YAML)
+* Column-level checks
+
+  * type validation
+  * not null
+  * min / max
+  * allowed values
+* Table-level checks
+
+  * row count
+  * freshness
+* Structured validation results (JSON-friendly)
+* Pandas engine
+* CLI support
+
+---
+
+## CLI usage
+
+```bash
+dq validate contracts/orders.yaml data/orders.parquet
+```
+
+---
+
+## Supported engines
+
+* ✅ Pandas
+* 🚧 PySpark (planned)
+* 🚧 SQL tables (planned)
+
+---
+
+## Philosophy
+
+* **Explicit is better than implicit**
+* **Bad data should break pipelines early**
+* **Quality rules are part of your system design**
+
+> dqflow is not a full data observability platform.
+> It is a small, opinionated library meant to be embedded directly into pipelines.
+
+---
+
+## Roadmap
+
+* PySpark engine
+* dbt / dlt integrations
+* Incremental & backfill-aware validation
+* Metrics export (Prometheus-compatible)
+
+---
+
+## License
+
+MIT
+
+---
+
+## Status
+
+🚧 Early development (v0.1.0)
+
+APIs may change. Feedback and contributions are welcome.

dqflow-0.1.0/README.md

@@ -0,0 +1,114 @@
+# dqflow
+
+**dqflow** is a lightweight, contract-first data quality engine for modern data pipelines.
+
+Define explicit expectations for your data (schema, validity, freshness) and **fail fast** when data breaks — before bad data reaches downstream systems.
+
+---
+
+## Why dqflow?
+
+Data quality issues are inevitable — silent failures are not.
+
+Most teams rely on ad-hoc checks, fragile assertions, or heavyweight frameworks that are hard to maintain. dqflow takes a different approach:
+
+* **Contracts over checks** — expectations are explicit and versionable
+* **Pipeline-first** — designed for ETL, ELT, and streaming workflows
+* **Lightweight & Pythonic** — minimal API, easy to embed
+* **Fail fast** — break pipelines intentionally, not silently
+
+---
+
+## Quick example
+
+```python
+from dqflow import Contract, Column
+
+orders = Contract(
+    name="orders",
+    columns={
+        "order_id": Column(str, not_null=True),
+        "amount": Column(float, min=0),
+        "currency": Column(str, allowed=["USD", "EUR"]),
+        "created_at": Column("timestamp", freshness_minutes=60),
+    },
+    rules=[
+        "row_count > 1000",
+        "null_rate(amount) < 0.01",
+    ],
+)
+
+result = orders.validate(df)
+
+if not result.ok:
+    raise Exception(result.summary())
+```
+
+---
+
+## Features (v0.1 scope)
+
+* Contract-as-code (Python & YAML)
+* Column-level checks
+
+  * type validation
+  * not null
+  * min / max
+  * allowed values
+* Table-level checks
+
+  * row count
+  * freshness
+* Structured validation results (JSON-friendly)
+* Pandas engine
+* CLI support
+
+---
+
+## CLI usage
+
+```bash
+dq validate contracts/orders.yaml data/orders.parquet
+```
+
+---
+
+## Supported engines
+
+* ✅ Pandas
+* 🚧 PySpark (planned)
+* 🚧 SQL tables (planned)
+
+---
+
+## Philosophy
+
+* **Explicit is better than implicit**
+* **Bad data should break pipelines early**
+* **Quality rules are part of your system design**
+
+> dqflow is not a full data observability platform.
+> It is a small, opinionated library meant to be embedded directly into pipelines.
+
+---
+
+## Roadmap
+
+* PySpark engine
+* dbt / dlt integrations
+* Incremental & backfill-aware validation
+* Metrics export (Prometheus-compatible)
+
+---
+
+## License
+
+MIT
+
+---
+
+## Status
+
+🚧 Early development (v0.1.0)
+
+APIs may change. Feedback and contributions are welcome.

dqflow-0.1.0/examples/orders.yaml

@@ -0,0 +1,33 @@
+name: orders
+description: E-commerce order data quality contract
+
+columns:
+  order_id:
+    type: string
+    not_null: true
+
+  customer_id:
+    type: string
+    not_null: true
+
+  amount:
+    type: float
+    min: 0
+    max: 100000
+
+  currency:
+    type: string
+    allowed: ["USD", "EUR", "GBP"]
+
+  status:
+    type: string
+    allowed: ["pending", "processing", "shipped", "delivered", "cancelled"]
+
+  created_at:
+    type: timestamp
+    freshness_minutes: 1440  # 24 hours
+
+rules:
+  - row_count > 0
+  - null_rate(amount) < 0.01
+  - null_rate(customer_id) < 0.001

dqflow-0.1.0/pyproject.toml

@@ -0,0 +1,84 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dqflow"
+version = "0.1.0"
+description = "Lightweight, contract-first data quality engine for modern data pipelines"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.9"
+authors = [
+    { name = "Quang", email = "nguyenvanquang247@gmail.com" },
+]
+keywords = [
+    "data-quality",
+    "data-validation",
+    "data-contracts",
+    "etl",
+    "pandas",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "pandas>=1.5.0",
+    "pyyaml>=6.0",
+    "click>=8.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "pytest-cov>=4.0",
+    "ruff>=0.1.0",
+    "mypy>=1.0",
+    "pre-commit>=3.0",
+]
+
+[project.scripts]
+dq = "dqflow.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/quang/dqflow"
+Repository = "https://github.com/quang/dqflow"
+Issues = "https://github.com/quang/dqflow/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dqflow"]
+
+[tool.ruff]
+target-version = "py39"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v --tb=short"
+
+[tool.mypy]
+python_version = "3.9"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+
+[tool.coverage.run]
+source = ["src/dqflow"]
+branch = true
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+]

dqflow-0.1.0/src/dqflow/__init__.py

@@ -0,0 +1,8 @@
+"""dqflow - Lightweight, contract-first data quality engine."""
+
+from dqflow.column import Column
+from dqflow.contract import Contract
+from dqflow.result import ValidationResult
+
+__version__ = "0.1.0"
+__all__ = ["Contract", "Column", "ValidationResult"]

dqflow-0.1.0/src/dqflow/cli.py

@@ -0,0 +1,130 @@
+"""Command-line interface for dqflow."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+import pandas as pd
+
+from dqflow import __version__
+from dqflow.contract import Contract
+
+
+@click.group()
+@click.version_option(version=__version__, prog_name="dqflow")
+def main() -> None:
+    """dqflow - Contract-first data quality for modern pipelines."""
+    pass
+
+
+@main.command()
+@click.argument("contract", type=click.Path(exists=True, path_type=Path))
+@click.argument("data", type=click.Path(exists=True, path_type=Path))
+@click.option("--output", "-o", type=click.Choice(["text", "json"]), default="text")
+@click.option("--fail-fast", is_flag=True, help="Exit with error code on validation failure")
+def validate(contract: Path, data: Path, output: str, fail_fast: bool) -> None:
+    """Validate DATA against CONTRACT.
+
+    CONTRACT: Path to contract YAML file
+    DATA: Path to data file (parquet, csv, json)
+    """
+    # Load contract
+    c = Contract.from_yaml(contract)
+
+    # Load data based on extension
+    df = _load_dataframe(data)
+
+    # Validate
+    result = c.validate(df)
+
+    # Output results
+    if output == "json":
+        click.echo(json.dumps(result.to_dict(), indent=2))
+    else:
+        click.echo(result.summary())
+
+    if fail_fast and not result.ok:
+        sys.exit(1)
+
+
+@main.command()
+@click.argument("contract", type=click.Path(exists=True, path_type=Path))
+def show(contract: Path) -> None:
+    """Show details of a CONTRACT."""
+    c = Contract.from_yaml(contract)
+
+    click.echo(f"Contract: {c.name}")
+    if c.description:
+        click.echo(f"Description: {c.description}")
+    click.echo()
+
+    click.echo("Columns:")
+    for col_name, col_def in c.columns.items():
+        constraints = []
+        if col_def.not_null:
+            constraints.append("NOT NULL")
+        if col_def.min is not None:
+            constraints.append(f"min={col_def.min}")
+        if col_def.max is not None:
+            constraints.append(f"max={col_def.max}")
+        if col_def.allowed:
+            constraints.append(f"allowed={col_def.allowed}")
+        if col_def.freshness_minutes:
+            constraints.append(f"freshness={col_def.freshness_minutes}m")
+
+        constraint_str = f" ({', '.join(constraints)})" if constraints else ""
+        click.echo(f"  {col_name}: {col_def.dtype}{constraint_str}")
+
+    if c.rules:
+        click.echo()
+        click.echo("Rules:")
+        for rule in c.rules:
+            click.echo(f"  - {rule}")
+
+
+@main.command()
+@click.argument("data", type=click.Path(exists=True, path_type=Path))
+@click.argument("output", type=click.Path(path_type=Path))
+def infer(data: Path, output: Path) -> None:
+    """Infer a contract from DATA and write to OUTPUT."""
+    df = _load_dataframe(data)
+
+    from dqflow.column import Column
+
+    columns = {}
+    for col in df.columns:
+        dtype = df[col].dtype
+        if pd.api.types.is_integer_dtype(dtype):
+            columns[col] = Column(dtype=int)
+        elif pd.api.types.is_float_dtype(dtype):
+            columns[col] = Column(dtype=float)
+        elif pd.api.types.is_bool_dtype(dtype):
+            columns[col] = Column(dtype=bool)
+        elif pd.api.types.is_datetime64_any_dtype(dtype):
+            columns[col] = Column(dtype="timestamp")
+        else:
+            columns[col] = Column(dtype=str)
+
+    contract = Contract(name=output.stem, columns=columns)
+    contract.to_yaml(output)
+    click.echo(f"Contract written to {output}")
+
+
+def _load_dataframe(path: Path) -> pd.DataFrame:
+    """Load DataFrame from file based on extension."""
+    suffix = path.suffix.lower()
+    if suffix == ".parquet":
+        return pd.read_parquet(path)
+    elif suffix == ".csv":
+        return pd.read_csv(path)
+    elif suffix == ".json":
+        return pd.read_json(path)
+    else:
+        raise click.ClickException(f"Unsupported file format: {suffix}")
+
+
+if __name__ == "__main__":
+    main()

dqflow-0.1.0/src/dqflow/column.py

@@ -0,0 +1,25 @@
+"""Column definition and validation logic."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Column:
+    """Define expectations for a single column."""
+
+    dtype: type | str
+    not_null: bool = False
+    min: float | None = None
+    max: float | None = None
+    allowed: Sequence[Any] | None = None
+    freshness_minutes: int | None = None
+    description: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.min is not None and self.max is not None and self.min > self.max:
+            raise ValueError(f"min ({self.min}) cannot be greater than max ({self.max})")