dqflow 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dqflow/__init__.py +8 -0
- dqflow/cli.py +130 -0
- dqflow/column.py +25 -0
- dqflow/contract.py +102 -0
- dqflow/engines/__init__.py +6 -0
- dqflow/engines/base.py +19 -0
- dqflow/engines/pandas.py +179 -0
- dqflow/result.py +68 -0
- dqflow-0.1.0.dist-info/METADATA +146 -0
- dqflow-0.1.0.dist-info/RECORD +13 -0
- dqflow-0.1.0.dist-info/WHEEL +4 -0
- dqflow-0.1.0.dist-info/entry_points.txt +2 -0
- dqflow-0.1.0.dist-info/licenses/LICENSE +21 -0
dqflow/__init__.py
ADDED
dqflow/cli.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Command-line interface for dqflow."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import click
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from dqflow import __version__
|
|
13
|
+
from dqflow.contract import Contract
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@click.group()
|
|
17
|
+
@click.version_option(version=__version__, prog_name="dqflow")
|
|
18
|
+
def main() -> None:
|
|
19
|
+
"""dqflow - Contract-first data quality for modern pipelines."""
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@main.command()
|
|
24
|
+
@click.argument("contract", type=click.Path(exists=True, path_type=Path))
|
|
25
|
+
@click.argument("data", type=click.Path(exists=True, path_type=Path))
|
|
26
|
+
@click.option("--output", "-o", type=click.Choice(["text", "json"]), default="text")
|
|
27
|
+
@click.option("--fail-fast", is_flag=True, help="Exit with error code on validation failure")
|
|
28
|
+
def validate(contract: Path, data: Path, output: str, fail_fast: bool) -> None:
|
|
29
|
+
"""Validate DATA against CONTRACT.
|
|
30
|
+
|
|
31
|
+
CONTRACT: Path to contract YAML file
|
|
32
|
+
DATA: Path to data file (parquet, csv, json)
|
|
33
|
+
"""
|
|
34
|
+
# Load contract
|
|
35
|
+
c = Contract.from_yaml(contract)
|
|
36
|
+
|
|
37
|
+
# Load data based on extension
|
|
38
|
+
df = _load_dataframe(data)
|
|
39
|
+
|
|
40
|
+
# Validate
|
|
41
|
+
result = c.validate(df)
|
|
42
|
+
|
|
43
|
+
# Output results
|
|
44
|
+
if output == "json":
|
|
45
|
+
click.echo(json.dumps(result.to_dict(), indent=2))
|
|
46
|
+
else:
|
|
47
|
+
click.echo(result.summary())
|
|
48
|
+
|
|
49
|
+
if fail_fast and not result.ok:
|
|
50
|
+
sys.exit(1)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@main.command()
|
|
54
|
+
@click.argument("contract", type=click.Path(exists=True, path_type=Path))
|
|
55
|
+
def show(contract: Path) -> None:
|
|
56
|
+
"""Show details of a CONTRACT."""
|
|
57
|
+
c = Contract.from_yaml(contract)
|
|
58
|
+
|
|
59
|
+
click.echo(f"Contract: {c.name}")
|
|
60
|
+
if c.description:
|
|
61
|
+
click.echo(f"Description: {c.description}")
|
|
62
|
+
click.echo()
|
|
63
|
+
|
|
64
|
+
click.echo("Columns:")
|
|
65
|
+
for col_name, col_def in c.columns.items():
|
|
66
|
+
constraints = []
|
|
67
|
+
if col_def.not_null:
|
|
68
|
+
constraints.append("NOT NULL")
|
|
69
|
+
if col_def.min is not None:
|
|
70
|
+
constraints.append(f"min={col_def.min}")
|
|
71
|
+
if col_def.max is not None:
|
|
72
|
+
constraints.append(f"max={col_def.max}")
|
|
73
|
+
if col_def.allowed:
|
|
74
|
+
constraints.append(f"allowed={col_def.allowed}")
|
|
75
|
+
if col_def.freshness_minutes:
|
|
76
|
+
constraints.append(f"freshness={col_def.freshness_minutes}m")
|
|
77
|
+
|
|
78
|
+
constraint_str = f" ({', '.join(constraints)})" if constraints else ""
|
|
79
|
+
click.echo(f" {col_name}: {col_def.dtype}{constraint_str}")
|
|
80
|
+
|
|
81
|
+
if c.rules:
|
|
82
|
+
click.echo()
|
|
83
|
+
click.echo("Rules:")
|
|
84
|
+
for rule in c.rules:
|
|
85
|
+
click.echo(f" - {rule}")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@main.command()
|
|
89
|
+
@click.argument("data", type=click.Path(exists=True, path_type=Path))
|
|
90
|
+
@click.argument("output", type=click.Path(path_type=Path))
|
|
91
|
+
def infer(data: Path, output: Path) -> None:
|
|
92
|
+
"""Infer a contract from DATA and write to OUTPUT."""
|
|
93
|
+
df = _load_dataframe(data)
|
|
94
|
+
|
|
95
|
+
from dqflow.column import Column
|
|
96
|
+
|
|
97
|
+
columns = {}
|
|
98
|
+
for col in df.columns:
|
|
99
|
+
dtype = df[col].dtype
|
|
100
|
+
if pd.api.types.is_integer_dtype(dtype):
|
|
101
|
+
columns[col] = Column(dtype=int)
|
|
102
|
+
elif pd.api.types.is_float_dtype(dtype):
|
|
103
|
+
columns[col] = Column(dtype=float)
|
|
104
|
+
elif pd.api.types.is_bool_dtype(dtype):
|
|
105
|
+
columns[col] = Column(dtype=bool)
|
|
106
|
+
elif pd.api.types.is_datetime64_any_dtype(dtype):
|
|
107
|
+
columns[col] = Column(dtype="timestamp")
|
|
108
|
+
else:
|
|
109
|
+
columns[col] = Column(dtype=str)
|
|
110
|
+
|
|
111
|
+
contract = Contract(name=output.stem, columns=columns)
|
|
112
|
+
contract.to_yaml(output)
|
|
113
|
+
click.echo(f"Contract written to {output}")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _load_dataframe(path: Path) -> pd.DataFrame:
|
|
117
|
+
"""Load DataFrame from file based on extension."""
|
|
118
|
+
suffix = path.suffix.lower()
|
|
119
|
+
if suffix == ".parquet":
|
|
120
|
+
return pd.read_parquet(path)
|
|
121
|
+
elif suffix == ".csv":
|
|
122
|
+
return pd.read_csv(path)
|
|
123
|
+
elif suffix == ".json":
|
|
124
|
+
return pd.read_json(path)
|
|
125
|
+
else:
|
|
126
|
+
raise click.ClickException(f"Unsupported file format: {suffix}")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
if __name__ == "__main__":
|
|
130
|
+
main()
|
dqflow/column.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Column definition and validation logic."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class Column:
|
|
12
|
+
"""Define expectations for a single column."""
|
|
13
|
+
|
|
14
|
+
dtype: type | str
|
|
15
|
+
not_null: bool = False
|
|
16
|
+
min: float | None = None
|
|
17
|
+
max: float | None = None
|
|
18
|
+
allowed: Sequence[Any] | None = None
|
|
19
|
+
freshness_minutes: int | None = None
|
|
20
|
+
description: str = ""
|
|
21
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
22
|
+
|
|
23
|
+
def __post_init__(self) -> None:
|
|
24
|
+
if self.min is not None and self.max is not None and self.min > self.max:
|
|
25
|
+
raise ValueError(f"min ({self.min}) cannot be greater than max ({self.max})")
|
dqflow/contract.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Contract definition and validation orchestration."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import TYPE_CHECKING, Any
|
|
8
|
+
|
|
9
|
+
import yaml
|
|
10
|
+
|
|
11
|
+
from dqflow.column import Column
|
|
12
|
+
from dqflow.result import ValidationResult
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
import pandas as pd
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Contract:
|
|
20
|
+
"""Data quality contract defining expectations for a dataset."""
|
|
21
|
+
|
|
22
|
+
name: str
|
|
23
|
+
columns: dict[str, Column] = field(default_factory=dict)
|
|
24
|
+
rules: list[str] = field(default_factory=list)
|
|
25
|
+
description: str = ""
|
|
26
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
27
|
+
|
|
28
|
+
def validate(self, df: pd.DataFrame) -> ValidationResult:
|
|
29
|
+
"""Validate a DataFrame against this contract."""
|
|
30
|
+
from dqflow.engines.pandas import PandasEngine
|
|
31
|
+
|
|
32
|
+
engine = PandasEngine()
|
|
33
|
+
return engine.validate(df, self)
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def from_yaml(cls, path: str | Path) -> Contract:
|
|
37
|
+
"""Load contract from YAML file."""
|
|
38
|
+
path = Path(path)
|
|
39
|
+
with path.open() as f:
|
|
40
|
+
data = yaml.safe_load(f)
|
|
41
|
+
|
|
42
|
+
columns = {}
|
|
43
|
+
for col_name, col_def in data.get("columns", {}).items():
|
|
44
|
+
if isinstance(col_def, dict):
|
|
45
|
+
dtype = col_def.pop("type", str)
|
|
46
|
+
columns[col_name] = Column(dtype=dtype, **col_def)
|
|
47
|
+
else:
|
|
48
|
+
columns[col_name] = Column(dtype=col_def)
|
|
49
|
+
|
|
50
|
+
return cls(
|
|
51
|
+
name=data.get("name", path.stem),
|
|
52
|
+
columns=columns,
|
|
53
|
+
rules=data.get("rules", []),
|
|
54
|
+
description=data.get("description", ""),
|
|
55
|
+
metadata=data.get("metadata", {}),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def to_yaml(self, path: str | Path) -> None:
|
|
59
|
+
"""Save contract to YAML file."""
|
|
60
|
+
path = Path(path)
|
|
61
|
+
|
|
62
|
+
columns_data = {}
|
|
63
|
+
for col_name, col in self.columns.items():
|
|
64
|
+
col_dict: dict[str, Any] = {"type": _dtype_to_str(col.dtype)}
|
|
65
|
+
if col.not_null:
|
|
66
|
+
col_dict["not_null"] = True
|
|
67
|
+
if col.min is not None:
|
|
68
|
+
col_dict["min"] = col.min
|
|
69
|
+
if col.max is not None:
|
|
70
|
+
col_dict["max"] = col.max
|
|
71
|
+
if col.allowed is not None:
|
|
72
|
+
col_dict["allowed"] = list(col.allowed)
|
|
73
|
+
if col.freshness_minutes is not None:
|
|
74
|
+
col_dict["freshness_minutes"] = col.freshness_minutes
|
|
75
|
+
columns_data[col_name] = col_dict
|
|
76
|
+
|
|
77
|
+
data = {
|
|
78
|
+
"name": self.name,
|
|
79
|
+
"columns": columns_data,
|
|
80
|
+
}
|
|
81
|
+
if self.rules:
|
|
82
|
+
data["rules"] = self.rules
|
|
83
|
+
if self.description:
|
|
84
|
+
data["description"] = self.description
|
|
85
|
+
|
|
86
|
+
with path.open("w") as f:
|
|
87
|
+
yaml.safe_dump(data, f, default_flow_style=False, sort_keys=False)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _dtype_to_str(dtype: type | str) -> str:
|
|
91
|
+
"""Convert dtype to string representation."""
|
|
92
|
+
if isinstance(dtype, str):
|
|
93
|
+
return dtype
|
|
94
|
+
if dtype is str:
|
|
95
|
+
return "string"
|
|
96
|
+
if dtype is int:
|
|
97
|
+
return "integer"
|
|
98
|
+
if dtype is float:
|
|
99
|
+
return "float"
|
|
100
|
+
if dtype is bool:
|
|
101
|
+
return "boolean"
|
|
102
|
+
return str(dtype)
|
dqflow/engines/base.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Abstract base engine."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from typing import TYPE_CHECKING, Any
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from dqflow.contract import Contract
|
|
10
|
+
from dqflow.result import ValidationResult
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Engine(ABC):
|
|
14
|
+
"""Abstract base class for validation engines."""
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def validate(self, data: Any, contract: Contract) -> ValidationResult:
|
|
18
|
+
"""Validate data against a contract."""
|
|
19
|
+
...
|
dqflow/engines/pandas.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Pandas validation engine."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import datetime, timedelta, timezone
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
from dqflow.column import Column
|
|
11
|
+
from dqflow.contract import Contract
|
|
12
|
+
from dqflow.engines.base import Engine
|
|
13
|
+
from dqflow.result import CheckResult, ValidationResult
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class PandasEngine(Engine):
|
|
17
|
+
"""Validation engine for pandas DataFrames."""
|
|
18
|
+
|
|
19
|
+
def validate(self, df: pd.DataFrame, contract: Contract) -> ValidationResult:
|
|
20
|
+
"""Validate a DataFrame against a contract."""
|
|
21
|
+
result = ValidationResult(contract_name=contract.name)
|
|
22
|
+
|
|
23
|
+
# Column existence checks
|
|
24
|
+
for col_name in contract.columns:
|
|
25
|
+
if col_name not in df.columns:
|
|
26
|
+
result.checks.append(
|
|
27
|
+
CheckResult(
|
|
28
|
+
name=f"column_exists:{col_name}",
|
|
29
|
+
passed=False,
|
|
30
|
+
message=f"Column '{col_name}' not found in DataFrame",
|
|
31
|
+
)
|
|
32
|
+
)
|
|
33
|
+
else:
|
|
34
|
+
result.checks.append(
|
|
35
|
+
CheckResult(name=f"column_exists:{col_name}", passed=True)
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
# Column-level checks
|
|
39
|
+
for col_name, col_def in contract.columns.items():
|
|
40
|
+
if col_name not in df.columns:
|
|
41
|
+
continue
|
|
42
|
+
result.checks.extend(self._validate_column(df[col_name], col_name, col_def))
|
|
43
|
+
|
|
44
|
+
# Table-level rules
|
|
45
|
+
for rule in contract.rules:
|
|
46
|
+
result.checks.append(self._evaluate_rule(df, rule))
|
|
47
|
+
|
|
48
|
+
return result
|
|
49
|
+
|
|
50
|
+
def _validate_column(
|
|
51
|
+
self, series: pd.Series, col_name: str, col_def: Column
|
|
52
|
+
) -> list[CheckResult]:
|
|
53
|
+
"""Validate a single column."""
|
|
54
|
+
checks = []
|
|
55
|
+
|
|
56
|
+
# Not null check
|
|
57
|
+
if col_def.not_null:
|
|
58
|
+
null_count = series.isna().sum()
|
|
59
|
+
checks.append(
|
|
60
|
+
CheckResult(
|
|
61
|
+
name=f"not_null:{col_name}",
|
|
62
|
+
passed=null_count == 0,
|
|
63
|
+
message=f"Found {null_count} null values" if null_count > 0 else "",
|
|
64
|
+
details={"null_count": int(null_count)},
|
|
65
|
+
)
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
# Min check
|
|
69
|
+
if col_def.min is not None:
|
|
70
|
+
min_val = series.min()
|
|
71
|
+
passed = pd.isna(min_val) or min_val >= col_def.min
|
|
72
|
+
checks.append(
|
|
73
|
+
CheckResult(
|
|
74
|
+
name=f"min:{col_name}",
|
|
75
|
+
passed=bool(passed),
|
|
76
|
+
message=f"Minimum value {min_val} is below {col_def.min}" if not passed else "",
|
|
77
|
+
details={"actual_min": float(min_val) if pd.notna(min_val) else None},
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
# Max check
|
|
82
|
+
if col_def.max is not None:
|
|
83
|
+
max_val = series.max()
|
|
84
|
+
passed = pd.isna(max_val) or max_val <= col_def.max
|
|
85
|
+
checks.append(
|
|
86
|
+
CheckResult(
|
|
87
|
+
name=f"max:{col_name}",
|
|
88
|
+
passed=bool(passed),
|
|
89
|
+
message=f"Maximum value {max_val} exceeds {col_def.max}" if not passed else "",
|
|
90
|
+
details={"actual_max": float(max_val) if pd.notna(max_val) else None},
|
|
91
|
+
)
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
# Allowed values check
|
|
95
|
+
if col_def.allowed is not None:
|
|
96
|
+
allowed_set = set(col_def.allowed)
|
|
97
|
+
unique_vals = set(series.dropna().unique())
|
|
98
|
+
invalid = unique_vals - allowed_set
|
|
99
|
+
checks.append(
|
|
100
|
+
CheckResult(
|
|
101
|
+
name=f"allowed:{col_name}",
|
|
102
|
+
passed=len(invalid) == 0,
|
|
103
|
+
message=f"Found invalid values: {invalid}" if invalid else "",
|
|
104
|
+
details={"invalid_values": list(invalid)},
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
# Freshness check
|
|
109
|
+
if col_def.freshness_minutes is not None:
|
|
110
|
+
checks.append(self._check_freshness(series, col_name, col_def.freshness_minutes))
|
|
111
|
+
|
|
112
|
+
return checks
|
|
113
|
+
|
|
114
|
+
def _check_freshness(
|
|
115
|
+
self, series: pd.Series, col_name: str, freshness_minutes: int
|
|
116
|
+
) -> CheckResult:
|
|
117
|
+
"""Check if timestamp column has recent data."""
|
|
118
|
+
try:
|
|
119
|
+
ts_series = pd.to_datetime(series)
|
|
120
|
+
max_ts = ts_series.max()
|
|
121
|
+
if pd.isna(max_ts):
|
|
122
|
+
return CheckResult(
|
|
123
|
+
name=f"freshness:{col_name}",
|
|
124
|
+
passed=False,
|
|
125
|
+
message="No valid timestamps found",
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
now = datetime.now(timezone.utc)
|
|
129
|
+
if max_ts.tzinfo is None:
|
|
130
|
+
max_ts = max_ts.replace(tzinfo=timezone.utc)
|
|
131
|
+
|
|
132
|
+
age = now - max_ts
|
|
133
|
+
threshold = timedelta(minutes=freshness_minutes)
|
|
134
|
+
passed = age <= threshold
|
|
135
|
+
|
|
136
|
+
return CheckResult(
|
|
137
|
+
name=f"freshness:{col_name}",
|
|
138
|
+
passed=passed,
|
|
139
|
+
message=f"Data is {age} old, threshold is {threshold}" if not passed else "",
|
|
140
|
+
details={
|
|
141
|
+
"max_timestamp": max_ts.isoformat(),
|
|
142
|
+
"age_minutes": age.total_seconds() / 60,
|
|
143
|
+
},
|
|
144
|
+
)
|
|
145
|
+
except Exception as e:
|
|
146
|
+
return CheckResult(
|
|
147
|
+
name=f"freshness:{col_name}",
|
|
148
|
+
passed=False,
|
|
149
|
+
message=f"Failed to check freshness: {e}",
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
def _evaluate_rule(self, df: pd.DataFrame, rule: str) -> CheckResult:
|
|
153
|
+
"""Evaluate a table-level rule."""
|
|
154
|
+
try:
|
|
155
|
+
result = self._parse_and_evaluate(df, rule)
|
|
156
|
+
return CheckResult(
|
|
157
|
+
name=f"rule:{rule}",
|
|
158
|
+
passed=bool(result),
|
|
159
|
+
message="" if result else f"Rule '{rule}' failed",
|
|
160
|
+
)
|
|
161
|
+
except Exception as e:
|
|
162
|
+
return CheckResult(
|
|
163
|
+
name=f"rule:{rule}",
|
|
164
|
+
passed=False,
|
|
165
|
+
message=f"Failed to evaluate rule: {e}",
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
def _parse_and_evaluate(self, df: pd.DataFrame, rule: str) -> Any:
|
|
169
|
+
"""Parse and evaluate a rule expression."""
|
|
170
|
+
# Simple rule parser - extend as needed
|
|
171
|
+
context = {
|
|
172
|
+
"row_count": len(df),
|
|
173
|
+
"null_rate": lambda col: df[col].isna().mean() if col in df.columns else 1.0,
|
|
174
|
+
"unique_count": lambda col: df[col].nunique() if col in df.columns else 0,
|
|
175
|
+
"duplicate_rate": lambda col: (
|
|
176
|
+
1 - (df[col].nunique() / len(df)) if col in df.columns and len(df) > 0 else 0
|
|
177
|
+
),
|
|
178
|
+
}
|
|
179
|
+
return eval(rule, {"__builtins__": {}}, context)
|
dqflow/result.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Validation result structures."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class CheckResult:
|
|
11
|
+
"""Result of a single validation check."""
|
|
12
|
+
|
|
13
|
+
name: str
|
|
14
|
+
passed: bool
|
|
15
|
+
message: str = ""
|
|
16
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class ValidationResult:
|
|
21
|
+
"""Aggregated result of contract validation."""
|
|
22
|
+
|
|
23
|
+
contract_name: str
|
|
24
|
+
checks: list[CheckResult] = field(default_factory=list)
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def ok(self) -> bool:
|
|
28
|
+
"""Return True if all checks passed."""
|
|
29
|
+
return all(check.passed for check in self.checks)
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def failed_checks(self) -> list[CheckResult]:
|
|
33
|
+
"""Return list of failed checks."""
|
|
34
|
+
return [check for check in self.checks if not check.passed]
|
|
35
|
+
|
|
36
|
+
def summary(self) -> str:
|
|
37
|
+
"""Return human-readable summary."""
|
|
38
|
+
total = len(self.checks)
|
|
39
|
+
passed = sum(1 for c in self.checks if c.passed)
|
|
40
|
+
failed = total - passed
|
|
41
|
+
|
|
42
|
+
lines = [f"Contract '{self.contract_name}': {passed}/{total} checks passed"]
|
|
43
|
+
|
|
44
|
+
if failed > 0:
|
|
45
|
+
lines.append("Failed checks:")
|
|
46
|
+
for check in self.failed_checks:
|
|
47
|
+
lines.append(f" - {check.name}: {check.message}")
|
|
48
|
+
|
|
49
|
+
return "\n".join(lines)
|
|
50
|
+
|
|
51
|
+
def to_dict(self) -> dict[str, Any]:
|
|
52
|
+
"""Return JSON-serializable dict."""
|
|
53
|
+
return {
|
|
54
|
+
"contract_name": self.contract_name,
|
|
55
|
+
"ok": self.ok,
|
|
56
|
+
"total_checks": len(self.checks),
|
|
57
|
+
"passed": sum(1 for c in self.checks if c.passed),
|
|
58
|
+
"failed": sum(1 for c in self.checks if not c.passed),
|
|
59
|
+
"checks": [
|
|
60
|
+
{
|
|
61
|
+
"name": c.name,
|
|
62
|
+
"passed": c.passed,
|
|
63
|
+
"message": c.message,
|
|
64
|
+
"details": c.details,
|
|
65
|
+
}
|
|
66
|
+
for c in self.checks
|
|
67
|
+
],
|
|
68
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dqflow
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Lightweight, contract-first data quality engine for modern data pipelines
|
|
5
|
+
Project-URL: Homepage, https://github.com/quang/dqflow
|
|
6
|
+
Project-URL: Repository, https://github.com/quang/dqflow
|
|
7
|
+
Project-URL: Issues, https://github.com/quang/dqflow/issues
|
|
8
|
+
Author-email: Quang <nguyenvanquang247@gmail.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: data-contracts,data-quality,data-validation,etl,pandas
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Requires-Dist: click>=8.0
|
|
23
|
+
Requires-Dist: pandas>=1.5.0
|
|
24
|
+
Requires-Dist: pyyaml>=6.0
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: mypy>=1.0; extra == 'dev'
|
|
27
|
+
Requires-Dist: pre-commit>=3.0; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# dqflow
|
|
34
|
+
|
|
35
|
+
**dqflow** is a lightweight, contract-first data quality engine for modern data pipelines.
|
|
36
|
+
|
|
37
|
+
Define explicit expectations for your data (schema, validity, freshness) and **fail fast** when data breaks — before bad data reaches downstream systems.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## Why dqflow?
|
|
42
|
+
|
|
43
|
+
Data quality issues are inevitable — silent failures are not.
|
|
44
|
+
|
|
45
|
+
Most teams rely on ad-hoc checks, fragile assertions, or heavyweight frameworks that are hard to maintain. dqflow takes a different approach:
|
|
46
|
+
|
|
47
|
+
* **Contracts over checks** — expectations are explicit and versionable
|
|
48
|
+
* **Pipeline-first** — designed for ETL, ELT, and streaming workflows
|
|
49
|
+
* **Lightweight & Pythonic** — minimal API, easy to embed
|
|
50
|
+
* **Fail fast** — break pipelines intentionally, not silently
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## Quick example
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from dqflow import Contract, Column
|
|
58
|
+
|
|
59
|
+
orders = Contract(
|
|
60
|
+
name="orders",
|
|
61
|
+
columns={
|
|
62
|
+
"order_id": Column(str, not_null=True),
|
|
63
|
+
"amount": Column(float, min=0),
|
|
64
|
+
"currency": Column(str, allowed=["USD", "EUR"]),
|
|
65
|
+
"created_at": Column("timestamp", freshness_minutes=60),
|
|
66
|
+
},
|
|
67
|
+
rules=[
|
|
68
|
+
"row_count > 1000",
|
|
69
|
+
"null_rate(amount) < 0.01",
|
|
70
|
+
],
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
result = orders.validate(df)
|
|
74
|
+
|
|
75
|
+
if not result.ok:
|
|
76
|
+
raise Exception(result.summary())
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
## Features (v0.1 scope)
|
|
82
|
+
|
|
83
|
+
* Contract-as-code (Python & YAML)
|
|
84
|
+
* Column-level checks
|
|
85
|
+
|
|
86
|
+
* type validation
|
|
87
|
+
* not null
|
|
88
|
+
* min / max
|
|
89
|
+
* allowed values
|
|
90
|
+
* Table-level checks
|
|
91
|
+
|
|
92
|
+
* row count
|
|
93
|
+
* freshness
|
|
94
|
+
* Structured validation results (JSON-friendly)
|
|
95
|
+
* Pandas engine
|
|
96
|
+
* CLI support
|
|
97
|
+
|
|
98
|
+
---
|
|
99
|
+
|
|
100
|
+
## CLI usage
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
dq validate contracts/orders.yaml data/orders.parquet
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## Supported engines
|
|
109
|
+
|
|
110
|
+
* ✅ Pandas
|
|
111
|
+
* 🚧 PySpark (planned)
|
|
112
|
+
* 🚧 SQL tables (planned)
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Philosophy
|
|
117
|
+
|
|
118
|
+
* **Explicit is better than implicit**
|
|
119
|
+
* **Bad data should break pipelines early**
|
|
120
|
+
* **Quality rules are part of your system design**
|
|
121
|
+
|
|
122
|
+
> dqflow is not a full data observability platform.
|
|
123
|
+
> It is a small, opinionated library meant to be embedded directly into pipelines.
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
## Roadmap
|
|
128
|
+
|
|
129
|
+
* PySpark engine
|
|
130
|
+
* dbt / dlt integrations
|
|
131
|
+
* Incremental & backfill-aware validation
|
|
132
|
+
* Metrics export (Prometheus-compatible)
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## License
|
|
137
|
+
|
|
138
|
+
MIT
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## Status
|
|
143
|
+
|
|
144
|
+
🚧 Early development (v0.1.0)
|
|
145
|
+
|
|
146
|
+
APIs may change. Feedback and contributions are welcome.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
dqflow/__init__.py,sha256=zHtLJbdDnm8eSEHnhzLqMD28dfOpINX4DGQkoNWWsWg,254
|
|
2
|
+
dqflow/cli.py,sha256=Nl8HYmPt-BV8r4BJo8zTJIwGZkJH_v3cNbqGDozVn8E,3945
|
|
3
|
+
dqflow/column.py,sha256=WkX7UwirIr7SPhBrcz7Ed0e0RPCQOsxIq7bOPtR8Scg,747
|
|
4
|
+
dqflow/contract.py,sha256=hXQpwt44SCKyAFO7dwITFzZo6NuGPOesWZOKMvCffIQ,3137
|
|
5
|
+
dqflow/result.py,sha256=kuXMgK8nPn52v6bbHfiqlpPdWKzoziwVQcXweEh3hto,1959
|
|
6
|
+
dqflow/engines/__init__.py,sha256=b-TtUG7wQc-Z1O42E5lhOvHU-XpTrxNiaRx2hnGv-74,151
|
|
7
|
+
dqflow/engines/base.py,sha256=5RnXf3CqQ73ScuvMRul7PqlFfRdqDljwUpg4VqZiYbo,477
|
|
8
|
+
dqflow/engines/pandas.py,sha256=8JT9OcssVCNdGBcaYPxB9bJyRwlEwjaHbzvrBR16jcI,6534
|
|
9
|
+
dqflow-0.1.0.dist-info/METADATA,sha256=nE1zRlHJ_9uaBHqNx_Uely4lcXWcPhNbeoeZZmtlFwk,3617
|
|
10
|
+
dqflow-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
11
|
+
dqflow-0.1.0.dist-info/entry_points.txt,sha256=sS9ovPRPoTFro_Pi-5cAhj7HJoilZTYQYGoBstFiXIU,39
|
|
12
|
+
dqflow-0.1.0.dist-info/licenses/LICENSE,sha256=csldIGJQ7mpAJegge6WFFzOIvLVGUO8CWz0xf57Mj9g,1062
|
|
13
|
+
dqflow-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Quang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|