csv-quality-gate 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csv_quality_gate-0.1.0/LICENSE +21 -0
- csv_quality_gate-0.1.0/PKG-INFO +148 -0
- csv_quality_gate-0.1.0/README.md +128 -0
- csv_quality_gate-0.1.0/pyproject.toml +42 -0
- csv_quality_gate-0.1.0/setup.cfg +4 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate/__init__.py +4 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate/cli.py +44 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate/models.py +30 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate/profiles.py +65 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate/report.py +41 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate/validator.py +104 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate.egg-info/PKG-INFO +148 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate.egg-info/SOURCES.txt +17 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate.egg-info/dependency_links.txt +1 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate.egg-info/entry_points.txt +2 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate.egg-info/requires.txt +4 -0
- csv_quality_gate-0.1.0/src/csv_quality_gate.egg-info/top_level.txt +1 -0
- csv_quality_gate-0.1.0/tests/test_cli.py +31 -0
- csv_quality_gate-0.1.0/tests/test_validator.py +29 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Rolando Bosch
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: csv-quality-gate
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CSV preflight validation and batch CSV quality checks that fail fast before pipeline runs.
|
|
5
|
+
Author: Rolando Bosch
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: csv preflight validation,batch CSV quality checks,fail fast,pipeline validation,csv lint
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
12
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
18
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# csv-quality-gate: CSV preflight validation before pipeline runs
|
|
22
|
+
|
|
23
|
+
Fail fast before pipeline runs when the input CSV is broken, incomplete, duplicated, or obviously junk.
|
|
24
|
+
|
|
25
|
+
`csv-quality-gate` runs batch CSV quality checks and returns `pass`, `warn`, or `fail` before expensive pipeline steps burn time on bad input.
|
|
26
|
+
|
|
27
|
+
- "We keep running expensive pipeline steps on broken CSVs."
|
|
28
|
+
- "A batch run fails 20 minutes in because the input CSV was junk."
|
|
29
|
+
- "We only discover missing required columns after the job already started."
|
|
30
|
+
- "Duplicate rows and empty contact fields keep polluting our batch runs."
|
|
31
|
+
- "I want CSV preflight validation, not a whole data platform."
|
|
32
|
+
|
|
33
|
+
Fastest install:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install csv-quality-gate
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Fastest real usage:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
csv-quality-gate check leads.csv --profile outreach
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Exact outcome:
|
|
46
|
+
|
|
47
|
+
```text
|
|
48
|
+
csv-quality-gate: FAIL
|
|
49
|
+
file: leads.csv
|
|
50
|
+
profile: outreach
|
|
51
|
+
rows: 125
|
|
52
|
+
ERROR: missing required column: person_name
|
|
53
|
+
WARNING: duplicate rate 12% exceeds warning threshold 10%
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+

|
|
57
|
+
|
|
58
|
+
It is designed for narrow, honest use as a preflight gate, not as a full data quality platform.
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install csv-quality-gate
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
For development:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install -e ".[dev]"
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Common search-intent use cases
|
|
73
|
+
|
|
74
|
+
- CSV preflight validation
|
|
75
|
+
- batch CSV quality checks
|
|
76
|
+
- fail fast before pipeline runs
|
|
77
|
+
- CSV validation before ETL or enrichment
|
|
78
|
+
- detect junk CSV rows before batch jobs
|
|
79
|
+
|
|
80
|
+
## Usage
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
csv-quality-gate check leads.csv
|
|
84
|
+
csv-quality-gate check leads.csv --profile outreach
|
|
85
|
+
csv-quality-gate check leads.csv --profile generic --json
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Exit codes:
|
|
89
|
+
|
|
90
|
+
- `0` pass
|
|
91
|
+
- `1` warnings only
|
|
92
|
+
- `2` fail
|
|
93
|
+
|
|
94
|
+
## Profiles
|
|
95
|
+
|
|
96
|
+
Built-in profiles:
|
|
97
|
+
|
|
98
|
+
- `generic`
|
|
99
|
+
- validates required columns, empties, duplicates, empty file
|
|
100
|
+
- `outreach`
|
|
101
|
+
- adds suspicious company-name heuristics for GTM/contact pipelines
|
|
102
|
+
|
|
103
|
+
## Output
|
|
104
|
+
|
|
105
|
+
```text
|
|
106
|
+
csv-quality-gate: FAIL
|
|
107
|
+
file: leads.csv
|
|
108
|
+
profile: outreach
|
|
109
|
+
rows: 125
|
|
110
|
+
ERROR: missing required column: person_name
|
|
111
|
+
WARNING: duplicate rate 12% exceeds warning threshold 10%
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## JSON mode
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
csv-quality-gate check leads.csv --json
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## Limitations
|
|
121
|
+
|
|
122
|
+
- Heuristics are intentionally simple.
|
|
123
|
+
- The `outreach` profile is opinionated and should not be treated as universal truth.
|
|
124
|
+
- The tool validates shape and obvious noise, not semantic correctness.
|
|
125
|
+
|
|
126
|
+
## When To Use It
|
|
127
|
+
|
|
128
|
+
- Before enrichment, outreach, ETL, or batch scoring runs
|
|
129
|
+
- In CI for checked-in CSV inputs
|
|
130
|
+
- As a preflight gate before expensive pipeline work
|
|
131
|
+
|
|
132
|
+
## When Not To Use It
|
|
133
|
+
|
|
134
|
+
- When you need semantic validation of the data itself
|
|
135
|
+
- When your input is not CSV
|
|
136
|
+
- When you need a full data quality framework with lineage and profiling
|
|
137
|
+
|
|
138
|
+
## More From Hermes Labs
|
|
139
|
+
|
|
140
|
+
- [intent-verify](https://github.com/roli-lpci/intent-verify): repo intent verification and spec drift checks
|
|
141
|
+
|
|
142
|
+
## Development
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
ruff check .
|
|
146
|
+
python3 -m pytest -q
|
|
147
|
+
python3 -m py_compile src/csv_quality_gate/*.py
|
|
148
|
+
```
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# csv-quality-gate: CSV preflight validation before pipeline runs
|
|
2
|
+
|
|
3
|
+
Fail fast before pipeline runs when the input CSV is broken, incomplete, duplicated, or obviously junk.
|
|
4
|
+
|
|
5
|
+
`csv-quality-gate` runs batch CSV quality checks and returns `pass`, `warn`, or `fail` before expensive pipeline steps burn time on bad input.
|
|
6
|
+
|
|
7
|
+
- "We keep running expensive pipeline steps on broken CSVs."
|
|
8
|
+
- "A batch run fails 20 minutes in because the input CSV was junk."
|
|
9
|
+
- "We only discover missing required columns after the job already started."
|
|
10
|
+
- "Duplicate rows and empty contact fields keep polluting our batch runs."
|
|
11
|
+
- "I want CSV preflight validation, not a whole data platform."
|
|
12
|
+
|
|
13
|
+
Fastest install:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install csv-quality-gate
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Fastest real usage:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
csv-quality-gate check leads.csv --profile outreach
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Exact outcome:
|
|
26
|
+
|
|
27
|
+
```text
|
|
28
|
+
csv-quality-gate: FAIL
|
|
29
|
+
file: leads.csv
|
|
30
|
+
profile: outreach
|
|
31
|
+
rows: 125
|
|
32
|
+
ERROR: missing required column: person_name
|
|
33
|
+
WARNING: duplicate rate 12% exceeds warning threshold 10%
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+

|
|
37
|
+
|
|
38
|
+
It is designed for narrow, honest use as a preflight gate, not as a full data quality platform.
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install csv-quality-gate
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
For development:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install -e ".[dev]"
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Common search-intent use cases
|
|
53
|
+
|
|
54
|
+
- CSV preflight validation
|
|
55
|
+
- batch CSV quality checks
|
|
56
|
+
- fail fast before pipeline runs
|
|
57
|
+
- CSV validation before ETL or enrichment
|
|
58
|
+
- detect junk CSV rows before batch jobs
|
|
59
|
+
|
|
60
|
+
## Usage
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
csv-quality-gate check leads.csv
|
|
64
|
+
csv-quality-gate check leads.csv --profile outreach
|
|
65
|
+
csv-quality-gate check leads.csv --profile generic --json
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Exit codes:
|
|
69
|
+
|
|
70
|
+
- `0` pass
|
|
71
|
+
- `1` warnings only
|
|
72
|
+
- `2` fail
|
|
73
|
+
|
|
74
|
+
## Profiles
|
|
75
|
+
|
|
76
|
+
Built-in profiles:
|
|
77
|
+
|
|
78
|
+
- `generic`
|
|
79
|
+
- validates required columns, empties, duplicates, empty file
|
|
80
|
+
- `outreach`
|
|
81
|
+
- adds suspicious company-name heuristics for GTM/contact pipelines
|
|
82
|
+
|
|
83
|
+
## Output
|
|
84
|
+
|
|
85
|
+
```text
|
|
86
|
+
csv-quality-gate: FAIL
|
|
87
|
+
file: leads.csv
|
|
88
|
+
profile: outreach
|
|
89
|
+
rows: 125
|
|
90
|
+
ERROR: missing required column: person_name
|
|
91
|
+
WARNING: duplicate rate 12% exceeds warning threshold 10%
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## JSON mode
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
csv-quality-gate check leads.csv --json
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Limitations
|
|
101
|
+
|
|
102
|
+
- Heuristics are intentionally simple.
|
|
103
|
+
- The `outreach` profile is opinionated and should not be treated as universal truth.
|
|
104
|
+
- The tool validates shape and obvious noise, not semantic correctness.
|
|
105
|
+
|
|
106
|
+
## When To Use It
|
|
107
|
+
|
|
108
|
+
- Before enrichment, outreach, ETL, or batch scoring runs
|
|
109
|
+
- In CI for checked-in CSV inputs
|
|
110
|
+
- As a preflight gate before expensive pipeline work
|
|
111
|
+
|
|
112
|
+
## When Not To Use It
|
|
113
|
+
|
|
114
|
+
- When you need semantic validation of the data itself
|
|
115
|
+
- When your input is not CSV
|
|
116
|
+
- When you need a full data quality framework with lineage and profiling
|
|
117
|
+
|
|
118
|
+
## More From Hermes Labs
|
|
119
|
+
|
|
120
|
+
- [intent-verify](https://github.com/roli-lpci/intent-verify): repo intent verification and spec drift checks
|
|
121
|
+
|
|
122
|
+
## Development
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
ruff check .
|
|
126
|
+
python3 -m pytest -q
|
|
127
|
+
python3 -m py_compile src/csv_quality_gate/*.py
|
|
128
|
+
```
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "csv-quality-gate"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "CSV preflight validation and batch CSV quality checks that fail fast before pipeline runs."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [{name = "Rolando Bosch"}]
|
|
13
|
+
keywords = ["csv preflight validation", "batch CSV quality checks", "fail fast", "pipeline validation", "csv lint"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
19
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.scripts]
|
|
23
|
+
csv-quality-gate = "csv_quality_gate.cli:main"
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
dev = ["pytest>=8.0", "ruff>=0.5"]
|
|
27
|
+
|
|
28
|
+
[tool.setuptools]
|
|
29
|
+
package-dir = {"" = "src"}
|
|
30
|
+
|
|
31
|
+
[tool.setuptools.packages.find]
|
|
32
|
+
where = ["src"]
|
|
33
|
+
|
|
34
|
+
[tool.pytest.ini_options]
|
|
35
|
+
testpaths = ["tests"]
|
|
36
|
+
|
|
37
|
+
[tool.ruff]
|
|
38
|
+
line-length = 100
|
|
39
|
+
target-version = "py310"
|
|
40
|
+
|
|
41
|
+
[tool.ruff.lint]
|
|
42
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from .profiles import PROFILES
|
|
8
|
+
from .report import exit_code, to_json, to_text
|
|
9
|
+
from .validator import validate_csv
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
13
|
+
parser = argparse.ArgumentParser(
|
|
14
|
+
prog="csv-quality-gate",
|
|
15
|
+
description="Run CSV preflight validation and fail fast before expensive pipeline runs.",
|
|
16
|
+
)
|
|
17
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
18
|
+
|
|
19
|
+
check = subparsers.add_parser("check", help="run batch CSV quality checks on one file")
|
|
20
|
+
check.add_argument("csv_path", help="path to the csv file to validate")
|
|
21
|
+
check.add_argument(
|
|
22
|
+
"--profile",
|
|
23
|
+
default="generic",
|
|
24
|
+
choices=sorted(PROFILES),
|
|
25
|
+
help="validation profile, for example generic or outreach",
|
|
26
|
+
)
|
|
27
|
+
check.add_argument("--json", action="store_true", help="emit machine-readable JSON output")
|
|
28
|
+
return parser
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def main(argv: list[str] | None = None) -> int:
|
|
32
|
+
parser = build_parser()
|
|
33
|
+
args = parser.parse_args(argv)
|
|
34
|
+
path = Path(args.csv_path)
|
|
35
|
+
if not path.exists():
|
|
36
|
+
print(f"csv-quality-gate: file not found: {path}", file=sys.stderr)
|
|
37
|
+
return 2
|
|
38
|
+
result = validate_csv(path, profile_name=args.profile)
|
|
39
|
+
print(to_json(result) if args.json else to_text(result))
|
|
40
|
+
return exit_code(result)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from enum import Enum
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Severity(str, Enum):
|
|
8
|
+
WARNING = "warning"
|
|
9
|
+
ERROR = "error"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Status(str, Enum):
|
|
13
|
+
PASS = "pass"
|
|
14
|
+
WARN = "warn"
|
|
15
|
+
FAIL = "fail"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class Issue:
|
|
20
|
+
severity: Severity
|
|
21
|
+
message: str
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class GateResult:
|
|
26
|
+
path: str
|
|
27
|
+
profile: str
|
|
28
|
+
row_count: int
|
|
29
|
+
issues: list[Issue]
|
|
30
|
+
status: Status
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class Profile:
|
|
9
|
+
name: str
|
|
10
|
+
required_columns: tuple[str, ...]
|
|
11
|
+
critical_columns: tuple[str, ...]
|
|
12
|
+
duplicate_column: str | None
|
|
13
|
+
empty_warning_rate: float
|
|
14
|
+
empty_fail_rate: float
|
|
15
|
+
duplicate_warning_rate: float
|
|
16
|
+
duplicate_fail_rate: float
|
|
17
|
+
suspicious_column: str | None = None
|
|
18
|
+
suspicious_patterns: tuple[str, ...] = ()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
OUTREACH_SUSPICIOUS_PATTERNS = (
|
|
22
|
+
r"^.{1,2}$",
|
|
23
|
+
r"^.{50,}$",
|
|
24
|
+
r"^\d+$",
|
|
25
|
+
r"^(the|this|that|these|those)$",
|
|
26
|
+
r"^(however|although|because|before|between)$",
|
|
27
|
+
r"^(january|february|march|april|may|june|july|august|september|october|november|december)$",
|
|
28
|
+
r"^(north|south|east|west|united|states|article|section|chapter)$",
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
PROFILES: dict[str, Profile] = {
|
|
32
|
+
"generic": Profile(
|
|
33
|
+
name="generic",
|
|
34
|
+
required_columns=("company",),
|
|
35
|
+
critical_columns=("company",),
|
|
36
|
+
duplicate_column="company",
|
|
37
|
+
empty_warning_rate=0.10,
|
|
38
|
+
empty_fail_rate=0.30,
|
|
39
|
+
duplicate_warning_rate=0.10,
|
|
40
|
+
duplicate_fail_rate=0.25,
|
|
41
|
+
),
|
|
42
|
+
"outreach": Profile(
|
|
43
|
+
name="outreach",
|
|
44
|
+
required_columns=("company", "person_name"),
|
|
45
|
+
critical_columns=("company", "person_name"),
|
|
46
|
+
duplicate_column="company",
|
|
47
|
+
empty_warning_rate=0.30,
|
|
48
|
+
empty_fail_rate=0.70,
|
|
49
|
+
duplicate_warning_rate=0.10,
|
|
50
|
+
duplicate_fail_rate=0.25,
|
|
51
|
+
suspicious_column="company",
|
|
52
|
+
suspicious_patterns=OUTREACH_SUSPICIOUS_PATTERNS,
|
|
53
|
+
),
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def get_profile(name: str) -> Profile:
|
|
58
|
+
try:
|
|
59
|
+
return PROFILES[name]
|
|
60
|
+
except KeyError as exc:
|
|
61
|
+
raise ValueError(f"unknown profile: {name}") from exc
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def compile_patterns(patterns: tuple[str, ...]) -> list[re.Pattern[str]]:
|
|
65
|
+
return [re.compile(pattern, re.IGNORECASE) for pattern in patterns]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
from .models import GateResult, Status
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def to_text(result: GateResult) -> str:
|
|
9
|
+
lines = [
|
|
10
|
+
f"csv-quality-gate: {result.status.value.upper()}",
|
|
11
|
+
f"file: {result.path}",
|
|
12
|
+
f"profile: {result.profile}",
|
|
13
|
+
f"rows: {result.row_count}",
|
|
14
|
+
]
|
|
15
|
+
for issue in result.issues:
|
|
16
|
+
lines.append(f" {issue.severity.value.upper()}: {issue.message}")
|
|
17
|
+
return "\n".join(lines)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def to_json(result: GateResult) -> str:
|
|
21
|
+
return json.dumps(
|
|
22
|
+
{
|
|
23
|
+
"path": result.path,
|
|
24
|
+
"profile": result.profile,
|
|
25
|
+
"rows": result.row_count,
|
|
26
|
+
"status": result.status.value,
|
|
27
|
+
"issues": [
|
|
28
|
+
{"severity": issue.severity.value, "message": issue.message}
|
|
29
|
+
for issue in result.issues
|
|
30
|
+
],
|
|
31
|
+
},
|
|
32
|
+
indent=2,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def exit_code(result: GateResult) -> int:
|
|
37
|
+
return {
|
|
38
|
+
Status.PASS: 0,
|
|
39
|
+
Status.WARN: 1,
|
|
40
|
+
Status.FAIL: 2,
|
|
41
|
+
}[result.status]
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from .models import GateResult, Issue, Severity, Status
|
|
7
|
+
from .profiles import compile_patterns, get_profile
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def validate_csv(path: Path, profile_name: str = "generic") -> GateResult:
|
|
11
|
+
profile = get_profile(profile_name)
|
|
12
|
+
issues: list[Issue] = []
|
|
13
|
+
|
|
14
|
+
with path.open(newline="", encoding="utf-8-sig") as handle:
|
|
15
|
+
rows = list(csv.DictReader(handle))
|
|
16
|
+
headers = tuple((field or "").strip() for field in (rows[0].keys() if rows else ()))
|
|
17
|
+
|
|
18
|
+
missing = [column for column in profile.required_columns if column not in headers]
|
|
19
|
+
for column in missing:
|
|
20
|
+
issues.append(Issue(Severity.ERROR, f"missing required column: {column}"))
|
|
21
|
+
|
|
22
|
+
if not rows:
|
|
23
|
+
issues.append(Issue(Severity.ERROR, "csv has no data rows"))
|
|
24
|
+
return GateResult(str(path), profile.name, 0, issues, Status.FAIL)
|
|
25
|
+
|
|
26
|
+
for column in profile.critical_columns:
|
|
27
|
+
if column not in headers:
|
|
28
|
+
continue
|
|
29
|
+
empty_count = sum(1 for row in rows if not (row.get(column) or "").strip())
|
|
30
|
+
empty_rate = empty_count / len(rows)
|
|
31
|
+
if empty_rate >= profile.empty_fail_rate:
|
|
32
|
+
issues.append(
|
|
33
|
+
Issue(
|
|
34
|
+
Severity.ERROR,
|
|
35
|
+
f"empty rate for {column} is {empty_rate:.0%}, "
|
|
36
|
+
f"exceeds fail threshold {profile.empty_fail_rate:.0%}",
|
|
37
|
+
)
|
|
38
|
+
)
|
|
39
|
+
elif empty_rate >= profile.empty_warning_rate:
|
|
40
|
+
issues.append(
|
|
41
|
+
Issue(
|
|
42
|
+
Severity.WARNING,
|
|
43
|
+
f"empty rate for {column} is {empty_rate:.0%}, "
|
|
44
|
+
f"exceeds warning threshold {profile.empty_warning_rate:.0%}",
|
|
45
|
+
)
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
if profile.duplicate_column and profile.duplicate_column in headers:
|
|
49
|
+
values = [
|
|
50
|
+
(row.get(profile.duplicate_column) or "").strip().casefold()
|
|
51
|
+
for row in rows
|
|
52
|
+
if (row.get(profile.duplicate_column) or "").strip()
|
|
53
|
+
]
|
|
54
|
+
duplicate_rate = (len(values) - len(set(values))) / len(rows)
|
|
55
|
+
if duplicate_rate >= profile.duplicate_fail_rate:
|
|
56
|
+
issues.append(
|
|
57
|
+
Issue(
|
|
58
|
+
Severity.ERROR,
|
|
59
|
+
f"duplicate rate {duplicate_rate:.0%} exceeds fail threshold "
|
|
60
|
+
f"{profile.duplicate_fail_rate:.0%}",
|
|
61
|
+
)
|
|
62
|
+
)
|
|
63
|
+
elif duplicate_rate >= profile.duplicate_warning_rate:
|
|
64
|
+
issues.append(
|
|
65
|
+
Issue(
|
|
66
|
+
Severity.WARNING,
|
|
67
|
+
f"duplicate rate {duplicate_rate:.0%} exceeds warning threshold "
|
|
68
|
+
f"{profile.duplicate_warning_rate:.0%}",
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
if profile.suspicious_column and profile.suspicious_column in headers:
|
|
73
|
+
patterns = compile_patterns(profile.suspicious_patterns)
|
|
74
|
+
suspicious = []
|
|
75
|
+
for row in rows:
|
|
76
|
+
value = (row.get(profile.suspicious_column) or "").strip()
|
|
77
|
+
if any(pattern.search(value) for pattern in patterns):
|
|
78
|
+
suspicious.append(value)
|
|
79
|
+
suspicious_rate = len(suspicious) / len(rows)
|
|
80
|
+
if suspicious_rate >= 0.30:
|
|
81
|
+
issues.append(
|
|
82
|
+
Issue(
|
|
83
|
+
Severity.ERROR,
|
|
84
|
+
f"suspicious {profile.suspicious_column} rate is {suspicious_rate:.0%}",
|
|
85
|
+
)
|
|
86
|
+
)
|
|
87
|
+
elif suspicious_rate >= 0.10:
|
|
88
|
+
issues.append(
|
|
89
|
+
Issue(
|
|
90
|
+
Severity.WARNING,
|
|
91
|
+
f"suspicious {profile.suspicious_column} rate is {suspicious_rate:.0%}",
|
|
92
|
+
)
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
status = _status_for_issues(issues)
|
|
96
|
+
return GateResult(str(path), profile.name, len(rows), issues, status)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _status_for_issues(issues: list[Issue]) -> Status:
|
|
100
|
+
if any(issue.severity is Severity.ERROR for issue in issues):
|
|
101
|
+
return Status.FAIL
|
|
102
|
+
if issues:
|
|
103
|
+
return Status.WARN
|
|
104
|
+
return Status.PASS
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: csv-quality-gate
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CSV preflight validation and batch CSV quality checks that fail fast before pipeline runs.
|
|
5
|
+
Author: Rolando Bosch
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: csv preflight validation,batch CSV quality checks,fail fast,pipeline validation,csv lint
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
12
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
18
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# csv-quality-gate: CSV preflight validation before pipeline runs
|
|
22
|
+
|
|
23
|
+
Fail fast before pipeline runs when the input CSV is broken, incomplete, duplicated, or obviously junk.
|
|
24
|
+
|
|
25
|
+
`csv-quality-gate` runs batch CSV quality checks and returns `pass`, `warn`, or `fail` before expensive pipeline steps burn time on bad input.
|
|
26
|
+
|
|
27
|
+
- "We keep running expensive pipeline steps on broken CSVs."
|
|
28
|
+
- "A batch run fails 20 minutes in because the input CSV was junk."
|
|
29
|
+
- "We only discover missing required columns after the job already started."
|
|
30
|
+
- "Duplicate rows and empty contact fields keep polluting our batch runs."
|
|
31
|
+
- "I want CSV preflight validation, not a whole data platform."
|
|
32
|
+
|
|
33
|
+
Fastest install:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install csv-quality-gate
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Fastest real usage:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
csv-quality-gate check leads.csv --profile outreach
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Exact outcome:
|
|
46
|
+
|
|
47
|
+
```text
|
|
48
|
+
csv-quality-gate: FAIL
|
|
49
|
+
file: leads.csv
|
|
50
|
+
profile: outreach
|
|
51
|
+
rows: 125
|
|
52
|
+
ERROR: missing required column: person_name
|
|
53
|
+
WARNING: duplicate rate 12% exceeds warning threshold 10%
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+

|
|
57
|
+
|
|
58
|
+
It is designed for narrow, honest use as a preflight gate, not as a full data quality platform.
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install csv-quality-gate
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
For development:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install -e ".[dev]"
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Common search-intent use cases
|
|
73
|
+
|
|
74
|
+
- CSV preflight validation
|
|
75
|
+
- batch CSV quality checks
|
|
76
|
+
- fail fast before pipeline runs
|
|
77
|
+
- CSV validation before ETL or enrichment
|
|
78
|
+
- detect junk CSV rows before batch jobs
|
|
79
|
+
|
|
80
|
+
## Usage
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
csv-quality-gate check leads.csv
|
|
84
|
+
csv-quality-gate check leads.csv --profile outreach
|
|
85
|
+
csv-quality-gate check leads.csv --profile generic --json
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Exit codes:
|
|
89
|
+
|
|
90
|
+
- `0` pass
|
|
91
|
+
- `1` warnings only
|
|
92
|
+
- `2` fail
|
|
93
|
+
|
|
94
|
+
## Profiles
|
|
95
|
+
|
|
96
|
+
Built-in profiles:
|
|
97
|
+
|
|
98
|
+
- `generic`
|
|
99
|
+
- validates required columns, empties, duplicates, empty file
|
|
100
|
+
- `outreach`
|
|
101
|
+
- adds suspicious company-name heuristics for GTM/contact pipelines
|
|
102
|
+
|
|
103
|
+
## Output
|
|
104
|
+
|
|
105
|
+
```text
|
|
106
|
+
csv-quality-gate: FAIL
|
|
107
|
+
file: leads.csv
|
|
108
|
+
profile: outreach
|
|
109
|
+
rows: 125
|
|
110
|
+
ERROR: missing required column: person_name
|
|
111
|
+
WARNING: duplicate rate 12% exceeds warning threshold 10%
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## JSON mode
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
csv-quality-gate check leads.csv --json
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## Limitations
|
|
121
|
+
|
|
122
|
+
- Heuristics are intentionally simple.
|
|
123
|
+
- The `outreach` profile is opinionated and should not be treated as universal truth.
|
|
124
|
+
- The tool validates shape and obvious noise, not semantic correctness.
|
|
125
|
+
|
|
126
|
+
## When To Use It
|
|
127
|
+
|
|
128
|
+
- Before enrichment, outreach, ETL, or batch scoring runs
|
|
129
|
+
- In CI for checked-in CSV inputs
|
|
130
|
+
- As a preflight gate before expensive pipeline work
|
|
131
|
+
|
|
132
|
+
## When Not To Use It
|
|
133
|
+
|
|
134
|
+
- When you need semantic validation of the data itself
|
|
135
|
+
- When your input is not CSV
|
|
136
|
+
- When you need a full data quality framework with lineage and profiling
|
|
137
|
+
|
|
138
|
+
## More From Hermes Labs
|
|
139
|
+
|
|
140
|
+
- [intent-verify](https://github.com/roli-lpci/intent-verify): repo intent verification and spec drift checks
|
|
141
|
+
|
|
142
|
+
## Development
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
ruff check .
|
|
146
|
+
python3 -m pytest -q
|
|
147
|
+
python3 -m py_compile src/csv_quality_gate/*.py
|
|
148
|
+
```
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/csv_quality_gate/__init__.py
|
|
5
|
+
src/csv_quality_gate/cli.py
|
|
6
|
+
src/csv_quality_gate/models.py
|
|
7
|
+
src/csv_quality_gate/profiles.py
|
|
8
|
+
src/csv_quality_gate/report.py
|
|
9
|
+
src/csv_quality_gate/validator.py
|
|
10
|
+
src/csv_quality_gate.egg-info/PKG-INFO
|
|
11
|
+
src/csv_quality_gate.egg-info/SOURCES.txt
|
|
12
|
+
src/csv_quality_gate.egg-info/dependency_links.txt
|
|
13
|
+
src/csv_quality_gate.egg-info/entry_points.txt
|
|
14
|
+
src/csv_quality_gate.egg-info/requires.txt
|
|
15
|
+
src/csv_quality_gate.egg-info/top_level.txt
|
|
16
|
+
tests/test_cli.py
|
|
17
|
+
tests/test_validator.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
csv_quality_gate
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import subprocess
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
7
|
+
FIXTURES = ROOT / "tests" / "fixtures"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def run_cli(*args: str) -> subprocess.CompletedProcess[str]:
|
|
11
|
+
return subprocess.run(
|
|
12
|
+
[sys.executable, "-m", "csv_quality_gate.cli", *args],
|
|
13
|
+
cwd=ROOT,
|
|
14
|
+
env={"PYTHONPATH": str(ROOT / "src")},
|
|
15
|
+
capture_output=True,
|
|
16
|
+
text=True,
|
|
17
|
+
check=False,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_cli_json_warn():
|
|
22
|
+
result = run_cli("check", str(FIXTURES / "duplicates.csv"), "--json")
|
|
23
|
+
assert result.returncode == 1
|
|
24
|
+
payload = json.loads(result.stdout)
|
|
25
|
+
assert payload["status"] == "warn"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_cli_fail():
|
|
29
|
+
result = run_cli("check", str(FIXTURES / "missing_people.csv"), "--profile", "outreach")
|
|
30
|
+
assert result.returncode == 2
|
|
31
|
+
assert "FAIL" in result.stdout
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from csv_quality_gate.models import Status
|
|
4
|
+
from csv_quality_gate.validator import validate_csv
|
|
5
|
+
|
|
6
|
+
FIXTURES = Path(__file__).parent / "fixtures"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_generic_pass():
|
|
10
|
+
result = validate_csv(FIXTURES / "clean.csv")
|
|
11
|
+
assert result.status is Status.PASS
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_generic_warn_duplicate_rate():
|
|
15
|
+
result = validate_csv(FIXTURES / "duplicates.csv")
|
|
16
|
+
assert result.status is Status.WARN
|
|
17
|
+
assert any("duplicate rate" in issue.message for issue in result.issues)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_outreach_fail_missing_people():
|
|
21
|
+
result = validate_csv(FIXTURES / "missing_people.csv", profile_name="outreach")
|
|
22
|
+
assert result.status is Status.FAIL
|
|
23
|
+
assert any("empty rate for person_name" in issue.message for issue in result.issues)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_outreach_fail_suspicious_company_values():
|
|
27
|
+
result = validate_csv(FIXTURES / "junk_companies.csv", profile_name="outreach")
|
|
28
|
+
assert result.status is Status.FAIL
|
|
29
|
+
assert any("suspicious company rate" in issue.message for issue in result.issues)
|