csv-quality-gate 0.1.0__py3-none-any.whl

csv_quality_gate/__init__.py

@@ -0,0 +1,4 @@
+from .models import GateResult, Issue, Severity, Status
+from .validator import validate_csv
+
+__all__ = ["GateResult", "Issue", "Severity", "Status", "validate_csv"]

csv_quality_gate/cli.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .profiles import PROFILES
+from .report import exit_code, to_json, to_text
+from .validator import validate_csv
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="csv-quality-gate",
+        description="Run CSV preflight validation and fail fast before expensive pipeline runs.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    check = subparsers.add_parser("check", help="run batch CSV quality checks on one file")
+    check.add_argument("csv_path", help="path to the csv file to validate")
+    check.add_argument(
+        "--profile",
+        default="generic",
+        choices=sorted(PROFILES),
+        help="validation profile, for example generic or outreach",
+    )
+    check.add_argument("--json", action="store_true", help="emit machine-readable JSON output")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    path = Path(args.csv_path)
+    if not path.exists():
+        print(f"csv-quality-gate: file not found: {path}", file=sys.stderr)
+        return 2
+    result = validate_csv(path, profile_name=args.profile)
+    print(to_json(result) if args.json else to_text(result))
+    return exit_code(result)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

csv_quality_gate/models.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class Severity(str, Enum):
+    WARNING = "warning"
+    ERROR = "error"
+
+
+class Status(str, Enum):
+    PASS = "pass"
+    WARN = "warn"
+    FAIL = "fail"
+
+
+@dataclass(frozen=True)
+class Issue:
+    severity: Severity
+    message: str
+
+
+@dataclass(frozen=True)
+class GateResult:
+    path: str
+    profile: str
+    row_count: int
+    issues: list[Issue]
+    status: Status

csv_quality_gate/profiles.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Profile:
+    name: str
+    required_columns: tuple[str, ...]
+    critical_columns: tuple[str, ...]
+    duplicate_column: str | None
+    empty_warning_rate: float
+    empty_fail_rate: float
+    duplicate_warning_rate: float
+    duplicate_fail_rate: float
+    suspicious_column: str | None = None
+    suspicious_patterns: tuple[str, ...] = ()
+
+
+OUTREACH_SUSPICIOUS_PATTERNS = (
+    r"^.{1,2}$",
+    r"^.{50,}$",
+    r"^\d+$",
+    r"^(the|this|that|these|those)$",
+    r"^(however|although|because|before|between)$",
+    r"^(january|february|march|april|may|june|july|august|september|october|november|december)$",
+    r"^(north|south|east|west|united|states|article|section|chapter)$",
+)
+
+PROFILES: dict[str, Profile] = {
+    "generic": Profile(
+        name="generic",
+        required_columns=("company",),
+        critical_columns=("company",),
+        duplicate_column="company",
+        empty_warning_rate=0.10,
+        empty_fail_rate=0.30,
+        duplicate_warning_rate=0.10,
+        duplicate_fail_rate=0.25,
+    ),
+    "outreach": Profile(
+        name="outreach",
+        required_columns=("company", "person_name"),
+        critical_columns=("company", "person_name"),
+        duplicate_column="company",
+        empty_warning_rate=0.30,
+        empty_fail_rate=0.70,
+        duplicate_warning_rate=0.10,
+        duplicate_fail_rate=0.25,
+        suspicious_column="company",
+        suspicious_patterns=OUTREACH_SUSPICIOUS_PATTERNS,
+    ),
+}
+
+
+def get_profile(name: str) -> Profile:
+    try:
+        return PROFILES[name]
+    except KeyError as exc:
+        raise ValueError(f"unknown profile: {name}") from exc
+
+
+def compile_patterns(patterns: tuple[str, ...]) -> list[re.Pattern[str]]:
+    return [re.compile(pattern, re.IGNORECASE) for pattern in patterns]

csv_quality_gate/report.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import json
+
+from .models import GateResult, Status
+
+
+def to_text(result: GateResult) -> str:
+    lines = [
+        f"csv-quality-gate: {result.status.value.upper()}",
+        f"file: {result.path}",
+        f"profile: {result.profile}",
+        f"rows: {result.row_count}",
+    ]
+    for issue in result.issues:
+        lines.append(f"  {issue.severity.value.upper()}: {issue.message}")
+    return "\n".join(lines)
+
+
+def to_json(result: GateResult) -> str:
+    return json.dumps(
+        {
+            "path": result.path,
+            "profile": result.profile,
+            "rows": result.row_count,
+            "status": result.status.value,
+            "issues": [
+                {"severity": issue.severity.value, "message": issue.message}
+                for issue in result.issues
+            ],
+        },
+        indent=2,
+    )
+
+
+def exit_code(result: GateResult) -> int:
+    return {
+        Status.PASS: 0,
+        Status.WARN: 1,
+        Status.FAIL: 2,
+    }[result.status]

csv_quality_gate/validator.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+from .models import GateResult, Issue, Severity, Status
+from .profiles import compile_patterns, get_profile
+
+
+def validate_csv(path: Path, profile_name: str = "generic") -> GateResult:
+    profile = get_profile(profile_name)
+    issues: list[Issue] = []
+
+    with path.open(newline="", encoding="utf-8-sig") as handle:
+        rows = list(csv.DictReader(handle))
+        headers = tuple((field or "").strip() for field in (rows[0].keys() if rows else ()))
+
+    missing = [column for column in profile.required_columns if column not in headers]
+    for column in missing:
+        issues.append(Issue(Severity.ERROR, f"missing required column: {column}"))
+
+    if not rows:
+        issues.append(Issue(Severity.ERROR, "csv has no data rows"))
+        return GateResult(str(path), profile.name, 0, issues, Status.FAIL)
+
+    for column in profile.critical_columns:
+        if column not in headers:
+            continue
+        empty_count = sum(1 for row in rows if not (row.get(column) or "").strip())
+        empty_rate = empty_count / len(rows)
+        if empty_rate >= profile.empty_fail_rate:
+            issues.append(
+                Issue(
+                    Severity.ERROR,
+                    f"empty rate for {column} is {empty_rate:.0%}, "
+                    f"exceeds fail threshold {profile.empty_fail_rate:.0%}",
+                )
+            )
+        elif empty_rate >= profile.empty_warning_rate:
+            issues.append(
+                Issue(
+                    Severity.WARNING,
+                    f"empty rate for {column} is {empty_rate:.0%}, "
+                    f"exceeds warning threshold {profile.empty_warning_rate:.0%}",
+                )
+            )
+
+    if profile.duplicate_column and profile.duplicate_column in headers:
+        values = [
+            (row.get(profile.duplicate_column) or "").strip().casefold()
+            for row in rows
+            if (row.get(profile.duplicate_column) or "").strip()
+        ]
+        duplicate_rate = (len(values) - len(set(values))) / len(rows)
+        if duplicate_rate >= profile.duplicate_fail_rate:
+            issues.append(
+                Issue(
+                    Severity.ERROR,
+                    f"duplicate rate {duplicate_rate:.0%} exceeds fail threshold "
+                    f"{profile.duplicate_fail_rate:.0%}",
+                )
+            )
+        elif duplicate_rate >= profile.duplicate_warning_rate:
+            issues.append(
+                Issue(
+                    Severity.WARNING,
+                    f"duplicate rate {duplicate_rate:.0%} exceeds warning threshold "
+                    f"{profile.duplicate_warning_rate:.0%}",
+                )
+            )
+
+    if profile.suspicious_column and profile.suspicious_column in headers:
+        patterns = compile_patterns(profile.suspicious_patterns)
+        suspicious = []
+        for row in rows:
+            value = (row.get(profile.suspicious_column) or "").strip()
+            if any(pattern.search(value) for pattern in patterns):
+                suspicious.append(value)
+        suspicious_rate = len(suspicious) / len(rows)
+        if suspicious_rate >= 0.30:
+            issues.append(
+                Issue(
+                    Severity.ERROR,
+                    f"suspicious {profile.suspicious_column} rate is {suspicious_rate:.0%}",
+                )
+            )
+        elif suspicious_rate >= 0.10:
+            issues.append(
+                Issue(
+                    Severity.WARNING,
+                    f"suspicious {profile.suspicious_column} rate is {suspicious_rate:.0%}",
+                )
+            )
+
+    status = _status_for_issues(issues)
+    return GateResult(str(path), profile.name, len(rows), issues, status)
+
+
+def _status_for_issues(issues: list[Issue]) -> Status:
+    if any(issue.severity is Severity.ERROR for issue in issues):
+        return Status.FAIL
+    if issues:
+        return Status.WARN
+    return Status.PASS

csv_quality_gate-0.1.0.dist-info/METADATA

@@ -0,0 +1,148 @@
+Metadata-Version: 2.4
+Name: csv-quality-gate
+Version: 0.1.0
+Summary: CSV preflight validation and batch CSV quality checks that fail fast before pipeline runs.
+Author: Rolando Bosch
+License: MIT
+Keywords: csv preflight validation,batch CSV quality checks,fail fast,pipeline validation,csv lint
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Dynamic: license-file
+
+# csv-quality-gate: CSV preflight validation before pipeline runs
+
+Fail fast before pipeline runs when the input CSV is broken, incomplete, duplicated, or obviously junk.
+
+`csv-quality-gate` runs batch CSV quality checks and returns `pass`, `warn`, or `fail` before expensive pipeline steps burn time on bad input.
+
+- "We keep running expensive pipeline steps on broken CSVs."
+- "A batch run fails 20 minutes in because the input CSV was junk."
+- "We only discover missing required columns after the job already started."
+- "Duplicate rows and empty contact fields keep polluting our batch runs."
+- "I want CSV preflight validation, not a whole data platform."
+
+Fastest install:
+
+```bash
+pip install csv-quality-gate
+```
+
+Fastest real usage:
+
+```bash
+csv-quality-gate check leads.csv --profile outreach
+```
+
+Exact outcome:
+
+```text
+csv-quality-gate: FAIL
+file: leads.csv
+profile: outreach
+rows: 125
+  ERROR: missing required column: person_name
+  WARNING: duplicate rate 12% exceeds warning threshold 10%
+```
+
+![csv-quality-gate preview](assets/preview.png)
+
+It is designed for narrow, honest use as a preflight gate, not as a full data quality platform.
+
+## Install
+
+```bash
+pip install csv-quality-gate
+```
+
+For development:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Common search-intent use cases
+
+- CSV preflight validation
+- batch CSV quality checks
+- fail fast before pipeline runs
+- CSV validation before ETL or enrichment
+- detect junk CSV rows before batch jobs
+
+## Usage
+
+```bash
+csv-quality-gate check leads.csv
+csv-quality-gate check leads.csv --profile outreach
+csv-quality-gate check leads.csv --profile generic --json
+```
+
+Exit codes:
+
+- `0` pass
+- `1` warnings only
+- `2` fail
+
+## Profiles
+
+Built-in profiles:
+
+- `generic`
+  - validates required columns, empties, duplicates, empty file
+- `outreach`
+  - adds suspicious company-name heuristics for GTM/contact pipelines
+
+## Output
+
+```text
+csv-quality-gate: FAIL
+file: leads.csv
+profile: outreach
+rows: 125
+  ERROR: missing required column: person_name
+  WARNING: duplicate rate 12% exceeds warning threshold 10%
+```
+
+## JSON mode
+
+```bash
+csv-quality-gate check leads.csv --json
+```
+
+## Limitations
+
+- Heuristics are intentionally simple.
+- The `outreach` profile is opinionated and should not be treated as universal truth.
+- The tool validates shape and obvious noise, not semantic correctness.
+
+## When To Use It
+
+- Before enrichment, outreach, ETL, or batch scoring runs
+- In CI for checked-in CSV inputs
+- As a preflight gate before expensive pipeline work
+
+## When Not To Use It
+
+- When you need semantic validation of the data itself
+- When your input is not CSV
+- When you need a full data quality framework with lineage and profiling
+
+## More From Hermes Labs
+
+- [intent-verify](https://github.com/roli-lpci/intent-verify): repo intent verification and spec drift checks
+
+## Development
+
+```bash
+ruff check .
+python3 -m pytest -q
+python3 -m py_compile src/csv_quality_gate/*.py
+```

csv_quality_gate-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+csv_quality_gate/__init__.py,sha256=Eii7WRh8OI_DYXVgHB_SbSROmvppFSSKi1Nk1yXgub0,165
+csv_quality_gate/cli.py,sha256=1IsK4-SScaCc3NFMEU4up18QsHKQhHkEx8NfcDF5Qpo,1431
+csv_quality_gate/models.py,sha256=_jO9m6J55yKfaGBFGu1xMSYh1TximawxjgmIMdXnf5g,462
+csv_quality_gate/profiles.py,sha256=EMYL3J_gKr-MRToUtSwV-ypw32uWKoVuN7idmV_rtCA,1883
+csv_quality_gate/report.py,sha256=zcye6IMSlJJYxSa4HpikgFxAHLFrYQ-bE6Y25fji_dY,1024
+csv_quality_gate/validator.py,sha256=RcYwxuigKdrBXEQz4B2JvXTEHLGIRRNrftMPuMYosXY,3996
+csv_quality_gate-0.1.0.dist-info/licenses/LICENSE,sha256=VOg1xj-037ncqAL2LUPDVkcLp4gGdk9nrYRx6YPMfvw,1070
+csv_quality_gate-0.1.0.dist-info/METADATA,sha256=LqiROao0wpXg0nvoRjIovVRA06RHfDjVLy9QM3tQBLs,3719
+csv_quality_gate-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+csv_quality_gate-0.1.0.dist-info/entry_points.txt,sha256=-ByTcV6TYrblyxw247qdjOFhdLrOSZPpXPl5xRvD1LA,63
+csv_quality_gate-0.1.0.dist-info/top_level.txt,sha256=j-y9EbI-uMDi4LrsMs9YI9JFPc_PxdNE9CUx8NZL4sw,17
+csv_quality_gate-0.1.0.dist-info/RECORD,,

csv_quality_gate-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

csv_quality_gate-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+csv-quality-gate = csv_quality_gate.cli:main

csv_quality_gate-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Rolando Bosch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

csv_quality_gate-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+csv_quality_gate