datacheck-cli 2.1.2__tar.gz → 2.1.4__tar.gz

datacheck_cli-2.1.4/PKG-INFO

@@ -0,0 +1,205 @@
+Metadata-Version: 2.4
+Name: datacheck-cli
+Version: 2.1.4
+Summary: A linter for data pipelines. Enforce deterministic validation rules in CI/CD, Airflow, and beyond.
+License: Apache-2.0
+License-File: LICENSE
+Keywords: data-validation,data-linter,cli,data-engineering,pipeline,ci-cd,yaml,testing,csv,parquet,postgres,data-testing,great-expectations-alternative,soda-alternative,dbt-testing,data-contracts,airflow,snowflake,bigquery,redshift,schema-contracts,schema-validation,data-pipeline,etl-testing
+Author: Squrtech
+Author-email: contact@squrtech.com
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database :: Database Engines/Servers
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Quality Assurance
+Provides-Extra: all
+Provides-Extra: bigquery
+Provides-Extra: cloud
+Provides-Extra: databases
+Provides-Extra: mysql
+Provides-Extra: postgres
+Provides-Extra: postgresql
+Provides-Extra: redshift
+Provides-Extra: s3
+Provides-Extra: snowflake
+Provides-Extra: statistical
+Provides-Extra: validation
+Provides-Extra: warehouses
+Requires-Dist: boto3 (>=1.34.0,<2.0.0) ; extra == "s3" or extra == "cloud" or extra == "redshift" or extra == "warehouses" or extra == "all"
+Requires-Dist: click (>=8.1.0,<9.0.0)
+Requires-Dist: duckdb (>=1.0.0,<2.0.0)
+Requires-Dist: google-auth (>=2.0.0,<3.0.0) ; extra == "bigquery" or extra == "warehouses" or extra == "all"
+Requires-Dist: google-cloud-bigquery (>=3.0.0,<4.0.0) ; extra == "bigquery" or extra == "warehouses" or extra == "all"
+Requires-Dist: jsonschema (>=4.17.0,<5.0.0) ; extra == "validation" or extra == "all"
+Requires-Dist: mysql-connector-python (>=8.2.0,<10.0.0) ; extra == "mysql" or extra == "databases" or extra == "all"
+Requires-Dist: numpy (>=1.24.0,<3.0.0)
+Requires-Dist: pandas (>=2.0.0,<3.0.0)
+Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0) ; extra == "postgresql" or extra == "postgres" or extra == "databases" or extra == "redshift" or extra == "warehouses" or extra == "all"
+Requires-Dist: pyarrow (>=14.0.0,<24.0.0)
+Requires-Dist: pyyaml (>=6.0,<7.0)
+Requires-Dist: rich (>=13,<15)
+Requires-Dist: scipy (>=1.11.0,<2.0.0) ; (python_version >= "3.11") and (extra == "statistical" or extra == "all")
+Requires-Dist: snowflake-connector-python (>=3.0.0,<4.0.0) ; extra == "snowflake" or extra == "warehouses" or extra == "all"
+Requires-Dist: sqlalchemy (>=2.0.23,<3.0.0) ; extra == "postgresql" or extra == "postgres" or extra == "mysql" or extra == "databases" or extra == "redshift" or extra == "warehouses" or extra == "all"
+Requires-Dist: typer (>=0.12,<1.0.0)
+Project-URL: Homepage, https://github.com/squrtech/datacheck
+Project-URL: Repository, https://github.com/squrtech/datacheck
+Description-Content-Type: text/markdown
+
+# DataCheck — Data Validation Made Easy
+
+[![PyPI version](https://img.shields.io/pypi/v/datacheck-cli.svg)](https://pypi.org/project/datacheck-cli/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+**DataCheck is a dataset validation tool.** Define rules in YAML, point it at your data, and it fails fast if anything is wrong — in CI or locally.
+
+---
+
+## Install
+
+```bash
+pip install datacheck-cli
+```
+
+For database connectivity, install the extras you need:
+
+```bash
+pip install datacheck-cli[postgresql]   # PostgreSQL
+pip install datacheck-cli[mysql]        # MySQL
+pip install datacheck-cli[snowflake]    # Snowflake
+pip install datacheck-cli[bigquery]     # BigQuery
+pip install datacheck-cli[redshift]     # Redshift
+pip install datacheck-cli[s3]           # S3 (CSV/Parquet)
+pip install datacheck-cli[all]          # Everything
+```
+
+---
+
+## Write a config
+
+```bash
+datacheck config init
+```
+
+This creates `datacheck.yaml`. Edit it to define your validation rules:
+
+```yaml
+# datacheck.yaml
+version: "1.0"
+sources_file: sources.yaml
+source: my_data
+
+checks:
+  - column: id
+    rules:
+      - not_null
+      - unique
+
+  - column: email
+    rules:
+      - not_null
+      - regex: "^[^@]+@[^@]+$"
+
+  - column: amount
+    rules:
+      - type: numeric
+      - positive
+```
+
+---
+
+## Add sources
+
+Create `sources.yaml` to define where your data lives:
+
+```yaml
+# sources.yaml
+version: "1.0"
+
+sources:
+  # Local CSV or Parquet file
+  my_data:
+    type: duckdb
+    path: ./data/customers.csv
+
+  # PostgreSQL
+  # my_data:
+  #   type: postgresql
+  #   host: ${PG_HOST}
+  #   database: ${PG_DATABASE}
+  #   user: ${PG_USER}
+  #   password: ${PG_PASSWORD}
+
+  # Snowflake
+  # my_data:
+  #   type: snowflake
+  #   account: ${SF_ACCOUNT}
+  #   user: ${SF_USER}
+  #   password: ${SF_PASSWORD}
+  #   warehouse: ${SF_WAREHOUSE}
+  #   database: ${SF_DATABASE}
+  #   schema: ${SF_SCHEMA}
+```
+
+Supported sources: **CSV/Parquet** (via DuckDB), **PostgreSQL**, **MySQL**, **Snowflake**, **BigQuery**, **Redshift**.
+
+Credentials use environment variables — `sources.yaml` never needs secrets hardcoded.
+
+---
+
+## Validate
+
+```bash
+datacheck validate
+```
+
+DataCheck runs all checks against your source and exits `0` on pass, `1` on failure.
+
+```
+✅  id         not_null    passed   10,000 rows
+✅  id         unique      passed   10,000 rows
+❌  email      regex       FAILED   142/10,000 rows (1.4%)
+✅  amount     type        passed   10,000 rows
+✅  amount     positive    passed   10,000 rows
+```
+
+For database sources validation runs as a single aggregate SQL query — no data is pulled out of your warehouse.
+
+---
+
+## Rules reference
+
+| Category  | Rules                                                                 |
+| :-------- | :-------------------------------------------------------------------- |
+| Presence  | `not_null`, `unique`                                                  |
+| Type      | `type: integer`, `type: numeric`, `type: string`, `type: date`        |
+| Numeric   | `positive`, `range: {min, max}`                                       |
+| String    | `regex`, `allowed_values`, `min_length`, `max_length`                 |
+| Boolean   | `boolean`                                                             |
+| Temporal  | `no_future_timestamps`, `date_range: {min, max}`                      |
+
+---
+
+## Documentation
+
+[squrtech.github.io/datacheck](https://squrtech.github.io/datacheck/)
+
+---
+
+## License
+
+Apache 2.0
+

datacheck_cli-2.1.4/README_PYPI.md

@@ -0,0 +1,144 @@
+# DataCheck — Data Validation Made Easy
+
+[![PyPI version](https://img.shields.io/pypi/v/datacheck-cli.svg)](https://pypi.org/project/datacheck-cli/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+**DataCheck is a dataset validation tool.** Define rules in YAML, point it at your data, and it fails fast if anything is wrong — in CI or locally.
+
+---
+
+## Install
+
+```bash
+pip install datacheck-cli
+```
+
+For database connectivity, install the extras you need:
+
+```bash
+pip install datacheck-cli[postgresql]   # PostgreSQL
+pip install datacheck-cli[mysql]        # MySQL
+pip install datacheck-cli[snowflake]    # Snowflake
+pip install datacheck-cli[bigquery]     # BigQuery
+pip install datacheck-cli[redshift]     # Redshift
+pip install datacheck-cli[s3]           # S3 (CSV/Parquet)
+pip install datacheck-cli[all]          # Everything
+```
+
+---
+
+## Write a config
+
+```bash
+datacheck config init
+```
+
+This creates `datacheck.yaml`. Edit it to define your validation rules:
+
+```yaml
+# datacheck.yaml
+version: "1.0"
+sources_file: sources.yaml
+source: my_data
+
+checks:
+  - column: id
+    rules:
+      - not_null
+      - unique
+
+  - column: email
+    rules:
+      - not_null
+      - regex: "^[^@]+@[^@]+$"
+
+  - column: amount
+    rules:
+      - type: numeric
+      - positive
+```
+
+---
+
+## Add sources
+
+Create `sources.yaml` to define where your data lives:
+
+```yaml
+# sources.yaml
+version: "1.0"
+
+sources:
+  # Local CSV or Parquet file
+  my_data:
+    type: duckdb
+    path: ./data/customers.csv
+
+  # PostgreSQL
+  # my_data:
+  #   type: postgresql
+  #   host: ${PG_HOST}
+  #   database: ${PG_DATABASE}
+  #   user: ${PG_USER}
+  #   password: ${PG_PASSWORD}
+
+  # Snowflake
+  # my_data:
+  #   type: snowflake
+  #   account: ${SF_ACCOUNT}
+  #   user: ${SF_USER}
+  #   password: ${SF_PASSWORD}
+  #   warehouse: ${SF_WAREHOUSE}
+  #   database: ${SF_DATABASE}
+  #   schema: ${SF_SCHEMA}
+```
+
+Supported sources: **CSV/Parquet** (via DuckDB), **PostgreSQL**, **MySQL**, **Snowflake**, **BigQuery**, **Redshift**.
+
+Credentials use environment variables — `sources.yaml` never needs secrets hardcoded.
+
+---
+
+## Validate
+
+```bash
+datacheck validate
+```
+
+DataCheck runs all checks against your source and exits `0` on pass, `1` on failure.
+
+```
+✅  id         not_null    passed   10,000 rows
+✅  id         unique      passed   10,000 rows
+❌  email      regex       FAILED   142/10,000 rows (1.4%)
+✅  amount     type        passed   10,000 rows
+✅  amount     positive    passed   10,000 rows
+```
+
+For database sources validation runs as a single aggregate SQL query — no data is pulled out of your warehouse.
+
+---
+
+## Rules reference
+
+| Category  | Rules                                                                 |
+| :-------- | :-------------------------------------------------------------------- |
+| Presence  | `not_null`, `unique`                                                  |
+| Type      | `type: integer`, `type: numeric`, `type: string`, `type: date`        |
+| Numeric   | `positive`, `range: {min, max}`                                       |
+| String    | `regex`, `allowed_values`, `min_length`, `max_length`                 |
+| Boolean   | `boolean`                                                             |
+| Temporal  | `no_future_timestamps`, `date_range: {min, max}`                      |
+
+---
+
+## Documentation
+
+[squrtech.github.io/datacheck](https://squrtech.github.io/datacheck/)
+
+---
+
+## License
+
+Apache 2.0

{datacheck_cli-2.1.2 → datacheck_cli-2.1.4}/datacheck/__init__.py

@@ -25,7 +25,7 @@ from datacheck.schema import (
     SchemaDetector,
 )
 
-__version__ = "2.1.1"
+__version__ = "2.1.4"
 __author__ = "Squrtech"
 __email__ = "contact@squrtech.com"
 

{datacheck_cli-2.1.2 → datacheck_cli-2.1.4}/datacheck/airflow/operators.py

@@ -8,11 +8,9 @@ Provides two operators for enforcing validation rules in Airflow DAGs:
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import Any
 from collections.abc import Sequence
 
-if TYPE_CHECKING:
-    import pandas as pd
 
 try:
     from airflow.models import BaseOperator
@@ -77,7 +75,6 @@ class DataCheckOperator(BaseOperator):
                 config_path="/config/checks.yaml",
                 sources_file="/config/sources.yaml",
                 source_name="orders",
-                where="created_at >= '{{ ds }}'",
             )
 
         With quality thresholds::
@@ -96,7 +93,6 @@ class DataCheckOperator(BaseOperator):
         sources_file: Path to named sources YAML file
         source_name: Named source to validate
         table: Database table name override
-        where: SQL WHERE clause for filtering
         query: Custom SQL query (alternative to table)
         min_pass_rate: Minimum rule pass rate to succeed (0-100)
         fail_on_error: Whether to fail the Airflow task on validation failure
@@ -108,7 +104,6 @@ class DataCheckOperator(BaseOperator):
         "sources_file",
         "source_name",
         "table",
-        "where",
         "query",
     )
     template_ext: Sequence[str] = (".yaml", ".yml")
@@ -122,7 +117,6 @@ class DataCheckOperator(BaseOperator):
         sources_file: str | None = None,
         source_name: str | None = None,
         table: str | None = None,
-        where: str | None = None,
         query: str | None = None,
         min_pass_rate: float = 0.0,
         fail_on_error: bool = True,
@@ -136,7 +130,6 @@ class DataCheckOperator(BaseOperator):
             sources_file: Path to sources YAML file (overrides config)
             source_name: Named source from sources.yaml
             table: Database table name (for database sources)
-            where: WHERE clause for filtering (for database sources)
             query: Custom SQL query (alternative to table)
             min_pass_rate: Minimum pass rate percentage (0-100, 0 = disabled)
             fail_on_error: Whether to raise AirflowException on failure
@@ -148,7 +141,6 @@ class DataCheckOperator(BaseOperator):
         self.sources_file = sources_file
         self.source_name = source_name
         self.table = table
-        self.where = where
         self.query = query
         self.min_pass_rate = min_pass_rate
         self.fail_on_error = fail_on_error
@@ -188,7 +180,6 @@ class DataCheckOperator(BaseOperator):
                 summary = engine.validate_sources(
                     source_name=self.source_name,
                     table=self.table,
-                    where=self.where,
                     query=self.query,
                 )
             else:
@@ -294,7 +285,7 @@ class DataCheckSchemaOperator(BaseOperator):
             )
 
     Attributes:
-        file_path: Path to a data file
+        file_path: Path to a data file (CSV, Parquet) — resolved via DuckDB
         sources_file: Path to named sources YAML file
         source_name: Named source to check
         table: Database table name
@@ -356,24 +347,26 @@ class DataCheckSchemaOperator(BaseOperator):
         self.fail_on_breaking = fail_on_breaking
         self.push_results = push_results
 
-    def _load_data(self) -> pd.DataFrame:
-        """Load data from file or named source.
+    def _resolve_source_config(self):
+        """Resolve SourceConfig from operator parameters.
 
         Returns:
-            DataFrame loaded from the configured data source
+            SourceConfig for the data source
 
         Raises:
-            AirflowException: If no data source is configured or loading fails
+            AirflowException: If no data source is configured or source not found
         """
-
         if self.file_path:
-            from datacheck.loader import LoaderFactory
+            from datacheck.config.source import SourceConfig
 
-            return LoaderFactory.load(self.file_path)
+            return SourceConfig(
+                name=str(self.file_path),
+                type="duckdb",
+                connection={"path": str(self.file_path)},
+            )
 
         if self.source_name and self.sources_file:
             from datacheck.config.source import load_sources
-            from datacheck.connectors.factory import load_source_data
 
             sources = load_sources(self.sources_file)
             if self.source_name not in sources:
@@ -381,7 +374,7 @@ class DataCheckSchemaOperator(BaseOperator):
                     f"Source '{self.source_name}' not found. "
                     f"Available: {', '.join(sorted(sources.keys()))}"
                 )
-            return load_source_data(sources[self.source_name], table=self.table, query=self.query)
+            return sources[self.source_name]
 
         raise AirflowException(
             "No data source specified. Provide file_path, "
@@ -394,6 +387,9 @@ class DataCheckSchemaOperator(BaseOperator):
         If a baseline exists, compares the current schema against it
         and reports changes. If no baseline exists, captures one.
 
+        Schema detection uses metadata-only queries (SUMMARIZE for DuckDB/S3,
+        information_schema for SQL databases) — no row data is transferred.
+
         Args:
             context: Airflow context dictionary
 
@@ -410,20 +406,26 @@ class DataCheckSchemaOperator(BaseOperator):
             f"source={self.source_name}, baseline={self.baseline_name}"
         )
 
-        # Load data
+        # Resolve source config (no data loaded yet)
         try:
-            df = self._load_data()
+            source_config = self._resolve_source_config()
         except AirflowException:
             raise
         except Exception as e:
-            raise AirflowException(f"Failed to load data: {e}")
+            raise AirflowException(f"Failed to resolve data source: {e}")
 
-        # Detect current schema
+        # Detect current schema via metadata-only queries
         detector = SchemaDetector()
-        source_label = self.file_path or self.source_name or "unknown"
-        current_schema = detector.detect(
-            df, name=self.baseline_name, source=source_label
-        )
+        try:
+            current_schema = detector.detect_from_source(
+                source_config,
+                name=self.baseline_name,
+                source_identifier=self.table or self.file_path or self.source_name,
+                table=self.table,
+                query=self.query,
+            )
+        except Exception as e:
+            raise AirflowException(f"Failed to detect schema: {e}")
 
         # Manage baseline
         manager = BaselineManager(baseline_dir=self.baseline_dir)