datacheck-cli 2.1.1__tar.gz → 2.1.3__tar.gz

{datacheck_cli-2.1.1 → datacheck_cli-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacheck-cli
-Version: 2.1.1
+Version: 2.1.3
 Summary: A linter for data pipelines. Enforce deterministic validation rules in CI/CD, Airflow, and beyond.
 License: Apache-2.0
 License-File: LICENSE
@@ -39,6 +39,7 @@ Provides-Extra: validation
 Provides-Extra: warehouses
 Requires-Dist: boto3 (>=1.34.0,<2.0.0) ; extra == "s3" or extra == "cloud" or extra == "redshift" or extra == "warehouses" or extra == "all"
 Requires-Dist: click (>=8.1.0,<9.0.0)
+Requires-Dist: duckdb (>=1.0.0,<2.0.0)
 Requires-Dist: google-auth (>=2.0.0,<3.0.0) ; extra == "bigquery" or extra == "warehouses" or extra == "all"
 Requires-Dist: google-cloud-bigquery (>=3.0.0,<4.0.0) ; extra == "bigquery" or extra == "warehouses" or extra == "all"
 Requires-Dist: jsonschema (>=4.17.0,<5.0.0) ; extra == "validation" or extra == "all"
@@ -111,12 +112,20 @@ datacheck config init --with-sample-data
 datacheck config init --template ecommerce --with-sample-data
 ```
 
-**Option 2 - Write manually.** Create a `.datacheck.yaml` config file with your data source and validation rules:
+**Option 2 - Write manually.** Create a `sources.yaml` and `.datacheck.yaml` with your data source and validation rules:
 
 ```yaml
-data_source:
-  type: csv
-  path: ./data/orders.csv
+# sources.yaml
+sources:
+  orders:
+    type: duckdb
+    path: ./data/orders.csv
+```
+
+```yaml
+# .datacheck.yaml
+sources_file: sources.yaml
+source: orders
 
 checks:
   - name: id_check
@@ -137,8 +146,7 @@ Run validation:
 
 ```bash
 datacheck validate                          # auto-discover config
-datacheck validate data.csv                 # direct file
-datacheck validate --config checks.yaml
+datacheck validate --config checks.yaml     # explicit config path
 echo $?  # 1 if any error-severity rule fails
 ```
 
@@ -252,7 +260,7 @@ datacheck schema compare --fail-on-breaking
 from datacheck import ValidationEngine
 
 engine = ValidationEngine(config_path=".datacheck.yaml")
-summary = engine.validate()
+summary = engine.validate_sources()
 
 print(f"Passed: {summary.passed_rules}/{summary.total_rules}")
 

{datacheck_cli-2.1.1 → datacheck_cli-2.1.3}/README_PYPI.md

@@ -52,12 +52,20 @@ datacheck config init --with-sample-data
 datacheck config init --template ecommerce --with-sample-data
 ```
 
-**Option 2 - Write manually.** Create a `.datacheck.yaml` config file with your data source and validation rules:
+**Option 2 - Write manually.** Create a `sources.yaml` and `.datacheck.yaml` with your data source and validation rules:
 
 ```yaml
-data_source:
-  type: csv
-  path: ./data/orders.csv
+# sources.yaml
+sources:
+  orders:
+    type: duckdb
+    path: ./data/orders.csv
+```
+
+```yaml
+# .datacheck.yaml
+sources_file: sources.yaml
+source: orders
 
 checks:
   - name: id_check
@@ -78,8 +86,7 @@ Run validation:
 
 ```bash
 datacheck validate                          # auto-discover config
-datacheck validate data.csv                 # direct file
-datacheck validate --config checks.yaml
+datacheck validate --config checks.yaml     # explicit config path
 echo $?  # 1 if any error-severity rule fails
 ```
 
@@ -193,7 +200,7 @@ datacheck schema compare --fail-on-breaking
 from datacheck import ValidationEngine
 
 engine = ValidationEngine(config_path=".datacheck.yaml")
-summary = engine.validate()
+summary = engine.validate_sources()
 
 print(f"Passed: {summary.passed_rules}/{summary.total_rules}")
 

{datacheck_cli-2.1.1 → datacheck_cli-2.1.3}/datacheck/__init__.py

@@ -25,7 +25,7 @@ from datacheck.schema import (
     SchemaDetector,
 )
 
-__version__ = "2.1.1"
+__version__ = "2.1.3"
 __author__ = "Squrtech"
 __email__ = "contact@squrtech.com"
 

{datacheck_cli-2.1.1 → datacheck_cli-2.1.3}/datacheck/airflow/operators.py

@@ -8,12 +8,9 @@ Provides two operators for enforcing validation rules in Airflow DAGs:
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import Any
 from collections.abc import Sequence
 
-if TYPE_CHECKING:
-    import pandas as pd
-from pathlib import Path
 
 try:
     from airflow.models import BaseOperator
@@ -63,32 +60,21 @@ class DataCheckOperator(BaseOperator):
     """Operator for running DataCheck validation in Airflow DAGs.
 
     Uses the full ValidationEngine to run config-based rules against
-    data from files or named database sources. Supports sampling,
-    parallel execution, and quality thresholds.
+    named data sources. All data sources are defined in a sources.yaml
+    file and referenced by name.
 
     Data source resolution (in order):
-        1. ``file_path`` — validate a local or cloud file
-        2. ``source_name`` — validate a named source from sources.yaml
-        3. Config default — uses ``source`` or ``data_source`` from config
+        1. ``source_name`` — validate a named source from sources.yaml
+        2. Config default — uses ``source`` defined in the config
 
     Examples:
-        Validate a file::
-
-            DataCheckOperator(
-                task_id="validate_orders",
-                config_path="/config/checks.yaml",
-                file_path="/data/orders_{{ ds }}.parquet",
-            )
-
-        Validate a database table::
+        Validate a named source::
 
             DataCheckOperator(
                 task_id="validate_orders",
                 config_path="/config/checks.yaml",
                 sources_file="/config/sources.yaml",
-                source_name="production_db",
-                table="orders",
-                where="created_at >= '{{ ds }}'",
+                source_name="orders",
             )
 
         With quality thresholds::
@@ -96,21 +82,18 @@ class DataCheckOperator(BaseOperator):
             DataCheckOperator(
                 task_id="validate_orders",
                 config_path="/config/checks.yaml",
-                file_path="/data/orders.parquet",
+                sources_file="/config/sources.yaml",
+                source_name="orders",
                 min_pass_rate=95.0,
                 fail_on_error=True,
             )
 
     Attributes:
         config_path: Path to the DataCheck validation config YAML
-        file_path: Path to a data file (CSV, Parquet)
         sources_file: Path to named sources YAML file
         source_name: Named source to validate
         table: Database table name override
-        where: SQL WHERE clause for filtering
         query: Custom SQL query (alternative to table)
-        parallel: Enable multi-core validation
-        workers: Number of worker processes
         min_pass_rate: Minimum rule pass rate to succeed (0-100)
         fail_on_error: Whether to fail the Airflow task on validation failure
         push_results: Whether to push results to XCom
@@ -118,11 +101,9 @@ class DataCheckOperator(BaseOperator):
 
     template_fields: Sequence[str] = (
         "config_path",
-        "file_path",
         "sources_file",
         "source_name",
         "table",
-        "where",
         "query",
     )
     template_ext: Sequence[str] = (".yaml", ".yml")
@@ -133,14 +114,10 @@ class DataCheckOperator(BaseOperator):
     def __init__(
         self,
         config_path: str,
-        file_path: str | None = None,
         sources_file: str | None = None,
         source_name: str | None = None,
         table: str | None = None,
-        where: str | None = None,
         query: str | None = None,
-        parallel: bool = False,
-        workers: int | None = None,
         min_pass_rate: float = 0.0,
         fail_on_error: bool = True,
         push_results: bool = True,
@@ -150,14 +127,10 @@ class DataCheckOperator(BaseOperator):
 
         Args:
             config_path: Path to DataCheck validation config YAML (required)
-            file_path: Path to data file (CSV, Parquet)
             sources_file: Path to sources YAML file (overrides config)
             source_name: Named source from sources.yaml
             table: Database table name (for database sources)
-            where: WHERE clause for filtering (for database sources)
             query: Custom SQL query (alternative to table)
-            parallel: Enable parallel execution
-            workers: Number of worker processes (default: CPU count)
             min_pass_rate: Minimum pass rate percentage (0-100, 0 = disabled)
             fail_on_error: Whether to raise AirflowException on failure
             push_results: Whether to push results to XCom
@@ -165,14 +138,10 @@ class DataCheckOperator(BaseOperator):
         """
         super().__init__(**kwargs)
         self.config_path = config_path
-        self.file_path = file_path
         self.sources_file = sources_file
         self.source_name = source_name
         self.table = table
-        self.where = where
         self.query = query
-        self.parallel = parallel
-        self.workers = workers
         self.min_pass_rate = min_pass_rate
         self.fail_on_error = fail_on_error
         self.push_results = push_results
@@ -193,7 +162,7 @@ class DataCheckOperator(BaseOperator):
 
         self.log.info(
             f"Running DataCheck validation: config={self.config_path}, "
-            f"file={self.file_path}, source={self.source_name}"
+            f"source={self.source_name}"
         )
 
         # Initialize engine
@@ -201,34 +170,22 @@ class DataCheckOperator(BaseOperator):
             engine = ValidationEngine(
                 config_path=self.config_path,
                 sources_file=self.sources_file,
-                parallel=self.parallel,
-                workers=self.workers,
             )
         except Exception as e:
             raise AirflowException(f"Failed to initialize ValidationEngine: {e}")
 
-        # Run validation based on data source
+        # Run validation
         try:
-            if self.file_path:
-                # File-based validation
-                summary = engine.validate_file(self.file_path)
-            elif self.source_name or engine.config.source:
-                # Named source validation
+            if self.source_name or engine.config.source:
                 summary = engine.validate_sources(
                     source_name=self.source_name,
                     table=self.table,
-                    where=self.where,
                     query=self.query,
                 )
-            elif engine.config.data_source is not None:
-                # Inline data_source from config
-                config_dir = Path(self.config_path).parent
-                source_path = config_dir / engine.config.data_source.path
-                summary = engine.validate_file(str(source_path))
             else:
                 raise AirflowException(
-                    "No data source specified. Provide file_path, "
-                    "source_name, or a config with data_source/sources_file."
+                    "No data source specified. Provide source_name "
+                    "or a config with a named source."
                 )
         except AirflowException:
             raise
@@ -249,7 +206,6 @@ class DataCheckOperator(BaseOperator):
         # Build results
         results = {
             "config_path": self.config_path,
-            "file_path": self.file_path,
             "source": self.source_name,
             "table": self.table,
             "passed": summary.all_passed,
@@ -329,7 +285,7 @@ class DataCheckSchemaOperator(BaseOperator):
             )
 
     Attributes:
-        file_path: Path to a data file
+        file_path: Path to a data file (CSV, Parquet) — resolved via DuckDB
         sources_file: Path to named sources YAML file
         source_name: Named source to check
         table: Database table name
@@ -391,24 +347,26 @@ class DataCheckSchemaOperator(BaseOperator):
         self.fail_on_breaking = fail_on_breaking
         self.push_results = push_results
 
-    def _load_data(self) -> pd.DataFrame:
-        """Load data from file or named source.
+    def _resolve_source_config(self):
+        """Resolve SourceConfig from operator parameters.
 
         Returns:
-            DataFrame loaded from the configured data source
+            SourceConfig for the data source
 
         Raises:
-            AirflowException: If no data source is configured or loading fails
+            AirflowException: If no data source is configured or source not found
         """
-
         if self.file_path:
-            from datacheck.loader import LoaderFactory
+            from datacheck.config.source import SourceConfig
 
-            return LoaderFactory.load(self.file_path)
+            return SourceConfig(
+                name=str(self.file_path),
+                type="duckdb",
+                connection={"path": str(self.file_path)},
+            )
 
         if self.source_name and self.sources_file:
             from datacheck.config.source import load_sources
-            from datacheck.connectors.factory import load_source_data
 
             sources = load_sources(self.sources_file)
             if self.source_name not in sources:
@@ -416,7 +374,7 @@ class DataCheckSchemaOperator(BaseOperator):
                     f"Source '{self.source_name}' not found. "
                     f"Available: {', '.join(sorted(sources.keys()))}"
                 )
-            return load_source_data(sources[self.source_name], table=self.table, query=self.query)
+            return sources[self.source_name]
 
         raise AirflowException(
             "No data source specified. Provide file_path, "
@@ -429,6 +387,9 @@ class DataCheckSchemaOperator(BaseOperator):
         If a baseline exists, compares the current schema against it
         and reports changes. If no baseline exists, captures one.
 
+        Schema detection uses metadata-only queries (SUMMARIZE for DuckDB/S3,
+        information_schema for SQL databases) — no row data is transferred.
+
         Args:
             context: Airflow context dictionary
 
@@ -445,20 +406,26 @@ class DataCheckSchemaOperator(BaseOperator):
             f"source={self.source_name}, baseline={self.baseline_name}"
         )
 
-        # Load data
+        # Resolve source config (no data loaded yet)
         try:
-            df = self._load_data()
+            source_config = self._resolve_source_config()
         except AirflowException:
             raise
         except Exception as e:
-            raise AirflowException(f"Failed to load data: {e}")
+            raise AirflowException(f"Failed to resolve data source: {e}")
 
-        # Detect current schema
+        # Detect current schema via metadata-only queries
         detector = SchemaDetector()
-        source_label = self.file_path or self.source_name or "unknown"
-        current_schema = detector.detect(
-            df, name=self.baseline_name, source=source_label
-        )
+        try:
+            current_schema = detector.detect_from_source(
+                source_config,
+                name=self.baseline_name,
+                source_identifier=self.table or self.file_path or self.source_name,
+                table=self.table,
+                query=self.query,
+            )
+        except Exception as e:
+            raise AirflowException(f"Failed to detect schema: {e}")
 
         # Manage baseline
         manager = BaselineManager(baseline_dir=self.baseline_dir)