datacheck-cli 2.1.1__tar.gz → 2.1.2__tar.gz

{datacheck_cli-2.1.1 → datacheck_cli-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacheck-cli
-Version: 2.1.1
+Version: 2.1.2
 Summary: A linter for data pipelines. Enforce deterministic validation rules in CI/CD, Airflow, and beyond.
 License: Apache-2.0
 License-File: LICENSE
@@ -39,6 +39,7 @@ Provides-Extra: validation
 Provides-Extra: warehouses
 Requires-Dist: boto3 (>=1.34.0,<2.0.0) ; extra == "s3" or extra == "cloud" or extra == "redshift" or extra == "warehouses" or extra == "all"
 Requires-Dist: click (>=8.1.0,<9.0.0)
+Requires-Dist: duckdb (>=1.0.0,<2.0.0)
 Requires-Dist: google-auth (>=2.0.0,<3.0.0) ; extra == "bigquery" or extra == "warehouses" or extra == "all"
 Requires-Dist: google-cloud-bigquery (>=3.0.0,<4.0.0) ; extra == "bigquery" or extra == "warehouses" or extra == "all"
 Requires-Dist: jsonschema (>=4.17.0,<5.0.0) ; extra == "validation" or extra == "all"
@@ -111,12 +112,20 @@ datacheck config init --with-sample-data
 datacheck config init --template ecommerce --with-sample-data
 ```
 
-**Option 2 - Write manually.** Create a `.datacheck.yaml` config file with your data source and validation rules:
+**Option 2 - Write manually.** Create a `sources.yaml` and `.datacheck.yaml` with your data source and validation rules:
 
 ```yaml
-data_source:
-  type: csv
-  path: ./data/orders.csv
+# sources.yaml
+sources:
+  orders:
+    type: duckdb
+    path: ./data/orders.csv
+```
+
+```yaml
+# .datacheck.yaml
+sources_file: sources.yaml
+source: orders
 
 checks:
   - name: id_check
@@ -137,8 +146,7 @@ Run validation:
 
 ```bash
 datacheck validate                          # auto-discover config
-datacheck validate data.csv                 # direct file
-datacheck validate --config checks.yaml
+datacheck validate --config checks.yaml     # explicit config path
 echo $?  # 1 if any error-severity rule fails
 ```
 
@@ -252,7 +260,7 @@ datacheck schema compare --fail-on-breaking
 from datacheck import ValidationEngine
 
 engine = ValidationEngine(config_path=".datacheck.yaml")
-summary = engine.validate()
+summary = engine.validate_sources()
 
 print(f"Passed: {summary.passed_rules}/{summary.total_rules}")
 

{datacheck_cli-2.1.1 → datacheck_cli-2.1.2}/README_PYPI.md

@@ -52,12 +52,20 @@ datacheck config init --with-sample-data
 datacheck config init --template ecommerce --with-sample-data
 ```
 
-**Option 2 - Write manually.** Create a `.datacheck.yaml` config file with your data source and validation rules:
+**Option 2 - Write manually.** Create a `sources.yaml` and `.datacheck.yaml` with your data source and validation rules:
 
 ```yaml
-data_source:
-  type: csv
-  path: ./data/orders.csv
+# sources.yaml
+sources:
+  orders:
+    type: duckdb
+    path: ./data/orders.csv
+```
+
+```yaml
+# .datacheck.yaml
+sources_file: sources.yaml
+source: orders
 
 checks:
   - name: id_check
@@ -78,8 +86,7 @@ Run validation:
 
 ```bash
 datacheck validate                          # auto-discover config
-datacheck validate data.csv                 # direct file
-datacheck validate --config checks.yaml
+datacheck validate --config checks.yaml     # explicit config path
 echo $?  # 1 if any error-severity rule fails
 ```
 
@@ -193,7 +200,7 @@ datacheck schema compare --fail-on-breaking
 from datacheck import ValidationEngine
 
 engine = ValidationEngine(config_path=".datacheck.yaml")
-summary = engine.validate()
+summary = engine.validate_sources()
 
 print(f"Passed: {summary.passed_rules}/{summary.total_rules}")
 

{datacheck_cli-2.1.1 → datacheck_cli-2.1.2}/datacheck/airflow/operators.py

@@ -13,7 +13,6 @@ from collections.abc import Sequence
 
 if TYPE_CHECKING:
     import pandas as pd
-from pathlib import Path
 
 try:
     from airflow.models import BaseOperator
@@ -63,31 +62,21 @@ class DataCheckOperator(BaseOperator):
     """Operator for running DataCheck validation in Airflow DAGs.
 
     Uses the full ValidationEngine to run config-based rules against
-    data from files or named database sources. Supports sampling,
-    parallel execution, and quality thresholds.
+    named data sources. All data sources are defined in a sources.yaml
+    file and referenced by name.
 
     Data source resolution (in order):
-        1. ``file_path`` — validate a local or cloud file
-        2. ``source_name`` — validate a named source from sources.yaml
-        3. Config default — uses ``source`` or ``data_source`` from config
+        1. ``source_name`` — validate a named source from sources.yaml
+        2. Config default — uses ``source`` defined in the config
 
     Examples:
-        Validate a file::
-
-            DataCheckOperator(
-                task_id="validate_orders",
-                config_path="/config/checks.yaml",
-                file_path="/data/orders_{{ ds }}.parquet",
-            )
-
-        Validate a database table::
+        Validate a named source::
 
             DataCheckOperator(
                 task_id="validate_orders",
                 config_path="/config/checks.yaml",
                 sources_file="/config/sources.yaml",
-                source_name="production_db",
-                table="orders",
+                source_name="orders",
                 where="created_at >= '{{ ds }}'",
             )
 
@@ -96,21 +85,19 @@ class DataCheckOperator(BaseOperator):
             DataCheckOperator(
                 task_id="validate_orders",
                 config_path="/config/checks.yaml",
-                file_path="/data/orders.parquet",
+                sources_file="/config/sources.yaml",
+                source_name="orders",
                 min_pass_rate=95.0,
                 fail_on_error=True,
             )
 
     Attributes:
         config_path: Path to the DataCheck validation config YAML
-        file_path: Path to a data file (CSV, Parquet)
         sources_file: Path to named sources YAML file
         source_name: Named source to validate
         table: Database table name override
         where: SQL WHERE clause for filtering
         query: Custom SQL query (alternative to table)
-        parallel: Enable multi-core validation
-        workers: Number of worker processes
         min_pass_rate: Minimum rule pass rate to succeed (0-100)
         fail_on_error: Whether to fail the Airflow task on validation failure
         push_results: Whether to push results to XCom
@@ -118,7 +105,6 @@ class DataCheckOperator(BaseOperator):
 
     template_fields: Sequence[str] = (
         "config_path",
-        "file_path",
         "sources_file",
         "source_name",
         "table",
@@ -133,14 +119,11 @@ class DataCheckOperator(BaseOperator):
     def __init__(
         self,
         config_path: str,
-        file_path: str | None = None,
         sources_file: str | None = None,
         source_name: str | None = None,
         table: str | None = None,
         where: str | None = None,
         query: str | None = None,
-        parallel: bool = False,
-        workers: int | None = None,
         min_pass_rate: float = 0.0,
         fail_on_error: bool = True,
         push_results: bool = True,
@@ -150,14 +133,11 @@ class DataCheckOperator(BaseOperator):
 
         Args:
             config_path: Path to DataCheck validation config YAML (required)
-            file_path: Path to data file (CSV, Parquet)
             sources_file: Path to sources YAML file (overrides config)
             source_name: Named source from sources.yaml
             table: Database table name (for database sources)
             where: WHERE clause for filtering (for database sources)
             query: Custom SQL query (alternative to table)
-            parallel: Enable parallel execution
-            workers: Number of worker processes (default: CPU count)
             min_pass_rate: Minimum pass rate percentage (0-100, 0 = disabled)
             fail_on_error: Whether to raise AirflowException on failure
             push_results: Whether to push results to XCom
@@ -165,14 +145,11 @@ class DataCheckOperator(BaseOperator):
         """
         super().__init__(**kwargs)
         self.config_path = config_path
-        self.file_path = file_path
         self.sources_file = sources_file
         self.source_name = source_name
         self.table = table
         self.where = where
         self.query = query
-        self.parallel = parallel
-        self.workers = workers
         self.min_pass_rate = min_pass_rate
         self.fail_on_error = fail_on_error
         self.push_results = push_results
@@ -193,7 +170,7 @@ class DataCheckOperator(BaseOperator):
 
         self.log.info(
             f"Running DataCheck validation: config={self.config_path}, "
-            f"file={self.file_path}, source={self.source_name}"
+            f"source={self.source_name}"
         )
 
         # Initialize engine
@@ -201,34 +178,23 @@ class DataCheckOperator(BaseOperator):
             engine = ValidationEngine(
                 config_path=self.config_path,
                 sources_file=self.sources_file,
-                parallel=self.parallel,
-                workers=self.workers,
             )
         except Exception as e:
             raise AirflowException(f"Failed to initialize ValidationEngine: {e}")
 
-        # Run validation based on data source
+        # Run validation
         try:
-            if self.file_path:
-                # File-based validation
-                summary = engine.validate_file(self.file_path)
-            elif self.source_name or engine.config.source:
-                # Named source validation
+            if self.source_name or engine.config.source:
                 summary = engine.validate_sources(
                     source_name=self.source_name,
                     table=self.table,
                     where=self.where,
                     query=self.query,
                 )
-            elif engine.config.data_source is not None:
-                # Inline data_source from config
-                config_dir = Path(self.config_path).parent
-                source_path = config_dir / engine.config.data_source.path
-                summary = engine.validate_file(str(source_path))
             else:
                 raise AirflowException(
-                    "No data source specified. Provide file_path, "
-                    "source_name, or a config with data_source/sources_file."
+                    "No data source specified. Provide source_name "
+                    "or a config with a named source."
                 )
         except AirflowException:
             raise
@@ -249,7 +215,6 @@ class DataCheckOperator(BaseOperator):
         # Build results
         results = {
             "config_path": self.config_path,
-            "file_path": self.file_path,
             "source": self.source_name,
             "table": self.table,
             "passed": summary.all_passed,

{datacheck_cli-2.1.1 → datacheck_cli-2.1.2}/datacheck/cli/schema.py

@@ -84,16 +84,11 @@ def _resolve_data_source(
         df = load_source_data(source_config, table=table, query=query)
         resolved_source_name = source
 
-    # Option 2: Inline data_source from config
+    # Option 2: Default source from config
     elif data_source is None and config:
         config_data = ConfigLoader.load(config)
         config_dir = Path(config).parent
-        if config_data.data_source:
-            source_path = config_dir / config_data.data_source.path
-            df = LoaderFactory.load(str(source_path), table=table, query=query)
-            resolved_source_name = str(source_path)
-        elif config_data.sources_file and config_data.source:
-            # Use default source from config
+        if config_data.sources_file and config_data.source:
             sources_path = config_dir / config_data.sources_file
             sources = load_sources(sources_path)
             if config_data.source not in sources:
@@ -107,7 +102,8 @@ def _resolve_data_source(
             resolved_source_name = config_data.source
         else:
             console.print(
-                "[red]Error:[/red] Config file has no data_source or sources_file defined",
+                "[red]Error:[/red] Config file has no sources_file/source defined. "
+                "Use --source with --sources-file.",
                 style="red",
             )
             raise typer.Exit(code=2)
@@ -118,11 +114,7 @@ def _resolve_data_source(
         if found_config:
             config_data = ConfigLoader.load(found_config)
             config_dir = found_config.parent
-            if config_data.data_source:
-                source_path = config_dir / config_data.data_source.path
-                df = LoaderFactory.load(str(source_path), table=table, query=query)
-                resolved_source_name = str(source_path)
-            elif config_data.sources_file and config_data.source:
+            if config_data.sources_file and config_data.source:
                 sources_path = config_dir / config_data.sources_file
                 sources = load_sources(sources_path)
                 if config_data.source in sources:
@@ -138,15 +130,14 @@ def _resolve_data_source(
             else:
                 console.print(
                     "[red]Error:[/red] No data source specified. "
-                    "Provide a file path as argument, use --config with data_source, "
-                    "or use --source with --sources-file.",
+                    "Provide a file path as argument, or use --source with --sources-file.",
                     style="red",
                 )
                 raise typer.Exit(code=2)
         else:
             console.print(
                 "[red]Error:[/red] No data source specified. "
-                "Provide a file path as argument, use --config, or use --source.",
+                "Provide a file path as argument, or use --source with --sources-file.",
                 style="red",
             )
             raise typer.Exit(code=2)