dequa-sdk 1.0.0__tar.gz

dequa_sdk-1.0.0/PKG-INFO

@@ -0,0 +1,115 @@
+Metadata-Version: 2.4
+Name: dequa-sdk
+Version: 1.0.0
+Summary: Native Python SDK & Integrations for Dequa ETL Data Quality Engine
+Author-email: Dequa ETL Team <support@dequa-etl.io>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/dequa-etl/dequa-etl-tool
+Project-URL: Bug Tracker, https://github.com/dequa-etl/dequa-etl-tool/issues
+Project-URL: Documentation, https://github.com/dequa-etl/dequa-etl-tool#readme
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.28.0
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: duckdb>=0.9.0
+Requires-Dist: pydantic>=2.0.0
+Provides-Extra: airflow
+Requires-Dist: apache-airflow>=2.4.0; extra == "airflow"
+Provides-Extra: dbt
+Requires-Dist: dbt-core>=1.5.0; extra == "dbt"
+Provides-Extra: all
+Requires-Dist: apache-airflow>=2.4.0; extra == "all"
+Requires-Dist: dbt-core>=1.5.0; extra == "all"
+
+# ⚡ Dequa Python SDK
+
+The official Python client and data validation SDK for **Dequa ETL**.
+
+[![PyPI](https://img.shields.io/pypi/v/dequa-sdk.svg)](https://pypi.org/project/dequa-sdk/)
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+---
+
+## 📦 Installation
+
+```bash
+pip install dequa-sdk
+```
+
+With optional integrations:
+```bash
+# Airflow integration
+pip install dequa-sdk[airflow]
+
+# dbt Core integration
+pip install dequa-sdk[dbt]
+```
+
+---
+
+## 🚀 Quick Usage
+
+### 1. Vectorized In-Memory DataFrame Reconciliation
+
+```python
+import pandas as pd
+from dequa_sdk import validate
+
+source_df = pd.read_csv("bronze_orders.csv")
+target_df = pd.read_csv("gold_orders.csv")
+
+# Run zero-config reconciliation
+report = validate(
+    source=source_df,
+    target=target_df,
+    key_columns=["order_id"],
+    compare_columns=["amount", "status"]
+)
+
+print(report.summary())
+if report.has_discrepancies():
+    print("Mismatched records found:", report.value_mismatches)
+```
+
+---
+
+### 2. Airflow DAG Gating Operator
+
+```python
+from airflow import DAG
+from dequa_sdk.integrations.airflow import DequaCheckOperator
+from datetime import datetime
+
+with DAG("daily_orders_pipeline", start_date=datetime(2026, 1, 1)) as dag:
+    assert_orders = DequaCheckOperator(
+        task_id="assert_orders_reconciliation",
+        pipeline_id=1,
+        dequa_url="http://dequa-backend:8000",
+        fail_on_mismatch=True
+    )
+```
+
+---
+
+### 3. dbt Manifest Parser
+
+```python
+from dequa_sdk.integrations.dbt import DbtManifestParser
+
+parser = DbtManifestParser(manifest_path="target/manifest.json")
+models = parser.extract_models()
+
+for model in models:
+    config = parser.generate_pipeline_config(
+        model_name=model["model_name"],
+        source_conn_id=1,
+        target_conn_id=2
+    )
+```

dequa_sdk-1.0.0/README.md

@@ -0,0 +1,85 @@
+# ⚡ Dequa Python SDK
+
+The official Python client and data validation SDK for **Dequa ETL**.
+
+[![PyPI](https://img.shields.io/pypi/v/dequa-sdk.svg)](https://pypi.org/project/dequa-sdk/)
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+---
+
+## 📦 Installation
+
+```bash
+pip install dequa-sdk
+```
+
+With optional integrations:
+```bash
+# Airflow integration
+pip install dequa-sdk[airflow]
+
+# dbt Core integration
+pip install dequa-sdk[dbt]
+```
+
+---
+
+## 🚀 Quick Usage
+
+### 1. Vectorized In-Memory DataFrame Reconciliation
+
+```python
+import pandas as pd
+from dequa_sdk import validate
+
+source_df = pd.read_csv("bronze_orders.csv")
+target_df = pd.read_csv("gold_orders.csv")
+
+# Run zero-config reconciliation
+report = validate(
+    source=source_df,
+    target=target_df,
+    key_columns=["order_id"],
+    compare_columns=["amount", "status"]
+)
+
+print(report.summary())
+if report.has_discrepancies():
+    print("Mismatched records found:", report.value_mismatches)
+```
+
+---
+
+### 2. Airflow DAG Gating Operator
+
+```python
+from airflow import DAG
+from dequa_sdk.integrations.airflow import DequaCheckOperator
+from datetime import datetime
+
+with DAG("daily_orders_pipeline", start_date=datetime(2026, 1, 1)) as dag:
+    assert_orders = DequaCheckOperator(
+        task_id="assert_orders_reconciliation",
+        pipeline_id=1,
+        dequa_url="http://dequa-backend:8000",
+        fail_on_mismatch=True
+    )
+```
+
+---
+
+### 3. dbt Manifest Parser
+
+```python
+from dequa_sdk.integrations.dbt import DbtManifestParser
+
+parser = DbtManifestParser(manifest_path="target/manifest.json")
+models = parser.extract_models()
+
+for model in models:
+    config = parser.generate_pipeline_config(
+        model_name=model["model_name"],
+        source_conn_id=1,
+        target_conn_id=2
+    )
+```

dequa_sdk-1.0.0/__init__.py

@@ -0,0 +1,10 @@
+"""
+Dequa SDK — Native Data Quality & Validation Engine
+"""
+
+from dequa_sdk.client import Client
+from dequa_sdk.validators import validate, ValidationReport
+from dequa_sdk.quarantine import QuarantineRouter
+
+__version__ = "1.0.0"
+__all__ = ["Client", "validate", "ValidationReport", "QuarantineRouter"]

dequa_sdk-1.0.0/client.py

@@ -0,0 +1,42 @@
+"""
+Dequa SDK Client Core
+
+Client interface for connecting Python data pipelines, PySpark jobs, and notebooks
+to the Dequa ETL backend control plane.
+"""
+
+from typing import Any, Dict, Optional
+import os
+import requests
+
+
+class Client:
+    """Dequa SDK Client for pipeline validation and observability sync."""
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        control_plane_url: Optional[str] = None,
+        offline_mode: bool = False
+    ):
+        self.api_key = api_key or os.getenv("DEQUA_API_KEY", "")
+        self.control_plane_url = (control_plane_url or os.getenv("DEQUA_API_URL", "http://localhost:8000/api/v1")).rstrip("/")
+        self.offline_mode = offline_mode
+
+    def _headers(self) -> Dict[str, str]:
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["X-API-Key"] = self.api_key
+        return headers
+
+    def sync_validation_report(self, pipeline_id: str, report_data: Dict[str, Any]) -> bool:
+        """Sends inline SDK validation results to Dequa Control Plane."""
+        if self.offline_mode:
+            return True
+
+        endpoint = f"{self.control_plane_url}/pipelines/{pipeline_id}/sdk-report"
+        try:
+            resp = requests.post(endpoint, json=report_data, headers=self._headers(), timeout=10)
+            return resp.status_code in [200, 201]
+        except Exception:
+            return False

dequa_sdk-1.0.0/dequa_sdk.egg-info/PKG-INFO

@@ -0,0 +1,115 @@
+Metadata-Version: 2.4
+Name: dequa-sdk
+Version: 1.0.0
+Summary: Native Python SDK & Integrations for Dequa ETL Data Quality Engine
+Author-email: Dequa ETL Team <support@dequa-etl.io>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/dequa-etl/dequa-etl-tool
+Project-URL: Bug Tracker, https://github.com/dequa-etl/dequa-etl-tool/issues
+Project-URL: Documentation, https://github.com/dequa-etl/dequa-etl-tool#readme
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.28.0
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: duckdb>=0.9.0
+Requires-Dist: pydantic>=2.0.0
+Provides-Extra: airflow
+Requires-Dist: apache-airflow>=2.4.0; extra == "airflow"
+Provides-Extra: dbt
+Requires-Dist: dbt-core>=1.5.0; extra == "dbt"
+Provides-Extra: all
+Requires-Dist: apache-airflow>=2.4.0; extra == "all"
+Requires-Dist: dbt-core>=1.5.0; extra == "all"
+
+# ⚡ Dequa Python SDK
+
+The official Python client and data validation SDK for **Dequa ETL**.
+
+[![PyPI](https://img.shields.io/pypi/v/dequa-sdk.svg)](https://pypi.org/project/dequa-sdk/)
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+---
+
+## 📦 Installation
+
+```bash
+pip install dequa-sdk
+```
+
+With optional integrations:
+```bash
+# Airflow integration
+pip install dequa-sdk[airflow]
+
+# dbt Core integration
+pip install dequa-sdk[dbt]
+```
+
+---
+
+## 🚀 Quick Usage
+
+### 1. Vectorized In-Memory DataFrame Reconciliation
+
+```python
+import pandas as pd
+from dequa_sdk import validate
+
+source_df = pd.read_csv("bronze_orders.csv")
+target_df = pd.read_csv("gold_orders.csv")
+
+# Run zero-config reconciliation
+report = validate(
+    source=source_df,
+    target=target_df,
+    key_columns=["order_id"],
+    compare_columns=["amount", "status"]
+)
+
+print(report.summary())
+if report.has_discrepancies():
+    print("Mismatched records found:", report.value_mismatches)
+```
+
+---
+
+### 2. Airflow DAG Gating Operator
+
+```python
+from airflow import DAG
+from dequa_sdk.integrations.airflow import DequaCheckOperator
+from datetime import datetime
+
+with DAG("daily_orders_pipeline", start_date=datetime(2026, 1, 1)) as dag:
+    assert_orders = DequaCheckOperator(
+        task_id="assert_orders_reconciliation",
+        pipeline_id=1,
+        dequa_url="http://dequa-backend:8000",
+        fail_on_mismatch=True
+    )
+```
+
+---
+
+### 3. dbt Manifest Parser
+
+```python
+from dequa_sdk.integrations.dbt import DbtManifestParser
+
+parser = DbtManifestParser(manifest_path="target/manifest.json")
+models = parser.extract_models()
+
+for model in models:
+    config = parser.generate_pipeline_config(
+        model_name=model["model_name"],
+        source_conn_id=1,
+        target_conn_id=2
+    )
+```

dequa_sdk-1.0.0/dequa_sdk.egg-info/SOURCES.txt

@@ -0,0 +1,23 @@
+README.md
+__init__.py
+client.py
+magics.py
+pyproject.toml
+quarantine.py
+validators.py
+./__init__.py
+./client.py
+./magics.py
+./quarantine.py
+./validators.py
+./integrations/__init__.py
+./integrations/airflow.py
+./integrations/dbt.py
+dequa_sdk.egg-info/PKG-INFO
+dequa_sdk.egg-info/SOURCES.txt
+dequa_sdk.egg-info/dependency_links.txt
+dequa_sdk.egg-info/requires.txt
+dequa_sdk.egg-info/top_level.txt
+integrations/__init__.py
+integrations/airflow.py
+integrations/dbt.py

dequa_sdk-1.0.0/dequa_sdk.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dequa_sdk-1.0.0/dequa_sdk.egg-info/requires.txt

@@ -0,0 +1,14 @@
+requests>=2.28.0
+pandas>=1.5.0
+duckdb>=0.9.0
+pydantic>=2.0.0
+
+[airflow]
+apache-airflow>=2.4.0
+
+[all]
+apache-airflow>=2.4.0
+dbt-core>=1.5.0
+
+[dbt]
+dbt-core>=1.5.0

dequa_sdk-1.0.0/dequa_sdk.egg-info/top_level.txt

@@ -0,0 +1 @@
+dequa_sdk

dequa_sdk-1.0.0/integrations/__init__.py

@@ -0,0 +1,8 @@
+"""
+Dequa SDK — Modern Data Stack Integrations (Airflow, dbt)
+"""
+
+from dequa_sdk.integrations.airflow import DequaCheckOperator
+from dequa_sdk.integrations.dbt import DbtManifestParser
+
+__all__ = ["DequaCheckOperator", "DbtManifestParser"]

dequa_sdk-1.0.0/integrations/airflow.py

@@ -0,0 +1,85 @@
+"""
+Dequa ETL — Apache Airflow Integration Operator
+Allows seamless gating and automated quality checks within Airflow DAGs.
+"""
+
+from typing import Optional, List, Dict, Any
+import time
+
+try:
+    from airflow.models import BaseOperator
+    from airflow.exceptions import AirflowException
+except ImportError:
+    # Graceful fallback for non-Airflow execution environments
+    class BaseOperator:
+        def __init__(self, **kwargs):
+            pass
+
+    class AirflowException(Exception):
+        pass
+
+from dequa_sdk.client import Client
+
+
+class DequaCheckOperator(BaseOperator):
+    """
+    Airflow Operator that triggers a Dequa pipeline validation and asserts quality SLA.
+
+    Usage:
+        from dequa_sdk.integrations.airflow import DequaCheckOperator
+
+        check_orders = DequaCheckOperator(
+            task_id="assert_orders_reconciliation",
+            pipeline_id=1,
+            fail_on_mismatch=True,
+            dequa_url="http://dequa-backend:8000",
+            api_key="your_api_key",
+        )
+    """
+
+    def __init__(
+        self,
+        pipeline_id: int,
+        dequa_url: str = "http://localhost:8000",
+        api_key: Optional[str] = None,
+        fail_on_mismatch: bool = True,
+        timeout_seconds: int = 300,
+        poll_interval: int = 5,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.pipeline_id = pipeline_id
+        self.dequa_url = dequa_url
+        self.api_key = api_key
+        self.fail_on_mismatch = fail_on_mismatch
+        self.timeout_seconds = timeout_seconds
+        self.poll_interval = poll_interval
+
+    def execute(self, context: Any) -> Dict[str, Any]:
+        self.log.info(f"Triggering Dequa Pipeline ID: {self.pipeline_id}")
+        client = Client(base_url=self.dequa_url, api_key=self.api_key)
+
+        job = client.trigger_pipeline(self.pipeline_id)
+        job_id = job.get("id") or job.get("job_id")
+        self.log.info(f"Dequa Job {job_id} initiated. Polling for completion...")
+
+        start_time = time.time()
+        while time.time() - start_time < self.timeout_seconds:
+            status_data = client.get_job_status(job_id)
+            status = status_data.get("status")
+
+            if status in ("passed", "success"):
+                self.log.info(f"Dequa Job {job_id} PASSED with 100% record parity.")
+                return status_data
+            elif status in ("failed", "warning"):
+                mismatch_count = status_data.get("value_mismatches_count", 0) + status_data.get("missing_in_target_count", 0)
+                msg = f"Dequa Job {job_id} failed with {mismatch_count} discrepancies."
+                if self.fail_on_mismatch:
+                    raise AirflowException(msg)
+                else:
+                    self.log.warning(msg)
+                    return status_data
+
+            time.sleep(self.poll_interval)
+
+        raise AirflowException(f"Dequa Job {job_id} timed out after {self.timeout_seconds} seconds.")

dequa_sdk-1.0.0/integrations/dbt.py

@@ -0,0 +1,80 @@
+"""
+Dequa ETL — dbt Core Integration
+Parses dbt manifest artifacts (manifest.json, catalog.json) to automatically generate
+Dequa schema reconciliation pipelines and assertions.
+"""
+
+import json
+import os
+from typing import Dict, List, Any, Optional
+
+
+class DbtManifestParser:
+    """
+    Introspects compiled dbt project manifests to extract models, primary keys,
+    and column schemas for automated Dequa pipeline generation.
+    """
+
+    def __init__(self, manifest_path: str, catalog_path: Optional[str] = None):
+        self.manifest_path = manifest_path
+        self.catalog_path = catalog_path
+        self.manifest_data = self._load_json(manifest_path)
+        self.catalog_data = self._load_json(catalog_path) if catalog_path and os.path.exists(catalog_path) else {}
+
+    def _load_json(self, path: str) -> Dict[str, Any]:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def extract_models(self) -> List[Dict[str, Any]]:
+        """
+        Extracts all model definitions and unique keys from the dbt manifest.
+        """
+        nodes = self.manifest_data.get("nodes", {})
+        models = []
+
+        for node_id, node in nodes.items():
+            if node.get("resource_type") == "model":
+                name = node.get("name")
+                schema = node.get("schema")
+                database = node.get("database")
+                alias = node.get("alias", name)
+
+                # Extract primary/unique keys if defined in dbt tests or meta
+                columns = list(node.get("columns", {}).keys())
+                meta = node.get("meta", {})
+                primary_keys = meta.get("primary_key", [])
+                if isinstance(primary_keys, str):
+                    primary_keys = [primary_keys]
+
+                models.append({
+                    "model_name": name,
+                    "target_table": f"{schema}.{alias}" if schema else alias,
+                    "database": database,
+                    "columns": columns,
+                    "primary_keys": primary_keys,
+                    "tags": node.get("tags", []),
+                })
+
+        return models
+
+    def generate_pipeline_config(self, model_name: str, source_conn_id: int, target_conn_id: int) -> Optional[Dict[str, Any]]:
+        """
+        Generates a complete Dequa pipeline configuration dict for a specific dbt model.
+        """
+        models = self.extract_models()
+        target = next((m for m in models if m["model_name"] == model_name), None)
+        if not target:
+            return None
+
+        primary_keys = target["primary_keys"] or (["id"] if "id" in target["columns"] else target["columns"][:1])
+
+        return {
+            "name": f"dbt Reconciliation: {model_name}",
+            "source_conn_id": source_conn_id,
+            "target_conn_id": target_conn_id,
+            "source_table": f"stg_{model_name}",
+            "target_table": target["target_table"],
+            "key_columns": primary_keys,
+            "compare_columns": [{"source": c, "target": c} for c in target["columns"] if c not in primary_keys],
+            "tags": ["dbt-generated", *target["tags"]],
+        }

dequa_sdk-1.0.0/magics.py

@@ -0,0 +1,67 @@
+"""
+Dequa IPython Magic Extensions for Jupyter & Databricks Notebooks
+"""
+
+from typing import Any
+import pandas as pd
+
+try:
+    from IPython.core.magic import Magics, magics_class, line_magic, cell_magic
+except ImportError:
+    # Graceful fallback if IPython is not installed
+    class Magics:
+        pass
+    def magics_class(cls): return cls
+    def line_magic(func): return func
+    def cell_magic(func): return func
+
+from dequa_sdk.validators import validate
+from dequa_sdk.quarantine import QuarantineRouter
+
+
+@magics_class
+class DequaMagics(Magics):
+    """IPython magic commands for interactive data quality testing."""
+
+    @line_magic
+    def dequa_profile(self, line: str):
+        """Line magic to quickly profile a pandas dataframe variable: %dequa_profile my_df"""
+        var_name = line.strip()
+        if not var_name or var_name not in self.shell.user_ns:
+            print(f"Error: Variable '{var_name}' not found in notebook namespace.")
+            return
+
+        val = self.shell.user_ns[var_name]
+        if isinstance(val, pd.DataFrame):
+            print(f"━━━ Dequa Profile: {var_name} ━━━")
+            print(f"Shape: {val.shape[0]} rows × {val.shape[1]} columns")
+            print(f"Null Cells Total: {val.isna().sum().sum()}")
+            print(f"Duplicate Rows: {val.duplicated().sum()}")
+            print("\nColumn Null Ratios:")
+            print((val.isna().mean() * 100.0).round(2).to_string())
+
+    @cell_magic
+    def dequa_test(self, line: str, cell: str):
+        """Cell magic to run assertions on a dataframe variable."""
+        var_name = line.strip()
+        if not var_name or var_name not in self.shell.user_ns:
+            print(f"Error: Variable '{var_name}' not found.")
+            return
+
+        df = self.shell.user_ns[var_name]
+        rules = [line.strip() for line in cell.strip().split("\n") if line.strip() and not line.startswith("#")]
+
+        report = validate(df, rules, dataset_name=var_name)
+        print(f"━━━ Dequa Validation Results for '{var_name}' ━━━")
+        print(f"Status: {'PASS ✅' if report.is_success else 'FAIL ❌'}")
+        print(f"Pass Rate: {report.pass_rate}% ({len(report.passed_rules)}/{len(rules)} rules passed)")
+
+        if report.failed_rules:
+            print(f"\nFailed Rules ({len(report.failed_rules)}):")
+            for r in report.failed_rules:
+                print(f"  - ❌ {r}")
+
+
+def load_ipython_extension(ipython):
+    """Entry point for %load_ext dequa_sdk.magics"""
+    ipython.register_magics(DequaMagics)

dequa_sdk-1.0.0/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dequa-sdk"
+version = "1.0.0"
+description = "Native Python SDK & Integrations for Dequa ETL Data Quality Engine"
+readme = "README.md"
+authors = [{ name = "Dequa ETL Team", email = "support@dequa-etl.io" }]
+license = "Apache-2.0"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "Topic :: Database",
+    "Topic :: Software Development :: Quality Assurance",
+]
+requires-python = ">=3.9"
+dependencies = [
+    "requests>=2.28.0",
+    "pandas>=1.5.0",
+    "duckdb>=0.9.0",
+    "pydantic>=2.0.0",
+]
+
+[project.optional-dependencies]
+airflow = [
+    "apache-airflow>=2.4.0",
+]
+dbt = [
+    "dbt-core>=1.5.0",
+]
+all = [
+    "apache-airflow>=2.4.0",
+    "dbt-core>=1.5.0",
+]
+
+[tool.setuptools]
+packages = ["dequa_sdk", "dequa_sdk.integrations"]
+package-dir = {"dequa_sdk" = "."}
+
+[project.urls]
+"Homepage" = "https://github.com/dequa-etl/dequa-etl-tool"
+"Bug Tracker" = "https://github.com/dequa-etl/dequa-etl-tool/issues"
+"Documentation" = "https://github.com/dequa-etl/dequa-etl-tool#readme"

dequa_sdk-1.0.0/quarantine.py

@@ -0,0 +1,44 @@
+"""
+Dequa SDK Dynamic Quarantine Router
+
+Splits clean rows from corrupted rows violating validation assertions,
+routing invalid records to quarantine storage (DataFrame, CSV, or DB table).
+"""
+
+from typing import Any, Tuple
+import pandas as pd
+from dequa_sdk.validators import ValidationReport
+
+
+class QuarantineRouter:
+    """Routes invalid records into quarantine isolated data streams."""
+
+    @staticmethod
+    def route(df: Any, report: ValidationReport) -> Tuple[pd.DataFrame, pd.DataFrame]:
+        """
+        Splits input DataFrame into (clean_df, quarantined_df).
+        """
+        if hasattr(df, "toPandas"):
+            pdf = df.toPandas()
+        elif isinstance(df, pd.DataFrame):
+            pdf = df
+        else:
+            pdf = pd.DataFrame(df)
+
+        if not report.quarantine_indices:
+            return pdf, pd.DataFrame(columns=pdf.columns)
+
+        quarantine_set = set(report.quarantine_indices)
+        clean_mask = ~pdf.index.isin(quarantine_set)
+        quarantine_mask = pdf.index.isin(quarantine_set)
+
+        clean_df = pdf[clean_mask].copy()
+        quarantined_df = pdf[quarantine_mask].copy()
+
+        return clean_df, quarantined_df
+
+    @staticmethod
+    def export_quarantine(quarantined_df: pd.DataFrame, output_path: str) -> str:
+        """Exports quarantined records to CSV file path."""
+        quarantined_df.to_csv(output_path, index=False)
+        return output_path

dequa_sdk-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

dequa_sdk-1.0.0/validators.py

@@ -0,0 +1,150 @@
+"""
+Dequa SDK DataFrame Validation Engine
+
+Provides inline validation rules for Pandas, Polars, and PySpark DataFrames.
+"""
+
+from typing import Any, Dict, List, Optional, Tuple, Union
+import re
+import pandas as pd
+
+
+class ValidationReport:
+    """Validation execution result details."""
+
+    def __init__(self, dataset_name: str, total_rows: int):
+        self.dataset_name = dataset_name
+        self.total_rows = total_rows
+        self.passed_rules: List[str] = []
+        self.failed_rules: List[str] = []
+        self.rule_details: List[Dict[str, Any]] = []
+        self.quarantine_indices: List[int] = []
+
+    @property
+    def is_success(self) -> bool:
+        return len(self.failed_rules) == 0
+
+    @property
+    def pass_rate(self) -> float:
+        total = len(self.passed_rules) + len(self.failed_rules)
+        return (len(self.passed_rules) / total * 100.0) if total > 0 else 100.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "dataset_name": self.dataset_name,
+            "total_rows": self.total_rows,
+            "is_success": self.is_success,
+            "pass_rate": round(self.pass_rate, 2),
+            "passed_rules_count": len(self.passed_rules),
+            "failed_rules_count": len(self.failed_rules),
+            "failed_rules": self.failed_rules,
+            "rule_details": self.rule_details,
+            "quarantined_rows_count": len(self.quarantine_indices)
+        }
+
+
+def _parse_rule(rule_str: str) -> Tuple[str, List[str]]:
+    """Parse rule strings like 'not_null(customer_id)' into ('not_null', ['customer_id'])."""
+    match = re.match(r"^(\w+)\((.*)\)$", rule_str.strip())
+    if not match:
+        return rule_str.strip(), []
+    func = match.group(1).lower()
+    args_raw = match.group(2)
+    args = [arg.strip().strip("'\"") for arg in args_raw.split(",") if arg.strip()]
+    return func, args
+
+
+def validate(
+    df: Any,
+    rules: List[str],
+    dataset_name: str = "dataset",
+    on_failure: str = "warn"
+) -> ValidationReport:
+    """
+    Validate a DataFrame against assertion rules.
+
+    :param df: Pandas DataFrame or PySpark DataFrame
+    :param rules: List of rule strings, e.g. ["not_null(id)", "min_value(amount, 0)", "unique(id)"]
+    :param dataset_name: Name identifier for the dataset
+    :param on_failure: Strategy when validation fails ("warn", "raise", "quarantine")
+    """
+    total_rows = len(df) if hasattr(df, "__len__") else 0
+    report = ValidationReport(dataset_name=dataset_name, total_rows=total_rows)
+
+    # Convert to Pandas if PySpark DataFrame passed
+    if hasattr(df, "toPandas"):
+        pdf = df.toPandas()
+    elif isinstance(df, pd.DataFrame):
+        pdf = df
+    else:
+        pdf = pd.DataFrame(df)
+
+    quarantine_indices = set()
+
+    for rule_str in rules:
+        func_name, args = _parse_rule(rule_str)
+        is_passed = True
+        failed_count = 0
+
+        if func_name == "not_null" and args:
+            col = args[0]
+            if col in pdf.columns:
+                null_mask = pdf[col].isna()
+                failed_count = int(null_mask.sum())
+                if failed_count > 0:
+                    is_passed = False
+                    quarantine_indices.update(pdf[null_mask].index.tolist())
+
+        elif func_name == "min_value" and len(args) >= 2:
+            col, min_val = args[0], float(args[1])
+            if col in pdf.columns:
+                min_mask = pdf[col] < min_val
+                failed_count = int(min_mask.sum())
+                if failed_count > 0:
+                    is_passed = False
+                    quarantine_indices.update(pdf[min_mask].index.tolist())
+
+        elif func_name == "max_value" and len(args) >= 2:
+            col, max_val = args[0], float(args[1])
+            if col in pdf.columns:
+                max_mask = pdf[col] > max_val
+                failed_count = int(max_mask.sum())
+                if failed_count > 0:
+                    is_passed = False
+                    quarantine_indices.update(pdf[max_mask].index.tolist())
+
+        elif func_name == "unique" and args:
+            col = args[0]
+            if col in pdf.columns:
+                dup_mask = pdf[col].duplicated(keep=False)
+                failed_count = int(dup_mask.sum())
+                if failed_count > 0:
+                    is_passed = False
+                    quarantine_indices.update(pdf[dup_mask].index.tolist())
+
+        elif func_name == "regex" and len(args) >= 2:
+            col, pattern = args[0], args[1]
+            if col in pdf.columns:
+                regex_mask = ~pdf[col].astype(str).str.contains(pattern, na=False, regex=True)
+                failed_count = int(regex_mask.sum())
+                if failed_count > 0:
+                    is_passed = False
+                    quarantine_indices.update(pdf[regex_mask].index.tolist())
+
+        if is_passed:
+            report.passed_rules.append(rule_str)
+        else:
+            report.failed_rules.append(rule_str)
+
+        report.rule_details.append({
+            "rule": rule_str,
+            "status": "PASS" if is_passed else "FAIL",
+            "failed_records_count": failed_count
+        })
+
+    report.quarantine_indices = sorted(list(quarantine_indices))
+
+    if not report.is_success and on_failure == "raise":
+        raise ValueError(f"Dequa Validation Failed for '{dataset_name}': {len(report.failed_rules)} rules violated.")
+
+    return report