dqtlib 0.1.0__tar.gz

dqtlib-0.1.0/.gitignore

@@ -0,0 +1,128 @@
+# --- Python ---
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# uv
+.venv/
+.uv-cache/
+
+# pip
+pip-log.txt
+pip-delete-this-directory.txt
+
+# pytest
+.pytest_cache/
+.cache/
+.coverage
+.coverage.*
+htmlcov/
+.tox/
+.nox/
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+
+# mypy / ruff
+.mypy_cache/
+.ruff_cache/
+.dmypy.json
+dmypy.json
+
+# --- Node / Next.js ---
+node_modules/
+.next/
+.turbo/
+.swc/
+out/
+dist/
+build/
+.npm/
+.pnpm-store/
+pnpm-debug.log*
+yarn-debug.log*
+npm-debug.log*
+
+# --- Generated artifacts (NEVER commit) ---
+packages/dqt-types/
+shared/generated/
+apps/web/src/generated/
+apps/web/src/lib/stats.generated.ts
+apps/web/src/components/connections/engines.generated.ts
+packages/dqt/src/dqt/generated/
+apps/server/openapi.json
+
+# --- Env / secrets ---
+.env
+.env.local
+.env.*.local
+*.pem
+*.key
+*.p12
+*.crt
+service-account*.json
+
+# --- OS / editor ---
+.DS_Store
+Thumbs.db
+.idea/
+.vscode/
+!.vscode/settings.json.example
+*.swp
+*.swo
+*~
+.history/
+
+# --- Local dev ---
+tmp/
+*.log
+logs/
+
+# --- Generated demo reports ---
+examples/*/reports/
+apps/web/tsconfig.tsbuildinfo
+
+# --- Unrelated legacy code ---
+dql/
+.cache/
+
+# --- Docker / DB volumes ---
+run_local/data/
+run_local/postgres-data/
+run_local/redis-data/
+
+# --- Test artifacts ---
+playwright-report/
+test-results/
+e2e-screenshots/
+*.pyc
+.benchmarks/
+
+# --- Built docs ---
+docs/_build/
+site/
+
+# --- Archives (created by build scripts) ---
+*.zip
+*.tar.gz
+!reference_data/**/*.zip

dqtlib-0.1.0/PKG-INFO

@@ -0,0 +1,64 @@
+Metadata-Version: 2.4
+Name: dqtlib
+Version: 0.1.0
+Summary: Data quality, observability, semantic, and causality library
+Project-URL: Homepage, https://github.com/antonbarr-data/dqt
+Project-URL: Repository, https://github.com/antonbarr-data/dqt
+Author-email: Anton Barr <antonbar@gmail.com>
+License: MIT
+Keywords: anomaly-detection,causal-inference,causality,data-drift,data-quality,observability
+Requires-Python: >=3.12
+Requires-Dist: diptest>=0.6
+Requires-Dist: duckdb>=0.9
+Requires-Dist: ibis-framework>=9.0
+Requires-Dist: jsonschema>=4.22
+Requires-Dist: numpy>=1.26
+Requires-Dist: pandas>=2.2
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pyod>=1.1
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: river>=0.21
+Requires-Dist: scikit-learn>=1.5
+Requires-Dist: scipy>=1.13
+Requires-Dist: statsmodels>=0.14
+Requires-Dist: structlog>=24.0
+Provides-Extra: causal
+Requires-Dist: causal-learn>=0.1; extra == 'causal'
+Requires-Dist: dowhy>=0.11; extra == 'causal'
+Requires-Dist: tigramite>=0.7; extra == 'causal'
+Provides-Extra: deep
+Requires-Dist: pyod[deep]>=1.1; extra == 'deep'
+Requires-Dist: torch>=2.3; extra == 'deep'
+Provides-Extra: explain
+Requires-Dist: pgmpy>=0.1; extra == 'explain'
+Requires-Dist: shap>=0.45; extra == 'explain'
+Provides-Extra: files
+Requires-Dist: openpyxl>=3.0; extra == 'files'
+Requires-Dist: pyarrow>=14.0; extra == 'files'
+Provides-Extra: forecast
+Requires-Dist: prophet>=1.1; extra == 'forecast'
+Requires-Dist: stumpy>=1.4; extra == 'forecast'
+Provides-Extra: postgres
+Requires-Dist: ibis-framework[postgres]>=9.0; extra == 'postgres'
+Requires-Dist: psycopg2-binary>=2.9; extra == 'postgres'
+Provides-Extra: reports
+Requires-Dist: matplotlib>=3.8; extra == 'reports'
+Description-Content-Type: text/markdown
+
+# dqtlib
+
+**Open-source data quality, lineage, semantic layer & causality — for dbt, warehouses and data lakes.**
+
+pip-installable Python library for watching dbt-built warehouses and any SQL warehouse for statistical drift, anomalies, silent regressions, and explaining *why* metrics moved.
+
+```bash
+pip install dqtlib
+```
+
+The import name is `dqt`:
+
+```python
+from dqt import Check, Runner, MemoryStore
+```
+
+Full documentation and examples: https://github.com/antonbarr-data/dqt

dqtlib-0.1.0/README.md

@@ -0,0 +1,17 @@
+# dqtlib
+
+**Open-source data quality, lineage, semantic layer & causality — for dbt, warehouses and data lakes.**
+
+pip-installable Python library for watching dbt-built warehouses and any SQL warehouse for statistical drift, anomalies, silent regressions, and explaining *why* metrics moved.
+
+```bash
+pip install dqtlib
+```
+
+The import name is `dqt`:
+
+```python
+from dqt import Check, Runner, MemoryStore
+```
+
+Full documentation and examples: https://github.com/antonbarr-data/dqt

dqtlib-0.1.0/pyproject.toml

@@ -0,0 +1,44 @@
+[project]
+name = "dqtlib"
+version = "0.1.0"
+description = "Data quality, observability, semantic, and causality library"
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "MIT" }
+keywords = ["data-quality", "observability", "anomaly-detection", "causality", "data-drift", "causal-inference"]
+authors = [
+    { name = "Anton Barr", email = "antonbar@gmail.com" },
+]
+urls = { Homepage = "https://github.com/antonbarr-data/dqt", Repository = "https://github.com/antonbarr-data/dqt" }
+dependencies = [
+    "numpy>=1.26",
+    "scipy>=1.13",
+    "pandas>=2.2",
+    "statsmodels>=0.14",
+    "scikit-learn>=1.5",
+    "pyod>=1.1",
+    "diptest>=0.6",
+    "river>=0.21",
+    "structlog>=24.0",
+    "pydantic>=2.7",
+    "ibis-framework>=9.0",
+    "duckdb>=0.9",
+    "jsonschema>=4.22",
+    "pyyaml>=6.0",
+]
+
+[project.optional-dependencies]
+postgres = ["psycopg2-binary>=2.9", "ibis-framework[postgres]>=9.0"]
+causal   = ["tigramite>=0.7", "dowhy>=0.11", "causal-learn>=0.1"]
+forecast = ["prophet>=1.1", "stumpy>=1.4"]
+deep     = ["torch>=2.3", "pyod[deep]>=1.1"]
+explain  = ["shap>=0.45", "pgmpy>=0.1"]
+files    = ["openpyxl>=3.0", "pyarrow>=14.0"]
+reports  = ["matplotlib>=3.8"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dqt"]

dqtlib-0.1.0/src/dqt/__init__.py

@@ -0,0 +1,49 @@
+# packages/dqt/src/dqt/__init__.py
+"""dqt — open-source data quality, observability, and causality library."""
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from dqt.algorithms._base import (
+    BaseAggregateDetector,
+    BaseDetector,
+    DetectorResult,
+    Verdict,
+    compute_verdict,
+)
+from dqt.adapters._protocol import AggExpr, ColumnMeta, HealthCheckResult, WarehouseAdapter
+from dqt.store._protocol import Incident, ResultsStore, RunResult
+from dqt.store.memory import MemoryStore
+from dqt.checks.models import BaselineConfig, Check, CheckFilter, CheckScope
+from dqt.runner.runner import Runner
+
+# Import all detector groups to trigger @registry.register side effects
+import dqt.algorithms.basic           # noqa: F401
+import dqt.algorithms.schema          # noqa: F401
+import dqt.algorithms.referential     # noqa: F401
+import dqt.algorithms.drift           # noqa: F401
+import dqt.algorithms.outliers_uni    # noqa: F401
+import dqt.algorithms.outliers_multi  # noqa: F401
+import dqt.algorithms.timeseries      # noqa: F401
+
+__all__ = [
+    "__version__",
+    "Verdict",
+    "DetectorResult",
+    "BaseDetector",
+    "BaseAggregateDetector",
+    "compute_verdict",
+    "AggExpr",
+    "ColumnMeta",
+    "HealthCheckResult",
+    "WarehouseAdapter",
+    "ResultsStore",
+    "RunResult",
+    "Incident",
+    "MemoryStore",
+    "Check",
+    "CheckScope",
+    "CheckFilter",
+    "BaselineConfig",
+    "Runner",
+]

dqtlib-0.1.0/src/dqt/adapters/_protocol.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal, Protocol, runtime_checkable
+
+import pandas as pd
+
+
+@dataclass
+class AggExpr:
+    name: str
+    sql: str
+
+
+@dataclass
+class HealthCheckStep:
+    name: str
+    status: Literal["pass", "fail", "skip"]
+    latency_ms: float
+    detail: str
+
+
+@dataclass
+class HealthCheckResult:
+    steps: list[HealthCheckStep] = field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        return all(s.status in ("pass", "skip") for s in self.steps)
+
+
+@dataclass
+class ColumnMeta:
+    name: str
+    data_type: str
+    nullable: bool
+    position: int
+
+
+@runtime_checkable
+class WarehouseAdapter(Protocol):
+    def health_check(self) -> HealthCheckResult: ...
+    def sample(self, schema: str, table: str, n: int = 100_000) -> pd.DataFrame: ...
+    def aggregate(self, schema: str, table: str, exprs: list[AggExpr]) -> dict[str, object]: ...
+    def describe_columns(self, schema: str, table: str) -> list[ColumnMeta]: ...
+    def list_schemas(self) -> list[str]: ...
+    def list_tables(self, schema: str) -> list[str]: ...

dqtlib-0.1.0/src/dqt/adapters/local/__init__.py

@@ -0,0 +1,3 @@
+from dqt.adapters.local.adapter import LocalFileAdapter
+
+__all__ = ["LocalFileAdapter"]

dqtlib-0.1.0/src/dqt/adapters/local/adapter.py

@@ -0,0 +1,118 @@
+# Ref: https://duckdb.org/docs/api/python/overview — used for SQL aggregations on DataFrames
+from __future__ import annotations
+
+import pathlib
+import time
+from typing import Any
+
+import pandas as pd
+
+from dqt.adapters._protocol import (
+    AggExpr,
+    ColumnMeta,
+    HealthCheckResult,
+    HealthCheckStep,
+)
+from dqt.utils.logging import get_logger
+
+_log = get_logger(__name__)
+
+_READERS: dict[str, Any] = {
+    ".csv":     lambda p: pd.read_csv(p),
+    ".tsv":     lambda p: pd.read_csv(p, sep="\t"),
+    ".xlsx":    lambda p: pd.read_excel(p),
+    ".xls":     lambda p: pd.read_excel(p),
+    ".parquet": lambda p: pd.read_parquet(p),
+    ".json":    lambda p: pd.read_json(p),
+    ".jsonl":   lambda p: pd.read_json(p, lines=True),
+    ".ndjson":  lambda p: pd.read_json(p, lines=True),
+    ".feather": lambda p: pd.read_feather(p),
+    ".arrow":   lambda p: pd.read_feather(p),
+}
+
+_HEALTH_STEPS = ("readable", "parseable", "columns", "sample_read", "row_count")
+
+
+class LocalFileAdapter:
+    """Reads a local file and exposes it as a single-table WarehouseAdapter."""
+
+    def __init__(self, path: str | pathlib.Path) -> None:
+        self._path = pathlib.Path(path)
+        self._suffix = self._path.suffix.lower()
+        if self._suffix not in _READERS:
+            supported = ", ".join(sorted(_READERS))
+            raise ValueError(f"Unsupported format '{self._suffix}'. Supported: {supported}")
+        self._table_name = self._path.stem
+
+    def _read(self) -> pd.DataFrame:
+        return _READERS[self._suffix](self._path)
+
+    def health_check(self) -> HealthCheckResult:
+        steps: list[HealthCheckStep] = []
+
+        t0 = time.perf_counter()
+        if not self._path.exists():
+            steps.append(HealthCheckStep("file_exists", "fail", 0.0, f"not found: {self._path}"))
+            for name in _HEALTH_STEPS:
+                steps.append(HealthCheckStep(name, "skip", 0.0, "skipped"))
+            return HealthCheckResult(steps=steps)
+        steps.append(HealthCheckStep("file_exists", "pass", (time.perf_counter() - t0) * 1000, str(self._path)))
+
+        t0 = time.perf_counter()
+        try:
+            self._path.read_bytes()[:1024]
+            steps.append(HealthCheckStep("readable", "pass", (time.perf_counter() - t0) * 1000, "ok"))
+        except Exception as exc:
+            steps.append(HealthCheckStep("readable", "fail", 0.0, str(exc)))
+            for name in ("parseable", "columns", "sample_read", "row_count"):
+                steps.append(HealthCheckStep(name, "skip", 0.0, "skipped"))
+            return HealthCheckResult(steps=steps)
+
+        t0 = time.perf_counter()
+        try:
+            df = self._read()
+        except Exception as exc:
+            steps.append(HealthCheckStep("parseable", "fail", 0.0, str(exc)))
+            for name in ("columns", "sample_read", "row_count"):
+                steps.append(HealthCheckStep(name, "skip", 0.0, "skipped"))
+            return HealthCheckResult(steps=steps)
+        steps.append(HealthCheckStep("parseable", "pass", (time.perf_counter() - t0) * 1000, f"{len(df.columns)} columns"))
+
+        steps.append(HealthCheckStep("columns", "pass", 0.0, str(list(df.columns)[:5])))
+        steps.append(HealthCheckStep("sample_read", "pass", 0.0, "ok"))
+        steps.append(HealthCheckStep("row_count", "pass", 0.0, f"{len(df)} rows"))
+        return HealthCheckResult(steps=steps)
+
+    def list_schemas(self) -> list[str]:
+        return ["default"]
+
+    def list_tables(self, schema: str) -> list[str]:
+        return [self._table_name]
+
+    def describe_columns(self, schema: str, table: str) -> list[ColumnMeta]:
+        df = self._read()
+        return [
+            ColumnMeta(
+                name=col,
+                data_type=str(df[col].dtype),
+                nullable=bool(df[col].isna().any()),
+                position=i + 1,
+            )
+            for i, col in enumerate(df.columns)
+        ]
+
+    def sample(self, schema: str, table: str, n: int = 100_000) -> pd.DataFrame:
+        df = self._read()
+        if len(df) <= n:
+            return df.reset_index(drop=True)
+        return df.sample(n=n, random_state=42).reset_index(drop=True)
+
+    def aggregate(self, schema: str, table: str, exprs: list[AggExpr]) -> dict[str, Any]:
+        import duckdb
+        df = self._read()
+        con = duckdb.connect()
+        con.register("_data", df)
+        cols = ", ".join(f"{e.sql} AS {e.name}" for e in exprs)
+        row = con.execute(f"SELECT {cols} FROM _data").fetchone()  # noqa: S608
+        con.close()
+        return dict(zip([e.name for e in exprs], row))

dqtlib-0.1.0/src/dqt/adapters/postgres/__init__.py

@@ -0,0 +1,4 @@
+from dqt.adapters.postgres.adapter import PostgresAdapter
+from dqt.adapters.postgres.config import PostgresConfig
+
+__all__ = ["PostgresAdapter", "PostgresConfig"]

dqtlib-0.1.0/src/dqt/adapters/postgres/adapter.py

@@ -0,0 +1,154 @@
+# PostgresAdapter wraps SQLAlchemy for all warehouse operations.
+# Sampling uses LIMIT for portable random rows; TABLESAMPLE BERNOULLI available as an option.
+from __future__ import annotations
+
+import datetime
+import time
+from typing import Any
+
+import pandas as pd
+import sqlalchemy as sa
+
+from dqt.adapters._protocol import (
+    AggExpr,
+    ColumnMeta,
+    HealthCheckResult,
+    HealthCheckStep,
+)
+from dqt.utils.logging import get_logger
+
+_log = get_logger(__name__)
+
+
+class PostgresAdapter:
+    def __init__(self, conn_str: str) -> None:
+        self._conn_str = conn_str
+        self._engine = sa.create_engine(
+            conn_str,
+            pool_pre_ping=True,
+            execution_options={"isolation_level": "READ COMMITTED"},
+        )
+
+    def health_check(self) -> HealthCheckResult:
+        steps: list[HealthCheckStep] = []
+        steps.append(self._step_tcp())
+        if steps[-1].status == "fail":
+            for name in ("auth", "info_schema", "sample_select", "latency_probe", "clock_skew"):
+                steps.append(HealthCheckStep(name=name, status="skip", latency_ms=0.0, detail="skipped"))
+            return HealthCheckResult(steps=steps)
+        steps.append(self._step_auth())
+        steps.append(self._step_info_schema())
+        steps.append(self._step_sample_select())
+        steps.append(self._step_latency())
+        steps.append(self._step_clock_skew())
+        return HealthCheckResult(steps=steps)
+
+    def _step_tcp(self) -> HealthCheckStep:
+        t0 = time.perf_counter()
+        try:
+            with self._engine.connect() as conn:
+                conn.execute(sa.text("SELECT 1"))
+            return HealthCheckStep("tcp_reach", "pass", (time.perf_counter() - t0) * 1000, "ok")
+        except Exception as exc:
+            return HealthCheckStep("tcp_reach", "fail", 0.0, str(exc))
+
+    def _step_auth(self) -> HealthCheckStep:
+        t0 = time.perf_counter()
+        try:
+            with self._engine.connect() as conn:
+                user = conn.execute(sa.text("SELECT current_user")).scalar()
+            return HealthCheckStep("auth", "pass", (time.perf_counter() - t0) * 1000, f"user={user}")
+        except Exception as exc:
+            return HealthCheckStep("auth", "fail", 0.0, str(exc))
+
+    def _step_info_schema(self) -> HealthCheckStep:
+        t0 = time.perf_counter()
+        try:
+            with self._engine.connect() as conn:
+                conn.execute(sa.text(
+                    "SELECT COUNT(*) FROM information_schema.tables "
+                    "WHERE table_schema NOT IN ('pg_catalog','information_schema')"
+                )).scalar()
+            return HealthCheckStep("info_schema", "pass", (time.perf_counter() - t0) * 1000, "readable")
+        except Exception as exc:
+            return HealthCheckStep("info_schema", "fail", 0.0, str(exc))
+
+    def _step_sample_select(self) -> HealthCheckStep:
+        t0 = time.perf_counter()
+        try:
+            with self._engine.connect() as conn:
+                conn.execute(sa.text(
+                    "SELECT table_name FROM information_schema.tables "
+                    "WHERE table_schema NOT IN ('pg_catalog','information_schema') LIMIT 1"
+                )).fetchone()
+            return HealthCheckStep("sample_select", "pass", (time.perf_counter() - t0) * 1000, "ok")
+        except Exception as exc:
+            return HealthCheckStep("sample_select", "fail", 0.0, str(exc))
+
+    def _step_latency(self) -> HealthCheckStep:
+        t0 = time.perf_counter()
+        try:
+            with self._engine.connect() as conn:
+                conn.execute(sa.text("SELECT 1"))
+            latency = (time.perf_counter() - t0) * 1000
+            return HealthCheckStep("latency_probe", "pass", latency, f"{latency:.1f}ms")
+        except Exception as exc:
+            return HealthCheckStep("latency_probe", "fail", 0.0, str(exc))
+
+    def _step_clock_skew(self) -> HealthCheckStep:
+        t0 = time.perf_counter()
+        try:
+            with self._engine.connect() as conn:
+                db_now = conn.execute(sa.text("SELECT NOW()")).scalar()
+            local_now = datetime.datetime.now(datetime.timezone.utc)
+            if db_now.tzinfo is None:
+                db_now = db_now.replace(tzinfo=datetime.timezone.utc)
+            skew_s = abs((db_now - local_now).total_seconds())
+            status = "pass" if skew_s < 60 else "fail"
+            return HealthCheckStep("clock_skew", status, (time.perf_counter() - t0) * 1000, f"skew={skew_s:.1f}s")
+        except Exception as exc:
+            return HealthCheckStep("clock_skew", "fail", 0.0, str(exc))
+
+    def list_schemas(self) -> list[str]:
+        with self._engine.connect() as conn:
+            rows = conn.execute(sa.text(
+                "SELECT DISTINCT table_schema FROM information_schema.tables "
+                "WHERE table_schema NOT IN ('pg_catalog','information_schema') ORDER BY 1"
+            )).fetchall()
+        return [r[0] for r in rows]
+
+    def list_tables(self, schema: str) -> list[str]:
+        with self._engine.connect() as conn:
+            rows = conn.execute(sa.text(
+                "SELECT table_name FROM information_schema.tables "
+                "WHERE table_schema = :schema ORDER BY 1"
+            ), {"schema": schema}).fetchall()
+        return [r[0] for r in rows]
+
+    def describe_columns(self, schema: str, table: str) -> list[ColumnMeta]:
+        with self._engine.connect() as conn:
+            rows = conn.execute(sa.text(
+                "SELECT column_name, data_type, is_nullable, ordinal_position "
+                "FROM information_schema.columns "
+                "WHERE table_schema = :schema AND table_name = :table "
+                "ORDER BY ordinal_position"
+            ), {"schema": schema, "table": table}).fetchall()
+        return [
+            ColumnMeta(name=r[0], data_type=r[1], nullable=(r[2] == "YES"), position=r[3])
+            for r in rows
+        ]
+
+    def sample(self, schema: str, table: str, n: int = 100_000) -> pd.DataFrame:
+        # Use ORDER BY random() to get a genuine random sample without TABLESAMPLE bias on small tables.
+        # schema/table are double-quoted identifiers, not user values in SQL context.
+        query = sa.text(f'SELECT * FROM "{schema}"."{table}" ORDER BY random() LIMIT :n')
+        with self._engine.connect() as conn:
+            return pd.read_sql(query, conn, params={"n": n})
+
+    def aggregate(self, schema: str, table: str, exprs: list[AggExpr]) -> dict[str, Any]:
+        cols = ", ".join(f"{e.sql} AS {e.name}" for e in exprs)
+        # schema/table are double-quoted identifiers; cols are built from AggExpr.sql (caller-controlled).
+        query = sa.text(f'SELECT {cols} FROM "{schema}"."{table}"')
+        with self._engine.connect() as conn:
+            row = conn.execute(query).fetchone()
+        return dict(zip([e.name for e in exprs], row))

dqtlib-0.1.0/src/dqt/adapters/postgres/config.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class PostgresConfig:
+    host: str = "localhost"
+    port: int = 5432
+    database: str = "postgres"
+    username: str = "postgres"
+    password: str = ""
+    ssl_mode: str = "prefer"
+
+    def to_conn_str(self) -> str:
+        return (
+            f"postgresql+psycopg2://{self.username}:{self.password}"
+            f"@{self.host}:{self.port}/{self.database}?sslmode={self.ssl_mode}"
+        )