qualis 0.3.1__py3-none-any.whl

qualis/__init__.py

@@ -0,0 +1,22 @@
+"""Qualis — Data quality framework that tells you WHAT failed."""
+
+from __future__ import annotations
+
+__version__ = "0.3.1"
+
+from qualis.domain.enums import DQDimension, RunStatus, Severity
+from qualis.domain.models import DatasetScore, Rule, Violation
+from qualis.ports.database import DatabasePort
+from qualis.ports.notifier import NotifierPort
+
+__all__ = [
+    "DQDimension",
+    "DatabasePort",
+    "DatasetScore",
+    "NotifierPort",
+    "Rule",
+    "RunStatus",
+    "Severity",
+    "Violation",
+    "__version__",
+]

qualis/adapters/console.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+if TYPE_CHECKING:
+    from qualis.domain.models import DatasetScore
+    from qualis.engine.diff import ScoreDiff
+
+
+def print_score(score: DatasetScore, console: Console | None = None) -> None:
+    """Print a rich-formatted terminal scorecard for *score*.
+
+    Parameters
+    ----------
+    score:
+        The ``DatasetScore`` to render.
+    console:
+        Optional ``rich.console.Console`` instance. A default ``Console``
+        (stdout, auto-detect colour) is created when not provided.
+    """
+    c = console or Console()
+
+    pct = int(score.aggregate_score * 100)
+    if pct >= 90:
+        color, status = "green", "PASSING"
+    elif pct >= 70:
+        color, status = "yellow", "WARNING"
+    else:
+        color, status = "red", "FAILING"
+
+    header = (
+        f"[bold magenta]QUALIS[/]  ·  Data Quality Report\n\n"
+        f"        Score:  [bold {color}]{pct} / 100[/]\n"
+        f"        Status: [{color}]● {status}[/]"
+    )
+    c.print(Panel(header, border_style="bright_black", padding=(1, 4)))
+
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("Dimension", style="cyan", min_width=16)
+    table.add_column("Score", justify="right", min_width=8)
+    table.add_column("Checks", justify="right", min_width=10)
+    table.add_column("", min_width=3)
+
+    for ds in score.dimension_scores:
+        pct_dim = int(ds.score * 100)
+        if ds.score >= 0.9:
+            indicator = "[green]✓[/]"
+        elif ds.score >= 0.7:
+            indicator = "[yellow]⚠[/]"
+        else:
+            indicator = "[red]✗[/]"
+        table.add_row(
+            ds.dimension.value.capitalize(),
+            f"{pct_dim}%",
+            f"{ds.passed}/{ds.total_checks}",
+            indicator,
+        )
+
+    c.print(table)
+
+    if score.total_violations > 0:
+        c.print(
+            f"\n[bold]{score.total_violations} violation(s)[/] "
+            f"({score.critical_violations} critical)"
+        )
+
+
+def print_diff(diff: ScoreDiff, console: Console | None = None) -> None:
+    """Print a rich-formatted terminal diff between two DatasetScore snapshots.
+
+    Parameters
+    ----------
+    diff:
+        The ``ScoreDiff`` to render.
+    console:
+        Optional ``rich.console.Console`` instance. A default ``Console``
+        (stdout, auto-detect colour) is created when not provided.
+    """
+    c = console or Console()
+
+    before_pct = int(diff.before_aggregate * 100)
+    after_pct = int(diff.after_aggregate * 100)
+    delta_pct = after_pct - before_pct
+
+    if delta_pct > 0:
+        delta_str = f"[green]↑ +{delta_pct}[/]"
+    elif delta_pct < 0:
+        delta_str = f"[red]↓ {delta_pct}[/]"
+    else:
+        delta_str = "[dim]—[/]"
+
+    header = (
+        "[bold magenta]QUALIS[/]  ·  Score Diff\n\n"
+        f"        Before:  [bold]{before_pct} / 100[/]\n"
+        f"        After:   [bold]{after_pct} / 100[/]\n"
+        f"        Delta:   {delta_str}"
+    )
+    c.print(Panel(header, border_style="bright_black", padding=(1, 4)))
+
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("Dimension", style="cyan", min_width=16)
+    table.add_column("Before", justify="right", min_width=8)
+    table.add_column("After", justify="right", min_width=8)
+    table.add_column("Delta", justify="right", min_width=8)
+    table.add_column("", min_width=3)
+
+    for dd in diff.dimension_deltas:
+        before_val = f"{int(dd.before_score * 100)}%" if dd.before_score is not None else "—"
+        after_val = f"{int(dd.after_score * 100)}%" if dd.after_score is not None else "—"
+
+        dim_delta = int(dd.delta * 100)
+        if dim_delta > 0:
+            delta_cell = f"[green]↑ +{dim_delta}[/]"
+            indicator = "[green]✓[/]"
+        elif dim_delta < 0:
+            delta_cell = f"[red]↓ {dim_delta}[/]"
+            indicator = "[red]✗[/]"
+        else:
+            delta_cell = "[dim]—[/]"
+            indicator = "[green]✓[/]"
+
+        table.add_row(
+            dd.dimension.value.capitalize(),
+            before_val,
+            after_val,
+            delta_cell,
+            indicator,
+        )
+
+    c.print(table)
+
+    viol_delta = diff.after_violations - diff.before_violations
+    if viol_delta != 0:
+        sign = "+" if viol_delta > 0 else ""
+        color = "red" if viol_delta > 0 else "green"
+        c.print(
+            f"\nViolations: [bold]{diff.before_violations}[/] → "
+            f"[bold]{diff.after_violations}[/]  "
+            f"([{color}]{sign}{viol_delta}[/])"
+        )

qualis/adapters/duckdb/adapter.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import duckdb
+
+from qualis.adapters.duckdb.sql_templates import (
+    BETWEEN_SQL,
+    IN_SET_SQL,
+    NOT_NEGATIVE_SQL,
+    NOT_NULL_SQL,
+    REFERENCE_LOOKUP_SQL,
+    REGEX_SQL,
+    ROW_COUNT_SQL,
+    TABLE_EXISTS_SQL,
+    UNIQUE_SQL,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
+def _qualified(schema: str, table: str) -> str:
+    """Return a qualified table reference for use inside SQL strings."""
+    if schema:
+        return f'"{schema}"."{table}"'
+    return f'"{table}"'
+
+
+class DuckDBAdapter:
+    """DuckDB-backed implementation of ``DatabasePort``.
+
+    Parameters
+    ----------
+    database:
+        Path to a persistent DuckDB file, or ``":memory:"`` (default) for an
+        in-process, session-only database.
+    """
+
+    def __init__(self, database: str = ":memory:") -> None:
+        self._con = duckdb.connect(database)
+
+    # ------------------------------------------------------------------
+    # Registration helpers
+    # ------------------------------------------------------------------
+
+    def register_csv(self, table_name: str, path: str) -> None:
+        """Create a table from a CSV file using DuckDB's auto-detection."""
+        self._con.execute(
+            f'CREATE TABLE "{table_name}" AS SELECT * FROM read_csv_auto(\'{path}\')'
+        )
+
+    def register_parquet(self, table_name: str, path: str) -> None:
+        """Create a table from a Parquet file."""
+        self._con.execute(
+            f'CREATE TABLE "{table_name}" AS SELECT * FROM read_parquet(\'{path}\')'
+        )
+
+    # ------------------------------------------------------------------
+    # DatabasePort implementation
+    # ------------------------------------------------------------------
+
+    def query(self, sql: str, params: dict[str, Any] | None = None) -> list[dict[str, Any]]:
+        """Execute *sql* and return a list of dicts keyed by column name."""
+        rel = self._con.execute(sql)
+        columns = [desc[0] for desc in rel.description or []]
+        return [dict(zip(columns, row, strict=True)) for row in rel.fetchall()]
+
+    def execute(self, sql: str, params: dict[str, Any] | None = None) -> int:
+        """Execute a DML statement; return the number of rows affected."""
+        self._con.execute(sql)
+        # DuckDB does not expose rowcount directly; return 0 as a safe default
+        return 0
+
+    def stream(
+        self,
+        sql: str,
+        params: dict[str, Any] | None = None,
+        chunk_size: int = 10_000,
+    ) -> Iterator[list[dict[str, Any]]]:
+        """Stream query results in chunks of *chunk_size* rows."""
+        rel = self._con.execute(sql)
+        columns = [desc[0] for desc in rel.description or []]
+        while True:
+            rows = rel.fetchmany(chunk_size)
+            if not rows:
+                break
+            yield [dict(zip(columns, row, strict=True)) for row in rows]
+
+    def table_exists(self, schema: str, table: str) -> bool:
+        """Return True when the named table is registered in the database."""
+        result = self._con.execute(
+            TABLE_EXISTS_SQL.format(table=table)
+        ).fetchone()
+        return bool(result and result[0] > 0)
+
+    # ------------------------------------------------------------------
+    # Check methods
+    # ------------------------------------------------------------------
+
+    def check_not_null(self, schema: str, table: str, column: str) -> dict[str, int]:
+        table_ref = _qualified(schema, table)
+        sql = NOT_NULL_SQL.format(column=column, table=table_ref)
+        row = self._con.execute(sql).fetchone()
+        null_count, total_count = (row[0], row[1]) if row else (0, 0)
+        return {"null_count": int(null_count), "total_count": int(total_count)}
+
+    def check_unique(self, schema: str, table: str, column: str) -> dict[str, int]:
+        table_ref = _qualified(schema, table)
+        sql = UNIQUE_SQL.format(column=column, table=table_ref)
+        row = self._con.execute(sql).fetchone()
+        if row is None:
+            # No duplicate groups at all — zero duplicates
+            total_row = self._con.execute(
+                f"SELECT COUNT(*) FROM {table_ref}"
+            ).fetchone()
+            total = int(total_row[0]) if total_row else 0
+            return {"duplicate_count": 0, "total_count": total}
+        dup_count = int(row[0])
+        # Re-query total so it is accurate regardless of UNIQUE_SQL shape
+        total_row = self._con.execute(
+            f"SELECT COUNT(*) FROM {table_ref}"
+        ).fetchone()
+        total = int(total_row[0]) if total_row else 0
+        return {"duplicate_count": dup_count, "total_count": total}
+
+    def check_between(
+        self,
+        schema: str,
+        table: str,
+        column: str,
+        min_val: str,
+        max_val: str,
+    ) -> dict[str, int]:
+        table_ref = _qualified(schema, table)
+        sql = BETWEEN_SQL.format(
+            column=column, table=table_ref, min_val=min_val, max_val=max_val
+        )
+        row = self._con.execute(sql).fetchone()
+        out_of_range, total = (row[0], row[1]) if row else (0, 0)
+        return {"out_of_range_count": int(out_of_range), "total_count": int(total)}
+
+    def check_regex(
+        self,
+        schema: str,
+        table: str,
+        column: str,
+        pattern: str,
+    ) -> dict[str, int]:
+        table_ref = _qualified(schema, table)
+        sql = REGEX_SQL.format(column=column, table=table_ref, pattern=pattern)
+        row = self._con.execute(sql).fetchone()
+        non_matching, total = (row[0], row[1]) if row else (0, 0)
+        return {"non_matching_count": int(non_matching), "total_count": int(total)}
+
+    def check_in_set(
+        self,
+        schema: str,
+        table: str,
+        column: str,
+        values: list[str],
+    ) -> dict[str, int]:
+        table_ref = _qualified(schema, table)
+        escaped = [v.replace("'", "''") for v in values]
+        value_list = ", ".join(f"'{v}'" for v in escaped)
+        sql = IN_SET_SQL.format(column=column, table=table_ref, value_list=value_list)
+        row = self._con.execute(sql).fetchone()
+        invalid, total = (row[0], row[1]) if row else (0, 0)
+        return {"invalid_count": int(invalid), "total_count": int(total)}
+
+    def check_row_count(self, schema: str, table: str) -> dict[str, int]:
+        table_ref = _qualified(schema, table)
+        sql = ROW_COUNT_SQL.format(table=table_ref)
+        row = self._con.execute(sql).fetchone()
+        count = int(row[0]) if row else 0
+        return {"row_count": count}
+
+    def check_not_negative(self, schema: str, table: str, column: str) -> dict[str, int]:
+        table_ref = _qualified(schema, table)
+        sql = NOT_NEGATIVE_SQL.format(column=column, table=table_ref)
+        row = self._con.execute(sql).fetchone()
+        negative, total = (row[0], row[1]) if row else (0, 0)
+        return {"negative_count": int(negative), "total_count": int(total)}
+
+    def check_reference_lookup(
+        self,
+        schema: str,
+        table: str,
+        column: str,
+        valid_values: list[str],
+    ) -> dict[str, int]:
+        table_ref = _qualified(schema, table)
+        escaped = [v.replace("'", "''") for v in valid_values]
+        value_list = ", ".join(f"'{v}'" for v in escaped) or "NULL"
+        sql = REFERENCE_LOOKUP_SQL.format(
+            column=column, table=table_ref, value_list=value_list,
+        )
+        row = self._con.execute(sql).fetchone()
+        invalid, total = (row[0], row[1]) if row else (0, 0)
+        return {"invalid_count": int(invalid), "total_count": int(total)}
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def close(self) -> None:
+        """Close the underlying DuckDB connection."""
+        self._con.close()

qualis/adapters/duckdb/sql_templates.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+# ---------------------------------------------------------------------------
+# DuckDB SQL templates
+# ---------------------------------------------------------------------------
+# All column references use double-quoted identifiers to handle names that
+# are reserved keywords or contain special characters.
+
+NOT_NULL_SQL = (
+    'SELECT COUNT(*) FILTER (WHERE "{column}" IS NULL) AS null_count, '
+    "COUNT(*) AS total_count "
+    "FROM {table}"
+)
+
+UNIQUE_SQL = (
+    "SELECT COUNT(*) AS duplicate_count, COUNT(*) + "
+    "(SELECT COUNT(*) FROM {table}) - COUNT(*) AS total_count "
+    "FROM ("
+    '  SELECT "{column}" FROM {table} '
+    '  GROUP BY "{column}" HAVING COUNT(*) > 1'
+    ") dup"
+)
+
+BETWEEN_SQL = (
+    "SELECT "
+    "  COUNT(*) FILTER ("
+    '    WHERE CAST("{column}" AS VARCHAR) < \'{min_val}\' '
+    '    OR CAST("{column}" AS VARCHAR) > \'{max_val}\''
+    "  ) AS out_of_range_count, "
+    "  COUNT(*) AS total_count "
+    "FROM {table}"
+)
+
+REGEX_SQL = (
+    "SELECT "
+    "  COUNT(*) FILTER ("
+    '    WHERE "{column}" IS NULL '
+    '    OR NOT regexp_matches(CAST("{column}" AS VARCHAR), \'{pattern}\')'
+    "  ) AS non_matching_count, "
+    "  COUNT(*) AS total_count "
+    "FROM {table}"
+)
+
+IN_SET_SQL = (
+    "SELECT "
+    "  COUNT(*) FILTER ("
+    '    WHERE "{column}" IS NULL '
+    '    OR CAST("{column}" AS VARCHAR) NOT IN ({value_list})'
+    "  ) AS invalid_count, "
+    "  COUNT(*) AS total_count "
+    "FROM {table}"
+)
+
+ROW_COUNT_SQL = "SELECT COUNT(*) AS row_count FROM {table}"
+
+NOT_NEGATIVE_SQL = (
+    "SELECT "
+    '  COUNT(*) FILTER (WHERE "{column}" IS NOT NULL AND "{column}" < 0) '
+    "    AS negative_count, "
+    "  COUNT(*) AS total_count "
+    "FROM {table}"
+)
+
+REFERENCE_LOOKUP_SQL = (
+    "SELECT "
+    "  COUNT(*) FILTER ("
+    '    WHERE "{column}" IS NOT NULL '
+    '    AND CAST("{column}" AS VARCHAR) NOT IN ({value_list})'
+    "  ) AS invalid_count, "
+    "  COUNT(*) AS total_count "
+    "FROM {table}"
+)
+
+TABLE_EXISTS_SQL = (
+    "SELECT COUNT(*) AS cnt FROM information_schema.tables "
+    "WHERE table_name = '{table}'"
+)

qualis/adapters/in_memory/adapter.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
+class InMemoryAdapter:
+    def __init__(self) -> None:
+        self._tables: dict[str, list[dict[str, Any]]] = {}
+
+    def add_table(self, schema: str, table: str, rows: list[dict[str, Any]]) -> None:
+        self._tables[f"{schema}.{table}"] = list(rows)
+
+    def _get_rows(self, schema: str, table: str) -> list[dict[str, Any]]:
+        key = f"{schema}.{table}"
+        if key not in self._tables:
+            raise ValueError(f"Table {key} not found")
+        return self._tables[key]
+
+    def query(self, sql: str, params: dict[str, Any] | None = None) -> list[dict[str, Any]]:
+        table_key = self._extract_table_from_sql(sql)
+        if table_key and table_key in self._tables:
+            return list(self._tables[table_key])
+        return []
+
+    def execute(self, sql: str, params: dict[str, Any] | None = None) -> int:
+        return 0
+
+    def stream(
+        self,
+        sql: str,
+        params: dict[str, Any] | None = None,
+        chunk_size: int = 10_000,
+    ) -> Iterator[list[dict[str, Any]]]:
+        table_key = self._extract_table_from_sql(sql)
+        if table_key and table_key in self._tables:
+            rows = self._tables[table_key]
+            for i in range(0, len(rows), chunk_size):
+                yield rows[i : i + chunk_size]
+
+    def table_exists(self, schema: str, table: str) -> bool:
+        return f"{schema}.{table}" in self._tables
+
+    def check_not_null(self, schema: str, table: str, column: str) -> dict[str, int]:
+        rows = self._get_rows(schema, table)
+        null_count = sum(1 for r in rows if r.get(column) is None)
+        return {"null_count": null_count, "total_count": len(rows)}
+
+    def check_unique(self, schema: str, table: str, column: str) -> dict[str, int]:
+        rows = self._get_rows(schema, table)
+        values = [r.get(column) for r in rows if r.get(column) is not None]
+        duplicate_count = len(values) - len(set(values))
+        return {"duplicate_count": duplicate_count, "total_count": len(rows)}
+
+    def check_between(
+        self,
+        schema: str,
+        table: str,
+        column: str,
+        min_val: str,
+        max_val: str,
+    ) -> dict[str, int]:
+        rows = self._get_rows(schema, table)
+        out_count = 0
+        checked = 0
+        for r in rows:
+            val = r.get(column)
+            if val is None:
+                continue
+            checked += 1
+            if str(val) < min_val or str(val) > max_val:
+                out_count += 1
+        return {"out_of_range_count": out_count, "total_count": len(rows), "checked": checked}
+
+    def check_regex(
+        self,
+        schema: str,
+        table: str,
+        column: str,
+        pattern: str,
+    ) -> dict[str, int]:
+        rows = self._get_rows(schema, table)
+        compiled = re.compile(pattern)
+        non_matching = sum(
+            1
+            for r in rows
+            if r.get(column) is None or not compiled.match(str(r.get(column)))
+        )
+        return {"non_matching_count": non_matching, "total_count": len(rows)}
+
+    def check_in_set(
+        self,
+        schema: str,
+        table: str,
+        column: str,
+        values: list[str],
+    ) -> dict[str, int]:
+        rows = self._get_rows(schema, table)
+        allowed = set(values)
+        invalid_count = sum(
+            1
+            for r in rows
+            if r.get(column) is None or str(r.get(column)) not in allowed
+        )
+        return {"invalid_count": invalid_count, "total_count": len(rows)}
+
+    def check_row_count(self, schema: str, table: str) -> dict[str, int]:
+        rows = self._get_rows(schema, table)
+        return {"row_count": len(rows)}
+
+    def check_not_negative(
+        self,
+        schema: str,
+        table: str,
+        column: str,
+    ) -> dict[str, int]:
+        rows = self._get_rows(schema, table)
+        negative_count = sum(
+            1
+            for r in rows
+            if r.get(column) is not None and (r[column]) < 0
+        )
+        return {"negative_count": negative_count, "total_count": len(rows)}
+
+    def check_reference_lookup(
+        self,
+        schema: str,
+        table: str,
+        column: str,
+        valid_values: list[str],
+    ) -> dict[str, int]:
+        rows = self._get_rows(schema, table)
+        valid_set = set(valid_values)
+        invalid_count = sum(
+            1
+            for r in rows
+            if r.get(column) is not None and r.get(column) not in valid_set
+        )
+        return {"invalid_count": invalid_count, "total_count": len(rows)}
+
+    def _extract_table_from_sql(self, sql: str) -> str | None:
+        match = re.search(r"FROM\s+(\S+)", sql, re.IGNORECASE)
+        return match.group(1) if match else None

qualis/adapters/in_memory/reference_data.py

@@ -0,0 +1,25 @@
+"""In-memory ReferenceDataPort -- dict-backed, for testing and small datasets."""
+
+from __future__ import annotations
+
+
+class InMemoryReferenceData:
+    """ReferenceDataPort backed by a dict.
+
+    Use ``register(reference, key_column, values)`` to load data;
+    ``load_values(reference, key_column)`` to retrieve it.
+    """
+
+    def __init__(self) -> None:
+        self._data: dict[tuple[str, str], list[str]] = {}
+
+    def register(self, reference: str, key_column: str, values: list[str]) -> None:
+        self._data[(reference, key_column)] = list(values)
+
+    def load_values(self, reference: str, key_column: str) -> list[str]:
+        key = (reference, key_column)
+        if key not in self._data:
+            raise KeyError(
+                f"Reference {reference!r} with key column {key_column!r} not registered"
+            )
+        return list(self._data[key])