aice-data-quality 0.1.0__py3-none-any.whl

aice_data_quality-0.1.0.dist-info/METADATA

@@ -0,0 +1,79 @@
+Metadata-Version: 2.5
+Name: aice-data-quality
+Version: 0.1.0
+Summary: Data quality SDK: checks, one result contract, one destination
+Author-email: Adanian Labs <engineering@adanianlabs.io>
+License: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Python: <3.14,>=3.10
+Requires-Dist: pydantic>=2
+Requires-Dist: pyyaml>=6
+Provides-Extra: dbt
+Provides-Extra: dev
+Requires-Dist: httpx>=0.27; extra == 'dev'
+Requires-Dist: jsonschema[format]>=4.21; extra == 'dev'
+Requires-Dist: openpyxl>=3.1; extra == 'dev'
+Requires-Dist: opentelemetry-api>=1.20; extra == 'dev'
+Requires-Dist: opentelemetry-sdk>=1.20; extra == 'dev'
+Requires-Dist: pandas>=2; extra == 'dev'
+Requires-Dist: psycopg[binary]>=3.1; extra == 'dev'
+Requires-Dist: pytest>=8.3; extra == 'dev'
+Requires-Dist: sqlalchemy>=2; extra == 'dev'
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.1; extra == 'excel'
+Requires-Dist: pandas>=2; extra == 'excel'
+Provides-Extra: http
+Requires-Dist: httpx>=0.27; extra == 'http'
+Provides-Extra: otel
+Requires-Dist: opentelemetry-api>=1.20; extra == 'otel'
+Provides-Extra: pandas
+Requires-Dist: pandas>=2; extra == 'pandas'
+Provides-Extra: sql
+Requires-Dist: psycopg[binary]>=3.1; extra == 'sql'
+Requires-Dist: sqlalchemy>=2; extra == 'sql'
+Description-Content-Type: text/markdown
+
+# aice-data-quality (SDK)
+
+Check data, record the outcome in the result contract, send it to one destination.
+Import name: `dq`. Produces records of contract `1.0` ([`contract/`](contract/README.md)).
+
+```python
+from dq import QualityClient, Dataset, Severity, checks as c
+from dq.sources import DataFrameSource
+
+client = QualityClient(project="my-project", environment="production")   # or DQ_* env vars
+with client.run(pipeline="uploads") as run:
+    report = run.validate(DataFrameSource(df), dataset=Dataset("finance_uploads", "statement", "bronze"),
+                          owner="finance",
+                          checks=[c.not_null("amount", name="amount_set", severity=Severity.BLOCK)])
+if report.blocked:
+    raise UploadRejected(report.messages())
+```
+
+- Full guide: [`docs/user-guide.md`](docs/user-guide.md)
+- Runnable example: [`examples/dataframe_example.py`](examples/dataframe_example.py)
+- Specification (shared with the server): [`../README.md`](../README.md)
+
+## What belongs here, and what does not
+
+| In the SDK (runs inside each project) | In the server (`../server/`, never here) |
+|---|---|
+| Sources, checks, suites, privacy rules | Storing results: schema, upserts, retention |
+| Contract models, run outcome, dbt mapping | Authentication and token scopes |
+| Sinks: `LogSink`, `HttpSink` (the server's client), the local spool | Queries, history for `vs_previous`, dataset and check listings |
+| `SqlSource`, which reads the project's own data tables | Alert routing, UI |
+
+The SDK has no results database and no results DDL. `SqlSource` is the only code that talks to a database,
+and only to read data being checked.
+
+## Develop
+
+```bash
+docker compose up -d --wait      # sample Postgres on :55432 for SqlSource tests only
+uv sync --extra dev
+uv run pytest                    # SqlSource tests skip themselves if the database is down
+```

aice_data_quality-0.1.0.dist-info/RECORD

@@ -0,0 +1,39 @@
+dq/__init__.py,sha256=Qx3tGTck4D3P-TfdTJY9yv32it96KDpywlFSHT3hYnM,833
+dq/cli.py,sha256=w-Gee9tKob5FDZjyORKxA47u0Zift-Jq6Nd34jl4n5U,2498
+dq/client.py,sha256=NvcCrpGHG-AzPU3oCIEi1pdq56Yu69rY0XsiqW1lenk,13669
+dq/config.py,sha256=j-CK4w1SBYLCpYwJdAAh3DkmTmK3ZfuKzk6XXm8Szhs,2239
+dq/contract.py,sha256=JxiamYeJop8AZWGC3qWeucCDFbHNMlU8YGspdeo9SWM,6262
+dq/dataset.py,sha256=3_F38kWN1SSDdasac6ykp8oYzD1BU_eM86Tu_MpSlcM,932
+dq/errors.py,sha256=RkuA-YmnKu3GgbX7Nn643whZxguIErxxfRCRraS3lAg,915
+dq/history.py,sha256=jYvHkMwYGTGSdOMKYZnQSzsI00U8jPyPWr4UaI7w8nE,735
+dq/privacy.py,sha256=vGnJGL2ykjBy_iWKIYcqaVGWVxW2BBK7OjLQf7MzHGk,1651
+dq/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dq/suites.py,sha256=GntFKg74wkgNfYMJeEqwEcf3gY7XW-eLnc2E89qfPs0,6338
+dq/telemetry.py,sha256=YtKgEvk71yt6ZgCpQ-vzUl9Actb2lxtLNhL5DAB4EGg,1059
+dq/checks/__init__.py,sha256=B_MGFdT7uYLN8hMvMpUy86r9i4nT70xrcjZyC1K2-Xo,1144
+dq/checks/accepted_values.py,sha256=ZHvJM37hKob_kvrWucuCRCVL3AME-bCbfnh-MAjc-18,2620
+dq/checks/base.py,sha256=IBLER1ABUjXeCwZHYk7Tz5LShE2jQWkwiIEW7taSpro,3296
+dq/checks/not_null.py,sha256=cIYJUEWu7USedEv0mUGfKQ12NGQqe8rNvBmbYXAyR0Q,1688
+dq/checks/parses_as.py,sha256=2o1T4CRiQxvoHvBCMYjN8m8jipDaSValWnnthxZ4rOw,3665
+dq/checks/row_count.py,sha256=L9ERmo-Jh47sTPl9SlJuidoYAeK3IOIDviizs4PIpO4,3238
+dq/checks/schema.py,sha256=I1tx8Ct-NjoqH6_2mGyj3XYFn6sU12ZSm3shnphv7jA,2684
+dq/checks/unique.py,sha256=A6JuwZ2PPDZ3hE0tWiZlLbPji9PQYbj9QCSDXTHlnHQ,2297
+dq/dbt/__init__.py,sha256=GEyKJbNqfqfef19yVRXmo04n1mzZocQtbsZJrHjXn1Y,84
+dq/dbt/artifacts.py,sha256=NK8v00_4UBzTSUSIabChfQrGe-Q4IKWzIiRAFk2U7dM,3396
+dq/dbt/publish.py,sha256=52ZDk_IPipnBPiGxQ4khHuPLgM6_901OtfOFr22LYO8,5909
+dq/sinks/__init__.py,sha256=seaI7CSGkOaP2-N4Y4gKbf_8nrBtl6UkTkypfNe3r7E,188
+dq/sinks/base.py,sha256=8z9DPUuMyAaMERRY9412GTeuPjVWQBooPXpjQtv_Hao,349
+dq/sinks/http.py,sha256=jdgK9Eqdpt8rifUIQBWKYPGQCQprNaCiHB95t-eqcwQ,2328
+dq/sinks/log.py,sha256=LSdFwDj-p2bgaDqdCRzt-bFUmPLbNPtGZzr471Yjrqg,788
+dq/sinks/spool.py,sha256=BMv18F0A02kdakEdsAr-Mq1LHKYyef7PuHBWINIONUc,3229
+dq/sources/__init__.py,sha256=41rDHbkQc1_32Z4zublIL9uxLVPTFlO9qEHBFUpxUsc,667
+dq/sources/_frame.py,sha256=PgMrrhzL1vp00vpPBXMQlbSnf7M3KJzHE_X_oVGOl6w,3714
+dq/sources/base.py,sha256=PW99ddBnc9oY-7GxsXFgZK60617drqhROkQvfdwM5XQ,366
+dq/sources/csv.py,sha256=Cr8cnXFBrcURYipgh7ZC4pNdKoeLZ623suSf24PFlK0,2093
+dq/sources/dataframe.py,sha256=Mrms18ttKL8wf5kClUcFXwloOxxKBFhy2mjBYLsdVRY,668
+dq/sources/excel.py,sha256=5_SbLawqW47R-fTgOHQ7UFaK41qpXrsh1FHgoWTb2Lo,4134
+dq/sources/sql.py,sha256=ydcsq6ke7-7SO2zJktVIocTCN19sdDNzwCeHsmoDI6I,5663
+aice_data_quality-0.1.0.dist-info/METADATA,sha256=ZLw8Kr0PEhniG1K8sTdTeh64NqBFzi1qbkbbn7oh_Cw,3272
+aice_data_quality-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+aice_data_quality-0.1.0.dist-info/entry_points.txt,sha256=U4tDvOjCv_IMzrDd3ethbmBPR3UeBP2doZZxOlykK6o,35
+aice_data_quality-0.1.0.dist-info/RECORD,,

aice_data_quality-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

aice_data_quality-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dq = dq.cli:main

dq/__init__.py

@@ -0,0 +1,19 @@
+"""Data quality SDK. Produces records of result contract v1.0 (see contract/)."""
+from __future__ import annotations
+
+__version__ = "0.1.0"
+CONTRACT_VERSIONS = ("1.0",)
+
+from . import checks  # noqa: E402
+from .client import QualityClient, Report, Run  # noqa: E402
+from .contract import ResultRecord, RunRecord, Severity, Status  # noqa: E402
+from .dataset import Dataset  # noqa: E402
+from .errors import (ConfigError, DQError, MissingColumn, SourceError, SuiteError,  # noqa: E402
+                     UnsupportedCheck)
+from .suites import Suite  # noqa: E402
+
+__all__ = [
+    "QualityClient", "Run", "Report", "Dataset", "Severity", "Status", "Suite", "checks",
+    "RunRecord", "ResultRecord", "DQError", "ConfigError", "SuiteError", "SourceError",
+    "UnsupportedCheck", "MissingColumn", "__version__", "CONTRACT_VERSIONS",
+]

dq/checks/__init__.py

@@ -0,0 +1,30 @@
+"""Check constructors (`from dq import checks as c`) and evaluator registries."""
+from __future__ import annotations
+
+from . import accepted_values as _accepted_values
+from . import not_null as _not_null
+from . import parses_as as _parses_as
+from . import row_count as _row_count
+from . import schema as _schema
+from . import unique as _unique
+from .base import Check, EvalContext, Measurement
+
+schema = _schema.build
+not_null = _not_null.build
+parses_as = _parses_as.build
+accepted_values = _accepted_values.build
+unique = _unique.build
+row_count = _row_count.build
+
+FRAME_EVALUATORS = {
+    "schema": _schema.evaluate, "not_null": _not_null.frame, "parses_as": _parses_as.frame,
+    "accepted_values": _accepted_values.frame, "unique": _unique.frame,
+    "row_count": _row_count.frame,
+}
+SQL_EVALUATORS = {
+    "schema": _schema.evaluate, "not_null": _not_null.sql,
+    "accepted_values": _accepted_values.sql, "unique": _unique.sql, "row_count": _row_count.sql,
+}
+
+__all__ = ["Check", "EvalContext", "Measurement", "schema", "not_null", "parses_as",
+           "accepted_values", "unique", "row_count", "FRAME_EVALUATORS", "SQL_EVALUATORS"]

dq/checks/accepted_values.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import Optional, Sequence
+
+from ..contract import Severity, Status
+from .base import Check, EvalContext, KeySpec, Measurement, make_check, rows_have, rows_hint
+
+
+def build(column: str, values: Sequence, case_sensitive: bool = True, *, name: str,
+          severity: Severity = Severity.FLAG, dimension: Optional[str] = None,
+          key: KeySpec = None) -> Check:
+    vals = tuple(str(v) for v in values)
+    return make_check("accepted_values", name=name, severity=severity, dimension=dimension,
+                      default_dimension="validity", key=key,
+                      params={"column": column, "values": vals, "case_sensitive": case_sensitive},
+                      expected={"column": column, "values": list(vals), "case_sensitive": case_sensitive})
+
+
+def _result(col: str, total: int, bad: int, keys: list[str]) -> Measurement:
+    return Measurement(
+        status=Status.FAIL if bad else Status.PASS, rows_evaluated=total, failing_count=bad,
+        observed_value=bad, failing_keys=keys,
+        message=(f"{rows_have(bad)} a value in '{col}' that is not in the accepted list"
+                 f"{rows_hint(keys, bad)}.") if bad else None)
+
+
+def frame(src, check: Check, ctx: EvalContext) -> Measurement:
+    p = check.params
+    src.require(p["column"])
+    series = src.df[p["column"]]
+    null = src.null_mask(p["column"]).tolist()
+    ok = set(p["values"] if p["case_sensitive"] else (v.lower() for v in p["values"]))
+    positions = []
+    for i, (v, is_null) in enumerate(zip(series.tolist(), null)):
+        if is_null:
+            continue
+        s = str(v) if p["case_sensitive"] else str(v).lower()
+        if s not in ok:
+            positions.append(i)
+    keys = src.keys_at(positions[: ctx.key_limit], check)
+    return _result(p["column"], len(series), len(positions), keys)
+
+
+def sql(src, check: Check, ctx: EvalContext) -> Measurement:
+    p = check.params
+    col = src.col(p["column"])
+    expr = f"cast({col} as text)" if p["case_sensitive"] else f"lower(cast({col} as text))"
+    values = p["values"] if p["case_sensitive"] else tuple(v.lower() for v in p["values"])
+    params = {f"_dq_v{i}": v for i, v in enumerate(values)}
+    if values:
+        bad_cond = f"{col} is not null and {expr} not in ({', '.join(':' + k for k in params)})"
+    else:
+        bad_cond = f"{col} is not null"
+    total, bad = src.one(
+        f"select count(*), count(*) filter (where {bad_cond}) from {src.base()}", params, ctx=ctx)
+    keys = src.keys(bad_cond, check, ctx, params) if bad else []
+    return _result(p["column"], total, bad, keys)

dq/checks/base.py

@@ -0,0 +1,102 @@
+"""Check definition, measurement and shared helpers."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any, Optional, Sequence, Union
+
+from ..contract import DIMENSIONS, Severity, Status
+from ..errors import SuiteError
+
+_SNAKE = re.compile(r"^[a-z0-9_]+$")
+
+KeySpec = Union[None, str, Sequence[str]]
+
+
+@dataclass(frozen=True)
+class Check:
+    type: str
+    name: str
+    severity: Severity
+    dimension: str
+    key: Optional[tuple[str, ...]]          # None: default (row number / primary key)
+    params: dict[str, Any] = field(default_factory=dict)
+    expected: Optional[dict[str, Any]] = None
+
+
+@dataclass
+class EvalContext:
+    key_limit: int = 100
+    timeout_s: float = 60.0
+    history: list[float] = field(default_factory=list)
+
+
+@dataclass
+class Measurement:
+    status: Status
+    rows_evaluated: Optional[int] = None
+    failing_count: Optional[int] = None
+    observed_value: Union[int, float, str, None] = None
+    failing_keys: list[str] = field(default_factory=list)
+    message: Optional[str] = None
+
+
+def make_check(
+    type: str,
+    *,
+    name: str,
+    severity: Union[Severity, str],
+    dimension: Optional[str],
+    default_dimension: str,
+    key: KeySpec,
+    params: dict[str, Any],
+    expected: Optional[dict[str, Any]] = None,
+) -> Check:
+    if not isinstance(name, str) or not _SNAKE.match(name):
+        raise SuiteError(f"check name {name!r} must be lowercase snake case (^[a-z0-9_]+$)")
+    try:
+        sev = Severity(severity)
+    except ValueError:
+        raise SuiteError(f"check {name!r}: severity must be block, flag or warn") from None
+    dim = dimension or default_dimension
+    if dim not in DIMENSIONS:
+        raise SuiteError(f"check {name!r}: unknown dimension {dim!r}")
+    return Check(type=type, name=name, severity=sev, dimension=dim,
+                 key=normalise_key(key), params=params, expected=expected)
+
+
+def normalise_key(key: KeySpec) -> Optional[tuple[str, ...]]:
+    if key is None or key == "row":
+        return None
+    if isinstance(key, str):
+        return (key,)
+    keys = tuple(key)
+    if not keys or not all(isinstance(k, str) and k for k in keys):
+        raise SuiteError("key must be 'row', a column name or a list of column names")
+    return keys
+
+
+def as_columns(columns: Union[str, Sequence[str]], what: str = "columns") -> tuple[str, ...]:
+    cols = (columns,) if isinstance(columns, str) else tuple(columns)
+    if not cols or not all(isinstance(c, str) and c for c in cols):
+        raise SuiteError(f"{what} must be a non-empty list of column names")
+    return cols
+
+
+def rows_hint(keys: Sequence[str], total: int, shown: int = 10) -> str:
+    """' (rows 14, 27, 301)' when every key is a row number; empty otherwise, so a
+    message never carries key values from the data."""
+    if not keys or not all(k.startswith("row:") for k in keys):
+        return ""
+    nums = [k[4:] for k in keys[:shown]]
+    more = f" and {total - len(nums)} more" if total > len(nums) else ""
+    return f" (rows {', '.join(nums)}{more})"
+
+
+def plural(n: int, word: str) -> str:
+    return f"{n} {word}" if n == 1 else f"{n} {word}s"
+
+
+def rows_have(n: int) -> str:
+    """'1 row has' / '3 rows have'."""
+    return f"{n} row has" if n == 1 else f"{n} rows have"

dq/checks/not_null.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from ..contract import Severity, Status
+from .base import Check, EvalContext, KeySpec, Measurement, make_check, rows_have, rows_hint
+
+
+def build(column: str, *, name: str, severity: Severity = Severity.FLAG,
+          dimension: Optional[str] = None, key: KeySpec = None) -> Check:
+    return make_check("not_null", name=name, severity=severity, dimension=dimension,
+                      default_dimension="completeness", key=key,
+                      params={"column": column}, expected={"column": column, "null_share": 0})
+
+
+def _result(col: str, total: int, nulls: int, keys: list[str]) -> Measurement:
+    return Measurement(
+        status=Status.FAIL if nulls else Status.PASS,
+        rows_evaluated=total, failing_count=nulls,
+        observed_value=(nulls / total) if total else 0.0, failing_keys=keys,
+        message=(f"{rows_have(nulls)} no value in '{col}'"
+                 f"{rows_hint(keys, nulls)}.") if nulls else None,
+    )
+
+
+def frame(src, check: Check, ctx: EvalContext) -> Measurement:
+    col = check.params["column"]
+    mask = src.null_mask(col)
+    positions = [i for i, f in enumerate(mask.tolist()) if f]
+    keys = src.keys_at(positions[: ctx.key_limit], check)
+    return _result(col, len(mask), len(positions), keys)
+
+
+def sql(src, check: Check, ctx: EvalContext) -> Measurement:
+    col = src.col(check.params["column"])
+    total, nulls = src.one(
+        f"select count(*), count(*) filter (where {col} is null) from {src.base()}", ctx=ctx)
+    keys = []
+    if nulls:
+        keys = src.keys(f"{col} is null", check, ctx)
+    return _result(check.params["column"], total, nulls, keys)

dq/checks/parses_as.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import datetime as dt
+import decimal
+import re
+from typing import Optional, Sequence
+
+from ..contract import Severity, Status
+from ..errors import SuiteError
+from .base import Check, EvalContext, KeySpec, Measurement, make_check, rows_have, rows_hint
+
+TYPES = ("date", "datetime", "integer", "decimal")
+DEFAULT_FORMATS = {
+    "date": ("%Y-%m-%d",),
+    "datetime": ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M"),
+}
+_INT = re.compile(r"^[+-]?\d+$")
+_DEC = re.compile(r"^[+-]?(\d+(\.\d*)?|\.\d+)$")
+
+
+def build(column: str, type: str, formats: Optional[Sequence[str]] = None,
+          allow_empty: bool = False, *, name: str, severity: Severity = Severity.FLAG,
+          dimension: Optional[str] = None, key: KeySpec = None) -> Check:
+    if type not in TYPES:
+        raise SuiteError(f"check {name!r}: parses_as type must be one of {', '.join(TYPES)}")
+    if formats and type not in DEFAULT_FORMATS:
+        raise SuiteError(f"check {name!r}: formats apply only to date and datetime")
+    fmts = tuple(formats) if formats else DEFAULT_FORMATS.get(type, ())
+    expected = {"column": column, "type": type, "allow_empty": allow_empty}
+    if fmts:
+        expected["formats"] = list(fmts)
+    return make_check("parses_as", name=name, severity=severity, dimension=dimension,
+                      default_dimension="validity", key=key,
+                      params={"column": column, "type": type, "formats": fmts,
+                              "allow_empty": allow_empty}, expected=expected)
+
+
+def parses(value, type: str, formats: Sequence[str], native_dates: bool) -> bool:
+    if isinstance(value, str):
+        s = value.strip()
+        if type in ("date", "datetime"):
+            fmts = list(formats)
+            if native_dates:  # ExcelSource renders real date cells as ISO text
+                fmts += list(DEFAULT_FORMATS["date"]) + list(DEFAULT_FORMATS["datetime"])
+            for f in fmts:
+                try:
+                    dt.datetime.strptime(s, f)
+                    return True
+                except ValueError:
+                    pass
+            return False
+        if type == "integer":
+            return bool(_INT.match(s))
+        return bool(_DEC.match(s))
+    if isinstance(value, bool):
+        return False
+    if type in ("date", "datetime"):
+        return isinstance(value, (dt.date, dt.datetime))  # includes pandas Timestamp
+    if type == "integer":
+        return isinstance(value, int) or (isinstance(value, float) and value.is_integer())
+    return isinstance(value, (int, float, decimal.Decimal)) and value == value  # not NaN
+
+
+def frame(src, check: Check, ctx: EvalContext) -> Measurement:
+    p = check.params
+    col = p["column"]
+    src.require(col)
+    values = src.df[col].tolist()
+    null = src.null_mask(col).tolist()
+    native = getattr(src, "native_dates", False)
+    positions = [
+        i for i, (v, is_null) in enumerate(zip(values, null))
+        if (is_null and not p["allow_empty"])
+        or (not is_null and not parses(v, p["type"], p["formats"], native))
+    ]
+    keys = src.keys_at(positions[: ctx.key_limit], check)
+    bad = len(positions)
+    fmt = f" in format {', '.join(p['formats'])}" if p["formats"] else ""
+    return Measurement(
+        status=Status.FAIL if bad else Status.PASS, rows_evaluated=len(values),
+        failing_count=bad, observed_value=bad, failing_keys=keys,
+        message=(f"{rows_have(bad)} a '{col}' that is empty or not a valid "
+                 f"{p['type']}{fmt}{rows_hint(keys, bad)}.") if bad else None)
+
+
+# Typed database columns are checked by `schema`; parses_as has no SQL pushdown.
+sql = None

dq/checks/row_count.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import statistics
+from typing import Optional
+
+from ..contract import Severity, Status
+from ..errors import SuiteError
+from ..history import MIN_HISTORY
+from .base import Check, EvalContext, KeySpec, Measurement, make_check
+
+
+def build(min: Optional[int] = None, max: Optional[int] = None,
+          vs_previous: Optional[float] = None, history: int = 14, *, name: str,
+          severity: Severity = Severity.FLAG, dimension: Optional[str] = None,
+          key: KeySpec = None) -> Check:
+    if min is None and max is None and vs_previous is None:
+        raise SuiteError(f"check {name!r}: row_count needs min, max or vs_previous")
+    if vs_previous is not None and vs_previous < 0:
+        raise SuiteError(f"check {name!r}: vs_previous must be a non-negative fraction")
+    if history < MIN_HISTORY:
+        raise SuiteError(f"check {name!r}: history must be at least {MIN_HISTORY}")
+    expected = {k: v for k, v in (("min", min), ("max", max), ("vs_previous", vs_previous)) if v is not None}
+    if vs_previous is not None:
+        expected["history"] = history
+    return make_check("row_count", name=name, severity=severity, dimension=dimension,
+                      default_dimension="volume", key=key,
+                      params={"min": min, "max": max, "vs_previous": vs_previous, "history": history},
+                      expected=expected)
+
+
+def judge(count: int, check: Check, ctx: EvalContext) -> Measurement:
+    p = check.params
+    problems = []
+    if p["min"] is not None and count < p["min"]:
+        problems.append(f"Row count {count} is below the minimum {p['min']}.")
+    if p["max"] is not None and count > p["max"]:
+        problems.append(f"Row count {count} is above the maximum {p['max']}.")
+    if problems:
+        return Measurement(Status.FAIL, rows_evaluated=count, observed_value=count,
+                           message=" ".join(problems))
+    if p["vs_previous"] is not None:
+        if len(ctx.history) < MIN_HISTORY:
+            return Measurement(
+                Status.SKIPPED, rows_evaluated=count, observed_value=count,
+                message=f"Not enough history to compare against ({len(ctx.history)} of "
+                        f"{MIN_HISTORY} earlier values).")
+        median = statistics.median(ctx.history[: p["history"]])
+        change = abs(count - median)
+        if change > p["vs_previous"] * median:
+            pct = f"{change / median:.0%}" if median else "n/a"
+            direction = "below" if count < median else "above"
+            return Measurement(
+                Status.FAIL, rows_evaluated=count, observed_value=count,
+                message=f"Row count {count} is {pct} {direction} the median {median:g} of the "
+                        f"last {min(len(ctx.history), p['history'])} runs "
+                        f"(allowed change {p['vs_previous']:.0%}).")
+    return Measurement(Status.PASS, rows_evaluated=count, observed_value=count)
+
+
+def frame(src, check: Check, ctx: EvalContext) -> Measurement:
+    return judge(len(src.df), check, ctx)
+
+
+def sql(src, check: Check, ctx: EvalContext) -> Measurement:
+    (count,) = src.one(f"select count(*) from {src.base()}", ctx=ctx)
+    return judge(int(count), check, ctx)

dq/checks/schema.py

@@ -0,0 +1,58 @@
+"""schema: expected columns and (optionally) types. Works on any source."""
+from __future__ import annotations
+
+from typing import Mapping, Optional, Sequence
+
+from ..contract import Severity, Status
+from ..errors import UnsupportedCheck
+from .base import Check, EvalContext, KeySpec, Measurement, as_columns, make_check
+
+TYPE_ALIASES = {
+    "int": "integer", "integer": "integer", "bigint": "integer",
+    "float": "decimal", "number": "decimal", "numeric": "decimal", "decimal": "decimal",
+    "str": "string", "string": "string", "text": "string",
+    "bool": "boolean", "boolean": "boolean",
+    "date": "date", "datetime": "datetime", "timestamp": "datetime",
+}
+
+
+def build(columns: Sequence[str], types: Optional[Mapping[str, str]] = None,
+          allow_extra: bool = True, *, name: str, severity: Severity = Severity.FLAG,
+          dimension: Optional[str] = None, key: KeySpec = None) -> Check:
+    cols = as_columns(columns)
+    norm_types: dict[str, str] = {}
+    for col, t in (types or {}).items():
+        if t not in TYPE_ALIASES:
+            raise ValueError(f"unknown type {t!r} for column {col!r}")
+        norm_types[col] = TYPE_ALIASES[t]
+    expected = {"columns": list(cols), "allow_extra": allow_extra}
+    if norm_types:
+        expected["types"] = norm_types
+    return make_check("schema", name=name, severity=severity, dimension=dimension,
+                      default_dimension="conformity", key=key,
+                      params={"columns": cols, "types": norm_types, "allow_extra": allow_extra},
+                      expected=expected)
+
+
+def evaluate(src, check: Check, ctx: EvalContext) -> Measurement:
+    p = check.params
+    actual = src.columns()
+    if p["types"] and not getattr(src, "supports_types", True):
+        raise UnsupportedCheck("types is not supported on file sources (all cells are text); use parses_as")
+    missing = [c for c in p["columns"] if c not in actual]
+    wrong = [c for c, t in p["types"].items() if c in actual and actual[c] != t]
+    extra = [] if p["allow_extra"] else [c for c in actual if c not in p["columns"]]
+    problems = len(missing) + len(wrong) + len(extra)
+    parts = []
+    if missing:
+        parts.append(f"Missing columns: {', '.join(missing)}.")
+    if wrong:
+        parts.append("Wrong type: " + ", ".join(f"{c} (expected {p['types'][c]})" for c in wrong) + ".")
+    if extra:
+        parts.append(f"Unexpected columns: {', '.join(extra)}.")
+    return Measurement(
+        status=Status.FAIL if problems else Status.PASS,
+        failing_count=problems,
+        observed_value=", ".join(missing + wrong + extra) if problems else None,
+        message=" ".join(parts) if parts else None,
+    )

dq/checks/unique.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from typing import Optional, Sequence, Union
+
+from ..contract import Severity, Status
+from .base import Check, EvalContext, KeySpec, Measurement, as_columns, make_check, plural, rows_hint
+
+
+def build(columns: Union[str, Sequence[str]], *, name: str, severity: Severity = Severity.FLAG,
+          dimension: Optional[str] = None, key: KeySpec = None) -> Check:
+    cols = as_columns(columns)
+    return make_check("unique", name=name, severity=severity, dimension=dimension,
+                      default_dimension="uniqueness", key=key,
+                      params={"columns": cols}, expected={"columns": list(cols)})
+
+
+def _result(cols: Sequence[str], total: int, dup: int, keys: list[str]) -> Measurement:
+    label = ", ".join(cols)
+    return Measurement(
+        status=Status.FAIL if dup else Status.PASS, rows_evaluated=total, failing_count=dup,
+        observed_value=dup, failing_keys=keys,
+        message=(f"{plural(dup, 'row')} share a duplicate value of ({label})"
+                 f"{rows_hint(keys, dup)}.") if dup else None)
+
+
+def frame(src, check: Check, ctx: EvalContext) -> Measurement:
+    cols = list(check.params["columns"])
+    df = src.df
+    for c in cols:
+        src.require(c)
+    nonnull = ~src.null_mask(cols[0])
+    for c in cols[1:]:
+        nonnull = nonnull & ~src.null_mask(c)
+    # Duplicates among rows where every column has a value; nulls never collide.
+    dup = nonnull & df.duplicated(subset=cols, keep=False)
+    positions = [i for i, f in enumerate(dup.tolist()) if f]
+    keys = src.keys_at(positions[: ctx.key_limit], check)
+    return _result(cols, len(df), len(positions), keys)
+
+
+def sql(src, check: Check, ctx: EvalContext) -> Measurement:
+    cols = [src.col(c) for c in check.params["columns"]]
+    joined = ", ".join(cols)
+    nonnull = " and ".join(f"{c} is not null" for c in cols)
+    dup_groups = (f"select {joined} from {src.base()} where {nonnull} "
+                  f"group by {joined} having count(*) > 1")
+    in_dup = f"{nonnull} and ({joined}) in ({dup_groups})"
+    total, dup = src.one(
+        f"select count(*), count(*) filter (where {in_dup}) from {src.base()}", ctx=ctx)
+    keys = src.keys(in_dup, check, ctx) if dup else []
+    return _result(check.params["columns"], total, dup, keys)