sqlassert 0.1.2__py3-none-any.whl → 0.3.1__py3-none-any.whl

sqlassert/dialects.py

@@ -0,0 +1,211 @@
+"""What each SQL dialect means by the constructs the analysis depends on.
+
+One `DialectSemantics` per dialect name (SQLGlot's), read by `ir/convert.py`
+while it lowers SQL: which Builtins raise an error rather than return NULL,
+which text compares by identity, what `||` and `concat()` do with NULL, and
+which constraints the database enforces. A dialect not listed gets
+`DialectSemantics()`, whose arithmetic, collation and `||` are the cautious
+choice; only its keys are taken as enforced, as before dialects were listed.
+
+Every flag is a claim that holds under all of that dialect's settings, since
+the SQL alone does not say which are in force (ADR 0005): MySQL's `||` is OR
+or concatenation depending on `sql_mode`, so it is neither; Spark's
+arithmetic raises or returns NULL depending on ANSI mode, so it may return
+NULL. `docs/dialects.md` argues each entry and says how soundness is judged
+without DuckDB; `docs/dialect-provenance.json` records, per fact, the engine
+versions it was certified on or the vendor doc it rests on
+(`scripts/certify_dialects.py`).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+# How a dialect treats a declared key (PRIMARY KEY, UNIQUE) or foreign key:
+# enforced; informational, trusted when marked `RELY`; or informational only.
+type KeyTrust = Literal["enforced", "rely", "unenforced"]
+
+# What `a || b` is: concatenation whose NULL operand makes it NULL ("strict"),
+# concatenation that may skip a NULL operand ("lenient", Postgres arrays), or
+# something whose meaning a setting decides ("ambiguous", MySQL); None when
+# it is not modelled.
+type PipesMeaning = Literal["strict", "lenient", "ambiguous"] | None
+
+# What `concat(a, ...)` does with NULL: makes the result NULL ("strict"), or
+# skips it, so the result is never NULL ("skips-null"); None when not modelled.
+type ConcatMeaning = Literal["strict", "skips-null"] | None
+
+
+@dataclass(frozen=True)
+class DialectSemantics:
+    # `+`, `-`, `*`, unary `-` and CAST raise an error on overflow or a bad
+    # value rather than wrapping, saturating, or returning NULL.
+    checked_arithmetic: bool = False
+    # `/`, integer division and `%` by zero raise an error rather than return NULL.
+    checked_division: bool = False
+    # An implicit cast that fails raises an error, so a comparison, LIKE,
+    # BETWEEN, NOT or bitwise operator of non-null operands is never NULL.
+    casts_raise: bool = True
+    # VARCHAR and TEXT with no COLLATE compare by identity under the engine's
+    # shipped default collation. A setting that changes that default (DuckDB's
+    # `default_collation`) is out of scope; text with any COLLATE is never
+    # taken to compare by identity (docs/dialects.md).
+    text_is_binary: bool = False
+    # A DECIMAL declared without a scale keeps each value's own scale, so
+    # equal values (1.0, 1.00) may differ: neither identity nor exact.
+    unscaled_decimal: bool = False
+    # A TIMESTAMP WITH TIME ZONE (SQLGlot's TIMESTAMPTZ: MySQL's, BigQuery's
+    # and Spark's TIMESTAMP too) holds an instant only, so equal values are
+    # identical. Not where it keeps the zone it was given (Trino, Snowflake's
+    # TIMESTAMP_TZ, Oracle): equal instants in two zones differ as text.
+    zoned_timestamp_is_instant: bool = False
+    pipes: PipesMeaning = None
+    concat: ConcatMeaning = None
+    # What `concat_ws(sep, a, ...)` does with a NULL value (a NULL separator
+    # makes it NULL either way); None where it does not exist or is not modelled.
+    concat_ws: ConcatMeaning = None
+    unique_keys: KeyTrust = "enforced"
+    foreign_keys: KeyTrust = "enforced"
+    # A PRIMARY KEY column is NOT NULL.
+    primary_key_not_null: bool = True
+    # A column-level `REFERENCES` is enforced (MySQL before 9.0 parses and ignores it).
+    inline_references: bool = True
+    # The table ENGINEs that enforce foreign keys (lower-cased), None for every table.
+    foreign_key_engines: frozenset[str] | None = None
+    # An ENUM's members lose their trailing spaces (MySQL). An ENUM holds only
+    # its members: MySQL's '' for any other value, outside strict mode or
+    # under INSERT IGNORE, is out of scope (docs/dialects.md).
+    enum_strips_trailing_spaces: bool = False
+    # A zero-length string is NULL (Oracle): the literal '' is NULL.
+    empty_string_is_null: bool = False
+    # `EXCEPT ALL` and `INTERSECT ALL` exist. Only gates suggesting them, so
+    # it is no soundness claim and is not certified: a wrong True suggests SQL
+    # the engine rejects, never a wrong proof.
+    set_operations_all: bool = False
+
+    @property
+    def rely(self) -> bool:
+        """Whether the dialect marks an informational constraint trusted with `RELY`."""
+        return "rely" in (self.unique_keys, self.foreign_keys)
+
+
+_SPARK = DialectSemantics(
+    # ANSI mode decides whether overflow, division by zero and a bad cast
+    # raise or return NULL; it is on by default only in recent versions.
+    casts_raise=False,
+    zoned_timestamp_is_instant=True,
+    pipes="strict",
+    concat="strict",
+    concat_ws="skips-null",
+    unique_keys="rely",
+    foreign_keys="rely",
+    set_operations_all=True,
+)
+
+_SEMANTICS: dict[str, DialectSemantics] = {
+    "duckdb": DialectSemantics(
+        checked_arithmetic=True,
+        text_is_binary=True,
+        zoned_timestamp_is_instant=True,
+        pipes="strict",
+        concat="skips-null",
+        concat_ws="skips-null",
+        set_operations_all=True,
+    ),
+    "postgres": DialectSemantics(
+        checked_arithmetic=True,
+        checked_division=True,
+        text_is_binary=True,
+        unscaled_decimal=True,
+        zoned_timestamp_is_instant=True,
+        pipes="lenient",  # `ARRAY[1] || NULL` is `{1}`.
+        concat="skips-null",
+        concat_ws="skips-null",
+        set_operations_all=True,
+    ),
+    "mysql": DialectSemantics(
+        # MySQL under its default strict sql_mode only (docs/dialects.md), so
+        # a comparison of non-null operands is non-null. Not `checked_arithmetic`:
+        # `-x` of the least BIGINT is an exact DECIMAL rather than an error, and
+        # division by zero is NULL in any mode.
+        # Not `text_is_binary`: the default collations ignore case (and
+        # accents, and some trailing spaces).
+        zoned_timestamp_is_instant=True,  # TIMESTAMP is converted to UTC.
+        pipes="ambiguous",
+        concat="strict",
+        concat_ws="skips-null",
+        inline_references=False,
+        foreign_key_engines=frozenset({"innodb", "ndb", "ndbcluster"}),
+        enum_strips_trailing_spaces=True,
+        set_operations_all=True,  # Since 8.0.31.
+    ),
+    "snowflake": DialectSemantics(
+        # Not `checked_arithmetic`: its docs never say integer overflow raises.
+        checked_division=True,
+        pipes="strict",
+        concat="strict",
+        concat_ws="strict",  # `concat_ws(',', 'a', NULL)` is NULL.
+        unique_keys="rely",
+        foreign_keys="rely",
+    ),
+    "bigquery": DialectSemantics(
+        checked_arithmetic=True,
+        checked_division=True,
+        zoned_timestamp_is_instant=True,
+        pipes="strict",
+        concat="strict",
+        unique_keys="unenforced",
+        foreign_keys="unenforced",
+    ),
+    "redshift": DialectSemantics(
+        checked_arithmetic=True,
+        checked_division=True,
+        zoned_timestamp_is_instant=True,
+        pipes="strict",
+        unique_keys="unenforced",
+        foreign_keys="unenforced",
+    ),
+    "databricks": _SPARK,
+    "spark": _SPARK,
+    "spark2": _SPARK,
+    "hive": DialectSemantics(casts_raise=False, pipes="strict", concat="strict", unique_keys="unenforced", foreign_keys="unenforced"),
+    "sqlite": DialectSemantics(
+        pipes="strict",
+        concat_ws="skips-null",
+        # Foreign keys are checked only under `PRAGMA foreign_keys = ON`, off by default.
+        foreign_keys="unenforced",
+        # Only an INTEGER PRIMARY KEY (the rowid) is never NULL.
+        primary_key_not_null=False,
+    ),
+    # A ClickHouse primary key orders and indexes a table; it never rejects a duplicate.
+    "clickhouse": DialectSemantics(concat_ws="strict", unique_keys="unenforced", foreign_keys="unenforced"),
+    "oracle": DialectSemantics(empty_string_is_null=True),
+    # Trino has no constraint but NOT NULL: its CREATE TABLE rejects a key, so
+    # one in the SQL describes a table some other engine holds.
+    "trino": DialectSemantics(
+        checked_arithmetic=True,
+        checked_division=True,
+        # VARCHAR compares by code point and Trino has no collations.
+        text_is_binary=True,
+        # Not `zoned_timestamp_is_instant`: a TIMESTAMP WITH TIME ZONE keeps
+        # its zone, and `CAST(ts AS VARCHAR)` writes it.
+        # Unlike Postgres, a NULL array or element operand makes `||` NULL too.
+        pipes="strict",
+        concat="strict",
+        # Not modelled: it skips a NULL VARCHAR, but its ARRAY(VARCHAR)
+        # overload is NULL for a NULL array, and SQLGlot parses both alike.
+        unique_keys="unenforced",
+        foreign_keys="unenforced",
+        set_operations_all=True,
+    ),
+}
+
+
+# The dialects whose semantics are argued here (docs/dialects.md).
+MODELLED_DIALECTS = frozenset(_SEMANTICS)
+
+
+def semantics(dialect: str) -> DialectSemantics:
+    """The semantics of a SQLGlot dialect name; cautious defaults for one not listed."""
+    return _SEMANTICS.get(dialect.lower(), DialectSemantics())

sqlassert/discovery/__init__.py

@@ -0,0 +1,30 @@
+"""Database Knowledge discovery: catalog metadata bound to IR nodes before the engine runs.
+
+Database adapters live in submodules, such as `sqlassert.discovery.duckdb`
+and `sqlassert.discovery.postgres`, so importing this package never imports
+a database driver. `information_schema.py` is the base of every DB-API one.
+"""
+
+from sqlassert.discovery.fetcher import (
+    CatalogFacts,
+    DatabaseColumn,
+    DatabaseForeignKey,
+    DatabaseHydration,
+    DatabaseKnowledgeFetcher,
+    DatabaseMetadataError,
+    DatabaseRelationKind,
+    DatabaseRelationMetadata,
+    DatabaseViewDefinition,
+)
+
+__all__ = [
+    "CatalogFacts",
+    "DatabaseColumn",
+    "DatabaseForeignKey",
+    "DatabaseHydration",
+    "DatabaseKnowledgeFetcher",
+    "DatabaseMetadataError",
+    "DatabaseRelationKind",
+    "DatabaseRelationMetadata",
+    "DatabaseViewDefinition",
+]

sqlassert/discovery/bigquery.py

@@ -0,0 +1,71 @@
+"""BigQuery: the standard discovery, one dataset's `INFORMATION_SCHEMA` at a time.
+
+Over `google.cloud.bigquery.dbapi` (`%s` parameters). BigQuery has no
+current dataset a query can ask for, so the fetcher is given the project and
+the default dataset unqualified names resolve in. Every view is read from
+`project.dataset.INFORMATION_SCHEMA`; a name may be qualified by any project.
+
+BigQuery has no `REFERENTIAL_CONSTRAINTS`: a foreign key's referenced table
+and columns come from `CONSTRAINT_COLUMN_USAGE`, which does not order them,
+so a foreign key over more than one column is skipped. Keys are not
+enforced: they hold only under `trust_unenforced_keys` (`dialects.py`).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from sqlassert.discovery.fetcher import DatabaseForeignKey, DatabaseRelationMetadata
+from sqlassert.discovery.information_schema import (
+    Connection,
+    InformationSchemaKnowledgeFetcher,
+    foreign_keys_from_pairs,
+    placeholders,
+    quoted,
+)
+
+
+class BigQueryKnowledgeFetcher(InformationSchemaKnowledgeFetcher):
+    type_dialect = "bigquery"
+    cross_catalog = True
+
+    def __init__(self, connection: Connection, project: str, dataset: str | None = None):
+        super().__init__(connection)
+        self.project = project
+        self.dataset = dataset
+
+    def session(self, scope: DatabaseRelationMetadata | None) -> tuple[str, tuple[str, ...]]:
+        if scope is not None:
+            return scope.catalog, (scope.schema,)
+        return self.project, (self.dataset,) if self.dataset else ()
+
+    def information_schema(self, catalog: str, schema: str) -> str:
+        return f"{quoted(catalog, '`')}.{quoted(schema, '`')}.INFORMATION_SCHEMA"
+
+    def foreign_keys(self, catalog: str, schema: str, tables: Sequence[str]) -> list[tuple[str, DatabaseForeignKey]]:
+        where = self.information_schema(catalog, schema)
+        own_sql = (
+            "SELECT k.TABLE_NAME, k.CONSTRAINT_NAME, k.COLUMN_NAME "
+            f"FROM {where}.TABLE_CONSTRAINTS c JOIN {where}.KEY_COLUMN_USAGE k "
+            "ON k.CONSTRAINT_NAME = c.CONSTRAINT_NAME AND k.TABLE_NAME = c.TABLE_NAME "
+            f"WHERE c.CONSTRAINT_TYPE = 'FOREIGN KEY' AND c.TABLE_NAME IN ({placeholders(tables)}) "
+            "ORDER BY k.TABLE_NAME, k.CONSTRAINT_NAME, k.ORDINAL_POSITION"
+        )
+        own: dict[str, tuple[str, list[str]]] = {}
+        for table, constraint, column in self._rows(own_sql, tuple(tables)):
+            own.setdefault(constraint, (table, []))[1].append(column)
+        single = sorted(constraint for constraint, (_, members) in own.items() if len(members) == 1)
+        if not single:
+            return []
+        referenced_sql = (
+            "SELECT CONSTRAINT_NAME, TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME "
+            f"FROM {where}.CONSTRAINT_COLUMN_USAGE WHERE CONSTRAINT_NAME IN ({placeholders(single)})"
+        )
+        referenced: dict[str, list[tuple[Any, ...]]] = {}
+        for constraint, *target in self._rows(referenced_sql, tuple(single)):
+            referenced.setdefault(constraint, []).append(tuple(target))
+        return foreign_keys_from_pairs([
+            (own[constraint][0], constraint, own[constraint][1][0], *referenced[constraint][0])
+            for constraint in single if len(referenced.get(constraint, ())) == 1
+        ])

sqlassert/discovery/databricks.py

@@ -0,0 +1,18 @@
+"""Databricks (Unity Catalog): the standard discovery, over `databricks-sql-connector`.
+
+Unity Catalog's `information_schema` has every standard view this reads, so
+only the session query and the parameter style (`?`) differ. Primary and
+foreign keys are informational: they hold only under
+`trust_unenforced_keys` (`dialects.py`); the `RELY` a constraint may carry is
+not read. Names qualified by another catalog are not looked up.
+"""
+
+from __future__ import annotations
+
+from sqlassert.discovery.information_schema import InformationSchemaKnowledgeFetcher
+
+
+class DatabricksKnowledgeFetcher(InformationSchemaKnowledgeFetcher):
+    type_dialect = "databricks"
+    paramstyle = "qmark"
+    session_sql = "SELECT current_catalog(), current_schema()"

sqlassert/discovery/duckdb.py

@@ -0,0 +1,163 @@
+"""DuckDB: the standard discovery, with DuckDB's `duckdb_*()` catalog functions.
+
+Every hook reads `duckdb_tables()`, `duckdb_views()`, `duckdb_columns()` or
+`duckdb_constraints()`; it never scans a table. Any DuckDB connection works
+(the `duckdb` module's, or one from any other client exposing `execute`), and
+sqlassert does not import a DuckDB driver.
+
+Queries run on the connection itself, not on a cursor: a DuckDB cursor is a
+new session, without the connection's temporary tables, `USE` or
+`search_path`, which decide what an unqualified name denotes.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any, Protocol
+
+import sqlglot
+from sqlglot import exp
+from sqlglot.errors import SqlglotError
+
+from sqlassert.discovery.fetcher import (
+    DatabaseColumn,
+    DatabaseForeignKey,
+    DatabaseMetadataError,
+    DatabaseRelationKind,
+    DatabaseRelationMetadata,
+)
+from sqlassert.discovery.information_schema import FoundRelation, InformationSchemaKnowledgeFetcher, Key
+
+_SESSION = "SELECT current_database(), current_schema(), current_setting('search_path')"
+
+_RELATIONS = """
+SELECT database_name, schema_name, table_name, 'table' FROM duckdb_tables()
+WHERE lower(database_name) = %s AND lower(schema_name) = %s AND list_contains(%s, lower(table_name))
+UNION ALL
+SELECT database_name, schema_name, view_name, 'view' FROM duckdb_views()
+WHERE NOT internal AND lower(database_name) = %s AND lower(schema_name) = %s AND list_contains(%s, lower(view_name))
+"""
+
+_TABLES = "database_name = %s AND schema_name = %s AND list_contains(%s, table_name)"
+
+_COLUMNS = f"""
+SELECT table_name, column_name, is_nullable, data_type FROM duckdb_columns()
+WHERE {_TABLES}
+ORDER BY table_name, column_index
+"""
+
+# duckdb_columns() drops collations; the stored CREATE TABLE keeps them.
+_TABLE_SQL = f"SELECT table_name, sql FROM duckdb_tables() WHERE {_TABLES}"
+
+# A foreign key's referenced table is in the referencing table's own catalog
+# and schema: DuckDB rejects foreign keys across them.
+_CONSTRAINTS = f"""
+SELECT table_name, constraint_type, constraint_column_names, referenced_table, referenced_column_names
+FROM duckdb_constraints()
+WHERE constraint_type IN ({{kinds}}) AND {_TABLES}
+ORDER BY table_name, constraint_index
+"""
+
+_VIEW = "SELECT sql FROM duckdb_views() WHERE database_name = %s AND schema_name = %s AND view_name = %s"
+
+
+class DuckDBConnection(Protocol):
+    def execute(self, query: str, parameters: Any = ..., /) -> Any: ...
+
+
+class DuckDBKnowledgeFetcher(InformationSchemaKnowledgeFetcher):
+    type_dialect = "duckdb"
+    cross_catalog = True
+    _search_path: tuple[tuple[str, str], ...] = ()
+
+    def __init__(self, connection: DuckDBConnection):
+        self.database = connection
+
+    def session(self, scope: DatabaseRelationMetadata | None) -> tuple[str, tuple[str, ...]]:
+        """Where an unqualified name is looked up: a view's own schema first,
+        then temporary objects, then the session's path. DuckDB's path spans
+        catalogs, so `candidates` reads it from `_search_path`."""
+        (database, schema, configured), = self._rows(_SESSION)
+        path: list[tuple[str, str]] = [(scope.catalog, scope.schema)] if scope is not None else []
+        path.append(("temp", "main"))
+        for entry in filter(None, (entry.strip() for entry in configured.split(","))):
+            qualifier, _, name = entry.rpartition(".")
+            path.append((qualifier or database, name))
+        path.append((database, schema))
+        self._search_path = tuple((catalog.lower(), schema.lower()) for catalog, schema in path)
+        return database, tuple(schema for _, schema in self._search_path)
+
+    def candidates(self, name: str, catalog: str, path: tuple[str, ...]) -> list[tuple[str, str, str]]:
+        """Every catalog path ``name`` may denote, in DuckDB's binding order."""
+        match name.lower().split("."):
+            case [table]:
+                return [(searched, schema, table) for searched, schema in self._search_path]
+            case [qualifier, table]:
+                # A schema in a searched catalog first, then a catalog's default schema.
+                catalogs = dict.fromkeys(searched for searched, _ in self._search_path)
+                return [(searched, qualifier, table) for searched in catalogs] + [(qualifier, "main", table)]
+            case [named, schema, table]:
+                return [(named, schema, table)]
+            case _:
+                return []
+
+    def relations(self, catalog: str, schema: str, names: Sequence[str]) -> list[FoundRelation]:
+        lookup = (catalog.lower(), schema.lower(), list(names))
+        return [
+            FoundRelation(database, found_schema, name, DatabaseRelationKind(kind))
+            for database, found_schema, name, kind in self._rows(_RELATIONS, lookup * 2)
+        ]
+
+    def columns(self, catalog: str, schema: str, tables: Sequence[str]) -> list[tuple[str, DatabaseColumn]]:
+        lookup = (catalog, schema, list(tables))
+        collations = self._collations(catalog, schema, lookup)
+        return [
+            (table, DatabaseColumn(column, nullable, data_type, collations.get(table, {}).get(column.lower())))
+            for table, column, nullable, data_type in self._rows(_COLUMNS, lookup)
+        ]
+
+    def keys(self, catalog: str, schema: str, tables: Sequence[str]) -> list[Key]:
+        return [
+            Key(table, tuple(members), kind == "PRIMARY KEY")
+            for table, kind, members, *_ in self._rows(_CONSTRAINTS.format(kinds="'PRIMARY KEY', 'UNIQUE'"), (catalog, schema, list(tables)))
+        ]
+
+    def foreign_keys(self, catalog: str, schema: str, tables: Sequence[str]) -> list[tuple[str, DatabaseForeignKey]]:
+        return [
+            (table, DatabaseForeignKey(tuple(members), f"{catalog}.{schema}.{referenced}", tuple(referenced_members)))
+            for table, _, members, referenced, referenced_members
+            in self._rows(_CONSTRAINTS.format(kinds="'FOREIGN KEY'"), (catalog, schema, list(tables)))
+        ]
+
+    def view_definition(self, relation: DatabaseRelationMetadata) -> str | None:
+        rows = self._rows(_VIEW, (relation.catalog, relation.schema, relation.name))
+        return rows[0][0] if rows else None
+
+    def _collations(self, catalog: str, schema: str, lookup: tuple[Any, ...]) -> dict[str, dict[str, str]]:
+        """Each table's columns declared with a COLLATE, by name. The
+        session's `default_collation` is not read: text without a COLLATE has
+        the engine's shipped (binary) default (docs/dialects.md)."""
+        collations: dict[str, dict[str, str]] = {}
+        for table, sql in self._rows(_TABLE_SQL, lookup):
+            try:
+                create = sqlglot.parse_one(sql, read="duckdb")
+            except SqlglotError as error:
+                raise DatabaseMetadataError(f"definition of table {catalog}.{schema}.{table} could not be parsed: {error}") from error
+            own: dict[str, str] = {}
+            for definition in create.find_all(exp.ColumnDef):
+                collate = next((
+                    constraint.args["kind"] for constraint in definition.constraints
+                    if isinstance(constraint.args.get("kind"), exp.CollateColumnConstraint)
+                ), None)
+                if collate is not None:
+                    own[definition.name.lower()] = collate.this.name
+            collations[table] = own
+        return collations
+
+    def _rows(self, sql: str, parameters: tuple[Any, ...] = ()) -> list[tuple[Any, ...]]:
+        """The rows of one catalog query, run on the connection (see the module
+        docstring). Any driver error is a DatabaseMetadataError."""
+        try:
+            return [tuple(row) for row in self.database.execute(sql.replace("%s", "?"), list(parameters)).fetchall()]
+        except Exception as error:
+            raise DatabaseMetadataError(str(error)) from error