pyoq-sql 1.0.2__py3-none-any.whl

pyoq/schema/mysql/reflection.py

@@ -0,0 +1,309 @@
+"""MySQL metadata reflection into the immutable schema model."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol, TypeVar, cast
+
+from pyoq.config import DatabaseDialect
+from pyoq.errors import SchemaInspectionError
+from pyoq.schema.models import (
+    Capability,
+    CapabilitySet,
+    Catalog,
+    CheckConstraint,
+    Column,
+    EnumType,
+    Identifier,
+    Index,
+    Key,
+    KeyKind,
+    ObjectReference,
+    ParameterMode,
+    Relation,
+    Routine,
+    RoutineKind,
+    RoutineParameter,
+    Schema,
+    SchemaSnapshot,
+    SqlExpression,
+    Table,
+    View,
+)
+from pyoq.schema.mysql import sql
+from pyoq.schema.mysql.records import (
+    CheckRecord,
+    ColumnRecord,
+    ConstraintRecord,
+    IndexRecord,
+    ObjectRecord,
+    RoutineRecord,
+    read_check,
+    read_column,
+    read_constraint,
+    read_index,
+    read_object,
+    read_routine,
+)
+from pyoq.schema.mysql.types import mysql_type
+
+if TYPE_CHECKING:
+    from pymysql.connections import Connection
+    from pymysql.cursors import Cursor
+
+
+class NamedRecord(Protocol):
+    @property
+    def name(self) -> str: ...
+
+
+Record = TypeVar("Record")
+Named = TypeVar("Named", bound=NamedRecord)
+
+_CAPABILITIES = CapabilitySet(
+    frozenset(
+        {
+            Capability.CHECK_CONSTRAINTS,
+            Capability.EXPRESSION_INDEXES,
+            Capability.GENERATED_COLUMNS,
+            Capability.IDENTITY_COLUMNS,
+            Capability.SCHEMAS,
+            Capability.VIEWS,
+        }
+    )
+)
+
+
+@dataclass(frozen=True, slots=True)
+class MySQLSchemaInspector:
+    def inspect(self, connection: Connection[Cursor]) -> SchemaSnapshot:
+        catalog = _read(connection)
+        return SchemaSnapshot(
+            DatabaseDialect.MYSQL,
+            (Catalog(None, (_schema(catalog),)),),
+            _CAPABILITIES,
+        )
+
+
+@dataclass(frozen=True, slots=True, eq=False)
+class _Catalog:
+    """Grouped catalog rows. An index, not a value, so it does not compare."""
+
+    database: str
+    objects: tuple[ObjectRecord, ...]
+    columns: dict[str, list[ColumnRecord]]
+    constraints: dict[str, list[ConstraintRecord]]
+    checks: dict[str, list[CheckRecord]]
+    indexes: dict[str, list[IndexRecord]]
+    routines: dict[str, list[RoutineRecord]]
+
+
+def _read(connection: Connection[Cursor]) -> _Catalog:
+    try:
+        database = _database_name(connection)
+        objects = tuple(map(read_object, _rows(connection, sql.SCHEMA_OBJECTS)))
+        columns = tuple(map(read_column, _rows(connection, sql.COLUMNS)))
+        constraints = tuple(map(read_constraint, _rows(connection, sql.CONSTRAINTS)))
+        checks = tuple(map(read_check, _rows(connection, sql.CHECKS)))
+        indexes = tuple(map(read_index, _rows(connection, sql.INDEXES)))
+        routines = tuple(map(read_routine, _rows(connection, sql.ROUTINES)))
+    except SchemaInspectionError:
+        raise
+    except Exception as error:
+        message = f"unable to inspect MySQL schema: {error}"
+        raise SchemaInspectionError(message) from error
+    return _Catalog(
+        database,
+        objects,
+        _grouped(columns, lambda record: record.table),
+        _grouped(constraints, lambda record: record.table),
+        _grouped(checks, lambda record: record.table),
+        _grouped(indexes, lambda record: record.table),
+        _grouped(routines, lambda record: record.name),
+    )
+
+
+def _database_name(connection: Connection[Cursor]) -> str:
+    rows = _rows(connection, sql.DATABASE_NAME)
+    name = rows[0][0] if rows and rows[0] else None
+    if not isinstance(name, str) or name == "":
+        message = "MySQL inspection requires a connection with a selected database"
+        raise SchemaInspectionError(message)
+    return name
+
+
+def _rows(
+    connection: Connection[Cursor],
+    query: str,
+) -> tuple[tuple[object, ...], ...]:
+    with connection.cursor() as cursor:
+        cursor.execute(query)
+        return cast("tuple[tuple[object, ...], ...]", cursor.fetchall())
+
+
+def _schema(catalog: _Catalog) -> Schema:
+    objects = catalog.objects
+    tables = tuple(_table(item, catalog) for item in objects if item.is_table)
+    views = tuple(_view(item, catalog) for item in objects if not item.is_table)
+    return Schema(
+        Identifier(catalog.database),
+        tables,
+        views,
+        _enums(catalog),
+        routines=_routines(catalog),
+    )
+
+
+_PARAMETER_MODES = {
+    "IN": ParameterMode.IN,
+    "OUT": ParameterMode.OUT,
+    "INOUT": ParameterMode.INOUT,
+}
+
+
+def _routines(catalog: _Catalog) -> tuple[Routine, ...]:
+    """Every procedure and function this database keeps, with its parameters.
+
+    MySQL refuses a second routine of one name, so a name identifies a routine
+    on its own and nothing here separates overloads.
+    """
+    return tuple(
+        _routine(records) for _, records in sorted(catalog.routines.items()) if records
+    )
+
+
+def _routine(records: list[RoutineRecord]) -> Routine:
+    first = records[0]
+    function = first.kind.upper() == "FUNCTION"
+    kind = RoutineKind.FUNCTION if function else RoutineKind.PROCEDURE
+    returns = mysql_type(first.returns) if function and first.returns else None
+    return Routine(
+        Identifier(first.name),
+        kind,
+        tuple(
+            _routine_parameter(record)
+            for record in records
+            if record.position is not None
+        ),
+        returns,
+    )
+
+
+def _routine_parameter(record: RoutineRecord) -> RoutineParameter:
+    """MySQL states no nullability for a parameter, so one may hold nothing."""
+    return RoutineParameter(
+        None if record.parameter_name is None else Identifier(record.parameter_name),
+        mysql_type(record.parameter_type or "text"),
+        _PARAMETER_MODES.get((record.mode or "IN").upper(), ParameterMode.IN),
+    )
+
+
+def _enums(catalog: _Catalog) -> tuple[EnumType, ...]:
+    """Every column-level enum, named for the column that carries it."""
+    enums = [
+        EnumType(record.enum_name, record.enum_values, record.comment)
+        for records in catalog.columns.values()
+        for record in records
+        if record.enum_values
+    ]
+    return tuple(sorted(enums, key=lambda enum: enum.name.value))
+
+
+def _table(record: ObjectRecord, catalog: _Catalog) -> Table:
+    constraints = _by_name(catalog.constraints.get(record.name, []))
+    indexes = _by_name(catalog.indexes.get(record.name, []))
+    return Table(
+        Identifier(record.name),
+        _columns(catalog, record.name),
+        tuple(
+            _key(rows, kind)
+            for rows in constraints.values()
+            if (kind := rows[0].key_kind) is not None
+        ),
+        tuple(
+            _relation(rows, target)
+            for rows in constraints.values()
+            if rows[0].is_foreign_key
+            and (target := rows[0].target_reference) is not None
+        ),
+        tuple(_index(rows) for rows in indexes.values()),
+        tuple(_check(item) for item in catalog.checks.get(record.name, [])),
+        record.comment,
+    )
+
+
+def _view(record: ObjectRecord, catalog: _Catalog) -> View:
+    return View(
+        Identifier(record.name),
+        _columns(catalog, record.name),
+        record.definition,
+        materialized=False,
+    )
+
+
+def _columns(catalog: _Catalog, table: str) -> tuple[Column, ...]:
+    return tuple(_column(record) for record in catalog.columns.get(table, []))
+
+
+def _column(record: ColumnRecord) -> Column:
+    return Column(
+        Identifier(record.name),
+        record.data_type,
+        record.nullable,
+        record.default,
+        record.generated,
+        record.comment,
+    )
+
+
+def _key(rows: list[ConstraintRecord], kind: KeyKind) -> Key:
+    return Key(
+        kind,
+        tuple(Identifier(row.column) for row in rows),
+        Identifier(rows[0].name),
+    )
+
+
+def _relation(rows: list[ConstraintRecord], target: ObjectReference) -> Relation:
+    return Relation(
+        tuple(Identifier(row.column) for row in rows),
+        target,
+        tuple(row.target_identifier for row in rows),
+        Identifier(rows[0].name),
+        rows[0].update_action,
+        rows[0].delete_action,
+    )
+
+
+def _check(record: CheckRecord) -> CheckConstraint:
+    return CheckConstraint(SqlExpression(record.clause), Identifier(record.name))
+
+
+def _index(rows: list[IndexRecord]) -> Index:
+    first = rows[0]
+    return Index(
+        Identifier(first.name),
+        tuple(row.term for row in rows),
+        first.unique,
+        None,
+        first.method,
+    )
+
+
+def _by_name(records: list[Named]) -> dict[str, list[Named]]:
+    return _grouped(tuple(records), lambda record: record.name)
+
+
+def _grouped(
+    records: tuple[Record, ...],
+    key: Callable[[Record], str],
+) -> dict[str, list[Record]]:
+    grouped: dict[str, list[Record]] = defaultdict(list)
+    for record in records:
+        grouped[key(record)].append(record)
+    return grouped
+
+
+__all__ = ("MySQLSchemaInspector",)

pyoq/schema/mysql/source.py

@@ -0,0 +1,30 @@
+"""Configuration-backed MySQL schema source."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from pyoq.config import Configuration, DatabaseDialect
+from pyoq.errors import OperationUnavailableError
+from pyoq.schema.models import SchemaSnapshot
+from pyoq.schema.mysql.connection import MySQLConnectionProvider
+from pyoq.schema.mysql.reflection import MySQLSchemaInspector
+
+
+@dataclass(frozen=True, slots=True)
+class MySQLSchemaSource:
+    connections: MySQLConnectionProvider = field(
+        default_factory=MySQLConnectionProvider
+    )
+    inspector: MySQLSchemaInspector = field(default_factory=MySQLSchemaInspector)
+
+    def load(self, configuration: Configuration) -> SchemaSnapshot:
+        dialect = configuration.active_profile.dialect
+        if dialect is not DatabaseDialect.MYSQL:
+            message = f"MySQL inspection is not available for {dialect.value}"
+            raise OperationUnavailableError(message)
+        with self.connections.open(configuration) as connection:
+            return self.inspector.inspect(connection)
+
+
+__all__ = ("MySQLSchemaSource",)

pyoq/schema/mysql/sql.py

@@ -0,0 +1,128 @@
+"""MySQL information schema queries."""
+
+from __future__ import annotations
+
+DATABASE_NAME = "SELECT DATABASE()"
+
+SCHEMA_OBJECTS = """
+SELECT
+    t.TABLE_NAME,
+    t.TABLE_TYPE,
+    t.TABLE_COMMENT,
+    v.VIEW_DEFINITION
+FROM information_schema.TABLES AS t
+LEFT JOIN information_schema.VIEWS AS v
+    ON v.TABLE_SCHEMA = t.TABLE_SCHEMA
+    AND v.TABLE_NAME = t.TABLE_NAME
+WHERE t.TABLE_SCHEMA = DATABASE()
+    AND t.TABLE_TYPE IN ('BASE TABLE', 'VIEW')
+ORDER BY t.TABLE_NAME
+"""
+
+COLUMNS = """
+SELECT
+    TABLE_NAME,
+    COLUMN_NAME,
+    COLUMN_TYPE,
+    IS_NULLABLE,
+    COLUMN_DEFAULT,
+    EXTRA,
+    GENERATION_EXPRESSION,
+    COLUMN_COMMENT
+FROM information_schema.COLUMNS
+WHERE TABLE_SCHEMA = DATABASE()
+ORDER BY TABLE_NAME, ORDINAL_POSITION
+"""
+
+CONSTRAINTS = """
+SELECT
+    tc.TABLE_NAME,
+    tc.CONSTRAINT_NAME,
+    tc.CONSTRAINT_TYPE,
+    k.COLUMN_NAME,
+    k.REFERENCED_TABLE_SCHEMA,
+    k.REFERENCED_TABLE_NAME,
+    k.REFERENCED_COLUMN_NAME,
+    rc.UPDATE_RULE,
+    rc.DELETE_RULE
+FROM information_schema.TABLE_CONSTRAINTS AS tc
+JOIN information_schema.KEY_COLUMN_USAGE AS k
+    ON k.CONSTRAINT_SCHEMA = tc.CONSTRAINT_SCHEMA
+    AND k.CONSTRAINT_NAME = tc.CONSTRAINT_NAME
+    AND k.TABLE_NAME = tc.TABLE_NAME
+LEFT JOIN information_schema.REFERENTIAL_CONSTRAINTS AS rc
+    ON rc.CONSTRAINT_SCHEMA = tc.CONSTRAINT_SCHEMA
+    AND rc.CONSTRAINT_NAME = tc.CONSTRAINT_NAME
+    AND rc.TABLE_NAME = tc.TABLE_NAME
+WHERE tc.TABLE_SCHEMA = DATABASE()
+    AND tc.CONSTRAINT_TYPE IN ('PRIMARY KEY', 'UNIQUE', 'FOREIGN KEY')
+ORDER BY
+    tc.TABLE_NAME,
+    tc.CONSTRAINT_TYPE,
+    tc.CONSTRAINT_NAME,
+    k.ORDINAL_POSITION
+"""
+
+CHECKS = """
+SELECT
+    tc.TABLE_NAME,
+    cc.CONSTRAINT_NAME,
+    cc.CHECK_CLAUSE
+FROM information_schema.CHECK_CONSTRAINTS AS cc
+JOIN information_schema.TABLE_CONSTRAINTS AS tc
+    ON tc.CONSTRAINT_SCHEMA = cc.CONSTRAINT_SCHEMA
+    AND tc.CONSTRAINT_NAME = cc.CONSTRAINT_NAME
+WHERE tc.TABLE_SCHEMA = DATABASE()
+    AND tc.CONSTRAINT_TYPE = 'CHECK'
+ORDER BY tc.TABLE_NAME, cc.CONSTRAINT_NAME
+"""
+
+INDEXES = """
+SELECT
+    TABLE_NAME,
+    INDEX_NAME,
+    NON_UNIQUE,
+    COLUMN_NAME,
+    COLLATION,
+    EXPRESSION,
+    INDEX_TYPE
+FROM information_schema.STATISTICS
+WHERE TABLE_SCHEMA = DATABASE()
+    AND INDEX_NAME <> 'PRIMARY'
+ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX
+"""
+
+__all__ = (
+    "CHECKS",
+    "COLUMNS",
+    "CONSTRAINTS",
+    "DATABASE_NAME",
+    "INDEXES",
+    "SCHEMA_OBJECTS",
+)
+
+
+ROUTINES = """
+SELECT
+    r.SPECIFIC_NAME,
+    r.ROUTINE_TYPE,
+    r.DTD_IDENTIFIER,
+    p.ORDINAL_POSITION,
+    p.PARAMETER_NAME,
+    p.PARAMETER_MODE,
+    p.DTD_IDENTIFIER
+FROM information_schema.ROUTINES AS r
+LEFT JOIN information_schema.PARAMETERS AS p
+    ON p.SPECIFIC_SCHEMA = r.ROUTINE_SCHEMA
+    AND p.SPECIFIC_NAME = r.SPECIFIC_NAME
+    AND p.ROUTINE_TYPE = r.ROUTINE_TYPE
+    AND p.ORDINAL_POSITION > 0
+WHERE r.ROUTINE_SCHEMA = DATABASE()
+ORDER BY r.SPECIFIC_NAME, p.ORDINAL_POSITION
+"""
+"""One row per parameter, so a routine of no parameters is still one row.
+
+A function's own result is ordinal zero in this catalog, and it is left out
+here because the routine already carries it. MySQL refuses two routines of one
+name, so a name is its own identity and no overload marker is needed.
+"""

pyoq/schema/mysql/types.py

@@ -0,0 +1,105 @@
+"""MySQL declared-type normalization."""
+
+from __future__ import annotations
+
+import re
+
+from pyoq.schema.models import SqlType, TypeKind
+
+_TYPE_ARGUMENTS = re.compile(
+    r"^\s*([A-Za-z][A-Za-z ]*?)\s*(?:\(([^()]*)\))?"
+    r"((?:\s+(?:unsigned|signed|zerofill))*)\s*$",
+    re.IGNORECASE,
+)
+_MODIFIERS = frozenset({"UNSIGNED", "SIGNED", "ZEROFILL"})
+
+_EXACT_KINDS = {
+    "BOOL": TypeKind.BOOLEAN,
+    "BOOLEAN": TypeKind.BOOLEAN,
+    "TINYINT": TypeKind.SMALL_INTEGER,
+    "SMALLINT": TypeKind.SMALL_INTEGER,
+    "MEDIUMINT": TypeKind.INTEGER,
+    "INT": TypeKind.INTEGER,
+    "INTEGER": TypeKind.INTEGER,
+    "BIGINT": TypeKind.BIG_INTEGER,
+    "DECIMAL": TypeKind.DECIMAL,
+    "NUMERIC": TypeKind.DECIMAL,
+    "FLOAT": TypeKind.REAL,
+    "DOUBLE": TypeKind.DOUBLE,
+    "DOUBLE PRECISION": TypeKind.DOUBLE,
+    "REAL": TypeKind.DOUBLE,
+    "CHAR": TypeKind.STRING,
+    "VARCHAR": TypeKind.STRING,
+    "TINYTEXT": TypeKind.STRING,
+    "TEXT": TypeKind.STRING,
+    "MEDIUMTEXT": TypeKind.STRING,
+    "LONGTEXT": TypeKind.STRING,
+    "BIT": TypeKind.BINARY,
+    "BINARY": TypeKind.BINARY,
+    "VARBINARY": TypeKind.BINARY,
+    "TINYBLOB": TypeKind.BINARY,
+    "BLOB": TypeKind.BINARY,
+    "MEDIUMBLOB": TypeKind.BINARY,
+    "LONGBLOB": TypeKind.BINARY,
+    "DATE": TypeKind.DATE,
+    "TIME": TypeKind.TIME,
+    "DATETIME": TypeKind.DATETIME,
+    "TIMESTAMP": TypeKind.DATETIME,
+    "YEAR": TypeKind.SMALL_INTEGER,
+    "JSON": TypeKind.JSON,
+    "SET": TypeKind.STRING,
+}
+
+_ENUM_VALUES = re.compile(r"(?i)^enum\((.*)\)$", re.S)
+_ENUM_LABEL = re.compile(r"'((?:[^']|'')*)'")
+
+_SIZED_KINDS = frozenset({TypeKind.STRING, TypeKind.BINARY})
+_PRECISION_KINDS = frozenset({TypeKind.DECIMAL})
+
+
+def mysql_enum_values(declared_type: str) -> tuple[str, ...]:
+    """The labels a column-level enum lists, in the order it lists them.
+
+    MySQL keeps an enum on the column rather than as a type of its own, and
+    reports it as the whole declaration. The labels are its own order, which
+    is what it sorts and compares by.
+    """
+    match = _ENUM_VALUES.fullmatch(declared_type.strip())
+    if match is None:
+        return ()
+    return tuple(
+        label.replace("''", "'") for label in _ENUM_LABEL.findall(match.group(1))
+    )
+
+
+def mysql_type(declared_type: str) -> SqlType:
+    source_name = declared_type.strip() or "text"
+    match = _TYPE_ARGUMENTS.fullmatch(source_name)
+    if match is None:
+        return SqlType(TypeKind.OTHER, source_name)
+    base_name, argument_text, trailing = match.groups()
+    kind = _EXACT_KINDS.get(_base_name(base_name, trailing), TypeKind.OTHER)
+    arguments = _numeric_arguments(argument_text)
+    if kind in _SIZED_KINDS and len(arguments) == 1:
+        return SqlType(kind, source_name, length=arguments[0])
+    if kind in _PRECISION_KINDS and arguments:
+        scale = arguments[1] if len(arguments) == 2 else None
+        return SqlType(kind, source_name, precision=arguments[0], scale=scale)
+    return SqlType(kind, source_name)
+
+
+def _base_name(base: str, trailing: str) -> str:
+    words = f"{base} {trailing}".upper().split()
+    return " ".join(word for word in words if word not in _MODIFIERS)
+
+
+def _numeric_arguments(value: str | None) -> tuple[int, ...]:
+    if value is None:
+        return ()
+    parts = tuple(part.strip() for part in value.split(","))
+    if not 1 <= len(parts) <= 2 or any(not part.isdecimal() for part in parts):
+        return ()
+    return tuple(int(part) for part in parts)
+
+
+__all__ = ("mysql_type",)

pyoq/schema/postgres/__init__.py

@@ -0,0 +1,13 @@
+"""PostgreSQL schema support."""
+
+from pyoq.schema.postgres.connection import PostgresConnectionProvider
+from pyoq.schema.postgres.reflection import PostgresSchemaInspector
+from pyoq.schema.postgres.source import PostgresSchemaSource
+from pyoq.schema.postgres.types import postgres_type
+
+__all__ = (
+    "PostgresConnectionProvider",
+    "PostgresSchemaInspector",
+    "PostgresSchemaSource",
+    "postgres_type",
+)

pyoq/schema/postgres/connection.py

@@ -0,0 +1,63 @@
+"""Read-only PostgreSQL connection ownership."""
+
+from __future__ import annotations
+
+from collections.abc import Generator
+from contextlib import AbstractContextManager, contextmanager
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pyoq.config import Configuration
+from pyoq.errors import (
+    ConfigurationValidationError,
+    OperationUnavailableError,
+    SchemaInspectionError,
+)
+
+if TYPE_CHECKING:
+    from psycopg import Connection
+
+_MISSING_DRIVER = (
+    "PostgreSQL support requires the psycopg driver; install pyoq-sql[postgres]"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class PostgresConnectionProvider:
+    def open(
+        self,
+        configuration: Configuration,
+    ) -> AbstractContextManager[Connection[tuple[object, ...]]]:
+        return self._connect(_data_source_name(configuration))
+
+    @contextmanager
+    def _connect(
+        self,
+        dsn: str,
+    ) -> Generator[Connection[tuple[object, ...]]]:
+        try:
+            import psycopg
+        except ImportError as error:
+            raise OperationUnavailableError(_MISSING_DRIVER) from error
+        try:
+            connection = psycopg.connect(dsn, autocommit=True)
+        except psycopg.Error as error:
+            message = f"unable to open PostgreSQL connection: {error}"
+            raise SchemaInspectionError(message) from error
+        try:
+            connection.execute("SET default_transaction_read_only = on")
+            yield connection
+        finally:
+            connection.close()
+
+
+def _data_source_name(configuration: Configuration) -> str:
+    profile = configuration.active_profile
+    resolved = profile.resolve_dsn()
+    if resolved is None:
+        message = "PostgreSQL profiles require a dsn setting"
+        raise ConfigurationValidationError(message)
+    return resolved.reveal()
+
+
+__all__ = ("PostgresConnectionProvider",)