pyoq-sql 1.0.2__py3-none-any.whl

pyoq/schema/sqlite/reflection.py

@@ -0,0 +1,393 @@
+"""SQLite metadata reflection into the immutable schema model."""
+
+from __future__ import annotations
+
+import sqlite3
+from collections import defaultdict
+from dataclasses import dataclass
+
+from pyoq.config import DatabaseDialect
+from pyoq.errors import SchemaInspectionError
+from pyoq.schema import (
+    Capability,
+    CapabilitySet,
+    Catalog,
+    CheckConstraint,
+    Column,
+    GeneratedKind,
+    GeneratedValue,
+    Identifier,
+    Index,
+    IndexTerm,
+    Key,
+    KeyKind,
+    MetadataEntry,
+    ObjectReference,
+    ReferentialAction,
+    Relation,
+    Schema,
+    SchemaSnapshot,
+    SqlExpression,
+    Table,
+    View,
+)
+from pyoq.schema.sqlite.records import (
+    ColumnRecord,
+    ForeignKeyRecord,
+    IndexRecord,
+    IndexTermRecord,
+    SchemaObjectRecord,
+    column,
+    foreign_key,
+    index,
+    index_term,
+    schema_object,
+)
+from pyoq.schema.sqlite.sql import (
+    ParsedIndexTerm,
+    check_constraints,
+    descending_primary_columns,
+    generated_columns,
+    index_parts,
+    table_options,
+    view_query,
+)
+from pyoq.schema.sqlite.types import sqlite_type
+
+_CAPABILITIES = CapabilitySet(
+    frozenset(
+        {
+            Capability.CHECK_CONSTRAINTS,
+            Capability.EXPRESSION_INDEXES,
+            Capability.GENERATED_COLUMNS,
+            Capability.PARTIAL_INDEXES,
+            Capability.VIEWS,
+        }
+    )
+)
+_SCHEMA_OBJECTS = """
+SELECT type, name, sql
+FROM sqlite_schema
+WHERE type IN ('index', 'table', 'view') AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'
+ORDER BY type, name
+"""
+
+
+@dataclass(frozen=True, slots=True)
+class SQLiteSchemaInspector:
+    def inspect(self, connection: sqlite3.Connection) -> SchemaSnapshot:
+        try:
+            objects = tuple(
+                schema_object(row) for row in connection.execute(_SCHEMA_OBJECTS)
+            )
+            object_map = {record.name: record for record in objects}
+            table_records = tuple(
+                record for record in objects if record.kind == "table"
+            )
+            tables = tuple(
+                self._table(connection, record, object_map) for record in table_records
+            )
+            views = tuple(
+                self._view(connection, record)
+                for record in objects
+                if record.kind == "view"
+            )
+        except sqlite3.Error as error:
+            message = f"unable to inspect SQLite schema: {error}"
+            raise SchemaInspectionError(message) from error
+        return SchemaSnapshot(
+            DatabaseDialect.SQLITE,
+            (Catalog(None, (Schema(None, tables, views),)),),
+            _CAPABILITIES,
+        )
+
+    def _table(
+        self,
+        connection: sqlite3.Connection,
+        record: SchemaObjectRecord,
+        objects: dict[str, SchemaObjectRecord],
+    ) -> Table:
+        column_records = _columns(connection, record.name)
+        columns = _model_columns(column_records, record.sql)
+        index_records = _indexes(connection, record.name)
+        keys = _keys(connection, column_records, index_records)
+        relations = _relations(connection, record.name, objects)
+        indexes = _model_indexes(connection, index_records, objects)
+        return Table(
+            Identifier(record.name),
+            columns,
+            keys=keys,
+            relations=relations,
+            indexes=indexes,
+            checks=_model_checks(record.sql),
+            metadata=_table_metadata(record.sql),
+        )
+
+    def _view(
+        self,
+        connection: sqlite3.Connection,
+        record: SchemaObjectRecord,
+    ) -> View:
+        columns = _model_columns(_columns(connection, record.name), None)
+        query = view_query(record.sql)
+        return View(
+            Identifier(record.name),
+            columns,
+            None if query is None else SqlExpression(query),
+        )
+
+
+def _columns(connection: sqlite3.Connection, name: str) -> tuple[ColumnRecord, ...]:
+    rows = connection.execute(
+        'SELECT cid, name, type, "notnull", dflt_value, pk, hidden '
+        "FROM pragma_table_xinfo(?) ORDER BY cid",
+        (name,),
+    )
+    return tuple(record for row in rows if (record := column(row)).hidden != 1)
+
+
+def _indexes(connection: sqlite3.Connection, name: str) -> tuple[IndexRecord, ...]:
+    rows = connection.execute(
+        'SELECT seq, name, "unique", origin, partial '
+        "FROM pragma_index_list(?) ORDER BY seq",
+        (name,),
+    )
+    return tuple(index(row) for row in rows)
+
+
+def _index_terms(
+    connection: sqlite3.Connection,
+    name: str,
+) -> tuple[IndexTermRecord, ...]:
+    rows = connection.execute(
+        "SELECT seqno, cid, name, desc, coll, key "
+        "FROM pragma_index_xinfo(?) ORDER BY seqno",
+        (name,),
+    )
+    return tuple(term for row in rows if (term := index_term(row)).key_term)
+
+
+def _model_columns(
+    records: tuple[ColumnRecord, ...],
+    create_sql: str | None,
+) -> tuple[Column, ...]:
+    generated = generated_columns(create_sql)
+    descending_primary = descending_primary_columns(create_sql)
+    single_primary = sum(record.primary_ordinal > 0 for record in records) == 1
+    without_rowid, _ = table_options(create_sql)
+    return tuple(
+        _model_column(
+            record,
+            generated.get(record.name),
+            single_primary,
+            without_rowid,
+            record.name in descending_primary,
+        )
+        for record in records
+    )
+
+
+def _model_column(
+    record: ColumnRecord,
+    generated: tuple[str, bool] | None,
+    single_primary: bool,
+    without_rowid: bool,
+    descending_primary: bool,
+) -> Column:
+    identity = (
+        record.primary_ordinal > 0
+        and single_primary
+        and not without_rowid
+        and not descending_primary
+        and record.declared_type.strip().upper() == "INTEGER"
+    )
+    generated_value = _generated_value(generated, identity)
+    return Column(
+        Identifier(record.name),
+        sqlite_type(record.declared_type),
+        nullable=not record.not_null and not identity,
+        default=None if record.default is None else SqlExpression(record.default),
+        generated=generated_value,
+    )
+
+
+def _generated_value(
+    generated: tuple[str, bool] | None,
+    identity: bool,
+) -> GeneratedValue | None:
+    if generated is not None:
+        expression, stored = generated
+        kind = (
+            GeneratedKind.COMPUTED_STORED if stored else GeneratedKind.COMPUTED_VIRTUAL
+        )
+        return GeneratedValue(kind, SqlExpression(expression))
+    if identity:
+        return GeneratedValue(GeneratedKind.IDENTITY_BY_DEFAULT)
+    return None
+
+
+def _keys(
+    connection: sqlite3.Connection,
+    columns: tuple[ColumnRecord, ...],
+    indexes: tuple[IndexRecord, ...],
+) -> tuple[Key, ...]:
+    primary_columns = tuple(
+        Identifier(record.name)
+        for record in sorted(columns, key=lambda candidate: candidate.primary_ordinal)
+        if record.primary_ordinal > 0
+    )
+    keys: list[Key] = []
+    if primary_columns:
+        keys.append(Key(KeyKind.PRIMARY, primary_columns))
+    for record in indexes:
+        if record.origin != "u":
+            continue
+        terms = _index_terms(connection, record.name)
+        names = tuple(term.column_name for term in terms)
+        if all(name is not None for name in names):
+            keys.append(
+                Key(
+                    KeyKind.UNIQUE,
+                    tuple(Identifier(name) for name in names if name is not None),
+                )
+            )
+    return tuple(keys)
+
+
+def _relations(
+    connection: sqlite3.Connection,
+    table_name: str,
+    objects: dict[str, SchemaObjectRecord],
+) -> tuple[Relation, ...]:
+    rows = connection.execute(
+        'SELECT id, seq, "table", "from", "to", on_update, on_delete, match '
+        "FROM pragma_foreign_key_list(?) ORDER BY id, seq",
+        (table_name,),
+    )
+    grouped: dict[int, list[ForeignKeyRecord]] = defaultdict(list)
+    for row in rows:
+        record = foreign_key(row)
+        grouped[record.relation_id].append(record)
+    return tuple(
+        _relation(connection, tuple(grouped[relation_id]), objects)
+        for relation_id in sorted(grouped)
+    )
+
+
+def _relation(
+    connection: sqlite3.Connection,
+    records: tuple[ForeignKeyRecord, ...],
+    objects: dict[str, SchemaObjectRecord],
+) -> Relation:
+    first = records[0]
+    target_columns = tuple(record.target_column for record in records)
+    if any(name is None for name in target_columns):
+        target_columns = _target_primary_columns(
+            connection, first.target_table, objects
+        )
+    return Relation(
+        tuple(Identifier(record.source_column) for record in records),
+        ObjectReference(Identifier(first.target_table)),
+        tuple(Identifier(name) for name in target_columns if name is not None),
+        on_update=_referential_action(first.on_update),
+        on_delete=_referential_action(first.on_delete),
+    )
+
+
+def _target_primary_columns(
+    connection: sqlite3.Connection,
+    table_name: str,
+    objects: dict[str, SchemaObjectRecord],
+) -> tuple[str, ...]:
+    if table_name not in objects:
+        message = f"foreign key references unavailable table: {table_name}"
+        raise SchemaInspectionError(message)
+    records = sorted(
+        _columns(connection, table_name), key=lambda item: item.primary_ordinal
+    )
+    names = tuple(record.name for record in records if record.primary_ordinal > 0)
+    if not names:
+        message = f"foreign key target has no primary key: {table_name}"
+        raise SchemaInspectionError(message)
+    return names
+
+
+def _model_indexes(
+    connection: sqlite3.Connection,
+    records: tuple[IndexRecord, ...],
+    objects: dict[str, SchemaObjectRecord],
+) -> tuple[Index, ...]:
+    indexes: list[Index] = []
+    for record in records:
+        if record.origin != "c":
+            continue
+        schema_record = objects.get(record.name)
+        indexes.append(_model_index(connection, record, schema_record))
+    return tuple(indexes)
+
+
+def _model_index(
+    connection: sqlite3.Connection,
+    record: IndexRecord,
+    schema_record: SchemaObjectRecord | None,
+) -> Index:
+    metadata_terms = _index_terms(connection, record.name)
+    sql = None if schema_record is None else schema_record.sql
+    parsed_terms, predicate = ((), None) if sql is None else index_parts(sql)
+    terms = tuple(
+        _model_index_term(
+            term, parsed_terms[index] if index < len(parsed_terms) else None
+        )
+        for index, term in enumerate(metadata_terms)
+    )
+    return Index(
+        Identifier(record.name),
+        terms,
+        unique=record.unique,
+        predicate=None if predicate is None else SqlExpression(predicate),
+    )
+
+
+def _model_index_term(
+    record: IndexTermRecord,
+    parsed: ParsedIndexTerm | None,
+) -> IndexTerm:
+    if record.column_name is not None:
+        return IndexTerm(Identifier(record.column_name), descending=record.descending)
+    if parsed is None:
+        message = "unable to recover SQLite expression index term"
+        raise SchemaInspectionError(message)
+    return IndexTerm(
+        expression=SqlExpression(parsed.expression),
+        descending=parsed.descending,
+    )
+
+
+def _referential_action(value: str) -> ReferentialAction:
+    normalized = value.lower().replace(" ", "-")
+    try:
+        return ReferentialAction(normalized)
+    except ValueError as error:
+        message = f"unsupported SQLite referential action: {value}"
+        raise SchemaInspectionError(message) from error
+
+
+def _model_checks(create_sql: str | None) -> tuple[CheckConstraint, ...]:
+    return tuple(
+        CheckConstraint(
+            SqlExpression(check.expression),
+            None if check.name is None else Identifier(check.name),
+        )
+        for check in check_constraints(create_sql)
+    )
+
+
+def _table_metadata(create_sql: str | None) -> tuple[MetadataEntry, ...]:
+    without_rowid, strict = table_options(create_sql)
+    return (
+        MetadataEntry("strict", str(strict).lower()),
+        MetadataEntry("without-rowid", str(without_rowid).lower()),
+    )
+
+
+__all__ = ("SQLiteSchemaInspector",)

pyoq/schema/sqlite/source.py

@@ -0,0 +1,30 @@
+"""Configuration-backed SQLite schema source."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from pyoq.config import Configuration, DatabaseDialect
+from pyoq.errors import OperationUnavailableError
+from pyoq.schema import SchemaSnapshot
+from pyoq.schema.sqlite.connection import SQLiteConnectionProvider
+from pyoq.schema.sqlite.reflection import SQLiteSchemaInspector
+
+
+@dataclass(frozen=True, slots=True)
+class SQLiteSchemaSource:
+    connections: SQLiteConnectionProvider = field(
+        default_factory=SQLiteConnectionProvider
+    )
+    inspector: SQLiteSchemaInspector = field(default_factory=SQLiteSchemaInspector)
+
+    def load(self, configuration: Configuration) -> SchemaSnapshot:
+        dialect = configuration.active_profile.dialect
+        if dialect is not DatabaseDialect.SQLITE:
+            message = f"schema inspection is not available for {dialect.value}"
+            raise OperationUnavailableError(message)
+        with self.connections.open(configuration) as connection:
+            return self.inspector.inspect(connection)
+
+
+__all__ = ("SQLiteSchemaSource",)

pyoq/schema/sqlite/sql.py

@@ -0,0 +1,254 @@
+"""Small SQLite DDL extraction helpers."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from pyoq.errors import SchemaInspectionError
+
+_IDENTIFIER = re.compile(
+    r"\s*(?:\"((?:\"\"|[^\"])*)\"|`((?:``|[^`])*)`|\[([^]]*)\]|([^\s,()]+))"
+)
+_GENERATED = re.compile(r"\b(?:GENERATED\s+ALWAYS\s+)?AS\s*\(", re.IGNORECASE)
+_CHECK = re.compile(r"\bCHECK\s*\(", re.IGNORECASE)
+_CONSTRAINT_NAME = re.compile(
+    r"\bCONSTRAINT\s+(?:\"((?:\"\"|[^\"])*)\"|`((?:``|[^`])*)`|"
+    r"\[([^]]*)\]|([^\s,()]+))\s*$",
+    re.IGNORECASE,
+)
+_ORDERING = re.compile(r"\s+(ASC|DESC)\s*$", re.IGNORECASE)
+_COLLATION = re.compile(r"\s+COLLATE\s+\S+\s*$", re.IGNORECASE)
+
+
+@dataclass(frozen=True, slots=True)
+class ParsedIndexTerm:
+    expression: str
+    descending: bool
+
+
+@dataclass(frozen=True, slots=True)
+class ParsedCheck:
+    name: str | None
+    expression: str
+
+
+def check_constraints(create_sql: str | None) -> tuple[ParsedCheck, ...]:
+    if create_sql is None:
+        return ()
+    body, _ = _first_parenthesized(create_sql)
+    checks: list[ParsedCheck] = []
+    for definition in split_top_level(body):
+        for match in _CHECK.finditer(definition):
+            if not _is_top_level(definition, match.start()):
+                continue
+            expression, _ = _balanced_from(definition, match.end() - 1)
+            checks.append(
+                ParsedCheck(
+                    _constraint_name(definition[: match.start()]), expression.strip()
+                )
+            )
+    return tuple(checks)
+
+
+def generated_columns(create_sql: str | None) -> dict[str, tuple[str, bool]]:
+    if create_sql is None:
+        return {}
+    body, _ = _first_parenthesized(create_sql)
+    generated: dict[str, tuple[str, bool]] = {}
+    for definition in split_top_level(body):
+        name, remainder = _column_definition(definition)
+        if name is None:
+            continue
+        match = _GENERATED.search(remainder)
+        if match is None or not _is_top_level(remainder, match.start()):
+            continue
+        expression, closing = _balanced_from(remainder, match.end() - 1)
+        storage = remainder[closing + 1 :].upper()
+        generated[name] = (expression.strip(), "STORED" in storage)
+    return generated
+
+
+def view_query(create_sql: str | None) -> str | None:
+    if create_sql is None:
+        return None
+    position = _top_level_keyword(create_sql, "AS")
+    if position is None:
+        return None
+    query = create_sql[position + 2 :].strip()
+    return query or None
+
+
+def index_parts(create_sql: str) -> tuple[tuple[ParsedIndexTerm, ...], str | None]:
+    body, closing = _first_parenthesized(create_sql)
+    terms = tuple(_index_term(term) for term in split_top_level(body))
+    suffix = create_sql[closing + 1 :]
+    where_position = _top_level_keyword(suffix, "WHERE")
+    predicate = None
+    if where_position is not None:
+        candidate = suffix[where_position + 5 :].strip()
+        predicate = candidate or None
+    return terms, predicate
+
+
+def table_options(create_sql: str | None) -> tuple[bool, bool]:
+    if create_sql is None:
+        return False, False
+    _, closing = _first_parenthesized(create_sql)
+    suffix = create_sql[closing + 1 :]
+    without_rowid = re.search(r"\bWITHOUT\s+ROWID\b", suffix, re.IGNORECASE)
+    strict = re.search(r"\bSTRICT\b", suffix, re.IGNORECASE)
+    return without_rowid is not None, strict is not None
+
+
+def descending_primary_columns(create_sql: str | None) -> frozenset[str]:
+    if create_sql is None:
+        return frozenset()
+    body, _ = _first_parenthesized(create_sql)
+    names: set[str] = set()
+    for definition in split_top_level(body):
+        name, remainder = _column_definition(definition)
+        if name is not None and re.search(
+            r"\bPRIMARY\s+KEY\s+DESC\b", remainder, re.IGNORECASE
+        ):
+            names.add(name)
+    return frozenset(names)
+
+
+def split_top_level(value: str) -> tuple[str, ...]:
+    parts: list[str] = []
+    start = 0
+    depth = 0
+    quote: str | None = None
+    index = 0
+    while index < len(value):
+        character = value[index]
+        if quote is not None:
+            index, quote = _advance_quote(value, index, quote)
+            continue
+        if character in {'"', "'", "`", "["}:
+            quote = "]" if character == "[" else character
+        elif character == "(":
+            depth += 1
+        elif character == ")":
+            depth -= 1
+        elif character == "," and depth == 0:
+            parts.append(value[start:index].strip())
+            start = index + 1
+        index += 1
+    parts.append(value[start:].strip())
+    return tuple(part for part in parts if part)
+
+
+def _column_definition(definition: str) -> tuple[str | None, str]:
+    match = _IDENTIFIER.match(definition)
+    if match is None:
+        return None, definition
+    raw_name = next(group for group in match.groups() if group is not None)
+    name = raw_name.replace('""', '"').replace("``", "`")
+    if name.upper() in {"CHECK", "CONSTRAINT", "FOREIGN", "PRIMARY", "UNIQUE"}:
+        return None, definition
+    return name, definition[match.end() :]
+
+
+def _index_term(value: str) -> ParsedIndexTerm:
+    order = _ORDERING.search(value)
+    descending = order is not None and order.group(1).upper() == "DESC"
+    expression = value if order is None else value[: order.start()]
+    expression = _COLLATION.sub("", expression).strip()
+    return ParsedIndexTerm(expression, descending)
+
+
+def _constraint_name(prefix: str) -> str | None:
+    match = _CONSTRAINT_NAME.search(prefix)
+    if match is None:
+        return None
+    raw_name = next(group for group in match.groups() if group is not None)
+    return raw_name.replace('""', '"').replace("``", "`")
+
+
+def _first_parenthesized(value: str) -> tuple[str, int]:
+    quote: str | None = None
+    index = 0
+    while index < len(value):
+        character = value[index]
+        if quote is not None:
+            index, quote = _advance_quote(value, index, quote)
+            continue
+        if character in {'"', "'", "`", "["}:
+            quote = "]" if character == "[" else character
+        elif character == "(":
+            return _balanced_from(value, index)
+        index += 1
+    message = "SQLite DDL is missing a parenthesized definition"
+    raise SchemaInspectionError(message)
+
+
+def _balanced_from(value: str, opening: int) -> tuple[str, int]:
+    depth = 0
+    quote: str | None = None
+    index = opening
+    while index < len(value):
+        character = value[index]
+        if quote is not None:
+            index, quote = _advance_quote(value, index, quote)
+            continue
+        if character in {'"', "'", "`", "["}:
+            quote = "]" if character == "[" else character
+        elif character == "(":
+            depth += 1
+        elif character == ")":
+            depth -= 1
+            if depth == 0:
+                return value[opening + 1 : index], index
+        index += 1
+    message = "SQLite DDL contains an unclosed parenthesis"
+    raise SchemaInspectionError(message)
+
+
+def _advance_quote(value: str, index: int, quote: str) -> tuple[int, str | None]:
+    if value[index] != quote:
+        return index + 1, quote
+    if quote != "]" and index + 1 < len(value) and value[index + 1] == quote:
+        return index + 2, quote
+    return index + 1, None
+
+
+def _top_level_keyword(value: str, keyword: str) -> int | None:
+    pattern = re.compile(rf"\b{keyword}\b", re.IGNORECASE)
+    for match in pattern.finditer(value):
+        if _is_top_level(value, match.start()):
+            return match.start()
+    return None
+
+
+def _is_top_level(value: str, stop: int) -> bool:
+    depth = 0
+    quote: str | None = None
+    index = 0
+    while index < stop:
+        character = value[index]
+        if quote is not None:
+            index, quote = _advance_quote(value, index, quote)
+            continue
+        if character in {'"', "'", "`", "["}:
+            quote = "]" if character == "[" else character
+        elif character == "(":
+            depth += 1
+        elif character == ")":
+            depth -= 1
+        index += 1
+    return depth == 0 and quote is None
+
+
+__all__ = (
+    "ParsedCheck",
+    "ParsedIndexTerm",
+    "check_constraints",
+    "descending_primary_columns",
+    "generated_columns",
+    "index_parts",
+    "split_top_level",
+    "table_options",
+    "view_query",
+)