instructure-pysqlsync 0.8.5.dev0__py3-none-any.whl

pysqlsync/factory.py

@@ -0,0 +1,169 @@
+"""
+pysqlsync: Synchronize schema and large volumes of data.
+
+This module helps discover and register database dialects.
+
+:see: https://github.com/instructure-internal/pysqlsync
+"""
+
+import importlib
+import importlib.resources
+import logging
+import re
+import typing
+from typing import Optional
+from urllib.parse import parse_qs, unquote, urlparse
+
+from strong_typing.inspection import get_module_classes
+
+from .base import BaseConnection, BaseEngine, BaseGenerator, Explorer
+from .connection import ConnectionParameters, ConnectionSSLMode
+
+LOGGER = logging.getLogger("pysqlsync")
+
+_engines: dict[str, BaseEngine] = {}
+
+
+def register_dialect(engine_name: str, engine_factory: BaseEngine) -> None:
+    """
+    Dynamically registers a new database engine dialect.
+
+    Connections strings to registered dialects are automatically recognized by pysqlsync.
+
+    :param engine_name: The dialect name such as `postgres` or `mysql`.
+    :param engine_factory: The engine factory to register, which creates connections, explorers and generators.
+    """
+
+    if engine_name in _engines:
+        raise ValueError(f"engine already registered: {engine_name}")
+
+    _engines[engine_name] = engine_factory
+
+
+def unregister_dialect(engine_name: str) -> None:
+    """
+    Dynamically removes a database engine dialect.
+
+    :param engine_name: The dialect name such as `postgres` or `mysql`.
+    """
+
+    _engines.pop(engine_name)
+
+
+def get_dialect(engine_name: str) -> BaseEngine:
+    "Looks up a database engine dialect based on its reference name."
+
+    try:
+        engine_factory = _engines[engine_name]
+    except KeyError:
+        raise ValueError(f"unrecognized dialect: {engine_name}") from None
+    else:
+        return engine_factory
+
+
+def get_parameters(url: str) -> tuple[str, ConnectionParameters]:
+    """
+    Extracts dialect and connection parameters from a connection string URL.
+
+    The connection string URL has the following structure:
+    ```
+    dialect://username:password@hostname:port/database
+    ```
+
+    For example,
+    ```
+    postgresql://root:%3C%3FYour%3AStrong%40Pass%2Fw0rd%3E@server.example.com:5432/public
+    ```
+
+    In the example above, dialect is `postgresql`, username is `root`, password is `<?Your:Strong@Pass/w0rd>`,
+    hostname is `server.example.com`, port is `5432` and the database name is `public`. Note how components are
+    URL-encoded (a.k.a. percent-encoded) to comply with RFC 3986.
+    """
+
+    parts = urlparse(url, allow_fragments=False)
+
+    ssl: Optional[ConnectionSSLMode] = None
+    if parts.query:
+        query = parse_qs(parts.query, strict_parsing=True)
+        if "ssl" in query:
+            if len(query["ssl"]) != 1:
+                raise ValueError("only a single `ssl` parameter is permitted in a connection string")
+            ssl_mode = query["ssl"][0]
+            for v in ConnectionSSLMode.__members__.values():
+                if ssl_mode == v.value:
+                    ssl = v
+                    break
+            else:
+                raise ValueError(f"unsupported SSL mode: {ssl_mode}")
+
+    return parts.scheme, ConnectionParameters(
+        host=parts.hostname,
+        port=parts.port,
+        username=unquote(parts.username) if parts.username else None,
+        password=unquote(parts.password) if parts.password else None,
+        database=parts.path.lstrip("/") if parts.path else None,
+        ssl=ssl,
+    )
+
+
+class UnavailableEngine(BaseEngine):
+    _name: str
+    _module: str
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def __init__(self, name: str, module: str) -> None:
+        self._name = name
+        self._module = module
+
+    def get_generator_type(self) -> type[BaseGenerator]:
+        self._raise_error()
+
+    def get_connection_type(self) -> type[BaseConnection]:
+        self._raise_error()
+
+    def get_explorer_type(self) -> type[Explorer]:
+        self._raise_error()
+
+    def _raise_error(self) -> typing.NoReturn:
+        raise RuntimeError(f"missing dependency: `{self._module}`; you may need to run `pip install pysqlsync[{self._name}]`")
+
+
+def discover_dialects() -> None:
+    "Discovers database engine dialects bundled with this package."
+
+    resources = importlib.resources.files(__package__).joinpath("dialect").iterdir()
+    for resource in resources:
+        if resource.name.startswith((".", "__")) or not resource.is_dir():
+            continue
+
+        # run a preliminary check importing only the module `dependency`;
+        # if the check fails, it indicates that required dependencies have not been installed (e.g. database driver)
+        try:
+            module = importlib.import_module(f".dialect.{resource.name}.dependency", package=__package__)
+        except ModuleNotFoundError as e:
+            LOGGER.debug(
+                "skipping dialect `%s`: missing dependency: `%s`; you may need to run `pip install pysqlsync[%s]`",
+                resource.name,
+                e.name,
+                resource.name,
+            )
+            register_dialect(resource.name, UnavailableEngine(resource.name, e.name or ""))
+            continue
+
+        # import the module `engine`, which acts as an entry point to connector, generator and explorer functionality
+        module = importlib.import_module(f".dialect.{resource.name}.engine", package=__package__)
+        classes = [cls for cls in get_module_classes(module) if re.match(r"^\w+Engine$", cls.__name__)]
+        engine_type = typing.cast(type[BaseEngine], classes.pop())
+        engine_factory = engine_type()
+        LOGGER.info(
+            "found dialect `%s` defined by `%s`",
+            engine_factory.name,
+            engine_type.__name__,
+        )
+        register_dialect(engine_factory.name, engine_factory)
+
+
+discover_dialects()

pysqlsync/formation/constraints.py

@@ -0,0 +1,78 @@
+from dataclasses import dataclass
+
+from ..model.id_types import LocalId, SupportsQualifiedId
+from .object_types import ConstraintReference, ForeignConstraint, UniqueConstraint
+
+
+class UniqueFactory:
+    "Builds a list of unique constraints from a table of constraint names and columns."
+
+    name_to_columns: dict[str, list[LocalId]]
+
+    def __init__(self) -> None:
+        self.name_to_columns = {}
+
+    def add(self, constraint_name: str, column: LocalId) -> None:
+        self.name_to_columns.setdefault(constraint_name, []).append(column)
+
+    def fetch(self) -> list[UniqueConstraint]:
+        return [UniqueConstraint(LocalId(name), tuple(columns)) for name, columns in self.name_to_columns.items()]
+
+
+@dataclass
+class ColumnPair:
+    source: LocalId
+    target: LocalId
+
+
+@dataclass
+class ConstraintLink:
+    target_table: SupportsQualifiedId
+    column_pairs: list[ColumnPair]
+
+    def __init__(self, target_table: SupportsQualifiedId) -> None:
+        self.target_table = target_table
+        self.column_pairs = []
+
+
+class ForeignFactory:
+    "Builds a list of foreign key constraints from a table of constraint names, source and target columns."
+
+    name_to_constraint: dict[str, ConstraintLink]
+
+    def __init__(self) -> None:
+        self.target_table = None
+        self.name_to_constraint = {}
+
+    def add(
+        self,
+        constraint_name: str,
+        source_column: LocalId,
+        target_table: SupportsQualifiedId,
+        target_column: LocalId,
+    ) -> None:
+        link = self.name_to_constraint.get(constraint_name)
+        if link is not None:
+            if target_table != link.target_table:
+                raise ValueError("inconsistent target table for foreign constraint")
+        else:
+            link = ConstraintLink(target_table)
+            self.name_to_constraint[constraint_name] = link
+
+        link.column_pairs.append(ColumnPair(source_column, target_column))
+
+    def fetch(self) -> list[ForeignConstraint]:
+        if not self.name_to_constraint:
+            return []
+
+        return [
+            ForeignConstraint(
+                LocalId(name),
+                tuple(item.source for item in link.column_pairs),
+                ConstraintReference(
+                    link.target_table,
+                    tuple(item.target for item in link.column_pairs),
+                ),
+            )
+            for name, link in self.name_to_constraint.items()
+        ]

pysqlsync/formation/data_types.py

@@ -0,0 +1,192 @@
+import copy
+import dataclasses
+import re
+from typing import Optional
+
+from ..model.data_types import (
+    SqlBooleanType,
+    SqlDataType,
+    SqlDateType,
+    SqlDecimalType,
+    SqlDoubleType,
+    SqlEnumType,
+    SqlFixedBinaryType,
+    SqlFixedCharacterType,
+    SqlFloatType,
+    SqlIntegerType,
+    SqlJsonType,
+    SqlRealType,
+    SqlTimestampType,
+    SqlTimeType,
+    SqlUserDefinedType,
+    SqlUuidType,
+    SqlVariableBinaryType,
+    SqlVariableCharacterType,
+)
+from ..model.id_types import QualifiedId
+
+
+@dataclasses.dataclass
+class SqlDiscoveryOptions:
+    substitutions: dict[str, SqlDataType] = dataclasses.field(default_factory=dict[str, SqlDataType])
+
+
+class SqlDiscovery:
+    options: SqlDiscoveryOptions
+
+    def __init__(self, options: Optional[SqlDiscoveryOptions] = None) -> None:
+        self.options = options if options is not None else SqlDiscoveryOptions()
+
+    def sql_data_type_from_name(
+        self,
+        type_name: str,
+        type_schema: Optional[str] = None,
+    ) -> Optional[SqlDataType]:
+        name = type_name.lower()
+        if name in ["bool", "boolean"]:
+            return SqlBooleanType()
+        elif name in ["tinyint", "tinyint(1)", "int1"]:
+            return SqlIntegerType(1)
+        elif name in ["smallint", "int2"]:
+            return SqlIntegerType(2)
+        elif name in ["integer", "int", "int4"]:
+            return SqlIntegerType(4)
+        elif name in ["bigint", "int8"]:
+            return SqlIntegerType(8)
+        elif name in ["decimal", "number", "numeric"]:
+            return SqlDecimalType()
+        elif name in ["real", "float4"]:
+            return SqlRealType()
+        elif name in ["double", "double precision", "float8"]:
+            return SqlDoubleType()
+        elif name == "float":
+            return SqlFloatType()
+        elif name in ["timestamp", "timestamp without time zone"]:
+            return SqlTimestampType(None, False)
+        elif name == "timestamp with time zone":
+            return SqlTimestampType(None, True)
+        elif name == "datetime":
+            return SqlTimestampType()
+        elif name == "date":
+            return SqlDateType()
+        elif name in ["time", "time without time zone"]:
+            return SqlTimeType(None, False)
+        elif name == "time with time zone":
+            return SqlTimeType(None, True)
+        elif name in ["char", "character"]:
+            return SqlFixedCharacterType()
+        elif name in ["varchar", "character varying", "text"]:
+            return SqlVariableCharacterType()
+        elif name == "binary":
+            return SqlFixedBinaryType()
+        elif name in ["varbinary", "binary varying", "bytea"]:
+            return SqlVariableBinaryType()
+        elif name in ["json", "jsonb"]:  # PostgreSQL-specific
+            return SqlJsonType()
+        elif name == "uuid":  # PostgreSQL-specific
+            return SqlUuidType()
+
+        if type_schema is not None:
+            return SqlUserDefinedType(QualifiedId(type_schema, type_name))
+
+        return None
+
+    def sql_data_type_from_def(self, type_def: str) -> Optional[SqlDataType]:
+        m = re.fullmatch(r"^(?:decimal|number|numeric)[(](\d+),\s*(\d+)[)]$", type_def, re.IGNORECASE)
+        if m is not None:
+            return SqlDecimalType(int(m.group(1)), int(m.group(2)))
+
+        m = re.fullmatch(
+            r"^timestamp[(](\d+)[)]\s*(with(?:out)? time zone)?$",
+            type_def,
+            re.IGNORECASE,
+        )
+        if m is not None:
+            if m.group(2) == "with time zone":
+                return SqlTimestampType(int(m.group(1)), True)
+            else:
+                return SqlTimestampType(int(m.group(1)), False)
+
+        m = re.fullmatch(r"^char(?:acter)?[(](\d+)[)]$", type_def, re.IGNORECASE)
+        if m is not None:
+            return SqlFixedCharacterType(int(m.group(1)))
+
+        m = re.fullmatch(r"^(?:varchar|character varying)[(](\d+)[)]$", type_def, re.IGNORECASE)
+        if m is not None:
+            return SqlVariableCharacterType(int(m.group(1)))
+
+        m = re.fullmatch(r"^binary[(](\d+)[)]$", type_def, re.IGNORECASE)
+        if m is not None:
+            return SqlFixedBinaryType(int(m.group(1)))
+
+        m = re.fullmatch(r"^(?:varbinary|binary varying)[(](\d+)[)]$", type_def, re.IGNORECASE)
+        if m is not None:
+            return SqlVariableBinaryType(int(m.group(1)))
+
+        # MySQL and Oracle
+        m = re.fullmatch(r"^enum[(](.+)[)]$", type_def, re.IGNORECASE)
+        if m is not None:
+            value_list = m.group(1)
+            values = [value for value in re.findall(r"'((?:[^']+|'')*)'(?:,|$)", value_list)]
+            return SqlEnumType(values)
+
+        return None
+
+    def sql_data_type_from_spec(
+        self,
+        *,
+        type_name: str,
+        type_schema: Optional[str] = None,
+        type_def: Optional[str] = None,
+        character_maximum_length: Optional[int] = None,
+        numeric_precision: Optional[int] = None,
+        numeric_scale: Optional[int] = None,
+        datetime_precision: Optional[int] = None,
+    ) -> SqlDataType:
+        "Determines the column type from SQL column attribute data extracted from the information schema table."
+
+        sql_type: Optional[SqlDataType] = None
+        if type_schema is None or type_schema == "pg_catalog" or type_schema == "SYS":
+            sql_type = copy.copy(self.options.substitutions.get(type_name))
+
+        if sql_type is None and type_def is not None:
+            sql_type = self.sql_data_type_from_def(type_def)
+
+        if sql_type is None:
+            sql_type = self.sql_data_type_from_name(type_name, type_schema)
+
+        if sql_type is None:
+            if type_schema is not None:
+                name = f"{type_schema}.{type_name}"
+            else:
+                name = type_name
+            raise TypeError(f"unrecognized SQL type: {name}")
+
+        if isinstance(sql_type, SqlDecimalType):
+            if numeric_precision is not None:
+                sql_type.precision = numeric_precision  # precision in base 10
+            if numeric_scale is not None:
+                sql_type.scale = numeric_scale
+        elif isinstance(sql_type, SqlFloatType):
+            if numeric_precision is not None:
+                sql_type.precision = numeric_precision  # precision in base 2
+        elif isinstance(sql_type, SqlTimestampType):
+            if datetime_precision is not None and datetime_precision > 0:
+                sql_type.precision = datetime_precision
+        elif isinstance(sql_type, SqlTimeType):
+            if datetime_precision is not None and datetime_precision > 0:
+                sql_type.precision = datetime_precision
+        elif isinstance(sql_type, SqlFixedCharacterType):
+            if character_maximum_length is not None:
+                sql_type.limit = character_maximum_length
+        elif isinstance(sql_type, SqlVariableCharacterType):
+            if character_maximum_length is not None and character_maximum_length > 0:
+                sql_type.limit = character_maximum_length
+        elif isinstance(sql_type, SqlFixedBinaryType):
+            if character_maximum_length is not None:
+                sql_type.storage = character_maximum_length
+        elif isinstance(sql_type, SqlVariableBinaryType):
+            if character_maximum_length is not None and character_maximum_length > 0:
+                sql_type.storage = character_maximum_length
+
+        return sql_type