instructure-pysqlsync 0.8.5.dev0__py3-none-any.whl

pysqlsync/dialect/mysql/mutation.py

@@ -0,0 +1,65 @@
+import typing
+from typing import Optional
+
+from pysqlsync.formation.mutation import Mutator
+from pysqlsync.formation.object_types import Column, StatementList, Table, deleted, join_or_none
+from pysqlsync.model.data_types import SqlEnumType, SqlVariableCharacterType, quote
+from pysqlsync.model.id_types import LocalId
+
+from .object_types import MySQLColumn, MySQLTable
+
+
+class MySQLMutator(Mutator):
+    def migrate_column_stmt(self, source_table: Table, source: Column, target_table: Table, target: Column) -> Optional[str]:
+        statements: list[str] = []
+        ref = target_table.get_constraint(target.name)
+        if isinstance(source.data_type, SqlEnumType):
+            enum_values = ", ".join(f"({quote(v)})" for v in source.data_type.values)
+            statements.append(f'INSERT INTO {ref.table} ("value") VALUES {enum_values}\nON DUPLICATE KEY UPDATE "value" = "value";')
+        elif isinstance(source.data_type, SqlVariableCharacterType):
+            statements.append(
+                f'INSERT INTO {ref.table} ("value") SELECT DISTINCT {LocalId(deleted(source.name.id))} FROM {source_table.name}\n'
+                'ON DUPLICATE KEY UPDATE "value" = "value";'
+            )
+        statements.append(
+            f"UPDATE {source_table.name} data_table\n"
+            f'JOIN {ref.table} enum_table ON data_table.{LocalId(deleted(source.name.id))} = enum_table."value"\n'
+            f'SET data_table.{target.name} = enum_table."id";'
+        )
+        return "\n".join(statements)
+
+    def mutate_table_stmt(self, source: Table, target: Table) -> Optional[str]:
+        source = typing.cast(MySQLTable, source)
+        target = typing.cast(MySQLTable, target)
+
+        statements: StatementList = StatementList()
+        statements.append(super().mutate_table_stmt(source, target))
+
+        source_desc = source.short_description
+        target_desc = target.short_description
+
+        if source_desc is None:
+            if target_desc is not None:
+                statements.append(f"ALTER TABLE {target.name} COMMENT = {quote(target_desc)};")
+        else:
+            if target_desc is None:
+                statements.append(f"ALTER TABLE {target.name} COMMENT = {quote('')};")
+            elif source_desc != target_desc:
+                statements.append(f"ALTER TABLE {target.name} COMMENT = {quote(target_desc)};")
+
+        return join_or_none(statements)
+
+    def mutate_column_stmt(self, source: Column, target: Column) -> Optional[str]:
+        source = typing.cast(MySQLColumn, source)
+        target = typing.cast(MySQLColumn, target)
+
+        if (
+            source.data_type != target.data_type
+            or source.nullable != target.nullable
+            or source.default != target.default
+            or source.identity != target.identity
+            or source.comment != target.comment
+        ):
+            return f"MODIFY COLUMN {source.name} {target.data_spec}"
+        else:
+            return None

pysqlsync/dialect/mysql/object_types.py

@@ -0,0 +1,75 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from pysqlsync.formation.object_types import (
+    Column,
+    EnumTable,
+    FormationError,
+    ObjectFactory,
+    Table,
+)
+from pysqlsync.model.data_types import SqlEnumType, quote
+from pysqlsync.model.id_types import LocalId
+
+
+class MySQLTable(Table):
+    @property
+    def short_description(self) -> Optional[str]:
+        if self.description is None:
+            return None
+
+        return self.description if "\n" not in self.description else self.description[: self.description.index("\n")]
+
+    def create_stmt(self) -> str:
+        defs: list[str] = []
+        defs.extend(str(c) for c in self.columns.values())
+        primary_columns = ", ".join(str(key) for key in self.primary_key)
+        defs.append(f"CONSTRAINT {self.primary_key_constraint_id} PRIMARY KEY ({primary_columns})")
+        definition = ",\n".join(defs)
+        comment = f"\nCOMMENT = {quote(self.short_description)}" if self.short_description else ""
+        return f"CREATE TABLE {self.name} (\n{definition}\n){comment};"
+
+
+class MySQLEnumTable(EnumTable, MySQLTable):
+    pass
+
+
+@dataclass(eq=True)
+class MySQLColumn(Column):
+    @property
+    def data_spec(self) -> str:
+        charset = " CHARACTER SET ascii COLLATE ascii_bin" if isinstance(self.data_type, SqlEnumType) else ""
+        nullable = " NOT NULL" if not self.nullable else ""
+        default = f" DEFAULT {self.default}" if self.default is not None else ""
+        identity = " AUTO_INCREMENT" if self.identity else ""
+        description = f" COMMENT {self.comment}" if self.description is not None else ""
+        return f"{self.data_type}{charset}{nullable}{default}{identity}{description}"
+
+    @property
+    def comment(self) -> Optional[str]:
+        if self.description is not None:
+            description = self.description if "\n" not in self.description else self.description[: self.description.index("\n")]
+
+            if len(description) > 1024:
+                raise FormationError(f"comment for column {self.name} too long, expected: maximum 1024; got: {len(description)}")
+
+            return quote(description)
+        else:
+            return None
+
+    def rename_stmt(self, name: str) -> str:
+        return f"CHANGE {self.name} {LocalId(name)} {self.data_spec}"
+
+
+class MySQLObjectFactory(ObjectFactory):
+    @property
+    def column_class(self) -> type[Column]:
+        return MySQLColumn
+
+    @property
+    def table_class(self) -> type[Table]:
+        return MySQLTable
+
+    @property
+    def enum_table_class(self) -> type[EnumTable]:
+        return MySQLEnumTable

pysqlsync/dialect/oracle/README.md

@@ -0,0 +1,22 @@
+# Oracle dialect for pysqlsync
+
+## Testing
+
+Refer to [Oracle Database Free Release Quick Start](https://www.oracle.com/database/free/get-started/) for setting up Oracle in Docker.
+
+Specifically, pull the Docker image with the command:
+```sh
+docker pull container-registry.oracle.com/database/free:latest
+```
+
+Next, start the Docker container with the command:
+```sh
+docker run -d --name oracle-db -e "ORACLE_PWD=<?YourStrong@Passw0rd>" -p 1521:1521 container-registry.oracle.com/database/free:latest
+```
+
+In order to inspect database startup logs, run:
+```sh
+docker logs oracle-db
+```
+
+Finally, run unit and integration tests.

pysqlsync/dialect/oracle/connection.py

@@ -0,0 +1,141 @@
+import logging
+import re
+import typing
+from typing import Iterable, Optional, TypeVar
+
+import oracledb
+from strong_typing.inspection import DataclassInstance, is_dataclass_type
+
+from pysqlsync.base import BaseConnection, BaseContext, DataSource, QueryException, RecordType
+from pysqlsync.formation.object_types import Table
+from pysqlsync.model.data_types import escape_like
+from pysqlsync.model.id_types import LocalId
+from pysqlsync.resultset import resultset_unwrap_tuple
+from pysqlsync.util.dispatch import thread_dispatch
+from pysqlsync.util.typing import override
+
+from .data_types import sql_to_oracle_type
+
+D = TypeVar("D", bound=DataclassInstance)
+T = TypeVar("T")
+
+LOGGER = logging.getLogger("pysqlsync.oracle")
+
+
+class OracleConnection(BaseConnection):
+    native: oracledb.Connection
+
+    @override
+    @thread_dispatch
+    def open(self) -> BaseContext:
+        LOGGER.info("connecting to %s", self.params)
+
+        host = self.params.host or "localhost"
+        port = self.params.port or 1521
+        database = self.params.database or "FREEPDB1"
+
+        conn = oracledb.connect(
+            user=self.params.username,
+            password=self.params.password,
+            dsn=f"{host}:{port}/{database}",
+        )
+        conn.autocommit = True
+        with conn.cursor() as cur:
+            cur.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS'")
+
+        self.native = conn
+        return OracleContext(self)
+
+    @override
+    @thread_dispatch
+    def close(self) -> None:
+        self.native.close()
+
+
+class OracleContext(BaseContext):
+    def __init__(self, connection: OracleConnection) -> None:
+        super().__init__(connection)
+
+    @property
+    def native_connection(self) -> oracledb.Connection:
+        return typing.cast(OracleConnection, self.connection).native
+
+    @override
+    @thread_dispatch
+    def _execute(self, statement: str) -> None:
+        with self.native_connection.cursor() as cur:
+            if statement.startswith("BEGIN"):
+                cur.execute(statement)
+            else:
+                statement = statement.rstrip("\r\n\t\v ;")
+                for s in re.split(r";$", statement, flags=re.MULTILINE):
+                    try:
+                        cur.execute(s)
+                    except Exception as e:
+                        raise QueryException(s) from e
+
+    @override
+    async def _execute_all(self, statement: str, source: DataSource) -> None:
+        async for batch in source.batches():
+            await self._internal_execute_all(statement, batch)
+
+    @override
+    async def _execute_typed(
+        self,
+        statement: str,
+        source: DataSource,
+        table: Table,
+        order: Optional[tuple[str, ...]] = None,
+    ) -> None:
+        async for batch in source.batches():
+            await self._internal_execute_typed(statement, batch, table, order)
+
+    @thread_dispatch
+    def _internal_execute_all(self, statement: str, records: Iterable[RecordType]) -> None:
+        statement = statement.rstrip("\r\n\t\v ;")
+        with self.native_connection.cursor() as cur:
+            cur.executemany(statement, list(records))
+
+    @thread_dispatch
+    def _internal_execute_typed(
+        self,
+        statement: str,
+        records: Iterable[RecordType],
+        table: Table,
+        order: Optional[tuple[str, ...]] = None,
+    ) -> None:
+        statement = statement.rstrip("\r\n\t\v ;")
+        with self.native_connection.cursor() as cur:
+            cur.setinputsizes(*tuple(sql_to_oracle_type(cur, column.data_type) for column in table.get_columns(order)))
+            cur.executemany(statement, list(records))
+
+    @override
+    @thread_dispatch
+    def _query_all(self, signature: type[T], statement: str) -> list[T]:
+        with self.native_connection.cursor() as cur:
+            cur.execute(statement)
+            if is_dataclass_type(signature):
+                cur.rowfactory = lambda *args: signature(*args)
+
+            records = cur.fetchall()
+
+            if is_dataclass_type(signature):
+                return typing.cast(list[T], records)
+            else:
+                return resultset_unwrap_tuple(signature, records)
+
+    @override
+    async def current_schema(self) -> Optional[str]:
+        return None
+
+    @override
+    async def drop_schema(self, namespace: LocalId) -> None:
+        LOGGER.debug("drop schema: %s", namespace)
+        condition = f"table_name LIKE '{escape_like(namespace.id, '~')}~_~_%' ESCAPE '~'"
+        tables = await self.query_all(
+            str,
+            f"SELECT table_name FROM dba_tables WHERE {condition}",
+        )
+        statement = "\n".join(f"DROP TABLE {LocalId(table)} CASCADE CONSTRAINTS PURGE;" for table in tables)
+        if statement:
+            await self.execute(statement)

pysqlsync/dialect/oracle/data_types.py

@@ -0,0 +1,104 @@
+import oracledb
+
+from pysqlsync.model.data_types import (
+    SqlBooleanType,
+    SqlDataType,
+    SqlDateType,
+    SqlDecimalType,
+    SqlDoubleType,
+    SqlFixedCharacterType,
+    SqlIntegerType,
+    SqlRealType,
+    SqlTimestampType,
+    SqlTimeType,
+    SqlVariableBinaryType,
+    SqlVariableCharacterType,
+)
+
+
+class OracleIntegerType(SqlIntegerType):
+    def __init__(self) -> None:
+        super().__init__(8)
+
+    def __str__(self) -> str:
+        return "number"
+
+
+class OracleTimestampType(SqlTimestampType):
+    def __str__(self) -> str:
+        if self.precision is not None and self.precision != 6:
+            precision = f"({self.precision})"
+        else:
+            precision = ""
+        if self.with_time_zone:
+            time_zone = " with time zone"
+        else:
+            time_zone = ""
+        return f"timestamp{precision}{time_zone}"
+
+
+class OracleTimeType(SqlTimeType):
+    def __str__(self) -> str:
+        return "interval day to second"
+
+
+class OracleVariableCharacterType(SqlVariableCharacterType):
+    def __str__(self) -> str:
+        if self.limit is not None and self.limit <= 4000:
+            return f"varchar2({self.limit})"
+        else:
+            return "clob"
+
+
+class OracleVariableBinaryType(SqlVariableBinaryType):
+    def __str__(self) -> str:
+        if self.storage is not None and self.storage <= 2000:
+            return f"raw({self.storage})"
+        else:
+            return "blob"
+
+
+def sql_to_oracle_type(cur: oracledb.Cursor, data_type: SqlDataType) -> oracledb.Var:
+    """
+    Returns the Oracle data type associated with the SQL data type.
+
+    Passing the right data types to `setinputsizes` eliminates data-based guessing, and can speed up `executemany`
+    by a significant factor.
+    """
+
+    if isinstance(data_type, SqlBooleanType):
+        return cur.var(oracledb.DB_TYPE_BOOLEAN)
+    elif isinstance(data_type, SqlRealType):
+        return cur.var(oracledb.DB_TYPE_BINARY_FLOAT)
+    elif isinstance(data_type, SqlDoubleType):
+        return cur.var(oracledb.DB_TYPE_BINARY_DOUBLE)
+    elif isinstance(data_type, SqlIntegerType):
+        if data_type.width > 4:
+            return cur.var(oracledb.DB_TYPE_NUMBER)
+        else:
+            return cur.var(oracledb.DB_TYPE_BINARY_INTEGER, data_type.width)
+    elif isinstance(data_type, SqlVariableBinaryType):
+        if data_type.storage is not None:
+            return cur.var(oracledb.DB_TYPE_RAW, data_type.storage)
+        else:
+            return cur.var(oracledb.DB_TYPE_BLOB)
+    elif isinstance(data_type, SqlFixedCharacterType):
+        if data_type.limit is not None:
+            return cur.var(oracledb.DB_TYPE_CHAR, data_type.limit)
+        else:
+            return cur.var(oracledb.DB_TYPE_CHAR, 4000)
+    elif isinstance(data_type, SqlVariableCharacterType):
+        if data_type.limit is not None:
+            return cur.var(oracledb.DB_TYPE_VARCHAR, data_type.limit)
+        else:
+            return cur.var(oracledb.DB_TYPE_CLOB)
+    elif isinstance(data_type, SqlDateType):
+        return cur.var(oracledb.DB_TYPE_DATE)
+    elif isinstance(data_type, SqlTimeType):
+        return cur.var(oracledb.DB_TYPE_INTERVAL_DS)
+    elif isinstance(data_type, SqlDecimalType):
+        return cur.var(oracledb.DB_TYPE_NUMBER)
+    elif isinstance(data_type, SqlTimestampType):
+        return cur.var(oracledb.DB_TYPE_TIMESTAMP)
+
+    return cur.var(oracledb.DB_TYPE_UNKNOWN)

pysqlsync/dialect/oracle/dependency.py

@@ -0,0 +1,9 @@
+"""
+pysqlsync: Synchronize schema and large volumes of data.
+
+This module defines dependencies required for Oracle.
+
+:see: https://github.com/instructure-internal/pysqlsync
+"""
+
+import oracledb  # pyright: ignore[reportUnusedImport] # noqa: F401

pysqlsync/dialect/oracle/discovery.py

@@ -0,0 +1,231 @@
+import logging
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+from pysqlsync.base import BaseContext, Explorer
+from pysqlsync.formation.constraints import ForeignFactory, UniqueFactory
+from pysqlsync.formation.data_types import SqlDiscovery, SqlDiscoveryOptions
+from pysqlsync.formation.object_types import Column, Constraint, Namespace, Table
+from pysqlsync.model.data_types import escape_like, quote
+from pysqlsync.model.id_types import (
+    LocalId,
+    PrefixedId,
+    QualifiedId,
+    SupportsQualifiedId,
+)
+
+from .data_types import (
+    OracleIntegerType,
+    OracleTimestampType,
+    OracleTimeType,
+    OracleVariableBinaryType,
+    OracleVariableCharacterType,
+)
+from .object_types import OracleObjectFactory
+
+LOGGER = logging.getLogger("pysqlsync.oracle")
+
+
+@dataclass
+class OracleColumnMeta:
+    column_name: str
+    data_type: str
+    data_type_owner: str
+    data_length: int
+    data_precision: int
+    data_scale: int
+    is_nullable: bool
+    data_default: str
+    char_length: int
+    is_identity: bool
+    comments: str
+
+
+@dataclass
+class OracleConstraintMeta:
+    constraint_name: str
+    constraint_type: str
+    source_column: str
+    target_table: str
+    target_column: str
+
+
+class OracleExplorer(Explorer):
+    discovery: SqlDiscovery
+    factory: OracleObjectFactory
+
+    def __init__(self, conn: BaseContext) -> None:
+        super().__init__(conn)
+        self.discovery = SqlDiscovery(
+            SqlDiscoveryOptions(
+                substitutions={
+                    "BLOB": OracleVariableBinaryType(),
+                    "CLOB": OracleVariableCharacterType(),
+                    "INTERVAL DAY": OracleTimeType(),
+                    "LONG": OracleVariableCharacterType(),
+                    "NUMBER": OracleIntegerType(),
+                    "RAW": OracleVariableBinaryType(),
+                    "TIMESTAMP": OracleTimestampType(),
+                    "VARCHAR2": OracleVariableCharacterType(),
+                }
+            )
+        )
+        self.factory = OracleObjectFactory()
+
+    def get_qualified_id(self, namespace: Optional[str], id: str) -> SupportsQualifiedId:
+        return PrefixedId(namespace, id)
+
+    def split_composite_id(self, name: str) -> SupportsQualifiedId:
+        if "__" in name:
+            parts = name.split("__", 1)
+            return PrefixedId(parts[0], parts[1])
+        else:
+            return PrefixedId(None, name)
+
+    async def get_table_names(self) -> list[QualifiedId]:
+        raise NotImplementedError()
+
+    async def has_table(self, table_id: SupportsQualifiedId) -> bool:
+        raise NotImplementedError()
+
+    async def has_column(self, table_id: SupportsQualifiedId, column_id: LocalId) -> bool:
+        raise NotImplementedError()
+
+    async def get_table(self, table_id: SupportsQualifiedId) -> Table:
+        condition = f"t.table_name = {quote(table_id.local_id)}"
+
+        table_comments = await self.conn.query_all(
+            str,
+            "SELECT tc.comments\n"
+            "FROM dba_tables t JOIN dba_tab_comments tc ON t.owner = tc.owner AND t.table_name = tc.table_name\n"
+            f"WHERE t.owner != 'SYS' AND {condition}\n AND t.tablespace_name != 'SYSAUX'\n"
+            "    AND t.status = 'VALID' AND t.temporary = 'N' AND t.secondary = 'N' AND t.dropped = 'NO' AND t.read_only = 'NO'",
+        )
+
+        column_records = await self.conn.query_all(
+            OracleColumnMeta,
+            "SELECT\n"
+            "    t.column_name,\n"
+            "    t.data_type,\n"
+            "    t.data_type_owner,\n"
+            "    t.data_length,\n"
+            "    t.data_precision,\n"
+            "    t.data_scale,\n"
+            "    t.nullable != 'N' AS is_nullable,\n"
+            "    t.data_default,\n"
+            "    t.char_length,\n"
+            "    t.identity_column = 'YES' AS is_identity,\n"
+            "    tc.comments\n"
+            "FROM dba_tab_columns t\n"
+            "    JOIN dba_col_comments tc ON t.owner = tc.owner AND t.table_name = tc.table_name AND t.column_name = tc.column_name\n"
+            f"WHERE {condition}\n"
+            "ORDER BY column_id",
+        )
+
+        columns: list[Column] = []
+        for col in column_records:
+            char_length: Optional[int]
+            if col.data_type in ["BLOB", "CLOB"]:
+                char_length = None
+            else:
+                char_length = col.char_length or col.data_length
+            m = re.match("^([^()]+)", col.data_type)
+            if m is not None:
+                data_type = m.group(1)
+            else:
+                data_type = col.data_type
+
+            data_type = self.discovery.sql_data_type_from_spec(
+                type_name=data_type,
+                type_schema=col.data_type_owner,
+                type_def=col.data_type,
+                character_maximum_length=char_length,
+                numeric_precision=col.data_precision,
+                numeric_scale=col.data_scale,
+            )
+            nullable = bool(col.is_nullable)
+            identity = bool(col.is_identity)
+            columns.append(
+                self.factory.column_class(
+                    LocalId(col.column_name),
+                    data_type,
+                    nullable,
+                    identity=identity,
+                    default=col.data_default if not identity else None,
+                    description=col.comments,
+                )
+            )
+
+        constraint_records = await self.conn.query_all(
+            OracleConstraintMeta,
+            f"SELECT\n"
+            "    t.constraint_name AS constraint_name,\n"
+            "    t.constraint_type AS constraint_type,\n"
+            "    tc.column_name AS source_column,\n"
+            "    r.table_name AS target_table,\n"
+            "    rc.column_name AS target_column\n"
+            "FROM dba_constraints t\n"
+            "    JOIN dba_cons_columns tc ON t.owner = tc.owner AND t.constraint_name = tc.constraint_name\n"
+            "    LEFT JOIN dba_constraints r ON t.r_owner = r.owner AND t.r_constraint_name = r.constraint_name\n"
+            "    LEFT JOIN dba_cons_columns rc ON r.owner = rc.owner AND r.constraint_name = rc.constraint_name\n"
+            f"WHERE {condition}\n"
+            "ORDER BY tc.position",
+        )
+
+        primary: list[LocalId] = []
+        unique = UniqueFactory()
+        foreign = ForeignFactory()
+        for con in constraint_records:
+            if con.constraint_type == "P":
+                primary.append(LocalId(con.source_column))
+            elif con.constraint_type == "U":
+                unique.add(con.constraint_name, LocalId(con.source_column))
+            elif con.constraint_type == "R":
+                foreign.add(
+                    con.constraint_name,
+                    LocalId(con.source_column),
+                    self.split_composite_id(con.target_table),
+                    LocalId(con.target_column),
+                )
+
+        constraints: list[Constraint] = []
+        constraints.extend(unique.fetch())
+        constraints.extend(foreign.fetch())
+
+        return self.factory.table_class(
+            name=table_id,
+            columns=columns,
+            primary_key=tuple(primary),
+            constraints=constraints or None,
+            description=table_comments[0] if table_comments else None,
+        )
+
+    async def get_namespace_current(self) -> Namespace:
+        return await self._get_namespace()
+
+    async def get_namespace(self, namespace_id: LocalId) -> Namespace:
+        return await self._get_namespace(namespace_id)
+
+    async def _get_namespace(self, namespace_id: Optional[LocalId] = None) -> Namespace:
+        if namespace_id is not None:
+            condition = f"table_name LIKE '{escape_like(namespace_id.id, '~')}~_~_%' ESCAPE '~'"
+        else:
+            condition = "table_name NOT LIKE '%~_~_%' ESCAPE '~'"
+
+        table_names = await self.conn.query_all(
+            str,
+            "SELECT table_name FROM dba_tables\n"
+            f"WHERE owner != 'SYS' AND {condition}\n AND table_name NOT LIKE '%$%' AND tablespace_name != 'SYSAUX'\n"
+            "    AND status = 'VALID' AND temporary = 'N' AND secondary = 'N' AND dropped = 'NO' AND read_only = 'NO'",
+        )
+
+        tables: list[Table] = []
+        for table_name in table_names:
+            table = await self.get_table(self.split_composite_id(table_name))
+            tables.append(table)
+
+        if tables:
+            return self.factory.namespace_class(enums=[], structs=[], tables=tables)
+        else:
+            return self.factory.namespace_class()

pysqlsync/dialect/oracle/engine.py

@@ -0,0 +1,20 @@
+from pysqlsync.base import BaseConnection, BaseEngine, BaseGenerator, Explorer
+
+from .connection import OracleConnection
+from .discovery import OracleExplorer
+from .generator import OracleGenerator
+
+
+class OracleEngine(BaseEngine):
+    @property
+    def name(self) -> str:
+        return "oracle"
+
+    def get_generator_type(self) -> type[BaseGenerator]:
+        return OracleGenerator
+
+    def get_connection_type(self) -> type[BaseConnection]:
+        return OracleConnection
+
+    def get_explorer_type(self) -> type[Explorer]:
+        return OracleExplorer