instructure-pysqlsync 0.8.5.dev0__py3-none-any.whl

pysqlsync/dialect/oracle/generator.py

@@ -0,0 +1,93 @@
+import datetime
+import ipaddress
+import uuid
+from typing import Any, Callable, Optional
+
+from strong_typing.auxiliary import int8, int16, int32, int64
+from strong_typing.core import JsonType
+
+from pysqlsync.base import BaseGenerator, GeneratorOptions
+from pysqlsync.dialect.oracle.mutation import OracleMutator
+from pysqlsync.formation.inspection import is_ip_address_type
+from pysqlsync.formation.object_types import Column
+from pysqlsync.formation.py_to_sql import ArrayMode, DataclassConverter, DataclassConverterOptions, EnumMode, NamespaceMapping, StructMode
+from pysqlsync.util.typing import override
+
+from .data_types import OracleIntegerType, OracleTimestampType, OracleTimeType, OracleVariableBinaryType, OracleVariableCharacterType
+from .object_types import OracleObjectFactory
+
+MIN_DATETIME = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
+MIN_DATE = datetime.date.min
+
+
+class OracleGenerator(BaseGenerator):
+    "Generator for Oracle."
+
+    converter: DataclassConverter
+
+    def __init__(self, options: GeneratorOptions) -> None:
+        super().__init__(options, OracleObjectFactory(), OracleMutator(options.synchronization))
+
+        self.check_enum_mode(exclude=[EnumMode.TYPE, EnumMode.INLINE])
+        self.check_struct_mode(matches=StructMode.JSON)
+        self.check_array_mode(include=[ArrayMode.JSON, ArrayMode.RELATION])
+
+        self.converter = DataclassConverter(
+            options=DataclassConverterOptions(
+                enum_mode=options.enum_mode or EnumMode.RELATION,
+                struct_mode=options.struct_mode or StructMode.JSON,
+                array_mode=options.array_mode or ArrayMode.JSON,
+                qualified_names=False,
+                namespaces=NamespaceMapping(options.namespaces),
+                foreign_constraints=options.foreign_constraints,
+                initialize_tables=options.initialize_tables,
+                substitutions={
+                    bytes: OracleVariableBinaryType(),
+                    datetime.time: OracleTimeType(),
+                    datetime.datetime: OracleTimestampType(),
+                    int: OracleIntegerType(),
+                    int8: OracleIntegerType(),
+                    int16: OracleIntegerType(),
+                    int32: OracleIntegerType(),
+                    int64: OracleIntegerType(),
+                    str: OracleVariableCharacterType(),
+                    uuid.UUID: OracleVariableBinaryType(16),
+                    JsonType: OracleVariableCharacterType(),
+                    ipaddress.IPv4Address: OracleVariableBinaryType(4),
+                    ipaddress.IPv6Address: OracleVariableBinaryType(16),
+                },
+                factory=self.factory,
+                skip_annotations=options.skip_annotations,
+                auto_default=options.auto_default,
+            )
+        )
+
+    @override
+    def get_current_schema_stmt(self) -> str:
+        return "SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')"
+
+    @override
+    def placeholder(self, index: int) -> str:
+        return f":{index}"
+
+    @override
+    def get_field_extractor(self, column: Column, field_name: str, field_type: type) -> Callable[[Any], Any]:
+        if field_type is uuid.UUID:
+            return lambda obj: getattr(obj, field_name).bytes
+        elif is_ip_address_type(field_type):
+            return lambda obj: getattr(obj, field_name).packed
+        elif field_type is datetime.time:
+            return lambda obj: datetime.datetime.combine(MIN_DATE, getattr(obj, field_name)) - MIN_DATETIME
+
+        return super().get_field_extractor(column, field_name, field_type)
+
+    @override
+    def get_value_transformer(self, column: Column, field_type: type) -> Optional[Callable[[Any], Any]]:
+        if field_type is uuid.UUID:
+            return lambda field: field.bytes
+        elif is_ip_address_type(field_type):
+            return lambda field: field.packed
+        elif field_type is datetime.time:
+            return lambda field: datetime.datetime.combine(datetime.date.min, field) - datetime.datetime.min
+
+        return super().get_value_transformer(column, field_type)

pysqlsync/dialect/oracle/mutation.py

@@ -0,0 +1,37 @@
+import typing
+from typing import Optional
+
+from pysqlsync.formation.mutation import Mutator
+from pysqlsync.formation.object_types import Column
+
+from .object_types import OracleColumn
+
+
+class OracleMutator(Mutator):
+    def mutate_column_stmt(self, source: Column, target: Column) -> Optional[str]:
+        source_column = typing.cast(OracleColumn, source)
+        target_column = typing.cast(OracleColumn, target)
+
+        changes: list[str] = []
+        if source_column.data_type != target_column.data_type:
+            changes.append(str(target_column.data_type))
+        if source_column.default != target_column.default:
+            if target_column.default is not None:
+                changes.append(f"DEFAULT {target_column.default_expr}")
+            else:
+                changes.append("DEFAULT NULL")
+        if source_column.nullable != target_column.nullable:
+            if not target_column.nullable:
+                changes.append("NOT NULL")
+            else:
+                changes.append("NULL")
+        if source_column.identity != target_column.identity:
+            if target_column.identity:
+                changes.append("GENERATED BY DEFAULT AS IDENTITY")
+            else:
+                changes.append("DROP IDENTITY")
+
+        if changes:
+            return f"MODIFY {source_column.name} {' '.join(changes)}"
+        else:
+            return None

pysqlsync/dialect/oracle/object_types.py

@@ -0,0 +1,59 @@
+import re
+
+from pysqlsync.formation.object_types import Column, EnumTable, ObjectFactory, Table
+from pysqlsync.model.data_types import SqlTimestampType
+
+
+class OracleColumn(Column):
+    @property
+    def default_expr(self) -> str:
+        if self.default is None:
+            raise ValueError("default value is NULL")
+
+        if isinstance(self.data_type, SqlTimestampType):
+            m = re.match(
+                r"^'(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2}) (?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})'$",
+                self.default,
+            )
+            if m:
+                return f"TIMESTAMP {self.default}"
+
+        return self.default
+
+    @property
+    def data_spec(self) -> str:
+        default = f" DEFAULT {self.default_expr}" if self.default is not None else ""
+        nullable = " NOT NULL" if not self.nullable and not self.identity else ""
+        identity = " GENERATED BY DEFAULT AS IDENTITY" if self.identity else ""
+        return f"{self.data_type}{default}{nullable}{identity}"
+
+
+class OracleTable(Table):
+    def alter_table_stmt(self, statements: list[str]) -> str:
+        return "\n".join(f"ALTER TABLE {self.name} {statement};" for statement in statements)
+
+    def drop_if_exists_stmt(self) -> str:
+        return (
+            "BEGIN\n"
+            f"    EXECUTE IMMEDIATE 'DROP TABLE {self.name} CASCADE CONSTRAINTS PURGE';\n"
+            "EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN RAISE; END IF;\n"
+            "END;"
+        )
+
+
+class OracleEnumTable(EnumTable, OracleTable):
+    pass
+
+
+class OracleObjectFactory(ObjectFactory):
+    @property
+    def column_class(self) -> type[Column]:
+        return OracleColumn
+
+    @property
+    def table_class(self) -> type[Table]:
+        return OracleTable
+
+    @property
+    def enum_table_class(self) -> type[EnumTable]:
+        return OracleEnumTable

pysqlsync/dialect/postgresql/connection.py

@@ -0,0 +1,133 @@
+import dataclasses
+import logging
+import ssl
+import typing
+from typing import Iterable, Optional, TypeVar
+
+import asyncpg
+from strong_typing.inspection import DataclassInstance, is_dataclass_type
+
+from pysqlsync.base import BaseConnection, BaseContext, ClassRef, DataSource
+from pysqlsync.connection import ConnectionSSLMode, create_context
+from pysqlsync.formation.object_types import Table
+from pysqlsync.model.properties import is_identity_type
+from pysqlsync.resultset import resultset_unwrap_dict, resultset_unwrap_tuple
+from pysqlsync.util.typing import override
+
+D = TypeVar("D", bound=DataclassInstance)
+T = TypeVar("T")
+
+LOGGER = logging.getLogger("pysqlsync.postgres")
+
+
+class PostgreSQLConnection(BaseConnection):
+    native: asyncpg.Connection
+
+    @override
+    async def open(self) -> BaseContext:
+        LOGGER.info("connecting to %s", self.params)
+
+        ssl_mode = self.params.ssl
+        if ssl_mode is None or ssl_mode is ConnectionSSLMode.disable:
+            return await self._open()
+        elif ssl_mode is ConnectionSSLMode.prefer:
+            try:
+                return await self._open(create_context(ssl_mode))
+            except ConnectionError:
+                return await self._open()
+        elif ssl_mode is ConnectionSSLMode.allow:
+            try:
+                return await self._open()
+            except ConnectionError:
+                return await self._open(create_context(ssl_mode))
+        elif ssl_mode is ConnectionSSLMode.require or ssl_mode is ConnectionSSLMode.verify_ca or ssl_mode is ConnectionSSLMode.verify_full:
+            return await self._open(create_context(ssl_mode))
+        else:
+            raise ValueError(f"unsupported SSL mode: {ssl_mode}")
+
+    async def _open(self, ctx: Optional[ssl.SSLContext] = None) -> BaseContext:
+        conn = await asyncpg.connect(
+            host=self.params.host,
+            port=self.params.port,
+            user=self.params.username,
+            password=self.params.password,
+            database=self.params.database,
+            ssl=ctx,
+        )
+
+        ver = conn.get_server_version()
+        LOGGER.info(
+            "PostgreSQL version %d.%d.%d %s",
+            ver.major,
+            ver.minor,
+            ver.micro,
+            ver.releaselevel,
+        )
+
+        self.native = conn
+        return PostgreSQLContext(self)
+
+    @override
+    async def close(self) -> None:
+        await self.native.close()
+
+
+class PostgreSQLContext(BaseContext):
+    def __init__(self, connection: PostgreSQLConnection) -> None:
+        super().__init__(connection)
+
+    @property
+    def native_connection(self) -> asyncpg.Connection:
+        return typing.cast(PostgreSQLConnection, self.connection).native
+
+    @override
+    async def _execute(self, statement: str) -> None:
+        await self.native_connection.execute(statement)
+
+    @override
+    async def _execute_all(self, statement: str, source: DataSource) -> None:
+        async for batch in source.batches():
+            await self.native_connection.executemany(statement, batch)
+
+    @override
+    async def _query_all(self, signature: type[T], statement: str) -> list[T]:
+        records: list[asyncpg.Record] = await self.native_connection.fetch(statement)
+        if is_dataclass_type(signature):
+            return resultset_unwrap_dict(signature, records)  # type: ignore
+        else:
+            return resultset_unwrap_tuple(signature, records)  # type: ignore
+
+    @override
+    async def insert_data(self, table: type[D], data: Iterable[D]) -> None:
+        if not is_dataclass_type(table):
+            raise TypeError(f"expected dataclass type, got: {table}")
+        generator = self.connection.generator
+        table_name = generator.get_qualified_id(ClassRef(table))
+        records = generator.get_dataclasses_as_records(table, data, skip_identity=True)
+        result = await self.native_connection.copy_records_to_table(
+            schema_name=table_name.scope_id,
+            table_name=table_name.local_id,
+            columns=tuple(field.name for field in dataclasses.fields(table) if not is_identity_type(field.type)),
+            records=records,
+        )
+        LOGGER.debug(result)
+
+    @override
+    async def _insert_rows(
+        self,
+        table: Table,
+        source: DataSource,
+        *,
+        field_types: tuple[type, ...],
+        field_names: Optional[tuple[str, ...]] = None,
+    ) -> None:
+        record_generator = await self._generate_records(table, source, field_types=field_types, field_names=field_names)
+        order = tuple(name for name in field_names if name) if field_names else None
+        async for batch in record_generator.batches():
+            result = await self.native_connection.copy_records_to_table(
+                schema_name=table.name.scope_id,
+                table_name=table.name.local_id,
+                columns=[str(col.name.local_id) for col in table.get_columns(order)],
+                records=batch,
+            )
+            LOGGER.debug(result)

pysqlsync/dialect/postgresql/data_types.py

@@ -0,0 +1,6 @@
+from pysqlsync.model.data_types import SqlJsonType
+
+
+class PostgreSQLJsonType(SqlJsonType):
+    def __str__(self) -> str:
+        return "jsonb"

pysqlsync/dialect/postgresql/dependency.py

@@ -0,0 +1,9 @@
+"""
+pysqlsync: Synchronize schema and large volumes of data.
+
+This module defines dependencies required for PostgreSQL.
+
+:see: https://github.com/instructure-internal/pysqlsync
+"""
+
+import asyncpg  # pyright: ignore[reportUnusedImport] # noqa: F401