instructure-pysqlsync 0.8.5.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- instructure_pysqlsync-0.8.5.dev0.dist-info/METADATA +244 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/RECORD +102 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/WHEEL +5 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/licenses/LICENSE +21 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/top_level.txt +1 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/zip-safe +1 -0
- pysqlsync/__init__.py +15 -0
- pysqlsync/base.py +1386 -0
- pysqlsync/connection.py +101 -0
- pysqlsync/data/__init__.py +0 -0
- pysqlsync/data/exchange.py +184 -0
- pysqlsync/data/generator.py +305 -0
- pysqlsync/dialect/README.md +35 -0
- pysqlsync/dialect/__init__.py +0 -0
- pysqlsync/dialect/delta/__init__.py +0 -0
- pysqlsync/dialect/delta/data_types.py +51 -0
- pysqlsync/dialect/delta/dependency.py +7 -0
- pysqlsync/dialect/delta/engine.py +18 -0
- pysqlsync/dialect/delta/generator.py +74 -0
- pysqlsync/dialect/delta/object_types.py +87 -0
- pysqlsync/dialect/mssql/README.md +23 -0
- pysqlsync/dialect/mssql/__init__.py +0 -0
- pysqlsync/dialect/mssql/connection.py +146 -0
- pysqlsync/dialect/mssql/data_types.py +113 -0
- pysqlsync/dialect/mssql/dependency.py +9 -0
- pysqlsync/dialect/mssql/discovery.py +99 -0
- pysqlsync/dialect/mssql/engine.py +20 -0
- pysqlsync/dialect/mssql/generator.py +88 -0
- pysqlsync/dialect/mssql/mutation.py +46 -0
- pysqlsync/dialect/mssql/object_types.py +135 -0
- pysqlsync/dialect/mysql/__init__.py +0 -0
- pysqlsync/dialect/mysql/connection.py +126 -0
- pysqlsync/dialect/mysql/data_types.py +40 -0
- pysqlsync/dialect/mysql/dependency.py +10 -0
- pysqlsync/dialect/mysql/discovery.py +145 -0
- pysqlsync/dialect/mysql/engine.py +20 -0
- pysqlsync/dialect/mysql/generator.py +140 -0
- pysqlsync/dialect/mysql/mutation.py +65 -0
- pysqlsync/dialect/mysql/object_types.py +75 -0
- pysqlsync/dialect/oracle/README.md +22 -0
- pysqlsync/dialect/oracle/__init__.py +0 -0
- pysqlsync/dialect/oracle/connection.py +141 -0
- pysqlsync/dialect/oracle/data_types.py +104 -0
- pysqlsync/dialect/oracle/dependency.py +9 -0
- pysqlsync/dialect/oracle/discovery.py +231 -0
- pysqlsync/dialect/oracle/engine.py +20 -0
- pysqlsync/dialect/oracle/generator.py +93 -0
- pysqlsync/dialect/oracle/mutation.py +37 -0
- pysqlsync/dialect/oracle/object_types.py +59 -0
- pysqlsync/dialect/postgresql/__init__.py +0 -0
- pysqlsync/dialect/postgresql/connection.py +133 -0
- pysqlsync/dialect/postgresql/data_types.py +6 -0
- pysqlsync/dialect/postgresql/dependency.py +9 -0
- pysqlsync/dialect/postgresql/discovery.py +392 -0
- pysqlsync/dialect/postgresql/engine.py +20 -0
- pysqlsync/dialect/postgresql/generator.py +99 -0
- pysqlsync/dialect/postgresql/mutation.py +82 -0
- pysqlsync/dialect/postgresql/object_types.py +99 -0
- pysqlsync/dialect/redshift/__init__.py +0 -0
- pysqlsync/dialect/redshift/connection.py +156 -0
- pysqlsync/dialect/redshift/data_types.py +10 -0
- pysqlsync/dialect/redshift/dependency.py +9 -0
- pysqlsync/dialect/redshift/engine.py +20 -0
- pysqlsync/dialect/redshift/generator.py +82 -0
- pysqlsync/dialect/snowflake/__init__.py +0 -0
- pysqlsync/dialect/snowflake/data_types.py +18 -0
- pysqlsync/dialect/snowflake/dependency.py +9 -0
- pysqlsync/dialect/snowflake/engine.py +18 -0
- pysqlsync/dialect/snowflake/generator.py +64 -0
- pysqlsync/dialect/snowflake/object_types.py +84 -0
- pysqlsync/dialect/test/__init__.py +0 -0
- pysqlsync/dialect/test/dependency.py +10 -0
- pysqlsync/dialect/trino/__init__.py +0 -0
- pysqlsync/dialect/trino/connection.py +86 -0
- pysqlsync/dialect/trino/dependency.py +9 -0
- pysqlsync/dialect/trino/discovery.py +21 -0
- pysqlsync/dialect/trino/engine.py +20 -0
- pysqlsync/dialect/trino/generator.py +31 -0
- pysqlsync/factory.py +169 -0
- pysqlsync/formation/__init__.py +0 -0
- pysqlsync/formation/constraints.py +78 -0
- pysqlsync/formation/data_types.py +192 -0
- pysqlsync/formation/discovery.py +390 -0
- pysqlsync/formation/inspection.py +179 -0
- pysqlsync/formation/mutation.py +320 -0
- pysqlsync/formation/object_dict.py +79 -0
- pysqlsync/formation/object_types.py +872 -0
- pysqlsync/formation/py_to_sql.py +1134 -0
- pysqlsync/formation/sql_to_py.py +194 -0
- pysqlsync/model/__init__.py +0 -0
- pysqlsync/model/data_types.py +397 -0
- pysqlsync/model/entity_types.py +68 -0
- pysqlsync/model/id_types.py +193 -0
- pysqlsync/model/key_types.py +38 -0
- pysqlsync/model/properties.py +234 -0
- pysqlsync/py.typed +0 -0
- pysqlsync/python_types.py +192 -0
- pysqlsync/resultset.py +106 -0
- pysqlsync/util/__init__.py +0 -0
- pysqlsync/util/dataclasses.py +80 -0
- pysqlsync/util/dispatch.py +28 -0
- pysqlsync/util/typing.py +13 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from pysqlsync.base import BaseConnection, BaseEngine, BaseGenerator, Explorer
|
|
2
|
+
|
|
3
|
+
from .generator import DeltaGenerator
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DeltaEngine(BaseEngine):
|
|
7
|
+
@property
|
|
8
|
+
def name(self) -> str:
|
|
9
|
+
return "delta"
|
|
10
|
+
|
|
11
|
+
def get_generator_type(self) -> type[BaseGenerator]:
|
|
12
|
+
return DeltaGenerator
|
|
13
|
+
|
|
14
|
+
def get_connection_type(self) -> type[BaseConnection]:
|
|
15
|
+
raise NotImplementedError()
|
|
16
|
+
|
|
17
|
+
def get_explorer_type(self) -> type[Explorer]:
|
|
18
|
+
raise NotImplementedError()
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import ipaddress
|
|
3
|
+
import uuid
|
|
4
|
+
|
|
5
|
+
from strong_typing.auxiliary import float32, float64
|
|
6
|
+
from strong_typing.core import JsonType
|
|
7
|
+
|
|
8
|
+
from pysqlsync.base import BaseGenerator, GeneratorOptions
|
|
9
|
+
from pysqlsync.formation.mutation import Mutator
|
|
10
|
+
from pysqlsync.formation.py_to_sql import (
|
|
11
|
+
ArrayMode,
|
|
12
|
+
DataclassConverter,
|
|
13
|
+
DataclassConverterOptions,
|
|
14
|
+
EnumMode,
|
|
15
|
+
NamespaceMapping,
|
|
16
|
+
StructMode,
|
|
17
|
+
)
|
|
18
|
+
from pysqlsync.model.data_types import SqlVariableCharacterType
|
|
19
|
+
from pysqlsync.util.typing import override
|
|
20
|
+
|
|
21
|
+
from .data_types import (
|
|
22
|
+
DeltaDoubleType,
|
|
23
|
+
DeltaFixedBinaryType,
|
|
24
|
+
DeltaRealType,
|
|
25
|
+
DeltaTimestampType,
|
|
26
|
+
DeltaVariableCharacterType,
|
|
27
|
+
)
|
|
28
|
+
from .object_types import DeltaObjectFactory
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class DeltaGenerator(BaseGenerator):
|
|
32
|
+
"Generator for Delta Lake on Databricks."
|
|
33
|
+
|
|
34
|
+
converter: DataclassConverter
|
|
35
|
+
|
|
36
|
+
def __init__(self, options: GeneratorOptions) -> None:
|
|
37
|
+
super().__init__(
|
|
38
|
+
options,
|
|
39
|
+
DeltaObjectFactory(),
|
|
40
|
+
Mutator(options.synchronization),
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
self.check_enum_mode(matches=EnumMode.CHECK)
|
|
44
|
+
self.check_struct_mode(exclude=[StructMode.TYPE])
|
|
45
|
+
|
|
46
|
+
self.converter = DataclassConverter(
|
|
47
|
+
options=DataclassConverterOptions(
|
|
48
|
+
enum_mode=options.enum_mode or EnumMode.CHECK,
|
|
49
|
+
struct_mode=options.struct_mode or StructMode.INLINE,
|
|
50
|
+
array_mode=options.array_mode or ArrayMode.ARRAY,
|
|
51
|
+
namespaces=NamespaceMapping(options.namespaces),
|
|
52
|
+
check_constraints=False,
|
|
53
|
+
foreign_constraints=False,
|
|
54
|
+
initialize_tables=options.initialize_tables,
|
|
55
|
+
substitutions={
|
|
56
|
+
datetime.datetime: DeltaTimestampType(),
|
|
57
|
+
float: DeltaDoubleType(),
|
|
58
|
+
float32: DeltaRealType(),
|
|
59
|
+
float64: DeltaDoubleType(),
|
|
60
|
+
str: DeltaVariableCharacterType(),
|
|
61
|
+
uuid.UUID: DeltaFixedBinaryType(16),
|
|
62
|
+
JsonType: SqlVariableCharacterType(),
|
|
63
|
+
ipaddress.IPv4Address: DeltaFixedBinaryType(4),
|
|
64
|
+
ipaddress.IPv6Address: DeltaFixedBinaryType(16),
|
|
65
|
+
},
|
|
66
|
+
factory=self.factory,
|
|
67
|
+
skip_annotations=options.skip_annotations,
|
|
68
|
+
auto_default=options.auto_default,
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
@override
|
|
73
|
+
def placeholder(self, index: int) -> str:
|
|
74
|
+
return f":{index}"
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from pysqlsync.formation.object_types import Column, Namespace, ObjectFactory, Table
|
|
5
|
+
from pysqlsync.model.id_types import LocalId
|
|
6
|
+
|
|
7
|
+
_sql_quoted_str_table = str.maketrans(
|
|
8
|
+
{
|
|
9
|
+
"\\": "\\\\",
|
|
10
|
+
"'": "\\'",
|
|
11
|
+
'"': '\\"',
|
|
12
|
+
"\0": "\\0",
|
|
13
|
+
"\b": "\\b",
|
|
14
|
+
"\n": "\\n",
|
|
15
|
+
"\r": "\\r",
|
|
16
|
+
"\t": "\\t",
|
|
17
|
+
"\u001a": "\\Z",
|
|
18
|
+
}
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def sql_quoted_string(text: str) -> str:
|
|
23
|
+
if re.search(r"[\\'\"\0\b\n\r\t\u001A]", text):
|
|
24
|
+
text = text.translate(_sql_quoted_str_table)
|
|
25
|
+
return f"'{text}'"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DeltaTable(Table):
|
|
29
|
+
def create_stmt(self) -> str:
|
|
30
|
+
defs: list[str] = []
|
|
31
|
+
defs.extend(str(c) for c in self.columns.values())
|
|
32
|
+
defs.append(self.create_keys())
|
|
33
|
+
definition = ",\n".join(defs)
|
|
34
|
+
comment = f"\nCOMMENT {sql_quoted_string(self.description)}" if self.description else ""
|
|
35
|
+
return f"CREATE TABLE {self.name} (\n{definition}\n){comment};"
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def primary_key_constraint_id(self) -> LocalId:
|
|
39
|
+
return LocalId(f"pk_{self.name.local_id.replace('.', '_')}")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class DeltaColumn(Column):
|
|
43
|
+
@property
|
|
44
|
+
def data_spec(self) -> str:
|
|
45
|
+
nullable = " NOT NULL" if not self.nullable and not self.identity else ""
|
|
46
|
+
identity = " GENERATED BY DEFAULT AS IDENTITY" if self.identity else ""
|
|
47
|
+
description = f" COMMENT {self.comment}" if self.description is not None else ""
|
|
48
|
+
|
|
49
|
+
# DEFAULT requires TBLPROPERTIES ('delta.feature.allowColumnDefaults' = 'enabled')
|
|
50
|
+
# default = f" DEFAULT {self.default}" if self.default is not None else ""
|
|
51
|
+
|
|
52
|
+
return f"{self.data_type}{nullable}{identity}{description}"
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def comment(self) -> Optional[str]:
|
|
56
|
+
if self.description is not None:
|
|
57
|
+
return sql_quoted_string(self.description)
|
|
58
|
+
else:
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class DeltaNamespace(Namespace):
|
|
63
|
+
def create_schema_stmt(self) -> Optional[str]:
|
|
64
|
+
if self.name.local_id:
|
|
65
|
+
return f"CREATE SCHEMA IF NOT EXISTS {self.name};"
|
|
66
|
+
else:
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
def drop_schema_stmt(self) -> Optional[str]:
|
|
70
|
+
if self.name.local_id:
|
|
71
|
+
return f"DROP SCHEMA IF EXISTS {self.name} CASCADE;"
|
|
72
|
+
else:
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class DeltaObjectFactory(ObjectFactory):
|
|
77
|
+
@property
|
|
78
|
+
def column_class(self) -> type[Column]:
|
|
79
|
+
return DeltaColumn
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def table_class(self) -> type[Table]:
|
|
83
|
+
return DeltaTable
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def namespace_class(self) -> type[Namespace]:
|
|
87
|
+
return DeltaNamespace
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Microsoft SQL Server dialect for pysqlsync
|
|
2
|
+
|
|
3
|
+
## Testing
|
|
4
|
+
|
|
5
|
+
Follow the steps of [running a SQL Server Linux container image with Docker](https://learn.microsoft.com/en-us/sql/linux/quickstart-install-connect-docker?view=sql-server-ver16&pivots=cs1-bash) to start testing with Microsoft SQL Server in a Linux/MacOS environment.
|
|
6
|
+
|
|
7
|
+
Specifically, pull the Docker image with the command:
|
|
8
|
+
```sh
|
|
9
|
+
docker pull mcr.microsoft.com/mssql/server:2022-latest
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Next, start the Docker container with the command:
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=<?YourStrong@Passw0rd>" -p 1433:1433 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2022-latest
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
In order to inspect database startup logs, run:
|
|
19
|
+
```sh
|
|
20
|
+
docker logs sql1
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Finally, run unit and integration tests.
|
|
File without changes
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import typing
|
|
3
|
+
from typing import Iterable, Optional, TypeVar
|
|
4
|
+
|
|
5
|
+
import pyodbc
|
|
6
|
+
from strong_typing.inspection import is_dataclass_type
|
|
7
|
+
|
|
8
|
+
from pysqlsync.base import BaseConnection, BaseContext, DataSource, RecordType
|
|
9
|
+
from pysqlsync.formation.object_types import Table
|
|
10
|
+
from pysqlsync.model.data_types import quote
|
|
11
|
+
from pysqlsync.model.id_types import LocalId, QualifiedId
|
|
12
|
+
from pysqlsync.resultset import resultset_unwrap_object, resultset_unwrap_tuple
|
|
13
|
+
from pysqlsync.util.dispatch import thread_dispatch
|
|
14
|
+
from pysqlsync.util.typing import override
|
|
15
|
+
|
|
16
|
+
from .data_types import sql_to_odbc_type
|
|
17
|
+
|
|
18
|
+
T = TypeVar("T")
|
|
19
|
+
|
|
20
|
+
LOGGER = logging.getLogger("pysqlsync.mssql")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class MSSQLConnection(BaseConnection):
|
|
24
|
+
"""
|
|
25
|
+
Represents a connection to a Microsoft SQL Server.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
native: pyodbc.Connection
|
|
29
|
+
|
|
30
|
+
@override
|
|
31
|
+
@thread_dispatch
|
|
32
|
+
def open(self) -> BaseContext:
|
|
33
|
+
LOGGER.info("connecting to %s", self.params)
|
|
34
|
+
params: dict[str, Optional[str]] = {
|
|
35
|
+
"DRIVER": "{ODBC Driver 18 for SQL Server}",
|
|
36
|
+
"APP": "pysqlsync",
|
|
37
|
+
"SERVER": (f"{self.params.host},{self.params.port}" if self.params.port is not None else self.params.host),
|
|
38
|
+
"UID": self.params.username,
|
|
39
|
+
"PWD": self.params.password,
|
|
40
|
+
"LongAsMax": "yes",
|
|
41
|
+
"TrustServerCertificate": "yes",
|
|
42
|
+
}
|
|
43
|
+
if self.params.database is not None:
|
|
44
|
+
params["DATABASE"] = self.params.database
|
|
45
|
+
conn_string = ";".join(f"{key}={value}" for key, value in params.items() if value is not None)
|
|
46
|
+
conn = pyodbc.connect(conn_string)
|
|
47
|
+
with conn.cursor() as cur:
|
|
48
|
+
rows = cur.execute("SELECT @@VERSION").fetchall()
|
|
49
|
+
for row in rows:
|
|
50
|
+
LOGGER.info(row)
|
|
51
|
+
|
|
52
|
+
self.native = conn
|
|
53
|
+
return MSSQLContext(self)
|
|
54
|
+
|
|
55
|
+
@override
|
|
56
|
+
@thread_dispatch
|
|
57
|
+
def close(self) -> None:
|
|
58
|
+
self.native.close()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class MSSQLContext(BaseContext):
|
|
62
|
+
def __init__(self, connection: MSSQLConnection) -> None:
|
|
63
|
+
super().__init__(connection)
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def native_connection(self) -> pyodbc.Connection:
|
|
67
|
+
return typing.cast(MSSQLConnection, self.connection).native
|
|
68
|
+
|
|
69
|
+
@override
|
|
70
|
+
@thread_dispatch
|
|
71
|
+
def _execute(self, statement: str) -> None:
|
|
72
|
+
with self.native_connection.cursor() as cur:
|
|
73
|
+
cur.execute(statement)
|
|
74
|
+
|
|
75
|
+
@override
|
|
76
|
+
async def _execute_all(self, statement: str, source: DataSource) -> None:
|
|
77
|
+
async for batch in source.batches():
|
|
78
|
+
await self._internal_execute_all(statement, batch)
|
|
79
|
+
|
|
80
|
+
@override
|
|
81
|
+
async def _execute_typed(
|
|
82
|
+
self,
|
|
83
|
+
statement: str,
|
|
84
|
+
source: DataSource,
|
|
85
|
+
table: Table,
|
|
86
|
+
order: Optional[tuple[str, ...]] = None,
|
|
87
|
+
) -> None:
|
|
88
|
+
async for batch in source.batches():
|
|
89
|
+
await self._internal_execute_typed(statement, batch, table, order)
|
|
90
|
+
|
|
91
|
+
@thread_dispatch
|
|
92
|
+
def _internal_execute_all(self, statement: str, records: Iterable[RecordType]) -> None:
|
|
93
|
+
with self.native_connection.cursor() as cur:
|
|
94
|
+
cur.fast_executemany = True
|
|
95
|
+
cur.executemany(statement, records)
|
|
96
|
+
|
|
97
|
+
@thread_dispatch
|
|
98
|
+
def _internal_execute_typed(
|
|
99
|
+
self,
|
|
100
|
+
statement: str,
|
|
101
|
+
records: Iterable[RecordType],
|
|
102
|
+
table: Table,
|
|
103
|
+
order: Optional[tuple[str, ...]] = None,
|
|
104
|
+
) -> None:
|
|
105
|
+
with self.native_connection.cursor() as cur:
|
|
106
|
+
cur.fast_executemany = True
|
|
107
|
+
cur.setinputsizes([sql_to_odbc_type(column.data_type) for column in table.get_columns(order)])
|
|
108
|
+
cur.executemany(statement, records)
|
|
109
|
+
|
|
110
|
+
@override
|
|
111
|
+
@thread_dispatch
|
|
112
|
+
def _query_all(self, signature: type[T], statement: str) -> list[T]:
|
|
113
|
+
with self.native_connection.cursor() as cur:
|
|
114
|
+
records = cur.execute(statement).fetchall()
|
|
115
|
+
|
|
116
|
+
if is_dataclass_type(signature):
|
|
117
|
+
return resultset_unwrap_object(signature, records) # type: ignore
|
|
118
|
+
else:
|
|
119
|
+
return resultset_unwrap_tuple(signature, records)
|
|
120
|
+
|
|
121
|
+
@override
|
|
122
|
+
async def drop_schema(self, namespace: LocalId) -> None:
|
|
123
|
+
LOGGER.debug("drop schema: %s", namespace)
|
|
124
|
+
|
|
125
|
+
constraints = await self.query_all(
|
|
126
|
+
tuple[str, str],
|
|
127
|
+
"SELECT table_name, constraint_name\n"
|
|
128
|
+
"FROM information_schema.table_constraints\n"
|
|
129
|
+
f"WHERE constraint_type = 'FOREIGN KEY' AND table_schema = {quote(namespace.id)};",
|
|
130
|
+
)
|
|
131
|
+
if constraints:
|
|
132
|
+
stmts: list[str] = []
|
|
133
|
+
for constraint in constraints:
|
|
134
|
+
table_name, constraint_name = constraint
|
|
135
|
+
stmts.append(f"ALTER TABLE {QualifiedId(namespace.id, table_name)} DROP CONSTRAINT {LocalId(constraint_name)};")
|
|
136
|
+
await self.execute("\n".join(stmts))
|
|
137
|
+
|
|
138
|
+
tables = await self.query_all(
|
|
139
|
+
str,
|
|
140
|
+
f"SELECT table_name\nFROM information_schema.tables\nWHERE table_schema = {quote(namespace.id)};",
|
|
141
|
+
)
|
|
142
|
+
if tables:
|
|
143
|
+
table_list = ", ".join(str(QualifiedId(namespace.local_id, table)) for table in tables)
|
|
144
|
+
await self.execute(f"DROP TABLE IF EXISTS {table_list};")
|
|
145
|
+
|
|
146
|
+
await self.execute(f"DROP SCHEMA IF EXISTS {namespace};")
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import enum
|
|
2
|
+
from typing import Any, Optional
|
|
3
|
+
|
|
4
|
+
import pyodbc
|
|
5
|
+
|
|
6
|
+
from pysqlsync.model.data_types import (
|
|
7
|
+
SqlBooleanType,
|
|
8
|
+
SqlDataType,
|
|
9
|
+
SqlDateType,
|
|
10
|
+
SqlDecimalType,
|
|
11
|
+
SqlDoubleType,
|
|
12
|
+
SqlFixedBinaryType,
|
|
13
|
+
SqlFixedCharacterType,
|
|
14
|
+
SqlFloatType,
|
|
15
|
+
SqlIntegerType,
|
|
16
|
+
SqlRealType,
|
|
17
|
+
SqlTimestampType,
|
|
18
|
+
SqlTimeType,
|
|
19
|
+
SqlVariableBinaryType,
|
|
20
|
+
SqlVariableCharacterType,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class MSSQLBooleanType(SqlBooleanType):
|
|
25
|
+
def __str__(self) -> str:
|
|
26
|
+
return "bit"
|
|
27
|
+
|
|
28
|
+
def value_to_sql_literal(self, value: Any) -> str:
|
|
29
|
+
if not isinstance(value, bool):
|
|
30
|
+
raise TypeError(f"expected: value of type `bool`, got: {type(value)}")
|
|
31
|
+
|
|
32
|
+
return "1" if value else "0"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class MSSQLEncoding(enum.Enum):
|
|
36
|
+
UTF8 = "utf-8"
|
|
37
|
+
UTF16 = "utf-16"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class MSSQLVariableCharacterType(SqlVariableCharacterType):
|
|
41
|
+
encoding: Optional[MSSQLEncoding] = None
|
|
42
|
+
|
|
43
|
+
def __init__(self, limit: Optional[int] = None, encoding: Optional[MSSQLEncoding] = None) -> None:
|
|
44
|
+
super().__init__(limit)
|
|
45
|
+
self.encoding = encoding
|
|
46
|
+
|
|
47
|
+
def __str__(self) -> str:
|
|
48
|
+
if self.encoding is MSSQLEncoding.UTF16:
|
|
49
|
+
char_type = "nvarchar"
|
|
50
|
+
else:
|
|
51
|
+
char_type = "varchar"
|
|
52
|
+
|
|
53
|
+
if self.limit is not None and self.limit > 0 and self.limit != 2147483647:
|
|
54
|
+
return f"{char_type}({self.limit})"
|
|
55
|
+
else:
|
|
56
|
+
return f"{char_type}(max)"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class MSSQLDateTimeType(SqlTimestampType):
|
|
60
|
+
def __init__(self) -> None:
|
|
61
|
+
self.precision = 7
|
|
62
|
+
|
|
63
|
+
def __str__(self) -> str:
|
|
64
|
+
return "datetime2"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def sql_to_odbc_type(data_type: SqlDataType) -> tuple[int, int, int]:
|
|
68
|
+
"""
|
|
69
|
+
Returns the ODBC data type associated with the SQL data type.
|
|
70
|
+
|
|
71
|
+
Passing the right data types to `setinputsizes` eliminates data-based guessing, and can speed up `executemany`
|
|
72
|
+
by a significant factor.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
if isinstance(data_type, MSSQLBooleanType):
|
|
76
|
+
return pyodbc.SQL_BIT, 0, 0
|
|
77
|
+
|
|
78
|
+
elif isinstance(data_type, SqlIntegerType):
|
|
79
|
+
if data_type.width == 1:
|
|
80
|
+
return pyodbc.SQL_TINYINT, 0, 0
|
|
81
|
+
elif data_type.width == 2:
|
|
82
|
+
return pyodbc.SQL_SMALLINT, 0, 0
|
|
83
|
+
elif data_type.width == 4:
|
|
84
|
+
return pyodbc.SQL_INTEGER, 0, 0
|
|
85
|
+
else:
|
|
86
|
+
return pyodbc.SQL_BIGINT, 0, 0
|
|
87
|
+
|
|
88
|
+
elif isinstance(data_type, SqlRealType):
|
|
89
|
+
return pyodbc.SQL_REAL, 0, 0
|
|
90
|
+
elif isinstance(data_type, SqlDoubleType):
|
|
91
|
+
return pyodbc.SQL_DOUBLE, 0, 0
|
|
92
|
+
elif isinstance(data_type, SqlFloatType):
|
|
93
|
+
return pyodbc.SQL_FLOAT, data_type.precision or 53, 0
|
|
94
|
+
elif isinstance(data_type, SqlDecimalType):
|
|
95
|
+
return pyodbc.SQL_DECIMAL, data_type.precision or 15, data_type.scale or 0
|
|
96
|
+
|
|
97
|
+
elif isinstance(data_type, SqlTimestampType):
|
|
98
|
+
return pyodbc.SQL_TYPE_TIMESTAMP, data_type.precision or 6, 0
|
|
99
|
+
elif isinstance(data_type, SqlDateType):
|
|
100
|
+
return pyodbc.SQL_TYPE_DATE, 0, 0
|
|
101
|
+
elif isinstance(data_type, SqlTimeType):
|
|
102
|
+
return pyodbc.SQL_TYPE_TIME, data_type.precision or 6, 0
|
|
103
|
+
|
|
104
|
+
elif isinstance(data_type, SqlFixedCharacterType):
|
|
105
|
+
return pyodbc.SQL_CHAR, data_type.limit or 0, 0
|
|
106
|
+
elif isinstance(data_type, SqlVariableCharacterType):
|
|
107
|
+
return pyodbc.SQL_VARCHAR, data_type.limit or 0, 0
|
|
108
|
+
elif isinstance(data_type, SqlFixedBinaryType):
|
|
109
|
+
return pyodbc.SQL_BINARY, data_type.storage or 0, 0
|
|
110
|
+
elif isinstance(data_type, SqlVariableBinaryType):
|
|
111
|
+
return pyodbc.SQL_VARBINARY, data_type.storage or 0, 0
|
|
112
|
+
|
|
113
|
+
return pyodbc.SQL_UNKNOWN_TYPE, 0, 0
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pysqlsync: Synchronize schema and large volumes of data.
|
|
3
|
+
|
|
4
|
+
This module defines dependencies required for Microsoft SQL Server.
|
|
5
|
+
|
|
6
|
+
:see: https://github.com/instructure-internal/pysqlsync
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import pyodbc # pyright: ignore[reportUnusedImport] # noqa: F401
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
from typing import Optional, Sequence
|
|
2
|
+
|
|
3
|
+
from pysqlsync.base import BaseContext
|
|
4
|
+
from pysqlsync.formation.data_types import SqlDiscovery, SqlDiscoveryOptions
|
|
5
|
+
from pysqlsync.formation.discovery import AnsiExplorer
|
|
6
|
+
from pysqlsync.formation.object_types import Namespace, Table
|
|
7
|
+
from pysqlsync.model.data_types import quote
|
|
8
|
+
from pysqlsync.model.id_types import LocalId, SupportsQualifiedId
|
|
9
|
+
|
|
10
|
+
from .data_types import MSSQLBooleanType, MSSQLDateTimeType, MSSQLEncoding, MSSQLVariableCharacterType
|
|
11
|
+
from .object_types import MSSQLColumn, MSSQLDefault, MSSQLObjectFactory
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MSSQLExplorer(AnsiExplorer):
|
|
15
|
+
def __init__(self, conn: BaseContext) -> None:
|
|
16
|
+
super().__init__(
|
|
17
|
+
conn,
|
|
18
|
+
SqlDiscovery(
|
|
19
|
+
SqlDiscoveryOptions(
|
|
20
|
+
substitutions={
|
|
21
|
+
"bit": MSSQLBooleanType(),
|
|
22
|
+
"datetime": MSSQLDateTimeType(),
|
|
23
|
+
"datetime2": MSSQLDateTimeType(),
|
|
24
|
+
"nvarchar": MSSQLVariableCharacterType(encoding=MSSQLEncoding.UTF16),
|
|
25
|
+
"varchar": MSSQLVariableCharacterType(),
|
|
26
|
+
"text": MSSQLVariableCharacterType(),
|
|
27
|
+
}
|
|
28
|
+
)
|
|
29
|
+
),
|
|
30
|
+
MSSQLObjectFactory(),
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
async def get_columns(self, table_id: SupportsQualifiedId) -> Sequence[MSSQLColumn]:
|
|
34
|
+
plain_columns = await super().get_columns(table_id)
|
|
35
|
+
identity_columns = await self.conn.query_all(
|
|
36
|
+
str,
|
|
37
|
+
"SELECT c.name\n"
|
|
38
|
+
"FROM sys.schemas AS s\n"
|
|
39
|
+
" INNER JOIN sys.tables AS t ON t.schema_id = s.schema_id\n"
|
|
40
|
+
" INNER JOIN sys.columns AS c ON c.object_id = t.object_id\n"
|
|
41
|
+
f"WHERE s.name = {quote(table_id.scope_id or 'dbo')} AND t.name = {quote(table_id.local_id)} AND c.is_identity = 1;",
|
|
42
|
+
)
|
|
43
|
+
default_columns = await self.conn.query_all(
|
|
44
|
+
tuple[str, str, str],
|
|
45
|
+
"SELECT c.name, d.name, d.definition\n"
|
|
46
|
+
"FROM sys.schemas AS s\n"
|
|
47
|
+
" INNER JOIN sys.tables AS t ON t.schema_id = s.schema_id\n"
|
|
48
|
+
" INNER JOIN sys.columns AS c ON c.object_id = t.object_id\n"
|
|
49
|
+
" INNER JOIN sys.default_constraints AS d ON d.object_id = c.default_object_id\n"
|
|
50
|
+
f"WHERE s.name = {quote(table_id.scope_id or 'dbo')} AND t.name = {quote(table_id.local_id)} AND c.default_object_id != 0;",
|
|
51
|
+
)
|
|
52
|
+
default_values = {col_name: MSSQLDefault(con_name, expr) for col_name, con_name, expr in default_columns}
|
|
53
|
+
|
|
54
|
+
columns: list[MSSQLColumn] = []
|
|
55
|
+
for col in plain_columns:
|
|
56
|
+
columns.append(
|
|
57
|
+
MSSQLColumn(
|
|
58
|
+
name=col.name,
|
|
59
|
+
data_type=col.data_type,
|
|
60
|
+
nullable=col.nullable,
|
|
61
|
+
default=default_values.get(col.name.id),
|
|
62
|
+
identity=col.name.id in identity_columns,
|
|
63
|
+
description=col.description,
|
|
64
|
+
)
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
return columns
|
|
68
|
+
|
|
69
|
+
async def get_namespace_current(self) -> Namespace:
|
|
70
|
+
return await self._get_namespace()
|
|
71
|
+
|
|
72
|
+
async def get_namespace(self, namespace_id: LocalId) -> Namespace:
|
|
73
|
+
return await self._get_namespace(namespace_id)
|
|
74
|
+
|
|
75
|
+
async def _get_namespace(self, namespace_id: Optional[LocalId] = None) -> Namespace:
|
|
76
|
+
tables: list[Table] = []
|
|
77
|
+
|
|
78
|
+
# create namespace using qualified IDs
|
|
79
|
+
if namespace_id is not None:
|
|
80
|
+
schema_id = f"SCHEMA_ID({quote(namespace_id.local_id)})"
|
|
81
|
+
else:
|
|
82
|
+
schema_id = "SCHEMA_ID()"
|
|
83
|
+
table_names = await self.conn.query_all(
|
|
84
|
+
str,
|
|
85
|
+
f"SELECT name\nFROM sys.tables\nWHERE schema_id = {schema_id} AND is_ms_shipped = 0\nORDER BY name ASC",
|
|
86
|
+
)
|
|
87
|
+
if table_names:
|
|
88
|
+
if namespace_id is not None:
|
|
89
|
+
scope_id = namespace_id.local_id
|
|
90
|
+
else:
|
|
91
|
+
scope_id = None
|
|
92
|
+
|
|
93
|
+
for table_name in table_names:
|
|
94
|
+
table = await self.get_table(self.get_qualified_id(scope_id, table_name))
|
|
95
|
+
tables.append(table)
|
|
96
|
+
|
|
97
|
+
return self.factory.namespace_class(name=LocalId(scope_id or ""), enums=[], structs=[], tables=tables)
|
|
98
|
+
else:
|
|
99
|
+
return self.factory.namespace_class()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from pysqlsync.base import BaseConnection, BaseEngine, BaseGenerator, Explorer
|
|
2
|
+
|
|
3
|
+
from .connection import MSSQLConnection
|
|
4
|
+
from .discovery import MSSQLExplorer
|
|
5
|
+
from .generator import MSSQLGenerator
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class MSSQLEngine(BaseEngine):
|
|
9
|
+
@property
|
|
10
|
+
def name(self) -> str:
|
|
11
|
+
return "mssql"
|
|
12
|
+
|
|
13
|
+
def get_generator_type(self) -> type[BaseGenerator]:
|
|
14
|
+
return MSSQLGenerator
|
|
15
|
+
|
|
16
|
+
def get_connection_type(self) -> type[BaseConnection]:
|
|
17
|
+
return MSSQLConnection
|
|
18
|
+
|
|
19
|
+
def get_explorer_type(self) -> type[Explorer]:
|
|
20
|
+
return MSSQLExplorer
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import ipaddress
|
|
3
|
+
import uuid
|
|
4
|
+
from typing import Any, Callable, Optional
|
|
5
|
+
|
|
6
|
+
from strong_typing.core import JsonType
|
|
7
|
+
|
|
8
|
+
from pysqlsync.base import BaseGenerator, GeneratorOptions
|
|
9
|
+
from pysqlsync.formation.inspection import is_ip_address_type
|
|
10
|
+
from pysqlsync.formation.object_types import Column
|
|
11
|
+
from pysqlsync.formation.py_to_sql import (
|
|
12
|
+
ArrayMode,
|
|
13
|
+
DataclassConverter,
|
|
14
|
+
DataclassConverterOptions,
|
|
15
|
+
EnumMode,
|
|
16
|
+
NamespaceMapping,
|
|
17
|
+
StructMode,
|
|
18
|
+
)
|
|
19
|
+
from pysqlsync.model.data_types import SqlFixedBinaryType
|
|
20
|
+
from pysqlsync.util.typing import override
|
|
21
|
+
|
|
22
|
+
from .data_types import MSSQLBooleanType, MSSQLDateTimeType, MSSQLVariableCharacterType
|
|
23
|
+
from .mutation import MSSQLMutator
|
|
24
|
+
from .object_types import MSSQLObjectFactory
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class MSSQLGenerator(BaseGenerator):
|
|
28
|
+
"""
|
|
29
|
+
Generator for Microsoft SQL Server (T-SQL).
|
|
30
|
+
|
|
31
|
+
Assumes a UTF-8 collation and `SET ANSI_DEFAULTS ON`. UTF-8 collation makes `varchar` store UTF-8 characters.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self, options: GeneratorOptions) -> None:
|
|
35
|
+
super().__init__(options, MSSQLObjectFactory(), MSSQLMutator(options.synchronization))
|
|
36
|
+
|
|
37
|
+
self.check_enum_mode(exclude=[EnumMode.TYPE, EnumMode.INLINE])
|
|
38
|
+
self.check_struct_mode(matches=StructMode.JSON)
|
|
39
|
+
self.check_array_mode(include=[ArrayMode.JSON, ArrayMode.RELATION])
|
|
40
|
+
|
|
41
|
+
self.converter = DataclassConverter(
|
|
42
|
+
options=DataclassConverterOptions(
|
|
43
|
+
enum_mode=options.enum_mode or EnumMode.RELATION,
|
|
44
|
+
struct_mode=options.struct_mode or StructMode.JSON,
|
|
45
|
+
array_mode=options.array_mode or ArrayMode.JSON,
|
|
46
|
+
namespaces=NamespaceMapping(options.namespaces),
|
|
47
|
+
foreign_constraints=options.foreign_constraints,
|
|
48
|
+
initialize_tables=options.initialize_tables,
|
|
49
|
+
substitutions={
|
|
50
|
+
bool: MSSQLBooleanType(),
|
|
51
|
+
datetime.datetime: MSSQLDateTimeType(),
|
|
52
|
+
str: MSSQLVariableCharacterType(),
|
|
53
|
+
uuid.UUID: SqlFixedBinaryType(16),
|
|
54
|
+
JsonType: MSSQLVariableCharacterType(),
|
|
55
|
+
ipaddress.IPv4Address: SqlFixedBinaryType(4),
|
|
56
|
+
ipaddress.IPv6Address: SqlFixedBinaryType(16),
|
|
57
|
+
},
|
|
58
|
+
factory=self.factory,
|
|
59
|
+
skip_annotations=options.skip_annotations,
|
|
60
|
+
auto_default=options.auto_default,
|
|
61
|
+
)
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
@override
|
|
65
|
+
def get_current_schema_stmt(self) -> str:
|
|
66
|
+
return "SCHEMA_NAME()"
|
|
67
|
+
|
|
68
|
+
@override
|
|
69
|
+
def placeholder(self, index: int) -> str:
|
|
70
|
+
return "?"
|
|
71
|
+
|
|
72
|
+
@override
|
|
73
|
+
def get_field_extractor(self, column: Column, field_name: str, field_type: type) -> Callable[[Any], Any]:
|
|
74
|
+
if field_type is uuid.UUID:
|
|
75
|
+
return lambda obj: getattr(obj, field_name).bytes
|
|
76
|
+
elif is_ip_address_type(field_type):
|
|
77
|
+
return lambda obj: getattr(obj, field_name).packed
|
|
78
|
+
|
|
79
|
+
return super().get_field_extractor(column, field_name, field_type)
|
|
80
|
+
|
|
81
|
+
@override
|
|
82
|
+
def get_value_transformer(self, column: Column, field_type: type) -> Optional[Callable[[Any], Any]]:
|
|
83
|
+
if field_type is uuid.UUID:
|
|
84
|
+
return lambda field: field.bytes
|
|
85
|
+
elif is_ip_address_type(field_type):
|
|
86
|
+
return lambda field: field.packed
|
|
87
|
+
|
|
88
|
+
return super().get_value_transformer(column, field_type)
|