instructure-pysqlsync 0.8.5.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. instructure_pysqlsync-0.8.5.dev0.dist-info/METADATA +244 -0
  2. instructure_pysqlsync-0.8.5.dev0.dist-info/RECORD +102 -0
  3. instructure_pysqlsync-0.8.5.dev0.dist-info/WHEEL +5 -0
  4. instructure_pysqlsync-0.8.5.dev0.dist-info/licenses/LICENSE +21 -0
  5. instructure_pysqlsync-0.8.5.dev0.dist-info/top_level.txt +1 -0
  6. instructure_pysqlsync-0.8.5.dev0.dist-info/zip-safe +1 -0
  7. pysqlsync/__init__.py +15 -0
  8. pysqlsync/base.py +1386 -0
  9. pysqlsync/connection.py +101 -0
  10. pysqlsync/data/__init__.py +0 -0
  11. pysqlsync/data/exchange.py +184 -0
  12. pysqlsync/data/generator.py +305 -0
  13. pysqlsync/dialect/README.md +35 -0
  14. pysqlsync/dialect/__init__.py +0 -0
  15. pysqlsync/dialect/delta/__init__.py +0 -0
  16. pysqlsync/dialect/delta/data_types.py +51 -0
  17. pysqlsync/dialect/delta/dependency.py +7 -0
  18. pysqlsync/dialect/delta/engine.py +18 -0
  19. pysqlsync/dialect/delta/generator.py +74 -0
  20. pysqlsync/dialect/delta/object_types.py +87 -0
  21. pysqlsync/dialect/mssql/README.md +23 -0
  22. pysqlsync/dialect/mssql/__init__.py +0 -0
  23. pysqlsync/dialect/mssql/connection.py +146 -0
  24. pysqlsync/dialect/mssql/data_types.py +113 -0
  25. pysqlsync/dialect/mssql/dependency.py +9 -0
  26. pysqlsync/dialect/mssql/discovery.py +99 -0
  27. pysqlsync/dialect/mssql/engine.py +20 -0
  28. pysqlsync/dialect/mssql/generator.py +88 -0
  29. pysqlsync/dialect/mssql/mutation.py +46 -0
  30. pysqlsync/dialect/mssql/object_types.py +135 -0
  31. pysqlsync/dialect/mysql/__init__.py +0 -0
  32. pysqlsync/dialect/mysql/connection.py +126 -0
  33. pysqlsync/dialect/mysql/data_types.py +40 -0
  34. pysqlsync/dialect/mysql/dependency.py +10 -0
  35. pysqlsync/dialect/mysql/discovery.py +145 -0
  36. pysqlsync/dialect/mysql/engine.py +20 -0
  37. pysqlsync/dialect/mysql/generator.py +140 -0
  38. pysqlsync/dialect/mysql/mutation.py +65 -0
  39. pysqlsync/dialect/mysql/object_types.py +75 -0
  40. pysqlsync/dialect/oracle/README.md +22 -0
  41. pysqlsync/dialect/oracle/__init__.py +0 -0
  42. pysqlsync/dialect/oracle/connection.py +141 -0
  43. pysqlsync/dialect/oracle/data_types.py +104 -0
  44. pysqlsync/dialect/oracle/dependency.py +9 -0
  45. pysqlsync/dialect/oracle/discovery.py +231 -0
  46. pysqlsync/dialect/oracle/engine.py +20 -0
  47. pysqlsync/dialect/oracle/generator.py +93 -0
  48. pysqlsync/dialect/oracle/mutation.py +37 -0
  49. pysqlsync/dialect/oracle/object_types.py +59 -0
  50. pysqlsync/dialect/postgresql/__init__.py +0 -0
  51. pysqlsync/dialect/postgresql/connection.py +133 -0
  52. pysqlsync/dialect/postgresql/data_types.py +6 -0
  53. pysqlsync/dialect/postgresql/dependency.py +9 -0
  54. pysqlsync/dialect/postgresql/discovery.py +392 -0
  55. pysqlsync/dialect/postgresql/engine.py +20 -0
  56. pysqlsync/dialect/postgresql/generator.py +99 -0
  57. pysqlsync/dialect/postgresql/mutation.py +82 -0
  58. pysqlsync/dialect/postgresql/object_types.py +99 -0
  59. pysqlsync/dialect/redshift/__init__.py +0 -0
  60. pysqlsync/dialect/redshift/connection.py +156 -0
  61. pysqlsync/dialect/redshift/data_types.py +10 -0
  62. pysqlsync/dialect/redshift/dependency.py +9 -0
  63. pysqlsync/dialect/redshift/engine.py +20 -0
  64. pysqlsync/dialect/redshift/generator.py +82 -0
  65. pysqlsync/dialect/snowflake/__init__.py +0 -0
  66. pysqlsync/dialect/snowflake/data_types.py +18 -0
  67. pysqlsync/dialect/snowflake/dependency.py +9 -0
  68. pysqlsync/dialect/snowflake/engine.py +18 -0
  69. pysqlsync/dialect/snowflake/generator.py +64 -0
  70. pysqlsync/dialect/snowflake/object_types.py +84 -0
  71. pysqlsync/dialect/test/__init__.py +0 -0
  72. pysqlsync/dialect/test/dependency.py +10 -0
  73. pysqlsync/dialect/trino/__init__.py +0 -0
  74. pysqlsync/dialect/trino/connection.py +86 -0
  75. pysqlsync/dialect/trino/dependency.py +9 -0
  76. pysqlsync/dialect/trino/discovery.py +21 -0
  77. pysqlsync/dialect/trino/engine.py +20 -0
  78. pysqlsync/dialect/trino/generator.py +31 -0
  79. pysqlsync/factory.py +169 -0
  80. pysqlsync/formation/__init__.py +0 -0
  81. pysqlsync/formation/constraints.py +78 -0
  82. pysqlsync/formation/data_types.py +192 -0
  83. pysqlsync/formation/discovery.py +390 -0
  84. pysqlsync/formation/inspection.py +179 -0
  85. pysqlsync/formation/mutation.py +320 -0
  86. pysqlsync/formation/object_dict.py +79 -0
  87. pysqlsync/formation/object_types.py +872 -0
  88. pysqlsync/formation/py_to_sql.py +1134 -0
  89. pysqlsync/formation/sql_to_py.py +194 -0
  90. pysqlsync/model/__init__.py +0 -0
  91. pysqlsync/model/data_types.py +397 -0
  92. pysqlsync/model/entity_types.py +68 -0
  93. pysqlsync/model/id_types.py +193 -0
  94. pysqlsync/model/key_types.py +38 -0
  95. pysqlsync/model/properties.py +234 -0
  96. pysqlsync/py.typed +0 -0
  97. pysqlsync/python_types.py +192 -0
  98. pysqlsync/resultset.py +106 -0
  99. pysqlsync/util/__init__.py +0 -0
  100. pysqlsync/util/dataclasses.py +80 -0
  101. pysqlsync/util/dispatch.py +28 -0
  102. pysqlsync/util/typing.py +13 -0
@@ -0,0 +1,18 @@
1
+ from pysqlsync.base import BaseConnection, BaseEngine, BaseGenerator, Explorer
2
+
3
+ from .generator import DeltaGenerator
4
+
5
+
6
+ class DeltaEngine(BaseEngine):
7
+ @property
8
+ def name(self) -> str:
9
+ return "delta"
10
+
11
+ def get_generator_type(self) -> type[BaseGenerator]:
12
+ return DeltaGenerator
13
+
14
+ def get_connection_type(self) -> type[BaseConnection]:
15
+ raise NotImplementedError()
16
+
17
+ def get_explorer_type(self) -> type[Explorer]:
18
+ raise NotImplementedError()
@@ -0,0 +1,74 @@
1
+ import datetime
2
+ import ipaddress
3
+ import uuid
4
+
5
+ from strong_typing.auxiliary import float32, float64
6
+ from strong_typing.core import JsonType
7
+
8
+ from pysqlsync.base import BaseGenerator, GeneratorOptions
9
+ from pysqlsync.formation.mutation import Mutator
10
+ from pysqlsync.formation.py_to_sql import (
11
+ ArrayMode,
12
+ DataclassConverter,
13
+ DataclassConverterOptions,
14
+ EnumMode,
15
+ NamespaceMapping,
16
+ StructMode,
17
+ )
18
+ from pysqlsync.model.data_types import SqlVariableCharacterType
19
+ from pysqlsync.util.typing import override
20
+
21
+ from .data_types import (
22
+ DeltaDoubleType,
23
+ DeltaFixedBinaryType,
24
+ DeltaRealType,
25
+ DeltaTimestampType,
26
+ DeltaVariableCharacterType,
27
+ )
28
+ from .object_types import DeltaObjectFactory
29
+
30
+
31
+ class DeltaGenerator(BaseGenerator):
32
+ "Generator for Delta Lake on Databricks."
33
+
34
+ converter: DataclassConverter
35
+
36
+ def __init__(self, options: GeneratorOptions) -> None:
37
+ super().__init__(
38
+ options,
39
+ DeltaObjectFactory(),
40
+ Mutator(options.synchronization),
41
+ )
42
+
43
+ self.check_enum_mode(matches=EnumMode.CHECK)
44
+ self.check_struct_mode(exclude=[StructMode.TYPE])
45
+
46
+ self.converter = DataclassConverter(
47
+ options=DataclassConverterOptions(
48
+ enum_mode=options.enum_mode or EnumMode.CHECK,
49
+ struct_mode=options.struct_mode or StructMode.INLINE,
50
+ array_mode=options.array_mode or ArrayMode.ARRAY,
51
+ namespaces=NamespaceMapping(options.namespaces),
52
+ check_constraints=False,
53
+ foreign_constraints=False,
54
+ initialize_tables=options.initialize_tables,
55
+ substitutions={
56
+ datetime.datetime: DeltaTimestampType(),
57
+ float: DeltaDoubleType(),
58
+ float32: DeltaRealType(),
59
+ float64: DeltaDoubleType(),
60
+ str: DeltaVariableCharacterType(),
61
+ uuid.UUID: DeltaFixedBinaryType(16),
62
+ JsonType: SqlVariableCharacterType(),
63
+ ipaddress.IPv4Address: DeltaFixedBinaryType(4),
64
+ ipaddress.IPv6Address: DeltaFixedBinaryType(16),
65
+ },
66
+ factory=self.factory,
67
+ skip_annotations=options.skip_annotations,
68
+ auto_default=options.auto_default,
69
+ )
70
+ )
71
+
72
+ @override
73
+ def placeholder(self, index: int) -> str:
74
+ return f":{index}"
@@ -0,0 +1,87 @@
1
+ import re
2
+ from typing import Optional
3
+
4
+ from pysqlsync.formation.object_types import Column, Namespace, ObjectFactory, Table
5
+ from pysqlsync.model.id_types import LocalId
6
+
7
+ _sql_quoted_str_table = str.maketrans(
8
+ {
9
+ "\\": "\\\\",
10
+ "'": "\\'",
11
+ '"': '\\"',
12
+ "\0": "\\0",
13
+ "\b": "\\b",
14
+ "\n": "\\n",
15
+ "\r": "\\r",
16
+ "\t": "\\t",
17
+ "\u001a": "\\Z",
18
+ }
19
+ )
20
+
21
+
22
+ def sql_quoted_string(text: str) -> str:
23
+ if re.search(r"[\\'\"\0\b\n\r\t\u001A]", text):
24
+ text = text.translate(_sql_quoted_str_table)
25
+ return f"'{text}'"
26
+
27
+
28
+ class DeltaTable(Table):
29
+ def create_stmt(self) -> str:
30
+ defs: list[str] = []
31
+ defs.extend(str(c) for c in self.columns.values())
32
+ defs.append(self.create_keys())
33
+ definition = ",\n".join(defs)
34
+ comment = f"\nCOMMENT {sql_quoted_string(self.description)}" if self.description else ""
35
+ return f"CREATE TABLE {self.name} (\n{definition}\n){comment};"
36
+
37
+ @property
38
+ def primary_key_constraint_id(self) -> LocalId:
39
+ return LocalId(f"pk_{self.name.local_id.replace('.', '_')}")
40
+
41
+
42
+ class DeltaColumn(Column):
43
+ @property
44
+ def data_spec(self) -> str:
45
+ nullable = " NOT NULL" if not self.nullable and not self.identity else ""
46
+ identity = " GENERATED BY DEFAULT AS IDENTITY" if self.identity else ""
47
+ description = f" COMMENT {self.comment}" if self.description is not None else ""
48
+
49
+ # DEFAULT requires TBLPROPERTIES ('delta.feature.allowColumnDefaults' = 'enabled')
50
+ # default = f" DEFAULT {self.default}" if self.default is not None else ""
51
+
52
+ return f"{self.data_type}{nullable}{identity}{description}"
53
+
54
+ @property
55
+ def comment(self) -> Optional[str]:
56
+ if self.description is not None:
57
+ return sql_quoted_string(self.description)
58
+ else:
59
+ return None
60
+
61
+
62
+ class DeltaNamespace(Namespace):
63
+ def create_schema_stmt(self) -> Optional[str]:
64
+ if self.name.local_id:
65
+ return f"CREATE SCHEMA IF NOT EXISTS {self.name};"
66
+ else:
67
+ return None
68
+
69
+ def drop_schema_stmt(self) -> Optional[str]:
70
+ if self.name.local_id:
71
+ return f"DROP SCHEMA IF EXISTS {self.name} CASCADE;"
72
+ else:
73
+ return None
74
+
75
+
76
+ class DeltaObjectFactory(ObjectFactory):
77
+ @property
78
+ def column_class(self) -> type[Column]:
79
+ return DeltaColumn
80
+
81
+ @property
82
+ def table_class(self) -> type[Table]:
83
+ return DeltaTable
84
+
85
+ @property
86
+ def namespace_class(self) -> type[Namespace]:
87
+ return DeltaNamespace
@@ -0,0 +1,23 @@
1
+ # Microsoft SQL Server dialect for pysqlsync
2
+
3
+ ## Testing
4
+
5
+ Follow the steps of [running a SQL Server Linux container image with Docker](https://learn.microsoft.com/en-us/sql/linux/quickstart-install-connect-docker?view=sql-server-ver16&pivots=cs1-bash) to start testing with Microsoft SQL Server in a Linux/MacOS environment.
6
+
7
+ Specifically, pull the Docker image with the command:
8
+ ```sh
9
+ docker pull mcr.microsoft.com/mssql/server:2022-latest
10
+ ```
11
+
12
+ Next, start the Docker container with the command:
13
+
14
+ ```sh
15
+ docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=<?YourStrong@Passw0rd>" -p 1433:1433 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2022-latest
16
+ ```
17
+
18
+ In order to inspect database startup logs, run:
19
+ ```sh
20
+ docker logs sql1
21
+ ```
22
+
23
+ Finally, run unit and integration tests.
File without changes
@@ -0,0 +1,146 @@
1
+ import logging
2
+ import typing
3
+ from typing import Iterable, Optional, TypeVar
4
+
5
+ import pyodbc
6
+ from strong_typing.inspection import is_dataclass_type
7
+
8
+ from pysqlsync.base import BaseConnection, BaseContext, DataSource, RecordType
9
+ from pysqlsync.formation.object_types import Table
10
+ from pysqlsync.model.data_types import quote
11
+ from pysqlsync.model.id_types import LocalId, QualifiedId
12
+ from pysqlsync.resultset import resultset_unwrap_object, resultset_unwrap_tuple
13
+ from pysqlsync.util.dispatch import thread_dispatch
14
+ from pysqlsync.util.typing import override
15
+
16
+ from .data_types import sql_to_odbc_type
17
+
18
+ T = TypeVar("T")
19
+
20
+ LOGGER = logging.getLogger("pysqlsync.mssql")
21
+
22
+
23
+ class MSSQLConnection(BaseConnection):
24
+ """
25
+ Represents a connection to a Microsoft SQL Server.
26
+ """
27
+
28
+ native: pyodbc.Connection
29
+
30
+ @override
31
+ @thread_dispatch
32
+ def open(self) -> BaseContext:
33
+ LOGGER.info("connecting to %s", self.params)
34
+ params: dict[str, Optional[str]] = {
35
+ "DRIVER": "{ODBC Driver 18 for SQL Server}",
36
+ "APP": "pysqlsync",
37
+ "SERVER": (f"{self.params.host},{self.params.port}" if self.params.port is not None else self.params.host),
38
+ "UID": self.params.username,
39
+ "PWD": self.params.password,
40
+ "LongAsMax": "yes",
41
+ "TrustServerCertificate": "yes",
42
+ }
43
+ if self.params.database is not None:
44
+ params["DATABASE"] = self.params.database
45
+ conn_string = ";".join(f"{key}={value}" for key, value in params.items() if value is not None)
46
+ conn = pyodbc.connect(conn_string)
47
+ with conn.cursor() as cur:
48
+ rows = cur.execute("SELECT @@VERSION").fetchall()
49
+ for row in rows:
50
+ LOGGER.info(row)
51
+
52
+ self.native = conn
53
+ return MSSQLContext(self)
54
+
55
+ @override
56
+ @thread_dispatch
57
+ def close(self) -> None:
58
+ self.native.close()
59
+
60
+
61
+ class MSSQLContext(BaseContext):
62
+ def __init__(self, connection: MSSQLConnection) -> None:
63
+ super().__init__(connection)
64
+
65
+ @property
66
+ def native_connection(self) -> pyodbc.Connection:
67
+ return typing.cast(MSSQLConnection, self.connection).native
68
+
69
+ @override
70
+ @thread_dispatch
71
+ def _execute(self, statement: str) -> None:
72
+ with self.native_connection.cursor() as cur:
73
+ cur.execute(statement)
74
+
75
+ @override
76
+ async def _execute_all(self, statement: str, source: DataSource) -> None:
77
+ async for batch in source.batches():
78
+ await self._internal_execute_all(statement, batch)
79
+
80
+ @override
81
+ async def _execute_typed(
82
+ self,
83
+ statement: str,
84
+ source: DataSource,
85
+ table: Table,
86
+ order: Optional[tuple[str, ...]] = None,
87
+ ) -> None:
88
+ async for batch in source.batches():
89
+ await self._internal_execute_typed(statement, batch, table, order)
90
+
91
+ @thread_dispatch
92
+ def _internal_execute_all(self, statement: str, records: Iterable[RecordType]) -> None:
93
+ with self.native_connection.cursor() as cur:
94
+ cur.fast_executemany = True
95
+ cur.executemany(statement, records)
96
+
97
+ @thread_dispatch
98
+ def _internal_execute_typed(
99
+ self,
100
+ statement: str,
101
+ records: Iterable[RecordType],
102
+ table: Table,
103
+ order: Optional[tuple[str, ...]] = None,
104
+ ) -> None:
105
+ with self.native_connection.cursor() as cur:
106
+ cur.fast_executemany = True
107
+ cur.setinputsizes([sql_to_odbc_type(column.data_type) for column in table.get_columns(order)])
108
+ cur.executemany(statement, records)
109
+
110
+ @override
111
+ @thread_dispatch
112
+ def _query_all(self, signature: type[T], statement: str) -> list[T]:
113
+ with self.native_connection.cursor() as cur:
114
+ records = cur.execute(statement).fetchall()
115
+
116
+ if is_dataclass_type(signature):
117
+ return resultset_unwrap_object(signature, records) # type: ignore
118
+ else:
119
+ return resultset_unwrap_tuple(signature, records)
120
+
121
+ @override
122
+ async def drop_schema(self, namespace: LocalId) -> None:
123
+ LOGGER.debug("drop schema: %s", namespace)
124
+
125
+ constraints = await self.query_all(
126
+ tuple[str, str],
127
+ "SELECT table_name, constraint_name\n"
128
+ "FROM information_schema.table_constraints\n"
129
+ f"WHERE constraint_type = 'FOREIGN KEY' AND table_schema = {quote(namespace.id)};",
130
+ )
131
+ if constraints:
132
+ stmts: list[str] = []
133
+ for constraint in constraints:
134
+ table_name, constraint_name = constraint
135
+ stmts.append(f"ALTER TABLE {QualifiedId(namespace.id, table_name)} DROP CONSTRAINT {LocalId(constraint_name)};")
136
+ await self.execute("\n".join(stmts))
137
+
138
+ tables = await self.query_all(
139
+ str,
140
+ f"SELECT table_name\nFROM information_schema.tables\nWHERE table_schema = {quote(namespace.id)};",
141
+ )
142
+ if tables:
143
+ table_list = ", ".join(str(QualifiedId(namespace.local_id, table)) for table in tables)
144
+ await self.execute(f"DROP TABLE IF EXISTS {table_list};")
145
+
146
+ await self.execute(f"DROP SCHEMA IF EXISTS {namespace};")
@@ -0,0 +1,113 @@
1
+ import enum
2
+ from typing import Any, Optional
3
+
4
+ import pyodbc
5
+
6
+ from pysqlsync.model.data_types import (
7
+ SqlBooleanType,
8
+ SqlDataType,
9
+ SqlDateType,
10
+ SqlDecimalType,
11
+ SqlDoubleType,
12
+ SqlFixedBinaryType,
13
+ SqlFixedCharacterType,
14
+ SqlFloatType,
15
+ SqlIntegerType,
16
+ SqlRealType,
17
+ SqlTimestampType,
18
+ SqlTimeType,
19
+ SqlVariableBinaryType,
20
+ SqlVariableCharacterType,
21
+ )
22
+
23
+
24
+ class MSSQLBooleanType(SqlBooleanType):
25
+ def __str__(self) -> str:
26
+ return "bit"
27
+
28
+ def value_to_sql_literal(self, value: Any) -> str:
29
+ if not isinstance(value, bool):
30
+ raise TypeError(f"expected: value of type `bool`, got: {type(value)}")
31
+
32
+ return "1" if value else "0"
33
+
34
+
35
+ class MSSQLEncoding(enum.Enum):
36
+ UTF8 = "utf-8"
37
+ UTF16 = "utf-16"
38
+
39
+
40
+ class MSSQLVariableCharacterType(SqlVariableCharacterType):
41
+ encoding: Optional[MSSQLEncoding] = None
42
+
43
+ def __init__(self, limit: Optional[int] = None, encoding: Optional[MSSQLEncoding] = None) -> None:
44
+ super().__init__(limit)
45
+ self.encoding = encoding
46
+
47
+ def __str__(self) -> str:
48
+ if self.encoding is MSSQLEncoding.UTF16:
49
+ char_type = "nvarchar"
50
+ else:
51
+ char_type = "varchar"
52
+
53
+ if self.limit is not None and self.limit > 0 and self.limit != 2147483647:
54
+ return f"{char_type}({self.limit})"
55
+ else:
56
+ return f"{char_type}(max)"
57
+
58
+
59
+ class MSSQLDateTimeType(SqlTimestampType):
60
+ def __init__(self) -> None:
61
+ self.precision = 7
62
+
63
+ def __str__(self) -> str:
64
+ return "datetime2"
65
+
66
+
67
+ def sql_to_odbc_type(data_type: SqlDataType) -> tuple[int, int, int]:
68
+ """
69
+ Returns the ODBC data type associated with the SQL data type.
70
+
71
+ Passing the right data types to `setinputsizes` eliminates data-based guessing, and can speed up `executemany`
72
+ by a significant factor.
73
+ """
74
+
75
+ if isinstance(data_type, MSSQLBooleanType):
76
+ return pyodbc.SQL_BIT, 0, 0
77
+
78
+ elif isinstance(data_type, SqlIntegerType):
79
+ if data_type.width == 1:
80
+ return pyodbc.SQL_TINYINT, 0, 0
81
+ elif data_type.width == 2:
82
+ return pyodbc.SQL_SMALLINT, 0, 0
83
+ elif data_type.width == 4:
84
+ return pyodbc.SQL_INTEGER, 0, 0
85
+ else:
86
+ return pyodbc.SQL_BIGINT, 0, 0
87
+
88
+ elif isinstance(data_type, SqlRealType):
89
+ return pyodbc.SQL_REAL, 0, 0
90
+ elif isinstance(data_type, SqlDoubleType):
91
+ return pyodbc.SQL_DOUBLE, 0, 0
92
+ elif isinstance(data_type, SqlFloatType):
93
+ return pyodbc.SQL_FLOAT, data_type.precision or 53, 0
94
+ elif isinstance(data_type, SqlDecimalType):
95
+ return pyodbc.SQL_DECIMAL, data_type.precision or 15, data_type.scale or 0
96
+
97
+ elif isinstance(data_type, SqlTimestampType):
98
+ return pyodbc.SQL_TYPE_TIMESTAMP, data_type.precision or 6, 0
99
+ elif isinstance(data_type, SqlDateType):
100
+ return pyodbc.SQL_TYPE_DATE, 0, 0
101
+ elif isinstance(data_type, SqlTimeType):
102
+ return pyodbc.SQL_TYPE_TIME, data_type.precision or 6, 0
103
+
104
+ elif isinstance(data_type, SqlFixedCharacterType):
105
+ return pyodbc.SQL_CHAR, data_type.limit or 0, 0
106
+ elif isinstance(data_type, SqlVariableCharacterType):
107
+ return pyodbc.SQL_VARCHAR, data_type.limit or 0, 0
108
+ elif isinstance(data_type, SqlFixedBinaryType):
109
+ return pyodbc.SQL_BINARY, data_type.storage or 0, 0
110
+ elif isinstance(data_type, SqlVariableBinaryType):
111
+ return pyodbc.SQL_VARBINARY, data_type.storage or 0, 0
112
+
113
+ return pyodbc.SQL_UNKNOWN_TYPE, 0, 0
@@ -0,0 +1,9 @@
1
+ """
2
+ pysqlsync: Synchronize schema and large volumes of data.
3
+
4
+ This module defines dependencies required for Microsoft SQL Server.
5
+
6
+ :see: https://github.com/instructure-internal/pysqlsync
7
+ """
8
+
9
+ import pyodbc # pyright: ignore[reportUnusedImport] # noqa: F401
@@ -0,0 +1,99 @@
1
+ from typing import Optional, Sequence
2
+
3
+ from pysqlsync.base import BaseContext
4
+ from pysqlsync.formation.data_types import SqlDiscovery, SqlDiscoveryOptions
5
+ from pysqlsync.formation.discovery import AnsiExplorer
6
+ from pysqlsync.formation.object_types import Namespace, Table
7
+ from pysqlsync.model.data_types import quote
8
+ from pysqlsync.model.id_types import LocalId, SupportsQualifiedId
9
+
10
+ from .data_types import MSSQLBooleanType, MSSQLDateTimeType, MSSQLEncoding, MSSQLVariableCharacterType
11
+ from .object_types import MSSQLColumn, MSSQLDefault, MSSQLObjectFactory
12
+
13
+
14
+ class MSSQLExplorer(AnsiExplorer):
15
+ def __init__(self, conn: BaseContext) -> None:
16
+ super().__init__(
17
+ conn,
18
+ SqlDiscovery(
19
+ SqlDiscoveryOptions(
20
+ substitutions={
21
+ "bit": MSSQLBooleanType(),
22
+ "datetime": MSSQLDateTimeType(),
23
+ "datetime2": MSSQLDateTimeType(),
24
+ "nvarchar": MSSQLVariableCharacterType(encoding=MSSQLEncoding.UTF16),
25
+ "varchar": MSSQLVariableCharacterType(),
26
+ "text": MSSQLVariableCharacterType(),
27
+ }
28
+ )
29
+ ),
30
+ MSSQLObjectFactory(),
31
+ )
32
+
33
+ async def get_columns(self, table_id: SupportsQualifiedId) -> Sequence[MSSQLColumn]:
34
+ plain_columns = await super().get_columns(table_id)
35
+ identity_columns = await self.conn.query_all(
36
+ str,
37
+ "SELECT c.name\n"
38
+ "FROM sys.schemas AS s\n"
39
+ " INNER JOIN sys.tables AS t ON t.schema_id = s.schema_id\n"
40
+ " INNER JOIN sys.columns AS c ON c.object_id = t.object_id\n"
41
+ f"WHERE s.name = {quote(table_id.scope_id or 'dbo')} AND t.name = {quote(table_id.local_id)} AND c.is_identity = 1;",
42
+ )
43
+ default_columns = await self.conn.query_all(
44
+ tuple[str, str, str],
45
+ "SELECT c.name, d.name, d.definition\n"
46
+ "FROM sys.schemas AS s\n"
47
+ " INNER JOIN sys.tables AS t ON t.schema_id = s.schema_id\n"
48
+ " INNER JOIN sys.columns AS c ON c.object_id = t.object_id\n"
49
+ " INNER JOIN sys.default_constraints AS d ON d.object_id = c.default_object_id\n"
50
+ f"WHERE s.name = {quote(table_id.scope_id or 'dbo')} AND t.name = {quote(table_id.local_id)} AND c.default_object_id != 0;",
51
+ )
52
+ default_values = {col_name: MSSQLDefault(con_name, expr) for col_name, con_name, expr in default_columns}
53
+
54
+ columns: list[MSSQLColumn] = []
55
+ for col in plain_columns:
56
+ columns.append(
57
+ MSSQLColumn(
58
+ name=col.name,
59
+ data_type=col.data_type,
60
+ nullable=col.nullable,
61
+ default=default_values.get(col.name.id),
62
+ identity=col.name.id in identity_columns,
63
+ description=col.description,
64
+ )
65
+ )
66
+
67
+ return columns
68
+
69
+ async def get_namespace_current(self) -> Namespace:
70
+ return await self._get_namespace()
71
+
72
+ async def get_namespace(self, namespace_id: LocalId) -> Namespace:
73
+ return await self._get_namespace(namespace_id)
74
+
75
+ async def _get_namespace(self, namespace_id: Optional[LocalId] = None) -> Namespace:
76
+ tables: list[Table] = []
77
+
78
+ # create namespace using qualified IDs
79
+ if namespace_id is not None:
80
+ schema_id = f"SCHEMA_ID({quote(namespace_id.local_id)})"
81
+ else:
82
+ schema_id = "SCHEMA_ID()"
83
+ table_names = await self.conn.query_all(
84
+ str,
85
+ f"SELECT name\nFROM sys.tables\nWHERE schema_id = {schema_id} AND is_ms_shipped = 0\nORDER BY name ASC",
86
+ )
87
+ if table_names:
88
+ if namespace_id is not None:
89
+ scope_id = namespace_id.local_id
90
+ else:
91
+ scope_id = None
92
+
93
+ for table_name in table_names:
94
+ table = await self.get_table(self.get_qualified_id(scope_id, table_name))
95
+ tables.append(table)
96
+
97
+ return self.factory.namespace_class(name=LocalId(scope_id or ""), enums=[], structs=[], tables=tables)
98
+ else:
99
+ return self.factory.namespace_class()
@@ -0,0 +1,20 @@
1
+ from pysqlsync.base import BaseConnection, BaseEngine, BaseGenerator, Explorer
2
+
3
+ from .connection import MSSQLConnection
4
+ from .discovery import MSSQLExplorer
5
+ from .generator import MSSQLGenerator
6
+
7
+
8
+ class MSSQLEngine(BaseEngine):
9
+ @property
10
+ def name(self) -> str:
11
+ return "mssql"
12
+
13
+ def get_generator_type(self) -> type[BaseGenerator]:
14
+ return MSSQLGenerator
15
+
16
+ def get_connection_type(self) -> type[BaseConnection]:
17
+ return MSSQLConnection
18
+
19
+ def get_explorer_type(self) -> type[Explorer]:
20
+ return MSSQLExplorer
@@ -0,0 +1,88 @@
1
+ import datetime
2
+ import ipaddress
3
+ import uuid
4
+ from typing import Any, Callable, Optional
5
+
6
+ from strong_typing.core import JsonType
7
+
8
+ from pysqlsync.base import BaseGenerator, GeneratorOptions
9
+ from pysqlsync.formation.inspection import is_ip_address_type
10
+ from pysqlsync.formation.object_types import Column
11
+ from pysqlsync.formation.py_to_sql import (
12
+ ArrayMode,
13
+ DataclassConverter,
14
+ DataclassConverterOptions,
15
+ EnumMode,
16
+ NamespaceMapping,
17
+ StructMode,
18
+ )
19
+ from pysqlsync.model.data_types import SqlFixedBinaryType
20
+ from pysqlsync.util.typing import override
21
+
22
+ from .data_types import MSSQLBooleanType, MSSQLDateTimeType, MSSQLVariableCharacterType
23
+ from .mutation import MSSQLMutator
24
+ from .object_types import MSSQLObjectFactory
25
+
26
+
27
+ class MSSQLGenerator(BaseGenerator):
28
+ """
29
+ Generator for Microsoft SQL Server (T-SQL).
30
+
31
+ Assumes a UTF-8 collation and `SET ANSI_DEFAULTS ON`. UTF-8 collation makes `varchar` store UTF-8 characters.
32
+ """
33
+
34
+ def __init__(self, options: GeneratorOptions) -> None:
35
+ super().__init__(options, MSSQLObjectFactory(), MSSQLMutator(options.synchronization))
36
+
37
+ self.check_enum_mode(exclude=[EnumMode.TYPE, EnumMode.INLINE])
38
+ self.check_struct_mode(matches=StructMode.JSON)
39
+ self.check_array_mode(include=[ArrayMode.JSON, ArrayMode.RELATION])
40
+
41
+ self.converter = DataclassConverter(
42
+ options=DataclassConverterOptions(
43
+ enum_mode=options.enum_mode or EnumMode.RELATION,
44
+ struct_mode=options.struct_mode or StructMode.JSON,
45
+ array_mode=options.array_mode or ArrayMode.JSON,
46
+ namespaces=NamespaceMapping(options.namespaces),
47
+ foreign_constraints=options.foreign_constraints,
48
+ initialize_tables=options.initialize_tables,
49
+ substitutions={
50
+ bool: MSSQLBooleanType(),
51
+ datetime.datetime: MSSQLDateTimeType(),
52
+ str: MSSQLVariableCharacterType(),
53
+ uuid.UUID: SqlFixedBinaryType(16),
54
+ JsonType: MSSQLVariableCharacterType(),
55
+ ipaddress.IPv4Address: SqlFixedBinaryType(4),
56
+ ipaddress.IPv6Address: SqlFixedBinaryType(16),
57
+ },
58
+ factory=self.factory,
59
+ skip_annotations=options.skip_annotations,
60
+ auto_default=options.auto_default,
61
+ )
62
+ )
63
+
64
+ @override
65
+ def get_current_schema_stmt(self) -> str:
66
+ return "SCHEMA_NAME()"
67
+
68
+ @override
69
+ def placeholder(self, index: int) -> str:
70
+ return "?"
71
+
72
+ @override
73
+ def get_field_extractor(self, column: Column, field_name: str, field_type: type) -> Callable[[Any], Any]:
74
+ if field_type is uuid.UUID:
75
+ return lambda obj: getattr(obj, field_name).bytes
76
+ elif is_ip_address_type(field_type):
77
+ return lambda obj: getattr(obj, field_name).packed
78
+
79
+ return super().get_field_extractor(column, field_name, field_type)
80
+
81
+ @override
82
+ def get_value_transformer(self, column: Column, field_type: type) -> Optional[Callable[[Any], Any]]:
83
+ if field_type is uuid.UUID:
84
+ return lambda field: field.bytes
85
+ elif is_ip_address_type(field_type):
86
+ return lambda field: field.packed
87
+
88
+ return super().get_value_transformer(column, field_type)