instructure-pysqlsync 0.8.5.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- instructure_pysqlsync-0.8.5.dev0.dist-info/METADATA +244 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/RECORD +102 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/WHEEL +5 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/licenses/LICENSE +21 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/top_level.txt +1 -0
- instructure_pysqlsync-0.8.5.dev0.dist-info/zip-safe +1 -0
- pysqlsync/__init__.py +15 -0
- pysqlsync/base.py +1386 -0
- pysqlsync/connection.py +101 -0
- pysqlsync/data/__init__.py +0 -0
- pysqlsync/data/exchange.py +184 -0
- pysqlsync/data/generator.py +305 -0
- pysqlsync/dialect/README.md +35 -0
- pysqlsync/dialect/__init__.py +0 -0
- pysqlsync/dialect/delta/__init__.py +0 -0
- pysqlsync/dialect/delta/data_types.py +51 -0
- pysqlsync/dialect/delta/dependency.py +7 -0
- pysqlsync/dialect/delta/engine.py +18 -0
- pysqlsync/dialect/delta/generator.py +74 -0
- pysqlsync/dialect/delta/object_types.py +87 -0
- pysqlsync/dialect/mssql/README.md +23 -0
- pysqlsync/dialect/mssql/__init__.py +0 -0
- pysqlsync/dialect/mssql/connection.py +146 -0
- pysqlsync/dialect/mssql/data_types.py +113 -0
- pysqlsync/dialect/mssql/dependency.py +9 -0
- pysqlsync/dialect/mssql/discovery.py +99 -0
- pysqlsync/dialect/mssql/engine.py +20 -0
- pysqlsync/dialect/mssql/generator.py +88 -0
- pysqlsync/dialect/mssql/mutation.py +46 -0
- pysqlsync/dialect/mssql/object_types.py +135 -0
- pysqlsync/dialect/mysql/__init__.py +0 -0
- pysqlsync/dialect/mysql/connection.py +126 -0
- pysqlsync/dialect/mysql/data_types.py +40 -0
- pysqlsync/dialect/mysql/dependency.py +10 -0
- pysqlsync/dialect/mysql/discovery.py +145 -0
- pysqlsync/dialect/mysql/engine.py +20 -0
- pysqlsync/dialect/mysql/generator.py +140 -0
- pysqlsync/dialect/mysql/mutation.py +65 -0
- pysqlsync/dialect/mysql/object_types.py +75 -0
- pysqlsync/dialect/oracle/README.md +22 -0
- pysqlsync/dialect/oracle/__init__.py +0 -0
- pysqlsync/dialect/oracle/connection.py +141 -0
- pysqlsync/dialect/oracle/data_types.py +104 -0
- pysqlsync/dialect/oracle/dependency.py +9 -0
- pysqlsync/dialect/oracle/discovery.py +231 -0
- pysqlsync/dialect/oracle/engine.py +20 -0
- pysqlsync/dialect/oracle/generator.py +93 -0
- pysqlsync/dialect/oracle/mutation.py +37 -0
- pysqlsync/dialect/oracle/object_types.py +59 -0
- pysqlsync/dialect/postgresql/__init__.py +0 -0
- pysqlsync/dialect/postgresql/connection.py +133 -0
- pysqlsync/dialect/postgresql/data_types.py +6 -0
- pysqlsync/dialect/postgresql/dependency.py +9 -0
- pysqlsync/dialect/postgresql/discovery.py +392 -0
- pysqlsync/dialect/postgresql/engine.py +20 -0
- pysqlsync/dialect/postgresql/generator.py +99 -0
- pysqlsync/dialect/postgresql/mutation.py +82 -0
- pysqlsync/dialect/postgresql/object_types.py +99 -0
- pysqlsync/dialect/redshift/__init__.py +0 -0
- pysqlsync/dialect/redshift/connection.py +156 -0
- pysqlsync/dialect/redshift/data_types.py +10 -0
- pysqlsync/dialect/redshift/dependency.py +9 -0
- pysqlsync/dialect/redshift/engine.py +20 -0
- pysqlsync/dialect/redshift/generator.py +82 -0
- pysqlsync/dialect/snowflake/__init__.py +0 -0
- pysqlsync/dialect/snowflake/data_types.py +18 -0
- pysqlsync/dialect/snowflake/dependency.py +9 -0
- pysqlsync/dialect/snowflake/engine.py +18 -0
- pysqlsync/dialect/snowflake/generator.py +64 -0
- pysqlsync/dialect/snowflake/object_types.py +84 -0
- pysqlsync/dialect/test/__init__.py +0 -0
- pysqlsync/dialect/test/dependency.py +10 -0
- pysqlsync/dialect/trino/__init__.py +0 -0
- pysqlsync/dialect/trino/connection.py +86 -0
- pysqlsync/dialect/trino/dependency.py +9 -0
- pysqlsync/dialect/trino/discovery.py +21 -0
- pysqlsync/dialect/trino/engine.py +20 -0
- pysqlsync/dialect/trino/generator.py +31 -0
- pysqlsync/factory.py +169 -0
- pysqlsync/formation/__init__.py +0 -0
- pysqlsync/formation/constraints.py +78 -0
- pysqlsync/formation/data_types.py +192 -0
- pysqlsync/formation/discovery.py +390 -0
- pysqlsync/formation/inspection.py +179 -0
- pysqlsync/formation/mutation.py +320 -0
- pysqlsync/formation/object_dict.py +79 -0
- pysqlsync/formation/object_types.py +872 -0
- pysqlsync/formation/py_to_sql.py +1134 -0
- pysqlsync/formation/sql_to_py.py +194 -0
- pysqlsync/model/__init__.py +0 -0
- pysqlsync/model/data_types.py +397 -0
- pysqlsync/model/entity_types.py +68 -0
- pysqlsync/model/id_types.py +193 -0
- pysqlsync/model/key_types.py +38 -0
- pysqlsync/model/properties.py +234 -0
- pysqlsync/py.typed +0 -0
- pysqlsync/python_types.py +192 -0
- pysqlsync/resultset.py +106 -0
- pysqlsync/util/__init__.py +0 -0
- pysqlsync/util/dataclasses.py +80 -0
- pysqlsync/util/dispatch.py +28 -0
- pysqlsync/util/typing.py +13 -0
|
@@ -0,0 +1,392 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import re
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from itertools import groupby
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
from pysqlsync.base import BaseContext, Explorer
|
|
8
|
+
from pysqlsync.formation.constraints import ForeignFactory, UniqueFactory
|
|
9
|
+
from pysqlsync.formation.data_types import SqlDiscovery, SqlDiscoveryOptions
|
|
10
|
+
from pysqlsync.formation.object_types import Column, Constraint, EnumType, Namespace, StructMember, StructType, Table
|
|
11
|
+
from pysqlsync.model.data_types import SqlArrayType, SqlUserDefinedType, quote
|
|
12
|
+
from pysqlsync.model.id_types import GlobalId, LocalId, QualifiedId, SupportsQualifiedId
|
|
13
|
+
|
|
14
|
+
from .data_types import PostgreSQLJsonType
|
|
15
|
+
from .object_types import PostgreSQLObjectFactory
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def to_default_expr(expr: Optional[str]) -> Optional[str]:
|
|
19
|
+
"Converts a PostgreSQL-specific default expression into a standard default expression."
|
|
20
|
+
|
|
21
|
+
if expr is None:
|
|
22
|
+
return None
|
|
23
|
+
|
|
24
|
+
if expr.lower() in ("true", "false"):
|
|
25
|
+
return expr.upper()
|
|
26
|
+
|
|
27
|
+
m = re.match(
|
|
28
|
+
r"^'(?P<year>\d{4})-(?P<mon>\d{2})-(?P<day>\d{2}) (?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})'::timestamp without time zone$",
|
|
29
|
+
expr,
|
|
30
|
+
)
|
|
31
|
+
if m:
|
|
32
|
+
timestamp = datetime.datetime(
|
|
33
|
+
int(m.group("year")),
|
|
34
|
+
int(m.group("mon")),
|
|
35
|
+
int(m.group("day")),
|
|
36
|
+
int(m.group("hour")),
|
|
37
|
+
int(m.group("min")),
|
|
38
|
+
int(m.group("sec")),
|
|
39
|
+
tzinfo=None,
|
|
40
|
+
)
|
|
41
|
+
return quote(timestamp.isoformat(sep=" "))
|
|
42
|
+
else:
|
|
43
|
+
return expr
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class PostgreSQLStructMeta:
|
|
48
|
+
description: str
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class PostgreSQLMemberMeta:
|
|
53
|
+
member_name: str
|
|
54
|
+
type_schema: str
|
|
55
|
+
type_name: str
|
|
56
|
+
type_def: str
|
|
57
|
+
description: str
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class PostgreSQLTableMeta:
|
|
62
|
+
description: str
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class PostgreSQLColumnMeta:
|
|
67
|
+
column_name: str
|
|
68
|
+
type_schema: str
|
|
69
|
+
type_name: str
|
|
70
|
+
type_def: str
|
|
71
|
+
is_nullable: bool
|
|
72
|
+
is_array: bool
|
|
73
|
+
is_identity: bool
|
|
74
|
+
default_value: str
|
|
75
|
+
description: str
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass
|
|
79
|
+
class PostgreSQLConstraintMeta:
|
|
80
|
+
constraint_type: bytes
|
|
81
|
+
constraint_name: str
|
|
82
|
+
source_column: str
|
|
83
|
+
target_namespace: str
|
|
84
|
+
target_table: str
|
|
85
|
+
target_column: str
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass
|
|
89
|
+
class PostgreSQLEnumMeta:
|
|
90
|
+
enum_name: str
|
|
91
|
+
enum_value: str
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class PostgreSQLExplorer(Explorer):
|
|
95
|
+
discovery: SqlDiscovery
|
|
96
|
+
factory: PostgreSQLObjectFactory
|
|
97
|
+
|
|
98
|
+
def __init__(self, conn: BaseContext) -> None:
|
|
99
|
+
super().__init__(conn)
|
|
100
|
+
self.discovery = SqlDiscovery(
|
|
101
|
+
SqlDiscoveryOptions(
|
|
102
|
+
substitutions={
|
|
103
|
+
"jsonb": PostgreSQLJsonType(),
|
|
104
|
+
"inet": SqlUserDefinedType(GlobalId("inet")),
|
|
105
|
+
}
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
self.factory = PostgreSQLObjectFactory()
|
|
109
|
+
|
|
110
|
+
async def get_table_names(self) -> list[QualifiedId]:
|
|
111
|
+
rows = await self.conn.query_all(
|
|
112
|
+
tuple[str, str],
|
|
113
|
+
"SELECT nsp.nspname, cls.relname\n"
|
|
114
|
+
"FROM pg_catalog.pg_class AS cls INNER JOIN pg_catalog.pg_namespace AS nsp ON cls.relnamespace = nsp.oid\n"
|
|
115
|
+
"WHERE cls.relkind = 'r' OR cls.relkind = 'v'",
|
|
116
|
+
)
|
|
117
|
+
return [QualifiedId(row[0], row[1]) for row in rows]
|
|
118
|
+
|
|
119
|
+
@classmethod
|
|
120
|
+
def _where_relation(cls, relation_id: SupportsQualifiedId, kinds: list[str]) -> str:
|
|
121
|
+
conditions: list[str] = []
|
|
122
|
+
conditions.append(f"cls.relname = {quote(relation_id.local_id)}")
|
|
123
|
+
if relation_id.scope_id is not None:
|
|
124
|
+
conditions.append(f"nsp.nspname = {quote(relation_id.scope_id)}")
|
|
125
|
+
conditions.append(" OR ".join(f"(cls.relkind = {quote(kind)})" for kind in kinds))
|
|
126
|
+
return " AND ".join(f"({c})" for c in conditions)
|
|
127
|
+
|
|
128
|
+
@classmethod
|
|
129
|
+
def _where_struct(cls, struct_id: SupportsQualifiedId) -> str:
|
|
130
|
+
return cls._where_relation(struct_id, ["c"])
|
|
131
|
+
|
|
132
|
+
@classmethod
|
|
133
|
+
def _where_table(cls, table_id: SupportsQualifiedId) -> str:
|
|
134
|
+
return cls._where_relation(table_id, ["r", "v"])
|
|
135
|
+
|
|
136
|
+
async def has_table(self, table_id: SupportsQualifiedId) -> bool:
|
|
137
|
+
rows = await self.conn.query_all(
|
|
138
|
+
int,
|
|
139
|
+
"SELECT COUNT(*)\n"
|
|
140
|
+
"FROM pg_catalog.pg_class AS cls INNER JOIN pg_catalog.pg_namespace AS nsp ON cls.relnamespace = nsp.oid\n"
|
|
141
|
+
f"WHERE {self._where_table(table_id)}",
|
|
142
|
+
)
|
|
143
|
+
return len(rows) > 0 and rows[0] > 0
|
|
144
|
+
|
|
145
|
+
async def has_column(self, table_id: SupportsQualifiedId, column_id: LocalId) -> bool:
|
|
146
|
+
conditions: list[str] = []
|
|
147
|
+
conditions.append(f"cls.relname = {quote(table_id.local_id)}")
|
|
148
|
+
if table_id.scope_id is not None:
|
|
149
|
+
conditions.append(f"nsp.nspname = {quote(table_id.scope_id)}")
|
|
150
|
+
conditions.append("cls.relkind = 'r' OR cls.relkind = 'v'")
|
|
151
|
+
conditions.append(f"att.attname = {quote(column_id.local_id)}")
|
|
152
|
+
condition = " AND ".join(f"({c})" for c in conditions)
|
|
153
|
+
|
|
154
|
+
rows = await self.conn.query_all(
|
|
155
|
+
int,
|
|
156
|
+
"SELECT COUNT(*)\n"
|
|
157
|
+
"FROM pg_catalog.pg_attribute AS att\n"
|
|
158
|
+
" INNER JOIN pg_catalog.pg_class AS cls ON att.attrelid = cls.oid\n"
|
|
159
|
+
" INNER JOIN pg_catalog.pg_namespace AS nsp ON cls.relnamespace = nsp.oid\n"
|
|
160
|
+
f"WHERE {condition}",
|
|
161
|
+
)
|
|
162
|
+
return len(rows) > 0 and rows[0] > 0
|
|
163
|
+
|
|
164
|
+
async def get_struct(self, struct_id: SupportsQualifiedId) -> StructType:
|
|
165
|
+
conditions: list[str] = []
|
|
166
|
+
conditions.append(f"typ.typname = {quote(struct_id.local_id)}")
|
|
167
|
+
if struct_id.scope_id is not None:
|
|
168
|
+
conditions.append(f"nsp.nspname = {quote(struct_id.scope_id)}")
|
|
169
|
+
condition = " AND ".join(f"({c})" for c in conditions)
|
|
170
|
+
|
|
171
|
+
struct_record = await self.conn.query_one(
|
|
172
|
+
PostgreSQLStructMeta,
|
|
173
|
+
"SELECT\n"
|
|
174
|
+
" dsc.description AS description\n"
|
|
175
|
+
"FROM pg_catalog.pg_type AS typ\n"
|
|
176
|
+
" INNER JOIN pg_catalog.pg_namespace AS nsp ON typ.typnamespace = nsp.oid\n"
|
|
177
|
+
" LEFT JOIN pg_catalog.pg_description AS dsc ON dsc.objoid = typ.oid AND dsc.objsubid = 0\n"
|
|
178
|
+
f"WHERE {condition}",
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
member_records = await self.conn.query_all(
|
|
182
|
+
PostgreSQLMemberMeta,
|
|
183
|
+
"SELECT\n"
|
|
184
|
+
" att.attname AS member_name,\n"
|
|
185
|
+
" typ_nsp.nspname AS type_schema,\n"
|
|
186
|
+
" typ.typname AS type_name,\n"
|
|
187
|
+
" CASE WHEN typ.typelem != 0 THEN typ_elem.typname ELSE typ.typname END AS type_name,\n"
|
|
188
|
+
" pg_catalog.format_type(att.atttypid, att.atttypmod) AS type_def,\n"
|
|
189
|
+
" dsc.description AS description\n"
|
|
190
|
+
"FROM pg_catalog.pg_attribute AS att\n"
|
|
191
|
+
" INNER JOIN pg_catalog.pg_type AS typ ON att.atttypid = typ.oid\n"
|
|
192
|
+
" INNER JOIN pg_catalog.pg_namespace AS typ_nsp ON typ.typnamespace = typ_nsp.oid\n"
|
|
193
|
+
" INNER JOIN pg_catalog.pg_class AS cls ON att.attrelid = cls.oid\n"
|
|
194
|
+
" INNER JOIN pg_catalog.pg_namespace AS nsp ON cls.relnamespace = nsp.oid\n"
|
|
195
|
+
" LEFT JOIN pg_catalog.pg_type AS typ_elem ON typ.typelem = typ_elem.oid\n"
|
|
196
|
+
" LEFT JOIN pg_catalog.pg_description AS dsc ON dsc.objoid = cls.oid AND dsc.objsubid = att.attnum\n"
|
|
197
|
+
f"WHERE {self._where_struct(struct_id)} AND (att.attnum > 0)\n"
|
|
198
|
+
"ORDER BY att.attnum",
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
members: list[StructMember] = []
|
|
202
|
+
for mem in member_records:
|
|
203
|
+
data_type = self.discovery.sql_data_type_from_spec(
|
|
204
|
+
type_name=mem.type_name,
|
|
205
|
+
type_schema=mem.type_schema,
|
|
206
|
+
type_def=mem.type_def,
|
|
207
|
+
)
|
|
208
|
+
members.append(
|
|
209
|
+
StructMember(
|
|
210
|
+
LocalId(mem.member_name),
|
|
211
|
+
data_type,
|
|
212
|
+
description=mem.description or None,
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
return self.factory.struct_class(
|
|
217
|
+
name=struct_id,
|
|
218
|
+
members=members,
|
|
219
|
+
description=struct_record.description,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
async def get_table(self, table_id: SupportsQualifiedId) -> Table:
|
|
223
|
+
table_record = await self.conn.query_one(
|
|
224
|
+
PostgreSQLTableMeta,
|
|
225
|
+
"SELECT\n"
|
|
226
|
+
" dsc.description AS description\n"
|
|
227
|
+
"FROM pg_catalog.pg_class AS cls\n"
|
|
228
|
+
" INNER JOIN pg_catalog.pg_namespace AS nsp ON cls.relnamespace = nsp.oid\n"
|
|
229
|
+
" LEFT JOIN pg_catalog.pg_description AS dsc ON dsc.objoid = cls.oid AND dsc.objsubid = 0\n"
|
|
230
|
+
f"WHERE {self._where_table(table_id)}",
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
column_records = await self.conn.query_all(
|
|
234
|
+
PostgreSQLColumnMeta,
|
|
235
|
+
"SELECT\n"
|
|
236
|
+
" att.attname AS column_name,\n"
|
|
237
|
+
" typ_nsp.nspname AS type_schema,\n"
|
|
238
|
+
" CASE WHEN typ.typelem != 0 THEN typ_elem.typname ELSE typ.typname END AS type_name,\n"
|
|
239
|
+
" pg_catalog.format_type(att.atttypid, att.atttypmod) AS type_def,\n"
|
|
240
|
+
" NOT att.attnotnull AS is_nullable,\n"
|
|
241
|
+
" att.attndims != 0 AS is_array,\n"
|
|
242
|
+
" att.attidentity IN ('a', 'd') AS is_identity,\n"
|
|
243
|
+
" pg_catalog.pg_get_expr(def.adbin, def.adrelid) AS default_value,\n"
|
|
244
|
+
" dsc.description AS description\n"
|
|
245
|
+
"FROM pg_catalog.pg_attribute AS att\n"
|
|
246
|
+
" INNER JOIN pg_catalog.pg_type AS typ ON att.atttypid = typ.oid\n"
|
|
247
|
+
" INNER JOIN pg_catalog.pg_namespace AS typ_nsp ON typ.typnamespace = typ_nsp.oid\n"
|
|
248
|
+
" INNER JOIN pg_catalog.pg_class AS cls ON att.attrelid = cls.oid\n"
|
|
249
|
+
" INNER JOIN pg_catalog.pg_namespace AS nsp ON cls.relnamespace = nsp.oid\n"
|
|
250
|
+
" LEFT JOIN pg_catalog.pg_type AS typ_elem ON typ.typelem = typ_elem.oid\n"
|
|
251
|
+
" LEFT JOIN pg_catalog.pg_attrdef AS def ON def.adrelid = cls.oid AND def.adnum = att.attnum\n"
|
|
252
|
+
" LEFT JOIN pg_catalog.pg_description AS dsc ON dsc.objoid = cls.oid AND dsc.objsubid = att.attnum\n"
|
|
253
|
+
f"WHERE {self._where_table(table_id)} AND (att.attnum > 0)\n"
|
|
254
|
+
"ORDER BY att.attnum",
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
columns: list[Column] = []
|
|
258
|
+
for col in column_records:
|
|
259
|
+
data_type = self.discovery.sql_data_type_from_spec(
|
|
260
|
+
type_name=col.type_name,
|
|
261
|
+
type_schema=col.type_schema,
|
|
262
|
+
type_def=col.type_def,
|
|
263
|
+
)
|
|
264
|
+
if col.is_array:
|
|
265
|
+
data_type = SqlArrayType(data_type)
|
|
266
|
+
columns.append(
|
|
267
|
+
self.factory.column_class(
|
|
268
|
+
LocalId(col.column_name),
|
|
269
|
+
data_type,
|
|
270
|
+
bool(col.is_nullable),
|
|
271
|
+
identity=col.is_identity,
|
|
272
|
+
default=to_default_expr(col.default_value),
|
|
273
|
+
description=col.description or None,
|
|
274
|
+
)
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
constraint_records = await self.conn.query_all(
|
|
278
|
+
PostgreSQLConstraintMeta,
|
|
279
|
+
"SELECT\n"
|
|
280
|
+
" cons.contype AS constraint_type,\n"
|
|
281
|
+
" cons.conname AS constraint_name,\n"
|
|
282
|
+
" att_s.attname AS source_column,\n"
|
|
283
|
+
" nsp_t.nspname AS target_namespace,\n"
|
|
284
|
+
" cls_t.relname AS target_table,\n"
|
|
285
|
+
" att_t.attname AS target_column\n"
|
|
286
|
+
"FROM (\n"
|
|
287
|
+
"SELECT\n"
|
|
288
|
+
" con.contype,\n"
|
|
289
|
+
" con.conname,\n"
|
|
290
|
+
" con.conrelid AS pkeyrel,\n"
|
|
291
|
+
" tgt.pkeycol,\n"
|
|
292
|
+
" con.confrelid AS fkeyrel,\n"
|
|
293
|
+
" tgt.fkeycol\n"
|
|
294
|
+
"FROM pg_catalog.pg_constraint AS con\n"
|
|
295
|
+
" CROSS JOIN UNNEST(con.conkey, con.confkey) AS tgt(pkeycol, fkeycol)\n"
|
|
296
|
+
" INNER JOIN pg_catalog.pg_class AS cls ON con.conrelid = cls.oid\n"
|
|
297
|
+
" INNER JOIN pg_catalog.pg_namespace AS nsp ON cls.relnamespace = nsp.oid\n"
|
|
298
|
+
f"WHERE {self._where_table(table_id)} AND con.contype IN ('p', 'u', 'f')\n"
|
|
299
|
+
") AS cons\n"
|
|
300
|
+
" INNER JOIN pg_catalog.pg_attribute AS att_s ON cons.pkeyrel = att_s.attrelid AND cons.pkeycol = att_s.attnum\n"
|
|
301
|
+
" LEFT JOIN pg_catalog.pg_class AS cls_t ON cons.fkeyrel = cls_t.oid\n"
|
|
302
|
+
" LEFT JOIN pg_catalog.pg_namespace AS nsp_t ON cls_t.relnamespace = nsp_t.oid\n"
|
|
303
|
+
" LEFT JOIN pg_catalog.pg_attribute AS att_t ON cons.fkeyrel = att_t.attrelid AND cons.fkeycol = att_t.attnum",
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
primary: list[LocalId] = []
|
|
307
|
+
unique = UniqueFactory()
|
|
308
|
+
foreign = ForeignFactory()
|
|
309
|
+
for con in constraint_records:
|
|
310
|
+
if con.constraint_type == b"p":
|
|
311
|
+
primary.append(LocalId(con.source_column))
|
|
312
|
+
elif con.constraint_type == b"u":
|
|
313
|
+
unique.add(con.constraint_name, LocalId(con.source_column))
|
|
314
|
+
elif con.constraint_type == b"f":
|
|
315
|
+
foreign.add(
|
|
316
|
+
con.constraint_name,
|
|
317
|
+
LocalId(con.source_column),
|
|
318
|
+
QualifiedId(con.target_namespace, con.target_table),
|
|
319
|
+
LocalId(con.target_column),
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
constraints: list[Constraint] = []
|
|
323
|
+
constraints.extend(unique.fetch())
|
|
324
|
+
constraints.extend(foreign.fetch())
|
|
325
|
+
|
|
326
|
+
return self.factory.table_class(
|
|
327
|
+
name=table_id,
|
|
328
|
+
columns=columns,
|
|
329
|
+
primary_key=tuple(primary),
|
|
330
|
+
constraints=constraints or None,
|
|
331
|
+
description=table_record.description,
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
async def get_namespace_current(self) -> Namespace:
|
|
335
|
+
schema = await self.conn.current_schema()
|
|
336
|
+
if schema is None:
|
|
337
|
+
raise RuntimeError("unable to fetch current schema")
|
|
338
|
+
return await self.get_namespace(LocalId(schema))
|
|
339
|
+
|
|
340
|
+
async def get_namespace(self, namespace_id: LocalId) -> Namespace:
|
|
341
|
+
enum_records = await self.conn.query_all(
|
|
342
|
+
PostgreSQLEnumMeta,
|
|
343
|
+
"SELECT\n"
|
|
344
|
+
"t.typname as enum_name,\n"
|
|
345
|
+
"e.enumlabel as enum_value\n"
|
|
346
|
+
"FROM pg_catalog.pg_type t\n"
|
|
347
|
+
" JOIN pg_catalog.pg_enum e ON t.oid = e.enumtypid\n"
|
|
348
|
+
" JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace\n"
|
|
349
|
+
f"WHERE n.nspname = {quote(namespace_id.id)}\n"
|
|
350
|
+
"ORDER BY enum_name, e.enumsortorder\n"
|
|
351
|
+
";",
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
enums: list[EnumType] = []
|
|
355
|
+
for enum_name, enum_groups in groupby(enum_records, key=lambda e: e.enum_name):
|
|
356
|
+
enums.append(
|
|
357
|
+
EnumType(
|
|
358
|
+
QualifiedId(namespace_id.id, enum_name),
|
|
359
|
+
[e.enum_value for e in enum_groups],
|
|
360
|
+
)
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
struct_names = await self.conn.query_all(
|
|
364
|
+
str,
|
|
365
|
+
"SELECT cls.relname\n"
|
|
366
|
+
"FROM pg_catalog.pg_class AS cls INNER JOIN pg_catalog.pg_namespace AS nsp ON cls.relnamespace = nsp.oid\n"
|
|
367
|
+
f"WHERE nsp.nspname = {quote(namespace_id.id)} AND (cls.relkind = 'c')\n"
|
|
368
|
+
";",
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
structs: list[StructType] = []
|
|
372
|
+
for struct_name in struct_names:
|
|
373
|
+
struct = await self.get_struct(QualifiedId(namespace_id.id, struct_name))
|
|
374
|
+
structs.append(struct)
|
|
375
|
+
|
|
376
|
+
table_names = await self.conn.query_all(
|
|
377
|
+
str,
|
|
378
|
+
"SELECT cls.relname\n"
|
|
379
|
+
"FROM pg_catalog.pg_class AS cls INNER JOIN pg_catalog.pg_namespace AS nsp ON cls.relnamespace = nsp.oid\n"
|
|
380
|
+
f"WHERE nsp.nspname = {quote(namespace_id.id)} AND (cls.relkind = 'r' OR cls.relkind = 'v')\n"
|
|
381
|
+
";",
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
tables: list[Table] = []
|
|
385
|
+
for table_name in table_names:
|
|
386
|
+
table = await self.get_table(QualifiedId(namespace_id.id, table_name))
|
|
387
|
+
tables.append(table)
|
|
388
|
+
|
|
389
|
+
if enums or structs or tables:
|
|
390
|
+
return self.factory.namespace_class(namespace_id, enums=enums, structs=structs, tables=tables)
|
|
391
|
+
else:
|
|
392
|
+
return self.factory.namespace_class()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from pysqlsync.base import BaseConnection, BaseEngine, BaseGenerator, Explorer
|
|
2
|
+
|
|
3
|
+
from .connection import PostgreSQLConnection
|
|
4
|
+
from .discovery import PostgreSQLExplorer
|
|
5
|
+
from .generator import PostgreSQLGenerator
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class PostgreSQLEngine(BaseEngine):
|
|
9
|
+
@property
|
|
10
|
+
def name(self) -> str:
|
|
11
|
+
return "postgresql"
|
|
12
|
+
|
|
13
|
+
def get_generator_type(self) -> type[BaseGenerator]:
|
|
14
|
+
return PostgreSQLGenerator
|
|
15
|
+
|
|
16
|
+
def get_connection_type(self) -> type[BaseConnection]:
|
|
17
|
+
return PostgreSQLConnection
|
|
18
|
+
|
|
19
|
+
def get_explorer_type(self) -> type[Explorer]:
|
|
20
|
+
return PostgreSQLExplorer
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
from strong_typing.core import JsonType
|
|
4
|
+
|
|
5
|
+
from pysqlsync.base import BaseGenerator, GeneratorOptions
|
|
6
|
+
from pysqlsync.formation.object_types import Table
|
|
7
|
+
from pysqlsync.formation.py_to_sql import (
|
|
8
|
+
ArrayMode,
|
|
9
|
+
DataclassConverter,
|
|
10
|
+
DataclassConverterOptions,
|
|
11
|
+
EnumMode,
|
|
12
|
+
NamespaceMapping,
|
|
13
|
+
StructMode,
|
|
14
|
+
)
|
|
15
|
+
from pysqlsync.util.typing import override
|
|
16
|
+
|
|
17
|
+
from .data_types import PostgreSQLJsonType
|
|
18
|
+
from .mutation import PostgreSQLMutator
|
|
19
|
+
from .object_types import PostgreSQLObjectFactory
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class PostgreSQLGenerator(BaseGenerator):
|
|
23
|
+
"Generator for PostgreSQL."
|
|
24
|
+
|
|
25
|
+
converter: DataclassConverter
|
|
26
|
+
|
|
27
|
+
def __init__(self, options: GeneratorOptions) -> None:
|
|
28
|
+
super().__init__(
|
|
29
|
+
options,
|
|
30
|
+
PostgreSQLObjectFactory(),
|
|
31
|
+
PostgreSQLMutator(options.synchronization),
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
self.check_enum_mode(exclude=[EnumMode.INLINE])
|
|
35
|
+
self.check_struct_mode(exclude=[StructMode.INLINE])
|
|
36
|
+
|
|
37
|
+
self.converter = DataclassConverter(
|
|
38
|
+
options=DataclassConverterOptions(
|
|
39
|
+
enum_mode=options.enum_mode or EnumMode.TYPE,
|
|
40
|
+
struct_mode=options.struct_mode or StructMode.TYPE,
|
|
41
|
+
array_mode=options.array_mode or ArrayMode.ARRAY,
|
|
42
|
+
unique_constraint_names=False,
|
|
43
|
+
namespaces=NamespaceMapping(options.namespaces),
|
|
44
|
+
foreign_constraints=options.foreign_constraints,
|
|
45
|
+
initialize_tables=options.initialize_tables,
|
|
46
|
+
substitutions={
|
|
47
|
+
JsonType: PostgreSQLJsonType(),
|
|
48
|
+
},
|
|
49
|
+
factory=self.factory,
|
|
50
|
+
skip_annotations=options.skip_annotations,
|
|
51
|
+
auto_default=options.auto_default,
|
|
52
|
+
)
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
@override
|
|
56
|
+
def placeholder(self, index: int) -> str:
|
|
57
|
+
return f"${index}"
|
|
58
|
+
|
|
59
|
+
@override
|
|
60
|
+
def get_table_insert_stmt(self, table: Table, order: Optional[tuple[str, ...]] = None) -> str:
|
|
61
|
+
statements: list[str] = []
|
|
62
|
+
statements.append(f"INSERT INTO {table.name}")
|
|
63
|
+
columns = [column for column in table.get_columns(order) if not column.identity]
|
|
64
|
+
column_list = ", ".join(str(column.name) for column in columns)
|
|
65
|
+
value_list = ", ".join(self.placeholder(index) for index, _ in enumerate(columns, start=1))
|
|
66
|
+
statements.append(f"({column_list}) VALUES ({value_list})")
|
|
67
|
+
statements.append(";")
|
|
68
|
+
return "\n".join(statements)
|
|
69
|
+
|
|
70
|
+
@override
|
|
71
|
+
def get_table_merge_stmt(self, table: Table, order: Optional[tuple[str, ...]] = None) -> str:
|
|
72
|
+
statements: list[str] = []
|
|
73
|
+
statements.append(f"INSERT INTO {table.name}")
|
|
74
|
+
columns = [column for column in table.get_columns(order) if not column.identity]
|
|
75
|
+
column_list = ", ".join(str(column.name) for column in columns)
|
|
76
|
+
value_list = ", ".join(self.placeholder(index) for index, _ in enumerate(columns, start=1))
|
|
77
|
+
statements.append(f"({column_list}) VALUES ({value_list})")
|
|
78
|
+
statements.append("ON CONFLICT DO NOTHING")
|
|
79
|
+
statements.append(";")
|
|
80
|
+
return "\n".join(statements)
|
|
81
|
+
|
|
82
|
+
@override
|
|
83
|
+
def get_table_upsert_stmt(self, table: Table, order: Optional[tuple[str, ...]] = None) -> str:
|
|
84
|
+
statements: list[str] = []
|
|
85
|
+
statements.append(f"INSERT INTO {table.name}")
|
|
86
|
+
columns = [column for column in table.get_columns(order)]
|
|
87
|
+
column_list = ", ".join(str(column.name) for column in columns)
|
|
88
|
+
value_list = ", ".join(self.placeholder(index) for index, _ in enumerate(columns, start=1))
|
|
89
|
+
statements.append(f"({column_list}) VALUES ({value_list})")
|
|
90
|
+
value_columns = table.get_value_columns()
|
|
91
|
+
keys = ", ".join(str(key) for key in table.primary_key)
|
|
92
|
+
if value_columns:
|
|
93
|
+
statements.append(f"ON CONFLICT ({keys}) DO UPDATE SET")
|
|
94
|
+
defs = [f"{column.name} = EXCLUDED.{column.name}" for column in value_columns]
|
|
95
|
+
statements.append(",\n".join(defs))
|
|
96
|
+
else:
|
|
97
|
+
statements.append(f"ON CONFLICT ({keys}) DO NOTHING")
|
|
98
|
+
statements.append(";")
|
|
99
|
+
return "\n".join(statements)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
from pysqlsync.formation.mutation import Mutator
|
|
4
|
+
from pysqlsync.formation.object_types import Column, EnumType, StatementList, StructType, Table, deleted, join_or_none
|
|
5
|
+
from pysqlsync.model.data_types import SqlIntegerType, SqlUserDefinedType, quote
|
|
6
|
+
from pysqlsync.model.id_types import LocalId
|
|
7
|
+
from pysqlsync.util.typing import override
|
|
8
|
+
|
|
9
|
+
from .object_types import sql_quoted_string
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class PostgreSQLMutator(Mutator):
|
|
13
|
+
@override
|
|
14
|
+
def is_column_migrated(self, source: Column, target: Column) -> Optional[bool]:
|
|
15
|
+
is_migrated = super().is_column_migrated(source, target)
|
|
16
|
+
if is_migrated is not None:
|
|
17
|
+
return is_migrated
|
|
18
|
+
|
|
19
|
+
if isinstance(target.data_type, SqlIntegerType):
|
|
20
|
+
# PostgreSQL defines a separate type for each enumeration type
|
|
21
|
+
if isinstance(source.data_type, SqlUserDefinedType):
|
|
22
|
+
return True
|
|
23
|
+
|
|
24
|
+
return None # undecided (converts to False in a Boolean expression)
|
|
25
|
+
|
|
26
|
+
def migrate_enum_stmt(self, enum_type: EnumType, table: Table) -> Optional[str]:
|
|
27
|
+
enum_values = ", ".join(f"({quote(v)})" for v in enum_type.values)
|
|
28
|
+
return f'INSERT INTO {table.name} ("value") VALUES {enum_values} ON CONFLICT ("value") DO NOTHING;'
|
|
29
|
+
|
|
30
|
+
def migrate_column_stmt(self, source_table: Table, source: Column, target_table: Table, target: Column) -> Optional[str]:
|
|
31
|
+
ref = target_table.get_constraint(target.name)
|
|
32
|
+
return (
|
|
33
|
+
f"UPDATE {source_table.name} data_table\n"
|
|
34
|
+
f'SET {target.name} = enum_table."id"\n'
|
|
35
|
+
f"FROM {ref.table} enum_table\n"
|
|
36
|
+
f'WHERE data_table.{LocalId(deleted(source.name.id))}::VARCHAR = enum_table."value";'
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
def mutate_table_stmt(self, source: Table, target: Table) -> Optional[str]:
|
|
40
|
+
statements: StatementList = StatementList()
|
|
41
|
+
statements.append(super().mutate_table_stmt(source, target))
|
|
42
|
+
|
|
43
|
+
for target_column in target.columns.values():
|
|
44
|
+
source_column = source.columns.get(target_column.name.id)
|
|
45
|
+
|
|
46
|
+
source_desc = source_column.description if source_column is not None else None
|
|
47
|
+
target_desc = target_column.description
|
|
48
|
+
|
|
49
|
+
if target_desc is None:
|
|
50
|
+
if source_desc is not None:
|
|
51
|
+
statements.append(f"COMMENT ON COLUMN {target.name}.{target_column.name} IS NULL;")
|
|
52
|
+
else:
|
|
53
|
+
if source_desc != target_desc:
|
|
54
|
+
statements.append(f"COMMENT ON COLUMN {target.name}.{target_column.name} IS {sql_quoted_string(target_desc)};")
|
|
55
|
+
|
|
56
|
+
if target.description is None:
|
|
57
|
+
if source.description is not None:
|
|
58
|
+
statements.append(f"COMMENT ON TABLE {target.name} IS NULL;")
|
|
59
|
+
else:
|
|
60
|
+
if source.description != target.description:
|
|
61
|
+
statements.append(f"COMMENT ON TABLE {target.name} IS {sql_quoted_string(target.description)};")
|
|
62
|
+
|
|
63
|
+
return join_or_none(statements)
|
|
64
|
+
|
|
65
|
+
def mutate_column_type(self, source: Column, target: Column) -> Optional[str]:
|
|
66
|
+
if source.data_type != target.data_type:
|
|
67
|
+
return f"SET DATA TYPE {target.data_type} USING {source.name}::{target.data_type}"
|
|
68
|
+
else:
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
def mutate_struct_stmt(self, source: StructType, target: StructType) -> Optional[str]:
|
|
72
|
+
statements: StatementList = StatementList()
|
|
73
|
+
statements.append(super().mutate_struct_stmt(source, target))
|
|
74
|
+
|
|
75
|
+
if target.description is None:
|
|
76
|
+
if source.description is not None:
|
|
77
|
+
statements.append(f"COMMENT ON TYPE {target.name} IS NULL;")
|
|
78
|
+
else:
|
|
79
|
+
if source.description != target.description:
|
|
80
|
+
statements.append(f"COMMENT ON TYPE {target.name} IS {sql_quoted_string(target.description)};")
|
|
81
|
+
|
|
82
|
+
return join_or_none(statements)
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from pysqlsync.formation.object_types import (
|
|
5
|
+
EnumTable,
|
|
6
|
+
Namespace,
|
|
7
|
+
ObjectFactory,
|
|
8
|
+
StructType,
|
|
9
|
+
Table,
|
|
10
|
+
)
|
|
11
|
+
from pysqlsync.model.id_types import LocalId
|
|
12
|
+
|
|
13
|
+
_sql_quoted_str_table = str.maketrans(
|
|
14
|
+
{
|
|
15
|
+
"\\": "\\\\",
|
|
16
|
+
"'": "\\'",
|
|
17
|
+
"\b": "\\b",
|
|
18
|
+
"\f": "\\f",
|
|
19
|
+
"\n": "\\n",
|
|
20
|
+
"\r": "\\r",
|
|
21
|
+
"\t": "\\t",
|
|
22
|
+
}
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def sql_quoted_string(text: str) -> str:
|
|
27
|
+
if re.search(r"[\b\f\n\r\t]", text):
|
|
28
|
+
string = text.translate(_sql_quoted_str_table)
|
|
29
|
+
return f"E'{string}'"
|
|
30
|
+
else:
|
|
31
|
+
string = text.replace("'", "''")
|
|
32
|
+
return f"'{string}'"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class PostgreSQLTable(Table):
|
|
36
|
+
def create_stmt(self) -> str:
|
|
37
|
+
statements: list[str] = []
|
|
38
|
+
statements.append(super().create_stmt())
|
|
39
|
+
|
|
40
|
+
# output comments for table and column objects
|
|
41
|
+
if self.description is not None:
|
|
42
|
+
statements.append(f"COMMENT ON TABLE {self.name} IS {sql_quoted_string(self.description)};")
|
|
43
|
+
for column in self.columns.values():
|
|
44
|
+
if column.description is not None:
|
|
45
|
+
statements.append(f"COMMENT ON COLUMN {self.name}.{column.name} IS {sql_quoted_string(column.description)};")
|
|
46
|
+
return "\n".join(statements)
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def primary_key_constraint_id(self) -> LocalId:
|
|
50
|
+
return LocalId(f"pk_{self.name.local_id.replace('.', '_')}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class PostgreSQLEnumTable(EnumTable, PostgreSQLTable):
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class PostgreSQLStructType(StructType):
|
|
58
|
+
def create_stmt(self) -> str:
|
|
59
|
+
statements: list[str] = []
|
|
60
|
+
statements.append(super().create_stmt())
|
|
61
|
+
|
|
62
|
+
if self.description is not None:
|
|
63
|
+
statements.append(f"COMMENT ON TYPE {self.name} IS {sql_quoted_string(self.description)};")
|
|
64
|
+
for member in self.members.values():
|
|
65
|
+
if member.description is not None:
|
|
66
|
+
statements.append(f"COMMENT ON COLUMN {self.name}.{member.name} IS {sql_quoted_string(member.description)};")
|
|
67
|
+
return "\n".join(statements)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class PostgreSQLNamespace(Namespace):
|
|
71
|
+
def create_schema_stmt(self) -> Optional[str]:
|
|
72
|
+
if self.name.local_id:
|
|
73
|
+
return f"CREATE SCHEMA IF NOT EXISTS {self.name};"
|
|
74
|
+
else:
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
def drop_schema_stmt(self) -> Optional[str]:
|
|
78
|
+
if self.name.local_id:
|
|
79
|
+
return f"DROP SCHEMA IF EXISTS {self.name} CASCADE;"
|
|
80
|
+
else:
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class PostgreSQLObjectFactory(ObjectFactory):
|
|
85
|
+
@property
|
|
86
|
+
def table_class(self) -> type[Table]:
|
|
87
|
+
return PostgreSQLTable
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def enum_table_class(self) -> type[EnumTable]:
|
|
91
|
+
return PostgreSQLEnumTable
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def struct_class(self) -> type[StructType]:
|
|
95
|
+
return PostgreSQLStructType
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def namespace_class(self) -> type[Namespace]:
|
|
99
|
+
return PostgreSQLNamespace
|