mtsql 1.11.18__py3-none-any.whl → 1.11.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mt/sql/redshift/__init__.py +2 -18
- mt/sql/redshift/commands.py +348 -261
- mt/sql/redshift/ddl.py +29 -27
- mt/sql/redshift/dialect.py +457 -345
- mt/sql/version.py +1 -1
- {mtsql-1.11.18.dist-info → mtsql-1.11.20.dist-info}/METADATA +2 -1
- mtsql-1.11.20.dist-info/RECORD +17 -0
- mtsql-1.11.18.dist-info/RECORD +0 -17
- {mtsql-1.11.18.dist-info → mtsql-1.11.20.dist-info}/LICENSE +0 -0
- {mtsql-1.11.18.dist-info → mtsql-1.11.20.dist-info}/WHEEL +0 -0
- {mtsql-1.11.18.dist-info → mtsql-1.11.20.dist-info}/top_level.txt +0 -0
mt/sql/redshift/dialect.py
CHANGED
|
@@ -9,26 +9,44 @@ import sqlalchemy as sa
|
|
|
9
9
|
from packaging.version import Version
|
|
10
10
|
from sqlalchemy import inspect
|
|
11
11
|
from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION
|
|
12
|
-
from sqlalchemy.dialects.postgresql.base import (
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
from sqlalchemy.dialects.postgresql.base import (
|
|
13
|
+
PGCompiler,
|
|
14
|
+
PGDDLCompiler,
|
|
15
|
+
PGDialect,
|
|
16
|
+
PGExecutionContext,
|
|
17
|
+
PGIdentifierPreparer,
|
|
18
|
+
PGTypeCompiler,
|
|
19
|
+
)
|
|
18
20
|
from sqlalchemy.engine import reflection
|
|
19
21
|
from sqlalchemy.engine.default import DefaultDialect
|
|
20
22
|
from sqlalchemy.ext.compiler import compiles
|
|
21
|
-
from sqlalchemy.sql.expression import
|
|
22
|
-
Delete)
|
|
23
|
+
from sqlalchemy.sql.expression import BinaryExpression, BooleanClauseList, Delete
|
|
23
24
|
from sqlalchemy.sql.type_api import TypeEngine
|
|
24
|
-
from sqlalchemy.types import (
|
|
25
|
-
|
|
25
|
+
from sqlalchemy.types import (
|
|
26
|
+
BIGINT,
|
|
27
|
+
BOOLEAN,
|
|
28
|
+
CHAR,
|
|
29
|
+
DATE,
|
|
30
|
+
DECIMAL,
|
|
31
|
+
INTEGER,
|
|
32
|
+
REAL,
|
|
33
|
+
SMALLINT,
|
|
34
|
+
TIMESTAMP,
|
|
35
|
+
VARCHAR,
|
|
36
|
+
NullType,
|
|
37
|
+
)
|
|
26
38
|
|
|
27
|
-
from .commands import (
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
39
|
+
from .commands import (
|
|
40
|
+
AlterTableAppendCommand,
|
|
41
|
+
Compression,
|
|
42
|
+
CopyCommand,
|
|
43
|
+
CreateLibraryCommand,
|
|
44
|
+
Encoding,
|
|
45
|
+
Format,
|
|
46
|
+
RefreshMaterializedView,
|
|
47
|
+
UnloadFromSelect,
|
|
48
|
+
)
|
|
49
|
+
from .ddl import CreateMaterializedView, DropMaterializedView, get_table_attributes
|
|
32
50
|
|
|
33
51
|
sa_version = Version(sa.__version__)
|
|
34
52
|
logger = getLogger(__name__)
|
|
@@ -40,50 +58,56 @@ except ImportError:
|
|
|
40
58
|
else:
|
|
41
59
|
from alembic.ddl import postgresql
|
|
42
60
|
from alembic.ddl.base import RenameTable
|
|
43
|
-
compiles(RenameTable, 'redshift')(postgresql.visit_rename_table)
|
|
44
61
|
|
|
45
|
-
|
|
62
|
+
compiles(RenameTable, "redshift")(postgresql.visit_rename_table)
|
|
63
|
+
|
|
64
|
+
if Version(alembic.__version__) >= Version("1.0.6"):
|
|
46
65
|
from alembic.ddl.base import ColumnComment
|
|
47
|
-
|
|
66
|
+
|
|
67
|
+
compiles(ColumnComment, "redshift")(postgresql.visit_column_comment)
|
|
48
68
|
|
|
49
69
|
class RedshiftImpl(postgresql.PostgresqlImpl):
|
|
50
|
-
__dialect__ =
|
|
70
|
+
__dialect__ = "redshift"
|
|
71
|
+
|
|
51
72
|
|
|
52
73
|
# "Each dialect provides the full set of typenames supported by that backend
|
|
53
74
|
# with its __all__ collection
|
|
54
75
|
# https://docs.sqlalchemy.org/en/13/core/type_basics.html#vendor-specific-types
|
|
55
76
|
__all__ = (
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
77
|
+
"SMALLINT",
|
|
78
|
+
"INTEGER",
|
|
79
|
+
"BIGINT",
|
|
80
|
+
"DECIMAL",
|
|
81
|
+
"REAL",
|
|
82
|
+
"BOOLEAN",
|
|
83
|
+
"CHAR",
|
|
84
|
+
"DATE",
|
|
85
|
+
"TIMESTAMP",
|
|
86
|
+
"VARCHAR",
|
|
87
|
+
"DOUBLE_PRECISION",
|
|
88
|
+
"GEOMETRY",
|
|
89
|
+
"SUPER",
|
|
90
|
+
"TIMESTAMPTZ",
|
|
91
|
+
"TIMETZ",
|
|
92
|
+
"HLLSKETCH",
|
|
93
|
+
"RedshiftDialect",
|
|
94
|
+
"CopyCommand",
|
|
95
|
+
"UnloadFromSelect",
|
|
96
|
+
"Compression",
|
|
97
|
+
"Encoding",
|
|
98
|
+
"Format",
|
|
99
|
+
"CreateLibraryCommand",
|
|
100
|
+
"AlterTableAppendCommand",
|
|
101
|
+
"RefreshMaterializedView",
|
|
102
|
+
"CreateMaterializedView",
|
|
103
|
+
"DropMaterializedView",
|
|
81
104
|
)
|
|
82
105
|
|
|
83
106
|
|
|
84
107
|
# Regex for parsing and identity constraint out of adsrc, e.g.:
|
|
85
108
|
# "identity"(445178, 0, '1,1'::text)
|
|
86
|
-
IDENTITY_RE = re.compile(
|
|
109
|
+
IDENTITY_RE = re.compile(
|
|
110
|
+
r"""
|
|
87
111
|
"identity" \(
|
|
88
112
|
(?P<current>-?\d+)
|
|
89
113
|
,\s
|
|
@@ -92,20 +116,26 @@ IDENTITY_RE = re.compile(r"""
|
|
|
92
116
|
'(?P<seed>-?\d+),(?P<step>-?\d+)'
|
|
93
117
|
.*
|
|
94
118
|
\)
|
|
95
|
-
""",
|
|
119
|
+
""",
|
|
120
|
+
re.VERBOSE,
|
|
121
|
+
)
|
|
96
122
|
|
|
97
123
|
# Regex for SQL identifiers (valid table and column names)
|
|
98
|
-
SQL_IDENTIFIER_RE = re.compile(
|
|
124
|
+
SQL_IDENTIFIER_RE = re.compile(
|
|
125
|
+
r"""
|
|
99
126
|
[_a-zA-Z][\w$]* # SQL standard identifier
|
|
100
127
|
| # or
|
|
101
128
|
(?:"[^"]+")+ # SQL delimited (quoted) identifier
|
|
102
|
-
""",
|
|
129
|
+
""",
|
|
130
|
+
re.VERBOSE,
|
|
131
|
+
)
|
|
103
132
|
|
|
104
133
|
# Regex for foreign key constraints, e.g.:
|
|
105
134
|
# FOREIGN KEY(col1) REFERENCES othertable (col2)
|
|
106
135
|
# See https://docs.aws.amazon.com/redshift/latest/dg/r_names.html
|
|
107
136
|
# for a definition of valid SQL identifiers.
|
|
108
|
-
FOREIGN_KEY_RE = re.compile(
|
|
137
|
+
FOREIGN_KEY_RE = re.compile(
|
|
138
|
+
r"""
|
|
109
139
|
^FOREIGN\ KEY \s* \( # FOREIGN KEY, arbitrary whitespace, literal '('
|
|
110
140
|
(?P<columns> # Start a group to capture the referring columns
|
|
111
141
|
(?: # Start a non-capturing group
|
|
@@ -129,11 +159,14 @@ FOREIGN_KEY_RE = re.compile(r"""
|
|
|
129
159
|
)+ # Close the non-capturing group; require at least one
|
|
130
160
|
) # Close the 'columns' group
|
|
131
161
|
\s* \) # Arbitrary whitespace and literal ')'
|
|
132
|
-
""",
|
|
162
|
+
""",
|
|
163
|
+
re.VERBOSE,
|
|
164
|
+
)
|
|
133
165
|
|
|
134
166
|
# Regex for primary key constraints, e.g.:
|
|
135
167
|
# PRIMARY KEY (col1, col2)
|
|
136
|
-
PRIMARY_KEY_RE = re.compile(
|
|
168
|
+
PRIMARY_KEY_RE = re.compile(
|
|
169
|
+
r"""
|
|
137
170
|
^PRIMARY \s* KEY \s* \( # FOREIGN KEY, arbitrary whitespace, literal '('
|
|
138
171
|
(?P<columns> # Start a group to capture column names
|
|
139
172
|
(?:
|
|
@@ -145,38 +178,173 @@ PRIMARY_KEY_RE = re.compile(r"""
|
|
|
145
178
|
)+ # Close the non-capturing group; require at least one
|
|
146
179
|
)
|
|
147
180
|
\s* \) \s* # Arbitrary whitespace and literal ')'
|
|
148
|
-
""",
|
|
181
|
+
""",
|
|
182
|
+
re.VERBOSE,
|
|
183
|
+
)
|
|
149
184
|
|
|
150
185
|
# Reserved words as extracted from Redshift docs.
|
|
151
186
|
# See pull_reserved_words.sh at the top level of this repository
|
|
152
187
|
# for the code used to generate this set.
|
|
153
|
-
RESERVED_WORDS = set(
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
188
|
+
RESERVED_WORDS = set(
|
|
189
|
+
[
|
|
190
|
+
"aes128",
|
|
191
|
+
"aes256",
|
|
192
|
+
"all",
|
|
193
|
+
"allowoverwrite",
|
|
194
|
+
"analyse",
|
|
195
|
+
"analyze",
|
|
196
|
+
"and",
|
|
197
|
+
"any",
|
|
198
|
+
"array",
|
|
199
|
+
"as",
|
|
200
|
+
"asc",
|
|
201
|
+
"authorization",
|
|
202
|
+
"az64",
|
|
203
|
+
"backup",
|
|
204
|
+
"between",
|
|
205
|
+
"binary",
|
|
206
|
+
"blanksasnull",
|
|
207
|
+
"both",
|
|
208
|
+
"bytedict",
|
|
209
|
+
"bzip2",
|
|
210
|
+
"case",
|
|
211
|
+
"cast",
|
|
212
|
+
"check",
|
|
213
|
+
"collate",
|
|
214
|
+
"column",
|
|
215
|
+
"constraint",
|
|
216
|
+
"create",
|
|
217
|
+
"credentials",
|
|
218
|
+
"cross",
|
|
219
|
+
"current_date",
|
|
220
|
+
"current_time",
|
|
221
|
+
"current_timestamp",
|
|
222
|
+
"current_user",
|
|
223
|
+
"current_user_id",
|
|
224
|
+
"default",
|
|
225
|
+
"deferrable",
|
|
226
|
+
"deflate",
|
|
227
|
+
"defrag",
|
|
228
|
+
"delta",
|
|
229
|
+
"delta32k",
|
|
230
|
+
"desc",
|
|
231
|
+
"disable",
|
|
232
|
+
"distinct",
|
|
233
|
+
"do",
|
|
234
|
+
"else",
|
|
235
|
+
"emptyasnull",
|
|
236
|
+
"enable",
|
|
237
|
+
"encode",
|
|
238
|
+
"encrypt",
|
|
239
|
+
"encryption",
|
|
240
|
+
"end",
|
|
241
|
+
"except",
|
|
242
|
+
"explicit",
|
|
243
|
+
"false",
|
|
244
|
+
"for",
|
|
245
|
+
"foreign",
|
|
246
|
+
"freeze",
|
|
247
|
+
"from",
|
|
248
|
+
"full",
|
|
249
|
+
"globaldict256",
|
|
250
|
+
"globaldict64k",
|
|
251
|
+
"grant",
|
|
252
|
+
"group",
|
|
253
|
+
"gzip",
|
|
254
|
+
"having",
|
|
255
|
+
"identity",
|
|
256
|
+
"ignore",
|
|
257
|
+
"ilike",
|
|
258
|
+
"in",
|
|
259
|
+
"initially",
|
|
260
|
+
"inner",
|
|
261
|
+
"intersect",
|
|
262
|
+
"into",
|
|
263
|
+
"is",
|
|
264
|
+
"isnull",
|
|
265
|
+
"join",
|
|
266
|
+
"language",
|
|
267
|
+
"leading",
|
|
268
|
+
"left",
|
|
269
|
+
"like",
|
|
270
|
+
"limit",
|
|
271
|
+
"localtime",
|
|
272
|
+
"localtimestamp",
|
|
273
|
+
"lun",
|
|
274
|
+
"luns",
|
|
275
|
+
"lzo",
|
|
276
|
+
"lzop",
|
|
277
|
+
"minus",
|
|
278
|
+
"mostly16",
|
|
279
|
+
"mostly32",
|
|
280
|
+
"mostly8",
|
|
281
|
+
"natural",
|
|
282
|
+
"new",
|
|
283
|
+
"not",
|
|
284
|
+
"notnull",
|
|
285
|
+
"null",
|
|
286
|
+
"nulls",
|
|
287
|
+
"off",
|
|
288
|
+
"offline",
|
|
289
|
+
"offset",
|
|
290
|
+
"oid",
|
|
291
|
+
"old",
|
|
292
|
+
"on",
|
|
293
|
+
"only",
|
|
294
|
+
"open",
|
|
295
|
+
"or",
|
|
296
|
+
"order",
|
|
297
|
+
"outer",
|
|
298
|
+
"overlaps",
|
|
299
|
+
"parallel",
|
|
300
|
+
"partition",
|
|
301
|
+
"percent",
|
|
302
|
+
"permissions",
|
|
303
|
+
"pivot",
|
|
304
|
+
"placing",
|
|
305
|
+
"primary",
|
|
306
|
+
"raw",
|
|
307
|
+
"readratio",
|
|
308
|
+
"recover",
|
|
309
|
+
"references",
|
|
310
|
+
"respect",
|
|
311
|
+
"rejectlog",
|
|
312
|
+
"resort",
|
|
313
|
+
"restore",
|
|
314
|
+
"right",
|
|
315
|
+
"select",
|
|
316
|
+
"session_user",
|
|
317
|
+
"similar",
|
|
318
|
+
"snapshot",
|
|
319
|
+
"some",
|
|
320
|
+
"sysdate",
|
|
321
|
+
"system",
|
|
322
|
+
"table",
|
|
323
|
+
"tag",
|
|
324
|
+
"tdes",
|
|
325
|
+
"text255",
|
|
326
|
+
"text32k",
|
|
327
|
+
"then",
|
|
328
|
+
"timestamp",
|
|
329
|
+
"to",
|
|
330
|
+
"top",
|
|
331
|
+
"trailing",
|
|
332
|
+
"true",
|
|
333
|
+
"truncatecolumns",
|
|
334
|
+
"union",
|
|
335
|
+
"unique",
|
|
336
|
+
"unnest",
|
|
337
|
+
"unpivot",
|
|
338
|
+
"user",
|
|
339
|
+
"using",
|
|
340
|
+
"verbose",
|
|
341
|
+
"wallet",
|
|
342
|
+
"when",
|
|
343
|
+
"where",
|
|
344
|
+
"with",
|
|
345
|
+
"without",
|
|
346
|
+
]
|
|
347
|
+
)
|
|
180
348
|
|
|
181
349
|
REFLECTION_SQL = """\
|
|
182
350
|
SELECT
|
|
@@ -314,7 +482,7 @@ class TIMESTAMPTZ(RedshiftTypeEngine, sa.dialects.postgresql.TIMESTAMP):
|
|
|
314
482
|
https://docs.sqlalchemy.org/en/13/core/type_basics.html#vendor-specific-types
|
|
315
483
|
"""
|
|
316
484
|
|
|
317
|
-
__visit_name__ =
|
|
485
|
+
__visit_name__ = "TIMESTAMPTZ"
|
|
318
486
|
|
|
319
487
|
def __init__(self, timezone=True, precision=None):
|
|
320
488
|
# timezone param must be present as it's provided in base class so the
|
|
@@ -335,7 +503,7 @@ class TIMETZ(RedshiftTypeEngine, sa.dialects.postgresql.TIME):
|
|
|
335
503
|
https://docs.sqlalchemy.org/en/13/core/type_basics.html#vendor-specific-types
|
|
336
504
|
"""
|
|
337
505
|
|
|
338
|
-
__visit_name__ =
|
|
506
|
+
__visit_name__ = "TIMETZ"
|
|
339
507
|
|
|
340
508
|
def __init__(self, timezone=True, precision=None):
|
|
341
509
|
# timezone param must be present as it's provided in base class so the
|
|
@@ -354,7 +522,8 @@ class GEOMETRY(RedshiftTypeEngine, sa.dialects.postgresql.TEXT):
|
|
|
354
522
|
|
|
355
523
|
https://docs.sqlalchemy.org/en/13/core/type_basics.html#vendor-specific-types
|
|
356
524
|
"""
|
|
357
|
-
|
|
525
|
+
|
|
526
|
+
__visit_name__ = "GEOMETRY"
|
|
358
527
|
|
|
359
528
|
def __init__(self):
|
|
360
529
|
super(GEOMETRY, self).__init__()
|
|
@@ -374,7 +543,7 @@ class SUPER(RedshiftTypeEngine, sa.dialects.postgresql.TEXT):
|
|
|
374
543
|
https://docs.sqlalchemy.org/en/13/core/type_basics.html#vendor-specific-types
|
|
375
544
|
"""
|
|
376
545
|
|
|
377
|
-
__visit_name__ =
|
|
546
|
+
__visit_name__ = "SUPER"
|
|
378
547
|
|
|
379
548
|
def __init__(self):
|
|
380
549
|
super(SUPER, self).__init__()
|
|
@@ -401,7 +570,8 @@ class HLLSKETCH(RedshiftTypeEngine, sa.dialects.postgresql.TEXT):
|
|
|
401
570
|
|
|
402
571
|
https://docs.sqlalchemy.org/en/13/core/type_basics.html#vendor-specific-types
|
|
403
572
|
"""
|
|
404
|
-
|
|
573
|
+
|
|
574
|
+
__visit_name__ = "HLLSKETCH"
|
|
405
575
|
|
|
406
576
|
def __init__(self):
|
|
407
577
|
super(HLLSKETCH, self).__init__()
|
|
@@ -420,10 +590,11 @@ REDSHIFT_ISCHEMA_NAMES = {
|
|
|
420
590
|
}
|
|
421
591
|
|
|
422
592
|
|
|
423
|
-
class RelationKey(namedtuple(
|
|
593
|
+
class RelationKey(namedtuple("RelationKey", ("name", "schema"))):
|
|
424
594
|
"""
|
|
425
595
|
Structured tuple of table/view name and schema name.
|
|
426
596
|
"""
|
|
597
|
+
|
|
427
598
|
__slots__ = ()
|
|
428
599
|
|
|
429
600
|
def __new__(cls, name, schema=None, connection=None):
|
|
@@ -444,10 +615,7 @@ class RelationKey(namedtuple('RelationKey', ('name', 'schema'))):
|
|
|
444
615
|
|
|
445
616
|
@staticmethod
|
|
446
617
|
def _unquote(part):
|
|
447
|
-
if (
|
|
448
|
-
part is not None and part.startswith('"') and
|
|
449
|
-
part.endswith('"')
|
|
450
|
-
):
|
|
618
|
+
if part is not None and part.startswith('"') and part.endswith('"'):
|
|
451
619
|
return part[1:-1]
|
|
452
620
|
return part
|
|
453
621
|
|
|
@@ -460,8 +628,7 @@ class RelationKey(namedtuple('RelationKey', ('name', 'schema'))):
|
|
|
460
628
|
In particular, this happens for tables named as a keyword.
|
|
461
629
|
"""
|
|
462
630
|
return RelationKey(
|
|
463
|
-
RelationKey._unquote(self.name),
|
|
464
|
-
RelationKey._unquote(self.schema)
|
|
631
|
+
RelationKey._unquote(self.name), RelationKey._unquote(self.schema)
|
|
465
632
|
)
|
|
466
633
|
|
|
467
634
|
|
|
@@ -483,7 +650,7 @@ class RedshiftDDLCompiler(PGDDLCompiler):
|
|
|
483
650
|
|
|
484
651
|
>>> import sqlalchemy as sa
|
|
485
652
|
>>> from sqlalchemy.schema import CreateTable
|
|
486
|
-
>>> engine = sa.create_engine('
|
|
653
|
+
>>> engine = sa.create_engine('mtsql_redshift://example')
|
|
487
654
|
>>> metadata = sa.MetaData()
|
|
488
655
|
>>> user = sa.Table(
|
|
489
656
|
... 'user',
|
|
@@ -582,7 +749,7 @@ class RedshiftDDLCompiler(PGDDLCompiler):
|
|
|
582
749
|
|
|
583
750
|
def post_create_table(self, table):
|
|
584
751
|
kwargs = ["diststyle", "distkey", "sortkey", "interleaved_sortkey"]
|
|
585
|
-
info = table.dialect_options[
|
|
752
|
+
info = table.dialect_options["redshift"]
|
|
586
753
|
info = {key: info.get(key) for key in kwargs}
|
|
587
754
|
return get_table_attributes(self.preparer, **info)
|
|
588
755
|
|
|
@@ -608,26 +775,26 @@ class RedshiftDDLCompiler(PGDDLCompiler):
|
|
|
608
775
|
|
|
609
776
|
def _fetch_redshift_column_attributes(self, column):
|
|
610
777
|
text = ""
|
|
611
|
-
if sa_version >= Version(
|
|
612
|
-
info = column.dialect_options[
|
|
778
|
+
if sa_version >= Version("1.3.0"):
|
|
779
|
+
info = column.dialect_options["redshift"]
|
|
613
780
|
else:
|
|
614
|
-
if not hasattr(column,
|
|
781
|
+
if not hasattr(column, "info"):
|
|
615
782
|
return text
|
|
616
783
|
info = column.info
|
|
617
784
|
|
|
618
|
-
identity = info.get(
|
|
785
|
+
identity = info.get("identity")
|
|
619
786
|
if identity:
|
|
620
787
|
text += " IDENTITY({0},{1})".format(identity[0], identity[1])
|
|
621
788
|
|
|
622
|
-
encode = info.get(
|
|
789
|
+
encode = info.get("encode")
|
|
623
790
|
if encode:
|
|
624
791
|
text += " ENCODE " + encode
|
|
625
792
|
|
|
626
|
-
distkey = info.get(
|
|
793
|
+
distkey = info.get("distkey")
|
|
627
794
|
if distkey:
|
|
628
795
|
text += " DISTKEY"
|
|
629
796
|
|
|
630
|
-
sortkey = info.get(
|
|
797
|
+
sortkey = info.get("sortkey")
|
|
631
798
|
if sortkey:
|
|
632
799
|
text += " SORTKEY"
|
|
633
800
|
return text
|
|
@@ -664,7 +831,7 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
664
831
|
:class:`~sqlalchemy.engine.Inspector`.
|
|
665
832
|
"""
|
|
666
833
|
|
|
667
|
-
name =
|
|
834
|
+
name = "redshift"
|
|
668
835
|
max_identifier_length = 127
|
|
669
836
|
|
|
670
837
|
statement_compiler = RedshiftCompiler
|
|
@@ -672,24 +839,26 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
672
839
|
preparer = RedshiftIdentifierPreparer
|
|
673
840
|
type_compiler = RedshiftTypeCompiler
|
|
674
841
|
construct_arguments = [
|
|
675
|
-
(sa.schema.Index, {
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
842
|
+
(sa.schema.Index, {"using": False, "where": None, "ops": {}}),
|
|
843
|
+
(
|
|
844
|
+
sa.schema.Table,
|
|
845
|
+
{
|
|
846
|
+
"ignore_search_path": False,
|
|
847
|
+
"diststyle": None,
|
|
848
|
+
"distkey": None,
|
|
849
|
+
"sortkey": None,
|
|
850
|
+
"interleaved_sortkey": None,
|
|
851
|
+
},
|
|
852
|
+
),
|
|
853
|
+
(
|
|
854
|
+
sa.schema.Column,
|
|
855
|
+
{
|
|
856
|
+
"encode": None,
|
|
857
|
+
"distkey": None,
|
|
858
|
+
"sortkey": None,
|
|
859
|
+
"identity": None,
|
|
860
|
+
},
|
|
861
|
+
),
|
|
693
862
|
]
|
|
694
863
|
|
|
695
864
|
def __init__(self, *args, **kw):
|
|
@@ -708,7 +877,7 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
708
877
|
"""
|
|
709
878
|
return {
|
|
710
879
|
**super(RedshiftDialectMixin, self).ischema_names,
|
|
711
|
-
**REDSHIFT_ISCHEMA_NAMES
|
|
880
|
+
**REDSHIFT_ISCHEMA_NAMES,
|
|
712
881
|
}
|
|
713
882
|
|
|
714
883
|
@reflection.cache
|
|
@@ -726,10 +895,16 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
726
895
|
columns = []
|
|
727
896
|
for col in cols:
|
|
728
897
|
column_info = self._get_column_info(
|
|
729
|
-
name=col.name,
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
898
|
+
name=col.name,
|
|
899
|
+
format_type=col.format_type,
|
|
900
|
+
default=col.default,
|
|
901
|
+
notnull=col.notnull,
|
|
902
|
+
domains=domains,
|
|
903
|
+
enums=[],
|
|
904
|
+
schema=col.schema,
|
|
905
|
+
encode=col.encode,
|
|
906
|
+
comment=col.comment,
|
|
907
|
+
)
|
|
733
908
|
columns.append(column_info)
|
|
734
909
|
return columns
|
|
735
910
|
|
|
@@ -738,11 +913,10 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
738
913
|
if not schema:
|
|
739
914
|
schema = inspect(connection).default_schema_name
|
|
740
915
|
|
|
741
|
-
info_cache = kw.get(
|
|
742
|
-
table = self._get_all_relation_info(
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
info_cache=info_cache)
|
|
916
|
+
info_cache = kw.get("info_cache")
|
|
917
|
+
table = self._get_all_relation_info(
|
|
918
|
+
connection, schema=schema, table_name=table_name, info_cache=info_cache
|
|
919
|
+
)
|
|
746
920
|
|
|
747
921
|
return True if table else False
|
|
748
922
|
|
|
@@ -751,9 +925,11 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
751
925
|
table_oid = self.get_table_oid(
|
|
752
926
|
connection, table_name, schema, info_cache=kw.get("info_cache")
|
|
753
927
|
)
|
|
754
|
-
table_oid =
|
|
928
|
+
table_oid = "NULL" if not table_oid else table_oid
|
|
755
929
|
|
|
756
|
-
result = connection.execute(
|
|
930
|
+
result = connection.execute(
|
|
931
|
+
sa.text(
|
|
932
|
+
"""
|
|
757
933
|
SELECT
|
|
758
934
|
cons.conname as name,
|
|
759
935
|
pg_get_constraintdef(cons.oid) as src
|
|
@@ -762,7 +938,11 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
762
938
|
WHERE
|
|
763
939
|
cons.conrelid = {} AND
|
|
764
940
|
cons.contype = 'c'
|
|
765
|
-
""".format(
|
|
941
|
+
""".format(
|
|
942
|
+
table_oid
|
|
943
|
+
)
|
|
944
|
+
)
|
|
945
|
+
)
|
|
766
946
|
ret = []
|
|
767
947
|
for name, src in result:
|
|
768
948
|
# samples:
|
|
@@ -772,16 +952,14 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
772
952
|
# "CHECK (some_boolean_function(a))"
|
|
773
953
|
# "CHECK (((a\n < 1)\n OR\n (a\n >= 5))\n)"
|
|
774
954
|
|
|
775
|
-
m = re.match(
|
|
776
|
-
r"^CHECK *\((.+)\)( NOT VALID)?$", src, flags=re.DOTALL
|
|
777
|
-
)
|
|
955
|
+
m = re.match(r"^CHECK *\((.+)\)( NOT VALID)?$", src, flags=re.DOTALL)
|
|
778
956
|
if not m:
|
|
779
957
|
logger.warning(f"Could not parse CHECK constraint text: {src}")
|
|
780
958
|
sqltext = ""
|
|
781
959
|
else:
|
|
782
|
-
sqltext = re.compile(
|
|
783
|
-
r"
|
|
784
|
-
)
|
|
960
|
+
sqltext = re.compile(r"^[\s\n]*\((.+)\)[\s\n]*$", flags=re.DOTALL).sub(
|
|
961
|
+
r"\1", m.group(1)
|
|
962
|
+
)
|
|
785
963
|
entry = {"name": name, "sqltext": sqltext}
|
|
786
964
|
if m and m.group(2):
|
|
787
965
|
entry["dialect_options"] = {"not_valid": True}
|
|
@@ -800,8 +978,7 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
800
978
|
"""
|
|
801
979
|
select '{schema_field}"{table_name}"'::regclass::oid;
|
|
802
980
|
""".format(
|
|
803
|
-
schema_field=schema_field,
|
|
804
|
-
table_name=table_name
|
|
981
|
+
schema_field=schema_field, table_name=table_name
|
|
805
982
|
)
|
|
806
983
|
)
|
|
807
984
|
)
|
|
@@ -816,18 +993,19 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
816
993
|
Overrides interface
|
|
817
994
|
:meth:`~sqlalchemy.engine.interfaces.Dialect.get_pk_constraint`.
|
|
818
995
|
"""
|
|
819
|
-
constraints = self._get_redshift_constraints(
|
|
820
|
-
|
|
821
|
-
|
|
996
|
+
constraints = self._get_redshift_constraints(
|
|
997
|
+
connection, table_name, schema, **kw
|
|
998
|
+
)
|
|
999
|
+
pk_constraints = [c for c in constraints if c.contype == "p"]
|
|
822
1000
|
if not pk_constraints:
|
|
823
|
-
return {
|
|
1001
|
+
return {"constrained_columns": [], "name": ""}
|
|
824
1002
|
pk_constraint = pk_constraints[0]
|
|
825
1003
|
m = PRIMARY_KEY_RE.match(pk_constraint.condef)
|
|
826
|
-
colstring = m.group(
|
|
1004
|
+
colstring = m.group("columns")
|
|
827
1005
|
constrained_columns = SQL_IDENTIFIER_RE.findall(colstring)
|
|
828
1006
|
return {
|
|
829
|
-
|
|
830
|
-
|
|
1007
|
+
"constrained_columns": constrained_columns,
|
|
1008
|
+
"name": pk_constraint.conname,
|
|
831
1009
|
}
|
|
832
1010
|
|
|
833
1011
|
@reflection.cache
|
|
@@ -838,28 +1016,29 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
838
1016
|
Overrides interface
|
|
839
1017
|
:meth:`~sqlalchemy.engine.interfaces.Dialect.get_pk_constraint`.
|
|
840
1018
|
"""
|
|
841
|
-
constraints = self._get_redshift_constraints(
|
|
842
|
-
|
|
843
|
-
|
|
1019
|
+
constraints = self._get_redshift_constraints(
|
|
1020
|
+
connection, table_name, schema, **kw
|
|
1021
|
+
)
|
|
1022
|
+
fk_constraints = [c for c in constraints if c.contype == "f"]
|
|
844
1023
|
uniques = defaultdict(lambda: defaultdict(dict))
|
|
845
1024
|
for con in fk_constraints:
|
|
846
1025
|
uniques[con.conname]["key"] = con.conkey
|
|
847
1026
|
uniques[con.conname]["condef"] = con.condef
|
|
848
1027
|
fkeys = []
|
|
849
1028
|
for conname, attrs in uniques.items():
|
|
850
|
-
m = FOREIGN_KEY_RE.match(attrs[
|
|
851
|
-
colstring = m.group(
|
|
1029
|
+
m = FOREIGN_KEY_RE.match(attrs["condef"])
|
|
1030
|
+
colstring = m.group("referred_columns")
|
|
852
1031
|
referred_columns = SQL_IDENTIFIER_RE.findall(colstring)
|
|
853
|
-
referred_table = m.group(
|
|
854
|
-
referred_schema = m.group(
|
|
855
|
-
colstring = m.group(
|
|
1032
|
+
referred_table = m.group("referred_table")
|
|
1033
|
+
referred_schema = m.group("referred_schema")
|
|
1034
|
+
colstring = m.group("columns")
|
|
856
1035
|
constrained_columns = SQL_IDENTIFIER_RE.findall(colstring)
|
|
857
1036
|
fkey_d = {
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
1037
|
+
"name": conname,
|
|
1038
|
+
"constrained_columns": constrained_columns,
|
|
1039
|
+
"referred_schema": referred_schema,
|
|
1040
|
+
"referred_table": referred_table,
|
|
1041
|
+
"referred_columns": referred_columns,
|
|
863
1042
|
}
|
|
864
1043
|
fkeys.append(fkey_d)
|
|
865
1044
|
return fkeys
|
|
@@ -872,7 +1051,7 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
872
1051
|
Overrides interface
|
|
873
1052
|
:meth:`~sqlalchemy.engine.interfaces.Dialect.get_table_names`.
|
|
874
1053
|
"""
|
|
875
|
-
return self._get_table_or_view_names(
|
|
1054
|
+
return self._get_table_or_view_names("r", connection, schema, **kw)
|
|
876
1055
|
|
|
877
1056
|
@reflection.cache
|
|
878
1057
|
def get_view_names(self, connection, schema=None, **kw):
|
|
@@ -882,7 +1061,7 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
882
1061
|
Overrides interface
|
|
883
1062
|
:meth:`~sqlalchemy.engine.interfaces.Dialect.get_view_names`.
|
|
884
1063
|
"""
|
|
885
|
-
return self._get_table_or_view_names(
|
|
1064
|
+
return self._get_table_or_view_names("v", connection, schema, **kw)
|
|
886
1065
|
|
|
887
1066
|
@reflection.cache
|
|
888
1067
|
def get_view_definition(self, connection, view_name, schema=None, **kw):
|
|
@@ -909,25 +1088,24 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
909
1088
|
return []
|
|
910
1089
|
|
|
911
1090
|
@reflection.cache
|
|
912
|
-
def get_unique_constraints(self, connection, table_name,
|
|
913
|
-
schema=None, **kw):
|
|
1091
|
+
def get_unique_constraints(self, connection, table_name, schema=None, **kw):
|
|
914
1092
|
"""
|
|
915
1093
|
Return information about unique constraints in `table_name`.
|
|
916
1094
|
|
|
917
1095
|
Overrides interface
|
|
918
1096
|
:meth:`~sqlalchemy.engine.interfaces.Dialect.get_unique_constraints`.
|
|
919
1097
|
"""
|
|
920
|
-
constraints = self._get_redshift_constraints(
|
|
921
|
-
|
|
922
|
-
|
|
1098
|
+
constraints = self._get_redshift_constraints(
|
|
1099
|
+
connection, table_name, schema, **kw
|
|
1100
|
+
)
|
|
1101
|
+
constraints = [c for c in constraints if c.contype == "u"]
|
|
923
1102
|
uniques = defaultdict(lambda: defaultdict(dict))
|
|
924
1103
|
for con in constraints:
|
|
925
1104
|
uniques[con.conname]["key"] = con.conkey
|
|
926
1105
|
uniques[con.conname]["cols"][con.attnum] = con.attname
|
|
927
1106
|
|
|
928
1107
|
return [
|
|
929
|
-
{
|
|
930
|
-
'column_names': [uc["cols"][i] for i in uc["key"]]}
|
|
1108
|
+
{"name": name, "column_names": [uc["cols"][i] for i in uc["key"]]}
|
|
931
1109
|
for name, uc in uniques.items()
|
|
932
1110
|
]
|
|
933
1111
|
|
|
@@ -940,17 +1118,16 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
940
1118
|
Overrides interface
|
|
941
1119
|
:meth:`~sqlalchemy.engine.Inspector.get_table_options`.
|
|
942
1120
|
"""
|
|
1121
|
+
|
|
943
1122
|
def keyfunc(column):
|
|
944
1123
|
num = int(column.sortkey)
|
|
945
1124
|
# If sortkey is interleaved, column numbers alternate
|
|
946
1125
|
# negative values, so take abs.
|
|
947
1126
|
return abs(num)
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
columns = self._get_redshift_columns(connection, table_name,
|
|
951
|
-
|
|
952
|
-
sortkey_cols = sorted([col for col in columns if col.sortkey],
|
|
953
|
-
key=keyfunc)
|
|
1127
|
+
|
|
1128
|
+
table = self._get_redshift_relation(connection, table_name, schema, **kw)
|
|
1129
|
+
columns = self._get_redshift_columns(connection, table_name, schema, **kw)
|
|
1130
|
+
sortkey_cols = sorted([col for col in columns if col.sortkey], key=keyfunc)
|
|
954
1131
|
interleaved = any([int(col.sortkey) < 0 for col in sortkey_cols])
|
|
955
1132
|
sortkey = tuple(col.name for col in sortkey_cols)
|
|
956
1133
|
interleaved_sortkey = None
|
|
@@ -960,20 +1137,20 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
960
1137
|
distkeys = [col.name for col in columns if col.distkey]
|
|
961
1138
|
distkey = distkeys[0] if distkeys else None
|
|
962
1139
|
return {
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
1140
|
+
"redshift_diststyle": table.diststyle,
|
|
1141
|
+
"redshift_distkey": distkey,
|
|
1142
|
+
"redshift_sortkey": sortkey,
|
|
1143
|
+
"redshift_interleaved_sortkey": interleaved_sortkey,
|
|
967
1144
|
}
|
|
968
1145
|
|
|
969
1146
|
def _get_table_or_view_names(self, relkind, connection, schema=None, **kw):
|
|
970
1147
|
default_schema = inspect(connection).default_schema_name
|
|
971
1148
|
if not schema:
|
|
972
1149
|
schema = default_schema
|
|
973
|
-
info_cache = kw.get(
|
|
974
|
-
all_relations = self._get_all_relation_info(
|
|
975
|
-
|
|
976
|
-
|
|
1150
|
+
info_cache = kw.get("info_cache")
|
|
1151
|
+
all_relations = self._get_all_relation_info(
|
|
1152
|
+
connection, schema=schema, info_cache=info_cache
|
|
1153
|
+
)
|
|
977
1154
|
relation_names = []
|
|
978
1155
|
for key, relation in all_relations.items():
|
|
979
1156
|
if key.schema == schema and relation.relkind == relkind:
|
|
@@ -982,37 +1159,32 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
982
1159
|
|
|
983
1160
|
def _get_column_info(self, *args, **kwargs):
|
|
984
1161
|
kw = kwargs.copy()
|
|
985
|
-
encode = kw.pop(
|
|
986
|
-
if sa_version >= Version(
|
|
1162
|
+
encode = kw.pop("encode", None)
|
|
1163
|
+
if sa_version >= Version("1.3.16"):
|
|
987
1164
|
# SQLAlchemy 1.3.16 introduced generated columns,
|
|
988
1165
|
# not supported in redshift
|
|
989
|
-
kw[
|
|
990
|
-
|
|
991
|
-
if sa_version < Version(
|
|
992
|
-
del kw[
|
|
993
|
-
elif sa_version >= Version(
|
|
994
|
-
kw[
|
|
995
|
-
|
|
996
|
-
column_info = super(RedshiftDialectMixin, self)._get_column_info(
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
if
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
column_info['info'] = {}
|
|
1005
|
-
if encode and encode != 'none':
|
|
1006
|
-
column_info['info']['encode'] = encode
|
|
1166
|
+
kw["generated"] = ""
|
|
1167
|
+
|
|
1168
|
+
if sa_version < Version("1.4.0") and "identity" in kw:
|
|
1169
|
+
del kw["identity"]
|
|
1170
|
+
elif sa_version >= Version("1.4.0") and "identity" not in kw:
|
|
1171
|
+
kw["identity"] = None
|
|
1172
|
+
|
|
1173
|
+
column_info = super(RedshiftDialectMixin, self)._get_column_info(*args, **kw)
|
|
1174
|
+
if isinstance(column_info["type"], VARCHAR):
|
|
1175
|
+
if column_info["type"].length is None:
|
|
1176
|
+
column_info["type"] = NullType()
|
|
1177
|
+
if "info" not in column_info:
|
|
1178
|
+
column_info["info"] = {}
|
|
1179
|
+
if encode and encode != "none":
|
|
1180
|
+
column_info["info"]["encode"] = encode
|
|
1007
1181
|
return column_info
|
|
1008
1182
|
|
|
1009
|
-
def _get_redshift_relation(self, connection, table_name,
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
table_name=table_name,
|
|
1015
|
-
info_cache=info_cache)
|
|
1183
|
+
def _get_redshift_relation(self, connection, table_name, schema=None, **kw):
|
|
1184
|
+
info_cache = kw.get("info_cache")
|
|
1185
|
+
all_relations = self._get_all_relation_info(
|
|
1186
|
+
connection, schema=schema, table_name=table_name, info_cache=info_cache
|
|
1187
|
+
)
|
|
1016
1188
|
key = RelationKey(table_name, schema, connection)
|
|
1017
1189
|
if key not in all_relations.keys():
|
|
1018
1190
|
key = key.unquoted()
|
|
@@ -1022,25 +1194,20 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
1022
1194
|
raise sa.exc.NoSuchTableError(key)
|
|
1023
1195
|
|
|
1024
1196
|
def _get_redshift_columns(self, connection, table_name, schema=None, **kw):
|
|
1025
|
-
info_cache = kw.get(
|
|
1197
|
+
info_cache = kw.get("info_cache")
|
|
1026
1198
|
all_schema_columns = self._get_schema_column_info(
|
|
1027
|
-
connection,
|
|
1028
|
-
schema=schema,
|
|
1029
|
-
table_name=table_name,
|
|
1030
|
-
info_cache=info_cache
|
|
1199
|
+
connection, schema=schema, table_name=table_name, info_cache=info_cache
|
|
1031
1200
|
)
|
|
1032
1201
|
key = RelationKey(table_name, schema, connection)
|
|
1033
1202
|
if key not in all_schema_columns.keys():
|
|
1034
1203
|
key = key.unquoted()
|
|
1035
1204
|
return all_schema_columns[key]
|
|
1036
1205
|
|
|
1037
|
-
def _get_redshift_constraints(self, connection, table_name,
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
table_name=table_name,
|
|
1043
|
-
info_cache=info_cache)
|
|
1206
|
+
def _get_redshift_constraints(self, connection, table_name, schema=None, **kw):
|
|
1207
|
+
info_cache = kw.get("info_cache")
|
|
1208
|
+
all_constraints = self._get_all_constraint_info(
|
|
1209
|
+
connection, schema=schema, table_name=table_name, info_cache=info_cache
|
|
1210
|
+
)
|
|
1044
1211
|
key = RelationKey(table_name, schema, connection)
|
|
1045
1212
|
if key not in all_constraints.keys():
|
|
1046
1213
|
key = key.unquoted()
|
|
@@ -1048,19 +1215,19 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
1048
1215
|
|
|
1049
1216
|
@reflection.cache
|
|
1050
1217
|
def _get_all_relation_info(self, connection, **kw):
|
|
1051
|
-
schema = kw.get(
|
|
1218
|
+
schema = kw.get("schema", None)
|
|
1052
1219
|
schema_clause = (
|
|
1053
1220
|
"AND schema = '{schema}'".format(schema=schema) if schema else ""
|
|
1054
1221
|
)
|
|
1055
1222
|
|
|
1056
|
-
table_name = kw.get(
|
|
1223
|
+
table_name = kw.get("table_name", None)
|
|
1057
1224
|
table_clause = (
|
|
1058
|
-
"AND relname = '{table}'".format(
|
|
1059
|
-
table=table_name
|
|
1060
|
-
) if table_name else ""
|
|
1225
|
+
"AND relname = '{table}'".format(table=table_name) if table_name else ""
|
|
1061
1226
|
)
|
|
1062
1227
|
|
|
1063
|
-
result = connection.execute(
|
|
1228
|
+
result = connection.execute(
|
|
1229
|
+
sa.text(
|
|
1230
|
+
"""
|
|
1064
1231
|
SELECT
|
|
1065
1232
|
c.relkind,
|
|
1066
1233
|
n.oid as "schema_oid",
|
|
@@ -1098,7 +1265,11 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
1098
1265
|
JOIN pg_catalog.pg_user u ON u.usesysid = s.esowner
|
|
1099
1266
|
where 1 {schema_clause} {table_clause}
|
|
1100
1267
|
ORDER BY "relkind", "schema_oid", "schema";
|
|
1101
|
-
""".format(
|
|
1268
|
+
""".format(
|
|
1269
|
+
schema_clause=schema_clause, table_clause=table_clause
|
|
1270
|
+
)
|
|
1271
|
+
)
|
|
1272
|
+
)
|
|
1102
1273
|
relations = {}
|
|
1103
1274
|
for rel in result:
|
|
1104
1275
|
key = RelationKey(rel.relname, rel.schema, connection)
|
|
@@ -1109,23 +1280,24 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
1109
1280
|
# when reflecting schema for multiple tables at once.
|
|
1110
1281
|
@reflection.cache
|
|
1111
1282
|
def _get_schema_column_info(self, connection, **kw):
|
|
1112
|
-
schema = kw.get(
|
|
1283
|
+
schema = kw.get("schema", None)
|
|
1113
1284
|
schema_clause = (
|
|
1114
1285
|
"AND schema = '{schema}'".format(schema=schema) if schema else ""
|
|
1115
1286
|
)
|
|
1116
1287
|
|
|
1117
|
-
table_name = kw.get(
|
|
1288
|
+
table_name = kw.get("table_name", None)
|
|
1118
1289
|
table_clause = (
|
|
1119
|
-
"AND table_name = '{table}'".format(
|
|
1120
|
-
table=table_name
|
|
1121
|
-
) if table_name else ""
|
|
1290
|
+
"AND table_name = '{table}'".format(table=table_name) if table_name else ""
|
|
1122
1291
|
)
|
|
1123
1292
|
|
|
1124
1293
|
all_columns = defaultdict(list)
|
|
1125
|
-
result = connection.execute(
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1294
|
+
result = connection.execute(
|
|
1295
|
+
sa.text(
|
|
1296
|
+
REFLECTION_SQL.format(
|
|
1297
|
+
schema_clause=schema_clause, table_clause=table_clause
|
|
1298
|
+
)
|
|
1299
|
+
)
|
|
1300
|
+
)
|
|
1129
1301
|
|
|
1130
1302
|
for col in result:
|
|
1131
1303
|
key = RelationKey(col.table_name, col.schema, connection)
|
|
@@ -1135,19 +1307,19 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
1135
1307
|
|
|
1136
1308
|
@reflection.cache
|
|
1137
1309
|
def _get_all_constraint_info(self, connection, **kw):
|
|
1138
|
-
schema = kw.get(
|
|
1310
|
+
schema = kw.get("schema", None)
|
|
1139
1311
|
schema_clause = (
|
|
1140
1312
|
"AND schema = '{schema}'".format(schema=schema) if schema else ""
|
|
1141
1313
|
)
|
|
1142
1314
|
|
|
1143
|
-
table_name = kw.get(
|
|
1315
|
+
table_name = kw.get("table_name", None)
|
|
1144
1316
|
table_clause = (
|
|
1145
|
-
"AND table_name = '{table}'".format(
|
|
1146
|
-
table=table_name
|
|
1147
|
-
) if table_name else ""
|
|
1317
|
+
"AND table_name = '{table}'".format(table=table_name) if table_name else ""
|
|
1148
1318
|
)
|
|
1149
1319
|
|
|
1150
|
-
result = connection.execute(
|
|
1320
|
+
result = connection.execute(
|
|
1321
|
+
sa.text(
|
|
1322
|
+
"""
|
|
1151
1323
|
SELECT
|
|
1152
1324
|
n.nspname as "schema",
|
|
1153
1325
|
c.relname as "table_name",
|
|
@@ -1184,7 +1356,11 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
1184
1356
|
JOIN svv_external_schemas s ON s.schemaname = c.schemaname
|
|
1185
1357
|
where 1 {schema_clause} {table_clause}
|
|
1186
1358
|
ORDER BY "schema", "table_name"
|
|
1187
|
-
""".format(
|
|
1359
|
+
""".format(
|
|
1360
|
+
schema_clause=schema_clause, table_clause=table_clause
|
|
1361
|
+
)
|
|
1362
|
+
)
|
|
1363
|
+
)
|
|
1188
1364
|
all_constraints = defaultdict(list)
|
|
1189
1365
|
for con in result:
|
|
1190
1366
|
key = RelationKey(con.table_name, con.schema, connection)
|
|
@@ -1195,65 +1371,9 @@ class RedshiftDialectMixin(DefaultDialect):
|
|
|
1195
1371
|
self._backslash_escapes = False
|
|
1196
1372
|
|
|
1197
1373
|
|
|
1198
|
-
class
|
|
1199
|
-
"""
|
|
1200
|
-
Define behavior specific to ``psycopg2``.
|
|
1201
|
-
|
|
1202
|
-
Most public methods are overrides of the underlying interfaces defined in
|
|
1203
|
-
:class:`~sqlalchemy.engine.interfaces.Dialect` and
|
|
1204
|
-
:class:`~sqlalchemy.engine.Inspector`.
|
|
1205
|
-
"""
|
|
1206
|
-
def create_connect_args(self, *args, **kwargs):
|
|
1207
|
-
"""
|
|
1208
|
-
Build DB-API compatible connection arguments.
|
|
1209
|
-
|
|
1210
|
-
Overrides interface
|
|
1211
|
-
:meth:`~sqlalchemy.engine.interfaces.Dialect.create_connect_args`.
|
|
1212
|
-
"""
|
|
1213
|
-
default_args = {
|
|
1214
|
-
'sslmode': 'verify-full',
|
|
1215
|
-
'sslrootcert': pkg_resources.resource_filename(
|
|
1216
|
-
__name__,
|
|
1217
|
-
'redshift-ca-bundle.crt'
|
|
1218
|
-
),
|
|
1219
|
-
}
|
|
1220
|
-
cargs, cparams = (
|
|
1221
|
-
super(Psycopg2RedshiftDialectMixin, self).create_connect_args(
|
|
1222
|
-
*args, **kwargs
|
|
1223
|
-
)
|
|
1224
|
-
)
|
|
1225
|
-
default_args.update(cparams)
|
|
1226
|
-
return cargs, default_args
|
|
1374
|
+
class RedshiftDialect(RedshiftDialectMixin, PGDialect):
|
|
1227
1375
|
|
|
1228
|
-
|
|
1229
|
-
def dbapi(cls):
|
|
1230
|
-
try:
|
|
1231
|
-
return importlib.import_module(cls.driver)
|
|
1232
|
-
except ImportError:
|
|
1233
|
-
raise ImportError(
|
|
1234
|
-
'No module named {}'.format(cls.driver)
|
|
1235
|
-
)
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
class RedshiftDialect_psycopg2(
|
|
1239
|
-
Psycopg2RedshiftDialectMixin, PGDialect_psycopg2
|
|
1240
|
-
):
|
|
1241
|
-
supports_statement_cache = False
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
# Add RedshiftDialect synonym for backwards compatibility.
|
|
1245
|
-
RedshiftDialect = RedshiftDialect_psycopg2
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
class RedshiftDialect_psycopg2cffi(
|
|
1249
|
-
Psycopg2RedshiftDialectMixin, PGDialect_psycopg2cffi
|
|
1250
|
-
):
|
|
1251
|
-
supports_statement_cache = False
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
class RedshiftDialect_redshift_connector(RedshiftDialectMixin, PGDialect):
|
|
1255
|
-
|
|
1256
|
-
class RedshiftCompiler_redshift_connector(RedshiftCompiler, PGCompiler):
|
|
1376
|
+
class RedshiftCompiler(RedshiftCompiler, PGCompiler):
|
|
1257
1377
|
def limit_clause(self, select, **kw):
|
|
1258
1378
|
text = ""
|
|
1259
1379
|
if select._limit_clause is not None:
|
|
@@ -1275,6 +1395,7 @@ class RedshiftDialect_redshift_connector(RedshiftDialectMixin, PGDialect):
|
|
|
1275
1395
|
|
|
1276
1396
|
def post_process_text(self, text):
|
|
1277
1397
|
from sqlalchemy import util
|
|
1398
|
+
|
|
1278
1399
|
if "%%" in text:
|
|
1279
1400
|
util.warn(
|
|
1280
1401
|
"The SQLAlchemy postgresql dialect "
|
|
@@ -1283,12 +1404,12 @@ class RedshiftDialect_redshift_connector(RedshiftDialectMixin, PGDialect):
|
|
|
1283
1404
|
)
|
|
1284
1405
|
return text.replace("%", "%%")
|
|
1285
1406
|
|
|
1286
|
-
class
|
|
1407
|
+
class RedshiftExecutionContext(PGExecutionContext):
|
|
1287
1408
|
def pre_exec(self):
|
|
1288
1409
|
if not self.compiled:
|
|
1289
1410
|
return
|
|
1290
1411
|
|
|
1291
|
-
driver =
|
|
1412
|
+
driver = "redshift_connector"
|
|
1292
1413
|
|
|
1293
1414
|
supports_unicode_statements = True
|
|
1294
1415
|
|
|
@@ -1296,16 +1417,14 @@ class RedshiftDialect_redshift_connector(RedshiftDialectMixin, PGDialect):
|
|
|
1296
1417
|
|
|
1297
1418
|
default_paramstyle = "format"
|
|
1298
1419
|
supports_sane_multi_rowcount = True
|
|
1299
|
-
statement_compiler =
|
|
1300
|
-
execution_ctx_cls =
|
|
1420
|
+
statement_compiler = RedshiftCompiler
|
|
1421
|
+
execution_ctx_cls = RedshiftExecutionContext
|
|
1301
1422
|
|
|
1302
1423
|
supports_statement_cache = False
|
|
1303
1424
|
use_setinputsizes = False # not implemented in redshift_connector
|
|
1304
1425
|
|
|
1305
1426
|
def __init__(self, client_encoding=None, **kwargs):
|
|
1306
|
-
super(
|
|
1307
|
-
RedshiftDialect_redshift_connector, self
|
|
1308
|
-
).__init__(client_encoding=client_encoding, **kwargs)
|
|
1427
|
+
super(RedshiftDialect, self).__init__(client_encoding=client_encoding, **kwargs)
|
|
1309
1428
|
self.client_encoding = client_encoding
|
|
1310
1429
|
|
|
1311
1430
|
@classmethod
|
|
@@ -1314,7 +1433,7 @@ class RedshiftDialect_redshift_connector(RedshiftDialectMixin, PGDialect):
|
|
|
1314
1433
|
driver_module = importlib.import_module(cls.driver)
|
|
1315
1434
|
|
|
1316
1435
|
# Starting v2.0.908 driver converts description column names to str
|
|
1317
|
-
if Version(driver_module.__version__) < Version(
|
|
1436
|
+
if Version(driver_module.__version__) < Version("2.0.908"):
|
|
1318
1437
|
cls.description_encoding = "use_encoding"
|
|
1319
1438
|
else:
|
|
1320
1439
|
cls.description_encoding = None
|
|
@@ -1322,8 +1441,8 @@ class RedshiftDialect_redshift_connector(RedshiftDialectMixin, PGDialect):
|
|
|
1322
1441
|
return driver_module
|
|
1323
1442
|
except ImportError:
|
|
1324
1443
|
raise ImportError(
|
|
1325
|
-
|
|
1326
|
-
|
|
1444
|
+
"No module named redshift_connector. Please install "
|
|
1445
|
+
"redshift_connector to use this sqlalchemy dialect."
|
|
1327
1446
|
)
|
|
1328
1447
|
|
|
1329
1448
|
def set_client_encoding(self, connection, client_encoding):
|
|
@@ -1360,9 +1479,7 @@ class RedshiftDialect_redshift_connector(RedshiftDialectMixin, PGDialect):
|
|
|
1360
1479
|
connection.autocommit = True
|
|
1361
1480
|
else:
|
|
1362
1481
|
connection.autocommit = False
|
|
1363
|
-
super(
|
|
1364
|
-
RedshiftDialect_redshift_connector, self
|
|
1365
|
-
).set_isolation_level(connection, level)
|
|
1482
|
+
super(RedshiftDialect, self).set_isolation_level(connection, level)
|
|
1366
1483
|
|
|
1367
1484
|
def on_connect(self):
|
|
1368
1485
|
fns = []
|
|
@@ -1370,6 +1487,7 @@ class RedshiftDialect_redshift_connector(RedshiftDialectMixin, PGDialect):
|
|
|
1370
1487
|
def on_connect(conn):
|
|
1371
1488
|
from sqlalchemy import util
|
|
1372
1489
|
from sqlalchemy.sql.elements import quoted_name
|
|
1490
|
+
|
|
1373
1491
|
conn.py_types[quoted_name] = conn.py_types[util.text_type]
|
|
1374
1492
|
|
|
1375
1493
|
fns.append(on_connect)
|
|
@@ -1406,25 +1524,23 @@ class RedshiftDialect_redshift_connector(RedshiftDialectMixin, PGDialect):
|
|
|
1406
1524
|
:meth:`~sqlalchemy.engine.interfaces.Dialect.create_connect_args`.
|
|
1407
1525
|
"""
|
|
1408
1526
|
default_args = {
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1527
|
+
"sslmode": "verify-full",
|
|
1528
|
+
"ssl": True,
|
|
1529
|
+
"application_name": "sqlalchemy-redshift",
|
|
1412
1530
|
}
|
|
1413
1531
|
cargs, cparams = super(RedshiftDialectMixin, self).create_connect_args(
|
|
1414
1532
|
*args, **kwargs
|
|
1415
1533
|
)
|
|
1416
1534
|
# set client_encoding so it is picked up by on_connect(), as
|
|
1417
1535
|
# redshift_connector does not have client_encoding connection parameter
|
|
1418
|
-
self.client_encoding = cparams.pop(
|
|
1419
|
-
'client_encoding', self.client_encoding
|
|
1420
|
-
)
|
|
1536
|
+
self.client_encoding = cparams.pop("client_encoding", self.client_encoding)
|
|
1421
1537
|
|
|
1422
|
-
if
|
|
1423
|
-
cparams[
|
|
1538
|
+
if "port" in cparams:
|
|
1539
|
+
cparams["port"] = int(cparams["port"])
|
|
1424
1540
|
|
|
1425
|
-
if
|
|
1426
|
-
cparams[
|
|
1427
|
-
del cparams[
|
|
1541
|
+
if "username" in cparams:
|
|
1542
|
+
cparams["user"] = cparams["username"]
|
|
1543
|
+
del cparams["username"]
|
|
1428
1544
|
|
|
1429
1545
|
default_args.update(cparams)
|
|
1430
1546
|
return cargs, default_args
|
|
@@ -1446,7 +1562,7 @@ def gen_columns_from_children(root):
|
|
|
1446
1562
|
yield root
|
|
1447
1563
|
|
|
1448
1564
|
|
|
1449
|
-
@compiles(Delete,
|
|
1565
|
+
@compiles(Delete, "redshift")
|
|
1450
1566
|
def visit_delete_stmt(element, compiler, **kwargs):
|
|
1451
1567
|
"""
|
|
1452
1568
|
Adds redshift-dialect specific compilation rule for the
|
|
@@ -1472,7 +1588,7 @@ def visit_delete_stmt(element, compiler, **kwargs):
|
|
|
1472
1588
|
problem illustration:
|
|
1473
1589
|
|
|
1474
1590
|
>>> from sqlalchemy import Table, Column, Integer, MetaData, delete
|
|
1475
|
-
>>> from sqlalchemy_redshift.dialect import
|
|
1591
|
+
>>> from sqlalchemy_redshift.dialect import RedshiftDialect
|
|
1476
1592
|
>>> meta = MetaData()
|
|
1477
1593
|
>>> table1 = Table(
|
|
1478
1594
|
... 'table_1',
|
|
@@ -1487,7 +1603,7 @@ def visit_delete_stmt(element, compiler, **kwargs):
|
|
|
1487
1603
|
... )
|
|
1488
1604
|
...
|
|
1489
1605
|
>>> del_stmt = delete(table1).where(table1.c.pk==table2.c.pk)
|
|
1490
|
-
>>> str(del_stmt.compile(dialect=
|
|
1606
|
+
>>> str(del_stmt.compile(dialect=RedshiftDialect()))
|
|
1491
1607
|
'DELETE FROM table_1 USING table_2 WHERE table_1.pk = table_2.pk'
|
|
1492
1608
|
>>> str(del_stmt)
|
|
1493
1609
|
'DELETE FROM table_1 , table_2 WHERE table_1.pk = table_2.pk'
|
|
@@ -1497,13 +1613,13 @@ def visit_delete_stmt(element, compiler, **kwargs):
|
|
|
1497
1613
|
>>> del_stmt3 = delete(table1).where(table1.c.pk > 1000)
|
|
1498
1614
|
>>> str(del_stmt3)
|
|
1499
1615
|
'DELETE FROM table_1 WHERE table_1.pk > :pk_1'
|
|
1500
|
-
>>> str(del_stmt3.compile(dialect=
|
|
1616
|
+
>>> str(del_stmt3.compile(dialect=RedshiftDialect()))
|
|
1501
1617
|
'DELETE FROM table_1 WHERE table_1.pk > %(pk_1)s'
|
|
1502
1618
|
"""
|
|
1503
1619
|
|
|
1504
1620
|
# Set empty strings for the default where clause and using clause
|
|
1505
|
-
whereclause =
|
|
1506
|
-
usingclause =
|
|
1621
|
+
whereclause = ""
|
|
1622
|
+
usingclause = ""
|
|
1507
1623
|
|
|
1508
1624
|
# determine if the delete query needs a ``USING`` injected
|
|
1509
1625
|
# by inspecting the whereclause's children & their children...
|
|
@@ -1514,15 +1630,15 @@ def visit_delete_stmt(element, compiler, **kwargs):
|
|
|
1514
1630
|
# which they first appear in the where clause.
|
|
1515
1631
|
delete_stmt_table = compiler.process(element.table, asfrom=True, **kwargs)
|
|
1516
1632
|
|
|
1517
|
-
if sa_version >= Version(
|
|
1633
|
+
if sa_version >= Version("1.4.0"):
|
|
1518
1634
|
if element.whereclause is not None:
|
|
1519
1635
|
clause = compiler.process(element.whereclause, **kwargs)
|
|
1520
1636
|
if clause:
|
|
1521
|
-
whereclause =
|
|
1637
|
+
whereclause = " WHERE {clause}".format(clause=clause)
|
|
1522
1638
|
else:
|
|
1523
1639
|
whereclause_tuple = element.get_children()
|
|
1524
1640
|
if whereclause_tuple:
|
|
1525
|
-
whereclause =
|
|
1641
|
+
whereclause = " WHERE {clause}".format(
|
|
1526
1642
|
clause=compiler.process(*whereclause_tuple, **kwargs)
|
|
1527
1643
|
)
|
|
1528
1644
|
|
|
@@ -1531,15 +1647,11 @@ def visit_delete_stmt(element, compiler, **kwargs):
|
|
|
1531
1647
|
whereclause_columns = gen_columns_from_children(element)
|
|
1532
1648
|
for col in whereclause_columns:
|
|
1533
1649
|
table = compiler.process(col.table, asfrom=True, **kwargs)
|
|
1534
|
-
if table != delete_stmt_table and
|
|
1535
|
-
table not in usingclause_tables:
|
|
1650
|
+
if table != delete_stmt_table and table not in usingclause_tables:
|
|
1536
1651
|
usingclause_tables.append(table)
|
|
1537
1652
|
if usingclause_tables:
|
|
1538
|
-
usingclause =
|
|
1539
|
-
clause=', '.join(usingclause_tables)
|
|
1540
|
-
)
|
|
1653
|
+
usingclause = " USING {clause}".format(clause=", ".join(usingclause_tables))
|
|
1541
1654
|
|
|
1542
|
-
return
|
|
1543
|
-
table=delete_stmt_table,
|
|
1544
|
-
|
|
1545
|
-
where=whereclause)
|
|
1655
|
+
return "DELETE FROM {table}{using}{where}".format(
|
|
1656
|
+
table=delete_stmt_table, using=usingclause, where=whereclause
|
|
1657
|
+
)
|