mtsql 1.11.18__py3-none-any.whl → 1.11.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mt/sql/redshift/__init__.py +2 -18
- mt/sql/redshift/commands.py +348 -261
- mt/sql/redshift/ddl.py +29 -27
- mt/sql/redshift/dialect.py +457 -345
- mt/sql/version.py +1 -1
- {mtsql-1.11.18.dist-info → mtsql-1.11.20.dist-info}/METADATA +2 -1
- mtsql-1.11.20.dist-info/RECORD +17 -0
- mtsql-1.11.18.dist-info/RECORD +0 -17
- {mtsql-1.11.18.dist-info → mtsql-1.11.20.dist-info}/LICENSE +0 -0
- {mtsql-1.11.18.dist-info → mtsql-1.11.20.dist-info}/WHEEL +0 -0
- {mtsql-1.11.18.dist-info → mtsql-1.11.20.dist-info}/top_level.txt +0 -0
mt/sql/redshift/commands.py
CHANGED
|
@@ -2,6 +2,7 @@ import enum
|
|
|
2
2
|
import numbers
|
|
3
3
|
import re
|
|
4
4
|
import warnings
|
|
5
|
+
|
|
5
6
|
try:
|
|
6
7
|
from collections.abc import Iterable
|
|
7
8
|
except ImportError:
|
|
@@ -23,49 +24,55 @@ from sqlalchemy.sql import expression as sa_expression
|
|
|
23
24
|
# The pattern of IAM role ARNs can be found here:
|
|
24
25
|
# http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-iam
|
|
25
26
|
|
|
26
|
-
ACCESS_KEY_ID_RE = re.compile(
|
|
27
|
-
SECRET_ACCESS_KEY_RE = re.compile(
|
|
28
|
-
TOKEN_RE = re.compile(
|
|
29
|
-
AWS_PARTITIONS = frozenset({
|
|
30
|
-
AWS_ACCOUNT_ID_RE = re.compile(
|
|
31
|
-
IAM_ROLE_NAME_RE = re.compile(
|
|
32
|
-
IAM_ROLE_ARN_RE = re.compile(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
27
|
+
ACCESS_KEY_ID_RE = re.compile("[A-Z0-9]{20}")
|
|
28
|
+
SECRET_ACCESS_KEY_RE = re.compile("[A-Za-z0-9/+=]{40}")
|
|
29
|
+
TOKEN_RE = re.compile("[A-Za-z0-9/+=]+")
|
|
30
|
+
AWS_PARTITIONS = frozenset({"aws", "aws-cn", "aws-us-gov"})
|
|
31
|
+
AWS_ACCOUNT_ID_RE = re.compile("[0-9]{12}")
|
|
32
|
+
IAM_ROLE_NAME_RE = re.compile("[A-Za-z0-9+=,.@\-_]{1,64}") # noqa
|
|
33
|
+
IAM_ROLE_ARN_RE = re.compile(
|
|
34
|
+
"arn:(aws|aws-cn|aws-us-gov):iam::" "[0-9]{12}:role/[A-Za-z0-9+=,.@\-_]{1,64}"
|
|
35
|
+
) # noqa
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _process_aws_credentials(
|
|
39
|
+
access_key_id=None,
|
|
40
|
+
secret_access_key=None,
|
|
41
|
+
session_token=None,
|
|
42
|
+
aws_partition="aws",
|
|
43
|
+
aws_account_id=None,
|
|
44
|
+
iam_role_name=None,
|
|
45
|
+
iam_role_arns=None,
|
|
46
|
+
):
|
|
40
47
|
uses_iam_role = aws_account_id is not None and iam_role_name is not None
|
|
41
48
|
uses_iam_roles = iam_role_arns is not None
|
|
42
49
|
uses_key = access_key_id is not None and secret_access_key is not None
|
|
43
50
|
|
|
44
51
|
if uses_iam_role + uses_iam_roles + uses_key > 1:
|
|
45
52
|
raise TypeError(
|
|
46
|
-
|
|
47
|
-
|
|
53
|
+
"Either access key based credentials or role based credentials "
|
|
54
|
+
"should be specified, but not both"
|
|
48
55
|
)
|
|
49
56
|
|
|
50
57
|
credentials = None
|
|
51
58
|
|
|
52
59
|
if aws_account_id is not None and iam_role_name is not None:
|
|
53
60
|
if aws_partition not in AWS_PARTITIONS:
|
|
54
|
-
raise ValueError(
|
|
61
|
+
raise ValueError("invalid AWS partition")
|
|
55
62
|
if not AWS_ACCOUNT_ID_RE.match(aws_account_id):
|
|
56
63
|
raise ValueError(
|
|
57
|
-
|
|
64
|
+
"invalid AWS account ID; does not match {pattern}".format(
|
|
58
65
|
pattern=AWS_ACCOUNT_ID_RE.pattern,
|
|
59
66
|
)
|
|
60
67
|
)
|
|
61
68
|
elif not IAM_ROLE_NAME_RE.match(iam_role_name):
|
|
62
69
|
raise ValueError(
|
|
63
|
-
|
|
70
|
+
"invalid IAM role name; does not match {pattern}".format(
|
|
64
71
|
pattern=IAM_ROLE_NAME_RE.pattern,
|
|
65
72
|
)
|
|
66
73
|
)
|
|
67
74
|
|
|
68
|
-
credentials =
|
|
75
|
+
credentials = "aws_iam_role=arn:{0}:iam::{1}:role/{2}".format(
|
|
69
76
|
aws_partition,
|
|
70
77
|
aws_account_id,
|
|
71
78
|
iam_role_name,
|
|
@@ -75,32 +82,32 @@ def _process_aws_credentials(access_key_id=None, secret_access_key=None,
|
|
|
75
82
|
if isinstance(iam_role_arns, str):
|
|
76
83
|
iam_role_arns = [iam_role_arns]
|
|
77
84
|
if not isinstance(iam_role_arns, list):
|
|
78
|
-
raise ValueError(
|
|
85
|
+
raise ValueError("iam_role_arns must be a list")
|
|
79
86
|
for arn in iam_role_arns:
|
|
80
87
|
if not IAM_ROLE_ARN_RE.match(arn):
|
|
81
88
|
raise ValueError(
|
|
82
|
-
|
|
89
|
+
"invalid AWS account ID; does not match {pattern}".format(
|
|
83
90
|
pattern=IAM_ROLE_ARN_RE.pattern,
|
|
84
91
|
)
|
|
85
92
|
)
|
|
86
93
|
|
|
87
|
-
credentials =
|
|
94
|
+
credentials = "aws_iam_role=" + ",".join(iam_role_arns)
|
|
88
95
|
|
|
89
96
|
if access_key_id is not None and secret_access_key is not None:
|
|
90
97
|
if not ACCESS_KEY_ID_RE.match(access_key_id):
|
|
91
98
|
raise ValueError(
|
|
92
|
-
|
|
99
|
+
"invalid access_key_id; does not match {pattern}".format(
|
|
93
100
|
pattern=ACCESS_KEY_ID_RE.pattern,
|
|
94
101
|
)
|
|
95
102
|
)
|
|
96
103
|
if not SECRET_ACCESS_KEY_RE.match(secret_access_key):
|
|
97
104
|
raise ValueError(
|
|
98
|
-
|
|
105
|
+
"invalid secret_access_key; does not match {pattern}".format(
|
|
99
106
|
pattern=SECRET_ACCESS_KEY_RE.pattern,
|
|
100
107
|
)
|
|
101
108
|
)
|
|
102
109
|
|
|
103
|
-
credentials =
|
|
110
|
+
credentials = "aws_access_key_id={0};aws_secret_access_key={1}".format(
|
|
104
111
|
access_key_id,
|
|
105
112
|
secret_access_key,
|
|
106
113
|
)
|
|
@@ -108,27 +115,26 @@ def _process_aws_credentials(access_key_id=None, secret_access_key=None,
|
|
|
108
115
|
if session_token is not None:
|
|
109
116
|
if not TOKEN_RE.match(session_token):
|
|
110
117
|
raise ValueError(
|
|
111
|
-
|
|
118
|
+
"invalid session_token; does not match {pattern}".format(
|
|
112
119
|
pattern=TOKEN_RE.pattern,
|
|
113
120
|
)
|
|
114
121
|
)
|
|
115
|
-
credentials +=
|
|
122
|
+
credentials += ";token={0}".format(session_token)
|
|
116
123
|
|
|
117
124
|
if credentials is None:
|
|
118
125
|
raise TypeError(
|
|
119
|
-
|
|
120
|
-
|
|
126
|
+
"Either access key based credentials or role based credentials "
|
|
127
|
+
"should be specified"
|
|
121
128
|
)
|
|
122
129
|
|
|
123
130
|
return credentials
|
|
124
131
|
|
|
125
132
|
|
|
126
133
|
def _process_fixed_width(spec):
|
|
127
|
-
return
|
|
134
|
+
return ",".join(("{0}:{1:d}".format(col, width) for col, width in spec))
|
|
128
135
|
|
|
129
136
|
|
|
130
|
-
class _ExecutableClause(sa_expression.Executable,
|
|
131
|
-
sa_expression.ClauseElement):
|
|
137
|
+
class _ExecutableClause(sa_expression.Executable, sa_expression.ClauseElement):
|
|
132
138
|
pass
|
|
133
139
|
|
|
134
140
|
|
|
@@ -159,10 +165,10 @@ class AlterTableAppendCommand(_ExecutableClause):
|
|
|
159
165
|
fill those columns with the default column value or NULL. Mutually
|
|
160
166
|
exclusive with `ignore_extra`.
|
|
161
167
|
"""
|
|
168
|
+
|
|
162
169
|
def __init__(self, source, target, ignore_extra=False, fill_target=False):
|
|
163
170
|
if ignore_extra and fill_target:
|
|
164
|
-
raise ValueError(
|
|
165
|
-
'"ignore_extra" cannot be used with "fill_target".')
|
|
171
|
+
raise ValueError('"ignore_extra" cannot be used with "fill_target".')
|
|
166
172
|
|
|
167
173
|
self.source = source
|
|
168
174
|
self.target = target
|
|
@@ -176,18 +182,17 @@ def visit_alter_table_append_command(element, compiler, **kw):
|
|
|
176
182
|
Returns the actual SQL query for the AlterTableAppendCommand class.
|
|
177
183
|
"""
|
|
178
184
|
if element.ignore_extra:
|
|
179
|
-
fill_option =
|
|
185
|
+
fill_option = "IGNOREEXTRA"
|
|
180
186
|
elif element.fill_target:
|
|
181
|
-
fill_option =
|
|
187
|
+
fill_option = "FILLTARGET"
|
|
182
188
|
else:
|
|
183
|
-
fill_option =
|
|
189
|
+
fill_option = ""
|
|
184
190
|
|
|
185
|
-
query_text =
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
)
|
|
191
|
+
query_text = "ALTER TABLE {target} APPEND FROM {source} {fill_option}".format(
|
|
192
|
+
target=compiler.preparer.format_table(element.target),
|
|
193
|
+
source=compiler.preparer.format_table(element.source),
|
|
194
|
+
fill_option=fill_option,
|
|
195
|
+
)
|
|
191
196
|
return compiler.process(sa.text(query_text), **kw)
|
|
192
197
|
|
|
193
198
|
|
|
@@ -269,24 +274,38 @@ class UnloadFromSelect(_ExecutableClause):
|
|
|
269
274
|
Indicates the type of file to unload to.
|
|
270
275
|
"""
|
|
271
276
|
|
|
272
|
-
def __init__(
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
277
|
+
def __init__(
|
|
278
|
+
self,
|
|
279
|
+
select,
|
|
280
|
+
unload_location,
|
|
281
|
+
access_key_id=None,
|
|
282
|
+
secret_access_key=None,
|
|
283
|
+
session_token=None,
|
|
284
|
+
aws_partition="aws",
|
|
285
|
+
aws_account_id=None,
|
|
286
|
+
iam_role_name=None,
|
|
287
|
+
manifest=False,
|
|
288
|
+
delimiter=None,
|
|
289
|
+
fixed_width=None,
|
|
290
|
+
encrypted=False,
|
|
291
|
+
gzip=False,
|
|
292
|
+
add_quotes=False,
|
|
293
|
+
null=None,
|
|
294
|
+
escape=False,
|
|
295
|
+
allow_overwrite=False,
|
|
296
|
+
parallel=True,
|
|
297
|
+
header=False,
|
|
298
|
+
region=None,
|
|
299
|
+
max_file_size=None,
|
|
300
|
+
format=None,
|
|
301
|
+
iam_role_arns=None,
|
|
302
|
+
):
|
|
280
303
|
|
|
281
304
|
if delimiter is not None and len(delimiter) != 1:
|
|
282
|
-
raise ValueError(
|
|
283
|
-
'"delimiter" parameter must be a single character'
|
|
284
|
-
)
|
|
305
|
+
raise ValueError('"delimiter" parameter must be a single character')
|
|
285
306
|
|
|
286
307
|
if header and fixed_width is not None:
|
|
287
|
-
raise ValueError(
|
|
288
|
-
"'header' cannot be used with 'fixed_width'"
|
|
289
|
-
)
|
|
308
|
+
raise ValueError("'header' cannot be used with 'fixed_width'")
|
|
290
309
|
|
|
291
310
|
credentials = _process_aws_credentials(
|
|
292
311
|
access_key_id=access_key_id,
|
|
@@ -342,87 +361,96 @@ def visit_unload_from_select(element, compiler, **kw):
|
|
|
342
361
|
el = element
|
|
343
362
|
|
|
344
363
|
if el.format is None:
|
|
345
|
-
format_ =
|
|
364
|
+
format_ = ""
|
|
346
365
|
elif el.format == Format.csv:
|
|
347
|
-
format_ =
|
|
366
|
+
format_ = "FORMAT AS {}".format(el.format.value)
|
|
348
367
|
if el.delimiter is not None or el.fixed_width is not None:
|
|
349
|
-
raise ValueError(
|
|
350
|
-
'CSV format cannot be used with delimiter or fixed_width')
|
|
368
|
+
raise ValueError("CSV format cannot be used with delimiter or fixed_width")
|
|
351
369
|
elif el.format == Format.parquet:
|
|
352
|
-
format_ =
|
|
353
|
-
if any(
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
370
|
+
format_ = "FORMAT AS {}".format(el.format.value)
|
|
371
|
+
if any(
|
|
372
|
+
(
|
|
373
|
+
el.delimiter,
|
|
374
|
+
el.fixed_width,
|
|
375
|
+
el.add_quotes,
|
|
376
|
+
el.escape,
|
|
377
|
+
el.null,
|
|
378
|
+
el.header,
|
|
379
|
+
el.gzip,
|
|
380
|
+
)
|
|
381
|
+
):
|
|
357
382
|
raise ValueError(
|
|
358
383
|
"Parquet format can't be used with `delimiter`, `fixed_width`,"
|
|
359
|
-
|
|
384
|
+
" `add_quotes`, `escape`, `null`, `header`, or `gzip`."
|
|
360
385
|
)
|
|
361
386
|
else:
|
|
362
|
-
raise ValueError(
|
|
363
|
-
'Only CSV and Parquet formats are currently supported.'
|
|
364
|
-
)
|
|
387
|
+
raise ValueError("Only CSV and Parquet formats are currently supported.")
|
|
365
388
|
|
|
366
389
|
qs = template.format(
|
|
367
|
-
manifest=
|
|
368
|
-
header=
|
|
390
|
+
manifest="MANIFEST" if el.manifest else "",
|
|
391
|
+
header="HEADER" if el.header else "",
|
|
369
392
|
format=format_,
|
|
370
|
-
delimiter=(
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
parallel='PARALLEL OFF' if not el.parallel else '',
|
|
381
|
-
region='REGION :region' if el.region is not None else '',
|
|
393
|
+
delimiter=("DELIMITER AS :delimiter" if el.delimiter is not None else ""),
|
|
394
|
+
encrypted="ENCRYPTED" if el.encrypted else "",
|
|
395
|
+
fixed_width="FIXEDWIDTH AS :fixed_width" if el.fixed_width else "",
|
|
396
|
+
gzip="GZIP" if el.gzip else "",
|
|
397
|
+
add_quotes="ADDQUOTES" if el.add_quotes else "",
|
|
398
|
+
escape="ESCAPE" if el.escape else "",
|
|
399
|
+
null="NULL AS :null_as" if el.null is not None else "",
|
|
400
|
+
allow_overwrite="ALLOWOVERWRITE" if el.allow_overwrite else "",
|
|
401
|
+
parallel="PARALLEL OFF" if not el.parallel else "",
|
|
402
|
+
region="REGION :region" if el.region is not None else "",
|
|
382
403
|
max_file_size=(
|
|
383
|
-
|
|
384
|
-
if el.max_file_size is not None else ''
|
|
404
|
+
"MAXFILESIZE :max_file_size MB" if el.max_file_size is not None else ""
|
|
385
405
|
),
|
|
386
406
|
)
|
|
387
407
|
|
|
388
408
|
query = sa.text(qs)
|
|
389
409
|
|
|
390
410
|
if el.delimiter is not None:
|
|
391
|
-
query = query.bindparams(
|
|
392
|
-
|
|
393
|
-
|
|
411
|
+
query = query.bindparams(
|
|
412
|
+
sa.bindparam(
|
|
413
|
+
"delimiter",
|
|
414
|
+
value=element.delimiter,
|
|
415
|
+
type_=sa.String,
|
|
416
|
+
)
|
|
417
|
+
)
|
|
394
418
|
|
|
395
419
|
if el.fixed_width:
|
|
396
|
-
query = query.bindparams(
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
420
|
+
query = query.bindparams(
|
|
421
|
+
sa.bindparam(
|
|
422
|
+
"fixed_width",
|
|
423
|
+
value=_process_fixed_width(el.fixed_width),
|
|
424
|
+
type_=sa.String,
|
|
425
|
+
)
|
|
426
|
+
)
|
|
401
427
|
|
|
402
428
|
if el.null is not None:
|
|
403
|
-
query = query.bindparams(
|
|
404
|
-
|
|
405
|
-
)
|
|
429
|
+
query = query.bindparams(
|
|
430
|
+
sa.bindparam("null_as", value=el.null, type_=sa.String)
|
|
431
|
+
)
|
|
406
432
|
|
|
407
433
|
if el.region is not None:
|
|
408
|
-
query = query.bindparams(
|
|
409
|
-
|
|
410
|
-
)
|
|
434
|
+
query = query.bindparams(
|
|
435
|
+
sa.bindparam("region", value=el.region, type_=sa.String)
|
|
436
|
+
)
|
|
411
437
|
|
|
412
438
|
if el.max_file_size is not None:
|
|
413
439
|
max_file_size_mib = float(el.max_file_size) / 1024 / 1024
|
|
414
|
-
query = query.bindparams(
|
|
415
|
-
|
|
416
|
-
)
|
|
440
|
+
query = query.bindparams(
|
|
441
|
+
sa.bindparam("max_file_size", value=max_file_size_mib, type_=sa.Float)
|
|
442
|
+
)
|
|
417
443
|
|
|
418
444
|
return compiler.process(
|
|
419
445
|
query.bindparams(
|
|
420
|
-
sa.bindparam(
|
|
446
|
+
sa.bindparam("credentials", value=el.credentials, type_=sa.String),
|
|
421
447
|
sa.bindparam(
|
|
422
|
-
|
|
448
|
+
"unload_location",
|
|
449
|
+
value=el.unload_location,
|
|
450
|
+
type_=sa.String,
|
|
423
451
|
),
|
|
424
452
|
sa.bindparam(
|
|
425
|
-
|
|
453
|
+
"select",
|
|
426
454
|
value=compiler.process(
|
|
427
455
|
el.select,
|
|
428
456
|
literal_binds=True,
|
|
@@ -435,25 +463,25 @@ def visit_unload_from_select(element, compiler, **kw):
|
|
|
435
463
|
|
|
436
464
|
|
|
437
465
|
class Format(enum.Enum):
|
|
438
|
-
csv =
|
|
439
|
-
json =
|
|
440
|
-
avro =
|
|
441
|
-
orc =
|
|
442
|
-
parquet =
|
|
443
|
-
fixed_width =
|
|
466
|
+
csv = "CSV"
|
|
467
|
+
json = "JSON"
|
|
468
|
+
avro = "AVRO"
|
|
469
|
+
orc = "ORC"
|
|
470
|
+
parquet = "PARQUET"
|
|
471
|
+
fixed_width = "FIXEDWIDTH"
|
|
444
472
|
|
|
445
473
|
|
|
446
474
|
class Compression(enum.Enum):
|
|
447
|
-
gzip =
|
|
448
|
-
lzop =
|
|
449
|
-
bzip2 =
|
|
475
|
+
gzip = "GZIP"
|
|
476
|
+
lzop = "LZOP"
|
|
477
|
+
bzip2 = "BZIP2"
|
|
450
478
|
|
|
451
479
|
|
|
452
480
|
class Encoding(enum.Enum):
|
|
453
|
-
utf8 =
|
|
454
|
-
utf16 =
|
|
455
|
-
utf16le =
|
|
456
|
-
utf16be =
|
|
481
|
+
utf8 = "UTF8"
|
|
482
|
+
utf16 = "UTF16"
|
|
483
|
+
utf16le = "UTF16LE"
|
|
484
|
+
utf16be = "UTF16BE"
|
|
457
485
|
|
|
458
486
|
|
|
459
487
|
def _check_enum(Enum, val):
|
|
@@ -462,7 +490,7 @@ def _check_enum(Enum, val):
|
|
|
462
490
|
|
|
463
491
|
cleaned = Enum(val)
|
|
464
492
|
if cleaned is not val:
|
|
465
|
-
tpl =
|
|
493
|
+
tpl = "{val!r} should be, {cleaned!r}, an instance of {Enum!r}"
|
|
466
494
|
msg = tpl.format(val=val, cleaned=cleaned, Enum=Enum)
|
|
467
495
|
warnings.warn(msg, DeprecationWarning)
|
|
468
496
|
|
|
@@ -610,21 +638,48 @@ class CopyCommand(_ExecutableClause):
|
|
|
610
638
|
cluster isn't in the same region as the S3 bucket.
|
|
611
639
|
"""
|
|
612
640
|
|
|
613
|
-
def __init__(
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
641
|
+
def __init__(
|
|
642
|
+
self,
|
|
643
|
+
to,
|
|
644
|
+
data_location,
|
|
645
|
+
access_key_id=None,
|
|
646
|
+
secret_access_key=None,
|
|
647
|
+
session_token=None,
|
|
648
|
+
aws_partition="aws",
|
|
649
|
+
aws_account_id=None,
|
|
650
|
+
iam_role_name=None,
|
|
651
|
+
format=None,
|
|
652
|
+
quote=None,
|
|
653
|
+
path_file="auto",
|
|
654
|
+
delimiter=None,
|
|
655
|
+
fixed_width=None,
|
|
656
|
+
compression=None,
|
|
657
|
+
accept_any_date=False,
|
|
658
|
+
accept_inv_chars=None,
|
|
659
|
+
blanks_as_null=False,
|
|
660
|
+
date_format=None,
|
|
661
|
+
empty_as_null=False,
|
|
662
|
+
encoding=None,
|
|
663
|
+
escape=False,
|
|
664
|
+
explicit_ids=False,
|
|
665
|
+
fill_record=False,
|
|
666
|
+
ignore_blank_lines=False,
|
|
667
|
+
ignore_header=None,
|
|
668
|
+
dangerous_null_delimiter=None,
|
|
669
|
+
remove_quotes=False,
|
|
670
|
+
roundec=False,
|
|
671
|
+
time_format=None,
|
|
672
|
+
trim_blanks=False,
|
|
673
|
+
truncate_columns=False,
|
|
674
|
+
comp_rows=None,
|
|
675
|
+
comp_update=None,
|
|
676
|
+
max_error=None,
|
|
677
|
+
no_load=False,
|
|
678
|
+
stat_update=None,
|
|
679
|
+
manifest=False,
|
|
680
|
+
region=None,
|
|
681
|
+
iam_role_arns=None,
|
|
682
|
+
):
|
|
628
683
|
|
|
629
684
|
credentials = _process_aws_credentials(
|
|
630
685
|
access_key_id=access_key_id,
|
|
@@ -637,14 +692,11 @@ class CopyCommand(_ExecutableClause):
|
|
|
637
692
|
)
|
|
638
693
|
|
|
639
694
|
if delimiter is not None and len(delimiter) != 1:
|
|
640
|
-
raise ValueError('"delimiter" parameter must be a single '
|
|
641
|
-
'character')
|
|
695
|
+
raise ValueError('"delimiter" parameter must be a single ' "character")
|
|
642
696
|
|
|
643
697
|
if ignore_header is not None:
|
|
644
698
|
if not isinstance(ignore_header, numbers.Integral):
|
|
645
|
-
raise TypeError(
|
|
646
|
-
'"ignore_header" parameter should be an integer'
|
|
647
|
-
)
|
|
699
|
+
raise TypeError('"ignore_header" parameter should be an integer')
|
|
648
700
|
|
|
649
701
|
table = None
|
|
650
702
|
columns = []
|
|
@@ -652,10 +704,8 @@ class CopyCommand(_ExecutableClause):
|
|
|
652
704
|
for column in to:
|
|
653
705
|
if table is not None and table != column.table:
|
|
654
706
|
raise ValueError(
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
column, column.table, table
|
|
658
|
-
),
|
|
707
|
+
"All columns must come from the same table: "
|
|
708
|
+
"%s comes from %s not %s" % (column, column.table, table),
|
|
659
709
|
)
|
|
660
710
|
columns.append(column)
|
|
661
711
|
table = column.table
|
|
@@ -710,189 +760,211 @@ def visit_copy_command(element, compiler, **kw):
|
|
|
710
760
|
parameters = []
|
|
711
761
|
bindparams = [
|
|
712
762
|
sa.bindparam(
|
|
713
|
-
|
|
763
|
+
"data_location",
|
|
714
764
|
value=element.data_location,
|
|
715
765
|
type_=sa.String,
|
|
716
766
|
),
|
|
717
767
|
sa.bindparam(
|
|
718
|
-
|
|
768
|
+
"credentials",
|
|
719
769
|
value=element.credentials,
|
|
720
770
|
type_=sa.String,
|
|
721
771
|
),
|
|
722
772
|
]
|
|
723
773
|
|
|
724
774
|
if element.format == Format.csv:
|
|
725
|
-
format_ =
|
|
775
|
+
format_ = "FORMAT AS CSV"
|
|
726
776
|
if element.quote is not None:
|
|
727
|
-
format_ +=
|
|
728
|
-
bindparams.append(
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
777
|
+
format_ += " QUOTE AS :quote_character"
|
|
778
|
+
bindparams.append(
|
|
779
|
+
sa.bindparam(
|
|
780
|
+
"quote_character",
|
|
781
|
+
value=element.quote,
|
|
782
|
+
type_=sa.String,
|
|
783
|
+
)
|
|
784
|
+
)
|
|
733
785
|
elif element.format == Format.json:
|
|
734
|
-
format_ =
|
|
735
|
-
bindparams.append(
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
786
|
+
format_ = "FORMAT AS JSON AS :json_option"
|
|
787
|
+
bindparams.append(
|
|
788
|
+
sa.bindparam(
|
|
789
|
+
"json_option",
|
|
790
|
+
value=element.path_file,
|
|
791
|
+
type_=sa.String,
|
|
792
|
+
)
|
|
793
|
+
)
|
|
740
794
|
elif element.format == Format.avro:
|
|
741
|
-
format_ =
|
|
742
|
-
bindparams.append(
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
795
|
+
format_ = "FORMAT AS AVRO AS :avro_option"
|
|
796
|
+
bindparams.append(
|
|
797
|
+
sa.bindparam(
|
|
798
|
+
"avro_option",
|
|
799
|
+
value=element.path_file,
|
|
800
|
+
type_=sa.String,
|
|
801
|
+
)
|
|
802
|
+
)
|
|
747
803
|
elif element.format == Format.orc:
|
|
748
|
-
format_ =
|
|
804
|
+
format_ = "FORMAT AS ORC"
|
|
749
805
|
elif element.format == Format.parquet:
|
|
750
|
-
format_ =
|
|
806
|
+
format_ = "FORMAT AS PARQUET"
|
|
751
807
|
elif element.format == Format.fixed_width and element.fixed_width is None:
|
|
752
808
|
raise sa_exc.CompileError(
|
|
753
|
-
"'fixed_width' argument required for format 'FIXEDWIDTH'."
|
|
809
|
+
"'fixed_width' argument required for format 'FIXEDWIDTH'."
|
|
810
|
+
)
|
|
754
811
|
else:
|
|
755
|
-
format_ =
|
|
812
|
+
format_ = ""
|
|
756
813
|
|
|
757
814
|
if element.delimiter is not None:
|
|
758
|
-
parameters.append(
|
|
759
|
-
bindparams.append(
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
815
|
+
parameters.append("DELIMITER AS :delimiter_char")
|
|
816
|
+
bindparams.append(
|
|
817
|
+
sa.bindparam(
|
|
818
|
+
"delimiter_char",
|
|
819
|
+
value=element.delimiter,
|
|
820
|
+
type_=sa.String,
|
|
821
|
+
)
|
|
822
|
+
)
|
|
764
823
|
|
|
765
824
|
if element.fixed_width is not None:
|
|
766
|
-
parameters.append(
|
|
767
|
-
bindparams.append(
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
825
|
+
parameters.append("FIXEDWIDTH AS :fixedwidth_spec")
|
|
826
|
+
bindparams.append(
|
|
827
|
+
sa.bindparam(
|
|
828
|
+
"fixedwidth_spec",
|
|
829
|
+
value=_process_fixed_width(element.fixed_width),
|
|
830
|
+
type_=sa.String,
|
|
831
|
+
)
|
|
832
|
+
)
|
|
772
833
|
|
|
773
834
|
if element.compression is not None:
|
|
774
835
|
parameters.append(Compression(element.compression).value)
|
|
775
836
|
|
|
776
837
|
if element.manifest:
|
|
777
|
-
parameters.append(
|
|
838
|
+
parameters.append("MANIFEST")
|
|
778
839
|
|
|
779
840
|
if element.accept_any_date:
|
|
780
|
-
parameters.append(
|
|
841
|
+
parameters.append("ACCEPTANYDATE")
|
|
781
842
|
|
|
782
843
|
if element.accept_inv_chars is not None:
|
|
783
|
-
parameters.append(
|
|
784
|
-
bindparams.append(
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
)
|
|
844
|
+
parameters.append("ACCEPTINVCHARS AS :replacement_char")
|
|
845
|
+
bindparams.append(
|
|
846
|
+
sa.bindparam(
|
|
847
|
+
"replacement_char", value=element.accept_inv_chars, type_=sa.String
|
|
848
|
+
)
|
|
849
|
+
)
|
|
789
850
|
|
|
790
851
|
if element.blanks_as_null:
|
|
791
|
-
parameters.append(
|
|
852
|
+
parameters.append("BLANKSASNULL")
|
|
792
853
|
|
|
793
854
|
if element.date_format is not None:
|
|
794
|
-
parameters.append(
|
|
795
|
-
bindparams.append(
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
855
|
+
parameters.append("DATEFORMAT AS :dateformat_string")
|
|
856
|
+
bindparams.append(
|
|
857
|
+
sa.bindparam(
|
|
858
|
+
"dateformat_string",
|
|
859
|
+
value=element.date_format,
|
|
860
|
+
type_=sa.String,
|
|
861
|
+
)
|
|
862
|
+
)
|
|
800
863
|
|
|
801
864
|
if element.empty_as_null:
|
|
802
|
-
parameters.append(
|
|
865
|
+
parameters.append("EMPTYASNULL")
|
|
803
866
|
|
|
804
867
|
if element.encoding is not None:
|
|
805
|
-
parameters.append(
|
|
868
|
+
parameters.append("ENCODING AS " + Encoding(element.encoding).value)
|
|
806
869
|
|
|
807
870
|
if element.escape:
|
|
808
|
-
parameters.append(
|
|
871
|
+
parameters.append("ESCAPE")
|
|
809
872
|
|
|
810
873
|
if element.explicit_ids:
|
|
811
|
-
parameters.append(
|
|
874
|
+
parameters.append("EXPLICIT_IDS")
|
|
812
875
|
|
|
813
876
|
if element.fill_record:
|
|
814
|
-
parameters.append(
|
|
877
|
+
parameters.append("FILLRECORD")
|
|
815
878
|
|
|
816
879
|
if element.ignore_blank_lines:
|
|
817
|
-
parameters.append(
|
|
880
|
+
parameters.append("IGNOREBLANKLINES")
|
|
818
881
|
|
|
819
882
|
if element.ignore_header is not None:
|
|
820
|
-
parameters.append(
|
|
821
|
-
bindparams.append(
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
883
|
+
parameters.append("IGNOREHEADER AS :number_rows")
|
|
884
|
+
bindparams.append(
|
|
885
|
+
sa.bindparam(
|
|
886
|
+
"number_rows",
|
|
887
|
+
value=element.ignore_header,
|
|
888
|
+
type_=sa.Integer,
|
|
889
|
+
)
|
|
890
|
+
)
|
|
826
891
|
|
|
827
892
|
if element.dangerous_null_delimiter is not None:
|
|
828
893
|
parameters.append("NULL AS '%s'" % element.dangerous_null_delimiter)
|
|
829
894
|
|
|
830
895
|
if element.remove_quotes:
|
|
831
|
-
parameters.append(
|
|
896
|
+
parameters.append("REMOVEQUOTES")
|
|
832
897
|
|
|
833
898
|
if element.roundec:
|
|
834
|
-
parameters.append(
|
|
899
|
+
parameters.append("ROUNDEC")
|
|
835
900
|
|
|
836
901
|
if element.time_format is not None:
|
|
837
|
-
parameters.append(
|
|
838
|
-
bindparams.append(
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
902
|
+
parameters.append("TIMEFORMAT AS :timeformat_string")
|
|
903
|
+
bindparams.append(
|
|
904
|
+
sa.bindparam(
|
|
905
|
+
"timeformat_string",
|
|
906
|
+
value=element.time_format,
|
|
907
|
+
type_=sa.String,
|
|
908
|
+
)
|
|
909
|
+
)
|
|
843
910
|
|
|
844
911
|
if element.trim_blanks:
|
|
845
|
-
parameters.append(
|
|
912
|
+
parameters.append("TRIMBLANKS")
|
|
846
913
|
|
|
847
914
|
if element.truncate_columns:
|
|
848
|
-
parameters.append(
|
|
915
|
+
parameters.append("TRUNCATECOLUMNS")
|
|
849
916
|
|
|
850
917
|
if element.comp_rows:
|
|
851
|
-
parameters.append(
|
|
852
|
-
bindparams.append(
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
918
|
+
parameters.append("COMPROWS :numrows")
|
|
919
|
+
bindparams.append(
|
|
920
|
+
sa.bindparam(
|
|
921
|
+
"numrows",
|
|
922
|
+
value=element.comp_rows,
|
|
923
|
+
type_=sa.Integer,
|
|
924
|
+
)
|
|
925
|
+
)
|
|
857
926
|
|
|
858
927
|
if element.comp_update:
|
|
859
|
-
parameters.append(
|
|
928
|
+
parameters.append("COMPUPDATE ON")
|
|
860
929
|
elif element.comp_update is not None:
|
|
861
|
-
parameters.append(
|
|
930
|
+
parameters.append("COMPUPDATE OFF")
|
|
862
931
|
|
|
863
932
|
if element.max_error is not None:
|
|
864
|
-
parameters.append(
|
|
865
|
-
bindparams.append(
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
933
|
+
parameters.append("MAXERROR AS :error_count")
|
|
934
|
+
bindparams.append(
|
|
935
|
+
sa.bindparam(
|
|
936
|
+
"error_count",
|
|
937
|
+
value=element.max_error,
|
|
938
|
+
type_=sa.Integer,
|
|
939
|
+
)
|
|
940
|
+
)
|
|
870
941
|
|
|
871
942
|
if element.no_load:
|
|
872
|
-
parameters.append(
|
|
943
|
+
parameters.append("NOLOAD")
|
|
873
944
|
|
|
874
945
|
if element.stat_update:
|
|
875
|
-
parameters.append(
|
|
946
|
+
parameters.append("STATUPDATE ON")
|
|
876
947
|
elif element.stat_update is not None:
|
|
877
|
-
parameters.append(
|
|
948
|
+
parameters.append("STATUPDATE OFF")
|
|
878
949
|
|
|
879
950
|
if element.region is not None:
|
|
880
|
-
parameters.append(
|
|
881
|
-
bindparams.append(sa.bindparam(
|
|
882
|
-
'region',
|
|
883
|
-
value=element.region,
|
|
884
|
-
type_=sa.String
|
|
885
|
-
))
|
|
951
|
+
parameters.append("REGION :region")
|
|
952
|
+
bindparams.append(sa.bindparam("region", value=element.region, type_=sa.String))
|
|
886
953
|
|
|
887
|
-
columns =
|
|
888
|
-
|
|
889
|
-
|
|
954
|
+
columns = (
|
|
955
|
+
" (%s)"
|
|
956
|
+
% ", ".join(
|
|
957
|
+
compiler.preparer.format_column(column) for column in element.columns
|
|
958
|
+
)
|
|
959
|
+
if element.columns
|
|
960
|
+
else ""
|
|
961
|
+
)
|
|
890
962
|
|
|
891
963
|
qs = qs.format(
|
|
892
964
|
table=compiler.preparer.format_table(element.table),
|
|
893
965
|
columns=columns,
|
|
894
966
|
format=format_,
|
|
895
|
-
parameters=
|
|
967
|
+
parameters="\n".join(parameters),
|
|
896
968
|
)
|
|
897
969
|
|
|
898
970
|
return compiler.process(sa.text(qs).bindparams(*bindparams), **kw)
|
|
@@ -941,10 +1013,20 @@ class CreateLibraryCommand(_ExecutableClause):
|
|
|
941
1013
|
The AWS region where the library's S3 bucket is located, if the
|
|
942
1014
|
Redshift cluster isn't in the same region as the S3 bucket.
|
|
943
1015
|
"""
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
1016
|
+
|
|
1017
|
+
def __init__(
|
|
1018
|
+
self,
|
|
1019
|
+
library_name,
|
|
1020
|
+
location,
|
|
1021
|
+
access_key_id=None,
|
|
1022
|
+
secret_access_key=None,
|
|
1023
|
+
session_token=None,
|
|
1024
|
+
aws_account_id=None,
|
|
1025
|
+
iam_role_name=None,
|
|
1026
|
+
replace=False,
|
|
1027
|
+
region=None,
|
|
1028
|
+
iam_role_arns=None,
|
|
1029
|
+
):
|
|
948
1030
|
self.library_name = library_name
|
|
949
1031
|
self.location = location
|
|
950
1032
|
self.credentials = _process_aws_credentials(
|
|
@@ -973,28 +1055,32 @@ def visit_create_library_command(element, compiler, **kw):
|
|
|
973
1055
|
"""
|
|
974
1056
|
bindparams = [
|
|
975
1057
|
sa.bindparam(
|
|
976
|
-
|
|
1058
|
+
"location",
|
|
977
1059
|
value=element.location,
|
|
978
1060
|
type_=sa.String,
|
|
979
1061
|
),
|
|
980
1062
|
sa.bindparam(
|
|
981
|
-
|
|
1063
|
+
"credentials",
|
|
982
1064
|
value=element.credentials,
|
|
983
1065
|
type_=sa.String,
|
|
984
1066
|
),
|
|
985
1067
|
]
|
|
986
1068
|
|
|
987
1069
|
if element.region is not None:
|
|
988
|
-
bindparams.append(
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1070
|
+
bindparams.append(
|
|
1071
|
+
sa.bindparam(
|
|
1072
|
+
"region",
|
|
1073
|
+
value=element.region,
|
|
1074
|
+
type_=sa.String,
|
|
1075
|
+
)
|
|
1076
|
+
)
|
|
993
1077
|
|
|
994
1078
|
quoted_lib_name = compiler.preparer.quote_identifier(element.library_name)
|
|
995
|
-
query = query.format(
|
|
996
|
-
|
|
997
|
-
|
|
1079
|
+
query = query.format(
|
|
1080
|
+
name=quoted_lib_name,
|
|
1081
|
+
or_replace="OR REPLACE" if element.replace else "",
|
|
1082
|
+
region="REGION :region" if element.region else "",
|
|
1083
|
+
)
|
|
998
1084
|
return compiler.process(sa.text(query).bindparams(*bindparams), **kw)
|
|
999
1085
|
|
|
1000
1086
|
|
|
@@ -1009,7 +1095,7 @@ class RefreshMaterializedView(_ExecutableClause):
|
|
|
1009
1095
|
|
|
1010
1096
|
>>> import sqlalchemy as sa
|
|
1011
1097
|
>>> from sqlalchemy_redshift.dialect import RefreshMaterializedView
|
|
1012
|
-
>>> engine = sa.create_engine('
|
|
1098
|
+
>>> engine = sa.create_engine('mtsql_redshift://example')
|
|
1013
1099
|
>>> refresh = RefreshMaterializedView('materialized_view_of_users')
|
|
1014
1100
|
>>> print(refresh.compile(engine))
|
|
1015
1101
|
<BLANKLINE>
|
|
@@ -1019,6 +1105,7 @@ class RefreshMaterializedView(_ExecutableClause):
|
|
|
1019
1105
|
|
|
1020
1106
|
This can be included in any execute() statement.
|
|
1021
1107
|
"""
|
|
1108
|
+
|
|
1022
1109
|
def __init__(self, name):
|
|
1023
1110
|
"""
|
|
1024
1111
|
Builds the Executable/ClauseElement that represents the refresh command
|