mtsql 1.6.202312111459__py3-none-any.whl → 1.8.202401021406__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mt/sql/base.py +52 -54
- mt/sql/psql.py +5 -53
- mt/sql/redshift.py +528 -0
- mt/sql/version.py +7 -7
- {mtsql-1.6.202312111459.dist-info → mtsql-1.8.202401021406.dist-info}/METADATA +1 -1
- mtsql-1.8.202401021406.dist-info/RECORD +12 -0
- mtsql-1.6.202312111459.dist-info/RECORD +0 -11
- {mtsql-1.6.202312111459.dist-info → mtsql-1.8.202401021406.dist-info}/LICENSE +0 -0
- {mtsql-1.6.202312111459.dist-info → mtsql-1.8.202401021406.dist-info}/WHEEL +0 -0
- {mtsql-1.6.202312111459.dist-info → mtsql-1.8.202401021406.dist-info}/top_level.txt +0 -0
mt/sql/base.py
CHANGED
|
@@ -11,6 +11,7 @@ from mt.base import deprecated_func
|
|
|
11
11
|
|
|
12
12
|
__all__ = [
|
|
13
13
|
"frame_sql",
|
|
14
|
+
"indices",
|
|
14
15
|
"run_func",
|
|
15
16
|
"conn_ctx",
|
|
16
17
|
"engine_execute",
|
|
@@ -19,6 +20,8 @@ __all__ = [
|
|
|
19
20
|
"exec_sql",
|
|
20
21
|
"list_schemas",
|
|
21
22
|
"list_tables",
|
|
23
|
+
"list_views",
|
|
24
|
+
"table_exists",
|
|
22
25
|
]
|
|
23
26
|
|
|
24
27
|
|
|
@@ -26,6 +29,12 @@ def frame_sql(frame_name, schema: tp.Optional[str] = None):
|
|
|
26
29
|
return frame_name if schema is None else "{}.{}".format(schema, frame_name)
|
|
27
30
|
|
|
28
31
|
|
|
32
|
+
def indices(df):
|
|
33
|
+
"""Returns the list of named indices of the dataframe, ignoring any unnamed index."""
|
|
34
|
+
a = list(df.index.names)
|
|
35
|
+
return a if a != [None] else []
|
|
36
|
+
|
|
37
|
+
|
|
29
38
|
# ----- functions dealing with sql queries to overcome OperationalError -----
|
|
30
39
|
|
|
31
40
|
|
|
@@ -200,60 +209,6 @@ def read_sql(
|
|
|
200
209
|
return df
|
|
201
210
|
|
|
202
211
|
|
|
203
|
-
@deprecated_func(
|
|
204
|
-
"1.0",
|
|
205
|
-
suggested_func="mt.sql.base.read_sql",
|
|
206
|
-
removed_version="2.0",
|
|
207
|
-
docstring_prefix=" ",
|
|
208
|
-
)
|
|
209
|
-
def read_sql_query(
|
|
210
|
-
sql,
|
|
211
|
-
engine,
|
|
212
|
-
index_col=None,
|
|
213
|
-
set_index_after=False,
|
|
214
|
-
nb_trials: int = 3,
|
|
215
|
-
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
216
|
-
**kwargs
|
|
217
|
-
):
|
|
218
|
-
"""Read an SQL query with a number of trials to overcome OperationalError.
|
|
219
|
-
|
|
220
|
-
Parameters
|
|
221
|
-
----------
|
|
222
|
-
sql : str
|
|
223
|
-
SQL query to be executed
|
|
224
|
-
engine : sqlalchemy.engine.Engine
|
|
225
|
-
connection engine to the server
|
|
226
|
-
index_col: string or list of strings, optional, default: None
|
|
227
|
-
Column(s) to set as index(MultiIndex). See :func:`pandas.read_sql_query`.
|
|
228
|
-
set_index_after: bool
|
|
229
|
-
whether to set index specified by index_col via the pandas.read_sql_query() function or
|
|
230
|
-
after the function has been invoked
|
|
231
|
-
nb_trials: int
|
|
232
|
-
number of query trials
|
|
233
|
-
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
234
|
-
logger for debugging
|
|
235
|
-
kwargs: dict
|
|
236
|
-
other keyword arguments to be passed directly to :func:`pandas.read_sql_query`
|
|
237
|
-
|
|
238
|
-
See Also
|
|
239
|
-
--------
|
|
240
|
-
pandas.read_sql_query
|
|
241
|
-
"""
|
|
242
|
-
|
|
243
|
-
df = read_sql(
|
|
244
|
-
sql,
|
|
245
|
-
engine,
|
|
246
|
-
index_col=index_col,
|
|
247
|
-
nb_trials=nb_trials,
|
|
248
|
-
exception_handling="raise",
|
|
249
|
-
logger=logger,
|
|
250
|
-
**kwargs
|
|
251
|
-
)
|
|
252
|
-
if index_col is None or not set_index_after:
|
|
253
|
-
return df
|
|
254
|
-
return df.set_index(index_col, drop=True)
|
|
255
|
-
|
|
256
|
-
|
|
257
212
|
def read_sql_table(
|
|
258
213
|
table_name,
|
|
259
214
|
engine,
|
|
@@ -358,3 +313,46 @@ def list_tables(engine, schema: tp.Optional[str] = None):
|
|
|
358
313
|
list of all table names
|
|
359
314
|
"""
|
|
360
315
|
return sa.inspect(engine).get_table_names(schema=schema)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def list_views(engine, schema: tp.Optional[str] = None):
|
|
319
|
+
"""Lists all views of a given schema.
|
|
320
|
+
|
|
321
|
+
Parameters
|
|
322
|
+
----------
|
|
323
|
+
engine : sqlalchemy.engine.Engine
|
|
324
|
+
connection engine to the server
|
|
325
|
+
schema: str, optional
|
|
326
|
+
a valid schema name returned from :func:`list_schemas`. Default to sqlalchemy
|
|
327
|
+
|
|
328
|
+
Returns
|
|
329
|
+
-------
|
|
330
|
+
list
|
|
331
|
+
list of all view names
|
|
332
|
+
"""
|
|
333
|
+
return sa.inspect(engine).get_view_names(schema=schema)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def table_exists(
|
|
337
|
+
table_name,
|
|
338
|
+
engine,
|
|
339
|
+
schema: tp.Optional[str] = None,
|
|
340
|
+
):
|
|
341
|
+
"""Checks if a table exists.
|
|
342
|
+
|
|
343
|
+
Parameters
|
|
344
|
+
----------
|
|
345
|
+
table_name: str
|
|
346
|
+
name of table
|
|
347
|
+
engine: sqlalchemy.engine.Engine
|
|
348
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
349
|
+
schema: str or None
|
|
350
|
+
a valid schema name returned from `list_schemas()`
|
|
351
|
+
|
|
352
|
+
Returns
|
|
353
|
+
-------
|
|
354
|
+
retval: bool
|
|
355
|
+
whether a table or a view exists with the given name
|
|
356
|
+
"""
|
|
357
|
+
|
|
358
|
+
return sa.inspect(engine).has_table(table_name, schema=schema)
|
mt/sql/psql.py
CHANGED
|
@@ -16,12 +16,10 @@ __all__ = [
|
|
|
16
16
|
"pg_get_locked_transactions",
|
|
17
17
|
"pg_cancel_backend",
|
|
18
18
|
"pg_cancel_all_backends",
|
|
19
|
-
"indices",
|
|
20
19
|
"compliance_check",
|
|
21
20
|
"as_column_name",
|
|
22
21
|
"to_sql",
|
|
23
22
|
"rename_schema",
|
|
24
|
-
"list_views",
|
|
25
23
|
"list_matviews",
|
|
26
24
|
"list_foreign_tables",
|
|
27
25
|
"list_frames",
|
|
@@ -137,12 +135,6 @@ def pg_cancel_all_backends(
|
|
|
137
135
|
# ----- functions dealing with sql queries to overcome OperationalError -----
|
|
138
136
|
|
|
139
137
|
|
|
140
|
-
def indices(df):
|
|
141
|
-
"""Returns the list of named indices of the dataframe, ignoring any unnamed index."""
|
|
142
|
-
a = list(df.index.names)
|
|
143
|
-
return a if a != [None] else []
|
|
144
|
-
|
|
145
|
-
|
|
146
138
|
def compliance_check(df: pd.DataFrame):
|
|
147
139
|
"""Checks if a dataframe is compliant to PSQL.
|
|
148
140
|
|
|
@@ -207,7 +199,7 @@ def to_sql(
|
|
|
207
199
|
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
208
200
|
**kwargs,
|
|
209
201
|
):
|
|
210
|
-
"""Writes records stored in a DataFrame to
|
|
202
|
+
"""Writes records stored in a DataFrame to a PostgreSQL database.
|
|
211
203
|
|
|
212
204
|
With a number of trials to overcome OperationalError.
|
|
213
205
|
|
|
@@ -391,42 +383,6 @@ def rename_schema(
|
|
|
391
383
|
)
|
|
392
384
|
|
|
393
385
|
|
|
394
|
-
def list_views(
|
|
395
|
-
engine,
|
|
396
|
-
schema: tp.Optional[str] = None,
|
|
397
|
-
nb_trials: int = 3,
|
|
398
|
-
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
399
|
-
):
|
|
400
|
-
"""Lists all views of a given schema.
|
|
401
|
-
|
|
402
|
-
Parameters
|
|
403
|
-
----------
|
|
404
|
-
engine: sqlalchemy.engine.Engine
|
|
405
|
-
an sqlalchemy connection engine created by function `create_engine()`
|
|
406
|
-
schema: str or None
|
|
407
|
-
a valid schema name returned from `list_schemas()`
|
|
408
|
-
nb_trials: int
|
|
409
|
-
number of query trials
|
|
410
|
-
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
411
|
-
logger for debugging
|
|
412
|
-
|
|
413
|
-
Returns
|
|
414
|
-
-------
|
|
415
|
-
out: list
|
|
416
|
-
list of all view names
|
|
417
|
-
"""
|
|
418
|
-
if schema is None:
|
|
419
|
-
query_str = "select distinct viewname from pg_views;"
|
|
420
|
-
else:
|
|
421
|
-
query_str = (
|
|
422
|
-
"select distinct viewname from pg_views where schemaname='{}';".format(
|
|
423
|
-
schema
|
|
424
|
-
)
|
|
425
|
-
)
|
|
426
|
-
df = read_sql(query_str, engine, nb_trials=nb_trials, logger=logger)
|
|
427
|
-
return df["viewname"].tolist()
|
|
428
|
-
|
|
429
|
-
|
|
430
386
|
def list_matviews(
|
|
431
387
|
engine,
|
|
432
388
|
schema: tp.Optional[str] = None,
|
|
@@ -521,7 +477,7 @@ def list_frames(
|
|
|
521
477
|
data = []
|
|
522
478
|
for item in list_tables(engine, schema=schema):
|
|
523
479
|
data.append((item, "table"))
|
|
524
|
-
for item in list_views(engine, schema=schema
|
|
480
|
+
for item in list_views(engine, schema=schema):
|
|
525
481
|
data.append((item, "view"))
|
|
526
482
|
for item in list_matviews(
|
|
527
483
|
engine, schema=schema, nb_trials=nb_trials, logger=logger
|
|
@@ -990,11 +946,9 @@ def frame_exists(
|
|
|
990
946
|
retval: bool
|
|
991
947
|
whether a table or a view exists with the given name
|
|
992
948
|
"""
|
|
993
|
-
if frame_name
|
|
949
|
+
if table_exists(frame_name, engine, schema=schema):
|
|
994
950
|
return True
|
|
995
|
-
if frame_name in list_views(
|
|
996
|
-
engine, schema=schema, nb_trials=nb_trials, logger=logger
|
|
997
|
-
):
|
|
951
|
+
if frame_name in list_views(engine, schema=schema):
|
|
998
952
|
return True
|
|
999
953
|
return frame_name in list_matviews(
|
|
1000
954
|
engine, schema=schema, nb_trials=nb_trials, logger=logger
|
|
@@ -1041,9 +995,7 @@ def drop_frame(
|
|
|
1041
995
|
nb_trials=nb_trials,
|
|
1042
996
|
logger=logger,
|
|
1043
997
|
)
|
|
1044
|
-
if frame_name in list_views(
|
|
1045
|
-
engine, schema=schema, nb_trials=nb_trials, logger=logger
|
|
1046
|
-
):
|
|
998
|
+
if frame_name in list_views(engine, schema=schema):
|
|
1047
999
|
return drop_view(
|
|
1048
1000
|
frame_name,
|
|
1049
1001
|
engine,
|
mt/sql/redshift.py
ADDED
|
@@ -0,0 +1,528 @@
|
|
|
1
|
+
"""Useful modules for accessing Redshift"""
|
|
2
|
+
|
|
3
|
+
from mt import tp, logg
|
|
4
|
+
|
|
5
|
+
from .base import *
|
|
6
|
+
from .psql import compliance_check
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
__api__ = [
|
|
10
|
+
"rename_schema",
|
|
11
|
+
"get_frame_length",
|
|
12
|
+
"rename_table",
|
|
13
|
+
"vacuum_table",
|
|
14
|
+
"drop_table",
|
|
15
|
+
"rename_view",
|
|
16
|
+
"drop_view",
|
|
17
|
+
"rename_matview",
|
|
18
|
+
"refresh_matview",
|
|
19
|
+
"drop_matview",
|
|
20
|
+
"rename_column",
|
|
21
|
+
"drop_column",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# ----- simple functions -----
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def rename_schema(
|
|
29
|
+
old_schema,
|
|
30
|
+
new_schema,
|
|
31
|
+
engine,
|
|
32
|
+
nb_trials: int = 3,
|
|
33
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
34
|
+
):
|
|
35
|
+
"""Renames a schema.
|
|
36
|
+
|
|
37
|
+
Parameters
|
|
38
|
+
----------
|
|
39
|
+
old_schema: str
|
|
40
|
+
old schema name
|
|
41
|
+
new_schema: str
|
|
42
|
+
new schema name
|
|
43
|
+
engine: sqlalchemy.engine.Engine
|
|
44
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
45
|
+
nb_trials: int
|
|
46
|
+
number of query trials
|
|
47
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
48
|
+
logger for debugging
|
|
49
|
+
"""
|
|
50
|
+
exec_sql(
|
|
51
|
+
'ALTER SCHEMA "{}" RENAME TO "{}";'.format(old_schema, new_schema),
|
|
52
|
+
engine,
|
|
53
|
+
nb_trials=nb_trials,
|
|
54
|
+
logger=logger,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def get_frame_length(
|
|
59
|
+
frame_name,
|
|
60
|
+
engine,
|
|
61
|
+
schema: tp.Optional[str] = None,
|
|
62
|
+
nb_trials: int = 3,
|
|
63
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
64
|
+
):
|
|
65
|
+
"""Gets the number of rows of a dataframes (tables/views/materialized views).
|
|
66
|
+
|
|
67
|
+
Parameters
|
|
68
|
+
----------
|
|
69
|
+
frame_name: str
|
|
70
|
+
name of the dataframe
|
|
71
|
+
engine: sqlalchemy.engine.Engine
|
|
72
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
73
|
+
nb_trials: int
|
|
74
|
+
number of query trials
|
|
75
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
76
|
+
logger for debugging
|
|
77
|
+
|
|
78
|
+
Returns
|
|
79
|
+
-------
|
|
80
|
+
out: int
|
|
81
|
+
number of rows
|
|
82
|
+
|
|
83
|
+
Notes
|
|
84
|
+
-----
|
|
85
|
+
The dataframe must exist.
|
|
86
|
+
"""
|
|
87
|
+
frame_sql_str = frame_sql(frame_name, schema=schema)
|
|
88
|
+
return read_sql(
|
|
89
|
+
"SELECT COUNT(*) a FROM {};".format(frame_sql_str),
|
|
90
|
+
engine,
|
|
91
|
+
nb_trials=nb_trials,
|
|
92
|
+
logger=logger,
|
|
93
|
+
)["a"][0]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def rename_table(
|
|
97
|
+
old_table_name,
|
|
98
|
+
new_table_name,
|
|
99
|
+
engine,
|
|
100
|
+
schema: tp.Optional[str] = None,
|
|
101
|
+
nb_trials: int = 3,
|
|
102
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
103
|
+
):
|
|
104
|
+
"""Renames a table of a schema.
|
|
105
|
+
|
|
106
|
+
Parameters
|
|
107
|
+
----------
|
|
108
|
+
old_table_name: str
|
|
109
|
+
old table name
|
|
110
|
+
new_table_name: str
|
|
111
|
+
new table name
|
|
112
|
+
engine: sqlalchemy.engine.Engine
|
|
113
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
114
|
+
schema: str or None
|
|
115
|
+
a valid schema name returned from `list_schemas()`
|
|
116
|
+
nb_trials: int
|
|
117
|
+
number of query trials
|
|
118
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
119
|
+
logger for debugging
|
|
120
|
+
|
|
121
|
+
Returns
|
|
122
|
+
-------
|
|
123
|
+
whatever exec_sql() returns
|
|
124
|
+
"""
|
|
125
|
+
frame_sql_str = frame_sql(old_table_name, schema=schema)
|
|
126
|
+
return exec_sql(
|
|
127
|
+
'ALTER TABLE {} RENAME TO "{}";'.format(frame_sql_str, new_table_name),
|
|
128
|
+
engine,
|
|
129
|
+
nb_trials=nb_trials,
|
|
130
|
+
logger=logger,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def drop_table(
|
|
135
|
+
table_name,
|
|
136
|
+
engine,
|
|
137
|
+
schema: tp.Optional[str] = None,
|
|
138
|
+
restrict=True,
|
|
139
|
+
nb_trials: int = 3,
|
|
140
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
141
|
+
):
|
|
142
|
+
"""Drops a table if it exists, with restrict or cascade options.
|
|
143
|
+
|
|
144
|
+
Parameters
|
|
145
|
+
----------
|
|
146
|
+
table_name : str
|
|
147
|
+
table name
|
|
148
|
+
engine: sqlalchemy.engine.Engine
|
|
149
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
150
|
+
schema: str or None
|
|
151
|
+
a valid schema name returned from `list_schemas()`
|
|
152
|
+
restrict: bool
|
|
153
|
+
If True, refuses to drop table if there is any object depending on it. Otherwise it is the
|
|
154
|
+
'cascade' option which allows you to remove those dependent objects together with the table
|
|
155
|
+
automatically.
|
|
156
|
+
nb_trials: int
|
|
157
|
+
number of query trials
|
|
158
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
159
|
+
logger for debugging
|
|
160
|
+
|
|
161
|
+
Returns
|
|
162
|
+
-------
|
|
163
|
+
whatever exec_sql() returns
|
|
164
|
+
"""
|
|
165
|
+
frame_sql_str = frame_sql(table_name, schema=schema)
|
|
166
|
+
query_str = "DROP TABLE IF EXISTS {} {};".format(
|
|
167
|
+
frame_sql_str, "RESTRICT" if restrict else "CASCADE"
|
|
168
|
+
)
|
|
169
|
+
return exec_sql(query_str, engine, nb_trials=nb_trials, logger=logger)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def rename_view(
|
|
173
|
+
old_view_name,
|
|
174
|
+
new_view_name,
|
|
175
|
+
engine,
|
|
176
|
+
schema: tp.Optional[str] = None,
|
|
177
|
+
nb_trials: int = 3,
|
|
178
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
179
|
+
):
|
|
180
|
+
"""Renames a view of a schema.
|
|
181
|
+
|
|
182
|
+
Parameters
|
|
183
|
+
----------
|
|
184
|
+
old_view_name: str
|
|
185
|
+
old view name
|
|
186
|
+
new_view_name: str
|
|
187
|
+
new view name
|
|
188
|
+
engine: sqlalchemy.engine.Engine
|
|
189
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
190
|
+
schema: str or None
|
|
191
|
+
a valid schema name returned from `list_schemas()`
|
|
192
|
+
nb_trials: int
|
|
193
|
+
number of query trials
|
|
194
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
195
|
+
logger for debugging
|
|
196
|
+
"""
|
|
197
|
+
frame_sql_str = frame_sql(old_view_name, schema=schema)
|
|
198
|
+
exec_sql(
|
|
199
|
+
'ALTER VIEW {} RENAME TO "{}";'.format(frame_sql_str, new_view_name),
|
|
200
|
+
engine,
|
|
201
|
+
nb_trials=nb_trials,
|
|
202
|
+
logger=logger,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def drop_view(
|
|
207
|
+
view_name,
|
|
208
|
+
engine,
|
|
209
|
+
schema: tp.Optional[str] = None,
|
|
210
|
+
restrict=True,
|
|
211
|
+
nb_trials: int = 3,
|
|
212
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
213
|
+
):
|
|
214
|
+
"""Drops a view if it exists, with restrict or cascade options.
|
|
215
|
+
|
|
216
|
+
Parameters
|
|
217
|
+
----------
|
|
218
|
+
view_name: str
|
|
219
|
+
view name
|
|
220
|
+
engine: sqlalchemy.engine.Engine
|
|
221
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
222
|
+
schema: str or None
|
|
223
|
+
a valid schema name returned from `list_schemas()`
|
|
224
|
+
restrict: bool
|
|
225
|
+
If True, refuses to drop table if there is any object depending on it. Otherwise it is the
|
|
226
|
+
'cascade' option which allows you to remove those dependent objects together with the table
|
|
227
|
+
automatically.
|
|
228
|
+
nb_trials: int
|
|
229
|
+
number of query trials
|
|
230
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
231
|
+
logger for debugging
|
|
232
|
+
|
|
233
|
+
Returns
|
|
234
|
+
-------
|
|
235
|
+
whatever exec_sql() returns
|
|
236
|
+
"""
|
|
237
|
+
frame_sql_str = frame_sql(view_name, schema=schema)
|
|
238
|
+
query_str = "DROP VIEW IF EXISTS {} {};".format(
|
|
239
|
+
frame_sql_str, "RESTRICT" if restrict else "CASCADE"
|
|
240
|
+
)
|
|
241
|
+
return exec_sql(query_str, engine, nb_trials=nb_trials, logger=logger)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def rename_matview(
|
|
245
|
+
old_matview_name,
|
|
246
|
+
new_matview_name,
|
|
247
|
+
engine,
|
|
248
|
+
schema: tp.Optional[str] = None,
|
|
249
|
+
nb_trials: int = 3,
|
|
250
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
251
|
+
):
|
|
252
|
+
"""Renames a materialized view of a schema.
|
|
253
|
+
|
|
254
|
+
Parameters
|
|
255
|
+
----------
|
|
256
|
+
old_matview_name: str
|
|
257
|
+
old materialized view name
|
|
258
|
+
new_matview_name: str
|
|
259
|
+
new materialized view name
|
|
260
|
+
engine: sqlalchemy.engine.Engine
|
|
261
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
262
|
+
schema: str or None
|
|
263
|
+
a valid schema name returned from `list_schemas()`
|
|
264
|
+
nb_trials: int
|
|
265
|
+
number of query trials
|
|
266
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
267
|
+
logger for debugging
|
|
268
|
+
"""
|
|
269
|
+
frame_sql_str = frame_sql(old_matview_name, schema=schema)
|
|
270
|
+
exec_sql(
|
|
271
|
+
'ALTER MATERIALIZED VIEW {} RENAME TO "{}";'.format(
|
|
272
|
+
frame_sql_str, new_matview_name
|
|
273
|
+
),
|
|
274
|
+
engine,
|
|
275
|
+
nb_trials=nb_trials,
|
|
276
|
+
logger=logger,
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def refresh_matview(
|
|
281
|
+
matview_name,
|
|
282
|
+
engine,
|
|
283
|
+
schema: tp.Optional[str] = None,
|
|
284
|
+
nb_trials: int = 3,
|
|
285
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
286
|
+
):
|
|
287
|
+
"""Refreshes a materialized view of a schema.
|
|
288
|
+
|
|
289
|
+
Parameters
|
|
290
|
+
----------
|
|
291
|
+
matview_name: str
|
|
292
|
+
materialized view name
|
|
293
|
+
engine: sqlalchemy.engine.Engine
|
|
294
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
295
|
+
schema: str or None
|
|
296
|
+
a valid schema name returned from `list_schemas()`
|
|
297
|
+
nb_trials: int
|
|
298
|
+
number of query trials
|
|
299
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
300
|
+
logger for debugging
|
|
301
|
+
"""
|
|
302
|
+
frame_sql_str = frame_sql(matview_name, schema=schema)
|
|
303
|
+
exec_sql(
|
|
304
|
+
f"REFRESH MATERIALIZED VIEW {frame_sql_str};",
|
|
305
|
+
engine,
|
|
306
|
+
nb_trials=nb_trials,
|
|
307
|
+
logger=logger,
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def drop_matview(
|
|
312
|
+
matview_name,
|
|
313
|
+
engine,
|
|
314
|
+
schema: tp.Optional[str] = None,
|
|
315
|
+
restrict=True,
|
|
316
|
+
nb_trials: int = 3,
|
|
317
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
318
|
+
):
|
|
319
|
+
"""Drops a mateiralized view if it exists, with restrict or cascade options.
|
|
320
|
+
|
|
321
|
+
Parameters
|
|
322
|
+
----------
|
|
323
|
+
matview_name: str
|
|
324
|
+
materialized view name
|
|
325
|
+
engine: sqlalchemy.engine.Engine
|
|
326
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
327
|
+
schema: str or None
|
|
328
|
+
a valid schema name returned from `list_schemas()`
|
|
329
|
+
restrict: bool
|
|
330
|
+
If True, refuses to drop table if there is any object depending on it. Otherwise it is the
|
|
331
|
+
'cascade' option which allows you to remove those dependent objects together with the table
|
|
332
|
+
automatically.
|
|
333
|
+
nb_trials: int
|
|
334
|
+
number of query trials
|
|
335
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
336
|
+
logger for debugging
|
|
337
|
+
|
|
338
|
+
Returns
|
|
339
|
+
-------
|
|
340
|
+
whatever exec_sql() returns
|
|
341
|
+
"""
|
|
342
|
+
frame_sql_str = frame_sql(matview_name, schema=schema)
|
|
343
|
+
query_str = "DROP MATERIALIZED VIEW IF EXISTS {} {};".format(
|
|
344
|
+
frame_sql_str, "RESTRICT" if restrict else "CASCADE"
|
|
345
|
+
)
|
|
346
|
+
return exec_sql(query_str, engine, nb_trials=nb_trials, logger=logger)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def rename_column(
|
|
350
|
+
table_name,
|
|
351
|
+
old_column_name,
|
|
352
|
+
new_column_name,
|
|
353
|
+
engine,
|
|
354
|
+
schema: tp.Optional[str] = None,
|
|
355
|
+
nb_trials: int = 3,
|
|
356
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
357
|
+
):
|
|
358
|
+
"""Renames a column of a table.
|
|
359
|
+
|
|
360
|
+
Parameters
|
|
361
|
+
----------
|
|
362
|
+
table_name: str
|
|
363
|
+
table name
|
|
364
|
+
old_column_name: str
|
|
365
|
+
old column name
|
|
366
|
+
new_column_name: str
|
|
367
|
+
new column name
|
|
368
|
+
engine: sqlalchemy.engine.Engine
|
|
369
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
370
|
+
schema: str or None
|
|
371
|
+
schema name
|
|
372
|
+
nb_trials: int
|
|
373
|
+
number of query trials
|
|
374
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
375
|
+
logger for debugging
|
|
376
|
+
"""
|
|
377
|
+
old_column_name = old_column_name.replace("%", "%%")
|
|
378
|
+
if schema is None:
|
|
379
|
+
query_str = 'ALTER TABLE "{}" RENAME COLUMN "{}" TO "{}";'.format(
|
|
380
|
+
table_name, old_column_name, new_column_name
|
|
381
|
+
)
|
|
382
|
+
else:
|
|
383
|
+
query_str = 'ALTER TABLE "{}"."{}" RENAME COLUMN "{}" TO "{}";'.format(
|
|
384
|
+
schema, table_name, old_column_name, new_column_name
|
|
385
|
+
)
|
|
386
|
+
exec_sql(query_str, engine, nb_trials=nb_trials, logger=logger)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def drop_column(
|
|
390
|
+
table_name,
|
|
391
|
+
column_name,
|
|
392
|
+
engine,
|
|
393
|
+
schema: tp.Optional[str] = None,
|
|
394
|
+
nb_trials: int = 3,
|
|
395
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
396
|
+
):
|
|
397
|
+
"""Drops a column of a table.
|
|
398
|
+
|
|
399
|
+
Parameters
|
|
400
|
+
----------
|
|
401
|
+
table_name: str
|
|
402
|
+
table name
|
|
403
|
+
column_name: str
|
|
404
|
+
column name
|
|
405
|
+
engine: sqlalchemy.engine.Engine
|
|
406
|
+
an sqlalchemy connection engine created by function `create_engine()`
|
|
407
|
+
schema: str or None
|
|
408
|
+
schema name
|
|
409
|
+
nb_trials: int
|
|
410
|
+
number of query trials
|
|
411
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
412
|
+
logger for debugging
|
|
413
|
+
"""
|
|
414
|
+
column_name = column_name.replace("%", "%%")
|
|
415
|
+
if schema is None:
|
|
416
|
+
query_str = 'ALTER TABLE "{}" DROP COLUMN "{}";'.format(table_name, column_name)
|
|
417
|
+
else:
|
|
418
|
+
query_str = 'ALTER TABLE "{}"."{}" DROP COLUMN "{}";'.format(
|
|
419
|
+
schema, table_name, column_name
|
|
420
|
+
)
|
|
421
|
+
exec_sql(query_str, engine, nb_trials=nb_trials, logger=logger)
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
# ----- functions dealing with sql queries to overcome OperationalError -----
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def to_sql(
|
|
428
|
+
df,
|
|
429
|
+
name,
|
|
430
|
+
engine,
|
|
431
|
+
schema: tp.Optional[str] = None,
|
|
432
|
+
if_exists="fail",
|
|
433
|
+
nb_trials: int = 3,
|
|
434
|
+
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
435
|
+
**kwargs,
|
|
436
|
+
):
|
|
437
|
+
"""Writes records stored in a DataFrame to a Redshift database.
|
|
438
|
+
|
|
439
|
+
With a number of trials to overcome OperationalError.
|
|
440
|
+
|
|
441
|
+
Parameters
|
|
442
|
+
----------
|
|
443
|
+
df : pandas.DataFrame
|
|
444
|
+
dataframe to be sent to the server
|
|
445
|
+
name : str
|
|
446
|
+
name of the table to be written to
|
|
447
|
+
engine : sqlalchemy.engine.Engine
|
|
448
|
+
connection engine to the server
|
|
449
|
+
schema: string, optional
|
|
450
|
+
Specify the schema. If None, use default schema.
|
|
451
|
+
if_exists: str
|
|
452
|
+
what to do when the table exists. Passed as-is to :func:`pandas.DataFrame.to_sql`.
|
|
453
|
+
nb_trials: int
|
|
454
|
+
number of query trials
|
|
455
|
+
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
456
|
+
logger for debugging
|
|
457
|
+
kwargs : dict
|
|
458
|
+
keyword arguments passed as-is to :func:`pandas.DataFrame.to_sql`
|
|
459
|
+
|
|
460
|
+
Raises
|
|
461
|
+
------
|
|
462
|
+
sqlalchemy.exc.ProgrammingError if the local and remote frames do not have the same structure
|
|
463
|
+
|
|
464
|
+
Notes
|
|
465
|
+
-----
|
|
466
|
+
The function takes as input a PSQL-compliant dataframe (see `compliance_check()`). It ignores
|
|
467
|
+
any input `index` or `index_label` keyword. Instead, it considers 2 cases. If the dataframe has
|
|
468
|
+
an index or indices, then the tuple of all indices is turned into the primary key. If not,
|
|
469
|
+
there is no primary key and no index is uploaded.
|
|
470
|
+
|
|
471
|
+
See Also
|
|
472
|
+
--------
|
|
473
|
+
pandas.DataFrame.to_sql()
|
|
474
|
+
|
|
475
|
+
"""
|
|
476
|
+
|
|
477
|
+
if kwargs:
|
|
478
|
+
if "index" in kwargs:
|
|
479
|
+
raise ValueError(
|
|
480
|
+
"The `mt.sql.psql.to_sql()` function does not accept `index` as a keyword."
|
|
481
|
+
)
|
|
482
|
+
if "index_label" in kwargs:
|
|
483
|
+
raise ValueError(
|
|
484
|
+
"This `mt.sql.psql.to_sql()` function does not accept `index_label` as a keyword."
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
compliance_check(df)
|
|
488
|
+
frame_sql_str = frame_sql(name, schema=schema)
|
|
489
|
+
|
|
490
|
+
# if the remote frame does not exist, force `if_exists` to 'replace'
|
|
491
|
+
if not table_exists(name, engine, schema=schema):
|
|
492
|
+
if_exists = "replace"
|
|
493
|
+
local_indices = indices(df)
|
|
494
|
+
|
|
495
|
+
if local_indices:
|
|
496
|
+
df = df.reset_index(drop=False)
|
|
497
|
+
retval = run_func(
|
|
498
|
+
df.to_sql,
|
|
499
|
+
name,
|
|
500
|
+
engine,
|
|
501
|
+
schema=schema,
|
|
502
|
+
if_exists=if_exists,
|
|
503
|
+
index=False,
|
|
504
|
+
index_label=None,
|
|
505
|
+
nb_trials=nb_trials,
|
|
506
|
+
logger=logger,
|
|
507
|
+
**kwargs,
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
query_str = (
|
|
511
|
+
f"ALTER TABLE {frame_sql_str} ADD PRIMARY KEY ({','.join(local_indices)});"
|
|
512
|
+
)
|
|
513
|
+
exec_sql(query_str, engine, nb_trials=nb_trials, logger=logger)
|
|
514
|
+
else:
|
|
515
|
+
retval = run_func(
|
|
516
|
+
df.to_sql,
|
|
517
|
+
name,
|
|
518
|
+
engine,
|
|
519
|
+
schema=schema,
|
|
520
|
+
if_exists=if_exists,
|
|
521
|
+
index=False,
|
|
522
|
+
index_label=None,
|
|
523
|
+
nb_trials=nb_trials,
|
|
524
|
+
logger=logger,
|
|
525
|
+
**kwargs,
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
return retval
|
mt/sql/version.py
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
VERSION_YEAR =
|
|
2
|
-
VERSION_MONTH = int('
|
|
3
|
-
VERSION_DAY = int('
|
|
1
|
+
VERSION_YEAR = 2024
|
|
2
|
+
VERSION_MONTH = int('01')
|
|
3
|
+
VERSION_DAY = int('02')
|
|
4
4
|
VERSION_HOUR = int('14')
|
|
5
|
-
VERSION_MINUTE = int('
|
|
5
|
+
VERSION_MINUTE = int('06')
|
|
6
6
|
MAJOR_VERSION = 1
|
|
7
|
-
MINOR_VERSION =
|
|
8
|
-
PATCH_VERSION =
|
|
9
|
-
version_date = '
|
|
7
|
+
MINOR_VERSION = 8
|
|
8
|
+
PATCH_VERSION = 202401021406
|
|
9
|
+
version_date = '2024/01/02 14:06'
|
|
10
10
|
version = '{}.{}.{}'.format(MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION)
|
|
11
11
|
__all__ = ['MAJOR_VERSION', 'MINOR_VERSION', 'PATCH_VERSION', 'version_date', 'version']
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mtsql
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.8.202401021406
|
|
4
4
|
Summary: Extra Python modules to deal with the interaction between pandas dataframes and remote SQL servers, for Minh-Tri Pham
|
|
5
5
|
Home-page: https://github.com/inteplus/mtsql
|
|
6
6
|
Author: ['Minh-Tri Pham']
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
mt/sql/__init__.py,sha256=b7zO50apZxt9Hg2eOkJhRLrXgACR8eS5b-Rphdn5qNQ,44
|
|
2
|
+
mt/sql/base.py,sha256=FoK42rNDxVkqwtGiDJGvW1g02FJtSoqkMtbPlxBElA0,9780
|
|
3
|
+
mt/sql/mysql.py,sha256=n2ENDctdUqZuSaDAcrqZYtPtawq3Wx4dOPCRsCB5Q4w,4894
|
|
4
|
+
mt/sql/psql.py,sha256=JBdwPKwk1G6QzOJGRjUbIcxKosjPQ_PADG2VwDxCR4M,66036
|
|
5
|
+
mt/sql/redshift.py,sha256=cDDSPUqy6vIyKcKUNsQnZLESGLhgC8NYGyyIWhFTrNk,14789
|
|
6
|
+
mt/sql/sqlite.py,sha256=T2ak_hhNi_zRfpg_gp8JhNHn7D2kl4i-Ey6-9ANMtz0,8678
|
|
7
|
+
mt/sql/version.py,sha256=gnhJJ-02mxUSXuUdVmoieNHefSOLt5692teFJSVykOA,396
|
|
8
|
+
mtsql-1.8.202401021406.dist-info/LICENSE,sha256=PojkRlQzTT5Eg6Nj03XoIVEefN3u8iiIFf1p4rqe_t4,1070
|
|
9
|
+
mtsql-1.8.202401021406.dist-info/METADATA,sha256=vlwY4ZndshEQJY6Ux4D1N907fFbl7b_7K1erSDkPkuA,589
|
|
10
|
+
mtsql-1.8.202401021406.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
|
11
|
+
mtsql-1.8.202401021406.dist-info/top_level.txt,sha256=WcqGFu9cV7iMZg09iam8eNxUvGpLSKKF2Iubf6SJVOo,3
|
|
12
|
+
mtsql-1.8.202401021406.dist-info/RECORD,,
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
mt/sql/__init__.py,sha256=b7zO50apZxt9Hg2eOkJhRLrXgACR8eS5b-Rphdn5qNQ,44
|
|
2
|
-
mt/sql/base.py,sha256=sFr7O_Odfsf2AHr9kq3DXGCAFInCKgHSgLJaen507_I,9994
|
|
3
|
-
mt/sql/mysql.py,sha256=n2ENDctdUqZuSaDAcrqZYtPtawq3Wx4dOPCRsCB5Q4w,4894
|
|
4
|
-
mt/sql/psql.py,sha256=dRN4wH1uQ-deGb2M-3PbdUfjHQ_1fbPXnR_94X1KMIU,67364
|
|
5
|
-
mt/sql/sqlite.py,sha256=T2ak_hhNi_zRfpg_gp8JhNHn7D2kl4i-Ey6-9ANMtz0,8678
|
|
6
|
-
mt/sql/version.py,sha256=Gw-cqAJUQkq15aZ7wH7Kko2b1pAD2Jx9PSnlpz_wnPU,396
|
|
7
|
-
mtsql-1.6.202312111459.dist-info/LICENSE,sha256=PojkRlQzTT5Eg6Nj03XoIVEefN3u8iiIFf1p4rqe_t4,1070
|
|
8
|
-
mtsql-1.6.202312111459.dist-info/METADATA,sha256=Nil4E7SpttMKbiwmhk_A0Kf07-tsW5528buQmE4NpxQ,589
|
|
9
|
-
mtsql-1.6.202312111459.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
|
10
|
-
mtsql-1.6.202312111459.dist-info/top_level.txt,sha256=WcqGFu9cV7iMZg09iam8eNxUvGpLSKKF2Iubf6SJVOo,3
|
|
11
|
-
mtsql-1.6.202312111459.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|