mtsql 1.12.9__py3-none-any.whl → 1.12.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mt/sql/base.py +27 -28
- mt/sql/version.py +1 -1
- {mtsql-1.12.9.dist-info → mtsql-1.12.11.dist-info}/METADATA +2 -2
- {mtsql-1.12.9.dist-info → mtsql-1.12.11.dist-info}/RECORD +7 -7
- {mtsql-1.12.9.dist-info → mtsql-1.12.11.dist-info}/WHEEL +0 -0
- {mtsql-1.12.9.dist-info → mtsql-1.12.11.dist-info}/licenses/LICENSE +0 -0
- {mtsql-1.12.9.dist-info → mtsql-1.12.11.dist-info}/top_level.txt +0 -0
mt/sql/base.py
CHANGED
|
@@ -31,7 +31,7 @@ __all__ = [
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
def frame_sql(frame_name, schema: tp.Optional[str] = None):
|
|
34
|
-
return frame_name if schema is None else "{}.{}"
|
|
34
|
+
return frame_name if schema is None else f"{schema}.{frame_name}"
|
|
35
35
|
|
|
36
36
|
|
|
37
37
|
def indices(df):
|
|
@@ -48,26 +48,26 @@ def run_func(
|
|
|
48
48
|
*args,
|
|
49
49
|
nb_trials: int = 3,
|
|
50
50
|
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
51
|
-
**
|
|
51
|
+
**kwds,
|
|
52
52
|
):
|
|
53
53
|
"""Attempt to run a function a number of times to overcome OperationalError exceptions.
|
|
54
54
|
|
|
55
55
|
Parameters
|
|
56
56
|
----------
|
|
57
|
-
func: function
|
|
57
|
+
func : function
|
|
58
58
|
function to be invoked
|
|
59
|
-
|
|
60
|
-
arguments to be passed to the function
|
|
61
|
-
nb_trials: int
|
|
59
|
+
nb_trials : int
|
|
62
60
|
number of query trials
|
|
63
|
-
logger: mt.logg.IndentedLoggerAdapter, optional
|
|
61
|
+
logger : mt.logg.IndentedLoggerAdapter, optional
|
|
64
62
|
logger for debugging
|
|
65
|
-
|
|
63
|
+
*args : iterable
|
|
64
|
+
arguments to be passed to the function
|
|
65
|
+
**kwds : dict
|
|
66
66
|
keyword arguments to be passed to the function
|
|
67
67
|
"""
|
|
68
68
|
for x in range(nb_trials):
|
|
69
69
|
try:
|
|
70
|
-
return func(*args, **
|
|
70
|
+
return func(*args, **kwds)
|
|
71
71
|
except (se.ProgrammingError, se.IntegrityError):
|
|
72
72
|
raise
|
|
73
73
|
except (
|
|
@@ -92,10 +92,10 @@ def conn_ctx(engine):
|
|
|
92
92
|
return ctx.nullcontext(engine)
|
|
93
93
|
|
|
94
94
|
|
|
95
|
-
def engine_execute(engine, sql, *args, **
|
|
95
|
+
def engine_execute(engine, sql, *args, **kwds):
|
|
96
96
|
text_sql = sa.text(sql) if isinstance(sql, str) else sql
|
|
97
97
|
with conn_ctx(engine) as conn:
|
|
98
|
-
return conn.execute(text_sql, *args, **
|
|
98
|
+
return conn.execute(text_sql, *args, **kwds)
|
|
99
99
|
|
|
100
100
|
|
|
101
101
|
def trim_sql_query(sql_query: str) -> str:
|
|
@@ -112,7 +112,7 @@ def read_sql(
|
|
|
112
112
|
nb_trials: int = 3,
|
|
113
113
|
exception_handling: str = "raise",
|
|
114
114
|
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
115
|
-
**
|
|
115
|
+
**kwds,
|
|
116
116
|
) -> pd.DataFrame:
|
|
117
117
|
"""Read an SQL query with a number of trials to overcome OperationalError.
|
|
118
118
|
|
|
@@ -165,7 +165,7 @@ def read_sql(
|
|
|
165
165
|
text_sql = trim_sql_query(text_sql)
|
|
166
166
|
|
|
167
167
|
if chunksize is not None:
|
|
168
|
-
s = "read_sql: '{}'"
|
|
168
|
+
s = f"read_sql: '{text_sql}'"
|
|
169
169
|
spinner = halo.Halo(s, spinner="dots", enabled=bool(logger))
|
|
170
170
|
spinner.start()
|
|
171
171
|
ts = pd.Timestamp.now()
|
|
@@ -180,7 +180,7 @@ def read_sql(
|
|
|
180
180
|
chunksize=chunksize,
|
|
181
181
|
nb_trials=nb_trials,
|
|
182
182
|
logger=logger,
|
|
183
|
-
**
|
|
183
|
+
**kwds,
|
|
184
184
|
)
|
|
185
185
|
|
|
186
186
|
if chunksize is None:
|
|
@@ -192,24 +192,21 @@ def read_sql(
|
|
|
192
192
|
dfs.append(df)
|
|
193
193
|
cnt += len(df)
|
|
194
194
|
td = (pd.Timestamp.now() - ts).total_seconds() + 0.001
|
|
195
|
-
s = "{} rows ({} rows/sec)"
|
|
195
|
+
s = f"{cnt} rows ({cnt / td} rows/sec)"
|
|
196
196
|
spinner.text = s
|
|
197
197
|
df = pd.concat(dfs)
|
|
198
|
-
s = "{} rows"
|
|
198
|
+
s = f"{cnt} rows"
|
|
199
199
|
spinner.succeed(s)
|
|
200
200
|
except:
|
|
201
|
-
s = "{} rows"
|
|
201
|
+
s = f"{cnt} rows"
|
|
202
202
|
spinner.fail(s)
|
|
203
203
|
if logger:
|
|
204
204
|
logger.warn_last_exception()
|
|
205
205
|
if exception_handling == "raise":
|
|
206
206
|
raise
|
|
207
207
|
if exception_handling != "warn":
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
exception_handling
|
|
211
|
-
)
|
|
212
|
-
)
|
|
208
|
+
msg = f"Unknown value for argument 'exception_handling': '{exception_handling}'."
|
|
209
|
+
raise ValueError(msg)
|
|
213
210
|
df = pd.concat(dfs)
|
|
214
211
|
|
|
215
212
|
return df
|
|
@@ -220,7 +217,7 @@ def read_sql_table(
|
|
|
220
217
|
engine,
|
|
221
218
|
nb_trials: int = 3,
|
|
222
219
|
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
223
|
-
**
|
|
220
|
+
**kwds,
|
|
224
221
|
):
|
|
225
222
|
"""Read an SQL table with a number of trials to overcome OperationalError.
|
|
226
223
|
|
|
@@ -246,7 +243,7 @@ def read_sql_table(
|
|
|
246
243
|
engine,
|
|
247
244
|
nb_trials=nb_trials,
|
|
248
245
|
logger=logger,
|
|
249
|
-
**
|
|
246
|
+
**kwds,
|
|
250
247
|
)
|
|
251
248
|
|
|
252
249
|
|
|
@@ -256,7 +253,7 @@ def exec_sql(
|
|
|
256
253
|
*args,
|
|
257
254
|
nb_trials: int = 3,
|
|
258
255
|
logger: tp.Optional[logg.IndentedLoggerAdapter] = None,
|
|
259
|
-
**
|
|
256
|
+
**kwds,
|
|
260
257
|
):
|
|
261
258
|
"""Execute an SQL query with a number of trials to overcome OperationalError.
|
|
262
259
|
|
|
@@ -280,7 +277,7 @@ def exec_sql(
|
|
|
280
277
|
"""
|
|
281
278
|
|
|
282
279
|
return run_func(
|
|
283
|
-
engine_execute, engine, sql, *args, nb_trials=nb_trials, logger=logger, **
|
|
280
|
+
engine_execute, engine, sql, *args, nb_trials=nb_trials, logger=logger, **kwds
|
|
284
281
|
)
|
|
285
282
|
|
|
286
283
|
|
|
@@ -456,7 +453,7 @@ def temp_table_drop(
|
|
|
456
453
|
|
|
457
454
|
|
|
458
455
|
@ctx.contextmanager
|
|
459
|
-
def to_temp_table(df: pd.DataFrame, engine: sa.engine.Engine):
|
|
456
|
+
def to_temp_table(df: pd.DataFrame, engine: sa.engine.Engine, **kwds):
|
|
460
457
|
"""
|
|
461
458
|
A context manager that uploads a dataframe to a temp table and cleans up the table when done.
|
|
462
459
|
|
|
@@ -469,12 +466,14 @@ def to_temp_table(df: pd.DataFrame, engine: sa.engine.Engine):
|
|
|
469
466
|
dataframe to be uploaded to the database as a temporary table
|
|
470
467
|
engine : sqlalchemy.engine.Engine
|
|
471
468
|
engine connrecting to the database
|
|
469
|
+
**kwds : dict
|
|
470
|
+
other keyword arguments passed as-is to :func:`pandas.DataFrame.to_sql`
|
|
472
471
|
"""
|
|
473
472
|
|
|
474
473
|
tid = temp_table_find_new_id(engine)
|
|
475
474
|
name = temp_table_name(tid)
|
|
476
475
|
try:
|
|
477
|
-
df.to_sql(name, engine)
|
|
476
|
+
df.to_sql(name, engine, **kwds)
|
|
478
477
|
yield name
|
|
479
478
|
finally:
|
|
480
479
|
temp_table_drop(engine, tid)
|
mt/sql/version.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mtsql
|
|
3
|
-
Version: 1.12.
|
|
3
|
+
Version: 1.12.11
|
|
4
4
|
Summary: Extra Python modules to deal with the interaction between pandas dataframes and remote SQL servers, for Minh-Tri Pham
|
|
5
5
|
Home-page: https://github.com/inteplus/mtsql
|
|
6
6
|
Author: ['Minh-Tri Pham']
|
|
@@ -12,7 +12,7 @@ Requires-Dist: tzlocal
|
|
|
12
12
|
Requires-Dist: tqdm
|
|
13
13
|
Requires-Dist: psycopg[binary]
|
|
14
14
|
Requires-Dist: redshift_connector>=2.1.5
|
|
15
|
-
Requires-Dist: mtbase>=4.32.
|
|
15
|
+
Requires-Dist: mtbase>=4.32.21
|
|
16
16
|
Requires-Dist: mtpandas>=1.17.10
|
|
17
17
|
Dynamic: author
|
|
18
18
|
Dynamic: home-page
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
mt/sql/__init__.py,sha256=b7zO50apZxt9Hg2eOkJhRLrXgACR8eS5b-Rphdn5qNQ,44
|
|
2
|
-
mt/sql/base.py,sha256=
|
|
2
|
+
mt/sql/base.py,sha256=rEDlXivxH06ZsdpSXkraiqBCDDYh3FLVZfuH-spVw8w,12976
|
|
3
3
|
mt/sql/mysql.py,sha256=n2ENDctdUqZuSaDAcrqZYtPtawq3Wx4dOPCRsCB5Q4w,4894
|
|
4
4
|
mt/sql/psql.py,sha256=dNPOvJBxZ8S88uOmIKnfjo1ssRiMLDLwVeJcVcdcNco,66580
|
|
5
5
|
mt/sql/sqlite.py,sha256=T2ak_hhNi_zRfpg_gp8JhNHn7D2kl4i-Ey6-9ANMtz0,8678
|
|
6
|
-
mt/sql/version.py,sha256=
|
|
6
|
+
mt/sql/version.py,sha256=w6SVgrZRGJU7ZkLGiZn_JhRKVxIF0tW5Z2yvN47_WVg,208
|
|
7
7
|
mt/sql/redshift/__init__.py,sha256=S-eRxJWcrvncF7LZXuulCdPV-UERu9eiw6uyDb5aGsM,443
|
|
8
8
|
mt/sql/redshift/commands.py,sha256=xhGUBf3bL66EYdZI5HCUtOx-XqPCnXT_P-LnhPgtzrY,40193
|
|
9
9
|
mt/sql/redshift/ddl.py,sha256=eUcZj9oIajiE1wKKBAP-V64gYJ7cVnSAt8dLFfluOJA,9777
|
|
10
10
|
mt/sql/redshift/dialect.py,sha256=oomLiQib0iMhotRY3GjDrQI3iWGXgcIQyK5WTbmj8kc,53431
|
|
11
11
|
mt/sql/redshift/main.py,sha256=6dwnwNJ1F0_V9o2oqrSOkyN_pAMrgE01CCoqAjoyOME,17116
|
|
12
12
|
mt/sql/redshift/redshift-ca-bundle.crt,sha256=532qYkOpQOstFE0mdXE1GVtL3v00XDKgZNTr6gK5-KE,8621
|
|
13
|
-
mtsql-1.12.
|
|
14
|
-
mtsql-1.12.
|
|
15
|
-
mtsql-1.12.
|
|
16
|
-
mtsql-1.12.
|
|
17
|
-
mtsql-1.12.
|
|
13
|
+
mtsql-1.12.11.dist-info/licenses/LICENSE,sha256=PojkRlQzTT5Eg6Nj03XoIVEefN3u8iiIFf1p4rqe_t4,1070
|
|
14
|
+
mtsql-1.12.11.dist-info/METADATA,sha256=ieYqOgiWjVGvSbp1kjcql2In-R36c6bB3UXr9GzcS6k,740
|
|
15
|
+
mtsql-1.12.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
16
|
+
mtsql-1.12.11.dist-info/top_level.txt,sha256=WcqGFu9cV7iMZg09iam8eNxUvGpLSKKF2Iubf6SJVOo,3
|
|
17
|
+
mtsql-1.12.11.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|