meerschaum 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/actions/copy.py +1 -0
- meerschaum/actions/drop.py +100 -22
- meerschaum/actions/index.py +71 -0
- meerschaum/actions/register.py +8 -12
- meerschaum/actions/sql.py +1 -1
- meerschaum/api/routes/_pipes.py +18 -0
- meerschaum/api/routes/_plugins.py +1 -1
- meerschaum/api/routes/_users.py +62 -61
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_pipes.py +20 -0
- meerschaum/connectors/sql/_SQLConnector.py +8 -12
- meerschaum/connectors/sql/_create_engine.py +1 -1
- meerschaum/connectors/sql/_fetch.py +9 -39
- meerschaum/connectors/sql/_instance.py +3 -3
- meerschaum/connectors/sql/_pipes.py +262 -70
- meerschaum/connectors/sql/_plugins.py +11 -16
- meerschaum/connectors/sql/_sql.py +60 -39
- meerschaum/connectors/sql/_uri.py +9 -9
- meerschaum/connectors/sql/_users.py +10 -12
- meerschaum/connectors/sql/tables/__init__.py +13 -14
- meerschaum/connectors/valkey/_ValkeyConnector.py +2 -2
- meerschaum/core/Pipe/__init__.py +12 -2
- meerschaum/core/Pipe/_attributes.py +32 -38
- meerschaum/core/Pipe/_drop.py +73 -2
- meerschaum/core/Pipe/_fetch.py +4 -0
- meerschaum/core/Pipe/_index.py +68 -0
- meerschaum/core/Pipe/_sync.py +16 -9
- meerschaum/utils/daemon/Daemon.py +9 -2
- meerschaum/utils/daemon/RotatingFile.py +3 -3
- meerschaum/utils/dataframe.py +42 -12
- meerschaum/utils/dtypes/__init__.py +144 -24
- meerschaum/utils/dtypes/sql.py +52 -9
- meerschaum/utils/formatting/__init__.py +2 -2
- meerschaum/utils/formatting/_pprint.py +12 -11
- meerschaum/utils/misc.py +16 -18
- meerschaum/utils/prompt.py +1 -1
- meerschaum/utils/sql.py +106 -42
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/METADATA +14 -2
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/RECORD +45 -43
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/WHEEL +1 -1
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/LICENSE +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/NOTICE +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/top_level.txt +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/zip-safe +0 -0
@@ -70,6 +70,9 @@ class SQLConnector(Connector):
|
|
70
70
|
create_pipe_table_from_df,
|
71
71
|
get_pipe_columns_indices,
|
72
72
|
get_temporary_target,
|
73
|
+
create_pipe_indices,
|
74
|
+
drop_pipe_indices,
|
75
|
+
get_pipe_index_names,
|
73
76
|
)
|
74
77
|
from ._plugins import (
|
75
78
|
register_plugin,
|
@@ -192,7 +195,8 @@ class SQLConnector(Connector):
|
|
192
195
|
self._debug = debug
|
193
196
|
### Store the PID and thread at initialization
|
194
197
|
### so we can dispose of the Pool in child processes or threads.
|
195
|
-
import os
|
198
|
+
import os
|
199
|
+
import threading
|
196
200
|
self._pid = os.getpid()
|
197
201
|
self._thread_ident = threading.current_thread().ident
|
198
202
|
self._sessions = {}
|
@@ -222,7 +226,7 @@ class SQLConnector(Connector):
|
|
222
226
|
return None
|
223
227
|
|
224
228
|
from meerschaum.utils.packages import attempt_import
|
225
|
-
sqlalchemy_orm = attempt_import('sqlalchemy.orm')
|
229
|
+
sqlalchemy_orm = attempt_import('sqlalchemy.orm', lazy=False)
|
226
230
|
session_factory = sqlalchemy_orm.sessionmaker(self.engine)
|
227
231
|
self._Session = sqlalchemy_orm.scoped_session(session_factory)
|
228
232
|
|
@@ -283,19 +287,17 @@ class SQLConnector(Connector):
|
|
283
287
|
return ':memory:' not in self.URI
|
284
288
|
return True
|
285
289
|
|
286
|
-
|
287
290
|
@property
|
288
291
|
def metadata(self):
|
289
292
|
"""
|
290
293
|
Return the metadata bound to this configured schema.
|
291
294
|
"""
|
292
295
|
from meerschaum.utils.packages import attempt_import
|
293
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
296
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
294
297
|
if '_metadata' not in self.__dict__:
|
295
298
|
self._metadata = sqlalchemy.MetaData(schema=self.schema)
|
296
299
|
return self._metadata
|
297
300
|
|
298
|
-
|
299
301
|
@property
|
300
302
|
def instance_schema(self):
|
301
303
|
"""
|
@@ -303,14 +305,12 @@ class SQLConnector(Connector):
|
|
303
305
|
"""
|
304
306
|
return self.schema
|
305
307
|
|
306
|
-
|
307
308
|
@property
|
308
309
|
def internal_schema(self):
|
309
310
|
"""
|
310
311
|
Return the schema name for internal tables.
|
311
312
|
"""
|
312
313
|
from meerschaum.config.static import STATIC_CONFIG
|
313
|
-
from meerschaum.utils.packages import attempt_import
|
314
314
|
from meerschaum.utils.sql import NO_SCHEMA_FLAVORS
|
315
315
|
schema_name = self.__dict__.get('internal_schema', None) or (
|
316
316
|
STATIC_CONFIG['sql']['internal_schema']
|
@@ -322,7 +322,6 @@ class SQLConnector(Connector):
|
|
322
322
|
self._internal_schema = schema_name
|
323
323
|
return self._internal_schema
|
324
324
|
|
325
|
-
|
326
325
|
@property
|
327
326
|
def db(self) -> Optional[databases.Database]:
|
328
327
|
from meerschaum.utils.packages import attempt_import
|
@@ -339,7 +338,6 @@ class SQLConnector(Connector):
|
|
339
338
|
self._db = None
|
340
339
|
return self._db
|
341
340
|
|
342
|
-
|
343
341
|
@property
|
344
342
|
def db_version(self) -> Union[str, None]:
|
345
343
|
"""
|
@@ -353,7 +351,6 @@ class SQLConnector(Connector):
|
|
353
351
|
self._db_version = get_db_version(self)
|
354
352
|
return self._db_version
|
355
353
|
|
356
|
-
|
357
354
|
@property
|
358
355
|
def schema(self) -> Union[str, None]:
|
359
356
|
"""
|
@@ -368,12 +365,11 @@ class SQLConnector(Connector):
|
|
368
365
|
self.__dict__['schema'] = None
|
369
366
|
return None
|
370
367
|
|
371
|
-
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
368
|
+
sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
|
372
369
|
_schema = sqlalchemy.inspect(self.engine).default_schema_name
|
373
370
|
self.__dict__['schema'] = _schema
|
374
371
|
return _schema
|
375
372
|
|
376
|
-
|
377
373
|
def __getstate__(self):
|
378
374
|
return self.__dict__
|
379
375
|
|
@@ -186,7 +186,7 @@ def create_engine(
|
|
186
186
|
"""Create a sqlalchemy engine by building the engine string."""
|
187
187
|
from meerschaum.utils.packages import attempt_import
|
188
188
|
from meerschaum.utils.warnings import error, warn
|
189
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
189
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
190
190
|
import urllib
|
191
191
|
import copy
|
192
192
|
### Install and patch required drivers.
|
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|
11
11
|
from datetime import datetime, timedelta
|
12
12
|
|
13
13
|
import meerschaum as mrsm
|
14
|
-
from meerschaum.utils.typing import Optional, Union,
|
14
|
+
from meerschaum.utils.typing import Optional, Union, Any, List, Dict
|
15
15
|
|
16
16
|
|
17
17
|
def fetch(
|
@@ -20,7 +20,6 @@ def fetch(
|
|
20
20
|
begin: Union[datetime, int, str, None] = '',
|
21
21
|
end: Union[datetime, int, str, None] = None,
|
22
22
|
check_existing: bool = True,
|
23
|
-
chunk_hook: Optional[Callable[['pd.DataFrame'], Any]] = None,
|
24
23
|
chunksize: Optional[int] = -1,
|
25
24
|
workers: Optional[int] = None,
|
26
25
|
debug: bool = False,
|
@@ -53,15 +52,12 @@ def fetch(
|
|
53
52
|
check_existing: bool, defult True
|
54
53
|
If `False`, use a backtrack interval of 0 minutes.
|
55
54
|
|
56
|
-
chunk_hook: Callable[[pd.DataFrame], Any], default None
|
57
|
-
A function to pass to `SQLConnector.read()` that accepts a Pandas DataFrame.
|
58
|
-
|
59
55
|
chunksize: Optional[int], default -1
|
60
|
-
How many rows to load into memory at once
|
56
|
+
How many rows to load into memory at once.
|
61
57
|
Otherwise the entire result set is loaded into memory.
|
62
58
|
|
63
59
|
workers: Optional[int], default None
|
64
|
-
How many threads to use when consuming the generator
|
60
|
+
How many threads to use when consuming the generator.
|
65
61
|
Defaults to the number of cores.
|
66
62
|
|
67
63
|
debug: bool, default False
|
@@ -69,8 +65,7 @@ def fetch(
|
|
69
65
|
|
70
66
|
Returns
|
71
67
|
-------
|
72
|
-
A pandas DataFrame
|
73
|
-
If `chunk_hook` is not None, return a list of the hook function's results.
|
68
|
+
A pandas DataFrame generator.
|
74
69
|
"""
|
75
70
|
meta_def = self.get_pipe_metadef(
|
76
71
|
pipe,
|
@@ -80,33 +75,13 @@ def fetch(
|
|
80
75
|
debug=debug,
|
81
76
|
**kw
|
82
77
|
)
|
83
|
-
as_hook_results = chunk_hook is not None
|
84
78
|
chunks = self.read(
|
85
79
|
meta_def,
|
86
|
-
chunk_hook=chunk_hook,
|
87
|
-
as_hook_results=as_hook_results,
|
88
80
|
chunksize=chunksize,
|
89
81
|
workers=workers,
|
82
|
+
as_iterator=True,
|
90
83
|
debug=debug,
|
91
84
|
)
|
92
|
-
### if sqlite, parse for datetimes
|
93
|
-
if not as_hook_results and self.flavor == 'sqlite':
|
94
|
-
from meerschaum.utils.dataframe import parse_df_datetimes
|
95
|
-
from meerschaum.utils.dtypes import are_dtypes_equal
|
96
|
-
ignore_cols = [
|
97
|
-
col
|
98
|
-
for col, dtype in pipe.dtypes.items()
|
99
|
-
if not are_dtypes_equal(str(dtype), 'datetime')
|
100
|
-
]
|
101
|
-
return (
|
102
|
-
parse_df_datetimes(
|
103
|
-
chunk,
|
104
|
-
ignore_cols=ignore_cols,
|
105
|
-
strip_timezone=(pipe.tzinfo is None),
|
106
|
-
debug=debug,
|
107
|
-
)
|
108
|
-
for chunk in chunks
|
109
|
-
)
|
110
85
|
return chunks
|
111
86
|
|
112
87
|
|
@@ -148,7 +123,6 @@ def get_pipe_metadef(
|
|
148
123
|
from meerschaum.utils.warnings import warn
|
149
124
|
from meerschaum.utils.sql import sql_item_name, dateadd_str, build_where
|
150
125
|
from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
|
151
|
-
from meerschaum.utils.misc import is_int
|
152
126
|
from meerschaum.config import get_config
|
153
127
|
|
154
128
|
dt_col = pipe.columns.get('datetime', None)
|
@@ -191,7 +165,7 @@ def get_pipe_metadef(
|
|
191
165
|
else begin
|
192
166
|
)
|
193
167
|
|
194
|
-
if begin and end and begin >= end:
|
168
|
+
if begin not in (None, '') and end is not None and begin >= end:
|
195
169
|
begin = None
|
196
170
|
|
197
171
|
if dt_name:
|
@@ -203,7 +177,7 @@ def get_pipe_metadef(
|
|
203
177
|
begin=begin,
|
204
178
|
db_type=db_dt_typ,
|
205
179
|
)
|
206
|
-
if begin
|
180
|
+
if begin not in ('', None)
|
207
181
|
else None
|
208
182
|
)
|
209
183
|
end_da = (
|
@@ -214,7 +188,7 @@ def get_pipe_metadef(
|
|
214
188
|
begin=end,
|
215
189
|
db_type=db_dt_typ,
|
216
190
|
)
|
217
|
-
if end
|
191
|
+
if end is not None
|
218
192
|
else None
|
219
193
|
)
|
220
194
|
|
@@ -228,11 +202,7 @@ def get_pipe_metadef(
|
|
228
202
|
|
229
203
|
has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
|
230
204
|
if dt_name and (begin_da or end_da):
|
231
|
-
definition_dt_name =
|
232
|
-
dateadd_str(self.flavor, 'minute', 0, f"{definition_name}.{dt_name}", db_type=db_dt_typ)
|
233
|
-
if not is_int((begin_da or end_da))
|
234
|
-
else f"{definition_name}.{dt_name}"
|
235
|
-
)
|
205
|
+
definition_dt_name = f"{definition_name}.{dt_name}"
|
236
206
|
meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
|
237
207
|
has_where = True
|
238
208
|
if begin_da:
|
@@ -25,7 +25,7 @@ def _log_temporary_tables_creation(
|
|
25
25
|
"""
|
26
26
|
from meerschaum.utils.misc import items_str
|
27
27
|
from meerschaum.connectors.sql.tables import get_tables
|
28
|
-
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
28
|
+
sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
|
29
29
|
temp_tables_table = get_tables(
|
30
30
|
mrsm_instance=self,
|
31
31
|
create=create,
|
@@ -86,7 +86,7 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
|
|
86
86
|
"""
|
87
87
|
from meerschaum.utils.misc import items_str
|
88
88
|
from meerschaum.connectors.sql.tables import get_tables
|
89
|
-
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
89
|
+
sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
|
90
90
|
temp_tables_table = get_tables(
|
91
91
|
mrsm_instance=self,
|
92
92
|
create=False,
|
@@ -150,7 +150,7 @@ def _drop_old_temporary_tables(
|
|
150
150
|
"""
|
151
151
|
from meerschaum.config import get_config
|
152
152
|
from meerschaum.connectors.sql.tables import get_tables
|
153
|
-
sqlalchemy = mrsm.attempt_import('sqlalchemy')
|
153
|
+
sqlalchemy = mrsm.attempt_import('sqlalchemy', lazy=False)
|
154
154
|
temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
|
155
155
|
last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
|
156
156
|
now_ts = time.perf_counter()
|