meerschaum 2.7.5__py3-none-any.whl → 2.7.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/shell/Shell.py +4 -6
- meerschaum/_internal/shell/ShellCompleter.py +6 -5
- meerschaum/actions/clear.py +6 -3
- meerschaum/actions/copy.py +33 -27
- meerschaum/actions/drop.py +100 -22
- meerschaum/actions/index.py +71 -0
- meerschaum/actions/register.py +8 -12
- meerschaum/actions/sql.py +1 -1
- meerschaum/actions/sync.py +22 -18
- meerschaum/api/dash/pipes.py +2 -3
- meerschaum/api/routes/_pipes.py +18 -0
- meerschaum/api/routes/_plugins.py +1 -1
- meerschaum/api/routes/_users.py +62 -61
- meerschaum/config/_default.py +5 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_misc.py +3 -2
- meerschaum/connectors/api/_pipes.py +28 -9
- meerschaum/connectors/sql/_SQLConnector.py +7 -3
- meerschaum/connectors/sql/_create_engine.py +1 -1
- meerschaum/connectors/sql/_fetch.py +4 -9
- meerschaum/connectors/sql/_instance.py +3 -3
- meerschaum/connectors/sql/_pipes.py +292 -76
- meerschaum/connectors/sql/_plugins.py +11 -16
- meerschaum/connectors/sql/_sql.py +13 -9
- meerschaum/connectors/sql/_uri.py +9 -9
- meerschaum/connectors/sql/_users.py +10 -12
- meerschaum/connectors/sql/tables/__init__.py +13 -14
- meerschaum/core/Pipe/__init__.py +12 -2
- meerschaum/core/Pipe/_attributes.py +32 -38
- meerschaum/core/Pipe/_drop.py +73 -2
- meerschaum/core/Pipe/_index.py +68 -0
- meerschaum/jobs/_Job.py +1 -0
- meerschaum/plugins/__init__.py +7 -3
- meerschaum/utils/daemon/Daemon.py +5 -1
- meerschaum/utils/daemon/__init__.py +2 -2
- meerschaum/utils/dtypes/sql.py +2 -2
- meerschaum/utils/misc.py +7 -6
- meerschaum/utils/packages/__init__.py +31 -27
- meerschaum/utils/packages/_packages.py +1 -1
- meerschaum/utils/prompt.py +54 -36
- meerschaum/utils/sql.py +80 -34
- meerschaum/utils/venv/__init__.py +12 -3
- {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/METADATA +17 -5
- {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/RECORD +50 -48
- {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/WHEEL +1 -1
- {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/LICENSE +0 -0
- {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/NOTICE +0 -0
- {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/top_level.txt +0 -0
- {meerschaum-2.7.5.dist-info → meerschaum-2.7.7.dist-info}/zip-safe +0 -0
@@ -21,9 +21,8 @@ def register_plugin(
|
|
21
21
|
**kw: Any
|
22
22
|
) -> SuccessTuple:
|
23
23
|
"""Register a new plugin to the plugins table."""
|
24
|
-
from meerschaum.utils.warnings import warn, error
|
25
24
|
from meerschaum.utils.packages import attempt_import
|
26
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
25
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
27
26
|
from meerschaum.utils.sql import json_flavors
|
28
27
|
from meerschaum.connectors.sql.tables import get_tables
|
29
28
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
@@ -85,7 +84,7 @@ def get_plugin_id(
|
|
85
84
|
from meerschaum.connectors.sql.tables import get_tables
|
86
85
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
87
86
|
from meerschaum.utils.packages import attempt_import
|
88
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
87
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
89
88
|
|
90
89
|
query = (
|
91
90
|
sqlalchemy
|
@@ -95,9 +94,10 @@ def get_plugin_id(
|
|
95
94
|
|
96
95
|
try:
|
97
96
|
return int(self.value(query, debug=debug))
|
98
|
-
except Exception
|
97
|
+
except Exception:
|
99
98
|
return None
|
100
99
|
|
100
|
+
|
101
101
|
def get_plugin_version(
|
102
102
|
self,
|
103
103
|
plugin: 'mrsm.core.Plugin',
|
@@ -110,7 +110,7 @@ def get_plugin_version(
|
|
110
110
|
from meerschaum.connectors.sql.tables import get_tables
|
111
111
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
112
112
|
from meerschaum.utils.packages import attempt_import
|
113
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
113
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
114
114
|
query = sqlalchemy.select(plugins_tbl.c.version).where(plugins_tbl.c.plugin_name == plugin.name)
|
115
115
|
return self.value(query, debug=debug)
|
116
116
|
|
@@ -126,7 +126,7 @@ def get_plugin_user_id(
|
|
126
126
|
from meerschaum.connectors.sql.tables import get_tables
|
127
127
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
128
128
|
from meerschaum.utils.packages import attempt_import
|
129
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
129
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
130
130
|
|
131
131
|
query = (
|
132
132
|
sqlalchemy
|
@@ -136,7 +136,7 @@ def get_plugin_user_id(
|
|
136
136
|
|
137
137
|
try:
|
138
138
|
return int(self.value(query, debug=debug))
|
139
|
-
except Exception
|
139
|
+
except Exception:
|
140
140
|
return None
|
141
141
|
|
142
142
|
def get_plugin_username(
|
@@ -152,7 +152,7 @@ def get_plugin_username(
|
|
152
152
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
153
153
|
users = get_tables(mrsm_instance=self, debug=debug)['users']
|
154
154
|
from meerschaum.utils.packages import attempt_import
|
155
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
155
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
156
156
|
|
157
157
|
query = (
|
158
158
|
sqlalchemy.select(users.c.username)
|
@@ -177,7 +177,7 @@ def get_plugin_attributes(
|
|
177
177
|
from meerschaum.connectors.sql.tables import get_tables
|
178
178
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
179
179
|
from meerschaum.utils.packages import attempt_import
|
180
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
180
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
181
181
|
|
182
182
|
query = (
|
183
183
|
sqlalchemy
|
@@ -219,7 +219,7 @@ def get_plugins(
|
|
219
219
|
from meerschaum.connectors.sql.tables import get_tables
|
220
220
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
221
221
|
from meerschaum.utils.packages import attempt_import
|
222
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
222
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
223
223
|
|
224
224
|
query = sqlalchemy.select(plugins_tbl.c.plugin_name)
|
225
225
|
if user_id is not None:
|
@@ -246,9 +246,8 @@ def delete_plugin(
|
|
246
246
|
**kw: Any
|
247
247
|
) -> SuccessTuple:
|
248
248
|
"""Delete a plugin from the plugins table."""
|
249
|
-
from meerschaum.utils.warnings import warn, error
|
250
249
|
from meerschaum.utils.packages import attempt_import
|
251
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
250
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
252
251
|
from meerschaum.connectors.sql.tables import get_tables
|
253
252
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
254
253
|
|
@@ -256,10 +255,6 @@ def delete_plugin(
|
|
256
255
|
if plugin_id is None:
|
257
256
|
return True, f"Plugin '{plugin}' was not registered."
|
258
257
|
|
259
|
-
bind_variables = {
|
260
|
-
'plugin_id' : plugin_id,
|
261
|
-
}
|
262
|
-
|
263
258
|
query = sqlalchemy.delete(plugins_tbl).where(plugins_tbl.c.plugin_id == plugin_id)
|
264
259
|
result = self.exec(query, debug=debug)
|
265
260
|
if result is None:
|
@@ -154,7 +154,7 @@ def read(
|
|
154
154
|
dtype[col] = 'datetime64[ns]'
|
155
155
|
|
156
156
|
pool = get_pool(workers=workers)
|
157
|
-
sqlalchemy = attempt_import("sqlalchemy")
|
157
|
+
sqlalchemy = attempt_import("sqlalchemy", lazy=False)
|
158
158
|
default_chunksize = self._sys_config.get('chunksize', None)
|
159
159
|
chunksize = chunksize if chunksize != -1 else default_chunksize
|
160
160
|
if chunksize is None and as_iterator:
|
@@ -443,7 +443,6 @@ def value(
|
|
443
443
|
|
444
444
|
"""
|
445
445
|
from meerschaum.utils.packages import attempt_import
|
446
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
447
446
|
if self.flavor == 'duckdb':
|
448
447
|
use_pandas = True
|
449
448
|
if use_pandas:
|
@@ -455,9 +454,6 @@ def value(
|
|
455
454
|
_close = kw.get('close', True)
|
456
455
|
_commit = kw.get('commit', (self.flavor != 'mssql'))
|
457
456
|
|
458
|
-
# _close = True
|
459
|
-
# _commit = True
|
460
|
-
|
461
457
|
try:
|
462
458
|
result, connection = self.exec(
|
463
459
|
query,
|
@@ -556,7 +552,7 @@ def exec(
|
|
556
552
|
)
|
557
553
|
|
558
554
|
from meerschaum.utils.packages import attempt_import
|
559
|
-
sqlalchemy = attempt_import("sqlalchemy")
|
555
|
+
sqlalchemy = attempt_import("sqlalchemy", lazy=False)
|
560
556
|
if debug:
|
561
557
|
dprint(f"[{self}] Executing query:\n{query}")
|
562
558
|
|
@@ -659,7 +655,7 @@ def exec_queries(
|
|
659
655
|
from meerschaum.utils.warnings import warn
|
660
656
|
from meerschaum.utils.debug import dprint
|
661
657
|
from meerschaum.utils.packages import attempt_import
|
662
|
-
sqlalchemy, sqlalchemy_orm = attempt_import('sqlalchemy', 'sqlalchemy.orm')
|
658
|
+
sqlalchemy, sqlalchemy_orm = attempt_import('sqlalchemy', 'sqlalchemy.orm', lazy=False)
|
663
659
|
session = sqlalchemy_orm.Session(self.engine)
|
664
660
|
|
665
661
|
result = None
|
@@ -778,6 +774,7 @@ def to_sql(
|
|
778
774
|
import time
|
779
775
|
import json
|
780
776
|
from decimal import Decimal
|
777
|
+
from datetime import timedelta
|
781
778
|
from meerschaum.utils.warnings import error, warn
|
782
779
|
import warnings
|
783
780
|
import functools
|
@@ -816,9 +813,10 @@ def to_sql(
|
|
816
813
|
PD_TO_SQLALCHEMY_DTYPES_FLAVORS,
|
817
814
|
get_db_type_from_pd_type,
|
818
815
|
)
|
816
|
+
from meerschaum.utils.misc import interval_str
|
819
817
|
from meerschaum.connectors.sql._create_engine import flavor_configs
|
820
818
|
from meerschaum.utils.packages import attempt_import, import_pandas
|
821
|
-
sqlalchemy = attempt_import('sqlalchemy', debug=debug)
|
819
|
+
sqlalchemy = attempt_import('sqlalchemy', debug=debug, lazy=False)
|
822
820
|
pd = import_pandas()
|
823
821
|
is_dask = 'dask' in df.__module__
|
824
822
|
|
@@ -998,7 +996,13 @@ def to_sql(
|
|
998
996
|
|
999
997
|
end = time.perf_counter()
|
1000
998
|
if success:
|
1001
|
-
|
999
|
+
num_rows = len(df)
|
1000
|
+
msg = (
|
1001
|
+
f"It took {interval_str(timedelta(seconds=(end - start)))} "
|
1002
|
+
+ f"to sync {num_rows:,} row"
|
1003
|
+
+ ('s' if num_rows != 1 else '')
|
1004
|
+
+ f" to {name}."
|
1005
|
+
)
|
1002
1006
|
stats['start'] = start
|
1003
1007
|
stats['end'] = end
|
1004
1008
|
stats['duration'] = end - start
|
@@ -13,14 +13,14 @@ from meerschaum.utils.packages import attempt_import
|
|
13
13
|
|
14
14
|
@classmethod
|
15
15
|
def from_uri(
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
16
|
+
cls,
|
17
|
+
uri: str,
|
18
|
+
label: Optional[str] = None,
|
19
|
+
as_dict: bool = False,
|
20
|
+
) -> Union[
|
21
|
+
'meerschaum.connectors.SQLConnector',
|
22
|
+
Dict[str, Union[str, int]],
|
23
|
+
]:
|
24
24
|
"""
|
25
25
|
Create a new SQLConnector from a URI string.
|
26
26
|
|
@@ -97,7 +97,7 @@ def parse_uri(uri: str) -> Dict[str, Any]:
|
|
97
97
|
>>>
|
98
98
|
"""
|
99
99
|
from urllib.parse import parse_qs, urlparse
|
100
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
100
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
101
101
|
parser = sqlalchemy.engine.url.make_url
|
102
102
|
params = parser(uri).translate_connect_args()
|
103
103
|
params['flavor'] = uri.split(':')[0].split('+')[0]
|
@@ -19,10 +19,9 @@ def register_user(
|
|
19
19
|
**kw: Any
|
20
20
|
) -> SuccessTuple:
|
21
21
|
"""Register a new user."""
|
22
|
-
from meerschaum.utils.warnings import warn, error, info
|
23
22
|
from meerschaum.utils.packages import attempt_import
|
24
23
|
from meerschaum.utils.sql import json_flavors
|
25
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
24
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
26
25
|
|
27
26
|
valid_tuple = valid_username(user.username)
|
28
27
|
if not valid_tuple[0]:
|
@@ -103,9 +102,8 @@ def edit_user(
|
|
103
102
|
) -> SuccessTuple:
|
104
103
|
"""Update an existing user's metadata."""
|
105
104
|
from meerschaum.utils.packages import attempt_import
|
106
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
105
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
107
106
|
from meerschaum.connectors.sql.tables import get_tables
|
108
|
-
from meerschaum.utils.sql import json_flavors
|
109
107
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
110
108
|
|
111
109
|
user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
|
@@ -158,7 +156,7 @@ def get_user_id(
|
|
158
156
|
"""If a user is registered, return the `user_id`."""
|
159
157
|
### ensure users table exists
|
160
158
|
from meerschaum.utils.packages import attempt_import
|
161
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
159
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
162
160
|
from meerschaum.connectors.sql.tables import get_tables
|
163
161
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
164
162
|
|
@@ -183,7 +181,7 @@ def get_user_attributes(
|
|
183
181
|
### ensure users table exists
|
184
182
|
from meerschaum.utils.warnings import warn
|
185
183
|
from meerschaum.utils.packages import attempt_import
|
186
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
184
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
187
185
|
from meerschaum.connectors.sql.tables import get_tables
|
188
186
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
189
187
|
|
@@ -199,14 +197,14 @@ def get_user_attributes(
|
|
199
197
|
try:
|
200
198
|
result = dict(result)
|
201
199
|
_parsed = True
|
202
|
-
except Exception
|
200
|
+
except Exception:
|
203
201
|
_parsed = False
|
204
202
|
if not _parsed:
|
205
203
|
try:
|
206
204
|
import json
|
207
205
|
result = json.loads(result)
|
208
206
|
_parsed = True
|
209
|
-
except Exception
|
207
|
+
except Exception:
|
210
208
|
_parsed = False
|
211
209
|
if not _parsed:
|
212
210
|
warn(f"Received unexpected type for attributes: {result}")
|
@@ -223,7 +221,7 @@ def delete_user(
|
|
223
221
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
224
222
|
plugins = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
225
223
|
from meerschaum.utils.packages import attempt_import
|
226
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
224
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
227
225
|
|
228
226
|
user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
|
229
227
|
|
@@ -256,7 +254,7 @@ def get_users(
|
|
256
254
|
from meerschaum.connectors.sql.tables import get_tables
|
257
255
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
258
256
|
from meerschaum.utils.packages import attempt_import
|
259
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
257
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
260
258
|
|
261
259
|
query = sqlalchemy.select(users_tbl.c.username)
|
262
260
|
|
@@ -277,7 +275,7 @@ def get_user_password_hash(
|
|
277
275
|
from meerschaum.connectors.sql.tables import get_tables
|
278
276
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
279
277
|
from meerschaum.utils.packages import attempt_import
|
280
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
278
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
281
279
|
|
282
280
|
if user.user_id is not None:
|
283
281
|
user_id = user.user_id
|
@@ -308,7 +306,7 @@ def get_user_type(
|
|
308
306
|
from meerschaum.connectors.sql.tables import get_tables
|
309
307
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
310
308
|
from meerschaum.utils.packages import attempt_import
|
311
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
309
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
312
310
|
|
313
311
|
user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
|
314
312
|
|
@@ -17,10 +17,10 @@ _sequence_flavors = {'duckdb', 'oracle'}
|
|
17
17
|
_skip_index_names_flavors = {'mssql',}
|
18
18
|
|
19
19
|
def get_tables(
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
20
|
+
mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
|
21
|
+
create: bool = True,
|
22
|
+
debug: Optional[bool] = None
|
23
|
+
) -> Union[Dict[str, 'sqlalchemy.Table'], bool]:
|
24
24
|
"""
|
25
25
|
Create tables on the database and return the `sqlalchemy` tables.
|
26
26
|
|
@@ -51,7 +51,7 @@ def get_tables(
|
|
51
51
|
sqlalchemy, sqlalchemy_dialects_postgresql = attempt_import(
|
52
52
|
'sqlalchemy',
|
53
53
|
'sqlalchemy.dialects.postgresql',
|
54
|
-
lazy
|
54
|
+
lazy=False,
|
55
55
|
)
|
56
56
|
if not sqlalchemy:
|
57
57
|
error(f"Failed to import sqlalchemy. Is sqlalchemy installed?")
|
@@ -205,13 +205,12 @@ def get_tables(
|
|
205
205
|
|
206
206
|
|
207
207
|
def create_tables(
|
208
|
-
|
209
|
-
|
210
|
-
|
208
|
+
conn: 'meerschaum.connectors.SQLConnector',
|
209
|
+
tables: Optional[Dict[str, 'sqlalchemy.Table']] = None,
|
210
|
+
) -> bool:
|
211
211
|
"""
|
212
212
|
Create the tables on the database.
|
213
213
|
"""
|
214
|
-
from meerschaum.utils.sql import get_rename_table_queries, table_exists
|
215
214
|
_tables = tables if tables is not None else get_tables(conn)
|
216
215
|
|
217
216
|
try:
|
@@ -225,10 +224,10 @@ def create_tables(
|
|
225
224
|
|
226
225
|
|
227
226
|
def create_schemas(
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
227
|
+
conn: 'meerschaum.connectors.SQLConnector',
|
228
|
+
schemas: List[str],
|
229
|
+
debug: bool = False,
|
230
|
+
) -> bool:
|
232
231
|
"""
|
233
232
|
Create the internal Meerschaum schema on the database.
|
234
233
|
"""
|
@@ -238,7 +237,7 @@ def create_schemas(
|
|
238
237
|
if conn.flavor in NO_SCHEMA_FLAVORS:
|
239
238
|
return True
|
240
239
|
|
241
|
-
|
240
|
+
_ = attempt_import('sqlalchemy.schema', lazy=False)
|
242
241
|
successes = {}
|
243
242
|
skip_if_not_exists = conn.flavor in SKIP_IF_EXISTS_FLAVORS
|
244
243
|
if_not_exists_str = ("IF NOT EXISTS " if not skip_if_not_exists else "")
|
meerschaum/core/Pipe/__init__.py
CHANGED
@@ -107,6 +107,7 @@ class Pipe:
|
|
107
107
|
static,
|
108
108
|
tzinfo,
|
109
109
|
enforce,
|
110
|
+
null_indices,
|
110
111
|
get_columns,
|
111
112
|
get_columns_types,
|
112
113
|
get_columns_indices,
|
@@ -141,7 +142,8 @@ class Pipe:
|
|
141
142
|
get_bound_time,
|
142
143
|
)
|
143
144
|
from ._delete import delete
|
144
|
-
from ._drop import drop
|
145
|
+
from ._drop import drop, drop_indices
|
146
|
+
from ._index import create_indices
|
145
147
|
from ._clear import clear
|
146
148
|
from ._deduplicate import deduplicate
|
147
149
|
from ._bootstrap import bootstrap
|
@@ -165,6 +167,7 @@ class Pipe:
|
|
165
167
|
autoincrement: Optional[bool] = None,
|
166
168
|
static: Optional[bool] = None,
|
167
169
|
enforce: Optional[bool] = None,
|
170
|
+
null_indices: Optional[bool] = None,
|
168
171
|
mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
|
169
172
|
cache: bool = False,
|
170
173
|
debug: bool = False,
|
@@ -223,10 +226,14 @@ class Pipe:
|
|
223
226
|
static: Optional[bool], default None
|
224
227
|
If `True`, set `static` in the parameters.
|
225
228
|
|
226
|
-
enforce:
|
229
|
+
enforce: Optional[bool], default None
|
227
230
|
If `False`, skip data type enforcement.
|
228
231
|
Default behavior is `True`.
|
229
232
|
|
233
|
+
null_indices: Optional[bool], default None
|
234
|
+
Set to `False` if there will be no null values in the index columns.
|
235
|
+
Defaults to `True`.
|
236
|
+
|
230
237
|
temporary: bool, default False
|
231
238
|
If `True`, prevent instance tables (pipes, users, plugins) from being created.
|
232
239
|
|
@@ -330,6 +337,9 @@ class Pipe:
|
|
330
337
|
if isinstance(enforce, bool):
|
331
338
|
self._attributes['parameters']['enforce'] = enforce
|
332
339
|
|
340
|
+
if isinstance(null_indices, bool):
|
341
|
+
self._attributes['parameters']['null_indices'] = null_indices
|
342
|
+
|
333
343
|
### NOTE: The parameters dictionary is {} by default.
|
334
344
|
### A Pipe may be registered without parameters, then edited,
|
335
345
|
### or a Pipe may be registered with parameters set in-memory first.
|
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|
11
11
|
from datetime import timezone
|
12
12
|
|
13
13
|
import meerschaum as mrsm
|
14
|
-
from meerschaum.utils.typing import Tuple, Dict,
|
14
|
+
from meerschaum.utils.typing import Tuple, Dict, Any, Union, Optional, List
|
15
15
|
from meerschaum.utils.warnings import warn
|
16
16
|
|
17
17
|
|
@@ -313,6 +313,25 @@ def enforce(self, _enforce: bool) -> None:
|
|
313
313
|
self.parameters['enforce'] = _enforce
|
314
314
|
|
315
315
|
|
316
|
+
@property
|
317
|
+
def null_indices(self) -> bool:
|
318
|
+
"""
|
319
|
+
Return the `null_indices` parameter for the pipe.
|
320
|
+
"""
|
321
|
+
if 'null_indices' not in self.parameters:
|
322
|
+
self.parameters['null_indices'] = True
|
323
|
+
|
324
|
+
return self.parameters['null_indices']
|
325
|
+
|
326
|
+
|
327
|
+
@null_indices.setter
|
328
|
+
def null_indices(self, _null_indices: bool) -> None:
|
329
|
+
"""
|
330
|
+
Set the `null_indices` parameter for the pipe.
|
331
|
+
"""
|
332
|
+
self.parameters['null_indices'] = _null_indices
|
333
|
+
|
334
|
+
|
316
335
|
def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
|
317
336
|
"""
|
318
337
|
Check if the requested columns are defined.
|
@@ -469,7 +488,7 @@ def get_columns_indices(
|
|
469
488
|
|
470
489
|
self.__dict__['_columns_indices'] = _columns_indices
|
471
490
|
self.__dict__['_columns_indices_timestamp'] = now
|
472
|
-
return _columns_indices or {}
|
491
|
+
return {k: v for k, v in _columns_indices.items() if k and v} or {}
|
473
492
|
|
474
493
|
|
475
494
|
def get_id(self, **kw: Any) -> Union[int, None]:
|
@@ -711,42 +730,17 @@ def guess_datetime(self) -> Union[str, None]:
|
|
711
730
|
|
712
731
|
def get_indices(self) -> Dict[str, str]:
|
713
732
|
"""
|
714
|
-
Return a dictionary mapping index keys to their names
|
733
|
+
Return a dictionary mapping index keys to their names in the database.
|
715
734
|
|
716
735
|
Returns
|
717
736
|
-------
|
718
|
-
A dictionary of index keys to
|
719
|
-
"""
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
else str(cols)
|
729
|
-
)
|
730
|
-
for ix, cols in _indices.items()
|
731
|
-
if cols
|
732
|
-
}
|
733
|
-
_index_names = {
|
734
|
-
ix: _index_template.format(
|
735
|
-
target=_target,
|
736
|
-
column_names=column_names,
|
737
|
-
connector_keys=self.connector_keys,
|
738
|
-
metric_key=self.connector_key,
|
739
|
-
location_key=self.location_key,
|
740
|
-
)
|
741
|
-
for ix, column_names in _column_names.items()
|
742
|
-
}
|
743
|
-
### NOTE: Skip any duplicate indices.
|
744
|
-
seen_index_names = {}
|
745
|
-
for ix, index_name in _index_names.items():
|
746
|
-
if index_name in seen_index_names:
|
747
|
-
continue
|
748
|
-
seen_index_names[index_name] = ix
|
749
|
-
return {
|
750
|
-
ix: index_name
|
751
|
-
for index_name, ix in seen_index_names.items()
|
752
|
-
}
|
737
|
+
A dictionary of index keys to index names.
|
738
|
+
"""
|
739
|
+
from meerschaum.connectors import get_connector_plugin
|
740
|
+
with mrsm.Venv(get_connector_plugin(self.instance_connector)):
|
741
|
+
if hasattr(self.instance_connector, 'get_pipe_index_names'):
|
742
|
+
result = self.instance_connector.get_pipe_index_names(self)
|
743
|
+
else:
|
744
|
+
result = {}
|
745
|
+
|
746
|
+
return result
|
meerschaum/core/Pipe/_drop.py
CHANGED
@@ -7,7 +7,7 @@ Drop a Pipe's table but keep its registration
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
from meerschaum.utils.typing import SuccessTuple, Any
|
10
|
+
from meerschaum.utils.typing import SuccessTuple, Any, Optional, List
|
11
11
|
|
12
12
|
|
13
13
|
def drop(
|
@@ -39,9 +39,80 @@ def drop(
|
|
39
39
|
warn(_drop_cache_tuple[1])
|
40
40
|
|
41
41
|
with Venv(get_connector_plugin(self.instance_connector)):
|
42
|
-
|
42
|
+
if hasattr(self.instance_connector, 'drop_pipe'):
|
43
|
+
result = self.instance_connector.drop_pipe(self, debug=debug, **kw)
|
44
|
+
else:
|
45
|
+
result = (
|
46
|
+
False,
|
47
|
+
(
|
48
|
+
"Cannot drop pipes for instance connectors of type "
|
49
|
+
f"'{self.instance_connector.type}'."
|
50
|
+
)
|
51
|
+
)
|
52
|
+
|
43
53
|
|
44
54
|
_ = self.__dict__.pop('_exists', None)
|
45
55
|
_ = self.__dict__.pop('_exists_timestamp', None)
|
46
56
|
|
47
57
|
return result
|
58
|
+
|
59
|
+
|
60
|
+
def drop_indices(
|
61
|
+
self,
|
62
|
+
columns: Optional[List[str]] = None,
|
63
|
+
debug: bool = False,
|
64
|
+
**kw: Any
|
65
|
+
) -> SuccessTuple:
|
66
|
+
"""
|
67
|
+
Call the Pipe's instance connector's `drop_indices()` method.
|
68
|
+
|
69
|
+
Parameters
|
70
|
+
----------
|
71
|
+
columns: Optional[List[str]] = None
|
72
|
+
If provided, only drop indices in the given list.
|
73
|
+
|
74
|
+
debug: bool, default False:
|
75
|
+
Verbosity toggle.
|
76
|
+
|
77
|
+
Returns
|
78
|
+
-------
|
79
|
+
A `SuccessTuple` of success, message.
|
80
|
+
|
81
|
+
"""
|
82
|
+
from meerschaum.utils.warnings import warn
|
83
|
+
from meerschaum.utils.venv import Venv
|
84
|
+
from meerschaum.connectors import get_connector_plugin
|
85
|
+
|
86
|
+
_ = self.__dict__.pop('_columns_indices', None)
|
87
|
+
_ = self.__dict__.pop('_columns_indices_timestamp', None)
|
88
|
+
_ = self.__dict__.pop('_columns_types_timestamp', None)
|
89
|
+
_ = self.__dict__.pop('_columns_types', None)
|
90
|
+
|
91
|
+
if self.cache_pipe is not None:
|
92
|
+
_drop_cache_tuple = self.cache_pipe.drop_indices(columns=columns, debug=debug, **kw)
|
93
|
+
if not _drop_cache_tuple[0]:
|
94
|
+
warn(_drop_cache_tuple[1])
|
95
|
+
|
96
|
+
with Venv(get_connector_plugin(self.instance_connector)):
|
97
|
+
if hasattr(self.instance_connector, 'drop_pipe_indices'):
|
98
|
+
result = self.instance_connector.drop_pipe_indices(
|
99
|
+
self,
|
100
|
+
columns=columns,
|
101
|
+
debug=debug,
|
102
|
+
**kw
|
103
|
+
)
|
104
|
+
else:
|
105
|
+
result = (
|
106
|
+
False,
|
107
|
+
(
|
108
|
+
"Cannot drop indices for instance connectors of type "
|
109
|
+
f"'{self.instance_connector.type}'."
|
110
|
+
)
|
111
|
+
)
|
112
|
+
|
113
|
+
_ = self.__dict__.pop('_columns_indices', None)
|
114
|
+
_ = self.__dict__.pop('_columns_indices_timestamp', None)
|
115
|
+
_ = self.__dict__.pop('_columns_types_timestamp', None)
|
116
|
+
_ = self.__dict__.pop('_columns_types', None)
|
117
|
+
|
118
|
+
return result
|
@@ -0,0 +1,68 @@
|
|
1
|
+
#! /usr/bin/env python
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# vim:fenc=utf-8
|
4
|
+
|
5
|
+
"""
|
6
|
+
Index a pipe's table.
|
7
|
+
"""
|
8
|
+
|
9
|
+
from __future__ import annotations
|
10
|
+
from meerschaum.utils.typing import SuccessTuple, Any, Optional, List
|
11
|
+
|
12
|
+
|
13
|
+
def create_indices(
|
14
|
+
self,
|
15
|
+
columns: Optional[List[str]] = None,
|
16
|
+
debug: bool = False,
|
17
|
+
**kw: Any
|
18
|
+
) -> SuccessTuple:
|
19
|
+
"""
|
20
|
+
Call the Pipe's instance connector's `create_pipe_indices()` method.
|
21
|
+
|
22
|
+
Parameters
|
23
|
+
----------
|
24
|
+
debug: bool, default False:
|
25
|
+
Verbosity toggle.
|
26
|
+
|
27
|
+
Returns
|
28
|
+
-------
|
29
|
+
A `SuccessTuple` of success, message.
|
30
|
+
|
31
|
+
"""
|
32
|
+
from meerschaum.utils.warnings import warn
|
33
|
+
from meerschaum.utils.venv import Venv
|
34
|
+
from meerschaum.connectors import get_connector_plugin
|
35
|
+
|
36
|
+
_ = self.__dict__.pop('_columns_indices', None)
|
37
|
+
_ = self.__dict__.pop('_columns_indices_timestamp', None)
|
38
|
+
_ = self.__dict__.pop('_columns_types_timestamp', None)
|
39
|
+
_ = self.__dict__.pop('_columns_types', None)
|
40
|
+
|
41
|
+
if self.cache_pipe is not None:
|
42
|
+
cache_success, cache_msg = self.cache_pipe.index(columns=columns, debug=debug, **kw)
|
43
|
+
if not cache_success:
|
44
|
+
warn(cache_msg)
|
45
|
+
|
46
|
+
with Venv(get_connector_plugin(self.instance_connector)):
|
47
|
+
if hasattr(self.instance_connector, 'create_pipe_indices'):
|
48
|
+
result = self.instance_connector.create_pipe_indices(
|
49
|
+
self,
|
50
|
+
columns=columns,
|
51
|
+
debug=debug,
|
52
|
+
**kw
|
53
|
+
)
|
54
|
+
else:
|
55
|
+
result = (
|
56
|
+
False,
|
57
|
+
(
|
58
|
+
"Cannot create indices for instance connectors of type "
|
59
|
+
f"'{self.instance_connector.type}'."
|
60
|
+
)
|
61
|
+
)
|
62
|
+
|
63
|
+
_ = self.__dict__.pop('_columns_indices', None)
|
64
|
+
_ = self.__dict__.pop('_columns_indices_timestamp', None)
|
65
|
+
_ = self.__dict__.pop('_columns_types_timestamp', None)
|
66
|
+
_ = self.__dict__.pop('_columns_types', None)
|
67
|
+
|
68
|
+
return result
|