meerschaum 2.7.6__py3-none-any.whl → 2.7.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/actions/copy.py +1 -0
- meerschaum/actions/drop.py +100 -22
- meerschaum/actions/index.py +71 -0
- meerschaum/actions/register.py +8 -12
- meerschaum/actions/sql.py +1 -1
- meerschaum/api/routes/_pipes.py +18 -0
- meerschaum/api/routes/_plugins.py +1 -1
- meerschaum/api/routes/_users.py +62 -61
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_pipes.py +20 -0
- meerschaum/connectors/sql/_SQLConnector.py +8 -12
- meerschaum/connectors/sql/_create_engine.py +1 -1
- meerschaum/connectors/sql/_fetch.py +9 -39
- meerschaum/connectors/sql/_instance.py +3 -3
- meerschaum/connectors/sql/_pipes.py +262 -70
- meerschaum/connectors/sql/_plugins.py +11 -16
- meerschaum/connectors/sql/_sql.py +60 -39
- meerschaum/connectors/sql/_uri.py +9 -9
- meerschaum/connectors/sql/_users.py +10 -12
- meerschaum/connectors/sql/tables/__init__.py +13 -14
- meerschaum/connectors/valkey/_ValkeyConnector.py +2 -2
- meerschaum/core/Pipe/__init__.py +12 -2
- meerschaum/core/Pipe/_attributes.py +32 -38
- meerschaum/core/Pipe/_drop.py +73 -2
- meerschaum/core/Pipe/_fetch.py +4 -0
- meerschaum/core/Pipe/_index.py +68 -0
- meerschaum/core/Pipe/_sync.py +16 -9
- meerschaum/utils/daemon/Daemon.py +9 -2
- meerschaum/utils/daemon/RotatingFile.py +3 -3
- meerschaum/utils/dataframe.py +42 -12
- meerschaum/utils/dtypes/__init__.py +144 -24
- meerschaum/utils/dtypes/sql.py +52 -9
- meerschaum/utils/formatting/__init__.py +2 -2
- meerschaum/utils/formatting/_pprint.py +12 -11
- meerschaum/utils/misc.py +16 -18
- meerschaum/utils/prompt.py +1 -1
- meerschaum/utils/sql.py +106 -42
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/METADATA +14 -2
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/RECORD +45 -43
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/WHEEL +1 -1
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/LICENSE +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/NOTICE +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/top_level.txt +0 -0
- {meerschaum-2.7.6.dist-info → meerschaum-2.7.8.dist-info}/zip-safe +0 -0
@@ -21,9 +21,8 @@ def register_plugin(
|
|
21
21
|
**kw: Any
|
22
22
|
) -> SuccessTuple:
|
23
23
|
"""Register a new plugin to the plugins table."""
|
24
|
-
from meerschaum.utils.warnings import warn, error
|
25
24
|
from meerschaum.utils.packages import attempt_import
|
26
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
25
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
27
26
|
from meerschaum.utils.sql import json_flavors
|
28
27
|
from meerschaum.connectors.sql.tables import get_tables
|
29
28
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
@@ -85,7 +84,7 @@ def get_plugin_id(
|
|
85
84
|
from meerschaum.connectors.sql.tables import get_tables
|
86
85
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
87
86
|
from meerschaum.utils.packages import attempt_import
|
88
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
87
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
89
88
|
|
90
89
|
query = (
|
91
90
|
sqlalchemy
|
@@ -95,9 +94,10 @@ def get_plugin_id(
|
|
95
94
|
|
96
95
|
try:
|
97
96
|
return int(self.value(query, debug=debug))
|
98
|
-
except Exception
|
97
|
+
except Exception:
|
99
98
|
return None
|
100
99
|
|
100
|
+
|
101
101
|
def get_plugin_version(
|
102
102
|
self,
|
103
103
|
plugin: 'mrsm.core.Plugin',
|
@@ -110,7 +110,7 @@ def get_plugin_version(
|
|
110
110
|
from meerschaum.connectors.sql.tables import get_tables
|
111
111
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
112
112
|
from meerschaum.utils.packages import attempt_import
|
113
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
113
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
114
114
|
query = sqlalchemy.select(plugins_tbl.c.version).where(plugins_tbl.c.plugin_name == plugin.name)
|
115
115
|
return self.value(query, debug=debug)
|
116
116
|
|
@@ -126,7 +126,7 @@ def get_plugin_user_id(
|
|
126
126
|
from meerschaum.connectors.sql.tables import get_tables
|
127
127
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
128
128
|
from meerschaum.utils.packages import attempt_import
|
129
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
129
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
130
130
|
|
131
131
|
query = (
|
132
132
|
sqlalchemy
|
@@ -136,7 +136,7 @@ def get_plugin_user_id(
|
|
136
136
|
|
137
137
|
try:
|
138
138
|
return int(self.value(query, debug=debug))
|
139
|
-
except Exception
|
139
|
+
except Exception:
|
140
140
|
return None
|
141
141
|
|
142
142
|
def get_plugin_username(
|
@@ -152,7 +152,7 @@ def get_plugin_username(
|
|
152
152
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
153
153
|
users = get_tables(mrsm_instance=self, debug=debug)['users']
|
154
154
|
from meerschaum.utils.packages import attempt_import
|
155
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
155
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
156
156
|
|
157
157
|
query = (
|
158
158
|
sqlalchemy.select(users.c.username)
|
@@ -177,7 +177,7 @@ def get_plugin_attributes(
|
|
177
177
|
from meerschaum.connectors.sql.tables import get_tables
|
178
178
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
179
179
|
from meerschaum.utils.packages import attempt_import
|
180
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
180
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
181
181
|
|
182
182
|
query = (
|
183
183
|
sqlalchemy
|
@@ -219,7 +219,7 @@ def get_plugins(
|
|
219
219
|
from meerschaum.connectors.sql.tables import get_tables
|
220
220
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
221
221
|
from meerschaum.utils.packages import attempt_import
|
222
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
222
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
223
223
|
|
224
224
|
query = sqlalchemy.select(plugins_tbl.c.plugin_name)
|
225
225
|
if user_id is not None:
|
@@ -246,9 +246,8 @@ def delete_plugin(
|
|
246
246
|
**kw: Any
|
247
247
|
) -> SuccessTuple:
|
248
248
|
"""Delete a plugin from the plugins table."""
|
249
|
-
from meerschaum.utils.warnings import warn, error
|
250
249
|
from meerschaum.utils.packages import attempt_import
|
251
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
250
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
252
251
|
from meerschaum.connectors.sql.tables import get_tables
|
253
252
|
plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
254
253
|
|
@@ -256,10 +255,6 @@ def delete_plugin(
|
|
256
255
|
if plugin_id is None:
|
257
256
|
return True, f"Plugin '{plugin}' was not registered."
|
258
257
|
|
259
|
-
bind_variables = {
|
260
|
-
'plugin_id' : plugin_id,
|
261
|
-
}
|
262
|
-
|
263
258
|
query = sqlalchemy.delete(plugins_tbl).where(plugins_tbl.c.plugin_id == plugin_id)
|
264
259
|
result = self.exec(query, debug=debug)
|
265
260
|
if result is None:
|
@@ -126,7 +126,7 @@ def read(
|
|
126
126
|
return []
|
127
127
|
from meerschaum.utils.sql import sql_item_name, truncate_item_name
|
128
128
|
from meerschaum.utils.dtypes import are_dtypes_equal, coerce_timezone
|
129
|
-
from meerschaum.utils.dtypes.sql import
|
129
|
+
from meerschaum.utils.dtypes.sql import TIMEZONE_NAIVE_FLAVORS
|
130
130
|
from meerschaum.utils.packages import attempt_import, import_pandas
|
131
131
|
from meerschaum.utils.pool import get_pool
|
132
132
|
from meerschaum.utils.dataframe import chunksize_to_npartitions, get_numeric_cols
|
@@ -154,7 +154,7 @@ def read(
|
|
154
154
|
dtype[col] = 'datetime64[ns]'
|
155
155
|
|
156
156
|
pool = get_pool(workers=workers)
|
157
|
-
sqlalchemy = attempt_import("sqlalchemy")
|
157
|
+
sqlalchemy = attempt_import("sqlalchemy", lazy=False)
|
158
158
|
default_chunksize = self._sys_config.get('chunksize', None)
|
159
159
|
chunksize = chunksize if chunksize != -1 else default_chunksize
|
160
160
|
if chunksize is None and as_iterator:
|
@@ -443,7 +443,6 @@ def value(
|
|
443
443
|
|
444
444
|
"""
|
445
445
|
from meerschaum.utils.packages import attempt_import
|
446
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
447
446
|
if self.flavor == 'duckdb':
|
448
447
|
use_pandas = True
|
449
448
|
if use_pandas:
|
@@ -455,9 +454,6 @@ def value(
|
|
455
454
|
_close = kw.get('close', True)
|
456
455
|
_commit = kw.get('commit', (self.flavor != 'mssql'))
|
457
456
|
|
458
|
-
# _close = True
|
459
|
-
# _commit = True
|
460
|
-
|
461
457
|
try:
|
462
458
|
result, connection = self.exec(
|
463
459
|
query,
|
@@ -556,7 +552,7 @@ def exec(
|
|
556
552
|
)
|
557
553
|
|
558
554
|
from meerschaum.utils.packages import attempt_import
|
559
|
-
sqlalchemy = attempt_import("sqlalchemy")
|
555
|
+
sqlalchemy = attempt_import("sqlalchemy", lazy=False)
|
560
556
|
if debug:
|
561
557
|
dprint(f"[{self}] Executing query:\n{query}")
|
562
558
|
|
@@ -659,7 +655,7 @@ def exec_queries(
|
|
659
655
|
from meerschaum.utils.warnings import warn
|
660
656
|
from meerschaum.utils.debug import dprint
|
661
657
|
from meerschaum.utils.packages import attempt_import
|
662
|
-
sqlalchemy, sqlalchemy_orm = attempt_import('sqlalchemy', 'sqlalchemy.orm')
|
658
|
+
sqlalchemy, sqlalchemy_orm = attempt_import('sqlalchemy', 'sqlalchemy.orm', lazy=False)
|
663
659
|
session = sqlalchemy_orm.Session(self.engine)
|
664
660
|
|
665
661
|
result = None
|
@@ -806,26 +802,37 @@ def to_sql(
|
|
806
802
|
)
|
807
803
|
from meerschaum.utils.dtypes import (
|
808
804
|
are_dtypes_equal,
|
809
|
-
quantize_decimal,
|
810
805
|
coerce_timezone,
|
811
806
|
encode_bytes_for_bytea,
|
812
807
|
serialize_bytes,
|
808
|
+
serialize_decimal,
|
809
|
+
json_serialize_value,
|
813
810
|
)
|
814
811
|
from meerschaum.utils.dtypes.sql import (
|
815
|
-
NUMERIC_PRECISION_FLAVORS,
|
816
|
-
NUMERIC_AS_TEXT_FLAVORS,
|
817
812
|
PD_TO_SQLALCHEMY_DTYPES_FLAVORS,
|
818
813
|
get_db_type_from_pd_type,
|
814
|
+
get_pd_type_from_db_type,
|
815
|
+
get_numeric_precision_scale,
|
819
816
|
)
|
820
817
|
from meerschaum.utils.misc import interval_str
|
821
818
|
from meerschaum.connectors.sql._create_engine import flavor_configs
|
822
819
|
from meerschaum.utils.packages import attempt_import, import_pandas
|
823
|
-
sqlalchemy = attempt_import('sqlalchemy', debug=debug)
|
820
|
+
sqlalchemy = attempt_import('sqlalchemy', debug=debug, lazy=False)
|
824
821
|
pd = import_pandas()
|
825
822
|
is_dask = 'dask' in df.__module__
|
826
823
|
|
827
824
|
bytes_cols = get_bytes_cols(df)
|
828
825
|
numeric_cols = get_numeric_cols(df)
|
826
|
+
numeric_cols_dtypes = {
|
827
|
+
col: typ
|
828
|
+
for col, typ in kw.get('dtype', {}).items()
|
829
|
+
if (
|
830
|
+
col in df.columns
|
831
|
+
and 'numeric' in str(typ).lower()
|
832
|
+
)
|
833
|
+
|
834
|
+
}
|
835
|
+
numeric_cols.extend([col for col in numeric_cols_dtypes if col not in numeric_cols])
|
829
836
|
|
830
837
|
enable_bulk_insert = mrsm.get_config(
|
831
838
|
'system', 'connectors', 'sql', 'bulk_insert'
|
@@ -858,12 +865,24 @@ def to_sql(
|
|
858
865
|
for col in bytes_cols:
|
859
866
|
df[col] = df[col].apply(bytes_serializer)
|
860
867
|
|
861
|
-
|
862
|
-
|
863
|
-
|
864
|
-
|
865
|
-
|
866
|
-
|
868
|
+
### Check for numeric columns.
|
869
|
+
for col in numeric_cols:
|
870
|
+
typ = numeric_cols_dtypes.get(col, None)
|
871
|
+
|
872
|
+
precision, scale = (
|
873
|
+
(typ.precision, typ.scale)
|
874
|
+
if hasattr(typ, 'precision')
|
875
|
+
else get_numeric_precision_scale(self.flavor)
|
876
|
+
)
|
877
|
+
|
878
|
+
df[col] = df[col].apply(
|
879
|
+
functools.partial(
|
880
|
+
serialize_decimal,
|
881
|
+
quantize=True,
|
882
|
+
precision=precision,
|
883
|
+
scale=scale,
|
884
|
+
)
|
885
|
+
)
|
867
886
|
|
868
887
|
stats['method'] = method.__name__ if hasattr(method, '__name__') else str(method)
|
869
888
|
|
@@ -893,7 +912,7 @@ def to_sql(
|
|
893
912
|
if name != truncated_name:
|
894
913
|
warn(
|
895
914
|
f"Table '{name}' is too long for '{self.flavor}',"
|
896
|
-
|
915
|
+
f" will instead create the table '{truncated_name}'."
|
897
916
|
)
|
898
917
|
|
899
918
|
### filter out non-pandas args
|
@@ -961,24 +980,11 @@ def to_sql(
|
|
961
980
|
### Check for JSON columns.
|
962
981
|
if self.flavor not in json_flavors:
|
963
982
|
json_cols = get_json_cols(df)
|
964
|
-
|
965
|
-
for col in json_cols:
|
966
|
-
df[col] = df[col].apply(
|
967
|
-
(
|
968
|
-
lambda x: json.dumps(x, default=str, sort_keys=True)
|
969
|
-
if not isinstance(x, Hashable)
|
970
|
-
else x
|
971
|
-
)
|
972
|
-
)
|
973
|
-
|
974
|
-
### Check for numeric columns.
|
975
|
-
numeric_scale, numeric_precision = NUMERIC_PRECISION_FLAVORS.get(self.flavor, (None, None))
|
976
|
-
if numeric_precision is not None and numeric_scale is not None:
|
977
|
-
for col in numeric_cols:
|
983
|
+
for col in json_cols:
|
978
984
|
df[col] = df[col].apply(
|
979
|
-
|
980
|
-
|
981
|
-
if isinstance(x,
|
985
|
+
(
|
986
|
+
lambda x: json.dumps(x, default=json_serialize_value, sort_keys=True)
|
987
|
+
if not isinstance(x, Hashable)
|
982
988
|
else x
|
983
989
|
)
|
984
990
|
)
|
@@ -1055,16 +1061,20 @@ def psql_insert_copy(
|
|
1055
1061
|
|
1056
1062
|
from meerschaum.utils.sql import sql_item_name
|
1057
1063
|
from meerschaum.utils.warnings import dprint
|
1064
|
+
from meerschaum.utils.dtypes import json_serialize_value
|
1058
1065
|
|
1059
1066
|
### NOTE: PostgreSQL doesn't support NUL chars in text, so they're removed from strings.
|
1060
1067
|
data_iter = (
|
1061
1068
|
(
|
1062
1069
|
(
|
1063
1070
|
(
|
1064
|
-
json.dumps(
|
1071
|
+
json.dumps(
|
1072
|
+
item,
|
1073
|
+
default=json_serialize_value,
|
1074
|
+
).replace('\0', '').replace('\\u0000', '')
|
1065
1075
|
if isinstance(item, (dict, list))
|
1066
1076
|
else (
|
1067
|
-
item
|
1077
|
+
json_serialize_value(item, default_to_str=False)
|
1068
1078
|
if not isinstance(item, str)
|
1069
1079
|
else item.replace('\0', '').replace('\\u0000', '')
|
1070
1080
|
)
|
@@ -1123,6 +1133,7 @@ def mssql_insert_json(
|
|
1123
1133
|
"""
|
1124
1134
|
import json
|
1125
1135
|
from meerschaum.utils.sql import sql_item_name
|
1136
|
+
from meerschaum.utils.dtypes import json_serialize_value
|
1126
1137
|
from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type, get_db_type_from_pd_type
|
1127
1138
|
from meerschaum.utils.warnings import dprint
|
1128
1139
|
table_name = sql_item_name(table.name, 'mssql', table.schema)
|
@@ -1131,6 +1142,15 @@ def mssql_insert_json(
|
|
1131
1142
|
str(column.name): get_pd_type_from_db_type(str(column.type))
|
1132
1143
|
for column in table.table.columns
|
1133
1144
|
}
|
1145
|
+
numeric_cols_types = {
|
1146
|
+
col: table.table.columns[col].type
|
1147
|
+
for col, typ in pd_types.items()
|
1148
|
+
if typ.startswith('numeric') and col in keys
|
1149
|
+
}
|
1150
|
+
pd_types.update({
|
1151
|
+
col: f'numeric[{typ.precision},{typ.scale}]'
|
1152
|
+
for col, typ in numeric_cols_types.items()
|
1153
|
+
})
|
1134
1154
|
cols_types = {
|
1135
1155
|
col: get_db_type_from_pd_type(typ, 'mssql')
|
1136
1156
|
for col, typ in pd_types.items()
|
@@ -1155,7 +1175,8 @@ def mssql_insert_json(
|
|
1155
1175
|
if debug:
|
1156
1176
|
dprint(sql)
|
1157
1177
|
|
1158
|
-
|
1178
|
+
serialized_data = json.dumps(json_data, default=json_serialize_value)
|
1179
|
+
conn.exec_driver_sql(sql, (serialized_data,))
|
1159
1180
|
|
1160
1181
|
|
1161
1182
|
def format_sql_query_for_dask(query: str) -> 'sqlalchemy.sql.selectable.Select':
|
@@ -13,14 +13,14 @@ from meerschaum.utils.packages import attempt_import
|
|
13
13
|
|
14
14
|
@classmethod
|
15
15
|
def from_uri(
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
16
|
+
cls,
|
17
|
+
uri: str,
|
18
|
+
label: Optional[str] = None,
|
19
|
+
as_dict: bool = False,
|
20
|
+
) -> Union[
|
21
|
+
'meerschaum.connectors.SQLConnector',
|
22
|
+
Dict[str, Union[str, int]],
|
23
|
+
]:
|
24
24
|
"""
|
25
25
|
Create a new SQLConnector from a URI string.
|
26
26
|
|
@@ -97,7 +97,7 @@ def parse_uri(uri: str) -> Dict[str, Any]:
|
|
97
97
|
>>>
|
98
98
|
"""
|
99
99
|
from urllib.parse import parse_qs, urlparse
|
100
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
100
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
101
101
|
parser = sqlalchemy.engine.url.make_url
|
102
102
|
params = parser(uri).translate_connect_args()
|
103
103
|
params['flavor'] = uri.split(':')[0].split('+')[0]
|
@@ -19,10 +19,9 @@ def register_user(
|
|
19
19
|
**kw: Any
|
20
20
|
) -> SuccessTuple:
|
21
21
|
"""Register a new user."""
|
22
|
-
from meerschaum.utils.warnings import warn, error, info
|
23
22
|
from meerschaum.utils.packages import attempt_import
|
24
23
|
from meerschaum.utils.sql import json_flavors
|
25
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
24
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
26
25
|
|
27
26
|
valid_tuple = valid_username(user.username)
|
28
27
|
if not valid_tuple[0]:
|
@@ -103,9 +102,8 @@ def edit_user(
|
|
103
102
|
) -> SuccessTuple:
|
104
103
|
"""Update an existing user's metadata."""
|
105
104
|
from meerschaum.utils.packages import attempt_import
|
106
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
105
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
107
106
|
from meerschaum.connectors.sql.tables import get_tables
|
108
|
-
from meerschaum.utils.sql import json_flavors
|
109
107
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
110
108
|
|
111
109
|
user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
|
@@ -158,7 +156,7 @@ def get_user_id(
|
|
158
156
|
"""If a user is registered, return the `user_id`."""
|
159
157
|
### ensure users table exists
|
160
158
|
from meerschaum.utils.packages import attempt_import
|
161
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
159
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
162
160
|
from meerschaum.connectors.sql.tables import get_tables
|
163
161
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
164
162
|
|
@@ -183,7 +181,7 @@ def get_user_attributes(
|
|
183
181
|
### ensure users table exists
|
184
182
|
from meerschaum.utils.warnings import warn
|
185
183
|
from meerschaum.utils.packages import attempt_import
|
186
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
184
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
187
185
|
from meerschaum.connectors.sql.tables import get_tables
|
188
186
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
189
187
|
|
@@ -199,14 +197,14 @@ def get_user_attributes(
|
|
199
197
|
try:
|
200
198
|
result = dict(result)
|
201
199
|
_parsed = True
|
202
|
-
except Exception
|
200
|
+
except Exception:
|
203
201
|
_parsed = False
|
204
202
|
if not _parsed:
|
205
203
|
try:
|
206
204
|
import json
|
207
205
|
result = json.loads(result)
|
208
206
|
_parsed = True
|
209
|
-
except Exception
|
207
|
+
except Exception:
|
210
208
|
_parsed = False
|
211
209
|
if not _parsed:
|
212
210
|
warn(f"Received unexpected type for attributes: {result}")
|
@@ -223,7 +221,7 @@ def delete_user(
|
|
223
221
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
224
222
|
plugins = get_tables(mrsm_instance=self, debug=debug)['plugins']
|
225
223
|
from meerschaum.utils.packages import attempt_import
|
226
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
224
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
227
225
|
|
228
226
|
user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
|
229
227
|
|
@@ -256,7 +254,7 @@ def get_users(
|
|
256
254
|
from meerschaum.connectors.sql.tables import get_tables
|
257
255
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
258
256
|
from meerschaum.utils.packages import attempt_import
|
259
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
257
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
260
258
|
|
261
259
|
query = sqlalchemy.select(users_tbl.c.username)
|
262
260
|
|
@@ -277,7 +275,7 @@ def get_user_password_hash(
|
|
277
275
|
from meerschaum.connectors.sql.tables import get_tables
|
278
276
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
279
277
|
from meerschaum.utils.packages import attempt_import
|
280
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
278
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
281
279
|
|
282
280
|
if user.user_id is not None:
|
283
281
|
user_id = user.user_id
|
@@ -308,7 +306,7 @@ def get_user_type(
|
|
308
306
|
from meerschaum.connectors.sql.tables import get_tables
|
309
307
|
users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
|
310
308
|
from meerschaum.utils.packages import attempt_import
|
311
|
-
sqlalchemy = attempt_import('sqlalchemy')
|
309
|
+
sqlalchemy = attempt_import('sqlalchemy', lazy=False)
|
312
310
|
|
313
311
|
user_id = user.user_id if user.user_id is not None else self.get_user_id(user, debug=debug)
|
314
312
|
|
@@ -17,10 +17,10 @@ _sequence_flavors = {'duckdb', 'oracle'}
|
|
17
17
|
_skip_index_names_flavors = {'mssql',}
|
18
18
|
|
19
19
|
def get_tables(
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
20
|
+
mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
|
21
|
+
create: bool = True,
|
22
|
+
debug: Optional[bool] = None
|
23
|
+
) -> Union[Dict[str, 'sqlalchemy.Table'], bool]:
|
24
24
|
"""
|
25
25
|
Create tables on the database and return the `sqlalchemy` tables.
|
26
26
|
|
@@ -51,7 +51,7 @@ def get_tables(
|
|
51
51
|
sqlalchemy, sqlalchemy_dialects_postgresql = attempt_import(
|
52
52
|
'sqlalchemy',
|
53
53
|
'sqlalchemy.dialects.postgresql',
|
54
|
-
lazy
|
54
|
+
lazy=False,
|
55
55
|
)
|
56
56
|
if not sqlalchemy:
|
57
57
|
error(f"Failed to import sqlalchemy. Is sqlalchemy installed?")
|
@@ -205,13 +205,12 @@ def get_tables(
|
|
205
205
|
|
206
206
|
|
207
207
|
def create_tables(
|
208
|
-
|
209
|
-
|
210
|
-
|
208
|
+
conn: 'meerschaum.connectors.SQLConnector',
|
209
|
+
tables: Optional[Dict[str, 'sqlalchemy.Table']] = None,
|
210
|
+
) -> bool:
|
211
211
|
"""
|
212
212
|
Create the tables on the database.
|
213
213
|
"""
|
214
|
-
from meerschaum.utils.sql import get_rename_table_queries, table_exists
|
215
214
|
_tables = tables if tables is not None else get_tables(conn)
|
216
215
|
|
217
216
|
try:
|
@@ -225,10 +224,10 @@ def create_tables(
|
|
225
224
|
|
226
225
|
|
227
226
|
def create_schemas(
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
227
|
+
conn: 'meerschaum.connectors.SQLConnector',
|
228
|
+
schemas: List[str],
|
229
|
+
debug: bool = False,
|
230
|
+
) -> bool:
|
232
231
|
"""
|
233
232
|
Create the internal Meerschaum schema on the database.
|
234
233
|
"""
|
@@ -238,7 +237,7 @@ def create_schemas(
|
|
238
237
|
if conn.flavor in NO_SCHEMA_FLAVORS:
|
239
238
|
return True
|
240
239
|
|
241
|
-
|
240
|
+
_ = attempt_import('sqlalchemy.schema', lazy=False)
|
242
241
|
successes = {}
|
243
242
|
skip_if_not_exists = conn.flavor in SKIP_IF_EXISTS_FLAVORS
|
244
243
|
if_not_exists_str = ("IF NOT EXISTS " if not skip_if_not_exists else "")
|
@@ -239,7 +239,7 @@ class ValkeyConnector(Connector):
|
|
239
239
|
-------
|
240
240
|
The current index counter value (how many docs have been pushed).
|
241
241
|
"""
|
242
|
-
from meerschaum.utils.
|
242
|
+
from meerschaum.utils.dtypes import json_serialize_value
|
243
243
|
table_name = self.quote_table(table)
|
244
244
|
datetime_column_key = self.get_datetime_column_key(table)
|
245
245
|
remote_datetime_column = self.get(datetime_column_key)
|
@@ -269,7 +269,7 @@ class ValkeyConnector(Connector):
|
|
269
269
|
) if datetime_column else None
|
270
270
|
doc_str = json.dumps(
|
271
271
|
doc,
|
272
|
-
default=
|
272
|
+
default=json_serialize_value,
|
273
273
|
separators=(',', ':'),
|
274
274
|
sort_keys=True,
|
275
275
|
)
|
meerschaum/core/Pipe/__init__.py
CHANGED
@@ -107,6 +107,7 @@ class Pipe:
|
|
107
107
|
static,
|
108
108
|
tzinfo,
|
109
109
|
enforce,
|
110
|
+
null_indices,
|
110
111
|
get_columns,
|
111
112
|
get_columns_types,
|
112
113
|
get_columns_indices,
|
@@ -141,7 +142,8 @@ class Pipe:
|
|
141
142
|
get_bound_time,
|
142
143
|
)
|
143
144
|
from ._delete import delete
|
144
|
-
from ._drop import drop
|
145
|
+
from ._drop import drop, drop_indices
|
146
|
+
from ._index import create_indices
|
145
147
|
from ._clear import clear
|
146
148
|
from ._deduplicate import deduplicate
|
147
149
|
from ._bootstrap import bootstrap
|
@@ -165,6 +167,7 @@ class Pipe:
|
|
165
167
|
autoincrement: Optional[bool] = None,
|
166
168
|
static: Optional[bool] = None,
|
167
169
|
enforce: Optional[bool] = None,
|
170
|
+
null_indices: Optional[bool] = None,
|
168
171
|
mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
|
169
172
|
cache: bool = False,
|
170
173
|
debug: bool = False,
|
@@ -223,10 +226,14 @@ class Pipe:
|
|
223
226
|
static: Optional[bool], default None
|
224
227
|
If `True`, set `static` in the parameters.
|
225
228
|
|
226
|
-
enforce:
|
229
|
+
enforce: Optional[bool], default None
|
227
230
|
If `False`, skip data type enforcement.
|
228
231
|
Default behavior is `True`.
|
229
232
|
|
233
|
+
null_indices: Optional[bool], default None
|
234
|
+
Set to `False` if there will be no null values in the index columns.
|
235
|
+
Defaults to `True`.
|
236
|
+
|
230
237
|
temporary: bool, default False
|
231
238
|
If `True`, prevent instance tables (pipes, users, plugins) from being created.
|
232
239
|
|
@@ -330,6 +337,9 @@ class Pipe:
|
|
330
337
|
if isinstance(enforce, bool):
|
331
338
|
self._attributes['parameters']['enforce'] = enforce
|
332
339
|
|
340
|
+
if isinstance(null_indices, bool):
|
341
|
+
self._attributes['parameters']['null_indices'] = null_indices
|
342
|
+
|
333
343
|
### NOTE: The parameters dictionary is {} by default.
|
334
344
|
### A Pipe may be registered without parameters, then edited,
|
335
345
|
### or a Pipe may be registered with parameters set in-memory first.
|
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|
11
11
|
from datetime import timezone
|
12
12
|
|
13
13
|
import meerschaum as mrsm
|
14
|
-
from meerschaum.utils.typing import Tuple, Dict,
|
14
|
+
from meerschaum.utils.typing import Tuple, Dict, Any, Union, Optional, List
|
15
15
|
from meerschaum.utils.warnings import warn
|
16
16
|
|
17
17
|
|
@@ -313,6 +313,25 @@ def enforce(self, _enforce: bool) -> None:
|
|
313
313
|
self.parameters['enforce'] = _enforce
|
314
314
|
|
315
315
|
|
316
|
+
@property
|
317
|
+
def null_indices(self) -> bool:
|
318
|
+
"""
|
319
|
+
Return the `null_indices` parameter for the pipe.
|
320
|
+
"""
|
321
|
+
if 'null_indices' not in self.parameters:
|
322
|
+
self.parameters['null_indices'] = True
|
323
|
+
|
324
|
+
return self.parameters['null_indices']
|
325
|
+
|
326
|
+
|
327
|
+
@null_indices.setter
|
328
|
+
def null_indices(self, _null_indices: bool) -> None:
|
329
|
+
"""
|
330
|
+
Set the `null_indices` parameter for the pipe.
|
331
|
+
"""
|
332
|
+
self.parameters['null_indices'] = _null_indices
|
333
|
+
|
334
|
+
|
316
335
|
def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
|
317
336
|
"""
|
318
337
|
Check if the requested columns are defined.
|
@@ -469,7 +488,7 @@ def get_columns_indices(
|
|
469
488
|
|
470
489
|
self.__dict__['_columns_indices'] = _columns_indices
|
471
490
|
self.__dict__['_columns_indices_timestamp'] = now
|
472
|
-
return _columns_indices or {}
|
491
|
+
return {k: v for k, v in _columns_indices.items() if k and v} or {}
|
473
492
|
|
474
493
|
|
475
494
|
def get_id(self, **kw: Any) -> Union[int, None]:
|
@@ -711,42 +730,17 @@ def guess_datetime(self) -> Union[str, None]:
|
|
711
730
|
|
712
731
|
def get_indices(self) -> Dict[str, str]:
|
713
732
|
"""
|
714
|
-
Return a dictionary mapping index keys to their names
|
733
|
+
Return a dictionary mapping index keys to their names in the database.
|
715
734
|
|
716
735
|
Returns
|
717
736
|
-------
|
718
|
-
A dictionary of index keys to
|
719
|
-
"""
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
else str(cols)
|
729
|
-
)
|
730
|
-
for ix, cols in _indices.items()
|
731
|
-
if cols
|
732
|
-
}
|
733
|
-
_index_names = {
|
734
|
-
ix: _index_template.format(
|
735
|
-
target=_target,
|
736
|
-
column_names=column_names,
|
737
|
-
connector_keys=self.connector_keys,
|
738
|
-
metric_key=self.connector_key,
|
739
|
-
location_key=self.location_key,
|
740
|
-
)
|
741
|
-
for ix, column_names in _column_names.items()
|
742
|
-
}
|
743
|
-
### NOTE: Skip any duplicate indices.
|
744
|
-
seen_index_names = {}
|
745
|
-
for ix, index_name in _index_names.items():
|
746
|
-
if index_name in seen_index_names:
|
747
|
-
continue
|
748
|
-
seen_index_names[index_name] = ix
|
749
|
-
return {
|
750
|
-
ix: index_name
|
751
|
-
for index_name, ix in seen_index_names.items()
|
752
|
-
}
|
737
|
+
A dictionary of index keys to index names.
|
738
|
+
"""
|
739
|
+
from meerschaum.connectors import get_connector_plugin
|
740
|
+
with mrsm.Venv(get_connector_plugin(self.instance_connector)):
|
741
|
+
if hasattr(self.instance_connector, 'get_pipe_index_names'):
|
742
|
+
result = self.instance_connector.get_pipe_index_names(self)
|
743
|
+
else:
|
744
|
+
result = {}
|
745
|
+
|
746
|
+
return result
|