meerschaum 3.0.0rc1__py3-none-any.whl → 3.0.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parser.py +2 -1
- meerschaum/_internal/docs/index.py +49 -2
- meerschaum/_internal/static.py +8 -24
- meerschaum/actions/verify.py +5 -8
- meerschaum/api/__init__.py +2 -1
- meerschaum/api/dash/__init__.py +0 -2
- meerschaum/api/dash/callbacks/dashboard.py +1 -1
- meerschaum/api/dash/tokens.py +2 -2
- meerschaum/api/routes/_pipes.py +47 -37
- meerschaum/config/_default.py +11 -1
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +9 -8
- meerschaum/connectors/api/_pipes.py +2 -18
- meerschaum/connectors/api/_tokens.py +2 -2
- meerschaum/connectors/instance/_tokens.py +4 -4
- meerschaum/connectors/sql/_create_engine.py +3 -14
- meerschaum/connectors/sql/_pipes.py +118 -163
- meerschaum/connectors/sql/_sql.py +38 -20
- meerschaum/connectors/valkey/_pipes.py +44 -16
- meerschaum/core/Pipe/__init__.py +28 -5
- meerschaum/core/Pipe/_attributes.py +270 -46
- meerschaum/core/Pipe/_data.py +55 -17
- meerschaum/core/Pipe/_dtypes.py +19 -4
- meerschaum/core/Pipe/_edit.py +2 -0
- meerschaum/core/Pipe/_fetch.py +1 -1
- meerschaum/core/Pipe/_sync.py +90 -160
- meerschaum/core/Pipe/_verify.py +3 -3
- meerschaum/core/Token/_Token.py +3 -4
- meerschaum/utils/dataframe.py +379 -68
- meerschaum/utils/debug.py +15 -15
- meerschaum/utils/dtypes/__init__.py +388 -22
- meerschaum/utils/dtypes/sql.py +326 -30
- meerschaum/utils/misc.py +9 -68
- meerschaum/utils/packages/__init__.py +7 -21
- meerschaum/utils/packages/_packages.py +7 -2
- meerschaum/utils/schedule.py +1 -1
- meerschaum/utils/sql.py +7 -7
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc2.dist-info}/METADATA +5 -17
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc2.dist-info}/RECORD +45 -44
- meerschaum-3.0.0rc2.dist-info/licenses/NOTICE +2 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc2.dist-info}/WHEEL +0 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc2.dist-info}/entry_points.txt +0 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc2.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc2.dist-info}/top_level.txt +0 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc2.dist-info}/zip-safe +0 -0
@@ -40,7 +40,8 @@ class ArgumentParser(argparse.ArgumentParser):
|
|
40
40
|
|
41
41
|
def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
|
42
42
|
"""Parse a string into a datetime."""
|
43
|
-
from meerschaum.utils.
|
43
|
+
from meerschaum.utils.dtypes import round_time
|
44
|
+
from meerschaum.utils.misc import is_int
|
44
45
|
if is_int(dt_str):
|
45
46
|
return int(dt_str)
|
46
47
|
|
@@ -98,7 +98,7 @@ Decorate your connector classes with `meerschaum.make_connector()` to designate
|
|
98
98
|
from datetime import datetime, timezone
|
99
99
|
from random import randint
|
100
100
|
import meerschaum as mrsm
|
101
|
-
from meerschaum.utils.
|
101
|
+
from meerschaum.utils.dtypes import round_time
|
102
102
|
|
103
103
|
@mrsm.make_connector
|
104
104
|
class FooConnector(mrsm.Connector):
|
@@ -535,14 +535,25 @@ def init_dash(dash_app):
|
|
535
535
|
<p></p>
|
536
536
|
<ul>
|
537
537
|
<li><code>meerschaum.utils.dataframe.add_missing_cols_to_df()</code></li>
|
538
|
+
<li><code>meerschaum.utils.dataframe.chunksize_to_npartitions()</code></li>
|
539
|
+
<li><code>meerschaum.utils.dataframe.df_from_literal()</code></li>
|
538
540
|
<li><code>meerschaum.utils.dataframe.df_is_chunk_generator()</code></li>
|
539
541
|
<li><code>meerschaum.utils.dataframe.enforce_dtypes()</code></li>
|
540
542
|
<li><code>meerschaum.utils.dataframe.filter_unseen_df()</code></li>
|
543
|
+
<li><code>meerschaum.utils.dataframe.get_bool_cols()</code></li>
|
544
|
+
<li><code>meerschaum.utils.dataframe.get_bytes_cols()</code></li>
|
541
545
|
<li><code>meerschaum.utils.dataframe.get_datetime_bound_from_df()</code></li>
|
546
|
+
<li><code>meerschaum.utils.dataframe.get_datetime_cols()</code></li>
|
547
|
+
<li><code>meerschaum.utils.dataframe.get_datetime_cols_types()</code></li>
|
542
548
|
<li><code>meerschaum.utils.dataframe.get_first_valid_dask_partition()</code></li>
|
549
|
+
<li><code>meerschaum.utils.dataframe.get_geometry_cols()</code></li>
|
550
|
+
<li><code>meerschaum.utils.dataframe.get_geometry_cols_types()</code></li>
|
543
551
|
<li><code>meerschaum.utils.dataframe.get_json_cols()</code></li>
|
544
552
|
<li><code>meerschaum.utils.dataframe.get_numeric_cols()</code></li>
|
553
|
+
<li><code>meerschaum.utils.dataframe.get_special_cols()</code></li>
|
545
554
|
<li><code>meerschaum.utils.dataframe.get_unhashable_cols()</code></li>
|
555
|
+
<li><code>meerschaum.utils.dataframe.get_unique_index_values()</code></li>
|
556
|
+
<li><code>meerschaum.utils.dataframe.get_uuid_cols()</code></li>
|
546
557
|
<li><code>meerschaum.utils.dataframe.parse_df_datetimes()</code></li>
|
547
558
|
<li><code>meerschaum.utils.dataframe.query_df()</code></li>
|
548
559
|
<li><code>meerschaum.utils.dataframe.to_json()</code></li>
|
@@ -559,14 +570,51 @@ def init_dash(dash_app):
|
|
559
570
|
<p></p>
|
560
571
|
<ul>
|
561
572
|
<li><code>meerschaum.utils.dtypes.are_dtypes_equal()</code></li>
|
573
|
+
<li><code>meerschaum.utils.dtypes.attempt_cast_to_bytes()</code></li>
|
574
|
+
<li><code>meerschaum.utils.dtypes.attempt_cast_to_geometry()</code></li>
|
562
575
|
<li><code>meerschaum.utils.dtypes.attempt_cast_to_numeric()</code></li>
|
576
|
+
<li><code>meerschaum.utils.dtypes.attempt_cast_to_uuid()</code></li>
|
577
|
+
<li><code>meerschaum.utils.dtypes.coerce_timezone()</code></li>
|
578
|
+
<li><code>meerschaum.utils.dtypes.deserialize_base64()</code></li>
|
579
|
+
<li><code>meerschaum.utils.dtypes.deserialize_bytes_string()</code></li>
|
580
|
+
<li><code>meerschaum.utils.dtypes.deserialize_geometry()</code></li>
|
581
|
+
<li><code>meerschaum.utils.dtypes.dtype_is_special()</code></li>
|
582
|
+
<li><code>meerschaum.utils.dtypes.encode_bytes_for_bytea()</code></li>
|
583
|
+
<li><code>meerschaum.utils.dtypes.geometry_is_wkt()</code></li>
|
584
|
+
<li><code>meerschaum.utils.dtypes.get_current_timestamp()</code></li>
|
585
|
+
<li><code>meerschaum.utils.dtypes.get_geometry_type_srid()</code></li>
|
563
586
|
<li><code>meerschaum.utils.dtypes.is_dtype_numeric()</code></li>
|
587
|
+
<li><code>meerschaum.utils.dtypes.json_serialize_value()</code></li>
|
564
588
|
<li><code>meerschaum.utils.dtypes.none_if_null()</code></li>
|
589
|
+
<li><code>meerschaum.utils.dtypes.project_geometry()</code></li>
|
565
590
|
<li><code>meerschaum.utils.dtypes.quantize_decimal()</code></li>
|
591
|
+
<li><code>meerschaum.utils.dtypes.serialize_bytes()</code></li>
|
592
|
+
<li><code>meerschaum.utils.dtypes.serialize_datetime()</code></li>
|
593
|
+
<li><code>meerschaum.utils.dtypes.serialize_date()</code></li>
|
594
|
+
<li><code>meerschaum.utils.dtypes.serialize_decimal()</code></li>
|
595
|
+
<li><code>meerschaum.utils.dtypes.serialize_geometry()</code></li>
|
596
|
+
<li><code>meerschaum.utils.dtypes.to_datetime()</code></li>
|
566
597
|
<li><code>meerschaum.utils.dtypes.to_pandas_dtype()</code></li>
|
567
598
|
<li><code>meerschaum.utils.dtypes.value_is_null()</code></li>
|
599
|
+
<li><code>meerschaum.utils.dtypes.get_current_timestamp()</code></li>
|
600
|
+
<li><code>meerschaum.utils.dtypes.dtype_is_special()</code></li>
|
601
|
+
<li><code>meerschaum.utils.dtypes.get_next_precision_unit()</code></li>
|
602
|
+
<li><code>meerschaum.utils.dtypes.round_time()</code></li>
|
603
|
+
</ul>
|
604
|
+
</details>
|
605
|
+
</ul>
|
606
|
+
|
607
|
+
<ul>
|
608
|
+
<details>
|
609
|
+
<summary>
|
610
|
+
<code>meerschaum.utils.dtypes.sql</code><br>
|
611
|
+
Work with SQL data types.<br>
|
612
|
+
</summary>
|
613
|
+
<p></p>
|
614
|
+
<ul>
|
568
615
|
<li><code>meerschaum.utils.dtypes.sql.get_pd_type_from_db_type()</code></li>
|
569
616
|
<li><code>meerschaum.utils.dtypes.sql.get_db_type_from_pd_type()</code></li>
|
617
|
+
<li><code>meerschaum.utils.dtypes.sql.get_numeric_precision_scale()</code></li>
|
570
618
|
</ul>
|
571
619
|
</details>
|
572
620
|
</ul>
|
@@ -605,7 +653,6 @@ def init_dash(dash_app):
|
|
605
653
|
<p></p>
|
606
654
|
<ul>
|
607
655
|
<li><code>meerschaum.utils.misc.items_str()</code></li>
|
608
|
-
<li><code>meerschaum.utils.misc.round_time()</code></li>
|
609
656
|
<li><code>meerschaum.utils.misc.is_int()</code></li>
|
610
657
|
<li><code>meerschaum.utils.misc.interval_str()</code></li>
|
611
658
|
<li><code>meerschaum.utils.misc.filter_keywords()</code></li>
|
meerschaum/_internal/static.py
CHANGED
@@ -14,8 +14,8 @@ __all__ = ('STATIC_CONFIG',)
|
|
14
14
|
|
15
15
|
_default_create_engine_args = {
|
16
16
|
# 'method': 'multi',
|
17
|
-
'pool_size': 5,
|
18
|
-
'max_overflow': 10,
|
17
|
+
'pool_size': (os.cpu_count() or 5),
|
18
|
+
'max_overflow': (os.cpu_count() or 10),
|
19
19
|
'pool_recycle': 3600,
|
20
20
|
'connect_args': {},
|
21
21
|
}
|
@@ -287,6 +287,11 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
287
287
|
'connectors': {
|
288
288
|
'default_label': 'main',
|
289
289
|
},
|
290
|
+
'dtypes': {
|
291
|
+
'datetime': {
|
292
|
+
'default_precision_unit': 'microsecond',
|
293
|
+
},
|
294
|
+
},
|
290
295
|
'stack': {
|
291
296
|
'dollar_standin': '<DOLLAR>', ### Temporary replacement for docker-compose.yaml.
|
292
297
|
},
|
@@ -313,8 +318,6 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
313
318
|
'dtypes': {
|
314
319
|
'min_ratio_columns_changed_for_full_astype': 0.5,
|
315
320
|
},
|
316
|
-
'exists_timeout_seconds': 5.0,
|
317
|
-
'static_schema_cache_seconds': 60.0,
|
318
321
|
'max_bound_time_days': 36525,
|
319
322
|
},
|
320
323
|
'jobs': {
|
@@ -349,24 +352,5 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
349
352
|
'users:delete': "Delete the associated user account (or other users for admins).",
|
350
353
|
},
|
351
354
|
},
|
352
|
-
|
353
|
-
'name': 'meerschaum',
|
354
|
-
'formal_name': 'Meerschaum',
|
355
|
-
'app_id': 'io.meerschaum',
|
356
|
-
'description': 'Sync Time-Series Pipes with Meerschaum',
|
357
|
-
'url': 'https://meerschaum.io',
|
358
|
-
'project_urls': {
|
359
|
-
'Documentation': 'https://docs.meerschaum.io',
|
360
|
-
'Changelog': 'https://meerschaum.io/news/changelog',
|
361
|
-
'GitHub': 'https://github.com/bmeares/Meerschaum',
|
362
|
-
'Homepage': 'https://meerschaum.io',
|
363
|
-
'Donate': 'https://github.com/sponsors/bmeares',
|
364
|
-
'Discord': 'https://discord.gg/8U8qMUjvcc',
|
365
|
-
},
|
366
|
-
'author': 'Bennett Meares',
|
367
|
-
'author_email': 'bennett.meares@gmail.com',
|
368
|
-
'maintainer_email': 'bennett.meares@gmail.com',
|
369
|
-
'license': 'Apache Software License 2.0',
|
370
|
-
'license_files': ('LICENSE',),
|
371
|
-
},
|
355
|
+
|
372
356
|
}
|
meerschaum/actions/verify.py
CHANGED
@@ -57,8 +57,11 @@ def _verify_packages(
|
|
57
57
|
Verify the versions of packages.
|
58
58
|
"""
|
59
59
|
from meerschaum.utils.packages import (
|
60
|
-
attempt_import,
|
61
|
-
|
60
|
+
attempt_import,
|
61
|
+
all_packages,
|
62
|
+
is_installed,
|
63
|
+
venv_contains_package,
|
64
|
+
manually_import_module,
|
62
65
|
)
|
63
66
|
|
64
67
|
venv_packages, base_packages, miss_packages = [], [], []
|
@@ -78,12 +81,6 @@ def _verify_packages(
|
|
78
81
|
)
|
79
82
|
_where_list.append(import_name)
|
80
83
|
|
81
|
-
if 'flask_compress' in venv_packages or 'dash' in venv_packages:
|
82
|
-
flask_compress = attempt_import('flask_compress', lazy=False, debug=debug)
|
83
|
-
_monkey_patch_get_distribution('flask-compress', flask_compress.__version__)
|
84
|
-
if 'flask_compress' in venv_packages:
|
85
|
-
venv_packages.remove('flask_compress')
|
86
|
-
|
87
84
|
for import_name in base_packages:
|
88
85
|
manually_import_module(import_name, debug=debug, venv=None)
|
89
86
|
for import_name in venv_packages:
|
meerschaum/api/__init__.py
CHANGED
@@ -217,8 +217,9 @@ def get_pipe(
|
|
217
217
|
status_code=403,
|
218
218
|
detail="Unable to serve any pipes with connector keys `mrsm` over the API.",
|
219
219
|
)
|
220
|
+
|
220
221
|
pipe = mrsm.Pipe(connector_keys, metric_key, location_key, mrsm_instance=instance_keys)
|
221
|
-
if is_pipe_registered(pipe, pipes(instance_keys)):
|
222
|
+
if is_pipe_registered(pipe, pipes(instance_keys, refresh=False)):
|
222
223
|
return pipes(instance_keys, refresh=refresh)[connector_keys][metric_key][location_key]
|
223
224
|
return pipe
|
224
225
|
|
meerschaum/api/dash/__init__.py
CHANGED
@@ -12,10 +12,8 @@ from meerschaum.utils.packages import (
|
|
12
12
|
attempt_import,
|
13
13
|
import_dcc,
|
14
14
|
import_html,
|
15
|
-
_monkey_patch_get_distribution,
|
16
15
|
)
|
17
16
|
flask_compress = attempt_import('flask_compress', lazy=False)
|
18
|
-
_monkey_patch_get_distribution('flask-compress', flask_compress.__version__)
|
19
17
|
dash = attempt_import('dash', lazy=False)
|
20
18
|
|
21
19
|
from meerschaum.utils.typing import List, Optional
|
@@ -1024,7 +1024,7 @@ def query_data_click(n_clicks, query_editor_text, limit_value, begin, end):
|
|
1024
1024
|
raise PreventUpdate
|
1025
1025
|
|
1026
1026
|
try:
|
1027
|
-
params_query =
|
1027
|
+
params_query = string_to_dict(query_editor_text)
|
1028
1028
|
except Exception as e:
|
1029
1029
|
return alert_from_success_tuple((False, f"Invalid query:\n{e}"))
|
1030
1030
|
|
meerschaum/api/dash/tokens.py
CHANGED
@@ -18,8 +18,8 @@ from meerschaum.api.dash.components import alert_from_success_tuple
|
|
18
18
|
from meerschaum.api.dash.sessions import get_user_from_session
|
19
19
|
from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple
|
20
20
|
from meerschaum.utils.packages import attempt_import, import_html, import_dcc
|
21
|
-
from meerschaum.utils.misc import interval_str
|
22
|
-
from meerschaum.utils.dtypes import value_is_null
|
21
|
+
from meerschaum.utils.misc import interval_str
|
22
|
+
from meerschaum.utils.dtypes import value_is_null, round_time
|
23
23
|
from meerschaum._internal.static import STATIC_CONFIG
|
24
24
|
from meerschaum.core import Token
|
25
25
|
from meerschaum.utils.daemon import get_new_daemon_name
|
meerschaum/api/routes/_pipes.py
CHANGED
@@ -23,9 +23,7 @@ from meerschaum.api import (
|
|
23
23
|
pipes,
|
24
24
|
get_pipe,
|
25
25
|
_get_pipes,
|
26
|
-
manager,
|
27
26
|
debug,
|
28
|
-
no_auth,
|
29
27
|
ScopedAuth,
|
30
28
|
)
|
31
29
|
from meerschaum.models import (
|
@@ -74,7 +72,7 @@ def register_pipe(
|
|
74
72
|
instance_keys: Optional[str] = None,
|
75
73
|
parameters: Optional[Dict[str, Any]] = None,
|
76
74
|
curr_user = fastapi.Security(ScopedAuth(['pipes:write']), scopes=['pipes:write']),
|
77
|
-
) ->
|
75
|
+
) -> mrsm.SuccessTuple:
|
78
76
|
"""
|
79
77
|
Register a new pipe.
|
80
78
|
"""
|
@@ -94,8 +92,9 @@ def register_pipe(
|
|
94
92
|
)
|
95
93
|
if parameters:
|
96
94
|
pipe.parameters = parameters
|
95
|
+
|
97
96
|
success, msg = get_api_connector(instance_keys).register_pipe(pipe, debug=debug)
|
98
|
-
pipes(instance_keys, refresh=True)
|
97
|
+
_ = pipes(instance_keys, refresh=True)
|
99
98
|
return success, msg
|
100
99
|
|
101
100
|
|
@@ -112,7 +111,7 @@ def edit_pipe(
|
|
112
111
|
instance_keys: Optional[str] = None,
|
113
112
|
patch: bool = False,
|
114
113
|
curr_user = fastapi.Security(ScopedAuth(['pipes:write'])),
|
115
|
-
) ->
|
114
|
+
) -> mrsm.SuccessTuple:
|
116
115
|
"""
|
117
116
|
Edit an existing pipe's parameters.
|
118
117
|
"""
|
@@ -125,14 +124,16 @@ def edit_pipe(
|
|
125
124
|
" Under the keys `api:permissions:actions`, "
|
126
125
|
"you can toggle non-admin actions."
|
127
126
|
)
|
127
|
+
|
128
128
|
pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
129
129
|
if not is_pipe_registered(pipe, pipes(instance_keys, refresh=True)):
|
130
130
|
raise fastapi.HTTPException(
|
131
131
|
status_code=409, detail=f"{pipe} is not registered."
|
132
132
|
)
|
133
|
+
|
133
134
|
pipe.parameters = parameters
|
134
135
|
success, msg = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
|
135
|
-
pipes(instance_keys, refresh=True)
|
136
|
+
_ = pipes(instance_keys, refresh=True)
|
136
137
|
return success, msg
|
137
138
|
|
138
139
|
|
@@ -147,7 +148,7 @@ def delete_pipe(
|
|
147
148
|
location_key: str,
|
148
149
|
instance_keys: Optional[str] = None,
|
149
150
|
curr_user = fastapi.Security(ScopedAuth(['pipes:delete'])),
|
150
|
-
) ->
|
151
|
+
) -> SuccessTuple:
|
151
152
|
"""
|
152
153
|
Delete a Pipe (without dropping its table).
|
153
154
|
"""
|
@@ -156,9 +157,10 @@ def delete_pipe(
|
|
156
157
|
raise fastapi.HTTPException(
|
157
158
|
status_code=409, detail=f"{pipe} is not registered."
|
158
159
|
)
|
159
|
-
|
160
|
-
|
161
|
-
|
160
|
+
|
161
|
+
success, msg = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
|
162
|
+
_ = pipes(instance_keys, refresh=True)
|
163
|
+
return success, msg
|
162
164
|
|
163
165
|
|
164
166
|
@app.get(
|
@@ -207,6 +209,7 @@ async def get_pipes(
|
|
207
209
|
kw['metric_keys'] = metric_keys
|
208
210
|
if location_keys != "":
|
209
211
|
kw['location_keys'] = location_keys
|
212
|
+
|
210
213
|
pipes_dict = replace_pipes_in_dict(_get_pipes(**kw), lambda p: p.attributes)
|
211
214
|
for metrics in pipes_dict.values():
|
212
215
|
for locations in metrics.values():
|
@@ -224,14 +227,19 @@ async def get_pipes_by_connector(
|
|
224
227
|
"""
|
225
228
|
Get all registered Pipes by connector_keys with metadata, excluding parameters.
|
226
229
|
"""
|
227
|
-
if connector_keys not in pipes(instance_keys):
|
230
|
+
if connector_keys not in pipes(instance_keys, refresh=True):
|
228
231
|
raise fastapi.HTTPException(
|
229
232
|
status_code=404, detail=f"Connector '{connector_keys}' not found."
|
230
233
|
)
|
231
|
-
|
234
|
+
|
235
|
+
metrics = replace_pipes_in_dict(
|
236
|
+
pipes(instance_keys, refresh=False)[connector_keys],
|
237
|
+
lambda p: p.attributes
|
238
|
+
)
|
232
239
|
for locations in metrics.values():
|
233
240
|
if None in locations:
|
234
241
|
locations['None'] = locations.pop(None)
|
242
|
+
|
235
243
|
return metrics
|
236
244
|
|
237
245
|
|
@@ -245,22 +253,26 @@ async def get_pipes_by_connector_and_metric(
|
|
245
253
|
"""
|
246
254
|
Get all registered Pipes by `connector_keys` and `metric_key` with metadata, excluding parameters.
|
247
255
|
"""
|
248
|
-
if connector_keys not in pipes(instance_keys):
|
256
|
+
if connector_keys not in pipes(instance_keys, refresh=True):
|
249
257
|
raise fastapi.HTTPException(
|
250
258
|
status_code=404,
|
251
259
|
detail=f"Connector '{connector_keys}' not found.",
|
252
260
|
)
|
253
|
-
|
261
|
+
|
262
|
+
if metric_key not in pipes(instance_keys, refresh=False)[connector_keys]:
|
254
263
|
raise fastapi.HTTPException(
|
255
264
|
status_code=404,
|
256
265
|
detail=f"Metric '{metric_key}' not found.",
|
257
266
|
)
|
267
|
+
|
258
268
|
locations = replace_pipes_in_dict(
|
259
|
-
pipes(instance_keys)[connector_keys][metric_key],
|
269
|
+
pipes(instance_keys, refresh=False)[connector_keys][metric_key],
|
260
270
|
lambda p: p.attributes
|
261
271
|
)
|
272
|
+
|
262
273
|
if None in locations:
|
263
274
|
locations['None'] = locations.pop(None)
|
275
|
+
|
264
276
|
return locations
|
265
277
|
|
266
278
|
|
@@ -278,22 +290,25 @@ async def get_pipe_by_connector_and_metric_and_location(
|
|
278
290
|
"""
|
279
291
|
Get a specific Pipe with metadata, excluding parameters.
|
280
292
|
"""
|
281
|
-
if connector_keys not in pipes(instance_keys):
|
293
|
+
if connector_keys not in pipes(instance_keys, refresh=True):
|
282
294
|
raise fastapi.HTTPException(
|
283
295
|
status_code=404,
|
284
296
|
detail=f"Connector '{connector_keys}' not found.",
|
285
297
|
)
|
286
|
-
|
298
|
+
|
299
|
+
if metric_key not in pipes(instance_keys, refresh=False)[connector_keys]:
|
287
300
|
raise fastapi.HTTPException(status_code=404, detail=f"Metric '{metric_key}' not found.")
|
301
|
+
|
288
302
|
if location_key in ('[None]', 'None', 'null'):
|
289
303
|
location_key = None
|
290
|
-
|
304
|
+
|
305
|
+
if location_key not in pipes(instance_keys, refresh=False)[connector_keys][metric_key]:
|
291
306
|
raise fastapi.HTTPException(
|
292
307
|
status_code=404,
|
293
308
|
detail=f"location_key '{location_key}' not found."
|
294
309
|
)
|
295
310
|
|
296
|
-
return pipes(instance_keys)[connector_keys][metric_key][location_key].attributes
|
311
|
+
return pipes(instance_keys, refresh=False)[connector_keys][metric_key][location_key].attributes
|
297
312
|
|
298
313
|
|
299
314
|
@app.get(
|
@@ -397,7 +412,7 @@ async def sync_pipe(
|
|
397
412
|
detail="Cannot sync given data.",
|
398
413
|
)
|
399
414
|
|
400
|
-
pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
415
|
+
pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys, refresh=True)
|
401
416
|
if pipe.target in ('mrsm_users', 'mrsm_plugins', 'mrsm_pipes', 'mrsm_tokens'):
|
402
417
|
raise fastapi.HTTPException(
|
403
418
|
status_code=409,
|
@@ -616,23 +631,14 @@ def drop_pipe(
|
|
616
631
|
location_key: str,
|
617
632
|
instance_keys: Optional[str] = None,
|
618
633
|
curr_user = fastapi.Security(ScopedAuth(['pipes:drop'])),
|
619
|
-
) ->
|
634
|
+
) -> mrsm.SuccessTuple:
|
620
635
|
"""
|
621
636
|
Drop a pipe's target table.
|
622
637
|
"""
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
"Please contact the system administrator, or if you are running this server, "
|
628
|
-
"open the configuration file with `edit config system` and search for 'permissions'."
|
629
|
-
" Under the keys `api:permissions:actions`, " +
|
630
|
-
"you can toggle non-admin actions."
|
631
|
-
)
|
632
|
-
pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
633
|
-
results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
|
634
|
-
pipes(instance_keys, refresh=True)
|
635
|
-
return results
|
638
|
+
pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
639
|
+
success, msg = pipe.drop(debug=debug)
|
640
|
+
_ = pipes(instance_keys, refresh=True)
|
641
|
+
return success, msg
|
636
642
|
|
637
643
|
|
638
644
|
@app.delete(
|
@@ -661,6 +667,7 @@ def clear_pipe(
|
|
661
667
|
_params = json.loads(params)
|
662
668
|
except Exception:
|
663
669
|
_params = None
|
670
|
+
|
664
671
|
if not isinstance(_params, dict):
|
665
672
|
raise fastapi.HTTPException(
|
666
673
|
status_code=409,
|
@@ -676,7 +683,7 @@ def clear_pipe(
|
|
676
683
|
params=_params,
|
677
684
|
debug=debug,
|
678
685
|
)
|
679
|
-
pipes(instance_keys, refresh=True)
|
686
|
+
_ = pipes(instance_keys, refresh=True)
|
680
687
|
return results
|
681
688
|
|
682
689
|
|
@@ -894,8 +901,11 @@ def get_pipe_columns_types(
|
|
894
901
|
```
|
895
902
|
"""
|
896
903
|
pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
|
897
|
-
columns_types =
|
898
|
-
|
904
|
+
columns_types = (
|
905
|
+
pipe.get_columns_types(debug=debug)
|
906
|
+
or pipe.get_dtypes(refresh=True, debug=debug)
|
907
|
+
)
|
908
|
+
return columns_types
|
899
909
|
|
900
910
|
|
901
911
|
@app.get(
|
meerschaum/config/_default.py
CHANGED
@@ -203,10 +203,21 @@ default_pipes_config = {
|
|
203
203
|
},
|
204
204
|
'sync': {
|
205
205
|
'filter_params_index_limit': 250,
|
206
|
+
'exists_cache_seconds': 5.0,
|
206
207
|
},
|
207
208
|
'verify': {
|
208
209
|
'max_chunks_syncs': 3,
|
209
210
|
},
|
211
|
+
'autotime': {
|
212
|
+
'column_name_if_datetime_missing': 'ts',
|
213
|
+
},
|
214
|
+
'dtypes': {
|
215
|
+
'min_ratio_columns_changed_for_full_astype': 0.5,
|
216
|
+
'columns_types_cache_seconds': 5.0,
|
217
|
+
},
|
218
|
+
'static': {
|
219
|
+
'static_schema_cache_seconds': 60.0,
|
220
|
+
},
|
210
221
|
}
|
211
222
|
default_plugins_config = {}
|
212
223
|
default_experimental_config = {
|
@@ -214,7 +225,6 @@ default_experimental_config = {
|
|
214
225
|
}
|
215
226
|
|
216
227
|
|
217
|
-
|
218
228
|
### build default config dictionary
|
219
229
|
default_config = {}
|
220
230
|
default_config['meerschaum'] = default_meerschaum_config
|
meerschaum/config/_version.py
CHANGED
@@ -122,9 +122,9 @@ default_docker_compose_config = {
|
|
122
122
|
'POSTGRES_DB': '<DOLLAR>POSTGRES_DB',
|
123
123
|
'POSTGRES_PASSWORD': '<DOLLAR>POSTGRES_PASSWORD',
|
124
124
|
'ALLOW_IP_RANGE': env_dict['ALLOW_IP_RANGE'],
|
125
|
-
'POSTGRES_INITDB_ARGS': '-c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100'
|
125
|
+
# 'POSTGRES_INITDB_ARGS': '-c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100'
|
126
126
|
},
|
127
|
-
|
127
|
+
'command': 'postgres -c max_connections=1000 -c shared_buffers=1024MB -c max_prepared_transactions=100',
|
128
128
|
'healthcheck': {
|
129
129
|
'test': [
|
130
130
|
'CMD-SHELL', 'pg_isready -d <DOLLAR>POSTGRES_DB -U <DOLLAR>POSTGRES_USER',
|
@@ -305,19 +305,20 @@ def get_necessary_files():
|
|
305
305
|
|
306
306
|
|
307
307
|
def write_stack(
|
308
|
-
|
309
|
-
|
308
|
+
debug: bool = False
|
309
|
+
):
|
310
310
|
"""Write Docker Compose configuration files."""
|
311
311
|
from meerschaum.config._edit import general_write_yaml_config
|
312
312
|
from meerschaum.config._sync import sync_files
|
313
313
|
general_write_yaml_config(get_necessary_files(), debug=debug)
|
314
314
|
return sync_files(['stack'])
|
315
315
|
|
316
|
+
|
316
317
|
def edit_stack(
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
318
|
+
action: Optional[List[str]] = None,
|
319
|
+
debug: bool = False,
|
320
|
+
**kw
|
321
|
+
):
|
321
322
|
"""Open docker-compose.yaml or .env for editing."""
|
322
323
|
from meerschaum.config._edit import general_edit_config
|
323
324
|
if action is None:
|
@@ -191,7 +191,7 @@ def sync_pipe(
|
|
191
191
|
from meerschaum.utils.misc import items_str, interval_str
|
192
192
|
from meerschaum.config import get_config
|
193
193
|
from meerschaum.utils.packages import attempt_import
|
194
|
-
from meerschaum.utils.dataframe import
|
194
|
+
from meerschaum.utils.dataframe import get_special_cols, to_json
|
195
195
|
begin = time.perf_counter()
|
196
196
|
more_itertools = attempt_import('more_itertools')
|
197
197
|
if df is None:
|
@@ -223,22 +223,6 @@ def sync_pipe(
|
|
223
223
|
else [partition.compute() for partition in df.partitions]
|
224
224
|
)
|
225
225
|
|
226
|
-
numeric_cols = get_numeric_cols(df)
|
227
|
-
if numeric_cols:
|
228
|
-
for col in numeric_cols:
|
229
|
-
df[col] = df[col].apply(lambda x: f'{x:f}' if isinstance(x, Decimal) else x)
|
230
|
-
pipe_dtypes = pipe.dtypes
|
231
|
-
new_numeric_cols = [
|
232
|
-
col
|
233
|
-
for col in numeric_cols
|
234
|
-
if pipe_dtypes.get(col, None) != 'numeric'
|
235
|
-
]
|
236
|
-
edit_success, edit_msg = pipe.update_parameters({'dtypes': {col: 'numeric' for col in new_numeric_cols}})
|
237
|
-
if not edit_success:
|
238
|
-
warn(
|
239
|
-
"Failed to update new numeric columns "
|
240
|
-
+ f"{items_str(new_numeric_cols)}:\n{edit_msg}"
|
241
|
-
)
|
242
226
|
elif isinstance(df, dict):
|
243
227
|
### `_chunks` is a dict of lists of dicts.
|
244
228
|
### e.g. {'a' : [ {'a':[1, 2]}, {'a':[3, 4]} ] }
|
@@ -325,7 +309,7 @@ def sync_pipe(
|
|
325
309
|
def delete_pipe(
|
326
310
|
self,
|
327
311
|
pipe: Optional[mrsm.Pipe] = None,
|
328
|
-
debug: bool =
|
312
|
+
debug: bool = False,
|
329
313
|
) -> SuccessTuple:
|
330
314
|
"""Delete a Pipe and drop its table."""
|
331
315
|
if pipe is None:
|
@@ -12,7 +12,6 @@ from typing import Union, List, Optional
|
|
12
12
|
|
13
13
|
import meerschaum as mrsm
|
14
14
|
from meerschaum.core import Token
|
15
|
-
from meerschaum.models import TokenModel
|
16
15
|
from meerschaum._internal.static import STATIC_CONFIG
|
17
16
|
tokens_endpoint = STATIC_CONFIG['api']['endpoints']['tokens']
|
18
17
|
|
@@ -45,10 +44,11 @@ def register_token(self, token: Token, debug: bool = False) -> mrsm.SuccessTuple
|
|
45
44
|
return True, f"Registered token '{token.label}'."
|
46
45
|
|
47
46
|
|
48
|
-
def get_token_model(self, token_id: uuid.UUID, debug: bool = False) -> Union[TokenModel, None]:
|
47
|
+
def get_token_model(self, token_id: uuid.UUID, debug: bool = False) -> 'Union[TokenModel, None]':
|
49
48
|
"""
|
50
49
|
Return a token's model from the API instance.
|
51
50
|
"""
|
51
|
+
from meerschaum.models import TokenModel
|
52
52
|
r_url = tokens_endpoint + f'/{token_id}'
|
53
53
|
response = self.get(r_url, debug=debug)
|
54
54
|
if not response:
|
@@ -14,7 +14,6 @@ from datetime import datetime, timezone
|
|
14
14
|
import meerschaum as mrsm
|
15
15
|
from meerschaum.core import Token, User
|
16
16
|
from meerschaum.core.User import hash_password
|
17
|
-
from meerschaum.models import TokenModel
|
18
17
|
from meerschaum._internal.static import STATIC_CONFIG
|
19
18
|
|
20
19
|
|
@@ -43,8 +42,8 @@ def get_tokens_pipe(self) -> mrsm.Pipe:
|
|
43
42
|
},
|
44
43
|
dtypes={
|
45
44
|
'id': 'uuid',
|
46
|
-
'creation': '
|
47
|
-
'expiration': '
|
45
|
+
'creation': 'datetime',
|
46
|
+
'expiration': 'datetime',
|
48
47
|
'is_valid': 'bool',
|
49
48
|
'label': 'string',
|
50
49
|
'user_id': user_id_dtype,
|
@@ -227,10 +226,11 @@ def get_token(self, token_id: Union[uuid.UUID, str], debug: bool = False) -> Uni
|
|
227
226
|
return Token(**dict(token_model))
|
228
227
|
|
229
228
|
|
230
|
-
def get_token_model(self, token_id: Union[uuid.UUID, Token], debug: bool = False) -> Union[TokenModel, None]:
|
229
|
+
def get_token_model(self, token_id: Union[uuid.UUID, Token], debug: bool = False) -> 'Union[TokenModel, None]':
|
231
230
|
"""
|
232
231
|
Return a token's model from the instance.
|
233
232
|
"""
|
233
|
+
from meerschaum.models import TokenModel
|
234
234
|
if isinstance(token_id, Token):
|
235
235
|
token_id = Token.id
|
236
236
|
if not token_id:
|