meerschaum 3.0.0rc1__py3-none-any.whl → 3.0.0rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parser.py +2 -1
- meerschaum/_internal/docs/index.py +49 -2
- meerschaum/_internal/shell/Shell.py +5 -4
- meerschaum/_internal/static.py +8 -24
- meerschaum/actions/bootstrap.py +1 -1
- meerschaum/actions/edit.py +6 -3
- meerschaum/actions/start.py +1 -1
- meerschaum/actions/verify.py +5 -8
- meerschaum/api/__init__.py +2 -1
- meerschaum/api/dash/__init__.py +0 -2
- meerschaum/api/dash/callbacks/__init__.py +1 -0
- meerschaum/api/dash/callbacks/dashboard.py +20 -19
- meerschaum/api/dash/callbacks/jobs.py +11 -5
- meerschaum/api/dash/callbacks/pipes.py +106 -5
- meerschaum/api/dash/callbacks/settings/__init__.py +0 -1
- meerschaum/api/dash/callbacks/{settings/tokens.py → tokens.py} +1 -1
- meerschaum/api/dash/jobs.py +1 -1
- meerschaum/api/dash/pages/__init__.py +2 -1
- meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
- meerschaum/api/dash/pages/pipes.py +4 -3
- meerschaum/api/dash/pages/settings/__init__.py +0 -1
- meerschaum/api/dash/pages/{settings/tokens.py → tokens.py} +6 -8
- meerschaum/api/dash/pipes.py +131 -0
- meerschaum/api/dash/tokens.py +28 -31
- meerschaum/api/routes/_pipes.py +47 -37
- meerschaum/config/_default.py +13 -2
- meerschaum/config/_paths.py +1 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +9 -8
- meerschaum/connectors/api/_pipes.py +2 -18
- meerschaum/connectors/api/_tokens.py +2 -2
- meerschaum/connectors/instance/_tokens.py +10 -6
- meerschaum/connectors/sql/_SQLConnector.py +14 -0
- meerschaum/connectors/sql/_create_engine.py +3 -14
- meerschaum/connectors/sql/_pipes.py +175 -185
- meerschaum/connectors/sql/_sql.py +38 -20
- meerschaum/connectors/sql/tables/__init__.py +237 -122
- meerschaum/connectors/valkey/_pipes.py +44 -16
- meerschaum/core/Pipe/__init__.py +28 -5
- meerschaum/core/Pipe/_attributes.py +273 -46
- meerschaum/core/Pipe/_data.py +55 -17
- meerschaum/core/Pipe/_dtypes.py +19 -4
- meerschaum/core/Pipe/_edit.py +2 -0
- meerschaum/core/Pipe/_fetch.py +1 -1
- meerschaum/core/Pipe/_sync.py +90 -160
- meerschaum/core/Pipe/_verify.py +3 -3
- meerschaum/core/Token/_Token.py +4 -5
- meerschaum/plugins/bootstrap.py +508 -3
- meerschaum/utils/_get_pipes.py +1 -1
- meerschaum/utils/dataframe.py +385 -68
- meerschaum/utils/debug.py +15 -15
- meerschaum/utils/dtypes/__init__.py +387 -22
- meerschaum/utils/dtypes/sql.py +327 -31
- meerschaum/utils/misc.py +9 -68
- meerschaum/utils/packages/__init__.py +7 -21
- meerschaum/utils/packages/_packages.py +7 -2
- meerschaum/utils/schedule.py +1 -1
- meerschaum/utils/sql.py +8 -8
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc3.dist-info}/METADATA +5 -17
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc3.dist-info}/RECORD +66 -65
- meerschaum-3.0.0rc3.dist-info/licenses/NOTICE +2 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc3.dist-info}/WHEEL +0 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc3.dist-info}/entry_points.txt +0 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc3.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc3.dist-info}/top_level.txt +0 -0
- {meerschaum-3.0.0rc1.dist-info → meerschaum-3.0.0rc3.dist-info}/zip-safe +0 -0
@@ -40,7 +40,8 @@ class ArgumentParser(argparse.ArgumentParser):
|
|
40
40
|
|
41
41
|
def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
|
42
42
|
"""Parse a string into a datetime."""
|
43
|
-
from meerschaum.utils.
|
43
|
+
from meerschaum.utils.dtypes import round_time
|
44
|
+
from meerschaum.utils.misc import is_int
|
44
45
|
if is_int(dt_str):
|
45
46
|
return int(dt_str)
|
46
47
|
|
@@ -98,7 +98,7 @@ Decorate your connector classes with `meerschaum.make_connector()` to designate
|
|
98
98
|
from datetime import datetime, timezone
|
99
99
|
from random import randint
|
100
100
|
import meerschaum as mrsm
|
101
|
-
from meerschaum.utils.
|
101
|
+
from meerschaum.utils.dtypes import round_time
|
102
102
|
|
103
103
|
@mrsm.make_connector
|
104
104
|
class FooConnector(mrsm.Connector):
|
@@ -535,14 +535,25 @@ def init_dash(dash_app):
|
|
535
535
|
<p></p>
|
536
536
|
<ul>
|
537
537
|
<li><code>meerschaum.utils.dataframe.add_missing_cols_to_df()</code></li>
|
538
|
+
<li><code>meerschaum.utils.dataframe.chunksize_to_npartitions()</code></li>
|
539
|
+
<li><code>meerschaum.utils.dataframe.df_from_literal()</code></li>
|
538
540
|
<li><code>meerschaum.utils.dataframe.df_is_chunk_generator()</code></li>
|
539
541
|
<li><code>meerschaum.utils.dataframe.enforce_dtypes()</code></li>
|
540
542
|
<li><code>meerschaum.utils.dataframe.filter_unseen_df()</code></li>
|
543
|
+
<li><code>meerschaum.utils.dataframe.get_bool_cols()</code></li>
|
544
|
+
<li><code>meerschaum.utils.dataframe.get_bytes_cols()</code></li>
|
541
545
|
<li><code>meerschaum.utils.dataframe.get_datetime_bound_from_df()</code></li>
|
546
|
+
<li><code>meerschaum.utils.dataframe.get_datetime_cols()</code></li>
|
547
|
+
<li><code>meerschaum.utils.dataframe.get_datetime_cols_types()</code></li>
|
542
548
|
<li><code>meerschaum.utils.dataframe.get_first_valid_dask_partition()</code></li>
|
549
|
+
<li><code>meerschaum.utils.dataframe.get_geometry_cols()</code></li>
|
550
|
+
<li><code>meerschaum.utils.dataframe.get_geometry_cols_types()</code></li>
|
543
551
|
<li><code>meerschaum.utils.dataframe.get_json_cols()</code></li>
|
544
552
|
<li><code>meerschaum.utils.dataframe.get_numeric_cols()</code></li>
|
553
|
+
<li><code>meerschaum.utils.dataframe.get_special_cols()</code></li>
|
545
554
|
<li><code>meerschaum.utils.dataframe.get_unhashable_cols()</code></li>
|
555
|
+
<li><code>meerschaum.utils.dataframe.get_unique_index_values()</code></li>
|
556
|
+
<li><code>meerschaum.utils.dataframe.get_uuid_cols()</code></li>
|
546
557
|
<li><code>meerschaum.utils.dataframe.parse_df_datetimes()</code></li>
|
547
558
|
<li><code>meerschaum.utils.dataframe.query_df()</code></li>
|
548
559
|
<li><code>meerschaum.utils.dataframe.to_json()</code></li>
|
@@ -559,14 +570,51 @@ def init_dash(dash_app):
|
|
559
570
|
<p></p>
|
560
571
|
<ul>
|
561
572
|
<li><code>meerschaum.utils.dtypes.are_dtypes_equal()</code></li>
|
573
|
+
<li><code>meerschaum.utils.dtypes.attempt_cast_to_bytes()</code></li>
|
574
|
+
<li><code>meerschaum.utils.dtypes.attempt_cast_to_geometry()</code></li>
|
562
575
|
<li><code>meerschaum.utils.dtypes.attempt_cast_to_numeric()</code></li>
|
576
|
+
<li><code>meerschaum.utils.dtypes.attempt_cast_to_uuid()</code></li>
|
577
|
+
<li><code>meerschaum.utils.dtypes.coerce_timezone()</code></li>
|
578
|
+
<li><code>meerschaum.utils.dtypes.deserialize_base64()</code></li>
|
579
|
+
<li><code>meerschaum.utils.dtypes.deserialize_bytes_string()</code></li>
|
580
|
+
<li><code>meerschaum.utils.dtypes.deserialize_geometry()</code></li>
|
581
|
+
<li><code>meerschaum.utils.dtypes.dtype_is_special()</code></li>
|
582
|
+
<li><code>meerschaum.utils.dtypes.encode_bytes_for_bytea()</code></li>
|
583
|
+
<li><code>meerschaum.utils.dtypes.geometry_is_wkt()</code></li>
|
584
|
+
<li><code>meerschaum.utils.dtypes.get_current_timestamp()</code></li>
|
585
|
+
<li><code>meerschaum.utils.dtypes.get_geometry_type_srid()</code></li>
|
563
586
|
<li><code>meerschaum.utils.dtypes.is_dtype_numeric()</code></li>
|
587
|
+
<li><code>meerschaum.utils.dtypes.json_serialize_value()</code></li>
|
564
588
|
<li><code>meerschaum.utils.dtypes.none_if_null()</code></li>
|
589
|
+
<li><code>meerschaum.utils.dtypes.project_geometry()</code></li>
|
565
590
|
<li><code>meerschaum.utils.dtypes.quantize_decimal()</code></li>
|
591
|
+
<li><code>meerschaum.utils.dtypes.serialize_bytes()</code></li>
|
592
|
+
<li><code>meerschaum.utils.dtypes.serialize_datetime()</code></li>
|
593
|
+
<li><code>meerschaum.utils.dtypes.serialize_date()</code></li>
|
594
|
+
<li><code>meerschaum.utils.dtypes.serialize_decimal()</code></li>
|
595
|
+
<li><code>meerschaum.utils.dtypes.serialize_geometry()</code></li>
|
596
|
+
<li><code>meerschaum.utils.dtypes.to_datetime()</code></li>
|
566
597
|
<li><code>meerschaum.utils.dtypes.to_pandas_dtype()</code></li>
|
567
598
|
<li><code>meerschaum.utils.dtypes.value_is_null()</code></li>
|
599
|
+
<li><code>meerschaum.utils.dtypes.get_current_timestamp()</code></li>
|
600
|
+
<li><code>meerschaum.utils.dtypes.dtype_is_special()</code></li>
|
601
|
+
<li><code>meerschaum.utils.dtypes.get_next_precision_unit()</code></li>
|
602
|
+
<li><code>meerschaum.utils.dtypes.round_time()</code></li>
|
603
|
+
</ul>
|
604
|
+
</details>
|
605
|
+
</ul>
|
606
|
+
|
607
|
+
<ul>
|
608
|
+
<details>
|
609
|
+
<summary>
|
610
|
+
<code>meerschaum.utils.dtypes.sql</code><br>
|
611
|
+
Work with SQL data types.<br>
|
612
|
+
</summary>
|
613
|
+
<p></p>
|
614
|
+
<ul>
|
568
615
|
<li><code>meerschaum.utils.dtypes.sql.get_pd_type_from_db_type()</code></li>
|
569
616
|
<li><code>meerschaum.utils.dtypes.sql.get_db_type_from_pd_type()</code></li>
|
617
|
+
<li><code>meerschaum.utils.dtypes.sql.get_numeric_precision_scale()</code></li>
|
570
618
|
</ul>
|
571
619
|
</details>
|
572
620
|
</ul>
|
@@ -605,7 +653,6 @@ def init_dash(dash_app):
|
|
605
653
|
<p></p>
|
606
654
|
<ul>
|
607
655
|
<li><code>meerschaum.utils.misc.items_str()</code></li>
|
608
|
-
<li><code>meerschaum.utils.misc.round_time()</code></li>
|
609
656
|
<li><code>meerschaum.utils.misc.is_int()</code></li>
|
610
657
|
<li><code>meerschaum.utils.misc.interval_str()</code></li>
|
611
658
|
<li><code>meerschaum.utils.misc.filter_keywords()</code></li>
|
@@ -781,10 +781,11 @@ class Shell(cmd.Cmd):
|
|
781
781
|
instance_keys += ':main'
|
782
782
|
|
783
783
|
conn_attrs = parse_instance_keys(instance_keys, construct=False, debug=debug)
|
784
|
-
|
785
|
-
|
786
|
-
|
787
|
-
|
784
|
+
conn_keys = (
|
785
|
+
str(get_connector(debug=debug))
|
786
|
+
if conn_attrs is None
|
787
|
+
else instance_keys
|
788
|
+
)
|
788
789
|
|
789
790
|
shell_attrs['instance_keys'] = conn_keys
|
790
791
|
|
meerschaum/_internal/static.py
CHANGED
@@ -14,8 +14,8 @@ __all__ = ('STATIC_CONFIG',)
|
|
14
14
|
|
15
15
|
_default_create_engine_args = {
|
16
16
|
# 'method': 'multi',
|
17
|
-
'pool_size': 5,
|
18
|
-
'max_overflow': 10,
|
17
|
+
'pool_size': (os.cpu_count() or 5),
|
18
|
+
'max_overflow': (os.cpu_count() or 10),
|
19
19
|
'pool_recycle': 3600,
|
20
20
|
'connect_args': {},
|
21
21
|
}
|
@@ -287,6 +287,11 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
287
287
|
'connectors': {
|
288
288
|
'default_label': 'main',
|
289
289
|
},
|
290
|
+
'dtypes': {
|
291
|
+
'datetime': {
|
292
|
+
'default_precision_unit': 'microsecond',
|
293
|
+
},
|
294
|
+
},
|
290
295
|
'stack': {
|
291
296
|
'dollar_standin': '<DOLLAR>', ### Temporary replacement for docker-compose.yaml.
|
292
297
|
},
|
@@ -313,8 +318,6 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
313
318
|
'dtypes': {
|
314
319
|
'min_ratio_columns_changed_for_full_astype': 0.5,
|
315
320
|
},
|
316
|
-
'exists_timeout_seconds': 5.0,
|
317
|
-
'static_schema_cache_seconds': 60.0,
|
318
321
|
'max_bound_time_days': 36525,
|
319
322
|
},
|
320
323
|
'jobs': {
|
@@ -349,24 +352,5 @@ STATIC_CONFIG: Dict[str, Any] = {
|
|
349
352
|
'users:delete': "Delete the associated user account (or other users for admins).",
|
350
353
|
},
|
351
354
|
},
|
352
|
-
|
353
|
-
'name': 'meerschaum',
|
354
|
-
'formal_name': 'Meerschaum',
|
355
|
-
'app_id': 'io.meerschaum',
|
356
|
-
'description': 'Sync Time-Series Pipes with Meerschaum',
|
357
|
-
'url': 'https://meerschaum.io',
|
358
|
-
'project_urls': {
|
359
|
-
'Documentation': 'https://docs.meerschaum.io',
|
360
|
-
'Changelog': 'https://meerschaum.io/news/changelog',
|
361
|
-
'GitHub': 'https://github.com/bmeares/Meerschaum',
|
362
|
-
'Homepage': 'https://meerschaum.io',
|
363
|
-
'Donate': 'https://github.com/sponsors/bmeares',
|
364
|
-
'Discord': 'https://discord.gg/8U8qMUjvcc',
|
365
|
-
},
|
366
|
-
'author': 'Bennett Meares',
|
367
|
-
'author_email': 'bennett.meares@gmail.com',
|
368
|
-
'maintainer_email': 'bennett.meares@gmail.com',
|
369
|
-
'license': 'Apache Software License 2.0',
|
370
|
-
'license_files': ('LICENSE',),
|
371
|
-
},
|
355
|
+
|
372
356
|
}
|
meerschaum/actions/bootstrap.py
CHANGED
@@ -429,7 +429,7 @@ def _bootstrap_plugins(
|
|
429
429
|
action = [prompt("Enter the name of your new plugin:")]
|
430
430
|
|
431
431
|
for plugin_name in action:
|
432
|
-
bootstrap_success, bootstrap_msg = bootstrap_plugin(plugin_name)
|
432
|
+
bootstrap_success, bootstrap_msg = bootstrap_plugin(plugin_name, debug=debug)
|
433
433
|
if not bootstrap_success:
|
434
434
|
return bootstrap_success, bootstrap_msg
|
435
435
|
|
meerschaum/actions/edit.py
CHANGED
@@ -552,7 +552,12 @@ def _edit_tokens(
|
|
552
552
|
dateutil_parser = mrsm.attempt_import('dateutil.parser')
|
553
553
|
|
554
554
|
if not action:
|
555
|
-
return
|
555
|
+
return (
|
556
|
+
False, (
|
557
|
+
"Provide token labels or IDs for the tokens to edit\n"
|
558
|
+
" (run `show tokens` to see registered tokens)."
|
559
|
+
)
|
560
|
+
)
|
556
561
|
|
557
562
|
conn = parse_instance_keys(mrsm_instance)
|
558
563
|
|
@@ -628,8 +633,6 @@ def _edit_tokens(
|
|
628
633
|
|
629
634
|
return True, msg
|
630
635
|
|
631
|
-
|
632
|
-
|
633
636
|
|
634
637
|
### NOTE: This must be the final statement of the module.
|
635
638
|
### Any subactions added below these lines will not
|
meerschaum/actions/start.py
CHANGED
meerschaum/actions/verify.py
CHANGED
@@ -57,8 +57,11 @@ def _verify_packages(
|
|
57
57
|
Verify the versions of packages.
|
58
58
|
"""
|
59
59
|
from meerschaum.utils.packages import (
|
60
|
-
attempt_import,
|
61
|
-
|
60
|
+
attempt_import,
|
61
|
+
all_packages,
|
62
|
+
is_installed,
|
63
|
+
venv_contains_package,
|
64
|
+
manually_import_module,
|
62
65
|
)
|
63
66
|
|
64
67
|
venv_packages, base_packages, miss_packages = [], [], []
|
@@ -78,12 +81,6 @@ def _verify_packages(
|
|
78
81
|
)
|
79
82
|
_where_list.append(import_name)
|
80
83
|
|
81
|
-
if 'flask_compress' in venv_packages or 'dash' in venv_packages:
|
82
|
-
flask_compress = attempt_import('flask_compress', lazy=False, debug=debug)
|
83
|
-
_monkey_patch_get_distribution('flask-compress', flask_compress.__version__)
|
84
|
-
if 'flask_compress' in venv_packages:
|
85
|
-
venv_packages.remove('flask_compress')
|
86
|
-
|
87
84
|
for import_name in base_packages:
|
88
85
|
manually_import_module(import_name, debug=debug, venv=None)
|
89
86
|
for import_name in venv_packages:
|
meerschaum/api/__init__.py
CHANGED
@@ -217,8 +217,9 @@ def get_pipe(
|
|
217
217
|
status_code=403,
|
218
218
|
detail="Unable to serve any pipes with connector keys `mrsm` over the API.",
|
219
219
|
)
|
220
|
+
|
220
221
|
pipe = mrsm.Pipe(connector_keys, metric_key, location_key, mrsm_instance=instance_keys)
|
221
|
-
if is_pipe_registered(pipe, pipes(instance_keys)):
|
222
|
+
if is_pipe_registered(pipe, pipes(instance_keys, refresh=False)):
|
222
223
|
return pipes(instance_keys, refresh=refresh)[connector_keys][metric_key][location_key]
|
223
224
|
return pipe
|
224
225
|
|
meerschaum/api/dash/__init__.py
CHANGED
@@ -12,10 +12,8 @@ from meerschaum.utils.packages import (
|
|
12
12
|
attempt_import,
|
13
13
|
import_dcc,
|
14
14
|
import_html,
|
15
|
-
_monkey_patch_get_distribution,
|
16
15
|
)
|
17
16
|
flask_compress = attempt_import('flask_compress', lazy=False)
|
18
|
-
_monkey_patch_get_distribution('flask-compress', flask_compress.__version__)
|
19
17
|
dash = attempt_import('dash', lazy=False)
|
20
18
|
|
21
19
|
from meerschaum.utils.typing import List, Optional
|
@@ -10,6 +10,7 @@ from meerschaum.api import debug as _debug
|
|
10
10
|
import meerschaum.api.dash.callbacks.dashboard
|
11
11
|
import meerschaum.api.dash.callbacks.login
|
12
12
|
import meerschaum.api.dash.callbacks.plugins
|
13
|
+
import meerschaum.api.dash.callbacks.tokens
|
13
14
|
import meerschaum.api.dash.callbacks.jobs
|
14
15
|
import meerschaum.api.dash.callbacks.register
|
15
16
|
import meerschaum.api.dash.callbacks.pipes
|
@@ -72,8 +72,6 @@ keys_state = (
|
|
72
72
|
State({'type': 'input-flags-dropdown', 'index': ALL}, 'value'),
|
73
73
|
State({'type': 'input-flags-dropdown-text', 'index': ALL}, 'value'),
|
74
74
|
State('instance-select', 'value'),
|
75
|
-
State('content-div-right', 'children'),
|
76
|
-
State('success-alert-div', 'children'),
|
77
75
|
State('session-store', 'data'),
|
78
76
|
)
|
79
77
|
|
@@ -103,15 +101,20 @@ omit_actions = {
|
|
103
101
|
_paths = {
|
104
102
|
'login' : pages.login.layout,
|
105
103
|
'' : pages.dashboard.layout,
|
104
|
+
'pipes' : pages.pipes.layout,
|
106
105
|
'plugins' : pages.plugins.layout,
|
106
|
+
'tokens' : pages.tokens.layout,
|
107
107
|
'register': pages.register.layout,
|
108
108
|
'pipes' : pages.pipes.layout,
|
109
|
-
'
|
109
|
+
'jobs' : pages.jobs.layout,
|
110
110
|
}
|
111
|
-
_required_login = {''}
|
111
|
+
_required_login = {'', 'tokens', 'jobs', 'pipes'}
|
112
112
|
_pages = {
|
113
113
|
'Web Console': '/dash/',
|
114
|
+
'Pipes': '/dash/pipes',
|
114
115
|
'Plugins': '/dash/plugins',
|
116
|
+
'Tokens': '/dash/tokens',
|
117
|
+
'Jobs': '/dash/jobs',
|
115
118
|
}
|
116
119
|
|
117
120
|
|
@@ -495,16 +498,12 @@ def update_flags(input_flags_dropdown_values, n_clicks, input_flags_texts):
|
|
495
498
|
|
496
499
|
@dash_app.callback(
|
497
500
|
Output('connector-keys-dropdown', 'options'),
|
498
|
-
Output('connector-keys-list', 'children'),
|
499
501
|
Output('connector-keys-dropdown', 'value'),
|
500
502
|
Output('metric-keys-dropdown', 'options'),
|
501
|
-
Output('metric-keys-list', 'children'),
|
502
503
|
Output('metric-keys-dropdown', 'value'),
|
503
504
|
Output('location-keys-dropdown', 'options'),
|
504
|
-
Output('location-keys-list', 'children'),
|
505
505
|
Output('location-keys-dropdown', 'value'),
|
506
506
|
Output('tags-dropdown', 'options'),
|
507
|
-
Output('tags-list', 'children'),
|
508
507
|
Output('tags-dropdown', 'value'),
|
509
508
|
Output('instance-select', 'value'),
|
510
509
|
Output('instance-alert-div', 'children'),
|
@@ -664,22 +663,14 @@ def update_keys_options(
|
|
664
663
|
for _tag in _tags_options
|
665
664
|
]
|
666
665
|
]
|
667
|
-
_connectors_datalist = [html.Option(value=o['value']) for o in _connectors_options]
|
668
|
-
_metrics_datalist = [html.Option(value=o['value']) for o in _metrics_options]
|
669
|
-
_locations_datalist = [html.Option(value=o['value']) for o in _locations_options]
|
670
|
-
_tags_datalist = [html.Option(value=o['value']) for o in _tags_options]
|
671
666
|
return (
|
672
667
|
_connectors_options,
|
673
|
-
_connectors_datalist,
|
674
668
|
connector_keys,
|
675
669
|
_metrics_options,
|
676
|
-
_metrics_datalist,
|
677
670
|
metric_keys,
|
678
671
|
_locations_options,
|
679
|
-
_locations_datalist,
|
680
672
|
location_keys,
|
681
673
|
_tags_options,
|
682
|
-
_tags_datalist,
|
683
674
|
tags,
|
684
675
|
(instance_keys if update_instance_keys else dash.no_update),
|
685
676
|
instance_alerts,
|
@@ -856,6 +847,7 @@ dash_app.clientside_callback(
|
|
856
847
|
@dash_app.callback(
|
857
848
|
Output("download-dataframe-csv", "data"),
|
858
849
|
Input({'type': 'pipe-download-csv-button', 'index': ALL}, 'n_clicks'),
|
850
|
+
prevent_initial_call=True,
|
859
851
|
)
|
860
852
|
def download_pipe_csv(n_clicks):
|
861
853
|
"""
|
@@ -885,6 +877,7 @@ def download_pipe_csv(n_clicks):
|
|
885
877
|
Output({'type': 'pipe-accordion', 'index': MATCH}, 'children'),
|
886
878
|
Input({'type': 'pipe-accordion', 'index': MATCH}, 'active_item'),
|
887
879
|
State('session-store', 'data'),
|
880
|
+
prevent_initial_call=True,
|
888
881
|
)
|
889
882
|
def update_pipe_accordion(item, session_store_data):
|
890
883
|
"""
|
@@ -908,7 +901,8 @@ def update_pipe_accordion(item, session_store_data):
|
|
908
901
|
@dash_app.callback(
|
909
902
|
Output({'type': 'update-parameters-success-div', 'index': MATCH}, 'children'),
|
910
903
|
Input({'type': 'update-parameters-button', 'index': MATCH}, 'n_clicks'),
|
911
|
-
State({'type': 'parameters-editor', 'index': MATCH}, 'value')
|
904
|
+
State({'type': 'parameters-editor', 'index': MATCH}, 'value'),
|
905
|
+
prevent_initial_call=True,
|
912
906
|
)
|
913
907
|
def update_pipe_parameters_click(n_clicks, parameters_editor_text):
|
914
908
|
if not n_clicks:
|
@@ -942,6 +936,7 @@ def update_pipe_parameters_click(n_clicks, parameters_editor_text):
|
|
942
936
|
Output({'type': 'update-sql-success-div', 'index': MATCH}, 'children'),
|
943
937
|
Input({'type': 'update-sql-button', 'index': MATCH}, 'n_clicks'),
|
944
938
|
State({'type': 'sql-editor', 'index': MATCH}, 'value'),
|
939
|
+
prevent_initial_call=True,
|
945
940
|
)
|
946
941
|
def update_pipe_sql_click(n_clicks, sql_editor_text):
|
947
942
|
if not n_clicks:
|
@@ -968,7 +963,8 @@ def update_pipe_sql_click(n_clicks, sql_editor_text):
|
|
968
963
|
@dash_app.callback(
|
969
964
|
Output({'type': 'sync-success-div', 'index': MATCH}, 'children'),
|
970
965
|
Input({'type': 'update-sync-button', 'index': MATCH}, 'n_clicks'),
|
971
|
-
State({'type': 'sync-editor', 'index': MATCH}, 'value')
|
966
|
+
State({'type': 'sync-editor', 'index': MATCH}, 'value'),
|
967
|
+
prevent_initial_call=True,
|
972
968
|
)
|
973
969
|
def sync_documents_click(n_clicks, sync_editor_text):
|
974
970
|
if not n_clicks:
|
@@ -1014,6 +1010,7 @@ def sync_documents_click(n_clicks, sync_editor_text):
|
|
1014
1010
|
State({'type': 'limit-input', 'index': MATCH}, 'value'),
|
1015
1011
|
State({'type': 'query-data-begin-input', 'index': MATCH}, 'value'),
|
1016
1012
|
State({'type': 'query-data-end-input', 'index': MATCH}, 'value'),
|
1013
|
+
prevent_initial_call=True,
|
1017
1014
|
)
|
1018
1015
|
def query_data_click(n_clicks, query_editor_text, limit_value, begin, end):
|
1019
1016
|
triggered = dash.callback_context.triggered
|
@@ -1024,7 +1021,7 @@ def query_data_click(n_clicks, query_editor_text, limit_value, begin, end):
|
|
1024
1021
|
raise PreventUpdate
|
1025
1022
|
|
1026
1023
|
try:
|
1027
|
-
params_query =
|
1024
|
+
params_query = string_to_dict(query_editor_text)
|
1028
1025
|
except Exception as e:
|
1029
1026
|
return alert_from_success_tuple((False, f"Invalid query:\n{e}"))
|
1030
1027
|
|
@@ -1130,6 +1127,7 @@ def toggle_navbar_collapse(n_clicks: Optional[int], is_open: bool) -> bool:
|
|
1130
1127
|
Output('session-store', 'data'),
|
1131
1128
|
Input("sign-out-button", "n_clicks"),
|
1132
1129
|
State('session-store', 'data'),
|
1130
|
+
prevent_initial_call=True,
|
1133
1131
|
)
|
1134
1132
|
def sign_out_button_click(
|
1135
1133
|
n_clicks: Optional[int],
|
@@ -1150,6 +1148,7 @@ def sign_out_button_click(
|
|
1150
1148
|
Output({'type': 'parameters-editor', 'index': MATCH}, 'value'),
|
1151
1149
|
Input({'type': 'parameters-as-yaml-button', 'index': MATCH}, 'n_clicks'),
|
1152
1150
|
Input({'type': 'parameters-as-json-button', 'index': MATCH}, 'n_clicks'),
|
1151
|
+
prevent_initial_call=True,
|
1153
1152
|
)
|
1154
1153
|
def parameters_as_yaml_or_json_click(
|
1155
1154
|
yaml_n_clicks: Optional[int],
|
@@ -1178,6 +1177,7 @@ def parameters_as_yaml_or_json_click(
|
|
1178
1177
|
Output({'type': 'sync-editor', 'index': MATCH}, 'value'),
|
1179
1178
|
Input({'type': 'sync-as-json-button', 'index': MATCH}, 'n_clicks'),
|
1180
1179
|
Input({'type': 'sync-as-lines-button', 'index': MATCH}, 'n_clicks'),
|
1180
|
+
prevent_initial_call=True,
|
1181
1181
|
)
|
1182
1182
|
def sync_as_json_or_lines_click(
|
1183
1183
|
json_n_clicks: Optional[int],
|
@@ -1206,6 +1206,7 @@ def sync_as_json_or_lines_click(
|
|
1206
1206
|
Output('pages-offcanvas', 'children'),
|
1207
1207
|
Input('logo-img', 'n_clicks'),
|
1208
1208
|
State('pages-offcanvas', 'is_open'),
|
1209
|
+
prevent_initial_call=True,
|
1209
1210
|
)
|
1210
1211
|
def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
|
1211
1212
|
"""
|
@@ -13,12 +13,13 @@ import time
|
|
13
13
|
import traceback
|
14
14
|
from datetime import datetime, timezone
|
15
15
|
|
16
|
+
from meerschaum.jobs import get_jobs
|
16
17
|
from meerschaum.utils.typing import Optional, Dict, Any
|
17
18
|
from meerschaum.api import CHECK_UPDATE
|
18
19
|
from meerschaum.api.dash import dash_app
|
19
20
|
from meerschaum.api.dash.sessions import get_username_from_session
|
20
21
|
from meerschaum.utils.packages import attempt_import, import_dcc, import_html
|
21
|
-
from meerschaum.api.dash.components import alert_from_success_tuple
|
22
|
+
from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
|
22
23
|
from meerschaum.api.dash.jobs import (
|
23
24
|
build_job_card,
|
24
25
|
build_manage_job_buttons_div_children,
|
@@ -251,17 +252,22 @@ def render_job_page_from_url(
|
|
251
252
|
session_data: Optional[Dict[str, Any]],
|
252
253
|
):
|
253
254
|
"""
|
254
|
-
Load the `/
|
255
|
+
Load the `/dash/jobs/{name}` page.
|
255
256
|
"""
|
256
|
-
if not str(pathname).startswith('/dash/
|
257
|
+
if not str(pathname).startswith('/dash/jobs'):
|
257
258
|
return no_update
|
258
259
|
|
259
260
|
session_id = (session_data or {}).get('session-id', None)
|
260
261
|
authenticated = is_session_authenticated(str(session_id))
|
261
262
|
|
262
|
-
job_name = pathname.replace('/dash/
|
263
|
+
job_name = pathname.replace('/dash/jobs', '').lstrip('/').rstrip('/')
|
263
264
|
if not job_name:
|
264
|
-
|
265
|
+
jobs = get_jobs(executor_keys='local', combine_local_and_systemd=True)
|
266
|
+
cards = [
|
267
|
+
build_job_card(job, authenticated=authenticated, include_follow=False)
|
268
|
+
for job in jobs.values()
|
269
|
+
]
|
270
|
+
return [html.Br(), build_cards_grid(cards, 3), html.Br()]
|
265
271
|
|
266
272
|
job = _get_job(job_name)
|
267
273
|
if not job.exists():
|
@@ -9,11 +9,17 @@ from urllib.parse import parse_qs
|
|
9
9
|
|
10
10
|
from dash.dependencies import Input, Output, State
|
11
11
|
from dash import no_update
|
12
|
+
from dash.exceptions import PreventUpdate
|
12
13
|
|
13
14
|
import meerschaum as mrsm
|
14
15
|
from meerschaum.api.dash import dash_app
|
15
|
-
from meerschaum.api.dash.
|
16
|
-
from meerschaum.api import
|
16
|
+
from meerschaum.api.dash.components import alert_from_success_tuple, build_cards_grid
|
17
|
+
from meerschaum.api.dash.pipes import (
|
18
|
+
build_pipe_card,
|
19
|
+
build_pipes_dropdown_keys_row,
|
20
|
+
build_pipes_tags_dropdown,
|
21
|
+
)
|
22
|
+
from meerschaum.api import CHECK_UPDATE, get_api_connector
|
17
23
|
from meerschaum.utils.packages import import_html, import_dcc
|
18
24
|
from meerschaum.api.dash.sessions import is_session_authenticated
|
19
25
|
from meerschaum.utils.typing import Optional, Dict, Any
|
@@ -36,16 +42,74 @@ def render_pipe_page_from_url(
|
|
36
42
|
|
37
43
|
session_id = (session_data or {}).get('session-id', None)
|
38
44
|
authenticated = is_session_authenticated(str(session_id))
|
45
|
+
query_params = parse_qs(pipe_search.lstrip('?')) if pipe_search else {}
|
46
|
+
instance = query_params.get('instance', [None])[0] or str(get_api_connector())
|
47
|
+
tags = query_params.get('tags', [None])[0] or []
|
48
|
+
if isinstance(tags, str):
|
49
|
+
tags = tags.split(',')
|
50
|
+
|
51
|
+
connector_keys = query_params.get('connector_keys', [None])[0] or []
|
52
|
+
if isinstance(connector_keys, str):
|
53
|
+
connector_keys = connector_keys.split(',')
|
54
|
+
|
55
|
+
metric_keys = query_params.get('metric_keys', [None])[0] or []
|
56
|
+
if isinstance(metric_keys, str):
|
57
|
+
metric_keys = metric_keys.split(',')
|
58
|
+
|
59
|
+
location_keys = query_params.get('location_keys', [None])[0] or []
|
60
|
+
if isinstance(location_keys, str):
|
61
|
+
location_keys = location_keys.split(',')
|
62
|
+
|
63
|
+
instance_connector = mrsm.get_connector(instance)
|
64
|
+
if instance_connector is None:
|
65
|
+
return [
|
66
|
+
html.Br(),
|
67
|
+
alert_from_success_tuple((False, f"Invalid instance keys '{instance}'.")),
|
68
|
+
html.Br(),
|
69
|
+
]
|
39
70
|
|
40
71
|
keys = pathname.replace('/dash/pipes', '').lstrip('/').rstrip('/').split('/')
|
41
72
|
if len(keys) not in (2, 3):
|
42
|
-
|
73
|
+
pipes = mrsm.get_pipes(
|
74
|
+
as_list=True,
|
75
|
+
connector_keys=connector_keys,
|
76
|
+
metric_keys=metric_keys,
|
77
|
+
location_keys=location_keys,
|
78
|
+
tags=tags,
|
79
|
+
instance=instance_connector,
|
80
|
+
)
|
81
|
+
cards = [
|
82
|
+
build_pipe_card(pipe, authenticated=authenticated, include_manage=False)
|
83
|
+
for pipe in pipes
|
84
|
+
]
|
85
|
+
return [
|
86
|
+
html.Div([
|
87
|
+
html.Br(),
|
88
|
+
build_pipes_dropdown_keys_row(
|
89
|
+
connector_keys,
|
90
|
+
metric_keys,
|
91
|
+
location_keys,
|
92
|
+
tags,
|
93
|
+
pipes,
|
94
|
+
instance_connector,
|
95
|
+
),
|
96
|
+
html.Br(),
|
97
|
+
build_pipes_tags_dropdown(
|
98
|
+
connector_keys,
|
99
|
+
metric_keys,
|
100
|
+
location_keys,
|
101
|
+
tags,
|
102
|
+
instance,
|
103
|
+
),
|
104
|
+
]),
|
105
|
+
html.Br(),
|
106
|
+
build_cards_grid(cards, 1),
|
107
|
+
html.Br(),
|
108
|
+
]
|
43
109
|
|
44
110
|
ck = keys[0]
|
45
111
|
mk = keys[1]
|
46
112
|
lk = keys[2] if len(keys) == 3 else None
|
47
|
-
query_params = parse_qs(pipe_search.lstrip('?')) if pipe_search else {}
|
48
|
-
instance = query_params.get('instance', [None])[0]
|
49
113
|
|
50
114
|
pipe = mrsm.Pipe(ck, mk, lk, instance=instance)
|
51
115
|
return [
|
@@ -53,3 +117,40 @@ def render_pipe_page_from_url(
|
|
53
117
|
build_pipe_card(pipe, authenticated=authenticated, include_manage=False),
|
54
118
|
html.Br(),
|
55
119
|
]
|
120
|
+
|
121
|
+
|
122
|
+
@dash_app.callback(
|
123
|
+
Output('pipes-location', 'search'),
|
124
|
+
Input('pipes-connector-keys-dropdown', 'value'),
|
125
|
+
Input('pipes-metric-keys-dropdown', 'value'),
|
126
|
+
Input('pipes-location-keys-dropdown', 'value'),
|
127
|
+
Input('pipes-tags-dropdown', 'value'),
|
128
|
+
)
|
129
|
+
def update_location_on_pipes_filter_change(connector_keys, metric_keys, location_keys, tags):
|
130
|
+
"""
|
131
|
+
Update the URL parameters when clicking the dropdowns.
|
132
|
+
"""
|
133
|
+
if not any((connector_keys or []) + (metric_keys or []) + (location_keys or []) + (tags or [])):
|
134
|
+
return ''
|
135
|
+
|
136
|
+
search_str = "?"
|
137
|
+
|
138
|
+
if connector_keys:
|
139
|
+
search_str += "connector_keys=" + ','.join(connector_keys)
|
140
|
+
if metric_keys or location_keys or tags:
|
141
|
+
search_str += '&'
|
142
|
+
|
143
|
+
if metric_keys:
|
144
|
+
search_str += "metric_keys=" + ','.join(metric_keys)
|
145
|
+
if location_keys or tags:
|
146
|
+
search_str += '&'
|
147
|
+
|
148
|
+
if location_keys:
|
149
|
+
search_str += "location_keys=" + ','.join(location_keys)
|
150
|
+
if tags:
|
151
|
+
search_str += '&'
|
152
|
+
|
153
|
+
if tags:
|
154
|
+
search_str += "tags=" + ','.join(tags)
|
155
|
+
|
156
|
+
return search_str
|
@@ -163,7 +163,7 @@ def register_token_click(
|
|
163
163
|
token = Token(
|
164
164
|
label=(name or None),
|
165
165
|
user=get_user_from_session(session_id),
|
166
|
-
expiration=(datetime.fromisoformat(f"{expiration}
|
166
|
+
expiration=(datetime.fromisoformat(f"{expiration}") if expiration is not None else None),
|
167
167
|
)
|
168
168
|
return False, True, build_tokens_register_output_modal(token)
|
169
169
|
|
meerschaum/api/dash/jobs.py
CHANGED
@@ -9,7 +9,8 @@ import meerschaum.api.dash.pages.error
|
|
9
9
|
import meerschaum.api.dash.pages.login
|
10
10
|
import meerschaum.api.dash.pages.dashboard
|
11
11
|
import meerschaum.api.dash.pages.plugins
|
12
|
+
import meerschaum.api.dash.pages.tokens
|
12
13
|
import meerschaum.api.dash.pages.register
|
13
14
|
import meerschaum.api.dash.pages.pipes
|
14
|
-
import meerschaum.api.dash.pages.
|
15
|
+
import meerschaum.api.dash.pages.jobs
|
15
16
|
import meerschaum.api.dash.pages.settings
|