meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +5 -2
- meerschaum/_internal/__init__.py +1 -0
- meerschaum/_internal/arguments/_parse_arguments.py +4 -4
- meerschaum/_internal/arguments/_parser.py +33 -4
- meerschaum/_internal/cli/__init__.py +6 -0
- meerschaum/_internal/cli/daemons.py +103 -0
- meerschaum/_internal/cli/entry.py +220 -0
- meerschaum/_internal/cli/workers.py +435 -0
- meerschaum/_internal/docs/index.py +48 -2
- meerschaum/_internal/entry.py +50 -14
- meerschaum/_internal/shell/Shell.py +121 -29
- meerschaum/_internal/shell/__init__.py +4 -1
- meerschaum/_internal/static.py +359 -0
- meerschaum/_internal/term/TermPageHandler.py +1 -2
- meerschaum/_internal/term/__init__.py +40 -6
- meerschaum/_internal/term/tools.py +33 -8
- meerschaum/actions/__init__.py +6 -4
- meerschaum/actions/api.py +53 -13
- meerschaum/actions/attach.py +1 -0
- meerschaum/actions/bootstrap.py +8 -8
- meerschaum/actions/delete.py +4 -2
- meerschaum/actions/edit.py +171 -25
- meerschaum/actions/login.py +8 -8
- meerschaum/actions/register.py +143 -6
- meerschaum/actions/reload.py +22 -5
- meerschaum/actions/restart.py +14 -0
- meerschaum/actions/show.py +184 -31
- meerschaum/actions/start.py +166 -17
- meerschaum/actions/stop.py +38 -2
- meerschaum/actions/sync.py +7 -2
- meerschaum/actions/tag.py +9 -8
- meerschaum/actions/verify.py +5 -8
- meerschaum/api/__init__.py +45 -15
- meerschaum/api/_events.py +46 -4
- meerschaum/api/_oauth2.py +162 -9
- meerschaum/api/_tokens.py +102 -0
- meerschaum/api/dash/__init__.py +0 -3
- meerschaum/api/dash/callbacks/__init__.py +1 -0
- meerschaum/api/dash/callbacks/custom.py +4 -3
- meerschaum/api/dash/callbacks/dashboard.py +228 -117
- meerschaum/api/dash/callbacks/jobs.py +14 -7
- meerschaum/api/dash/callbacks/login.py +10 -1
- meerschaum/api/dash/callbacks/pipes.py +194 -14
- meerschaum/api/dash/callbacks/plugins.py +0 -1
- meerschaum/api/dash/callbacks/register.py +10 -3
- meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
- meerschaum/api/dash/callbacks/tokens.py +389 -0
- meerschaum/api/dash/components.py +36 -15
- meerschaum/api/dash/jobs.py +1 -1
- meerschaum/api/dash/keys.py +35 -93
- meerschaum/api/dash/pages/__init__.py +2 -1
- meerschaum/api/dash/pages/dashboard.py +1 -20
- meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
- meerschaum/api/dash/pages/login.py +2 -2
- meerschaum/api/dash/pages/pipes.py +16 -5
- meerschaum/api/dash/pages/settings/password_reset.py +1 -1
- meerschaum/api/dash/pages/tokens.py +53 -0
- meerschaum/api/dash/pipes.py +438 -88
- meerschaum/api/dash/sessions.py +12 -0
- meerschaum/api/dash/tokens.py +603 -0
- meerschaum/api/dash/websockets.py +1 -1
- meerschaum/api/dash/webterm.py +18 -6
- meerschaum/api/models/__init__.py +23 -3
- meerschaum/api/models/_actions.py +22 -0
- meerschaum/api/models/_pipes.py +91 -7
- meerschaum/api/models/_tokens.py +81 -0
- meerschaum/api/resources/static/css/dash.css +16 -0
- meerschaum/api/resources/static/js/terminado.js +3 -0
- meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
- meerschaum/api/resources/templates/termpage.html +13 -0
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +3 -4
- meerschaum/api/routes/_connectors.py +3 -7
- meerschaum/api/routes/_jobs.py +26 -35
- meerschaum/api/routes/_login.py +120 -15
- meerschaum/api/routes/_misc.py +5 -10
- meerschaum/api/routes/_pipes.py +178 -143
- meerschaum/api/routes/_plugins.py +38 -28
- meerschaum/api/routes/_tokens.py +236 -0
- meerschaum/api/routes/_users.py +47 -35
- meerschaum/api/routes/_version.py +3 -3
- meerschaum/api/routes/_webterm.py +3 -3
- meerschaum/config/__init__.py +100 -30
- meerschaum/config/_default.py +132 -64
- meerschaum/config/_edit.py +38 -32
- meerschaum/config/_formatting.py +2 -0
- meerschaum/config/_patch.py +10 -8
- meerschaum/config/_paths.py +133 -13
- meerschaum/config/_read_config.py +87 -36
- meerschaum/config/_sync.py +6 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/environment.py +262 -0
- meerschaum/config/stack/__init__.py +37 -15
- meerschaum/config/static.py +18 -0
- meerschaum/connectors/_Connector.py +11 -6
- meerschaum/connectors/__init__.py +41 -22
- meerschaum/connectors/api/_APIConnector.py +34 -6
- meerschaum/connectors/api/_actions.py +2 -2
- meerschaum/connectors/api/_jobs.py +12 -1
- meerschaum/connectors/api/_login.py +33 -7
- meerschaum/connectors/api/_misc.py +2 -2
- meerschaum/connectors/api/_pipes.py +23 -32
- meerschaum/connectors/api/_plugins.py +2 -2
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/api/_tokens.py +146 -0
- meerschaum/connectors/api/_users.py +70 -58
- meerschaum/connectors/instance/_InstanceConnector.py +83 -0
- meerschaum/connectors/instance/__init__.py +10 -0
- meerschaum/connectors/instance/_pipes.py +442 -0
- meerschaum/connectors/instance/_plugins.py +159 -0
- meerschaum/connectors/instance/_tokens.py +317 -0
- meerschaum/connectors/instance/_users.py +188 -0
- meerschaum/connectors/parse.py +5 -2
- meerschaum/connectors/sql/_SQLConnector.py +22 -5
- meerschaum/connectors/sql/_cli.py +12 -11
- meerschaum/connectors/sql/_create_engine.py +12 -168
- meerschaum/connectors/sql/_fetch.py +2 -18
- meerschaum/connectors/sql/_pipes.py +295 -278
- meerschaum/connectors/sql/_plugins.py +29 -0
- meerschaum/connectors/sql/_sql.py +47 -22
- meerschaum/connectors/sql/_users.py +36 -2
- meerschaum/connectors/sql/tables/__init__.py +254 -122
- meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
- meerschaum/connectors/valkey/_pipes.py +60 -31
- meerschaum/connectors/valkey/_plugins.py +2 -26
- meerschaum/core/Pipe/__init__.py +115 -85
- meerschaum/core/Pipe/_attributes.py +425 -124
- meerschaum/core/Pipe/_bootstrap.py +54 -24
- meerschaum/core/Pipe/_cache.py +555 -0
- meerschaum/core/Pipe/_clear.py +0 -11
- meerschaum/core/Pipe/_data.py +96 -68
- meerschaum/core/Pipe/_deduplicate.py +0 -13
- meerschaum/core/Pipe/_delete.py +12 -21
- meerschaum/core/Pipe/_drop.py +11 -23
- meerschaum/core/Pipe/_dtypes.py +49 -19
- meerschaum/core/Pipe/_edit.py +14 -4
- meerschaum/core/Pipe/_fetch.py +1 -1
- meerschaum/core/Pipe/_index.py +8 -14
- meerschaum/core/Pipe/_show.py +5 -5
- meerschaum/core/Pipe/_sync.py +123 -204
- meerschaum/core/Pipe/_verify.py +4 -4
- meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
- meerschaum/core/Plugin/__init__.py +1 -1
- meerschaum/core/Token/_Token.py +220 -0
- meerschaum/core/Token/__init__.py +12 -0
- meerschaum/core/User/_User.py +35 -10
- meerschaum/core/User/__init__.py +9 -1
- meerschaum/core/__init__.py +1 -0
- meerschaum/jobs/_Executor.py +88 -4
- meerschaum/jobs/_Job.py +149 -38
- meerschaum/jobs/__init__.py +3 -2
- meerschaum/jobs/systemd.py +8 -3
- meerschaum/models/__init__.py +35 -0
- meerschaum/models/pipes.py +247 -0
- meerschaum/models/tokens.py +38 -0
- meerschaum/models/users.py +26 -0
- meerschaum/plugins/__init__.py +301 -88
- meerschaum/plugins/bootstrap.py +510 -4
- meerschaum/utils/_get_pipes.py +97 -30
- meerschaum/utils/daemon/Daemon.py +199 -43
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
- meerschaum/utils/daemon/RotatingFile.py +63 -36
- meerschaum/utils/daemon/StdinFile.py +53 -13
- meerschaum/utils/daemon/__init__.py +47 -6
- meerschaum/utils/daemon/_names.py +6 -3
- meerschaum/utils/dataframe.py +480 -82
- meerschaum/utils/debug.py +49 -19
- meerschaum/utils/dtypes/__init__.py +478 -37
- meerschaum/utils/dtypes/sql.py +369 -29
- meerschaum/utils/formatting/__init__.py +5 -2
- meerschaum/utils/formatting/_jobs.py +1 -1
- meerschaum/utils/formatting/_pipes.py +52 -50
- meerschaum/utils/formatting/_pprint.py +1 -0
- meerschaum/utils/formatting/_shell.py +44 -18
- meerschaum/utils/misc.py +268 -186
- meerschaum/utils/packages/__init__.py +25 -40
- meerschaum/utils/packages/_packages.py +42 -34
- meerschaum/utils/pipes.py +213 -0
- meerschaum/utils/process.py +2 -2
- meerschaum/utils/prompt.py +175 -144
- meerschaum/utils/schedule.py +2 -1
- meerschaum/utils/sql.py +135 -49
- meerschaum/utils/threading.py +42 -0
- meerschaum/utils/typing.py +1 -4
- meerschaum/utils/venv/_Venv.py +2 -2
- meerschaum/utils/venv/__init__.py +7 -7
- meerschaum/utils/warnings.py +19 -13
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
- meerschaum-3.0.0.dist-info/RECORD +289 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
- meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
- meerschaum/api/models/_interfaces.py +0 -15
- meerschaum/api/models/_locations.py +0 -15
- meerschaum/api/models/_metrics.py +0 -15
- meerschaum/config/_environment.py +0 -145
- meerschaum/config/static/__init__.py +0 -186
- meerschaum-2.9.4.dist-info/RECORD +0 -263
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.9.4.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
meerschaum/api/dash/pipes.py
CHANGED
@@ -12,18 +12,26 @@ import shlex
|
|
12
12
|
from textwrap import dedent
|
13
13
|
from urllib.parse import urlencode
|
14
14
|
|
15
|
+
from meerschaum.utils import fetch_pipes_keys
|
15
16
|
from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
|
16
|
-
from meerschaum.utils.misc import
|
17
|
+
from meerschaum.utils.misc import get_connector_labels
|
18
|
+
from meerschaum.connectors import instance_types
|
17
19
|
from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
|
18
20
|
from meerschaum.utils.sql import get_pd_type
|
19
21
|
from meerschaum.utils.yaml import yaml
|
20
22
|
from meerschaum.utils.warnings import warn
|
21
|
-
from meerschaum.utils.dataframe import to_json
|
23
|
+
from meerschaum.utils.dataframe import to_json, to_simple_lines
|
22
24
|
from meerschaum.connectors.sql._fetch import get_pipe_query
|
23
|
-
from meerschaum.api import CHECK_UPDATE
|
25
|
+
from meerschaum.api import CHECK_UPDATE, get_api_connector
|
24
26
|
from meerschaum.api.dash import debug, _get_pipes
|
25
27
|
from meerschaum.api.dash.connectors import get_web_connector
|
26
|
-
from meerschaum.api.dash.components import
|
28
|
+
from meerschaum.api.dash.components import (
|
29
|
+
alert_from_success_tuple,
|
30
|
+
build_cards_grid,
|
31
|
+
sign_out_button,
|
32
|
+
logo_row,
|
33
|
+
pages_offcanvas,
|
34
|
+
)
|
27
35
|
from meerschaum.api.dash.sessions import is_session_authenticated
|
28
36
|
from meerschaum.config import get_config
|
29
37
|
import meerschaum as mrsm
|
@@ -58,28 +66,20 @@ def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, N
|
|
58
66
|
|
59
67
|
def keys_from_state(
|
60
68
|
state: Dict[str, Any],
|
61
|
-
|
69
|
+
with_tags: bool = False,
|
62
70
|
) -> Union[
|
63
71
|
Tuple[List[str], List[str], List[str]],
|
64
|
-
Tuple[List[str], List[str], List[str], str],
|
72
|
+
Tuple[List[str], List[str], List[str], List[str]],
|
65
73
|
]:
|
66
74
|
"""
|
67
75
|
Read the current state and return the selected keys lists.
|
68
76
|
"""
|
69
77
|
_filters = {
|
70
|
-
'ck'
|
71
|
-
'mk'
|
72
|
-
'lk'
|
78
|
+
'ck': state.get("connector-keys-dropdown.value", None),
|
79
|
+
'mk': state.get("metric-keys-dropdown.value", None),
|
80
|
+
'lk': state.get("location-keys-dropdown.value", None),
|
81
|
+
'tags': state.get("tags-dropdown.value", None),
|
73
82
|
}
|
74
|
-
if state['pipes-filter-tabs.active_tab'] == 'input':
|
75
|
-
try:
|
76
|
-
# params = string_to_dict(state['params-textarea.value'])
|
77
|
-
params = string_to_dict(state['search-parameters-editor.value'])
|
78
|
-
except Exception:
|
79
|
-
params = None
|
80
|
-
else:
|
81
|
-
params = None
|
82
|
-
|
83
83
|
for k in _filters:
|
84
84
|
_filters[k] = [] if _filters[k] is None else _filters[k]
|
85
85
|
if not isinstance(_filters[k], list):
|
@@ -89,8 +89,8 @@ def keys_from_state(
|
|
89
89
|
print(e)
|
90
90
|
_filters[k] = []
|
91
91
|
keys = [_filters['ck'], _filters['mk'], _filters['lk']]
|
92
|
-
if
|
93
|
-
keys.append(
|
92
|
+
if with_tags:
|
93
|
+
keys.append(_filters['tags'])
|
94
94
|
return tuple(keys)
|
95
95
|
|
96
96
|
|
@@ -98,12 +98,12 @@ def pipes_from_state(
|
|
98
98
|
state: Dict[str, Any],
|
99
99
|
**kw
|
100
100
|
):
|
101
|
-
_ck, _mk, _lk,
|
101
|
+
_ck, _mk, _lk, _tags = keys_from_state(state, with_tags=True)
|
102
102
|
try:
|
103
103
|
_pipes = _get_pipes(
|
104
104
|
_ck, _mk, _lk,
|
105
|
-
|
106
|
-
mrsm_instance
|
105
|
+
tags=(_tags or []),
|
106
|
+
mrsm_instance=get_web_connector(state),
|
107
107
|
**kw
|
108
108
|
)
|
109
109
|
except Exception as e:
|
@@ -343,6 +343,7 @@ def accordion_items_from_pipe(
|
|
343
343
|
items_titles['sql'] = '📃 SQL Query'
|
344
344
|
items_titles.update({
|
345
345
|
'recent-data': '🗃️ Recent Data',
|
346
|
+
'query-data': '🔍 Query Data',
|
346
347
|
'sync-data': '📝 Sync Documents',
|
347
348
|
})
|
348
349
|
|
@@ -376,48 +377,48 @@ def accordion_items_from_pipe(
|
|
376
377
|
html.Th(
|
377
378
|
html.Span(
|
378
379
|
"Key",
|
379
|
-
id={'type': 'key-table-header', '
|
380
|
+
id={'type': 'key-table-header', 'index': pipe_meta_str},
|
380
381
|
style={"textDecoration": "underline", "cursor": "pointer"},
|
381
382
|
),
|
382
383
|
),
|
383
384
|
html.Th(
|
384
385
|
html.Span(
|
385
386
|
"Column",
|
386
|
-
id={'type': 'column-table-header', '
|
387
|
+
id={'type': 'column-table-header', 'index': pipe_meta_str},
|
387
388
|
style={"textDecoration": "underline", "cursor": "pointer"},
|
388
389
|
),
|
389
390
|
),
|
390
391
|
html.Th(
|
391
392
|
html.Span(
|
392
393
|
"Index",
|
393
|
-
id={'type': 'index-table-header', '
|
394
|
+
id={'type': 'index-table-header', 'index': pipe_meta_str},
|
394
395
|
style={"textDecoration": "underline", "cursor": "pointer"},
|
395
396
|
),
|
396
397
|
),
|
397
398
|
html.Th(
|
398
399
|
html.Span(
|
399
400
|
"Is Composite",
|
400
|
-
id={'type': 'is-composite-table-header', '
|
401
|
+
id={'type': 'is-composite-table-header', 'index': pipe_meta_str},
|
401
402
|
style={"textDecoration": "underline", "cursor": "pointer"},
|
402
403
|
),
|
403
404
|
),
|
404
405
|
dbc.Tooltip(
|
405
406
|
"Unique reference name for the index "
|
406
407
|
"(e.g. `datetime` for the range axis)",
|
407
|
-
target={'type': 'key-table-header', '
|
408
|
+
target={'type': 'key-table-header', 'index': pipe_meta_str},
|
408
409
|
),
|
409
410
|
dbc.Tooltip(
|
410
411
|
"The actual column (field name) in the target dataset.",
|
411
|
-
target={'type': 'column-table-header', '
|
412
|
+
target={'type': 'column-table-header', 'index': pipe_meta_str},
|
412
413
|
),
|
413
414
|
dbc.Tooltip(
|
414
415
|
"The name of the index created on the given columns.",
|
415
|
-
target={'type': 'index-table-header', '
|
416
|
+
target={'type': 'index-table-header', 'index': pipe_meta_str},
|
416
417
|
),
|
417
418
|
dbc.Tooltip(
|
418
419
|
"Whether the column is used in the composite primary key "
|
419
420
|
"to determine updates.",
|
420
|
-
target={'type': 'is-composite-table-header', '
|
421
|
+
target={'type': 'is-composite-table-header', 'index': pipe_meta_str},
|
421
422
|
),
|
422
423
|
]
|
423
424
|
)
|
@@ -465,7 +466,7 @@ def accordion_items_from_pipe(
|
|
465
466
|
overview_rows.append(
|
466
467
|
html.Tr([
|
467
468
|
html.Td("Indices" if len(indices_rows) != 1 else "Index"),
|
468
|
-
html.Td(indices_table),
|
469
|
+
html.Td(html.Div(indices_table, style={'overflowX': 'auto'})),
|
469
470
|
])
|
470
471
|
)
|
471
472
|
|
@@ -482,9 +483,14 @@ def accordion_items_from_pipe(
|
|
482
483
|
])
|
483
484
|
)
|
484
485
|
|
485
|
-
items_bodies['overview'] =
|
486
|
-
|
487
|
-
|
486
|
+
items_bodies['overview'] = html.Div(
|
487
|
+
dbc.Table(
|
488
|
+
overview_header + [html.Tbody(overview_rows)],
|
489
|
+
bordered=False,
|
490
|
+
hover=True,
|
491
|
+
striped=False,
|
492
|
+
),
|
493
|
+
style={'overflowX': 'auto'},
|
488
494
|
)
|
489
495
|
|
490
496
|
if 'stats' in active_items:
|
@@ -496,17 +502,13 @@ def accordion_items_from_pipe(
|
|
496
502
|
(newest_time - oldest_time) if newest_time is not None and oldest_time is not None
|
497
503
|
else None
|
498
504
|
)
|
499
|
-
rowcount = pipe.get_rowcount(debug=debug)
|
500
505
|
except Exception:
|
501
506
|
oldest_time = None
|
502
507
|
newest_time = None
|
503
508
|
interval = None
|
504
|
-
rowcount = None
|
505
509
|
|
506
510
|
stats_rows = []
|
507
|
-
if
|
508
|
-
stats_rows.append(html.Tr([html.Td("Row Count"), html.Td(f"{rowcount:,}")]))
|
509
|
-
if interval is not None:
|
511
|
+
if interval is not None and not isinstance(interval, int):
|
510
512
|
stats_rows.append(
|
511
513
|
html.Tr([html.Td("Timespan"), html.Td(humanfriendly.format_timespan(interval))])
|
512
514
|
)
|
@@ -515,7 +517,39 @@ def accordion_items_from_pipe(
|
|
515
517
|
if newest_time is not None:
|
516
518
|
stats_rows.append(html.Tr([html.Td("Newest time"), html.Td(str(newest_time))]))
|
517
519
|
|
518
|
-
|
520
|
+
precision = pipe.precision
|
521
|
+
if precision:
|
522
|
+
stats_rows.append(
|
523
|
+
html.Tr([
|
524
|
+
html.Td("Precision"),
|
525
|
+
html.Td(str(precision.get('interval', 1)) + ' ' + str(precision.get('unit', 'unit')))
|
526
|
+
])
|
527
|
+
)
|
528
|
+
|
529
|
+
stats_rows.append(
|
530
|
+
html.Tr([
|
531
|
+
html.Td("Row count"),
|
532
|
+
html.Td(
|
533
|
+
html.Div(
|
534
|
+
dbc.Button(
|
535
|
+
"Calculate",
|
536
|
+
color='link',
|
537
|
+
size='sm',
|
538
|
+
style={'text-decoration': 'none'},
|
539
|
+
id={'type': 'calculate-rowcount-button', 'index': pipe_meta_str},
|
540
|
+
)
|
541
|
+
if pipe.exists(debug=debug)
|
542
|
+
else '0'
|
543
|
+
),
|
544
|
+
id={'type': 'calculate-rowcount-div', 'index': pipe_meta_str},
|
545
|
+
)
|
546
|
+
])
|
547
|
+
)
|
548
|
+
|
549
|
+
items_bodies['stats'] = html.Div(
|
550
|
+
dbc.Table(stats_header + [html.Tbody(stats_rows)], hover=True),
|
551
|
+
style={'overflowX': 'auto'},
|
552
|
+
)
|
519
553
|
|
520
554
|
if 'columns' in active_items:
|
521
555
|
try:
|
@@ -535,9 +569,9 @@ def accordion_items_from_pipe(
|
|
535
569
|
]
|
536
570
|
columns_body = [html.Tbody(columns_rows)]
|
537
571
|
columns_table = dbc.Table(columns_header + columns_body, bordered=False, hover=True)
|
572
|
+
items_bodies['columns'] = html.Div(columns_table, style={'overflowX': 'auto'})
|
538
573
|
except Exception:
|
539
|
-
|
540
|
-
items_bodies['columns'] = columns_table
|
574
|
+
items_bodies['columns'] = html.P("Could not retrieve columns ― please try again.")
|
541
575
|
|
542
576
|
if 'parameters' in active_items:
|
543
577
|
parameters_editor = dash_ace.DashAceEditor(
|
@@ -545,7 +579,7 @@ def accordion_items_from_pipe(
|
|
545
579
|
mode='norm',
|
546
580
|
tabSize=4,
|
547
581
|
theme='twilight',
|
548
|
-
id={'type': 'parameters-editor', 'index':
|
582
|
+
id={'type': 'parameters-editor', 'index': pipe_meta_str},
|
549
583
|
width='100%',
|
550
584
|
height='500px',
|
551
585
|
readOnly=False,
|
@@ -557,19 +591,19 @@ def accordion_items_from_pipe(
|
|
557
591
|
)
|
558
592
|
update_parameters_button = dbc.Button(
|
559
593
|
"Update",
|
560
|
-
id={'type': 'update-parameters-button', 'index':
|
594
|
+
id={'type': 'update-parameters-button', 'index': pipe_meta_str},
|
561
595
|
)
|
562
596
|
|
563
597
|
as_yaml_button = dbc.Button(
|
564
598
|
"YAML",
|
565
|
-
id={'type': 'parameters-as-yaml-button', 'index':
|
599
|
+
id={'type': 'parameters-as-yaml-button', 'index': pipe_meta_str},
|
566
600
|
color='link',
|
567
601
|
size='sm',
|
568
602
|
style={'text-decoration': 'none'},
|
569
603
|
)
|
570
604
|
as_json_button = dbc.Button(
|
571
605
|
"JSON",
|
572
|
-
id={'type': 'parameters-as-json-button', 'index':
|
606
|
+
id={'type': 'parameters-as-json-button', 'index': pipe_meta_str},
|
573
607
|
color='link',
|
574
608
|
size='sm',
|
575
609
|
style={'text-decoration': 'none', 'margin-left': '10px'},
|
@@ -578,21 +612,28 @@ def accordion_items_from_pipe(
|
|
578
612
|
parameters_editor,
|
579
613
|
html.Br(),
|
580
614
|
dbc.Row([
|
581
|
-
dbc.Col(
|
582
|
-
(
|
583
|
-
(
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
|
594
|
-
|
595
|
-
|
615
|
+
dbc.Col(
|
616
|
+
html.Span(
|
617
|
+
(
|
618
|
+
([update_parameters_button] if authenticated else []) +
|
619
|
+
[
|
620
|
+
as_json_button,
|
621
|
+
as_yaml_button,
|
622
|
+
]
|
623
|
+
)
|
624
|
+
),
|
625
|
+
width=4,
|
626
|
+
),
|
627
|
+
dbc.Col(
|
628
|
+
[
|
629
|
+
html.Div(
|
630
|
+
id={
|
631
|
+
'type': 'update-parameters-success-div',
|
632
|
+
'index': pipe_meta_str,
|
633
|
+
}
|
634
|
+
)
|
635
|
+
],
|
636
|
+
width=True,
|
596
637
|
)
|
597
638
|
]),
|
598
639
|
]
|
@@ -632,7 +673,7 @@ def accordion_items_from_pipe(
|
|
632
673
|
mode='sql',
|
633
674
|
tabSize=4,
|
634
675
|
theme='twilight',
|
635
|
-
id={'type': 'sql-editor', 'index':
|
676
|
+
id={'type': 'sql-editor', 'index': pipe_meta_str},
|
636
677
|
width='100%',
|
637
678
|
height='500px',
|
638
679
|
readOnly=False,
|
@@ -644,7 +685,7 @@ def accordion_items_from_pipe(
|
|
644
685
|
)
|
645
686
|
update_sql_button = dbc.Button(
|
646
687
|
"Update",
|
647
|
-
id={'type': 'update-sql-button', 'index':
|
688
|
+
id={'type': 'update-sql-button', 'index': pipe_meta_str},
|
648
689
|
)
|
649
690
|
items_bodies['sql'] = html.Div([
|
650
691
|
sql_editor,
|
@@ -653,7 +694,7 @@ def accordion_items_from_pipe(
|
|
653
694
|
dbc.Col([update_sql_button], width=2),
|
654
695
|
dbc.Col([
|
655
696
|
html.Div(
|
656
|
-
id={'type': 'update-sql-success-div', 'index':
|
697
|
+
id={'type': 'update-sql-success-div', 'index': pipe_meta_str}
|
657
698
|
)
|
658
699
|
],
|
659
700
|
width=True,
|
@@ -664,33 +705,84 @@ def accordion_items_from_pipe(
|
|
664
705
|
if 'recent-data' in active_items:
|
665
706
|
try:
|
666
707
|
df = pipe.get_backtrack_data(backtrack_minutes=10, limit=10, debug=debug).astype(str)
|
667
|
-
table = dbc.Table.from_dataframe(df, bordered=False, hover=True)
|
708
|
+
table = dbc.Table.from_dataframe(df, bordered=False, hover=True)
|
709
|
+
items_bodies['recent-data'] = html.Div(table, style={'overflowX': 'auto'})
|
668
710
|
except Exception:
|
669
|
-
|
670
|
-
|
711
|
+
items_bodies['recent-data'] = html.P("Could not retrieve recent data.")
|
712
|
+
|
713
|
+
if 'query-data' in active_items:
|
714
|
+
query_editor = dash_ace.DashAceEditor(
|
715
|
+
value='{\n \n}',
|
716
|
+
mode='norm',
|
717
|
+
tabSize=4,
|
718
|
+
theme='twilight',
|
719
|
+
id={'type': 'query-editor', 'index': pipe_meta_str},
|
720
|
+
width='100%',
|
721
|
+
height='200px',
|
722
|
+
readOnly=False,
|
723
|
+
showGutter=True,
|
724
|
+
showPrintMargin=False,
|
725
|
+
highlightActiveLine=True,
|
726
|
+
wrapEnabled=True,
|
727
|
+
style={'min-height': '120px'},
|
728
|
+
)
|
729
|
+
query_data_button = dbc.Button(
|
730
|
+
"Query",
|
731
|
+
id={'type': 'query-data-button', 'index': pipe_meta_str},
|
732
|
+
)
|
733
|
+
|
734
|
+
begin_end_input_group = dbc.InputGroup(
|
735
|
+
[
|
736
|
+
dbc.Input(
|
737
|
+
id={'type': 'query-data-begin-input', 'index': pipe_meta_str},
|
738
|
+
placeholder="Begin",
|
739
|
+
),
|
740
|
+
dbc.Input(
|
741
|
+
id={'type': 'query-data-end-input', 'index': pipe_meta_str},
|
742
|
+
placeholder="End",
|
743
|
+
),
|
744
|
+
],
|
745
|
+
size="sm",
|
746
|
+
)
|
747
|
+
|
748
|
+
limit_input = dbc.Input(
|
749
|
+
type="number",
|
750
|
+
min=0,
|
751
|
+
max=100,
|
752
|
+
value=10,
|
753
|
+
step=1,
|
754
|
+
placeholder="Limit",
|
755
|
+
id={'type': 'limit-input', 'index': pipe_meta_str},
|
756
|
+
)
|
757
|
+
query_result_div = html.Div(
|
758
|
+
id={'type': 'query-result-div', 'index': pipe_meta_str},
|
759
|
+
style={'overflowX': 'auto'},
|
760
|
+
)
|
761
|
+
|
762
|
+
items_bodies['query-data'] = html.Div([
|
763
|
+
query_editor,
|
764
|
+
html.Br(),
|
765
|
+
dbc.Row(
|
766
|
+
[
|
767
|
+
dbc.Col([query_data_button], lg=2, md=3, sm=4, xs=6, width=2),
|
768
|
+
dbc.Col([begin_end_input_group], lg=6, md=6, sm=4, width=6),
|
769
|
+
dbc.Col(html.Div([limit_input, dbc.FormText("Row Limit")]), lg=2, md=3, sm=4, xs=6, width=2),
|
770
|
+
],
|
771
|
+
justify="between",
|
772
|
+
),
|
773
|
+
dbc.Row([
|
774
|
+
dbc.Col([query_result_div], width=True),
|
775
|
+
]),
|
776
|
+
])
|
671
777
|
|
672
778
|
if 'sync-data' in active_items:
|
673
|
-
|
674
|
-
try:
|
675
|
-
json_text = to_json(
|
676
|
-
backtrack_df,
|
677
|
-
orient='records',
|
678
|
-
date_format='iso',
|
679
|
-
force_ascii=False,
|
680
|
-
indent=4,
|
681
|
-
date_unit='us',
|
682
|
-
) if backtrack_df is not None else '[]'
|
683
|
-
except Exception as e:
|
684
|
-
warn(e)
|
685
|
-
json_text = '[]'
|
686
|
-
|
687
|
-
json_text = json.dumps(json.loads(json_text), indent=4, separators=(',', ': '))
|
779
|
+
backtrack_text = get_backtrack_text(pipe)
|
688
780
|
sync_editor = dash_ace.DashAceEditor(
|
689
|
-
value =
|
781
|
+
value = backtrack_text,
|
690
782
|
mode = 'norm',
|
691
783
|
tabSize = 4,
|
692
784
|
theme = 'twilight',
|
693
|
-
id = {'type': 'sync-editor', 'index':
|
785
|
+
id = {'type': 'sync-editor', 'index': pipe_meta_str},
|
694
786
|
width = '100%',
|
695
787
|
height = '500px',
|
696
788
|
readOnly = False,
|
@@ -700,16 +792,40 @@ def accordion_items_from_pipe(
|
|
700
792
|
wrapEnabled = True,
|
701
793
|
style = {'min-height': '120px'},
|
702
794
|
)
|
795
|
+
|
796
|
+
sync_as_json_button = dbc.Button(
|
797
|
+
"JSON",
|
798
|
+
id={'type': 'sync-as-json-button', 'index': pipe_meta_str},
|
799
|
+
color='link',
|
800
|
+
size='sm',
|
801
|
+
style={'text-decoration': 'none', 'margin-left': '10px'},
|
802
|
+
)
|
803
|
+
sync_as_lines_button = dbc.Button(
|
804
|
+
"Lines",
|
805
|
+
id={'type': 'sync-as-lines-button', 'index': pipe_meta_str},
|
806
|
+
color='link',
|
807
|
+
size='sm',
|
808
|
+
style={'text-decoration': 'none', 'margin-left': '10px'},
|
809
|
+
)
|
810
|
+
|
703
811
|
update_sync_button = dbc.Button(
|
704
812
|
"Sync",
|
705
|
-
id = {'type': 'update-sync-button', 'index':
|
813
|
+
id = {'type': 'update-sync-button', 'index': pipe_meta_str},
|
706
814
|
)
|
707
|
-
sync_success_div = html.Div(id={'type': 'sync-success-div', 'index':
|
815
|
+
sync_success_div = html.Div(id={'type': 'sync-success-div', 'index': pipe_meta_str})
|
708
816
|
items_bodies['sync-data'] = html.Div([
|
709
817
|
sync_editor,
|
710
818
|
html.Br(),
|
711
819
|
dbc.Row([
|
712
|
-
dbc.Col(
|
820
|
+
dbc.Col(html.Span(
|
821
|
+
(
|
822
|
+
([update_sync_button] if authenticated else []) +
|
823
|
+
[
|
824
|
+
sync_as_json_button,
|
825
|
+
sync_as_lines_button,
|
826
|
+
]
|
827
|
+
)
|
828
|
+
), width=4),
|
713
829
|
dbc.Col([sync_success_div], width=True),
|
714
830
|
]),
|
715
831
|
])
|
@@ -719,3 +835,237 @@ def accordion_items_from_pipe(
|
|
719
835
|
for item_id, title in items_titles.items()
|
720
836
|
]
|
721
837
|
|
838
|
+
|
839
|
+
def get_backtrack_text(
|
840
|
+
pipe: mrsm.Pipe,
|
841
|
+
lines: bool = False,
|
842
|
+
limit: int = 5,
|
843
|
+
) -> str:
|
844
|
+
"""
|
845
|
+
Return the backtrack documents as text for the sync editor.
|
846
|
+
"""
|
847
|
+
backtrack_df = pipe.get_backtrack_data(debug=debug, limit=limit)
|
848
|
+
if lines:
|
849
|
+
return to_simple_lines(backtrack_df)
|
850
|
+
try:
|
851
|
+
json_text = to_json(
|
852
|
+
backtrack_df,
|
853
|
+
orient='records',
|
854
|
+
date_format='iso',
|
855
|
+
force_ascii=False,
|
856
|
+
indent=4,
|
857
|
+
date_unit='us',
|
858
|
+
) if backtrack_df is not None else '[]'
|
859
|
+
except Exception as e:
|
860
|
+
warn(e)
|
861
|
+
json_text = '[]'
|
862
|
+
|
863
|
+
return json.dumps(json.loads(json_text), indent=4, separators=(',', ': '))
|
864
|
+
|
865
|
+
|
866
|
+
def build_pipes_dropdown_keys_row(
|
867
|
+
connector_keys: List[str],
|
868
|
+
metric_keys: List[str],
|
869
|
+
location_keys: List[str],
|
870
|
+
tags: List[str],
|
871
|
+
pipes: List[mrsm.Pipe],
|
872
|
+
instance_connector: mrsm.connectors.InstanceConnector,
|
873
|
+
) -> dbc.Row:
|
874
|
+
"""
|
875
|
+
Return the dropdown keys row for the dedicated pipes page.
|
876
|
+
"""
|
877
|
+
ck_alone = connector_keys and not any([str(x) for x in (tags + metric_keys + location_keys)])
|
878
|
+
mk_alone = metric_keys and not any([str(x) for x in (connector_keys + tags + location_keys)])
|
879
|
+
lk_alone = location_keys and not any([str(x) for x in (connector_keys + metric_keys + tags)])
|
880
|
+
all_keys = fetch_pipes_keys('registered', instance_connector)
|
881
|
+
|
882
|
+
ck_options_source = (
|
883
|
+
{keys_tuple[0] for keys_tuple in all_keys}
|
884
|
+
if ck_alone
|
885
|
+
else {p.connector_keys for p in pipes}
|
886
|
+
)
|
887
|
+
ck_options = sorted(ck_options_source.union(connector_keys))
|
888
|
+
|
889
|
+
mk_options_source = (
|
890
|
+
{keys_tuple[1] for keys_tuple in all_keys}
|
891
|
+
if mk_alone
|
892
|
+
else {p.metric_key for p in pipes}
|
893
|
+
)
|
894
|
+
mk_options = sorted(mk_options_source.union(metric_keys))
|
895
|
+
|
896
|
+
lk_options_source = (
|
897
|
+
{str(keys_tuple[2]) for keys_tuple in all_keys}
|
898
|
+
if lk_alone
|
899
|
+
else {str(p.location_key) for p in pipes}
|
900
|
+
)
|
901
|
+
lk_options = sorted(lk_options_source.union({str(lk) for lk in location_keys}))
|
902
|
+
|
903
|
+
return dbc.Row(
|
904
|
+
[
|
905
|
+
dbc.Col(
|
906
|
+
html.Div(
|
907
|
+
[
|
908
|
+
dcc.Dropdown(
|
909
|
+
id='pipes-connector-keys-dropdown',
|
910
|
+
options=ck_options,
|
911
|
+
value=[str(ck) for ck in connector_keys],
|
912
|
+
placeholder='Connectors',
|
913
|
+
multi=True,
|
914
|
+
),
|
915
|
+
],
|
916
|
+
className='dbc_dark',
|
917
|
+
),
|
918
|
+
lg=4,
|
919
|
+
md=12,
|
920
|
+
sm=12,
|
921
|
+
),
|
922
|
+
dbc.Col(
|
923
|
+
html.Div(
|
924
|
+
[
|
925
|
+
dcc.Dropdown(
|
926
|
+
id='pipes-metric-keys-dropdown',
|
927
|
+
options=mk_options,
|
928
|
+
value=[str(mk) for mk in metric_keys],
|
929
|
+
placeholder='Metrics',
|
930
|
+
multi=True,
|
931
|
+
),
|
932
|
+
],
|
933
|
+
className='dbc_dark'
|
934
|
+
),
|
935
|
+
lg=4,
|
936
|
+
md=12,
|
937
|
+
sm=12,
|
938
|
+
),
|
939
|
+
dbc.Col(
|
940
|
+
html.Div(
|
941
|
+
[
|
942
|
+
dcc.Dropdown(
|
943
|
+
id='pipes-location-keys-dropdown',
|
944
|
+
options=lk_options,
|
945
|
+
value=[str(lk) for lk in location_keys],
|
946
|
+
placeholder='Locations',
|
947
|
+
multi=True,
|
948
|
+
),
|
949
|
+
],
|
950
|
+
className='dbc_dark'
|
951
|
+
),
|
952
|
+
lg=4,
|
953
|
+
md=12,
|
954
|
+
sm=12,
|
955
|
+
),
|
956
|
+
] ### end of filters row children
|
957
|
+
)
|
958
|
+
|
959
|
+
|
960
|
+
def build_pipes_tags_dropdown(
|
961
|
+
connector_keys: List[str],
|
962
|
+
metric_keys: List[str],
|
963
|
+
location_keys: List[str],
|
964
|
+
tags: List[str],
|
965
|
+
instance: str,
|
966
|
+
) -> dbc.Row:
|
967
|
+
"""
|
968
|
+
Build the tags dropdown for the dedicated pipes page.
|
969
|
+
"""
|
970
|
+
_tags_alone = tags and not any([str(x) for x in (connector_keys + metric_keys + location_keys)])
|
971
|
+
_tags_pipes = mrsm.get_pipes(
|
972
|
+
connector_keys=connector_keys,
|
973
|
+
metric_keys=metric_keys,
|
974
|
+
location_keys=location_keys,
|
975
|
+
tags=tags,
|
976
|
+
instance=instance,
|
977
|
+
as_tags_dict=True,
|
978
|
+
)
|
979
|
+
|
980
|
+
_all_tags = list(
|
981
|
+
mrsm.get_pipes(
|
982
|
+
instance=instance,
|
983
|
+
as_tags_dict=True,
|
984
|
+
)
|
985
|
+
) if _tags_alone else []
|
986
|
+
|
987
|
+
tags_options = [
|
988
|
+
str(tag)
|
989
|
+
for tag in (_all_tags if _tags_alone else _tags_pipes)
|
990
|
+
]
|
991
|
+
if tags:
|
992
|
+
tags_options += [tag for tag in tags if tag not in tags_options]
|
993
|
+
|
994
|
+
return dbc.Row(
|
995
|
+
[
|
996
|
+
dbc.Col(
|
997
|
+
html.Div(
|
998
|
+
dcc.Dropdown(
|
999
|
+
id='pipes-tags-dropdown',
|
1000
|
+
options=tags_options,
|
1001
|
+
value=tags,
|
1002
|
+
placeholder='Tags',
|
1003
|
+
multi=True,
|
1004
|
+
searchable=True,
|
1005
|
+
),
|
1006
|
+
className="dbc_dark",
|
1007
|
+
id="pipes-tags-dropdown-div",
|
1008
|
+
),
|
1009
|
+
width=True,
|
1010
|
+
),
|
1011
|
+
dbc.Col(
|
1012
|
+
dbc.Button(
|
1013
|
+
"Clear all",
|
1014
|
+
color='link',
|
1015
|
+
size='sm',
|
1016
|
+
style={'text-decoration': 'none'},
|
1017
|
+
id='pipes-clear-all-button',
|
1018
|
+
),
|
1019
|
+
width='auto',
|
1020
|
+
),
|
1021
|
+
],
|
1022
|
+
className='g-0',
|
1023
|
+
align='center',
|
1024
|
+
)
|
1025
|
+
|
1026
|
+
|
1027
|
+
def build_pipes_navbar(instance_keys: Optional[str] = None, with_instance_select: bool = True):
|
1028
|
+
"""
|
1029
|
+
Build the navbar from the selected instance keys.
|
1030
|
+
"""
|
1031
|
+
instance_select = dbc.Select(
|
1032
|
+
id='instance-select',
|
1033
|
+
size='sm',
|
1034
|
+
value=instance_keys or str(get_api_connector()),
|
1035
|
+
options=[
|
1036
|
+
{'label': (i[:32] + '…') if len(i) > 32 else i, 'value': i}
|
1037
|
+
for i in get_connector_labels(*instance_types)
|
1038
|
+
],
|
1039
|
+
class_name='dbc_dark custom-select custom-select-sm',
|
1040
|
+
)
|
1041
|
+
instance_select_div_style = {} if with_instance_select else {'visibility': 'hidden'}
|
1042
|
+
instance_select_div = html.Div(instance_select, style=instance_select_div_style)
|
1043
|
+
return html.Div(
|
1044
|
+
[
|
1045
|
+
pages_offcanvas,
|
1046
|
+
dbc.Navbar(
|
1047
|
+
dbc.Container(
|
1048
|
+
[
|
1049
|
+
logo_row,
|
1050
|
+
dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
|
1051
|
+
dbc.Collapse(
|
1052
|
+
dbc.Row(
|
1053
|
+
[
|
1054
|
+
dbc.Col(instance_select_div, width='auto'),
|
1055
|
+
dbc.Col(sign_out_button, width='auto'),
|
1056
|
+
],
|
1057
|
+
className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",
|
1058
|
+
align='center',
|
1059
|
+
),
|
1060
|
+
id='navbar-collapse',
|
1061
|
+
is_open=False,
|
1062
|
+
navbar=True,
|
1063
|
+
),
|
1064
|
+
]
|
1065
|
+
),
|
1066
|
+
dark=True,
|
1067
|
+
color='dark'
|
1068
|
+
),
|
1069
|
+
],
|
1070
|
+
id='pages-navbar-div',
|
1071
|
+
)
|