meerschaum 2.9.5__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +5 -2
- meerschaum/_internal/__init__.py +1 -0
- meerschaum/_internal/arguments/_parse_arguments.py +4 -4
- meerschaum/_internal/arguments/_parser.py +33 -4
- meerschaum/_internal/cli/__init__.py +6 -0
- meerschaum/_internal/cli/daemons.py +103 -0
- meerschaum/_internal/cli/entry.py +220 -0
- meerschaum/_internal/cli/workers.py +435 -0
- meerschaum/_internal/docs/index.py +48 -2
- meerschaum/_internal/entry.py +50 -14
- meerschaum/_internal/shell/Shell.py +121 -29
- meerschaum/_internal/shell/__init__.py +4 -1
- meerschaum/_internal/static.py +359 -0
- meerschaum/_internal/term/TermPageHandler.py +1 -2
- meerschaum/_internal/term/__init__.py +40 -6
- meerschaum/_internal/term/tools.py +33 -8
- meerschaum/actions/__init__.py +6 -4
- meerschaum/actions/api.py +53 -13
- meerschaum/actions/attach.py +1 -0
- meerschaum/actions/bootstrap.py +8 -8
- meerschaum/actions/delete.py +4 -2
- meerschaum/actions/edit.py +171 -25
- meerschaum/actions/login.py +8 -8
- meerschaum/actions/register.py +143 -6
- meerschaum/actions/reload.py +22 -5
- meerschaum/actions/restart.py +14 -0
- meerschaum/actions/show.py +184 -31
- meerschaum/actions/start.py +166 -17
- meerschaum/actions/stop.py +38 -2
- meerschaum/actions/sync.py +7 -2
- meerschaum/actions/tag.py +9 -8
- meerschaum/actions/verify.py +5 -8
- meerschaum/api/__init__.py +45 -15
- meerschaum/api/_events.py +46 -4
- meerschaum/api/_oauth2.py +162 -9
- meerschaum/api/_tokens.py +102 -0
- meerschaum/api/dash/__init__.py +0 -3
- meerschaum/api/dash/callbacks/__init__.py +1 -0
- meerschaum/api/dash/callbacks/custom.py +4 -3
- meerschaum/api/dash/callbacks/dashboard.py +198 -118
- meerschaum/api/dash/callbacks/jobs.py +14 -7
- meerschaum/api/dash/callbacks/login.py +10 -1
- meerschaum/api/dash/callbacks/pipes.py +194 -14
- meerschaum/api/dash/callbacks/plugins.py +0 -1
- meerschaum/api/dash/callbacks/register.py +10 -3
- meerschaum/api/dash/callbacks/settings/password_reset.py +2 -2
- meerschaum/api/dash/callbacks/tokens.py +389 -0
- meerschaum/api/dash/components.py +36 -15
- meerschaum/api/dash/jobs.py +1 -1
- meerschaum/api/dash/keys.py +35 -93
- meerschaum/api/dash/pages/__init__.py +2 -1
- meerschaum/api/dash/pages/dashboard.py +1 -20
- meerschaum/api/dash/pages/{job.py → jobs.py} +10 -7
- meerschaum/api/dash/pages/login.py +2 -2
- meerschaum/api/dash/pages/pipes.py +16 -5
- meerschaum/api/dash/pages/settings/password_reset.py +1 -1
- meerschaum/api/dash/pages/tokens.py +53 -0
- meerschaum/api/dash/pipes.py +382 -95
- meerschaum/api/dash/sessions.py +12 -0
- meerschaum/api/dash/tokens.py +603 -0
- meerschaum/api/dash/websockets.py +1 -1
- meerschaum/api/dash/webterm.py +18 -6
- meerschaum/api/models/__init__.py +23 -3
- meerschaum/api/models/_actions.py +22 -0
- meerschaum/api/models/_pipes.py +91 -7
- meerschaum/api/models/_tokens.py +81 -0
- meerschaum/api/resources/static/js/terminado.js +3 -0
- meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
- meerschaum/api/resources/templates/termpage.html +13 -0
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +3 -4
- meerschaum/api/routes/_connectors.py +3 -7
- meerschaum/api/routes/_jobs.py +26 -35
- meerschaum/api/routes/_login.py +120 -15
- meerschaum/api/routes/_misc.py +5 -10
- meerschaum/api/routes/_pipes.py +178 -143
- meerschaum/api/routes/_plugins.py +38 -28
- meerschaum/api/routes/_tokens.py +236 -0
- meerschaum/api/routes/_users.py +47 -35
- meerschaum/api/routes/_version.py +3 -3
- meerschaum/api/routes/_webterm.py +3 -3
- meerschaum/config/__init__.py +100 -30
- meerschaum/config/_default.py +132 -64
- meerschaum/config/_edit.py +38 -32
- meerschaum/config/_formatting.py +2 -0
- meerschaum/config/_patch.py +10 -8
- meerschaum/config/_paths.py +133 -13
- meerschaum/config/_read_config.py +87 -36
- meerschaum/config/_sync.py +6 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/environment.py +262 -0
- meerschaum/config/stack/__init__.py +37 -15
- meerschaum/config/static.py +18 -0
- meerschaum/connectors/_Connector.py +11 -6
- meerschaum/connectors/__init__.py +41 -22
- meerschaum/connectors/api/_APIConnector.py +34 -6
- meerschaum/connectors/api/_actions.py +2 -2
- meerschaum/connectors/api/_jobs.py +12 -1
- meerschaum/connectors/api/_login.py +33 -7
- meerschaum/connectors/api/_misc.py +2 -2
- meerschaum/connectors/api/_pipes.py +23 -32
- meerschaum/connectors/api/_plugins.py +2 -2
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/api/_tokens.py +146 -0
- meerschaum/connectors/api/_users.py +70 -58
- meerschaum/connectors/instance/_InstanceConnector.py +83 -0
- meerschaum/connectors/instance/__init__.py +10 -0
- meerschaum/connectors/instance/_pipes.py +442 -0
- meerschaum/connectors/instance/_plugins.py +159 -0
- meerschaum/connectors/instance/_tokens.py +317 -0
- meerschaum/connectors/instance/_users.py +188 -0
- meerschaum/connectors/parse.py +5 -2
- meerschaum/connectors/sql/_SQLConnector.py +22 -5
- meerschaum/connectors/sql/_cli.py +12 -11
- meerschaum/connectors/sql/_create_engine.py +12 -168
- meerschaum/connectors/sql/_fetch.py +2 -18
- meerschaum/connectors/sql/_pipes.py +295 -278
- meerschaum/connectors/sql/_plugins.py +29 -0
- meerschaum/connectors/sql/_sql.py +46 -21
- meerschaum/connectors/sql/_users.py +36 -2
- meerschaum/connectors/sql/tables/__init__.py +254 -122
- meerschaum/connectors/valkey/_ValkeyConnector.py +5 -7
- meerschaum/connectors/valkey/_pipes.py +60 -31
- meerschaum/connectors/valkey/_plugins.py +2 -26
- meerschaum/core/Pipe/__init__.py +115 -85
- meerschaum/core/Pipe/_attributes.py +425 -124
- meerschaum/core/Pipe/_bootstrap.py +54 -24
- meerschaum/core/Pipe/_cache.py +555 -0
- meerschaum/core/Pipe/_clear.py +0 -11
- meerschaum/core/Pipe/_data.py +96 -68
- meerschaum/core/Pipe/_deduplicate.py +0 -13
- meerschaum/core/Pipe/_delete.py +12 -21
- meerschaum/core/Pipe/_drop.py +11 -23
- meerschaum/core/Pipe/_dtypes.py +49 -19
- meerschaum/core/Pipe/_edit.py +14 -4
- meerschaum/core/Pipe/_fetch.py +1 -1
- meerschaum/core/Pipe/_index.py +8 -14
- meerschaum/core/Pipe/_show.py +5 -5
- meerschaum/core/Pipe/_sync.py +123 -204
- meerschaum/core/Pipe/_verify.py +4 -4
- meerschaum/{plugins → core/Plugin}/_Plugin.py +16 -12
- meerschaum/core/Plugin/__init__.py +1 -1
- meerschaum/core/Token/_Token.py +220 -0
- meerschaum/core/Token/__init__.py +12 -0
- meerschaum/core/User/_User.py +35 -10
- meerschaum/core/User/__init__.py +9 -1
- meerschaum/core/__init__.py +1 -0
- meerschaum/jobs/_Executor.py +88 -4
- meerschaum/jobs/_Job.py +149 -38
- meerschaum/jobs/__init__.py +3 -2
- meerschaum/jobs/systemd.py +8 -3
- meerschaum/models/__init__.py +35 -0
- meerschaum/models/pipes.py +247 -0
- meerschaum/models/tokens.py +38 -0
- meerschaum/models/users.py +26 -0
- meerschaum/plugins/__init__.py +301 -88
- meerschaum/plugins/bootstrap.py +510 -4
- meerschaum/utils/_get_pipes.py +97 -30
- meerschaum/utils/daemon/Daemon.py +199 -43
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
- meerschaum/utils/daemon/RotatingFile.py +63 -36
- meerschaum/utils/daemon/StdinFile.py +53 -13
- meerschaum/utils/daemon/__init__.py +47 -6
- meerschaum/utils/daemon/_names.py +6 -3
- meerschaum/utils/dataframe.py +479 -81
- meerschaum/utils/debug.py +49 -19
- meerschaum/utils/dtypes/__init__.py +476 -34
- meerschaum/utils/dtypes/sql.py +369 -29
- meerschaum/utils/formatting/__init__.py +5 -2
- meerschaum/utils/formatting/_jobs.py +1 -1
- meerschaum/utils/formatting/_pipes.py +52 -50
- meerschaum/utils/formatting/_pprint.py +1 -0
- meerschaum/utils/formatting/_shell.py +44 -18
- meerschaum/utils/misc.py +268 -186
- meerschaum/utils/packages/__init__.py +25 -40
- meerschaum/utils/packages/_packages.py +42 -34
- meerschaum/utils/pipes.py +213 -0
- meerschaum/utils/process.py +2 -2
- meerschaum/utils/prompt.py +175 -144
- meerschaum/utils/schedule.py +2 -1
- meerschaum/utils/sql.py +134 -47
- meerschaum/utils/threading.py +42 -0
- meerschaum/utils/typing.py +1 -4
- meerschaum/utils/venv/_Venv.py +2 -2
- meerschaum/utils/venv/__init__.py +7 -7
- meerschaum/utils/warnings.py +19 -13
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/METADATA +94 -96
- meerschaum-3.0.0.dist-info/RECORD +289 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/WHEEL +1 -1
- meerschaum-3.0.0.dist-info/licenses/NOTICE +2 -0
- meerschaum/api/models/_interfaces.py +0 -15
- meerschaum/api/models/_locations.py +0 -15
- meerschaum/api/models/_metrics.py +0 -15
- meerschaum/config/_environment.py +0 -145
- meerschaum/config/static/__init__.py +0 -186
- meerschaum-2.9.5.dist-info/RECORD +0 -263
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.9.5.dist-info → meerschaum-3.0.0.dist-info}/zip-safe +0 -0
meerschaum/api/dash/pipes.py
CHANGED
@@ -12,18 +12,26 @@ import shlex
|
|
12
12
|
from textwrap import dedent
|
13
13
|
from urllib.parse import urlencode
|
14
14
|
|
15
|
+
from meerschaum.utils import fetch_pipes_keys
|
15
16
|
from meerschaum.utils.typing import List, Optional, Dict, Any, Tuple, Union
|
16
|
-
from meerschaum.utils.misc import
|
17
|
+
from meerschaum.utils.misc import get_connector_labels
|
18
|
+
from meerschaum.connectors import instance_types
|
17
19
|
from meerschaum.utils.packages import attempt_import, import_dcc, import_html, import_pandas
|
18
20
|
from meerschaum.utils.sql import get_pd_type
|
19
21
|
from meerschaum.utils.yaml import yaml
|
20
22
|
from meerschaum.utils.warnings import warn
|
21
|
-
from meerschaum.utils.dataframe import to_json
|
23
|
+
from meerschaum.utils.dataframe import to_json, to_simple_lines
|
22
24
|
from meerschaum.connectors.sql._fetch import get_pipe_query
|
23
|
-
from meerschaum.api import CHECK_UPDATE
|
25
|
+
from meerschaum.api import CHECK_UPDATE, get_api_connector
|
24
26
|
from meerschaum.api.dash import debug, _get_pipes
|
25
27
|
from meerschaum.api.dash.connectors import get_web_connector
|
26
|
-
from meerschaum.api.dash.components import
|
28
|
+
from meerschaum.api.dash.components import (
|
29
|
+
alert_from_success_tuple,
|
30
|
+
build_cards_grid,
|
31
|
+
sign_out_button,
|
32
|
+
logo_row,
|
33
|
+
pages_offcanvas,
|
34
|
+
)
|
27
35
|
from meerschaum.api.dash.sessions import is_session_authenticated
|
28
36
|
from meerschaum.config import get_config
|
29
37
|
import meerschaum as mrsm
|
@@ -58,28 +66,20 @@ def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, N
|
|
58
66
|
|
59
67
|
def keys_from_state(
|
60
68
|
state: Dict[str, Any],
|
61
|
-
|
69
|
+
with_tags: bool = False,
|
62
70
|
) -> Union[
|
63
71
|
Tuple[List[str], List[str], List[str]],
|
64
|
-
Tuple[List[str], List[str], List[str], str],
|
72
|
+
Tuple[List[str], List[str], List[str], List[str]],
|
65
73
|
]:
|
66
74
|
"""
|
67
75
|
Read the current state and return the selected keys lists.
|
68
76
|
"""
|
69
77
|
_filters = {
|
70
|
-
'ck'
|
71
|
-
'mk'
|
72
|
-
'lk'
|
78
|
+
'ck': state.get("connector-keys-dropdown.value", None),
|
79
|
+
'mk': state.get("metric-keys-dropdown.value", None),
|
80
|
+
'lk': state.get("location-keys-dropdown.value", None),
|
81
|
+
'tags': state.get("tags-dropdown.value", None),
|
73
82
|
}
|
74
|
-
if state['pipes-filter-tabs.active_tab'] == 'input':
|
75
|
-
try:
|
76
|
-
# params = string_to_dict(state['params-textarea.value'])
|
77
|
-
params = string_to_dict(state['search-parameters-editor.value'])
|
78
|
-
except Exception:
|
79
|
-
params = None
|
80
|
-
else:
|
81
|
-
params = None
|
82
|
-
|
83
83
|
for k in _filters:
|
84
84
|
_filters[k] = [] if _filters[k] is None else _filters[k]
|
85
85
|
if not isinstance(_filters[k], list):
|
@@ -89,8 +89,8 @@ def keys_from_state(
|
|
89
89
|
print(e)
|
90
90
|
_filters[k] = []
|
91
91
|
keys = [_filters['ck'], _filters['mk'], _filters['lk']]
|
92
|
-
if
|
93
|
-
keys.append(
|
92
|
+
if with_tags:
|
93
|
+
keys.append(_filters['tags'])
|
94
94
|
return tuple(keys)
|
95
95
|
|
96
96
|
|
@@ -98,12 +98,12 @@ def pipes_from_state(
|
|
98
98
|
state: Dict[str, Any],
|
99
99
|
**kw
|
100
100
|
):
|
101
|
-
_ck, _mk, _lk,
|
101
|
+
_ck, _mk, _lk, _tags = keys_from_state(state, with_tags=True)
|
102
102
|
try:
|
103
103
|
_pipes = _get_pipes(
|
104
104
|
_ck, _mk, _lk,
|
105
|
-
|
106
|
-
mrsm_instance
|
105
|
+
tags=(_tags or []),
|
106
|
+
mrsm_instance=get_web_connector(state),
|
107
107
|
**kw
|
108
108
|
)
|
109
109
|
except Exception as e:
|
@@ -377,48 +377,48 @@ def accordion_items_from_pipe(
|
|
377
377
|
html.Th(
|
378
378
|
html.Span(
|
379
379
|
"Key",
|
380
|
-
id={'type': 'key-table-header', '
|
380
|
+
id={'type': 'key-table-header', 'index': pipe_meta_str},
|
381
381
|
style={"textDecoration": "underline", "cursor": "pointer"},
|
382
382
|
),
|
383
383
|
),
|
384
384
|
html.Th(
|
385
385
|
html.Span(
|
386
386
|
"Column",
|
387
|
-
id={'type': 'column-table-header', '
|
387
|
+
id={'type': 'column-table-header', 'index': pipe_meta_str},
|
388
388
|
style={"textDecoration": "underline", "cursor": "pointer"},
|
389
389
|
),
|
390
390
|
),
|
391
391
|
html.Th(
|
392
392
|
html.Span(
|
393
393
|
"Index",
|
394
|
-
id={'type': 'index-table-header', '
|
394
|
+
id={'type': 'index-table-header', 'index': pipe_meta_str},
|
395
395
|
style={"textDecoration": "underline", "cursor": "pointer"},
|
396
396
|
),
|
397
397
|
),
|
398
398
|
html.Th(
|
399
399
|
html.Span(
|
400
400
|
"Is Composite",
|
401
|
-
id={'type': 'is-composite-table-header', '
|
401
|
+
id={'type': 'is-composite-table-header', 'index': pipe_meta_str},
|
402
402
|
style={"textDecoration": "underline", "cursor": "pointer"},
|
403
403
|
),
|
404
404
|
),
|
405
405
|
dbc.Tooltip(
|
406
406
|
"Unique reference name for the index "
|
407
407
|
"(e.g. `datetime` for the range axis)",
|
408
|
-
target={'type': 'key-table-header', '
|
408
|
+
target={'type': 'key-table-header', 'index': pipe_meta_str},
|
409
409
|
),
|
410
410
|
dbc.Tooltip(
|
411
411
|
"The actual column (field name) in the target dataset.",
|
412
|
-
target={'type': 'column-table-header', '
|
412
|
+
target={'type': 'column-table-header', 'index': pipe_meta_str},
|
413
413
|
),
|
414
414
|
dbc.Tooltip(
|
415
415
|
"The name of the index created on the given columns.",
|
416
|
-
target={'type': 'index-table-header', '
|
416
|
+
target={'type': 'index-table-header', 'index': pipe_meta_str},
|
417
417
|
),
|
418
418
|
dbc.Tooltip(
|
419
419
|
"Whether the column is used in the composite primary key "
|
420
420
|
"to determine updates.",
|
421
|
-
target={'type': 'is-composite-table-header', '
|
421
|
+
target={'type': 'is-composite-table-header', 'index': pipe_meta_str},
|
422
422
|
),
|
423
423
|
]
|
424
424
|
)
|
@@ -466,7 +466,7 @@ def accordion_items_from_pipe(
|
|
466
466
|
overview_rows.append(
|
467
467
|
html.Tr([
|
468
468
|
html.Td("Indices" if len(indices_rows) != 1 else "Index"),
|
469
|
-
html.Td(indices_table),
|
469
|
+
html.Td(html.Div(indices_table, style={'overflowX': 'auto'})),
|
470
470
|
])
|
471
471
|
)
|
472
472
|
|
@@ -483,9 +483,14 @@ def accordion_items_from_pipe(
|
|
483
483
|
])
|
484
484
|
)
|
485
485
|
|
486
|
-
items_bodies['overview'] =
|
487
|
-
|
488
|
-
|
486
|
+
items_bodies['overview'] = html.Div(
|
487
|
+
dbc.Table(
|
488
|
+
overview_header + [html.Tbody(overview_rows)],
|
489
|
+
bordered=False,
|
490
|
+
hover=True,
|
491
|
+
striped=False,
|
492
|
+
),
|
493
|
+
style={'overflowX': 'auto'},
|
489
494
|
)
|
490
495
|
|
491
496
|
if 'stats' in active_items:
|
@@ -497,17 +502,13 @@ def accordion_items_from_pipe(
|
|
497
502
|
(newest_time - oldest_time) if newest_time is not None and oldest_time is not None
|
498
503
|
else None
|
499
504
|
)
|
500
|
-
rowcount = pipe.get_rowcount(debug=debug)
|
501
505
|
except Exception:
|
502
506
|
oldest_time = None
|
503
507
|
newest_time = None
|
504
508
|
interval = None
|
505
|
-
rowcount = None
|
506
509
|
|
507
510
|
stats_rows = []
|
508
|
-
if
|
509
|
-
stats_rows.append(html.Tr([html.Td("Row Count"), html.Td(f"{rowcount:,}")]))
|
510
|
-
if interval is not None:
|
511
|
+
if interval is not None and not isinstance(interval, int):
|
511
512
|
stats_rows.append(
|
512
513
|
html.Tr([html.Td("Timespan"), html.Td(humanfriendly.format_timespan(interval))])
|
513
514
|
)
|
@@ -516,7 +517,39 @@ def accordion_items_from_pipe(
|
|
516
517
|
if newest_time is not None:
|
517
518
|
stats_rows.append(html.Tr([html.Td("Newest time"), html.Td(str(newest_time))]))
|
518
519
|
|
519
|
-
|
520
|
+
precision = pipe.precision
|
521
|
+
if precision:
|
522
|
+
stats_rows.append(
|
523
|
+
html.Tr([
|
524
|
+
html.Td("Precision"),
|
525
|
+
html.Td(str(precision.get('interval', 1)) + ' ' + str(precision.get('unit', 'unit')))
|
526
|
+
])
|
527
|
+
)
|
528
|
+
|
529
|
+
stats_rows.append(
|
530
|
+
html.Tr([
|
531
|
+
html.Td("Row count"),
|
532
|
+
html.Td(
|
533
|
+
html.Div(
|
534
|
+
dbc.Button(
|
535
|
+
"Calculate",
|
536
|
+
color='link',
|
537
|
+
size='sm',
|
538
|
+
style={'text-decoration': 'none'},
|
539
|
+
id={'type': 'calculate-rowcount-button', 'index': pipe_meta_str},
|
540
|
+
)
|
541
|
+
if pipe.exists(debug=debug)
|
542
|
+
else '0'
|
543
|
+
),
|
544
|
+
id={'type': 'calculate-rowcount-div', 'index': pipe_meta_str},
|
545
|
+
)
|
546
|
+
])
|
547
|
+
)
|
548
|
+
|
549
|
+
items_bodies['stats'] = html.Div(
|
550
|
+
dbc.Table(stats_header + [html.Tbody(stats_rows)], hover=True),
|
551
|
+
style={'overflowX': 'auto'},
|
552
|
+
)
|
520
553
|
|
521
554
|
if 'columns' in active_items:
|
522
555
|
try:
|
@@ -536,9 +569,9 @@ def accordion_items_from_pipe(
|
|
536
569
|
]
|
537
570
|
columns_body = [html.Tbody(columns_rows)]
|
538
571
|
columns_table = dbc.Table(columns_header + columns_body, bordered=False, hover=True)
|
572
|
+
items_bodies['columns'] = html.Div(columns_table, style={'overflowX': 'auto'})
|
539
573
|
except Exception:
|
540
|
-
|
541
|
-
items_bodies['columns'] = columns_table
|
574
|
+
items_bodies['columns'] = html.P("Could not retrieve columns ― please try again.")
|
542
575
|
|
543
576
|
if 'parameters' in active_items:
|
544
577
|
parameters_editor = dash_ace.DashAceEditor(
|
@@ -546,7 +579,7 @@ def accordion_items_from_pipe(
|
|
546
579
|
mode='norm',
|
547
580
|
tabSize=4,
|
548
581
|
theme='twilight',
|
549
|
-
id={'type': 'parameters-editor', 'index':
|
582
|
+
id={'type': 'parameters-editor', 'index': pipe_meta_str},
|
550
583
|
width='100%',
|
551
584
|
height='500px',
|
552
585
|
readOnly=False,
|
@@ -558,19 +591,19 @@ def accordion_items_from_pipe(
|
|
558
591
|
)
|
559
592
|
update_parameters_button = dbc.Button(
|
560
593
|
"Update",
|
561
|
-
id={'type': 'update-parameters-button', 'index':
|
594
|
+
id={'type': 'update-parameters-button', 'index': pipe_meta_str},
|
562
595
|
)
|
563
596
|
|
564
597
|
as_yaml_button = dbc.Button(
|
565
598
|
"YAML",
|
566
|
-
id={'type': 'parameters-as-yaml-button', 'index':
|
599
|
+
id={'type': 'parameters-as-yaml-button', 'index': pipe_meta_str},
|
567
600
|
color='link',
|
568
601
|
size='sm',
|
569
602
|
style={'text-decoration': 'none'},
|
570
603
|
)
|
571
604
|
as_json_button = dbc.Button(
|
572
605
|
"JSON",
|
573
|
-
id={'type': 'parameters-as-json-button', 'index':
|
606
|
+
id={'type': 'parameters-as-json-button', 'index': pipe_meta_str},
|
574
607
|
color='link',
|
575
608
|
size='sm',
|
576
609
|
style={'text-decoration': 'none', 'margin-left': '10px'},
|
@@ -579,21 +612,28 @@ def accordion_items_from_pipe(
|
|
579
612
|
parameters_editor,
|
580
613
|
html.Br(),
|
581
614
|
dbc.Row([
|
582
|
-
dbc.Col(
|
583
|
-
(
|
584
|
-
(
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
|
594
|
-
|
595
|
-
|
596
|
-
|
615
|
+
dbc.Col(
|
616
|
+
html.Span(
|
617
|
+
(
|
618
|
+
([update_parameters_button] if authenticated else []) +
|
619
|
+
[
|
620
|
+
as_json_button,
|
621
|
+
as_yaml_button,
|
622
|
+
]
|
623
|
+
)
|
624
|
+
),
|
625
|
+
width=4,
|
626
|
+
),
|
627
|
+
dbc.Col(
|
628
|
+
[
|
629
|
+
html.Div(
|
630
|
+
id={
|
631
|
+
'type': 'update-parameters-success-div',
|
632
|
+
'index': pipe_meta_str,
|
633
|
+
}
|
634
|
+
)
|
635
|
+
],
|
636
|
+
width=True,
|
597
637
|
)
|
598
638
|
]),
|
599
639
|
]
|
@@ -633,7 +673,7 @@ def accordion_items_from_pipe(
|
|
633
673
|
mode='sql',
|
634
674
|
tabSize=4,
|
635
675
|
theme='twilight',
|
636
|
-
id={'type': 'sql-editor', 'index':
|
676
|
+
id={'type': 'sql-editor', 'index': pipe_meta_str},
|
637
677
|
width='100%',
|
638
678
|
height='500px',
|
639
679
|
readOnly=False,
|
@@ -645,7 +685,7 @@ def accordion_items_from_pipe(
|
|
645
685
|
)
|
646
686
|
update_sql_button = dbc.Button(
|
647
687
|
"Update",
|
648
|
-
id={'type': 'update-sql-button', 'index':
|
688
|
+
id={'type': 'update-sql-button', 'index': pipe_meta_str},
|
649
689
|
)
|
650
690
|
items_bodies['sql'] = html.Div([
|
651
691
|
sql_editor,
|
@@ -654,7 +694,7 @@ def accordion_items_from_pipe(
|
|
654
694
|
dbc.Col([update_sql_button], width=2),
|
655
695
|
dbc.Col([
|
656
696
|
html.Div(
|
657
|
-
id={'type': 'update-sql-success-div', 'index':
|
697
|
+
id={'type': 'update-sql-success-div', 'index': pipe_meta_str}
|
658
698
|
)
|
659
699
|
],
|
660
700
|
width=True,
|
@@ -665,10 +705,10 @@ def accordion_items_from_pipe(
|
|
665
705
|
if 'recent-data' in active_items:
|
666
706
|
try:
|
667
707
|
df = pipe.get_backtrack_data(backtrack_minutes=10, limit=10, debug=debug).astype(str)
|
668
|
-
table = dbc.Table.from_dataframe(df, bordered=False, hover=True)
|
708
|
+
table = dbc.Table.from_dataframe(df, bordered=False, hover=True)
|
709
|
+
items_bodies['recent-data'] = html.Div(table, style={'overflowX': 'auto'})
|
669
710
|
except Exception:
|
670
|
-
|
671
|
-
items_bodies['recent-data'] = table
|
711
|
+
items_bodies['recent-data'] = html.P("Could not retrieve recent data.")
|
672
712
|
|
673
713
|
if 'query-data' in active_items:
|
674
714
|
query_editor = dash_ace.DashAceEditor(
|
@@ -676,7 +716,7 @@ def accordion_items_from_pipe(
|
|
676
716
|
mode='norm',
|
677
717
|
tabSize=4,
|
678
718
|
theme='twilight',
|
679
|
-
id={'type': 'query-editor', 'index':
|
719
|
+
id={'type': 'query-editor', 'index': pipe_meta_str},
|
680
720
|
width='100%',
|
681
721
|
height='200px',
|
682
722
|
readOnly=False,
|
@@ -688,17 +728,17 @@ def accordion_items_from_pipe(
|
|
688
728
|
)
|
689
729
|
query_data_button = dbc.Button(
|
690
730
|
"Query",
|
691
|
-
id={'type': 'query-data-button', 'index':
|
731
|
+
id={'type': 'query-data-button', 'index': pipe_meta_str},
|
692
732
|
)
|
693
733
|
|
694
734
|
begin_end_input_group = dbc.InputGroup(
|
695
735
|
[
|
696
736
|
dbc.Input(
|
697
|
-
id={'type': 'query-data-begin-input', 'index':
|
737
|
+
id={'type': 'query-data-begin-input', 'index': pipe_meta_str},
|
698
738
|
placeholder="Begin",
|
699
739
|
),
|
700
740
|
dbc.Input(
|
701
|
-
id={'type': 'query-data-end-input', 'index':
|
741
|
+
id={'type': 'query-data-end-input', 'index': pipe_meta_str},
|
702
742
|
placeholder="End",
|
703
743
|
),
|
704
744
|
],
|
@@ -712,9 +752,12 @@ def accordion_items_from_pipe(
|
|
712
752
|
value=10,
|
713
753
|
step=1,
|
714
754
|
placeholder="Limit",
|
715
|
-
id={'type': 'limit-input', 'index':
|
755
|
+
id={'type': 'limit-input', 'index': pipe_meta_str},
|
756
|
+
)
|
757
|
+
query_result_div = html.Div(
|
758
|
+
id={'type': 'query-result-div', 'index': pipe_meta_str},
|
759
|
+
style={'overflowX': 'auto'},
|
716
760
|
)
|
717
|
-
query_result_div = html.Div(id={'type': 'query-result-div', 'index': json.dumps(pipe.meta)})
|
718
761
|
|
719
762
|
items_bodies['query-data'] = html.Div([
|
720
763
|
query_editor,
|
@@ -722,7 +765,7 @@ def accordion_items_from_pipe(
|
|
722
765
|
dbc.Row(
|
723
766
|
[
|
724
767
|
dbc.Col([query_data_button], lg=2, md=3, sm=4, xs=6, width=2),
|
725
|
-
dbc.Col([begin_end_input_group], lg=
|
768
|
+
dbc.Col([begin_end_input_group], lg=6, md=6, sm=4, width=6),
|
726
769
|
dbc.Col(html.Div([limit_input, dbc.FormText("Row Limit")]), lg=2, md=3, sm=4, xs=6, width=2),
|
727
770
|
],
|
728
771
|
justify="between",
|
@@ -733,27 +776,13 @@ def accordion_items_from_pipe(
|
|
733
776
|
])
|
734
777
|
|
735
778
|
if 'sync-data' in active_items:
|
736
|
-
|
737
|
-
try:
|
738
|
-
json_text = to_json(
|
739
|
-
backtrack_df,
|
740
|
-
orient='records',
|
741
|
-
date_format='iso',
|
742
|
-
force_ascii=False,
|
743
|
-
indent=4,
|
744
|
-
date_unit='us',
|
745
|
-
) if backtrack_df is not None else '[]'
|
746
|
-
except Exception as e:
|
747
|
-
warn(e)
|
748
|
-
json_text = '[]'
|
749
|
-
|
750
|
-
json_text = json.dumps(json.loads(json_text), indent=4, separators=(',', ': '))
|
779
|
+
backtrack_text = get_backtrack_text(pipe)
|
751
780
|
sync_editor = dash_ace.DashAceEditor(
|
752
|
-
value =
|
781
|
+
value = backtrack_text,
|
753
782
|
mode = 'norm',
|
754
783
|
tabSize = 4,
|
755
784
|
theme = 'twilight',
|
756
|
-
id = {'type': 'sync-editor', 'index':
|
785
|
+
id = {'type': 'sync-editor', 'index': pipe_meta_str},
|
757
786
|
width = '100%',
|
758
787
|
height = '500px',
|
759
788
|
readOnly = False,
|
@@ -763,16 +792,40 @@ def accordion_items_from_pipe(
|
|
763
792
|
wrapEnabled = True,
|
764
793
|
style = {'min-height': '120px'},
|
765
794
|
)
|
795
|
+
|
796
|
+
sync_as_json_button = dbc.Button(
|
797
|
+
"JSON",
|
798
|
+
id={'type': 'sync-as-json-button', 'index': pipe_meta_str},
|
799
|
+
color='link',
|
800
|
+
size='sm',
|
801
|
+
style={'text-decoration': 'none', 'margin-left': '10px'},
|
802
|
+
)
|
803
|
+
sync_as_lines_button = dbc.Button(
|
804
|
+
"Lines",
|
805
|
+
id={'type': 'sync-as-lines-button', 'index': pipe_meta_str},
|
806
|
+
color='link',
|
807
|
+
size='sm',
|
808
|
+
style={'text-decoration': 'none', 'margin-left': '10px'},
|
809
|
+
)
|
810
|
+
|
766
811
|
update_sync_button = dbc.Button(
|
767
812
|
"Sync",
|
768
|
-
id = {'type': 'update-sync-button', 'index':
|
813
|
+
id = {'type': 'update-sync-button', 'index': pipe_meta_str},
|
769
814
|
)
|
770
|
-
sync_success_div = html.Div(id={'type': 'sync-success-div', 'index':
|
815
|
+
sync_success_div = html.Div(id={'type': 'sync-success-div', 'index': pipe_meta_str})
|
771
816
|
items_bodies['sync-data'] = html.Div([
|
772
817
|
sync_editor,
|
773
818
|
html.Br(),
|
774
819
|
dbc.Row([
|
775
|
-
dbc.Col(
|
820
|
+
dbc.Col(html.Span(
|
821
|
+
(
|
822
|
+
([update_sync_button] if authenticated else []) +
|
823
|
+
[
|
824
|
+
sync_as_json_button,
|
825
|
+
sync_as_lines_button,
|
826
|
+
]
|
827
|
+
)
|
828
|
+
), width=4),
|
776
829
|
dbc.Col([sync_success_div], width=True),
|
777
830
|
]),
|
778
831
|
])
|
@@ -782,3 +835,237 @@ def accordion_items_from_pipe(
|
|
782
835
|
for item_id, title in items_titles.items()
|
783
836
|
]
|
784
837
|
|
838
|
+
|
839
|
+
def get_backtrack_text(
|
840
|
+
pipe: mrsm.Pipe,
|
841
|
+
lines: bool = False,
|
842
|
+
limit: int = 5,
|
843
|
+
) -> str:
|
844
|
+
"""
|
845
|
+
Return the backtrack documents as text for the sync editor.
|
846
|
+
"""
|
847
|
+
backtrack_df = pipe.get_backtrack_data(debug=debug, limit=limit)
|
848
|
+
if lines:
|
849
|
+
return to_simple_lines(backtrack_df)
|
850
|
+
try:
|
851
|
+
json_text = to_json(
|
852
|
+
backtrack_df,
|
853
|
+
orient='records',
|
854
|
+
date_format='iso',
|
855
|
+
force_ascii=False,
|
856
|
+
indent=4,
|
857
|
+
date_unit='us',
|
858
|
+
) if backtrack_df is not None else '[]'
|
859
|
+
except Exception as e:
|
860
|
+
warn(e)
|
861
|
+
json_text = '[]'
|
862
|
+
|
863
|
+
return json.dumps(json.loads(json_text), indent=4, separators=(',', ': '))
|
864
|
+
|
865
|
+
|
866
|
+
def build_pipes_dropdown_keys_row(
|
867
|
+
connector_keys: List[str],
|
868
|
+
metric_keys: List[str],
|
869
|
+
location_keys: List[str],
|
870
|
+
tags: List[str],
|
871
|
+
pipes: List[mrsm.Pipe],
|
872
|
+
instance_connector: mrsm.connectors.InstanceConnector,
|
873
|
+
) -> dbc.Row:
|
874
|
+
"""
|
875
|
+
Return the dropdown keys row for the dedicated pipes page.
|
876
|
+
"""
|
877
|
+
ck_alone = connector_keys and not any([str(x) for x in (tags + metric_keys + location_keys)])
|
878
|
+
mk_alone = metric_keys and not any([str(x) for x in (connector_keys + tags + location_keys)])
|
879
|
+
lk_alone = location_keys and not any([str(x) for x in (connector_keys + metric_keys + tags)])
|
880
|
+
all_keys = fetch_pipes_keys('registered', instance_connector)
|
881
|
+
|
882
|
+
ck_options_source = (
|
883
|
+
{keys_tuple[0] for keys_tuple in all_keys}
|
884
|
+
if ck_alone
|
885
|
+
else {p.connector_keys for p in pipes}
|
886
|
+
)
|
887
|
+
ck_options = sorted(ck_options_source.union(connector_keys))
|
888
|
+
|
889
|
+
mk_options_source = (
|
890
|
+
{keys_tuple[1] for keys_tuple in all_keys}
|
891
|
+
if mk_alone
|
892
|
+
else {p.metric_key for p in pipes}
|
893
|
+
)
|
894
|
+
mk_options = sorted(mk_options_source.union(metric_keys))
|
895
|
+
|
896
|
+
lk_options_source = (
|
897
|
+
{str(keys_tuple[2]) for keys_tuple in all_keys}
|
898
|
+
if lk_alone
|
899
|
+
else {str(p.location_key) for p in pipes}
|
900
|
+
)
|
901
|
+
lk_options = sorted(lk_options_source.union({str(lk) for lk in location_keys}))
|
902
|
+
|
903
|
+
return dbc.Row(
|
904
|
+
[
|
905
|
+
dbc.Col(
|
906
|
+
html.Div(
|
907
|
+
[
|
908
|
+
dcc.Dropdown(
|
909
|
+
id='pipes-connector-keys-dropdown',
|
910
|
+
options=ck_options,
|
911
|
+
value=[str(ck) for ck in connector_keys],
|
912
|
+
placeholder='Connectors',
|
913
|
+
multi=True,
|
914
|
+
),
|
915
|
+
],
|
916
|
+
className='dbc_dark',
|
917
|
+
),
|
918
|
+
lg=4,
|
919
|
+
md=12,
|
920
|
+
sm=12,
|
921
|
+
),
|
922
|
+
dbc.Col(
|
923
|
+
html.Div(
|
924
|
+
[
|
925
|
+
dcc.Dropdown(
|
926
|
+
id='pipes-metric-keys-dropdown',
|
927
|
+
options=mk_options,
|
928
|
+
value=[str(mk) for mk in metric_keys],
|
929
|
+
placeholder='Metrics',
|
930
|
+
multi=True,
|
931
|
+
),
|
932
|
+
],
|
933
|
+
className='dbc_dark'
|
934
|
+
),
|
935
|
+
lg=4,
|
936
|
+
md=12,
|
937
|
+
sm=12,
|
938
|
+
),
|
939
|
+
dbc.Col(
|
940
|
+
html.Div(
|
941
|
+
[
|
942
|
+
dcc.Dropdown(
|
943
|
+
id='pipes-location-keys-dropdown',
|
944
|
+
options=lk_options,
|
945
|
+
value=[str(lk) for lk in location_keys],
|
946
|
+
placeholder='Locations',
|
947
|
+
multi=True,
|
948
|
+
),
|
949
|
+
],
|
950
|
+
className='dbc_dark'
|
951
|
+
),
|
952
|
+
lg=4,
|
953
|
+
md=12,
|
954
|
+
sm=12,
|
955
|
+
),
|
956
|
+
] ### end of filters row children
|
957
|
+
)
|
958
|
+
|
959
|
+
|
960
|
+
def build_pipes_tags_dropdown(
|
961
|
+
connector_keys: List[str],
|
962
|
+
metric_keys: List[str],
|
963
|
+
location_keys: List[str],
|
964
|
+
tags: List[str],
|
965
|
+
instance: str,
|
966
|
+
) -> dbc.Row:
|
967
|
+
"""
|
968
|
+
Build the tags dropdown for the dedicated pipes page.
|
969
|
+
"""
|
970
|
+
_tags_alone = tags and not any([str(x) for x in (connector_keys + metric_keys + location_keys)])
|
971
|
+
_tags_pipes = mrsm.get_pipes(
|
972
|
+
connector_keys=connector_keys,
|
973
|
+
metric_keys=metric_keys,
|
974
|
+
location_keys=location_keys,
|
975
|
+
tags=tags,
|
976
|
+
instance=instance,
|
977
|
+
as_tags_dict=True,
|
978
|
+
)
|
979
|
+
|
980
|
+
_all_tags = list(
|
981
|
+
mrsm.get_pipes(
|
982
|
+
instance=instance,
|
983
|
+
as_tags_dict=True,
|
984
|
+
)
|
985
|
+
) if _tags_alone else []
|
986
|
+
|
987
|
+
tags_options = [
|
988
|
+
str(tag)
|
989
|
+
for tag in (_all_tags if _tags_alone else _tags_pipes)
|
990
|
+
]
|
991
|
+
if tags:
|
992
|
+
tags_options += [tag for tag in tags if tag not in tags_options]
|
993
|
+
|
994
|
+
return dbc.Row(
|
995
|
+
[
|
996
|
+
dbc.Col(
|
997
|
+
html.Div(
|
998
|
+
dcc.Dropdown(
|
999
|
+
id='pipes-tags-dropdown',
|
1000
|
+
options=tags_options,
|
1001
|
+
value=tags,
|
1002
|
+
placeholder='Tags',
|
1003
|
+
multi=True,
|
1004
|
+
searchable=True,
|
1005
|
+
),
|
1006
|
+
className="dbc_dark",
|
1007
|
+
id="pipes-tags-dropdown-div",
|
1008
|
+
),
|
1009
|
+
width=True,
|
1010
|
+
),
|
1011
|
+
dbc.Col(
|
1012
|
+
dbc.Button(
|
1013
|
+
"Clear all",
|
1014
|
+
color='link',
|
1015
|
+
size='sm',
|
1016
|
+
style={'text-decoration': 'none'},
|
1017
|
+
id='pipes-clear-all-button',
|
1018
|
+
),
|
1019
|
+
width='auto',
|
1020
|
+
),
|
1021
|
+
],
|
1022
|
+
className='g-0',
|
1023
|
+
align='center',
|
1024
|
+
)
|
1025
|
+
|
1026
|
+
|
1027
|
+
def build_pipes_navbar(instance_keys: Optional[str] = None, with_instance_select: bool = True):
|
1028
|
+
"""
|
1029
|
+
Build the navbar from the selected instance keys.
|
1030
|
+
"""
|
1031
|
+
instance_select = dbc.Select(
|
1032
|
+
id='instance-select',
|
1033
|
+
size='sm',
|
1034
|
+
value=instance_keys or str(get_api_connector()),
|
1035
|
+
options=[
|
1036
|
+
{'label': (i[:32] + '…') if len(i) > 32 else i, 'value': i}
|
1037
|
+
for i in get_connector_labels(*instance_types)
|
1038
|
+
],
|
1039
|
+
class_name='dbc_dark custom-select custom-select-sm',
|
1040
|
+
)
|
1041
|
+
instance_select_div_style = {} if with_instance_select else {'visibility': 'hidden'}
|
1042
|
+
instance_select_div = html.Div(instance_select, style=instance_select_div_style)
|
1043
|
+
return html.Div(
|
1044
|
+
[
|
1045
|
+
pages_offcanvas,
|
1046
|
+
dbc.Navbar(
|
1047
|
+
dbc.Container(
|
1048
|
+
[
|
1049
|
+
logo_row,
|
1050
|
+
dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
|
1051
|
+
dbc.Collapse(
|
1052
|
+
dbc.Row(
|
1053
|
+
[
|
1054
|
+
dbc.Col(instance_select_div, width='auto'),
|
1055
|
+
dbc.Col(sign_out_button, width='auto'),
|
1056
|
+
],
|
1057
|
+
className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",
|
1058
|
+
align='center',
|
1059
|
+
),
|
1060
|
+
id='navbar-collapse',
|
1061
|
+
is_open=False,
|
1062
|
+
navbar=True,
|
1063
|
+
),
|
1064
|
+
]
|
1065
|
+
),
|
1066
|
+
dark=True,
|
1067
|
+
color='dark'
|
1068
|
+
),
|
1069
|
+
],
|
1070
|
+
id='pages-navbar-div',
|
1071
|
+
)
|