meerschaum 2.8.4__py3-none-any.whl → 2.9.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/api/_chunks.py +67 -0
- meerschaum/api/dash/callbacks/custom.py +23 -2
- meerschaum/api/dash/callbacks/dashboard.py +41 -3
- meerschaum/api/dash/components.py +27 -19
- meerschaum/api/dash/pages/dashboard.py +11 -9
- meerschaum/api/dash/pages/plugins.py +31 -27
- meerschaum/api/dash/webterm.py +6 -3
- meerschaum/api/resources/static/css/dash.css +1 -1
- meerschaum/api/resources/templates/termpage.html +4 -0
- meerschaum/api/routes/_pipes.py +191 -78
- meerschaum/config/_default.py +3 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/connectors/api/_APIConnector.py +12 -1
- meerschaum/connectors/api/_pipes.py +27 -15
- meerschaum/connectors/api/_plugins.py +51 -45
- meerschaum/connectors/api/_request.py +1 -1
- meerschaum/connectors/parse.py +1 -2
- meerschaum/core/Pipe/_data.py +1 -2
- meerschaum/plugins/_Plugin.py +21 -5
- meerschaum/plugins/__init__.py +6 -4
- meerschaum/utils/formatting/_shell.py +1 -4
- meerschaum/utils/packages/_packages.py +1 -0
- meerschaum/utils/venv/__init__.py +2 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/METADATA +4 -1
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/RECORD +31 -30
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/WHEEL +1 -1
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/LICENSE +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/NOTICE +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.8.4.dist-info → meerschaum-2.9.0.dev1.dist-info}/zip-safe +0 -0
@@ -0,0 +1,67 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# vim:fenc=utf-8
|
3
|
+
|
4
|
+
"""
|
5
|
+
Utility functions for the retrieval, caching, and response of chunk data.
|
6
|
+
"""
|
7
|
+
|
8
|
+
import random
|
9
|
+
from datetime import datetime, timedelta, timezone
|
10
|
+
from typing import Dict, Generator, Any, Union, Optional, List
|
11
|
+
|
12
|
+
import meerschaum as mrsm
|
13
|
+
from meerschaum.api import get_cache_connector
|
14
|
+
from meerschaum.utils.misc import generate_password
|
15
|
+
|
16
|
+
CHUNKS_TOKENS_GENERATORS: Dict[str, Dict[str, Union[Generator[Any, None, None], datetime, int]]]
|
17
|
+
DEFAULT_TTL_SECONDS = mrsm.get_config('system', 'api', 'data', 'chunks', 'ttl_seconds')
|
18
|
+
|
19
|
+
|
20
|
+
def generate_chunks_cursor_token(
|
21
|
+
pipe: mrsm.Pipe,
|
22
|
+
select_columns: Optional[List[str]] = None,
|
23
|
+
omit_columns: Optional[List[str]] = None,
|
24
|
+
begin: Union[datetime, int, None] = None,
|
25
|
+
end: Union[datetime, int, None] = None,
|
26
|
+
params: Optional[Dict[str, Any]] = None,
|
27
|
+
limit: Optional[int] = None,
|
28
|
+
order: Optional[str] = 'asc',
|
29
|
+
ttl_seconds: Optional[int] = None,
|
30
|
+
debug: bool = False,
|
31
|
+
) -> str:
|
32
|
+
"""
|
33
|
+
Store a generator in the `CHUNKS_TOKENS_GENERATORS`
|
34
|
+
"""
|
35
|
+
now = datetime.now(timezone.utc)
|
36
|
+
cache_connector = get_cache_connector()
|
37
|
+
if cache_connector is None:
|
38
|
+
pass
|
39
|
+
|
40
|
+
ttl_seconds = ttl_seconds or DEFAULT_TTL_SECONDS
|
41
|
+
chunk_bounds = pipe.get_chunk_bounds(
|
42
|
+
begin=begin,
|
43
|
+
end=end,
|
44
|
+
bounded=True,
|
45
|
+
)
|
46
|
+
|
47
|
+
while True:
|
48
|
+
chunk_token = prefix + generate_password(random.randint(6, 12))
|
49
|
+
if chunk_token in CHUNKS_TOKENS_GENERATORS:
|
50
|
+
continue
|
51
|
+
break
|
52
|
+
|
53
|
+
CHUNKS_TOKENS_GENERATORS[chunk_token] = {
|
54
|
+
'generator': chunk_generator,
|
55
|
+
'created': now,
|
56
|
+
'ttl': ttl_seconds,
|
57
|
+
'last_accessed': now,
|
58
|
+
}
|
59
|
+
|
60
|
+
return chunk_token
|
61
|
+
|
62
|
+
|
63
|
+
def deallocate_expired_generators():
|
64
|
+
"""
|
65
|
+
Periodically delete chunk tokens with an expired ttl timestamp.
|
66
|
+
"""
|
67
|
+
chunk_tokens = list(CHUNKS_TOKENS_GENERATORS)
|
@@ -7,10 +7,13 @@ Import custom callbacks created by plugins.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
import traceback
|
10
|
+
from typing import Any, Dict
|
11
|
+
|
10
12
|
from meerschaum.api.dash import dash_app
|
11
13
|
from meerschaum.plugins import _dash_plugins, _plugin_endpoints_to_pages
|
12
14
|
from meerschaum.utils.warnings import warn
|
13
|
-
from meerschaum.api.dash.callbacks.dashboard import _paths, _required_login
|
15
|
+
from meerschaum.api.dash.callbacks.dashboard import _paths, _required_login, _pages
|
16
|
+
from meerschaum.api.dash.components import pages_navbar
|
14
17
|
|
15
18
|
|
16
19
|
def init_dash_plugins():
|
@@ -34,6 +37,24 @@ def add_plugin_pages():
|
|
34
37
|
Allow users to add pages via the `@web_page` decorator.
|
35
38
|
"""
|
36
39
|
for _endpoint, _page_dict in _plugin_endpoints_to_pages.items():
|
37
|
-
|
40
|
+
page_layout = _page_dict['function']()
|
41
|
+
if not _page_dict['skip_navbar']:
|
42
|
+
if isinstance(page_layout, list):
|
43
|
+
page_layout = [pages_navbar] + page_layout
|
44
|
+
else:
|
45
|
+
page_layout = [pages_navbar, page_layout]
|
46
|
+
page_key = (
|
47
|
+
' '.join(
|
48
|
+
[
|
49
|
+
word.capitalize()
|
50
|
+
for word in (
|
51
|
+
_endpoint.replace('/dash', '').lstrip('/').rstrip('/').strip()
|
52
|
+
.replace('-', ' ').replace('_', ' ').split(' ')
|
53
|
+
)
|
54
|
+
]
|
55
|
+
)
|
56
|
+
)
|
57
|
+
_pages[page_key] = _endpoint
|
58
|
+
_paths[_endpoint] = page_layout
|
38
59
|
if _page_dict['login_required']:
|
39
60
|
_required_login.add(_endpoint)
|
@@ -103,6 +103,10 @@ _paths = {
|
|
103
103
|
'job' : pages.job.layout,
|
104
104
|
}
|
105
105
|
_required_login = {''}
|
106
|
+
_pages = {
|
107
|
+
'Web Console': '/dash/',
|
108
|
+
'Plugins': '/dash/plugins',
|
109
|
+
}
|
106
110
|
|
107
111
|
|
108
112
|
@dash_app.callback(
|
@@ -696,7 +700,6 @@ dash_app.clientside_callback(
|
|
696
700
|
dash_app.clientside_callback(
|
697
701
|
"""
|
698
702
|
function(n_clicks){
|
699
|
-
console.log('fullscreen');
|
700
703
|
if (!n_clicks) { return dash_clientside.no_update; }
|
701
704
|
iframe = document.getElementById('webterm-iframe');
|
702
705
|
if (!iframe){ return dash_clientside.no_update; }
|
@@ -707,11 +710,11 @@ dash_app.clientside_callback(
|
|
707
710
|
if (leftCol.style.display === 'none') {
|
708
711
|
leftCol.style.display = '';
|
709
712
|
rightCol.className = 'col-6';
|
710
|
-
button.innerHTML = "
|
713
|
+
button.innerHTML = "⛶";
|
711
714
|
} else {
|
712
715
|
leftCol.style.display = 'none';
|
713
716
|
rightCol.className = 'col-12';
|
714
|
-
button.innerHTML = "
|
717
|
+
button.innerHTML = "🀲";
|
715
718
|
}
|
716
719
|
|
717
720
|
return dash_clientside.no_update;
|
@@ -1078,3 +1081,38 @@ def parameters_as_yaml_or_json_click(
|
|
1078
1081
|
if as_yaml:
|
1079
1082
|
return yaml.dump(pipe.parameters)
|
1080
1083
|
return json.dumps(pipe.parameters, indent=4, separators=(',', ': '), sort_keys=True)
|
1084
|
+
|
1085
|
+
|
1086
|
+
@dash_app.callback(
|
1087
|
+
Output('pages-offcanvas', 'is_open'),
|
1088
|
+
Output('pages-offcanvas', 'children'),
|
1089
|
+
Input('logo-img', 'n_clicks'),
|
1090
|
+
State('pages-offcanvas', 'is_open'),
|
1091
|
+
)
|
1092
|
+
def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
|
1093
|
+
"""
|
1094
|
+
Toggle the pages sidebar.
|
1095
|
+
"""
|
1096
|
+
pages_children = dbc.Card(
|
1097
|
+
dbc.ListGroup(
|
1098
|
+
[
|
1099
|
+
dbc.ListGroupItem(
|
1100
|
+
dbc.Button(
|
1101
|
+
html.P(
|
1102
|
+
' '.join([word.capitalize() for word in page_key.split(' ')]),
|
1103
|
+
style={'text-decoration': 'none', 'fontSize': '18px'},
|
1104
|
+
),
|
1105
|
+
style={'width': '100%', 'text-align': 'left'},
|
1106
|
+
href=page_href,
|
1107
|
+
color='dark',
|
1108
|
+
)
|
1109
|
+
)
|
1110
|
+
for page_key, page_href in _pages.items()
|
1111
|
+
],
|
1112
|
+
flush=True,
|
1113
|
+
),
|
1114
|
+
outline=True,
|
1115
|
+
)
|
1116
|
+
if n_clicks:
|
1117
|
+
return not is_open, pages_children
|
1118
|
+
return is_open, pages_children
|
@@ -92,9 +92,9 @@ search_parameters_editor = dash_ace.DashAceEditor(
|
|
92
92
|
style={'height': 100},
|
93
93
|
)
|
94
94
|
|
95
|
-
|
96
|
-
|
97
|
-
|
95
|
+
pages_offcanvas = dbc.Offcanvas(
|
96
|
+
title='',
|
97
|
+
id='pages-offcanvas',
|
98
98
|
)
|
99
99
|
|
100
100
|
download_dataframe = dcc.Download(id='download-dataframe-csv')
|
@@ -110,26 +110,33 @@ instance_select = dbc.Select(
|
|
110
110
|
class_name='dbc_dark custom-select custom-select-sm',
|
111
111
|
)
|
112
112
|
|
113
|
+
logo_row = dbc.Row(
|
114
|
+
[
|
115
|
+
dbc.Col(
|
116
|
+
html.Img(
|
117
|
+
src=endpoints['dash'] + "/assets/logo_48x48.png",
|
118
|
+
title=doc,
|
119
|
+
id="logo-img",
|
120
|
+
style={'cursor': 'pointer'},
|
121
|
+
),
|
122
|
+
),
|
123
|
+
],
|
124
|
+
align='center',
|
125
|
+
className='g-0 navbar-logo-row',
|
126
|
+
)
|
127
|
+
|
128
|
+
pages_navbar = html.Div(
|
129
|
+
[
|
130
|
+
pages_offcanvas,
|
131
|
+
dbc.Navbar(dbc.Container(logo_row), dark=True, color='dark'),
|
132
|
+
],
|
133
|
+
id='pages-navbar-div',
|
134
|
+
)
|
113
135
|
|
114
136
|
navbar = dbc.Navbar(
|
115
137
|
dbc.Container(
|
116
138
|
[
|
117
|
-
|
118
|
-
dbc.Row(
|
119
|
-
[
|
120
|
-
dbc.Col(
|
121
|
-
html.Img(
|
122
|
-
src=endpoints['dash'] + "/assets/logo_48x48.png",
|
123
|
-
title=doc,
|
124
|
-
),
|
125
|
-
),
|
126
|
-
],
|
127
|
-
align='center',
|
128
|
-
className='g-0 navbar-logo-row',
|
129
|
-
),
|
130
|
-
href=('/docs' if docs_enabled else '#'),
|
131
|
-
style={"textDecoration": "none"},
|
132
|
-
),
|
139
|
+
logo_row,
|
133
140
|
dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
|
134
141
|
dbc.Collapse(
|
135
142
|
dbc.Row(
|
@@ -142,6 +149,7 @@ navbar = dbc.Navbar(
|
|
142
149
|
style={'margin-left': '30px'},
|
143
150
|
id='sign-out-button',
|
144
151
|
),
|
152
|
+
className="ms-auto",
|
145
153
|
),
|
146
154
|
],
|
147
155
|
className="g-0 ms-auto flex-nowrap mt-3 mt-md-0",
|
@@ -37,6 +37,7 @@ from meerschaum.api.dash.components import (
|
|
37
37
|
console_div,
|
38
38
|
download_dataframe,
|
39
39
|
navbar,
|
40
|
+
pages_offcanvas,
|
40
41
|
download_logs,
|
41
42
|
refresh_jobs_interval,
|
42
43
|
)
|
@@ -47,22 +48,23 @@ from meerschaum.api.dash.keys import (
|
|
47
48
|
)
|
48
49
|
|
49
50
|
layout = html.Div(
|
50
|
-
id
|
51
|
-
children
|
51
|
+
id='main-div',
|
52
|
+
children=[
|
52
53
|
keys_lists_content,
|
53
54
|
download_dataframe,
|
54
55
|
download_logs,
|
55
56
|
refresh_jobs_interval,
|
56
57
|
navbar,
|
58
|
+
pages_offcanvas,
|
57
59
|
html.Div(
|
58
60
|
dbc.Row(
|
59
|
-
id
|
60
|
-
children
|
61
|
+
id='content-row',
|
62
|
+
children=[
|
61
63
|
dbc.Col(
|
62
|
-
children
|
64
|
+
children=[
|
63
65
|
dbc.Tabs(
|
64
|
-
id
|
65
|
-
children
|
66
|
+
id='pipes-filter-tabs',
|
67
|
+
children=[
|
66
68
|
dbc.Tab(
|
67
69
|
dropdown_tab_content,
|
68
70
|
label='Filters',
|
@@ -107,9 +109,9 @@ layout = html.Div(
|
|
107
109
|
id='content-col-right',
|
108
110
|
),
|
109
111
|
],
|
110
|
-
style
|
112
|
+
style={'max-width': '100%', 'padding': '15px'},
|
111
113
|
), ### end of Row
|
112
|
-
className
|
114
|
+
className='container-fluid',
|
113
115
|
), ### end of Div
|
114
116
|
html.P('', id='line-buffer', style={'display': 'none'}),
|
115
117
|
],
|
@@ -13,6 +13,7 @@ html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHEC
|
|
13
13
|
import dash_bootstrap_components as dbc
|
14
14
|
from meerschaum.core import Plugin
|
15
15
|
from meerschaum.utils.typing import Optional
|
16
|
+
from meerschaum.api.dash.components import pages_navbar
|
16
17
|
|
17
18
|
search_box = dbc.Input(
|
18
19
|
id = "search-plugins-input",
|
@@ -20,32 +21,35 @@ search_box = dbc.Input(
|
|
20
21
|
type = "text",
|
21
22
|
)
|
22
23
|
|
23
|
-
layout =
|
24
|
-
|
25
|
-
|
26
|
-
html.Div(
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
(
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
24
|
+
layout = [
|
25
|
+
pages_navbar,
|
26
|
+
dbc.Container([
|
27
|
+
html.Div([
|
28
|
+
html.Br(),
|
29
|
+
html.Div(
|
30
|
+
dbc.Container([
|
31
|
+
html.H3('Plugins'),
|
32
|
+
html.P([
|
33
|
+
(
|
34
|
+
'Plugins extend the functionality of Meerschaum.'
|
35
|
+
' To find out more, check out the '
|
36
|
+
),
|
37
|
+
html.A(
|
38
|
+
'plugins documentation',
|
39
|
+
href='https://meerschaum.io/reference/plugins/',
|
40
|
+
rel="noreferrer noopener",
|
41
|
+
target="_blank",
|
42
|
+
),
|
43
|
+
'.',
|
44
|
+
]),
|
41
45
|
]),
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
46
|
+
className='page-header',
|
47
|
+
style={'background-color': 'var(--dark)', 'padding': '1em'},
|
48
|
+
),
|
49
|
+
html.Br(),
|
50
|
+
search_box,
|
51
|
+
html.Br(),
|
52
|
+
html.Div([], id='plugins-cards-div'),
|
53
|
+
])
|
50
54
|
])
|
51
|
-
]
|
55
|
+
]
|
meerschaum/api/dash/webterm.py
CHANGED
@@ -53,19 +53,22 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
|
|
53
53
|
html.Div(
|
54
54
|
[
|
55
55
|
dbc.Button(
|
56
|
-
|
56
|
+
"⟳",
|
57
57
|
color='black',
|
58
|
+
size='sm',
|
58
59
|
id='webterm-refresh-button',
|
59
60
|
),
|
60
61
|
dbc.Button(
|
61
|
-
'
|
62
|
+
'⛶',
|
62
63
|
color='black',
|
64
|
+
size='sm',
|
63
65
|
id='webterm-fullscreen-button',
|
64
66
|
),
|
65
67
|
] + [
|
66
68
|
dbc.Button(
|
67
|
-
'
|
69
|
+
html.B('+'),
|
68
70
|
color='black',
|
71
|
+
size='sm',
|
69
72
|
id='webterm-new-tab-button',
|
70
73
|
),
|
71
74
|
] if TMUX_IS_ENABLED else [],
|