meerschaum 2.3.6__py3-none-any.whl → 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parse_arguments.py +2 -5
- meerschaum/_internal/docs/index.py +3 -2
- meerschaum/_internal/entry.py +13 -7
- meerschaum/_internal/shell/Shell.py +38 -44
- meerschaum/_internal/term/TermPageHandler.py +2 -3
- meerschaum/_internal/term/__init__.py +13 -11
- meerschaum/actions/api.py +10 -7
- meerschaum/actions/bootstrap.py +38 -11
- meerschaum/actions/copy.py +3 -3
- meerschaum/actions/delete.py +4 -1
- meerschaum/actions/register.py +1 -3
- meerschaum/actions/stack.py +24 -19
- meerschaum/actions/start.py +38 -40
- meerschaum/actions/sync.py +53 -52
- meerschaum/api/__init__.py +48 -14
- meerschaum/api/_events.py +15 -10
- meerschaum/api/_oauth2.py +2 -2
- meerschaum/api/_websockets.py +5 -4
- meerschaum/api/dash/__init__.py +7 -16
- meerschaum/api/dash/callbacks/__init__.py +1 -0
- meerschaum/api/dash/callbacks/dashboard.py +52 -58
- meerschaum/api/dash/callbacks/jobs.py +15 -16
- meerschaum/api/dash/callbacks/login.py +16 -10
- meerschaum/api/dash/callbacks/pipes.py +41 -0
- meerschaum/api/dash/callbacks/plugins.py +1 -1
- meerschaum/api/dash/callbacks/register.py +15 -11
- meerschaum/api/dash/components.py +54 -59
- meerschaum/api/dash/jobs.py +5 -9
- meerschaum/api/dash/pages/__init__.py +1 -0
- meerschaum/api/dash/pages/pipes.py +19 -0
- meerschaum/api/dash/pipes.py +86 -58
- meerschaum/api/dash/plugins.py +6 -4
- meerschaum/api/dash/sessions.py +176 -0
- meerschaum/api/dash/users.py +3 -41
- meerschaum/api/dash/webterm.py +12 -17
- meerschaum/api/resources/static/js/terminado.js +1 -1
- meerschaum/api/routes/_actions.py +4 -20
- meerschaum/api/routes/_jobs.py +8 -7
- meerschaum/api/routes/_login.py +4 -4
- meerschaum/api/routes/_pipes.py +3 -3
- meerschaum/api/routes/_webterm.py +5 -6
- meerschaum/config/_default.py +15 -2
- meerschaum/config/_version.py +1 -1
- meerschaum/config/stack/__init__.py +64 -19
- meerschaum/config/static/__init__.py +4 -0
- meerschaum/connectors/{Connector.py → _Connector.py} +19 -13
- meerschaum/connectors/__init__.py +24 -14
- meerschaum/connectors/api/{APIConnector.py → _APIConnector.py} +3 -1
- meerschaum/connectors/api/__init__.py +2 -1
- meerschaum/connectors/parse.py +18 -16
- meerschaum/connectors/poll.py +30 -24
- meerschaum/connectors/sql/__init__.py +3 -1
- meerschaum/connectors/sql/_pipes.py +172 -197
- meerschaum/connectors/sql/_plugins.py +45 -43
- meerschaum/connectors/sql/_users.py +46 -38
- meerschaum/connectors/valkey/_ValkeyConnector.py +535 -0
- meerschaum/connectors/valkey/__init__.py +10 -0
- meerschaum/connectors/valkey/_fetch.py +75 -0
- meerschaum/connectors/valkey/_pipes.py +844 -0
- meerschaum/connectors/valkey/_plugins.py +265 -0
- meerschaum/connectors/valkey/_users.py +305 -0
- meerschaum/core/Pipe/__init__.py +3 -0
- meerschaum/core/Pipe/_attributes.py +1 -2
- meerschaum/core/Pipe/_clear.py +16 -13
- meerschaum/core/Pipe/_copy.py +106 -0
- meerschaum/core/Pipe/_data.py +165 -101
- meerschaum/core/Pipe/_drop.py +4 -4
- meerschaum/core/Pipe/_dtypes.py +14 -14
- meerschaum/core/Pipe/_edit.py +15 -14
- meerschaum/core/Pipe/_sync.py +134 -53
- meerschaum/core/Pipe/_verify.py +11 -11
- meerschaum/core/User/_User.py +14 -12
- meerschaum/jobs/_Job.py +1 -6
- meerschaum/jobs/__init__.py +7 -2
- meerschaum/plugins/_Plugin.py +17 -13
- meerschaum/utils/_get_pipes.py +14 -20
- meerschaum/utils/dataframe.py +291 -101
- meerschaum/utils/dtypes/__init__.py +31 -6
- meerschaum/utils/dtypes/sql.py +4 -4
- meerschaum/utils/formatting/_shell.py +5 -6
- meerschaum/utils/misc.py +3 -3
- meerschaum/utils/packages/__init__.py +14 -9
- meerschaum/utils/packages/_packages.py +2 -0
- meerschaum/utils/schedule.py +1 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/METADATA +7 -1
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/RECORD +93 -84
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/WHEEL +1 -1
- meerschaum/api/dash/actions.py +0 -255
- /meerschaum/connectors/sql/{SQLConnector.py → _SQLConnector.py} +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.3.6.dist-info → meerschaum-2.4.0.dist-info}/zip-safe +0 -0
@@ -0,0 +1,176 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# vim:fenc=utf-8
|
3
|
+
|
4
|
+
"""
|
5
|
+
Define caching functions for session management.
|
6
|
+
"""
|
7
|
+
|
8
|
+
import json
|
9
|
+
|
10
|
+
from meerschaum.utils.typing import Union, Dict, Any, Optional
|
11
|
+
from meerschaum.api import debug, no_auth
|
12
|
+
from meerschaum.api import get_cache_connector, get_api_connector
|
13
|
+
from meerschaum.core import User
|
14
|
+
from meerschaum.config import get_config
|
15
|
+
from meerschaum.utils.warnings import dprint
|
16
|
+
|
17
|
+
SESSION_KEY_TEMPLATE: str = 'mrsm_session_id:{session_id}'
|
18
|
+
EXPIRES_SECONDS: int = get_config('system', 'api', 'cache', 'session_expires_minutes') * 60
|
19
|
+
_active_sessions: Dict[str, Dict[str, Any]] = {}
|
20
|
+
|
21
|
+
|
22
|
+
def get_session_key(session_id: str) -> str:
|
23
|
+
"""
|
24
|
+
Return the session key for the cache connector.
|
25
|
+
"""
|
26
|
+
return SESSION_KEY_TEMPLATE.format(session_id=session_id)
|
27
|
+
|
28
|
+
|
29
|
+
def set_session(session_id: str, session_data: Dict[str, Any]):
|
30
|
+
"""
|
31
|
+
Set a `session_id` to a dictionary.
|
32
|
+
"""
|
33
|
+
conn = get_cache_connector()
|
34
|
+
if conn is None:
|
35
|
+
_active_sessions[session_id] = session_data
|
36
|
+
if debug:
|
37
|
+
dprint(f"Setting in-memory data for {session_id}:\n{session_data}")
|
38
|
+
return
|
39
|
+
|
40
|
+
session_key = get_session_key(session_id)
|
41
|
+
session_data_str = json.dumps(session_data, separators=(',', ':'))
|
42
|
+
if debug:
|
43
|
+
dprint(f"Setting production data for {session_id=}:\n{session_data_str}")
|
44
|
+
|
45
|
+
conn.set(session_key, session_data_str, ex=EXPIRES_SECONDS)
|
46
|
+
|
47
|
+
|
48
|
+
def update_session(session_id: Optional[str], session_data: Dict[str, Any]):
|
49
|
+
"""
|
50
|
+
Update the session's data dictionary.
|
51
|
+
"""
|
52
|
+
existing_session_data = get_session_data(session_id) or {}
|
53
|
+
existing_session_data.update(session_data)
|
54
|
+
set_session(str(session_id), existing_session_data)
|
55
|
+
|
56
|
+
|
57
|
+
def get_session_data(session_id: Optional[str]) -> Union[Dict[str, Any], None]:
|
58
|
+
"""
|
59
|
+
Return the session data dictionary.
|
60
|
+
"""
|
61
|
+
if debug:
|
62
|
+
dprint(f"Getting session data for {session_id=}")
|
63
|
+
conn = get_cache_connector()
|
64
|
+
if conn is None:
|
65
|
+
return _active_sessions.get(str(session_id), None)
|
66
|
+
|
67
|
+
session_key = get_session_key(str(session_id))
|
68
|
+
session_data_str = conn.get(session_key)
|
69
|
+
if not session_data_str:
|
70
|
+
return None
|
71
|
+
|
72
|
+
return json.loads(session_data_str)
|
73
|
+
|
74
|
+
|
75
|
+
def get_username_from_session(session_id: Optional[str]) -> Union[str, None]:
|
76
|
+
"""
|
77
|
+
If a `session_id` has been set, return the username.
|
78
|
+
Otherwise return `None`.
|
79
|
+
"""
|
80
|
+
if debug:
|
81
|
+
dprint(f"Getting username for {session_id=}")
|
82
|
+
session_data = get_session_data(session_id)
|
83
|
+
if session_data is None:
|
84
|
+
return None
|
85
|
+
|
86
|
+
return session_data.get('username', None)
|
87
|
+
|
88
|
+
|
89
|
+
def is_session_active(session_id: Union[str, None]) -> bool:
|
90
|
+
"""
|
91
|
+
Return whether a given `session_id` has been set.
|
92
|
+
"""
|
93
|
+
return get_username_from_session(str(session_id)) is not None
|
94
|
+
|
95
|
+
|
96
|
+
def delete_session(session_id: str):
|
97
|
+
"""
|
98
|
+
Delete a session if it's been set.
|
99
|
+
"""
|
100
|
+
if debug:
|
101
|
+
dprint(f"Deleting {session_id=}")
|
102
|
+
conn = get_cache_connector()
|
103
|
+
if conn is None:
|
104
|
+
_ = _active_sessions.pop(session_id, None)
|
105
|
+
return
|
106
|
+
|
107
|
+
session_key = get_session_key(session_id)
|
108
|
+
conn.client.delete(session_key)
|
109
|
+
|
110
|
+
|
111
|
+
def is_session_authenticated(session_id: Optional[str]) -> bool:
|
112
|
+
"""
|
113
|
+
Check is a session ID is active.
|
114
|
+
If running in secure mode, check whether a session ID corresponds to an admin.
|
115
|
+
|
116
|
+
Parameters
|
117
|
+
----------
|
118
|
+
session_id: str
|
119
|
+
The session UUID.
|
120
|
+
|
121
|
+
Returns
|
122
|
+
-------
|
123
|
+
A bool whether the session is authenticated to perform actions.
|
124
|
+
"""
|
125
|
+
if debug:
|
126
|
+
dprint(f"Checking authentication for {session_id=}")
|
127
|
+
|
128
|
+
if no_auth:
|
129
|
+
return True
|
130
|
+
|
131
|
+
if session_id is None:
|
132
|
+
return False
|
133
|
+
|
134
|
+
session_data = get_session_data(session_id)
|
135
|
+
if session_data is None:
|
136
|
+
return False
|
137
|
+
username = session_data.get('username', None)
|
138
|
+
if username is None:
|
139
|
+
return False
|
140
|
+
|
141
|
+
cached_auth = session_data.get('authenticated', None)
|
142
|
+
if cached_auth is not None:
|
143
|
+
return cached_auth
|
144
|
+
|
145
|
+
permissions = get_config('system', 'api', 'permissions')
|
146
|
+
allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
|
147
|
+
|
148
|
+
is_auth = True if allow_non_admin else session_is_admin(session_id)
|
149
|
+
update_session(session_id, {'authenticated': is_auth})
|
150
|
+
return is_auth
|
151
|
+
|
152
|
+
|
153
|
+
def session_is_admin(session_id: str) -> bool:
|
154
|
+
"""
|
155
|
+
Check whether a session ID corresponds to an admin user.
|
156
|
+
"""
|
157
|
+
if debug:
|
158
|
+
dprint(f"Check admin for {session_id=}")
|
159
|
+
session_data = get_session_data(session_id)
|
160
|
+
if session_data is None:
|
161
|
+
return False
|
162
|
+
|
163
|
+
username = session_data.get('username', None)
|
164
|
+
if username is None:
|
165
|
+
return False
|
166
|
+
|
167
|
+
cached_admin = session_data.get('admin', None)
|
168
|
+
if cached_admin is not None:
|
169
|
+
return cached_admin
|
170
|
+
|
171
|
+
conn = get_api_connector()
|
172
|
+
user = User(username, instance=conn)
|
173
|
+
user_type = conn.get_user_type(user, debug=debug)
|
174
|
+
is_admin = user_type == 'admin'
|
175
|
+
update_session(session_id, {'admin': is_admin})
|
176
|
+
return is_admin
|
meerschaum/api/dash/users.py
CHANGED
@@ -8,16 +8,14 @@ Functions for interacting with users.
|
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
10
|
|
11
|
-
from meerschaum.api import debug, CHECK_UPDATE
|
12
|
-
from meerschaum.api.dash import active_sessions, authenticated_sessions, unauthenticated_sessions
|
11
|
+
from meerschaum.api import debug, CHECK_UPDATE
|
13
12
|
from meerschaum.api.dash.connectors import get_web_connector
|
14
|
-
from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple
|
13
|
+
from meerschaum.utils.typing import WebState, SuccessTuple, List, Tuple
|
15
14
|
from meerschaum.utils.packages import attempt_import, import_html, import_dcc
|
16
|
-
from meerschaum.config import get_config
|
17
|
-
from meerschaum.core import User
|
18
15
|
dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
|
19
16
|
dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
|
20
17
|
|
18
|
+
|
21
19
|
def get_users_cards(state: WebState) -> Tuple[List[dbc.Card], List[SuccessTuple]]:
|
22
20
|
"""
|
23
21
|
Return the cards and alerts for users.
|
@@ -34,39 +32,3 @@ def get_users_cards(state: WebState) -> Tuple[List[dbc.Card], List[SuccessTuple]
|
|
34
32
|
)
|
35
33
|
|
36
34
|
return cards, alerts
|
37
|
-
|
38
|
-
|
39
|
-
def is_session_authenticated(session_id: str) -> bool:
|
40
|
-
"""
|
41
|
-
Check is a session ID is active.
|
42
|
-
If running in secure mode, check whether a session ID corresponds to an admin.
|
43
|
-
|
44
|
-
Parameters
|
45
|
-
----------
|
46
|
-
session_id: str
|
47
|
-
The session UUID.
|
48
|
-
|
49
|
-
Returns
|
50
|
-
-------
|
51
|
-
A bool whether the session is authenticated to perform actions.
|
52
|
-
"""
|
53
|
-
if no_auth:
|
54
|
-
return True
|
55
|
-
if session_id in unauthenticated_sessions:
|
56
|
-
return False
|
57
|
-
if session_id in authenticated_sessions:
|
58
|
-
return True
|
59
|
-
permissions = get_config('system', 'api', 'permissions')
|
60
|
-
allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
|
61
|
-
if allow_non_admin:
|
62
|
-
return True
|
63
|
-
conn = get_api_connector()
|
64
|
-
username = active_sessions.get(session_id, {}).get('username', None)
|
65
|
-
user = User(username, instance=conn)
|
66
|
-
user_type = conn.get_user_type(user, debug=debug)
|
67
|
-
is_auth = user_type == 'admin'
|
68
|
-
if is_auth:
|
69
|
-
authenticated_sessions[session_id] = username
|
70
|
-
else:
|
71
|
-
unauthenticated_sessions[session_id] = username
|
72
|
-
return is_auth
|
meerschaum/api/dash/webterm.py
CHANGED
@@ -7,16 +7,12 @@ Functions for interacting with the Webterm via the dashboard.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
import time
|
10
|
-
from
|
11
|
-
from meerschaum.
|
12
|
-
from meerschaum.api import debug, CHECK_UPDATE, get_api_connector, no_auth
|
13
|
-
from meerschaum.api.dash import active_sessions
|
14
|
-
from meerschaum.api.dash.users import is_session_authenticated
|
10
|
+
from meerschaum.api import CHECK_UPDATE, get_api_connector
|
11
|
+
from meerschaum.api.dash.sessions import is_session_authenticated, get_username_from_session
|
15
12
|
from meerschaum.api.dash.components import alert_from_success_tuple, console_div
|
16
|
-
from meerschaum.utils.typing import WebState,
|
17
|
-
from meerschaum.utils.packages import attempt_import, import_html, import_dcc
|
13
|
+
from meerschaum.utils.typing import WebState, Tuple, Any
|
14
|
+
from meerschaum.utils.packages import attempt_import, import_html, import_dcc
|
18
15
|
from meerschaum._internal.term.tools import is_webterm_running
|
19
|
-
from meerschaum.config.static import STATIC_CONFIG
|
20
16
|
from meerschaum.utils.threading import Thread, RLock
|
21
17
|
dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
|
22
18
|
dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
|
@@ -30,7 +26,7 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
|
|
30
26
|
Start the webterm and return its iframe.
|
31
27
|
"""
|
32
28
|
session_id = state['session-store.data'].get('session-id', None)
|
33
|
-
username =
|
29
|
+
username = get_username_from_session(session_id)
|
34
30
|
if not is_session_authenticated(session_id):
|
35
31
|
msg = f"User '{username}' is not authorized to access the webterm."
|
36
32
|
return (
|
@@ -57,24 +53,23 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
|
|
57
53
|
return console_div, [alert_from_success_tuple((False, "Could not start the webterm server."))]
|
58
54
|
|
59
55
|
|
60
|
-
|
61
56
|
webterm_procs = {}
|
62
57
|
def start_webterm() -> None:
|
63
58
|
"""
|
64
59
|
Start the webterm thread.
|
65
60
|
"""
|
66
|
-
from meerschaum._internal.entry import entry
|
67
61
|
from meerschaum.utils.packages import run_python_package
|
68
62
|
|
69
63
|
def run():
|
64
|
+
conn = get_api_connector()
|
70
65
|
_ = run_python_package(
|
71
66
|
'meerschaum',
|
72
|
-
['start', 'webterm'],
|
73
|
-
capture_output
|
74
|
-
as_proc
|
75
|
-
store_proc_dict
|
76
|
-
store_proc_key
|
77
|
-
venv
|
67
|
+
['start', 'webterm', '-i', str(conn)],
|
68
|
+
capture_output=True,
|
69
|
+
as_proc=True,
|
70
|
+
store_proc_dict=webterm_procs,
|
71
|
+
store_proc_key='process',
|
72
|
+
venv=None,
|
78
73
|
)
|
79
74
|
|
80
75
|
with _locks['webterm_thread']:
|
@@ -15,7 +15,7 @@ function make_terminal(element, size, ws_url) {
|
|
15
15
|
term.attachCustomKeyEventHandler(copyPasteKeyEventHandler);
|
16
16
|
term.open(element);
|
17
17
|
|
18
|
-
sessionStore =
|
18
|
+
sessionStore = localStorage.getItem("session-store");
|
19
19
|
ws.onopen = function (event) {
|
20
20
|
ws.send(sessionStore);
|
21
21
|
ws.send(
|
@@ -7,28 +7,13 @@ Execute Meerschaum Actions via the API
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
|
11
|
-
import
|
12
|
-
import shlex
|
13
|
-
from functools import partial
|
14
|
-
from datetime import datetime, timezone
|
15
|
-
|
16
|
-
from fastapi import WebSocket, WebSocketDisconnect
|
17
|
-
from websockets.exceptions import ConnectionClosedError
|
18
|
-
|
19
|
-
from meerschaum.utils.misc import generate_password
|
20
|
-
from meerschaum.jobs import Job
|
21
|
-
from meerschaum.utils.warnings import warn
|
22
|
-
from meerschaum.utils.typing import SuccessTuple, Union, List, Dict, Any
|
10
|
+
|
11
|
+
from meerschaum.utils.typing import SuccessTuple, List, Dict, Any
|
23
12
|
from meerschaum.api import (
|
24
13
|
fastapi, app, endpoints, get_api_connector, debug, manager, private, no_auth
|
25
14
|
)
|
26
15
|
from meerschaum.actions import actions
|
27
|
-
import meerschaum.core
|
28
16
|
from meerschaum.config import get_config
|
29
|
-
from meerschaum._internal.arguments._parse_arguments import parse_dict_to_sysargs, parse_arguments
|
30
|
-
from meerschaum.api.routes._jobs import clean_sysargs
|
31
|
-
from meerschaum.jobs._Job import StopMonitoringLogs
|
32
17
|
|
33
18
|
actions_endpoint = endpoints['actions']
|
34
19
|
|
@@ -82,7 +67,7 @@ def do_action_legacy(
|
|
82
67
|
----------
|
83
68
|
action: str
|
84
69
|
The action to perform.
|
85
|
-
|
70
|
+
|
86
71
|
keywords: Dict[str, Any]
|
87
72
|
The keywords dictionary to pass to the action.
|
88
73
|
|
@@ -91,7 +76,6 @@ def do_action_legacy(
|
|
91
76
|
A `SuccessTuple`.
|
92
77
|
"""
|
93
78
|
if curr_user is not None and curr_user.type != 'admin':
|
94
|
-
from meerschaum.config import get_config
|
95
79
|
allow_non_admin = get_config(
|
96
80
|
'system', 'api', 'permissions', 'actions', 'non_admin', patch=True
|
97
81
|
)
|
@@ -103,7 +87,7 @@ def do_action_legacy(
|
|
103
87
|
+ "and search for 'permissions'. "
|
104
88
|
+ "\nUnder the keys 'api:permissions:actions', "
|
105
89
|
+ "you can allow non-admin users to perform actions."
|
106
|
-
|
90
|
+
)
|
107
91
|
|
108
92
|
if action not in actions:
|
109
93
|
return False, f"Invalid action '{action}'."
|
meerschaum/api/routes/_jobs.py
CHANGED
@@ -9,7 +9,6 @@ Manage jobs via the Meerschaum API.
|
|
9
9
|
from __future__ import annotations
|
10
10
|
|
11
11
|
import os
|
12
|
-
import select
|
13
12
|
import asyncio
|
14
13
|
import traceback
|
15
14
|
from datetime import datetime
|
@@ -18,12 +17,10 @@ from functools import partial
|
|
18
17
|
|
19
18
|
from fastapi import WebSocket, WebSocketDisconnect
|
20
19
|
|
21
|
-
from meerschaum.utils.typing import Dict, Any, SuccessTuple, List,
|
20
|
+
from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union
|
22
21
|
from meerschaum.jobs import (
|
23
22
|
get_jobs as _get_jobs,
|
24
23
|
Job,
|
25
|
-
StopMonitoringLogs,
|
26
|
-
get_executor_keys_from_context,
|
27
24
|
)
|
28
25
|
from meerschaum.utils.warnings import warn
|
29
26
|
|
@@ -32,14 +29,13 @@ from meerschaum.api import (
|
|
32
29
|
app,
|
33
30
|
endpoints,
|
34
31
|
manager,
|
35
|
-
debug,
|
36
32
|
no_auth,
|
37
|
-
private,
|
38
33
|
)
|
39
34
|
from meerschaum.config.static import STATIC_CONFIG
|
40
35
|
|
41
36
|
JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
|
42
37
|
JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
|
38
|
+
NONINTERACTIVE_ENV: str = STATIC_CONFIG['environment']['noninteractive']
|
43
39
|
EXECUTOR_KEYS: str = 'local'
|
44
40
|
|
45
41
|
|
@@ -48,7 +44,8 @@ def _get_job(name: str):
|
|
48
44
|
if systemd_job.exists():
|
49
45
|
return systemd_job
|
50
46
|
|
51
|
-
|
47
|
+
job = Job(name, executor_keys=EXECUTOR_KEYS)
|
48
|
+
return job
|
52
49
|
|
53
50
|
|
54
51
|
@app.get(endpoints['jobs'], tags=['Jobs'])
|
@@ -150,6 +147,10 @@ def create_job(
|
|
150
147
|
name,
|
151
148
|
clean_sysargs(sysargs),
|
152
149
|
executor_keys=EXECUTOR_KEYS,
|
150
|
+
env={
|
151
|
+
NONINTERACTIVE_ENV: '1',
|
152
|
+
**dict(os.environ)
|
153
|
+
},
|
153
154
|
_properties=properties,
|
154
155
|
)
|
155
156
|
if job.exists():
|
meerschaum/api/routes/_login.py
CHANGED
@@ -23,8 +23,8 @@ from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
|
|
23
23
|
|
24
24
|
@manager.user_loader()
|
25
25
|
def load_user(
|
26
|
-
|
27
|
-
|
26
|
+
username: str
|
27
|
+
) -> User:
|
28
28
|
"""
|
29
29
|
Create the `meerschaum.core.User` object from the username.
|
30
30
|
"""
|
@@ -33,8 +33,8 @@ def load_user(
|
|
33
33
|
|
34
34
|
@app.post(endpoints['login'], tags=['Users'])
|
35
35
|
def login(
|
36
|
-
|
37
|
-
|
36
|
+
data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
|
37
|
+
) -> Dict[str, Any]:
|
38
38
|
"""
|
39
39
|
Login and set the session token.
|
40
40
|
"""
|
meerschaum/api/routes/_pipes.py
CHANGED
@@ -474,9 +474,9 @@ def get_pipe_data(
|
|
474
474
|
df[col] = df[col].apply(lambda x: f'{x:f}' if isinstance(x, Decimal) else x)
|
475
475
|
|
476
476
|
json_content = df.to_json(
|
477
|
-
date_format
|
478
|
-
orient
|
479
|
-
date_unit
|
477
|
+
date_format='iso',
|
478
|
+
orient='records',
|
479
|
+
date_unit='us',
|
480
480
|
)
|
481
481
|
|
482
482
|
return fastapi.Response(
|
@@ -8,10 +8,9 @@ Routes to the Webterm proxy.
|
|
8
8
|
|
9
9
|
import asyncio
|
10
10
|
from meerschaum.utils.typing import Optional
|
11
|
-
from meerschaum.api import app,
|
11
|
+
from meerschaum.api import app, endpoints
|
12
12
|
from meerschaum.utils.packages import attempt_import
|
13
|
-
from meerschaum.api.dash import
|
14
|
-
from meerschaum.api.dash.users import is_session_authenticated
|
13
|
+
from meerschaum.api.dash.sessions import is_session_authenticated
|
15
14
|
fastapi, fastapi_responses = attempt_import('fastapi', 'fastapi.responses')
|
16
15
|
import starlette
|
17
16
|
httpx = attempt_import('httpx')
|
@@ -25,9 +24,9 @@ PlainTextResponse = fastapi_responses.PlainTextResponse
|
|
25
24
|
|
26
25
|
@app.get(endpoints['webterm'], tags=["Webterm"])
|
27
26
|
async def get_webterm(
|
28
|
-
|
29
|
-
|
30
|
-
|
27
|
+
request: Request,
|
28
|
+
s: Optional[str] = None,
|
29
|
+
) -> HTMLResponse:
|
31
30
|
"""
|
32
31
|
Get the main HTML template for the Webterm.
|
33
32
|
"""
|
meerschaum/config/_default.py
CHANGED
@@ -52,6 +52,14 @@ default_meerschaum_config = {
|
|
52
52
|
'protocol': 'https',
|
53
53
|
},
|
54
54
|
},
|
55
|
+
'valkey': {
|
56
|
+
'main': {
|
57
|
+
'host': 'localhost',
|
58
|
+
'username': 'default',
|
59
|
+
'password': 'mrsm',
|
60
|
+
'port': 6379,
|
61
|
+
},
|
62
|
+
},
|
55
63
|
},
|
56
64
|
}
|
57
65
|
default_system_config = {
|
@@ -74,7 +82,7 @@ default_system_config = {
|
|
74
82
|
},
|
75
83
|
},
|
76
84
|
|
77
|
-
'api'
|
85
|
+
'api': {
|
78
86
|
},
|
79
87
|
},
|
80
88
|
### not to be confused with system_config['connectors']['api'], this is the configuration
|
@@ -88,6 +96,10 @@ default_system_config = {
|
|
88
96
|
'proxy_headers': True,
|
89
97
|
'forwarded_allow_ips': '*',
|
90
98
|
},
|
99
|
+
'cache': {
|
100
|
+
'connector': 'valkey:main',
|
101
|
+
'session_expires_minutes': 43200,
|
102
|
+
},
|
91
103
|
'permissions': {
|
92
104
|
'registration': {
|
93
105
|
'users': True,
|
@@ -97,7 +109,7 @@ default_system_config = {
|
|
97
109
|
'actions': {
|
98
110
|
'non_admin': True,
|
99
111
|
},
|
100
|
-
'chaining'
|
112
|
+
'chaining': {
|
101
113
|
'insecure_parent_instance': False,
|
102
114
|
'child_apis': False,
|
103
115
|
},
|
@@ -112,6 +124,7 @@ default_system_config = {
|
|
112
124
|
'inplace_sync': True,
|
113
125
|
'uv_pip': True,
|
114
126
|
'systemd_healthcheck': False,
|
127
|
+
'valkey_session_cache': True,
|
115
128
|
},
|
116
129
|
}
|
117
130
|
default_pipes_config = {
|
meerschaum/config/_version.py
CHANGED
@@ -31,21 +31,29 @@ db_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:db:hostname}'
|
|
31
31
|
api_port = "MRSM{meerschaum:connectors:api:main:port}"
|
32
32
|
api_host = "api"
|
33
33
|
|
34
|
+
valkey_hostname = "valkey"
|
35
|
+
valkey_host = 'MRSM{stack:' + str(STACK_COMPOSE_FILENAME) + ':services:valkey:hostname}'
|
36
|
+
valkey_port = "MRSM{meerschaum:connectors:valkey:main:port}"
|
37
|
+
valkey_username = 'MRSM{meerschaum:connectors:valkey:main:username}'
|
38
|
+
valkey_password = 'MRSM{meerschaum:connectors:valkey:main:password}'
|
39
|
+
|
34
40
|
env_dict = {
|
35
|
-
'COMPOSE_PROJECT_NAME'
|
36
|
-
'TIMESCALEDB_VERSION'
|
37
|
-
'POSTGRES_USER'
|
38
|
-
'POSTGRES_PASSWORD'
|
39
|
-
'POSTGRES_DB'
|
40
|
-
'
|
41
|
-
'
|
42
|
-
'
|
41
|
+
'COMPOSE_PROJECT_NAME': 'mrsm',
|
42
|
+
'TIMESCALEDB_VERSION': 'latest-pg16-oss',
|
43
|
+
'POSTGRES_USER': db_user,
|
44
|
+
'POSTGRES_PASSWORD': db_pass,
|
45
|
+
'POSTGRES_DB': db_base,
|
46
|
+
'VALKEY_USERNAME': valkey_username,
|
47
|
+
'VALKEY_PASSWORD': valkey_password,
|
48
|
+
'MEERSCHAUM_API_HOSTNAME': api_host,
|
49
|
+
'ALLOW_IP_RANGE': '0.0.0.0/0',
|
50
|
+
'MEERSCHAUM_API_CONFIG_RESOURCES': '/meerschaum',
|
43
51
|
}
|
44
52
|
### apply patch to host config to change hostname to the Docker service name
|
45
53
|
env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
|
46
54
|
{
|
47
|
-
'meerschaum'
|
48
|
-
'system'
|
55
|
+
'meerschaum': 'MRSM{!meerschaum}',
|
56
|
+
'system': 'MRSM{!system}',
|
49
57
|
},
|
50
58
|
indent = 4,
|
51
59
|
).replace(
|
@@ -58,6 +66,7 @@ volumes = {
|
|
58
66
|
'api_root': '/meerschaum',
|
59
67
|
'meerschaum_db_data': '/var/lib/postgresql/data',
|
60
68
|
'grafana_storage': '/var/lib/grafana',
|
69
|
+
'valkey_data': '/bitnami/valkey/data',
|
61
70
|
}
|
62
71
|
networks = {
|
63
72
|
'frontend': None,
|
@@ -73,13 +82,19 @@ env_dict['MEERSCHAUM_API_PATCH'] = json.dumps(
|
|
73
82
|
'port': 5432,
|
74
83
|
},
|
75
84
|
'local': {
|
76
|
-
'database': volumes['api_root'] + '/sqlite/mrsm_local.db'
|
85
|
+
'database': volumes['api_root'] + '/sqlite/mrsm_local.db',
|
86
|
+
},
|
87
|
+
},
|
88
|
+
'valkey': {
|
89
|
+
'main': {
|
90
|
+
'host': valkey_host,
|
91
|
+
'port': 6379,
|
77
92
|
},
|
78
93
|
},
|
79
94
|
},
|
80
95
|
},
|
81
96
|
},
|
82
|
-
indent
|
97
|
+
indent=4,
|
83
98
|
)
|
84
99
|
|
85
100
|
compose_header = """
|
@@ -113,19 +128,19 @@ default_docker_compose_config = {
|
|
113
128
|
],
|
114
129
|
'interval': '5s',
|
115
130
|
'timeout': '3s',
|
116
|
-
'retries':
|
131
|
+
'retries': 5
|
117
132
|
},
|
118
133
|
'restart': 'always',
|
119
|
-
'image'
|
120
|
-
'ports'
|
134
|
+
'image': 'timescale/timescaledb:' + env_dict['TIMESCALEDB_VERSION'],
|
135
|
+
'ports': [
|
121
136
|
f'{db_port}:5432',
|
122
137
|
],
|
123
|
-
'hostname'
|
124
|
-
'volumes'
|
138
|
+
'hostname': db_hostname,
|
139
|
+
'volumes': [
|
125
140
|
'meerschaum_db_data:' + volumes['meerschaum_db_data'],
|
126
141
|
],
|
127
142
|
'shm_size': '1024m',
|
128
|
-
'networks'
|
143
|
+
'networks': [
|
129
144
|
'backend',
|
130
145
|
],
|
131
146
|
},
|
@@ -156,11 +171,41 @@ default_docker_compose_config = {
|
|
156
171
|
'db': {
|
157
172
|
'condition': 'service_healthy',
|
158
173
|
},
|
174
|
+
'valkey': {
|
175
|
+
'condition': 'service_healthy',
|
176
|
+
},
|
159
177
|
},
|
160
|
-
'volumes'
|
178
|
+
'volumes': [
|
161
179
|
'api_root:' + volumes['api_root'],
|
162
180
|
],
|
163
181
|
},
|
182
|
+
'valkey': {
|
183
|
+
'image': 'bitnami/valkey:latest',
|
184
|
+
'restart': 'always',
|
185
|
+
'environment': {
|
186
|
+
'VALKEY_PASSWORD': '<DOLLAR>VALKEY_PASSWORD',
|
187
|
+
'VALKEY_RDB_POLICY_DISABLED': 'no',
|
188
|
+
'VALKEY_RDB_POLICY': '900#1 600#5 300#10 120#50 60#1000 30#10000',
|
189
|
+
},
|
190
|
+
'hostname': valkey_hostname,
|
191
|
+
'ports': [
|
192
|
+
f'{valkey_port}:6379',
|
193
|
+
],
|
194
|
+
'volumes': [
|
195
|
+
'valkey_data:' + volumes['valkey_data'],
|
196
|
+
],
|
197
|
+
'healthcheck': {
|
198
|
+
'test': [
|
199
|
+
'CMD', 'valkey-cli', 'ping',
|
200
|
+
],
|
201
|
+
'interval': '5s',
|
202
|
+
'timeout': '3s',
|
203
|
+
'retries': 5,
|
204
|
+
},
|
205
|
+
'networks': [
|
206
|
+
'backend',
|
207
|
+
],
|
208
|
+
},
|
164
209
|
'grafana': {
|
165
210
|
'image': 'grafana/grafana:latest',
|
166
211
|
'ports': [
|