meerschaum 2.7.9__py3-none-any.whl → 2.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parser.py +17 -5
- meerschaum/_internal/term/TermPageHandler.py +1 -1
- meerschaum/_internal/term/__init__.py +1 -1
- meerschaum/actions/api.py +36 -10
- meerschaum/actions/copy.py +3 -1
- meerschaum/actions/index.py +1 -1
- meerschaum/actions/show.py +7 -7
- meerschaum/actions/sync.py +5 -1
- meerschaum/actions/verify.py +14 -1
- meerschaum/api/__init__.py +77 -41
- meerschaum/api/_exceptions.py +18 -0
- meerschaum/api/dash/__init__.py +4 -2
- meerschaum/api/dash/callbacks/dashboard.py +30 -1
- meerschaum/api/dash/components.py +2 -2
- meerschaum/api/dash/webterm.py +23 -4
- meerschaum/api/models/_pipes.py +8 -8
- meerschaum/api/resources/static/css/dash.css +2 -2
- meerschaum/api/resources/templates/termpage.html +5 -1
- meerschaum/api/routes/__init__.py +15 -12
- meerschaum/api/routes/_connectors.py +30 -28
- meerschaum/api/routes/_index.py +16 -7
- meerschaum/api/routes/_misc.py +30 -22
- meerschaum/api/routes/_pipes.py +244 -148
- meerschaum/api/routes/_plugins.py +58 -47
- meerschaum/api/routes/_users.py +39 -31
- meerschaum/api/routes/_version.py +8 -10
- meerschaum/api/routes/_webterm.py +2 -2
- meerschaum/config/_default.py +10 -0
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +5 -2
- meerschaum/connectors/api/_APIConnector.py +4 -3
- meerschaum/connectors/api/_login.py +21 -17
- meerschaum/connectors/api/_pipes.py +1 -0
- meerschaum/connectors/api/_request.py +9 -10
- meerschaum/connectors/sql/_cli.py +11 -3
- meerschaum/connectors/sql/_instance.py +1 -1
- meerschaum/connectors/sql/_pipes.py +77 -57
- meerschaum/connectors/sql/_sql.py +26 -9
- meerschaum/core/Pipe/__init__.py +2 -0
- meerschaum/core/Pipe/_attributes.py +13 -2
- meerschaum/core/Pipe/_data.py +85 -0
- meerschaum/core/Pipe/_deduplicate.py +6 -8
- meerschaum/core/Pipe/_sync.py +63 -30
- meerschaum/core/Pipe/_verify.py +242 -77
- meerschaum/core/User/__init__.py +2 -6
- meerschaum/jobs/_Job.py +1 -1
- meerschaum/jobs/__init__.py +15 -0
- meerschaum/utils/dataframe.py +2 -0
- meerschaum/utils/dtypes/sql.py +26 -0
- meerschaum/utils/formatting/_pipes.py +1 -1
- meerschaum/utils/misc.py +11 -7
- meerschaum/utils/packages/_packages.py +1 -1
- meerschaum/utils/sql.py +6 -2
- {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/METADATA +4 -4
- {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/RECORD +61 -60
- {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/WHEEL +0 -0
- {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.7.9.dist-info → meerschaum-2.8.0.dist-info}/zip-safe +0 -0
@@ -302,14 +302,17 @@ groups['sync'].add_argument(
|
|
302
302
|
help="Run the action asynchronously, if possible. Alias for --unblock",
|
303
303
|
)
|
304
304
|
groups['sync'].add_argument(
|
305
|
-
'--begin', type=parse_datetime, help="
|
305
|
+
'--begin', type=parse_datetime, help="The begin datetime when syncing, fetching data."
|
306
306
|
)
|
307
307
|
groups['sync'].add_argument(
|
308
|
-
'--end', type=parse_datetime, help="
|
308
|
+
'--end', type=parse_datetime, help="The end datetime when syncing, fetching data."
|
309
309
|
)
|
310
310
|
groups['sync'].add_argument(
|
311
|
-
'--chunksize', type=int, help=
|
312
|
-
|
311
|
+
'--chunksize', type=int, help="How many rows per chunk. Defaults to 100,000."
|
312
|
+
)
|
313
|
+
groups['sync'].add_argument(
|
314
|
+
'--batchsize', type=int, help=(
|
315
|
+
"How many chunks to process in parallel. Defaults to number of CPUs."
|
313
316
|
),
|
314
317
|
)
|
315
318
|
groups['sync'].add_argument(
|
@@ -355,9 +358,18 @@ groups['sync'].add_argument(
|
|
355
358
|
"This improves performance when all rows are expected to already be of the correct type."
|
356
359
|
),
|
357
360
|
)
|
361
|
+
groups['sync'].add_argument(
|
362
|
+
'--skip-chunks-with-greater-rowcounts', action='store_true',
|
363
|
+
help="When verifying, skip chunks with rowcounts greater than the remote's."
|
364
|
+
)
|
365
|
+
groups['sync'].add_argument(
|
366
|
+
'--check-rowcounts-only', action='store_true', help=(
|
367
|
+
"Only compare row-counts when verifying pipes."
|
368
|
+
),
|
369
|
+
)
|
358
370
|
groups['sync'].add_argument(
|
359
371
|
'--cache', action='store_true',
|
360
|
-
help
|
372
|
+
help=(
|
361
373
|
"When syncing or viewing a pipe's data, sync to a local database for later analysis."
|
362
374
|
)
|
363
375
|
)
|
meerschaum/actions/api.py
CHANGED
@@ -140,16 +140,28 @@ def _api_start(
|
|
140
140
|
If provided, serve over HTTPS with this certfile.
|
141
141
|
Requires `--keyfile`.
|
142
142
|
"""
|
143
|
+
import json
|
144
|
+
import sys
|
145
|
+
import shutil
|
146
|
+
import pathlib
|
147
|
+
from copy import deepcopy
|
148
|
+
|
143
149
|
from meerschaum.utils.packages import (
|
144
|
-
attempt_import,
|
150
|
+
attempt_import,
|
151
|
+
venv_contains_package,
|
152
|
+
pip_install,
|
153
|
+
run_python_package,
|
145
154
|
)
|
146
155
|
from meerschaum.utils.misc import is_int, filter_keywords
|
156
|
+
from meerschaum.utils.dtypes import json_serialize_value
|
147
157
|
from meerschaum.utils.formatting import pprint, ANSI, _init
|
148
158
|
from meerschaum.utils.debug import dprint
|
149
159
|
from meerschaum.utils.warnings import error, warn
|
150
160
|
from meerschaum.config import get_config, _config
|
151
161
|
from meerschaum.config._paths import (
|
152
|
-
API_UVICORN_RESOURCES_PATH,
|
162
|
+
API_UVICORN_RESOURCES_PATH,
|
163
|
+
API_UVICORN_CONFIG_PATH,
|
164
|
+
CACHE_RESOURCES_PATH,
|
153
165
|
PACKAGE_ROOT_PATH,
|
154
166
|
)
|
155
167
|
from meerschaum.config._patch import apply_patch_to_config
|
@@ -157,8 +169,6 @@ def _api_start(
|
|
157
169
|
from meerschaum.config.static import STATIC_CONFIG, SERVER_ID
|
158
170
|
from meerschaum.connectors.parse import parse_instance_keys
|
159
171
|
from meerschaum.utils.pool import get_pool
|
160
|
-
import shutil
|
161
|
-
from copy import deepcopy
|
162
172
|
|
163
173
|
if action is None:
|
164
174
|
action = []
|
@@ -256,7 +266,6 @@ def _api_start(
|
|
256
266
|
custom_keys = ['mrsm_instance', 'no_dash', 'no_auth', 'private', 'debug', 'production']
|
257
267
|
|
258
268
|
### write config to a temporary file to communicate with uvicorn threads
|
259
|
-
import json, sys
|
260
269
|
try:
|
261
270
|
if uvicorn_config_path.exists():
|
262
271
|
os.remove(uvicorn_config_path)
|
@@ -275,12 +284,25 @@ def _api_start(
|
|
275
284
|
MRSM_RUNTIME = STATIC_CONFIG['environment']['runtime']
|
276
285
|
MRSM_PATCH = STATIC_CONFIG['environment']['patch']
|
277
286
|
MRSM_ROOT_DIR = STATIC_CONFIG['environment']['root']
|
278
|
-
env_dict = {
|
287
|
+
env_dict = {}
|
288
|
+
env_dict.update({
|
279
289
|
MRSM_SERVER_ID: SERVER_ID,
|
280
290
|
MRSM_RUNTIME: 'api',
|
281
291
|
MRSM_CONFIG: json.loads(os.environ.get(MRSM_CONFIG, '{}')),
|
282
292
|
'FORWARDED_ALLOW_IPS': forwarded_allow_ips,
|
283
|
-
|
293
|
+
'TERM': os.environ.get('TERM', 'screen-256color'),
|
294
|
+
'SHELL': os.environ.get('SHELL', '/bin/bash'),
|
295
|
+
'LANG': os.environ.get('LANG', 'C.UTF-8'),
|
296
|
+
'HOME': os.environ.get('HOME', pathlib.Path.home().as_posix()),
|
297
|
+
'PATH': os.environ.get(
|
298
|
+
'PATH',
|
299
|
+
(
|
300
|
+
'/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin:'
|
301
|
+
f'{pathlib.Path.home().as_posix().rstrip("/")}/.local/bin'
|
302
|
+
)
|
303
|
+
),
|
304
|
+
'HOSTNAME': os.environ.get('HOSTNAME', 'api'),
|
305
|
+
})
|
284
306
|
for env_var in get_env_vars():
|
285
307
|
if env_var in env_dict:
|
286
308
|
continue
|
@@ -294,10 +316,10 @@ def _api_start(
|
|
294
316
|
env_text = ''
|
295
317
|
for key, val in env_dict.items():
|
296
318
|
value = str(
|
297
|
-
json.dumps(val)
|
319
|
+
json.dumps(val, default=json_serialize_value)
|
298
320
|
if isinstance(val, (dict))
|
299
321
|
else val
|
300
|
-
).replace('\\', '\\\\')
|
322
|
+
).replace('\\', '\\\\').replace("'", "'\\''")
|
301
323
|
env_text += f"{key}='{value}'\n"
|
302
324
|
with open(uvicorn_env_path, 'w+', encoding='utf-8') as f:
|
303
325
|
if debug:
|
@@ -329,7 +351,11 @@ def _api_start(
|
|
329
351
|
for key, val in env_dict.items():
|
330
352
|
gunicorn_args += [
|
331
353
|
'--env', key + "="
|
332
|
-
+ (
|
354
|
+
+ (
|
355
|
+
json.dumps(val, default=json_serialize_value)
|
356
|
+
if isinstance(val, (dict, list))
|
357
|
+
else val
|
358
|
+
)
|
333
359
|
]
|
334
360
|
if workers is not None:
|
335
361
|
gunicorn_args += ['--workers', str(workers)]
|
meerschaum/actions/copy.py
CHANGED
@@ -70,6 +70,7 @@ def _copy_pipes(
|
|
70
70
|
Copy pipes' attributes and make new pipes.
|
71
71
|
"""
|
72
72
|
from meerschaum import get_pipes, Pipe
|
73
|
+
from meerschaum.connectors import instance_types
|
73
74
|
from meerschaum.utils.prompt import prompt, yes_no, get_connectors_completer
|
74
75
|
from meerschaum.utils.warnings import warn
|
75
76
|
from meerschaum.utils.formatting import print_tuple
|
@@ -92,7 +93,8 @@ def _copy_pipes(
|
|
92
93
|
|
93
94
|
instance_keys = prompt(
|
94
95
|
f"Meerschaum instance for copy of {pipe}:",
|
95
|
-
default=pipe.instance_keys
|
96
|
+
default=pipe.instance_keys,
|
97
|
+
completer=get_connectors_completer(*instance_types),
|
96
98
|
)
|
97
99
|
new_pipe = Pipe(
|
98
100
|
ck, mk, lk,
|
meerschaum/actions/index.py
CHANGED
@@ -48,7 +48,7 @@ def _index_pipes(
|
|
48
48
|
|
49
49
|
for pipe in pipes:
|
50
50
|
info(f"Creating indices for {pipe}...")
|
51
|
-
index_success, index_msg = pipe.create_indices(debug=debug)
|
51
|
+
index_success, index_msg = pipe.create_indices(columns=(action or None), debug=debug)
|
52
52
|
success_dict[pipe] = index_msg
|
53
53
|
if index_success:
|
54
54
|
successes += 1
|
meerschaum/actions/show.py
CHANGED
@@ -442,7 +442,7 @@ def _show_rowcounts(
|
|
442
442
|
|
443
443
|
msgs = []
|
444
444
|
for p, rc in rc_dict.items():
|
445
|
-
msgs.append(f'{p}\n{rc}\n')
|
445
|
+
msgs.append(f'{p}\n{rc:,}\n')
|
446
446
|
|
447
447
|
header = "Remote row-counts:" if remote else "Pipe row-counts:"
|
448
448
|
|
@@ -581,10 +581,10 @@ def _show_jobs(
|
|
581
581
|
from meerschaum.utils.warnings import info
|
582
582
|
info('No running or stopped jobs.')
|
583
583
|
print(
|
584
|
-
|
585
|
-
" or run the command `start job` before action commands.\n\n"
|
586
|
-
" Examples:\n"
|
587
|
-
" - start api -d\n"
|
584
|
+
" You can start a background job with `-d` or `--daemon`,\n"
|
585
|
+
" or run the command `start job` before action commands.\n\n"
|
586
|
+
" Examples:\n"
|
587
|
+
" - start api -d\n"
|
588
588
|
" - start job sync pipes --loop"
|
589
589
|
)
|
590
590
|
return True, "No jobs to show."
|
@@ -610,7 +610,7 @@ def _show_logs(
|
|
610
610
|
`show logs --nopretty`
|
611
611
|
`show logs myjob myotherjob`
|
612
612
|
"""
|
613
|
-
import
|
613
|
+
import asyncio
|
614
614
|
from functools import partial
|
615
615
|
from datetime import datetime, timezone
|
616
616
|
from meerschaum.utils.packages import attempt_import, import_rich
|
@@ -676,7 +676,7 @@ def _show_logs(
|
|
676
676
|
try:
|
677
677
|
line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
|
678
678
|
previous_line_timestamp = line_timestamp
|
679
|
-
except Exception
|
679
|
+
except Exception:
|
680
680
|
line_timestamp = None
|
681
681
|
if line_timestamp:
|
682
682
|
line = line[(len(now_str) + 3):]
|
meerschaum/actions/sync.py
CHANGED
@@ -49,6 +49,7 @@ def _pipes_lap(
|
|
49
49
|
deduplicate: bool = False,
|
50
50
|
bounded: Optional[bool] = None,
|
51
51
|
chunk_interval: Union[timedelta, int, None] = None,
|
52
|
+
check_rowcounts_only: bool = False,
|
52
53
|
mrsm_instance: Optional[str] = None,
|
53
54
|
timeout_seconds: Optional[int] = None,
|
54
55
|
nopretty: bool = False,
|
@@ -93,6 +94,7 @@ def _pipes_lap(
|
|
93
94
|
'deduplicate': deduplicate,
|
94
95
|
'bounded': bounded,
|
95
96
|
'chunk_interval': chunk_interval,
|
97
|
+
'check_rowcounts_only': check_rowcounts_only,
|
96
98
|
})
|
97
99
|
locks = {'remaining_count': Lock(), 'results_dict': Lock(), 'pipes_threads': Lock(),}
|
98
100
|
pipes = get_pipes(
|
@@ -254,6 +256,7 @@ def _sync_pipes(
|
|
254
256
|
deduplicate: bool = False,
|
255
257
|
bounded: Optional[bool] = None,
|
256
258
|
chunk_interval: Union[timedelta, int, None] = None,
|
259
|
+
check_rowcounts_only: bool = False,
|
257
260
|
shell: bool = False,
|
258
261
|
nopretty: bool = False,
|
259
262
|
debug: bool = False,
|
@@ -308,6 +311,7 @@ def _sync_pipes(
|
|
308
311
|
deduplicate=deduplicate,
|
309
312
|
bounded=bounded,
|
310
313
|
chunk_interval=chunk_interval,
|
314
|
+
check_rowcounts_only=check_rowcounts_only,
|
311
315
|
unblock=unblock,
|
312
316
|
debug=debug,
|
313
317
|
nopretty=nopretty,
|
@@ -447,7 +451,7 @@ def _wrap_pipe(
|
|
447
451
|
sync_hook_result = sync_hook(pipe, **filter_keywords(sync_hook, **sync_kwargs))
|
448
452
|
if is_success_tuple(sync_hook_result):
|
449
453
|
return sync_hook_result
|
450
|
-
except Exception
|
454
|
+
except Exception:
|
451
455
|
msg = (
|
452
456
|
f"Failed to execute sync hook '{sync_hook.__name__}' "
|
453
457
|
+ f"from plugin '{plugin}':\n{traceback.format_exc()}"
|
meerschaum/actions/verify.py
CHANGED
@@ -7,7 +7,9 @@ Verify the states of pipes, pacakages, and more.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
|
10
|
+
|
11
|
+
from meerschaum.utils.typing import Any, SuccessTuple, Optional, List
|
12
|
+
|
11
13
|
|
12
14
|
def verify(
|
13
15
|
action: Optional[List[str]] = None,
|
@@ -22,6 +24,7 @@ def verify(
|
|
22
24
|
'packages': _verify_packages,
|
23
25
|
'venvs': _verify_venvs,
|
24
26
|
'plugins': _verify_plugins,
|
27
|
+
'rowcounts': _verify_rowcounts,
|
25
28
|
}
|
26
29
|
return choose_subaction(action, options, **kwargs)
|
27
30
|
|
@@ -35,6 +38,16 @@ def _verify_pipes(**kwargs) -> SuccessTuple:
|
|
35
38
|
return _sync_pipes(**kwargs)
|
36
39
|
|
37
40
|
|
41
|
+
def _verify_rowcounts(**kwargs) -> SuccessTuple:
|
42
|
+
"""
|
43
|
+
Verify the contents of pipes, syncing across their entire datetime intervals.
|
44
|
+
"""
|
45
|
+
from meerschaum.actions.sync import _sync_pipes
|
46
|
+
kwargs['verify'] = True
|
47
|
+
kwargs['check_rowcounts_only'] = True
|
48
|
+
return _sync_pipes(**kwargs)
|
49
|
+
|
50
|
+
|
38
51
|
def _verify_packages(
|
39
52
|
debug: bool = False,
|
40
53
|
venv: Optional[str] = 'mrsm',
|
meerschaum/api/__init__.py
CHANGED
@@ -6,15 +6,12 @@
|
|
6
6
|
Meerschaum API backend. Start an API instance with `start api`.
|
7
7
|
"""
|
8
8
|
from __future__ import annotations
|
9
|
-
import os
|
10
|
-
from meerschaum.utils.typing import Dict, Any, Optional
|
11
9
|
|
12
|
-
|
13
|
-
|
14
|
-
__doc__ = """
|
15
|
-
The Meerschaum Web API lets you access and control your data over the Internet.
|
16
|
-
"""
|
10
|
+
import os
|
11
|
+
from collections import defaultdict
|
17
12
|
|
13
|
+
import meerschaum as mrsm
|
14
|
+
from meerschaum.utils.typing import Dict, Any, Optional, PipesDict
|
18
15
|
from meerschaum.config import get_config
|
19
16
|
from meerschaum.config.static import STATIC_CONFIG, SERVER_ID
|
20
17
|
from meerschaum.utils.packages import attempt_import
|
@@ -23,8 +20,14 @@ from meerschaum.config._paths import API_UVICORN_CONFIG_PATH, API_UVICORN_RESOUR
|
|
23
20
|
from meerschaum.plugins import _api_plugins
|
24
21
|
from meerschaum.utils.warnings import warn, dprint
|
25
22
|
from meerschaum.utils.threading import RLock
|
23
|
+
from meerschaum.utils.misc import is_pipe_registered
|
24
|
+
|
25
|
+
from meerschaum import __version__ as version
|
26
|
+
__version__ = version
|
27
|
+
__doc__ = """The Meerschaum Web API lets you manage your pipes over the Internet."""
|
28
|
+
|
26
29
|
|
27
|
-
_locks =
|
30
|
+
_locks = defaultdict(lambda: RLock())
|
28
31
|
|
29
32
|
### Skip verifying packages in the docker image.
|
30
33
|
CHECK_UPDATE = os.environ.get(STATIC_CONFIG['environment']['runtime'], None) != 'docker'
|
@@ -58,6 +61,7 @@ uv = attempt_import('uv', lazy=False, check_update=CHECK_UPDATE)
|
|
58
61
|
venv=None,
|
59
62
|
)
|
60
63
|
from meerschaum.api._chain import check_allow_chaining, DISALLOW_CHAINING_MESSAGE
|
64
|
+
from meerschaum.api._exceptions import APIPermissionError
|
61
65
|
uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
|
62
66
|
|
63
67
|
uvicorn_config = None
|
@@ -78,13 +82,11 @@ def get_uvicorn_config() -> Dict[str, Any]:
|
|
78
82
|
with open(uvicorn_config_path, 'r', encoding='utf-8') as f:
|
79
83
|
uvicorn_config = json.load(f)
|
80
84
|
_uvicorn_config = uvicorn_config
|
81
|
-
except Exception
|
85
|
+
except Exception:
|
82
86
|
_uvicorn_config = sys_config.get('uvicorn', None)
|
83
87
|
|
84
88
|
if _uvicorn_config is None:
|
85
89
|
_uvicorn_config = {}
|
86
|
-
|
87
|
-
### Default: main SQL connector
|
88
90
|
if 'mrsm_instance' not in _uvicorn_config:
|
89
91
|
_uvicorn_config['mrsm_instance'] = get_config('meerschaum', 'api_instance')
|
90
92
|
return _uvicorn_config
|
@@ -95,22 +97,47 @@ no_auth = get_uvicorn_config().get('no_auth', False)
|
|
95
97
|
private = get_uvicorn_config().get('private', False)
|
96
98
|
production = get_uvicorn_config().get('production', False)
|
97
99
|
_include_dash = (not no_dash)
|
100
|
+
docs_enabled = not production or sys_config.get('endpoints', {}).get('docs_in_production', True)
|
98
101
|
|
99
102
|
connector = None
|
103
|
+
default_instance_keys = None
|
104
|
+
_instance_connectors = defaultdict(lambda: None)
|
100
105
|
def get_api_connector(instance_keys: Optional[str] = None):
|
101
|
-
"""Create the instance
|
102
|
-
global
|
103
|
-
|
104
|
-
if
|
105
|
-
|
106
|
-
|
106
|
+
"""Create the instance connectors."""
|
107
|
+
global default_instance_keys
|
108
|
+
if instance_keys is None:
|
109
|
+
if default_instance_keys is None:
|
110
|
+
default_instance_keys = get_uvicorn_config().get('mrsm_instance', None)
|
111
|
+
instance_keys = default_instance_keys
|
112
|
+
|
113
|
+
allow_multiple_instances = permissions_config.get(
|
114
|
+
'instances', {}
|
115
|
+
).get('allow_multiple_instances', False)
|
116
|
+
if not allow_multiple_instances and instance_keys != default_instance_keys:
|
117
|
+
raise APIPermissionError(
|
118
|
+
"This API instance does not allow for accessing additional instances."
|
119
|
+
)
|
120
|
+
|
121
|
+
allowed_instance_keys = permissions_config.get(
|
122
|
+
'instance', {}
|
123
|
+
).get(
|
124
|
+
'allowed_instance_keys',
|
125
|
+
['*']
|
126
|
+
)
|
127
|
+
if allowed_instance_keys != ['*'] and instance_keys not in allowed_instance_keys:
|
128
|
+
raise APIPermissionError(
|
129
|
+
f"Instance keys '{instance_keys}' not in list of allowed instances."
|
130
|
+
)
|
107
131
|
|
132
|
+
with _locks[f'instance-{instance_keys}']:
|
133
|
+
connector = _instance_connectors[instance_keys]
|
134
|
+
if connector is None:
|
108
135
|
from meerschaum.connectors.parse import parse_instance_keys
|
109
136
|
connector = parse_instance_keys(instance_keys, debug=debug)
|
110
|
-
|
111
|
-
dprint(f"API instance connector: {connector}")
|
137
|
+
_instance_connectors[instance_keys] = connector
|
112
138
|
return connector
|
113
139
|
|
140
|
+
|
114
141
|
cache_connector = None
|
115
142
|
def get_cache_connector(connector_keys: Optional[str] = None):
|
116
143
|
"""Return the `valkey` connector if running in production."""
|
@@ -146,28 +173,35 @@ def get_cache_connector(connector_keys: Optional[str] = None):
|
|
146
173
|
return cache_connector
|
147
174
|
|
148
175
|
|
149
|
-
|
150
|
-
def pipes(refresh=False):
|
176
|
+
_instance_pipes = defaultdict(lambda: None)
|
177
|
+
def pipes(instance_keys: Optional[str] = None, refresh: bool = False) -> PipesDict:
|
151
178
|
"""
|
152
|
-
Manage the global pipes
|
179
|
+
Manage the global pipes dictionaries.
|
153
180
|
"""
|
154
|
-
|
155
|
-
with _locks['pipes']:
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
181
|
+
instance_keys = str(get_api_connector(instance_keys))
|
182
|
+
with _locks['pipes-' + instance_keys]:
|
183
|
+
pipes = _instance_pipes[instance_keys]
|
184
|
+
if pipes is None or refresh:
|
185
|
+
pipes = _get_pipes(mrsm_instance=instance_keys)
|
186
|
+
_instance_pipes[instance_keys] = pipes
|
187
|
+
return pipes
|
188
|
+
|
189
|
+
|
190
|
+
def get_pipe(
|
191
|
+
connector_keys: str,
|
192
|
+
metric_key: str,
|
193
|
+
location_key: Optional[str],
|
194
|
+
instance_keys: Optional[str] = None,
|
195
|
+
refresh: bool = False
|
196
|
+
) -> mrsm.Pipe:
|
162
197
|
"""Index the pipes dictionary or create a new Pipe object."""
|
163
|
-
from meerschaum.utils.misc import is_pipe_registered
|
164
|
-
from meerschaum import Pipe
|
165
198
|
if location_key in ('[None]', 'None', 'null'):
|
166
199
|
location_key = None
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
200
|
+
instance_keys = str(get_api_connector(instance_keys))
|
201
|
+
pipe = mrsm.Pipe(connector_keys, metric_key, location_key, mrsm_instance=instance_keys)
|
202
|
+
if is_pipe_registered(pipe, pipes(instance_keys)):
|
203
|
+
return pipes(instance_keys, refresh=refresh)[connector_keys][metric_key][location_key]
|
204
|
+
return pipe
|
171
205
|
|
172
206
|
|
173
207
|
app = fastapi.FastAPI(
|
@@ -182,6 +216,9 @@ app = fastapi.FastAPI(
|
|
182
216
|
'name': 'Apache 2.0',
|
183
217
|
'url': 'https://www.apache.org/licenses/LICENSE-2.0.html',
|
184
218
|
},
|
219
|
+
docs_url=(None if not docs_enabled else endpoints['docs']),
|
220
|
+
redoc_url=(None if not docs_enabled else endpoints['redoc']),
|
221
|
+
openapi_url=endpoints['openapi'],
|
185
222
|
open_api_tags=[
|
186
223
|
{
|
187
224
|
'name': 'Pipes',
|
@@ -193,7 +230,7 @@ app = fastapi.FastAPI(
|
|
193
230
|
},
|
194
231
|
{
|
195
232
|
'name': 'Connectors',
|
196
|
-
'description': 'Get information about the registered connectors.'
|
233
|
+
'description': 'Get information about the registered connectors.',
|
197
234
|
},
|
198
235
|
{
|
199
236
|
'name': 'Users',
|
@@ -209,8 +246,8 @@ app = fastapi.FastAPI(
|
|
209
246
|
},
|
210
247
|
{
|
211
248
|
'name': 'Version',
|
212
|
-
'description': 'Version information.'
|
213
|
-
}
|
249
|
+
'description': 'Version information.',
|
250
|
+
},
|
214
251
|
],
|
215
252
|
)
|
216
253
|
|
@@ -228,7 +265,7 @@ app = fastapi.FastAPI(
|
|
228
265
|
HTMLResponse = fastapi_responses.HTMLResponse
|
229
266
|
Request = fastapi.Request
|
230
267
|
|
231
|
-
from meerschaum.config.
|
268
|
+
from meerschaum.config.paths import API_RESOURCES_PATH, API_STATIC_PATH, API_TEMPLATES_PATH
|
232
269
|
app.mount('/static', fastapi_staticfiles.StaticFiles(directory=str(API_STATIC_PATH)), name='static')
|
233
270
|
|
234
271
|
_custom_kwargs = {'mrsm_instance'}
|
@@ -252,7 +289,6 @@ if _include_dash:
|
|
252
289
|
import meerschaum.api.dash
|
253
290
|
|
254
291
|
### Execute the API plugins functions.
|
255
|
-
import meerschaum as mrsm
|
256
292
|
for module_name, functions_list in _api_plugins.items():
|
257
293
|
plugin_name = module_name.split('.')[-1] if module_name.startswith('plugins.') else None
|
258
294
|
plugin = mrsm.Plugin(plugin_name) if plugin_name else None
|
@@ -0,0 +1,18 @@
|
|
1
|
+
#! /usr/bin/env python3
|
2
|
+
# vim:fenc=utf-8
|
3
|
+
|
4
|
+
"""
|
5
|
+
Define custom API exceptions.
|
6
|
+
"""
|
7
|
+
|
8
|
+
import meerschaum as mrsm
|
9
|
+
|
10
|
+
_ = mrsm.attempt_import('fastapi', lazy=False)
|
11
|
+
from fastapi import HTTPException
|
12
|
+
|
13
|
+
|
14
|
+
class APIPermissionError(HTTPException):
|
15
|
+
"""Raise if the configured Meerschaum API permissions disallow an action."""
|
16
|
+
|
17
|
+
def __init__(self, detail: str = "Permission denied.", status_code: int = 403):
|
18
|
+
super().__init__(status_code=status_code, detail=detail)
|
meerschaum/api/dash/__init__.py
CHANGED
@@ -7,10 +7,12 @@ Build the Dash app to be hooked into FastAPI.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
import uuid
|
11
10
|
|
12
11
|
from meerschaum.utils.packages import (
|
13
|
-
attempt_import,
|
12
|
+
attempt_import,
|
13
|
+
import_dcc,
|
14
|
+
import_html,
|
15
|
+
_monkey_patch_get_distribution,
|
14
16
|
)
|
15
17
|
flask_compress = attempt_import('flask_compress', lazy=False)
|
16
18
|
_monkey_patch_get_distribution('flask-compress', flask_compress.__version__)
|
@@ -665,7 +665,8 @@ dash_app.clientside_callback(
|
|
665
665
|
|
666
666
|
iframe.contentWindow.postMessage(
|
667
667
|
{
|
668
|
-
action: "__TMUX_NEW_WINDOW"
|
668
|
+
action: "__TMUX_NEW_WINDOW",
|
669
|
+
instance: window.instance
|
669
670
|
},
|
670
671
|
url
|
671
672
|
);
|
@@ -692,6 +693,34 @@ dash_app.clientside_callback(
|
|
692
693
|
State('mrsm-location', 'href'),
|
693
694
|
)
|
694
695
|
|
696
|
+
dash_app.clientside_callback(
|
697
|
+
"""
|
698
|
+
function(n_clicks){
|
699
|
+
console.log('fullscreen');
|
700
|
+
if (!n_clicks) { return dash_clientside.no_update; }
|
701
|
+
iframe = document.getElementById('webterm-iframe');
|
702
|
+
if (!iframe){ return dash_clientside.no_update; }
|
703
|
+
const leftCol = document.getElementById('content-col-left');
|
704
|
+
const rightCol = document.getElementById('content-col-right');
|
705
|
+
const button = document.getElementById('webterm-fullscreen-button');
|
706
|
+
|
707
|
+
if (leftCol.style.display === 'none') {
|
708
|
+
leftCol.style.display = '';
|
709
|
+
rightCol.className = 'col-6';
|
710
|
+
button.innerHTML = "Full View";
|
711
|
+
} else {
|
712
|
+
leftCol.style.display = 'none';
|
713
|
+
rightCol.className = 'col-12';
|
714
|
+
button.innerHTML = "Side-by-side View";
|
715
|
+
}
|
716
|
+
|
717
|
+
return dash_clientside.no_update;
|
718
|
+
}
|
719
|
+
""",
|
720
|
+
Output('webterm-fullscreen-button', 'n_clicks'),
|
721
|
+
Input('webterm-fullscreen-button', 'n_clicks'),
|
722
|
+
)
|
723
|
+
|
695
724
|
@dash_app.callback(
|
696
725
|
Output(component_id='connector-keys-input', component_property='value'),
|
697
726
|
Input(component_id='clear-connector-keys-input-button', component_property='n_clicks'),
|
@@ -13,7 +13,7 @@ from meerschaum.utils.typing import SuccessTuple, List
|
|
13
13
|
from meerschaum.config.static import STATIC_CONFIG
|
14
14
|
from meerschaum.utils.misc import remove_ansi
|
15
15
|
from meerschaum._internal.shell.Shell import get_shell_intro
|
16
|
-
from meerschaum.api import endpoints, CHECK_UPDATE
|
16
|
+
from meerschaum.api import endpoints, CHECK_UPDATE, docs_enabled
|
17
17
|
from meerschaum.connectors import instance_types, _load_builtin_custom_connectors
|
18
18
|
from meerschaum.utils.misc import get_connector_labels
|
19
19
|
from meerschaum.config import __doc__ as doc
|
@@ -127,7 +127,7 @@ navbar = dbc.Navbar(
|
|
127
127
|
align='center',
|
128
128
|
className='g-0 navbar-logo-row',
|
129
129
|
),
|
130
|
-
href='/docs',
|
130
|
+
href=('/docs' if docs_enabled else '#'),
|
131
131
|
style={"textDecoration": "none"},
|
132
132
|
),
|
133
133
|
dbc.NavbarToggler(id="navbar-toggler", n_clicks=0),
|
meerschaum/api/dash/webterm.py
CHANGED
@@ -7,6 +7,8 @@ Functions for interacting with the Webterm via the dashboard.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
import time
|
10
|
+
|
11
|
+
import meerschaum as mrsm
|
10
12
|
from meerschaum.api import CHECK_UPDATE, get_api_connector
|
11
13
|
from meerschaum.api.dash.sessions import is_session_authenticated, get_username_from_session
|
12
14
|
from meerschaum.api.dash.components import alert_from_success_tuple, console_div
|
@@ -19,7 +21,9 @@ dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHEC
|
|
19
21
|
dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
|
20
22
|
|
21
23
|
MAX_WEBTERM_ATTEMPTS: int = 10
|
22
|
-
|
24
|
+
TMUX_IS_ENABLED: bool = (
|
25
|
+
is_tmux_available() and mrsm.get_config('system', 'webterm', 'tmux', 'enabled')
|
26
|
+
)
|
23
27
|
|
24
28
|
_locks = {'webterm_thread': RLock()}
|
25
29
|
|
@@ -48,10 +52,25 @@ def get_webterm(state: WebState) -> Tuple[Any, Any]:
|
|
48
52
|
[
|
49
53
|
html.Div(
|
50
54
|
[
|
51
|
-
dbc.Button(
|
52
|
-
|
53
|
-
|
55
|
+
dbc.Button(
|
56
|
+
'Refresh',
|
57
|
+
color='black',
|
58
|
+
id='webterm-refresh-button',
|
59
|
+
),
|
60
|
+
dbc.Button(
|
61
|
+
'Full View',
|
62
|
+
color='black',
|
63
|
+
id='webterm-fullscreen-button',
|
64
|
+
),
|
65
|
+
] + [
|
66
|
+
dbc.Button(
|
67
|
+
'New Tab',
|
68
|
+
color='black',
|
69
|
+
id='webterm-new-tab-button',
|
70
|
+
),
|
71
|
+
] if TMUX_IS_ENABLED else [],
|
54
72
|
id='webterm-controls-div',
|
73
|
+
style={'text-align': 'right'},
|
55
74
|
),
|
56
75
|
html.Iframe(
|
57
76
|
src=f"/webterm/{session_id}",
|