meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/_internal/arguments/_parser.py +14 -2
- meerschaum/_internal/cli/__init__.py +6 -0
- meerschaum/_internal/cli/daemons.py +103 -0
- meerschaum/_internal/cli/entry.py +220 -0
- meerschaum/_internal/cli/workers.py +435 -0
- meerschaum/_internal/docs/index.py +1 -2
- meerschaum/_internal/entry.py +44 -8
- meerschaum/_internal/shell/Shell.py +115 -24
- meerschaum/_internal/shell/__init__.py +4 -1
- meerschaum/_internal/static.py +4 -1
- meerschaum/_internal/term/TermPageHandler.py +1 -2
- meerschaum/_internal/term/__init__.py +40 -6
- meerschaum/_internal/term/tools.py +33 -8
- meerschaum/actions/__init__.py +6 -4
- meerschaum/actions/api.py +39 -11
- meerschaum/actions/attach.py +1 -0
- meerschaum/actions/delete.py +4 -2
- meerschaum/actions/edit.py +27 -8
- meerschaum/actions/login.py +8 -8
- meerschaum/actions/register.py +13 -7
- meerschaum/actions/reload.py +22 -5
- meerschaum/actions/restart.py +14 -0
- meerschaum/actions/show.py +69 -4
- meerschaum/actions/start.py +135 -14
- meerschaum/actions/stop.py +36 -3
- meerschaum/actions/sync.py +6 -1
- meerschaum/api/__init__.py +35 -13
- meerschaum/api/_events.py +2 -2
- meerschaum/api/_oauth2.py +47 -4
- meerschaum/api/dash/callbacks/dashboard.py +29 -0
- meerschaum/api/dash/callbacks/jobs.py +3 -2
- meerschaum/api/dash/callbacks/login.py +10 -1
- meerschaum/api/dash/callbacks/register.py +9 -2
- meerschaum/api/dash/pages/login.py +2 -2
- meerschaum/api/dash/pipes.py +72 -36
- meerschaum/api/dash/webterm.py +14 -6
- meerschaum/api/models/_pipes.py +7 -1
- meerschaum/api/resources/static/js/terminado.js +3 -0
- meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
- meerschaum/api/resources/templates/termpage.html +1 -0
- meerschaum/api/routes/_jobs.py +23 -11
- meerschaum/api/routes/_login.py +73 -5
- meerschaum/api/routes/_pipes.py +6 -4
- meerschaum/api/routes/_webterm.py +3 -3
- meerschaum/config/__init__.py +60 -13
- meerschaum/config/_default.py +89 -61
- meerschaum/config/_edit.py +10 -8
- meerschaum/config/_formatting.py +2 -0
- meerschaum/config/_patch.py +4 -2
- meerschaum/config/_paths.py +127 -12
- meerschaum/config/_read_config.py +32 -12
- meerschaum/config/_version.py +1 -1
- meerschaum/config/environment.py +262 -0
- meerschaum/config/stack/__init__.py +7 -5
- meerschaum/connectors/_Connector.py +1 -2
- meerschaum/connectors/__init__.py +37 -2
- meerschaum/connectors/api/_APIConnector.py +1 -1
- meerschaum/connectors/api/_jobs.py +11 -0
- meerschaum/connectors/api/_pipes.py +7 -1
- meerschaum/connectors/instance/_plugins.py +9 -1
- meerschaum/connectors/instance/_tokens.py +20 -3
- meerschaum/connectors/instance/_users.py +8 -1
- meerschaum/connectors/parse.py +1 -1
- meerschaum/connectors/sql/_create_engine.py +3 -0
- meerschaum/connectors/sql/_pipes.py +93 -79
- meerschaum/connectors/sql/_users.py +8 -1
- meerschaum/connectors/valkey/_ValkeyConnector.py +3 -3
- meerschaum/connectors/valkey/_pipes.py +7 -5
- meerschaum/core/Pipe/__init__.py +45 -71
- meerschaum/core/Pipe/_attributes.py +66 -90
- meerschaum/core/Pipe/_cache.py +555 -0
- meerschaum/core/Pipe/_clear.py +0 -11
- meerschaum/core/Pipe/_data.py +0 -50
- meerschaum/core/Pipe/_deduplicate.py +0 -13
- meerschaum/core/Pipe/_delete.py +12 -21
- meerschaum/core/Pipe/_drop.py +11 -23
- meerschaum/core/Pipe/_dtypes.py +1 -1
- meerschaum/core/Pipe/_index.py +8 -14
- meerschaum/core/Pipe/_sync.py +12 -18
- meerschaum/core/Plugin/_Plugin.py +7 -1
- meerschaum/core/Token/_Token.py +1 -1
- meerschaum/core/User/_User.py +1 -2
- meerschaum/jobs/_Executor.py +88 -4
- meerschaum/jobs/_Job.py +146 -36
- meerschaum/jobs/systemd.py +7 -2
- meerschaum/plugins/__init__.py +277 -81
- meerschaum/utils/daemon/Daemon.py +197 -42
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
- meerschaum/utils/daemon/RotatingFile.py +63 -36
- meerschaum/utils/daemon/StdinFile.py +53 -13
- meerschaum/utils/daemon/__init__.py +18 -5
- meerschaum/utils/daemon/_names.py +6 -3
- meerschaum/utils/debug.py +34 -4
- meerschaum/utils/dtypes/__init__.py +5 -1
- meerschaum/utils/formatting/__init__.py +4 -1
- meerschaum/utils/formatting/_jobs.py +1 -1
- meerschaum/utils/formatting/_pipes.py +47 -46
- meerschaum/utils/formatting/_shell.py +33 -9
- meerschaum/utils/misc.py +22 -38
- meerschaum/utils/packages/__init__.py +15 -13
- meerschaum/utils/packages/_packages.py +1 -0
- meerschaum/utils/pipes.py +33 -5
- meerschaum/utils/process.py +1 -1
- meerschaum/utils/prompt.py +172 -143
- meerschaum/utils/sql.py +12 -2
- meerschaum/utils/threading.py +42 -0
- meerschaum/utils/venv/__init__.py +2 -0
- meerschaum/utils/warnings.py +19 -13
- {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/METADATA +3 -1
- {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/RECORD +116 -110
- meerschaum/config/_environment.py +0 -145
- {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/WHEEL +0 -0
- {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/entry_points.txt +0 -0
- {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/licenses/LICENSE +0 -0
- {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/licenses/NOTICE +0 -0
- {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/top_level.txt +0 -0
- {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/zip-safe +0 -0
meerschaum/actions/start.py
CHANGED
@@ -7,6 +7,8 @@ Start subsystems (API server, logging daemon, etc.).
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
+
|
11
|
+
import meerschaum as mrsm
|
10
12
|
from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union, Dict
|
11
13
|
|
12
14
|
|
@@ -25,6 +27,7 @@ def start(
|
|
25
27
|
'webterm': _start_webterm,
|
26
28
|
'connectors': _start_connectors,
|
27
29
|
'pipeline': _start_pipeline,
|
30
|
+
'daemons': _start_daemons,
|
28
31
|
}
|
29
32
|
return choose_subaction(action, options, **kw)
|
30
33
|
|
@@ -308,6 +311,7 @@ def _start_gui(
|
|
308
311
|
action: Optional[List[str]] = None,
|
309
312
|
mrsm_instance: Optional[str] = None,
|
310
313
|
port: Optional[int] = None,
|
314
|
+
webterm_port: Optional[int] = None,
|
311
315
|
debug: bool = False,
|
312
316
|
**kw
|
313
317
|
) -> SuccessTuple:
|
@@ -321,7 +325,6 @@ def _start_gui(
|
|
321
325
|
|
322
326
|
from meerschaum.utils.venv import venv_exec
|
323
327
|
from meerschaum.utils.packages import attempt_import
|
324
|
-
from meerschaum.utils.warnings import warn
|
325
328
|
from meerschaum.utils.debug import dprint
|
326
329
|
from meerschaum.utils.networking import find_open_ports
|
327
330
|
from meerschaum.connectors.parse import parse_instance_keys
|
@@ -329,9 +332,10 @@ def _start_gui(
|
|
329
332
|
webview, requests = attempt_import('webview', 'requests')
|
330
333
|
|
331
334
|
success, msg = True, "Success"
|
332
|
-
host =
|
333
|
-
|
334
|
-
|
335
|
+
host = mrsm.get_config('webterm', 'host')
|
336
|
+
port = port or webterm_port
|
337
|
+
if not port:
|
338
|
+
port = mrsm.get_config('webterm', 'port')
|
335
339
|
|
336
340
|
if not is_webterm_running(host, port, session_id='mrsm'):
|
337
341
|
port = next(find_open_ports(port, 9000))
|
@@ -369,7 +373,7 @@ def _start_gui(
|
|
369
373
|
break
|
370
374
|
except Exception as e:
|
371
375
|
if debug:
|
372
|
-
dprint(e)
|
376
|
+
dprint(str(e))
|
373
377
|
continue
|
374
378
|
if starting_up is False:
|
375
379
|
return False, f"The webterm failed to start within {timeout} seconds."
|
@@ -417,7 +421,6 @@ def _start_webterm(
|
|
417
421
|
- `-i`, '--instance'
|
418
422
|
The default instance to use for the Webterm shell.
|
419
423
|
"""
|
420
|
-
import uuid
|
421
424
|
from meerschaum._internal.term import get_webterm_app_and_manager, tornado_ioloop
|
422
425
|
from meerschaum._internal.term.tools import (
|
423
426
|
is_webterm_running,
|
@@ -426,15 +429,17 @@ def _start_webterm(
|
|
426
429
|
)
|
427
430
|
from meerschaum.utils.networking import find_open_ports
|
428
431
|
from meerschaum.utils.warnings import info
|
432
|
+
from meerschaum.config.paths import WEBTERM_INTERNAL_RESOURCES_PATH
|
429
433
|
|
430
434
|
if host is None:
|
431
|
-
host =
|
435
|
+
host = mrsm.get_config('api', 'webterm', 'host')
|
432
436
|
if port is None:
|
433
|
-
port =
|
437
|
+
port = mrsm.get_config('api', 'webterm', 'port')
|
434
438
|
if sysargs is None:
|
435
439
|
sysargs = ['start', 'webterm']
|
436
440
|
session_id = 'mrsm'
|
437
|
-
|
441
|
+
|
442
|
+
env_path = WEBTERM_INTERNAL_RESOURCES_PATH / (str(port) + '.json')
|
438
443
|
|
439
444
|
if is_webterm_running(host, port, session_id=session_id):
|
440
445
|
if force:
|
@@ -445,11 +450,18 @@ def _start_webterm(
|
|
445
450
|
+ " Include `-f` to start another server on a new port\n"
|
446
451
|
+ " or specify a different port with `-p`."
|
447
452
|
)
|
453
|
+
|
454
|
+
tornado_app, term_manager = get_webterm_app_and_manager(
|
455
|
+
instance_keys=mrsm_instance,
|
456
|
+
port=port,
|
457
|
+
env_path=env_path,
|
458
|
+
)
|
448
459
|
if not nopretty:
|
449
460
|
info(
|
450
461
|
f"Starting the webterm at http://{host}:{port}/webterm/{session_id} ..."
|
451
462
|
"\n Press CTRL+C to quit."
|
452
463
|
)
|
464
|
+
|
453
465
|
tornado_app.listen(port, host)
|
454
466
|
loop = tornado_ioloop.IOLoop.instance()
|
455
467
|
try:
|
@@ -462,10 +474,15 @@ def _start_webterm(
|
|
462
474
|
term_manager.shutdown()
|
463
475
|
loop.close()
|
464
476
|
|
465
|
-
sessions = get_mrsm_tmux_sessions()
|
477
|
+
sessions = get_mrsm_tmux_sessions(port=port)
|
466
478
|
for session in sessions:
|
467
479
|
kill_tmux_session(session)
|
468
480
|
|
481
|
+
try:
|
482
|
+
env_path.unlink()
|
483
|
+
except Exception:
|
484
|
+
pass
|
485
|
+
|
469
486
|
return True, "Success"
|
470
487
|
|
471
488
|
|
@@ -483,7 +500,6 @@ def _start_connectors(
|
|
483
500
|
from meerschaum.connectors.parse import parse_instance_keys
|
484
501
|
from meerschaum.utils.pool import get_pool
|
485
502
|
from meerschaum.utils.warnings import warn
|
486
|
-
from meerschaum.utils.formatting import pprint
|
487
503
|
from meerschaum.utils.misc import items_str
|
488
504
|
if action is None:
|
489
505
|
action = []
|
@@ -575,7 +591,6 @@ def _start_pipeline(
|
|
575
591
|
"""
|
576
592
|
import json
|
577
593
|
import time
|
578
|
-
import sys
|
579
594
|
from meerschaum._internal.entry import entry
|
580
595
|
from meerschaum.utils.warnings import info, warn
|
581
596
|
from meerschaum.utils.misc import is_int
|
@@ -630,7 +645,7 @@ def _start_pipeline(
|
|
630
645
|
def do_entry() -> None:
|
631
646
|
nonlocal success, msg, proc
|
632
647
|
if timeout_seconds is None:
|
633
|
-
success, msg = entry(sub_args_line, _patch_args=patch_args)
|
648
|
+
success, msg = entry(sub_args_line, _patch_args=patch_args, _use_cli_daemon=False)
|
634
649
|
return
|
635
650
|
|
636
651
|
sub_args_line_escaped = sub_args_line.replace("'", "<QUOTE>")
|
@@ -640,7 +655,7 @@ def _start_pipeline(
|
|
640
655
|
"from meerschaum._internal.entry import entry\n\n"
|
641
656
|
f"sub_args_line = '{sub_args_line_escaped}'.replace(\"<QUOTE>\", \"'\")\n"
|
642
657
|
f"patch_args = json.loads('{patch_args_escaped_str}'.replace('<QUOTE>', \"'\"))\n"
|
643
|
-
"success, msg = entry(sub_args_line, _patch_args=patch_args)\n"
|
658
|
+
"success, msg = entry(sub_args_line, _patch_args=patch_args, _use_cli_daemon=False)\n"
|
644
659
|
f"print('{fence_begin}' + json.dumps((success, msg)) + '{fence_end}')"
|
645
660
|
)
|
646
661
|
proc = venv_exec(src, venv=None, as_proc=True)
|
@@ -688,6 +703,112 @@ def _start_pipeline(
|
|
688
703
|
return success, msg
|
689
704
|
|
690
705
|
|
706
|
+
def _start_daemons(
|
707
|
+
timeout_seconds: Union[int, float, None] = None,
|
708
|
+
yes: bool = False,
|
709
|
+
force: bool = False,
|
710
|
+
noask: bool = False,
|
711
|
+
debug: bool = False,
|
712
|
+
**kwargs
|
713
|
+
) -> SuccessTuple:
|
714
|
+
"""
|
715
|
+
Start the Meerschaum CLI daemon processes.
|
716
|
+
"""
|
717
|
+
from meerschaum.utils.warnings import warn, dprint
|
718
|
+
from meerschaum._internal.cli.daemons import (
|
719
|
+
get_cli_daemon,
|
720
|
+
get_cli_lock_path,
|
721
|
+
)
|
722
|
+
from meerschaum._internal.cli.workers import (
|
723
|
+
get_existing_cli_workers,
|
724
|
+
get_existing_cli_worker_indices,
|
725
|
+
)
|
726
|
+
from meerschaum.utils.prompt import yes_no
|
727
|
+
from meerschaum.actions import actions
|
728
|
+
|
729
|
+
workers = get_existing_cli_workers()
|
730
|
+
if not workers:
|
731
|
+
if debug:
|
732
|
+
dprint("No daemons are running, spawning a new process...")
|
733
|
+
workers = [get_cli_daemon()]
|
734
|
+
|
735
|
+
accepted_restart = False
|
736
|
+
any_daemons_are_running = any((worker.job.status == 'running') for worker in workers)
|
737
|
+
lock_paths = [get_cli_lock_path(ix) for ix in get_existing_cli_worker_indices()]
|
738
|
+
any_locks_exist = any(lock_path.exists() for lock_path in lock_paths)
|
739
|
+
|
740
|
+
if any_locks_exist:
|
741
|
+
warn(
|
742
|
+
"Locks are currently held by the CLI daemons.\n"
|
743
|
+
"Run again with `--force` to remove the locks.",
|
744
|
+
stack=False,
|
745
|
+
)
|
746
|
+
|
747
|
+
if not force:
|
748
|
+
return False, "Actions are currently running."
|
749
|
+
|
750
|
+
for lock_path in lock_paths:
|
751
|
+
try:
|
752
|
+
if lock_path.exists():
|
753
|
+
lock_path.unlink()
|
754
|
+
except Exception as e:
|
755
|
+
warn(f"Failed to release lock:\n{e}")
|
756
|
+
|
757
|
+
if any_daemons_are_running:
|
758
|
+
accepted_restart = force or yes_no(
|
759
|
+
"Restart running CLI daemons?",
|
760
|
+
yes=yes,
|
761
|
+
noask=noask,
|
762
|
+
default='n',
|
763
|
+
)
|
764
|
+
|
765
|
+
if not accepted_restart:
|
766
|
+
return True, "Daemons are already running; nothing to do."
|
767
|
+
|
768
|
+
stop_success, stop_msg = actions['stop'](
|
769
|
+
['daemons'],
|
770
|
+
timeout_seconds=timeout_seconds,
|
771
|
+
debug=debug,
|
772
|
+
)
|
773
|
+
if not stop_success:
|
774
|
+
return stop_success, stop_msg
|
775
|
+
|
776
|
+
worker = get_cli_daemon()
|
777
|
+
|
778
|
+
if debug:
|
779
|
+
dprint("Cleaning up CLI daemon...")
|
780
|
+
cleanup_success, cleanup_msg = worker.cleanup()
|
781
|
+
if not cleanup_success:
|
782
|
+
return cleanup_success, cleanup_msg
|
783
|
+
|
784
|
+
if debug:
|
785
|
+
dprint("Starting CLI daemon...")
|
786
|
+
|
787
|
+
start_success, start_msg = worker.job.start()
|
788
|
+
if not start_success:
|
789
|
+
return start_success, start_msg
|
790
|
+
|
791
|
+
return True, "Success"
|
792
|
+
|
793
|
+
|
794
|
+
def _start_worker(action: Optional[List[str]] = None, **kwargs: Any) -> SuccessTuple:
|
795
|
+
"""
|
796
|
+
Start a CLI worker process. This is intended for internal use only.
|
797
|
+
"""
|
798
|
+
from meerschaum._internal.cli.workers import ActionWorker
|
799
|
+
from meerschaum.utils.misc import is_int
|
800
|
+
|
801
|
+
if not action:
|
802
|
+
return False, "No worker index is provided."
|
803
|
+
|
804
|
+
if not is_int(action[0]):
|
805
|
+
return False, "Invalid worker index."
|
806
|
+
|
807
|
+
ix = int(action[0])
|
808
|
+
worker = ActionWorker(ix)
|
809
|
+
return worker.run()
|
810
|
+
|
811
|
+
|
691
812
|
### NOTE: This must be the final statement of the module.
|
692
813
|
### Any subactions added below these lines will not
|
693
814
|
### be added to the `help` docstring.
|
meerschaum/actions/stop.py
CHANGED
@@ -7,7 +7,7 @@ Stop running jobs that were started with `-d` or `start job`.
|
|
7
7
|
"""
|
8
8
|
|
9
9
|
from __future__ import annotations
|
10
|
-
from meerschaum.utils.typing import Optional, List, SuccessTuple, Any
|
10
|
+
from meerschaum.utils.typing import Optional, List, SuccessTuple, Any, Union
|
11
11
|
|
12
12
|
|
13
13
|
def stop(action: Optional[List[str]] = None, **kw) -> SuccessTuple:
|
@@ -17,6 +17,7 @@ def stop(action: Optional[List[str]] = None, **kw) -> SuccessTuple:
|
|
17
17
|
from meerschaum.actions import choose_subaction
|
18
18
|
options = {
|
19
19
|
'jobs': _stop_jobs,
|
20
|
+
'daemons': _stop_daemons,
|
20
21
|
}
|
21
22
|
return choose_subaction(action, options, **kw)
|
22
23
|
|
@@ -53,7 +54,7 @@ def _complete_stop(
|
|
53
54
|
return options[sub](action=action, **kw)
|
54
55
|
|
55
56
|
from meerschaum._internal.shell import default_action_completer
|
56
|
-
return default_action_completer(action=(['
|
57
|
+
return default_action_completer(action=(['stop'] + action), **kw)
|
57
58
|
|
58
59
|
|
59
60
|
def _stop_jobs(
|
@@ -113,7 +114,7 @@ def _stop_jobs(
|
|
113
114
|
|
114
115
|
if not jobs_to_stop:
|
115
116
|
if jobs:
|
116
|
-
return True, "The selected jobs are currently
|
117
|
+
return True, "The selected jobs are currently stopped."
|
117
118
|
return False, "No running, paused or restarting jobs to stop."
|
118
119
|
|
119
120
|
if not action:
|
@@ -163,6 +164,38 @@ def _stop_jobs(
|
|
163
164
|
return success, msg
|
164
165
|
|
165
166
|
|
167
|
+
def _stop_daemons(
|
168
|
+
timeout_seconds: Union[int, float, None] = None,
|
169
|
+
debug: bool = False,
|
170
|
+
**kwargs
|
171
|
+
) -> SuccessTuple:
|
172
|
+
"""
|
173
|
+
Stop the Meerschaum CLI daemon.
|
174
|
+
"""
|
175
|
+
import shutil
|
176
|
+
from meerschaum._internal.cli.workers import get_existing_cli_workers
|
177
|
+
from meerschaum.config.paths import CLI_RESOURCES_PATH
|
178
|
+
workers = get_existing_cli_workers()
|
179
|
+
|
180
|
+
for worker in workers:
|
181
|
+
stop_success, stop_msg = worker.job.stop(timeout_seconds=timeout_seconds, debug=debug)
|
182
|
+
if not stop_success:
|
183
|
+
return stop_success, stop_msg
|
184
|
+
|
185
|
+
cleanup_success, cleanup_msg = worker.cleanup(debug=debug)
|
186
|
+
if not cleanup_success:
|
187
|
+
return cleanup_success, cleanup_msg
|
188
|
+
|
189
|
+
try:
|
190
|
+
if CLI_RESOURCES_PATH.exists():
|
191
|
+
shutil.rmtree(CLI_RESOURCES_PATH)
|
192
|
+
CLI_RESOURCES_PATH.mkdir(parents=True, exist_ok=True)
|
193
|
+
except Exception as e:
|
194
|
+
return False, f"Failed to clean up CLI resources directory.\n{e}"
|
195
|
+
|
196
|
+
return True, "Success"
|
197
|
+
|
198
|
+
|
166
199
|
### NOTE: This must be the final statement of the module.
|
167
200
|
### Any subactions added below these lines will not
|
168
201
|
### be added to the `help` docstring.
|
meerschaum/actions/sync.py
CHANGED
@@ -289,6 +289,7 @@ def _sync_pipes(
|
|
289
289
|
from meerschaum.utils.formatting import print_pipes_results
|
290
290
|
from meerschaum._internal.static import STATIC_CONFIG
|
291
291
|
from meerschaum.utils.misc import interval_str
|
292
|
+
from meerschaum.utils.daemon import running_in_daemon
|
292
293
|
|
293
294
|
noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
|
294
295
|
noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
|
@@ -301,7 +302,11 @@ def _sync_pipes(
|
|
301
302
|
cooldown = 2 * (min_seconds + 1)
|
302
303
|
success_pipes, failure_pipes = [], []
|
303
304
|
while run:
|
304
|
-
_progress =
|
305
|
+
_progress = (
|
306
|
+
progress()
|
307
|
+
if (shell and not noninteractive and not running_in_daemon())
|
308
|
+
else None
|
309
|
+
)
|
305
310
|
cm = _progress if _progress is not None else contextlib.nullcontext()
|
306
311
|
|
307
312
|
lap_begin = time.perf_counter()
|
meerschaum/api/__init__.py
CHANGED
@@ -21,7 +21,6 @@ from meerschaum.config._paths import API_UVICORN_CONFIG_PATH, API_UVICORN_RESOUR
|
|
21
21
|
from meerschaum.plugins import _api_plugins
|
22
22
|
from meerschaum.utils.warnings import warn, dprint
|
23
23
|
from meerschaum.utils.threading import RLock
|
24
|
-
from meerschaum.utils.misc import is_pipe_registered
|
25
24
|
from meerschaum.connectors.parse import parse_instance_keys
|
26
25
|
|
27
26
|
from meerschaum import __version__ as version
|
@@ -67,16 +66,13 @@ from meerschaum.api._exceptions import APIPermissionError
|
|
67
66
|
uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
|
68
67
|
|
69
68
|
uvicorn_config = None
|
70
|
-
sys_config = get_config('
|
71
|
-
permissions_config = get_config('
|
69
|
+
sys_config = get_config('api')
|
70
|
+
permissions_config = get_config('api', 'permissions')
|
72
71
|
|
73
72
|
def get_uvicorn_config() -> Dict[str, Any]:
|
74
73
|
"""Read the Uvicorn configuration JSON and return a dictionary."""
|
75
74
|
global uvicorn_config
|
76
75
|
import json
|
77
|
-
runtime = os.environ.get(STATIC_CONFIG['environment']['runtime'], None)
|
78
|
-
if runtime == 'api':
|
79
|
-
return get_config('system', 'api', 'uvicorn')
|
80
76
|
_uvicorn_config = uvicorn_config
|
81
77
|
with _locks['uvicorn_config']:
|
82
78
|
if uvicorn_config is None:
|
@@ -85,6 +81,8 @@ def get_uvicorn_config() -> Dict[str, Any]:
|
|
85
81
|
uvicorn_config = json.load(f)
|
86
82
|
_uvicorn_config = uvicorn_config
|
87
83
|
except Exception:
|
84
|
+
import traceback
|
85
|
+
traceback.print_exc()
|
88
86
|
_uvicorn_config = sys_config.get('uvicorn', None)
|
89
87
|
|
90
88
|
if _uvicorn_config is None:
|
@@ -102,6 +100,10 @@ production = get_uvicorn_config().get('production', False)
|
|
102
100
|
_include_dash = (not no_dash)
|
103
101
|
_include_webterm = (not no_webterm) and _include_dash
|
104
102
|
docs_enabled = not production or sys_config.get('endpoints', {}).get('docs_in_production', True)
|
103
|
+
webterm_port = (
|
104
|
+
get_uvicorn_config().get('webterm_port', None)
|
105
|
+
or mrsm.get_config('api', 'webterm', 'port')
|
106
|
+
)
|
105
107
|
|
106
108
|
default_instance_keys = None
|
107
109
|
_instance_connectors = defaultdict(lambda: None)
|
@@ -129,7 +131,7 @@ def get_api_connector(instance_keys: Optional[str] = None):
|
|
129
131
|
)
|
130
132
|
found_match: bool = False
|
131
133
|
for allowed_keys_pattern in allowed_instance_keys:
|
132
|
-
if fnmatch(instance_keys, allowed_keys_pattern):
|
134
|
+
if fnmatch(str(instance_keys), allowed_keys_pattern):
|
133
135
|
found_match = True
|
134
136
|
break
|
135
137
|
if not found_match:
|
@@ -141,7 +143,9 @@ def get_api_connector(instance_keys: Optional[str] = None):
|
|
141
143
|
if _instance_connectors[instance_keys] is None:
|
142
144
|
try:
|
143
145
|
is_valid_connector = True
|
144
|
-
|
146
|
+
instance_connector = parse_instance_keys(instance_keys, debug=debug)
|
147
|
+
instance_connector._cache_connector = get_cache_connector()
|
148
|
+
_instance_connectors[instance_keys] = instance_connector
|
145
149
|
except Exception:
|
146
150
|
is_valid_connector = False
|
147
151
|
|
@@ -168,7 +172,7 @@ def get_cache_connector(connector_keys: Optional[str] = None):
|
|
168
172
|
return None
|
169
173
|
|
170
174
|
connector_keys = connector_keys or get_config(
|
171
|
-
'
|
175
|
+
'api', 'cache', 'connector',
|
172
176
|
warn=False,
|
173
177
|
)
|
174
178
|
if connector_keys is None:
|
@@ -196,7 +200,11 @@ def pipes(instance_keys: Optional[str] = None, refresh: bool = False) -> PipesDi
|
|
196
200
|
with _locks['pipes-' + instance_keys]:
|
197
201
|
pipes = _instance_pipes[instance_keys]
|
198
202
|
if pipes is None or refresh:
|
199
|
-
pipes = _get_pipes(
|
203
|
+
pipes = _get_pipes(
|
204
|
+
mrsm_instance=instance_keys,
|
205
|
+
cache=True,
|
206
|
+
cache_connector_keys=get_cache_connector(),
|
207
|
+
)
|
200
208
|
_instance_pipes[instance_keys] = pipes
|
201
209
|
return pipes
|
202
210
|
|
@@ -218,9 +226,23 @@ def get_pipe(
|
|
218
226
|
detail="Unable to serve any pipes with connector keys `mrsm` over the API.",
|
219
227
|
)
|
220
228
|
|
221
|
-
|
222
|
-
if
|
223
|
-
|
229
|
+
pipes_dict = pipes(instance_keys)
|
230
|
+
if (
|
231
|
+
not refresh
|
232
|
+
and connector_keys in pipes_dict
|
233
|
+
and metric_key in pipes_dict[connector_keys]
|
234
|
+
and location_key in pipes_dict[connector_keys][metric_key]
|
235
|
+
):
|
236
|
+
return pipes_dict[connector_keys][metric_key][location_key]
|
237
|
+
|
238
|
+
pipe = mrsm.Pipe(
|
239
|
+
connector_keys,
|
240
|
+
metric_key,
|
241
|
+
location_key,
|
242
|
+
mrsm_instance=instance_keys,
|
243
|
+
cache=True,
|
244
|
+
cache_connector_keys=get_cache_connector(),
|
245
|
+
)
|
224
246
|
return pipe
|
225
247
|
|
226
248
|
|
meerschaum/api/_events.py
CHANGED
@@ -8,13 +8,13 @@ Declare FastAPI events in this module (startup, shutdown, etc.).
|
|
8
8
|
|
9
9
|
import sys
|
10
10
|
import os
|
11
|
-
import time
|
12
11
|
from meerschaum.api import (
|
13
12
|
app,
|
14
13
|
get_api_connector,
|
15
14
|
get_cache_connector,
|
16
15
|
get_uvicorn_config,
|
17
16
|
debug,
|
17
|
+
webterm_port,
|
18
18
|
no_dash,
|
19
19
|
_include_dash,
|
20
20
|
_include_webterm,
|
@@ -75,7 +75,7 @@ async def startup():
|
|
75
75
|
try:
|
76
76
|
if _include_webterm:
|
77
77
|
from meerschaum.api.dash.webterm import start_webterm
|
78
|
-
start_webterm()
|
78
|
+
start_webterm(webterm_port=webterm_port)
|
79
79
|
|
80
80
|
connected = retry_connect(
|
81
81
|
get_api_connector(),
|
meerschaum/api/_oauth2.py
CHANGED
@@ -21,6 +21,7 @@ from meerschaum.core import User, Token
|
|
21
21
|
fastapi, starlette = attempt_import('fastapi', 'starlette', lazy=False, check_update=CHECK_UPDATE)
|
22
22
|
fastapi_responses = attempt_import('fastapi.responses', lazy=False, check_update=CHECK_UPDATE)
|
23
23
|
fastapi_login = attempt_import('fastapi_login', check_update=CHECK_UPDATE)
|
24
|
+
jose_jwt, jose_exceptions = attempt_import('jose.jwt', 'jose.exceptions', lazy=False, check_update=CHECK_UPDATE)
|
24
25
|
from fastapi import Depends, HTTPException, Request
|
25
26
|
from starlette import status
|
26
27
|
|
@@ -91,6 +92,7 @@ async def load_user_or_token(
|
|
91
92
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
92
93
|
detail="Not authenticated.",
|
93
94
|
)
|
95
|
+
|
94
96
|
authorization = authorization.replace('Basic ', '').replace('Bearer ', '')
|
95
97
|
if not authorization.startswith('mrsm-key:'):
|
96
98
|
if not users:
|
@@ -98,12 +100,15 @@ async def load_user_or_token(
|
|
98
100
|
status=status.HTTP_401_UNAUTHORIZED,
|
99
101
|
detail="Users not authenticated for this endpoint.",
|
100
102
|
)
|
103
|
+
|
101
104
|
return await manager(request)
|
105
|
+
|
102
106
|
if not tokens:
|
103
107
|
raise HTTPException(
|
104
108
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
105
109
|
detail="Tokens not authenticated for this endpoint.",
|
106
110
|
)
|
111
|
+
|
107
112
|
return get_token_from_authorization(authorization)
|
108
113
|
|
109
114
|
|
@@ -112,6 +117,7 @@ def ScopedAuth(scopes: List[str]):
|
|
112
117
|
Dependency factory for authenticating with either a user session or a scoped token.
|
113
118
|
"""
|
114
119
|
async def _authenticate(
|
120
|
+
request: Request,
|
115
121
|
user_or_token: Union[User, Token, None] = Depends(
|
116
122
|
load_user_or_token,
|
117
123
|
),
|
@@ -126,12 +132,45 @@ def ScopedAuth(scopes: List[str]):
|
|
126
132
|
headers={"WWW-Authenticate": "Basic"},
|
127
133
|
)
|
128
134
|
|
129
|
-
|
130
|
-
|
135
|
+
authorization = request.headers.get('authorization', request.headers.get('Authorization', None))
|
136
|
+
is_long_lived = authorization and 'mrsm-key:' in authorization
|
137
|
+
|
138
|
+
current_scopes = []
|
139
|
+
### For long-lived API tokens, always hit the database.
|
140
|
+
if is_long_lived:
|
141
|
+
current_scopes = user_or_token.get_scopes(refresh=True, debug=debug)
|
142
|
+
|
143
|
+
### For JWTs, trust the scopes in the token.
|
144
|
+
else:
|
145
|
+
if not authorization:
|
146
|
+
# This should be caught by `load_user_or_token` but we can be safe.
|
147
|
+
raise HTTPException(
|
148
|
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
149
|
+
detail="Not authenticated.",
|
150
|
+
)
|
151
|
+
|
152
|
+
scheme, _, token_str = authorization.partition(' ')
|
153
|
+
if not token_str or scheme.lower() != 'bearer':
|
154
|
+
raise HTTPException(
|
155
|
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
156
|
+
detail="Unsupported authentication scheme.",
|
157
|
+
)
|
158
|
+
|
159
|
+
try:
|
160
|
+
payload = jose_jwt.decode(token_str, SECRET, algorithms=['HS256'])
|
161
|
+
current_scopes = payload.get('scopes', [])
|
162
|
+
except jose_exceptions.JWTError:
|
163
|
+
raise HTTPException(
|
164
|
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
165
|
+
detail="Invalid access token.",
|
166
|
+
headers={"WWW-Authenticate": "Bearer"},
|
167
|
+
)
|
168
|
+
|
169
|
+
if '*' in current_scopes:
|
131
170
|
return user_or_token
|
132
171
|
|
133
172
|
for scope in scopes:
|
134
|
-
if scope not in
|
173
|
+
if scope not in current_scopes:
|
135
174
|
raise HTTPException(
|
136
175
|
status_code=status.HTTP_403_FORBIDDEN,
|
137
176
|
detail=f"Missing required scope: '{scope}'",
|
@@ -159,4 +198,8 @@ def generate_secret_key() -> bytes:
|
|
159
198
|
|
160
199
|
|
161
200
|
SECRET = generate_secret_key()
|
162
|
-
manager = LoginManager(
|
201
|
+
manager = LoginManager(
|
202
|
+
SECRET,
|
203
|
+
token_url=endpoints['login'],
|
204
|
+
scopes=STATIC_CONFIG['tokens']['scopes'],
|
205
|
+
)
|
@@ -1193,3 +1193,32 @@ def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
|
|
1193
1193
|
if n_clicks:
|
1194
1194
|
return not is_open, pages_children
|
1195
1195
|
return is_open, pages_children
|
1196
|
+
|
1197
|
+
|
1198
|
+
@dash_app.callback(
|
1199
|
+
Output({'type': 'calculate-rowcount-div', 'index': MATCH}, 'children'),
|
1200
|
+
Input({'type': 'calculate-rowcount-button', 'index': MATCH}, 'n_clicks'),
|
1201
|
+
prevent_initial_call=True,
|
1202
|
+
)
|
1203
|
+
def calculate_rowcount_button_click(n_clicks: int):
|
1204
|
+
"""
|
1205
|
+
Calculate the rowcount for the pipe.
|
1206
|
+
"""
|
1207
|
+
if not n_clicks:
|
1208
|
+
raise PreventUpdate
|
1209
|
+
|
1210
|
+
triggered = dash.callback_context.triggered
|
1211
|
+
if triggered[0]['value'] is None:
|
1212
|
+
raise PreventUpdate
|
1213
|
+
|
1214
|
+
pipe = pipe_from_ctx(triggered, 'n_clicks')
|
1215
|
+
if pipe is None:
|
1216
|
+
raise PreventUpdate
|
1217
|
+
|
1218
|
+
try:
|
1219
|
+
rowcount = pipe.get_rowcount(debug=debug)
|
1220
|
+
return f"{rowcount:,}"
|
1221
|
+
except Exception as e:
|
1222
|
+
return (
|
1223
|
+
alert_from_success_tuple((False, f"Failed to calculate row count: {e}"))
|
1224
|
+
)
|
@@ -13,6 +13,7 @@ import time
|
|
13
13
|
import traceback
|
14
14
|
from datetime import datetime, timezone
|
15
15
|
|
16
|
+
import meerschaum as mrsm
|
16
17
|
from meerschaum.jobs import get_jobs
|
17
18
|
from meerschaum.utils.typing import Optional, Dict, Any
|
18
19
|
from meerschaum.api import CHECK_UPDATE
|
@@ -136,12 +137,12 @@ def manage_job_button_click(
|
|
136
137
|
old_status = job.status
|
137
138
|
try:
|
138
139
|
success, msg = manage_functions[manage_job_action]()
|
139
|
-
except Exception
|
140
|
+
except Exception:
|
140
141
|
success, msg = False, traceback.format_exc()
|
141
142
|
|
142
143
|
### Wait for a status change before building the elements.
|
143
144
|
timeout_seconds = 1.0
|
144
|
-
check_interval_seconds =
|
145
|
+
check_interval_seconds = mrsm.get_config('system', 'cli', 'refresh_seconds')
|
145
146
|
begin = time.perf_counter()
|
146
147
|
while (time.perf_counter() - begin) < timeout_seconds:
|
147
148
|
if job.status != old_status:
|
@@ -16,11 +16,14 @@ from meerschaum.utils.typing import Optional
|
|
16
16
|
dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
|
17
17
|
from dash.exceptions import PreventUpdate
|
18
18
|
from dash.dependencies import Input, Output, State
|
19
|
+
|
19
20
|
from meerschaum.api.dash import dash_app, debug, pipes, _get_pipes
|
20
21
|
from meerschaum.api.dash.sessions import set_session
|
21
22
|
from meerschaum.api.dash.connectors import get_web_connector
|
22
23
|
from meerschaum.api.routes._login import login
|
23
24
|
from meerschaum.api.dash.components import alert_from_success_tuple
|
25
|
+
from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
|
26
|
+
from meerschaum._internal.static import STATIC_CONFIG
|
24
27
|
from fastapi_login.exceptions import InvalidCredentialsException
|
25
28
|
from fastapi.exceptions import HTTPException
|
26
29
|
dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
|
@@ -73,7 +76,13 @@ def login_button_click(
|
|
73
76
|
raise PreventUpdate
|
74
77
|
|
75
78
|
try:
|
76
|
-
|
79
|
+
form = CustomOAuth2PasswordRequestForm(
|
80
|
+
grant_type='password',
|
81
|
+
username=username,
|
82
|
+
password=password,
|
83
|
+
scope=' '.join(STATIC_CONFIG['tokens']['scopes'])
|
84
|
+
)
|
85
|
+
_ = login(form)
|
77
86
|
session_id = str(uuid.uuid4())
|
78
87
|
session_data = {
|
79
88
|
'session-id': session_id,
|