meerschaum 3.0.0rc4__py3-none-any.whl → 3.0.0rc8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. meerschaum/_internal/arguments/_parser.py +14 -2
  2. meerschaum/_internal/cli/__init__.py +6 -0
  3. meerschaum/_internal/cli/daemons.py +103 -0
  4. meerschaum/_internal/cli/entry.py +220 -0
  5. meerschaum/_internal/cli/workers.py +435 -0
  6. meerschaum/_internal/docs/index.py +1 -2
  7. meerschaum/_internal/entry.py +44 -8
  8. meerschaum/_internal/shell/Shell.py +115 -24
  9. meerschaum/_internal/shell/__init__.py +4 -1
  10. meerschaum/_internal/static.py +4 -1
  11. meerschaum/_internal/term/TermPageHandler.py +1 -2
  12. meerschaum/_internal/term/__init__.py +40 -6
  13. meerschaum/_internal/term/tools.py +33 -8
  14. meerschaum/actions/__init__.py +6 -4
  15. meerschaum/actions/api.py +39 -11
  16. meerschaum/actions/attach.py +1 -0
  17. meerschaum/actions/delete.py +4 -2
  18. meerschaum/actions/edit.py +27 -8
  19. meerschaum/actions/login.py +8 -8
  20. meerschaum/actions/register.py +13 -7
  21. meerschaum/actions/reload.py +22 -5
  22. meerschaum/actions/restart.py +14 -0
  23. meerschaum/actions/show.py +69 -4
  24. meerschaum/actions/start.py +135 -14
  25. meerschaum/actions/stop.py +36 -3
  26. meerschaum/actions/sync.py +6 -1
  27. meerschaum/api/__init__.py +35 -13
  28. meerschaum/api/_events.py +2 -2
  29. meerschaum/api/_oauth2.py +47 -4
  30. meerschaum/api/dash/callbacks/dashboard.py +29 -0
  31. meerschaum/api/dash/callbacks/jobs.py +3 -2
  32. meerschaum/api/dash/callbacks/login.py +10 -1
  33. meerschaum/api/dash/callbacks/register.py +9 -2
  34. meerschaum/api/dash/pages/login.py +2 -2
  35. meerschaum/api/dash/pipes.py +72 -36
  36. meerschaum/api/dash/webterm.py +14 -6
  37. meerschaum/api/models/_pipes.py +7 -1
  38. meerschaum/api/resources/static/js/terminado.js +3 -0
  39. meerschaum/api/resources/static/js/xterm-addon-unicode11.js +2 -0
  40. meerschaum/api/resources/templates/termpage.html +1 -0
  41. meerschaum/api/routes/_jobs.py +23 -11
  42. meerschaum/api/routes/_login.py +73 -5
  43. meerschaum/api/routes/_pipes.py +6 -4
  44. meerschaum/api/routes/_webterm.py +3 -3
  45. meerschaum/config/__init__.py +60 -13
  46. meerschaum/config/_default.py +89 -61
  47. meerschaum/config/_edit.py +10 -8
  48. meerschaum/config/_formatting.py +2 -0
  49. meerschaum/config/_patch.py +4 -2
  50. meerschaum/config/_paths.py +127 -12
  51. meerschaum/config/_read_config.py +32 -12
  52. meerschaum/config/_version.py +1 -1
  53. meerschaum/config/environment.py +262 -0
  54. meerschaum/config/stack/__init__.py +7 -5
  55. meerschaum/connectors/_Connector.py +1 -2
  56. meerschaum/connectors/__init__.py +37 -2
  57. meerschaum/connectors/api/_APIConnector.py +1 -1
  58. meerschaum/connectors/api/_jobs.py +11 -0
  59. meerschaum/connectors/api/_pipes.py +7 -1
  60. meerschaum/connectors/instance/_plugins.py +9 -1
  61. meerschaum/connectors/instance/_tokens.py +20 -3
  62. meerschaum/connectors/instance/_users.py +8 -1
  63. meerschaum/connectors/parse.py +1 -1
  64. meerschaum/connectors/sql/_create_engine.py +3 -0
  65. meerschaum/connectors/sql/_pipes.py +93 -79
  66. meerschaum/connectors/sql/_users.py +8 -1
  67. meerschaum/connectors/valkey/_ValkeyConnector.py +3 -3
  68. meerschaum/connectors/valkey/_pipes.py +7 -5
  69. meerschaum/core/Pipe/__init__.py +45 -71
  70. meerschaum/core/Pipe/_attributes.py +66 -90
  71. meerschaum/core/Pipe/_cache.py +555 -0
  72. meerschaum/core/Pipe/_clear.py +0 -11
  73. meerschaum/core/Pipe/_data.py +0 -50
  74. meerschaum/core/Pipe/_deduplicate.py +0 -13
  75. meerschaum/core/Pipe/_delete.py +12 -21
  76. meerschaum/core/Pipe/_drop.py +11 -23
  77. meerschaum/core/Pipe/_dtypes.py +1 -1
  78. meerschaum/core/Pipe/_index.py +8 -14
  79. meerschaum/core/Pipe/_sync.py +12 -18
  80. meerschaum/core/Plugin/_Plugin.py +7 -1
  81. meerschaum/core/Token/_Token.py +1 -1
  82. meerschaum/core/User/_User.py +1 -2
  83. meerschaum/jobs/_Executor.py +88 -4
  84. meerschaum/jobs/_Job.py +146 -36
  85. meerschaum/jobs/systemd.py +7 -2
  86. meerschaum/plugins/__init__.py +277 -81
  87. meerschaum/utils/daemon/Daemon.py +197 -42
  88. meerschaum/utils/daemon/FileDescriptorInterceptor.py +0 -1
  89. meerschaum/utils/daemon/RotatingFile.py +63 -36
  90. meerschaum/utils/daemon/StdinFile.py +53 -13
  91. meerschaum/utils/daemon/__init__.py +18 -5
  92. meerschaum/utils/daemon/_names.py +6 -3
  93. meerschaum/utils/debug.py +34 -4
  94. meerschaum/utils/dtypes/__init__.py +5 -1
  95. meerschaum/utils/formatting/__init__.py +4 -1
  96. meerschaum/utils/formatting/_jobs.py +1 -1
  97. meerschaum/utils/formatting/_pipes.py +47 -46
  98. meerschaum/utils/formatting/_shell.py +33 -9
  99. meerschaum/utils/misc.py +22 -38
  100. meerschaum/utils/packages/__init__.py +15 -13
  101. meerschaum/utils/packages/_packages.py +1 -0
  102. meerschaum/utils/pipes.py +33 -5
  103. meerschaum/utils/process.py +1 -1
  104. meerschaum/utils/prompt.py +172 -143
  105. meerschaum/utils/sql.py +12 -2
  106. meerschaum/utils/threading.py +42 -0
  107. meerschaum/utils/venv/__init__.py +2 -0
  108. meerschaum/utils/warnings.py +19 -13
  109. {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/METADATA +3 -1
  110. {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/RECORD +116 -110
  111. meerschaum/config/_environment.py +0 -145
  112. {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/WHEEL +0 -0
  113. {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/entry_points.txt +0 -0
  114. {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/licenses/LICENSE +0 -0
  115. {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/licenses/NOTICE +0 -0
  116. {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/top_level.txt +0 -0
  117. {meerschaum-3.0.0rc4.dist-info → meerschaum-3.0.0rc8.dist-info}/zip-safe +0 -0
@@ -7,6 +7,8 @@ Start subsystems (API server, logging daemon, etc.).
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
+
11
+ import meerschaum as mrsm
10
12
  from meerschaum.utils.typing import SuccessTuple, Optional, List, Any, Union, Dict
11
13
 
12
14
 
@@ -25,6 +27,7 @@ def start(
25
27
  'webterm': _start_webterm,
26
28
  'connectors': _start_connectors,
27
29
  'pipeline': _start_pipeline,
30
+ 'daemons': _start_daemons,
28
31
  }
29
32
  return choose_subaction(action, options, **kw)
30
33
 
@@ -308,6 +311,7 @@ def _start_gui(
308
311
  action: Optional[List[str]] = None,
309
312
  mrsm_instance: Optional[str] = None,
310
313
  port: Optional[int] = None,
314
+ webterm_port: Optional[int] = None,
311
315
  debug: bool = False,
312
316
  **kw
313
317
  ) -> SuccessTuple:
@@ -321,7 +325,6 @@ def _start_gui(
321
325
 
322
326
  from meerschaum.utils.venv import venv_exec
323
327
  from meerschaum.utils.packages import attempt_import
324
- from meerschaum.utils.warnings import warn
325
328
  from meerschaum.utils.debug import dprint
326
329
  from meerschaum.utils.networking import find_open_ports
327
330
  from meerschaum.connectors.parse import parse_instance_keys
@@ -329,9 +332,10 @@ def _start_gui(
329
332
  webview, requests = attempt_import('webview', 'requests')
330
333
 
331
334
  success, msg = True, "Success"
332
- host = '127.0.0.1'
333
- if port is None:
334
- port = 8765
335
+ host = mrsm.get_config('webterm', 'host')
336
+ port = port or webterm_port
337
+ if not port:
338
+ port = mrsm.get_config('webterm', 'port')
335
339
 
336
340
  if not is_webterm_running(host, port, session_id='mrsm'):
337
341
  port = next(find_open_ports(port, 9000))
@@ -369,7 +373,7 @@ def _start_gui(
369
373
  break
370
374
  except Exception as e:
371
375
  if debug:
372
- dprint(e)
376
+ dprint(str(e))
373
377
  continue
374
378
  if starting_up is False:
375
379
  return False, f"The webterm failed to start within {timeout} seconds."
@@ -417,7 +421,6 @@ def _start_webterm(
417
421
  - `-i`, '--instance'
418
422
  The default instance to use for the Webterm shell.
419
423
  """
420
- import uuid
421
424
  from meerschaum._internal.term import get_webterm_app_and_manager, tornado_ioloop
422
425
  from meerschaum._internal.term.tools import (
423
426
  is_webterm_running,
@@ -426,15 +429,17 @@ def _start_webterm(
426
429
  )
427
430
  from meerschaum.utils.networking import find_open_ports
428
431
  from meerschaum.utils.warnings import info
432
+ from meerschaum.config.paths import WEBTERM_INTERNAL_RESOURCES_PATH
429
433
 
430
434
  if host is None:
431
- host = '127.0.0.1'
435
+ host = mrsm.get_config('api', 'webterm', 'host')
432
436
  if port is None:
433
- port = 8765
437
+ port = mrsm.get_config('api', 'webterm', 'port')
434
438
  if sysargs is None:
435
439
  sysargs = ['start', 'webterm']
436
440
  session_id = 'mrsm'
437
- tornado_app, term_manager = get_webterm_app_and_manager(instance_keys=mrsm_instance)
441
+
442
+ env_path = WEBTERM_INTERNAL_RESOURCES_PATH / (str(port) + '.json')
438
443
 
439
444
  if is_webterm_running(host, port, session_id=session_id):
440
445
  if force:
@@ -445,11 +450,18 @@ def _start_webterm(
445
450
  + " Include `-f` to start another server on a new port\n"
446
451
  + " or specify a different port with `-p`."
447
452
  )
453
+
454
+ tornado_app, term_manager = get_webterm_app_and_manager(
455
+ instance_keys=mrsm_instance,
456
+ port=port,
457
+ env_path=env_path,
458
+ )
448
459
  if not nopretty:
449
460
  info(
450
461
  f"Starting the webterm at http://{host}:{port}/webterm/{session_id} ..."
451
462
  "\n Press CTRL+C to quit."
452
463
  )
464
+
453
465
  tornado_app.listen(port, host)
454
466
  loop = tornado_ioloop.IOLoop.instance()
455
467
  try:
@@ -462,10 +474,15 @@ def _start_webterm(
462
474
  term_manager.shutdown()
463
475
  loop.close()
464
476
 
465
- sessions = get_mrsm_tmux_sessions()
477
+ sessions = get_mrsm_tmux_sessions(port=port)
466
478
  for session in sessions:
467
479
  kill_tmux_session(session)
468
480
 
481
+ try:
482
+ env_path.unlink()
483
+ except Exception:
484
+ pass
485
+
469
486
  return True, "Success"
470
487
 
471
488
 
@@ -483,7 +500,6 @@ def _start_connectors(
483
500
  from meerschaum.connectors.parse import parse_instance_keys
484
501
  from meerschaum.utils.pool import get_pool
485
502
  from meerschaum.utils.warnings import warn
486
- from meerschaum.utils.formatting import pprint
487
503
  from meerschaum.utils.misc import items_str
488
504
  if action is None:
489
505
  action = []
@@ -575,7 +591,6 @@ def _start_pipeline(
575
591
  """
576
592
  import json
577
593
  import time
578
- import sys
579
594
  from meerschaum._internal.entry import entry
580
595
  from meerschaum.utils.warnings import info, warn
581
596
  from meerschaum.utils.misc import is_int
@@ -630,7 +645,7 @@ def _start_pipeline(
630
645
  def do_entry() -> None:
631
646
  nonlocal success, msg, proc
632
647
  if timeout_seconds is None:
633
- success, msg = entry(sub_args_line, _patch_args=patch_args)
648
+ success, msg = entry(sub_args_line, _patch_args=patch_args, _use_cli_daemon=False)
634
649
  return
635
650
 
636
651
  sub_args_line_escaped = sub_args_line.replace("'", "<QUOTE>")
@@ -640,7 +655,7 @@ def _start_pipeline(
640
655
  "from meerschaum._internal.entry import entry\n\n"
641
656
  f"sub_args_line = '{sub_args_line_escaped}'.replace(\"<QUOTE>\", \"'\")\n"
642
657
  f"patch_args = json.loads('{patch_args_escaped_str}'.replace('<QUOTE>', \"'\"))\n"
643
- "success, msg = entry(sub_args_line, _patch_args=patch_args)\n"
658
+ "success, msg = entry(sub_args_line, _patch_args=patch_args, _use_cli_daemon=False)\n"
644
659
  f"print('{fence_begin}' + json.dumps((success, msg)) + '{fence_end}')"
645
660
  )
646
661
  proc = venv_exec(src, venv=None, as_proc=True)
@@ -688,6 +703,112 @@ def _start_pipeline(
688
703
  return success, msg
689
704
 
690
705
 
706
+ def _start_daemons(
707
+ timeout_seconds: Union[int, float, None] = None,
708
+ yes: bool = False,
709
+ force: bool = False,
710
+ noask: bool = False,
711
+ debug: bool = False,
712
+ **kwargs
713
+ ) -> SuccessTuple:
714
+ """
715
+ Start the Meerschaum CLI daemon processes.
716
+ """
717
+ from meerschaum.utils.warnings import warn, dprint
718
+ from meerschaum._internal.cli.daemons import (
719
+ get_cli_daemon,
720
+ get_cli_lock_path,
721
+ )
722
+ from meerschaum._internal.cli.workers import (
723
+ get_existing_cli_workers,
724
+ get_existing_cli_worker_indices,
725
+ )
726
+ from meerschaum.utils.prompt import yes_no
727
+ from meerschaum.actions import actions
728
+
729
+ workers = get_existing_cli_workers()
730
+ if not workers:
731
+ if debug:
732
+ dprint("No daemons are running, spawning a new process...")
733
+ workers = [get_cli_daemon()]
734
+
735
+ accepted_restart = False
736
+ any_daemons_are_running = any((worker.job.status == 'running') for worker in workers)
737
+ lock_paths = [get_cli_lock_path(ix) for ix in get_existing_cli_worker_indices()]
738
+ any_locks_exist = any(lock_path.exists() for lock_path in lock_paths)
739
+
740
+ if any_locks_exist:
741
+ warn(
742
+ "Locks are currently held by the CLI daemons.\n"
743
+ "Run again with `--force` to remove the locks.",
744
+ stack=False,
745
+ )
746
+
747
+ if not force:
748
+ return False, "Actions are currently running."
749
+
750
+ for lock_path in lock_paths:
751
+ try:
752
+ if lock_path.exists():
753
+ lock_path.unlink()
754
+ except Exception as e:
755
+ warn(f"Failed to release lock:\n{e}")
756
+
757
+ if any_daemons_are_running:
758
+ accepted_restart = force or yes_no(
759
+ "Restart running CLI daemons?",
760
+ yes=yes,
761
+ noask=noask,
762
+ default='n',
763
+ )
764
+
765
+ if not accepted_restart:
766
+ return True, "Daemons are already running; nothing to do."
767
+
768
+ stop_success, stop_msg = actions['stop'](
769
+ ['daemons'],
770
+ timeout_seconds=timeout_seconds,
771
+ debug=debug,
772
+ )
773
+ if not stop_success:
774
+ return stop_success, stop_msg
775
+
776
+ worker = get_cli_daemon()
777
+
778
+ if debug:
779
+ dprint("Cleaning up CLI daemon...")
780
+ cleanup_success, cleanup_msg = worker.cleanup()
781
+ if not cleanup_success:
782
+ return cleanup_success, cleanup_msg
783
+
784
+ if debug:
785
+ dprint("Starting CLI daemon...")
786
+
787
+ start_success, start_msg = worker.job.start()
788
+ if not start_success:
789
+ return start_success, start_msg
790
+
791
+ return True, "Success"
792
+
793
+
794
+ def _start_worker(action: Optional[List[str]] = None, **kwargs: Any) -> SuccessTuple:
795
+ """
796
+ Start a CLI worker process. This is intended for internal use only.
797
+ """
798
+ from meerschaum._internal.cli.workers import ActionWorker
799
+ from meerschaum.utils.misc import is_int
800
+
801
+ if not action:
802
+ return False, "No worker index is provided."
803
+
804
+ if not is_int(action[0]):
805
+ return False, "Invalid worker index."
806
+
807
+ ix = int(action[0])
808
+ worker = ActionWorker(ix)
809
+ return worker.run()
810
+
811
+
691
812
  ### NOTE: This must be the final statement of the module.
692
813
  ### Any subactions added below these lines will not
693
814
  ### be added to the `help` docstring.
@@ -7,7 +7,7 @@ Stop running jobs that were started with `-d` or `start job`.
7
7
  """
8
8
 
9
9
  from __future__ import annotations
10
- from meerschaum.utils.typing import Optional, List, SuccessTuple, Any
10
+ from meerschaum.utils.typing import Optional, List, SuccessTuple, Any, Union
11
11
 
12
12
 
13
13
  def stop(action: Optional[List[str]] = None, **kw) -> SuccessTuple:
@@ -17,6 +17,7 @@ def stop(action: Optional[List[str]] = None, **kw) -> SuccessTuple:
17
17
  from meerschaum.actions import choose_subaction
18
18
  options = {
19
19
  'jobs': _stop_jobs,
20
+ 'daemons': _stop_daemons,
20
21
  }
21
22
  return choose_subaction(action, options, **kw)
22
23
 
@@ -53,7 +54,7 @@ def _complete_stop(
53
54
  return options[sub](action=action, **kw)
54
55
 
55
56
  from meerschaum._internal.shell import default_action_completer
56
- return default_action_completer(action=(['start'] + action), **kw)
57
+ return default_action_completer(action=(['stop'] + action), **kw)
57
58
 
58
59
 
59
60
  def _stop_jobs(
@@ -113,7 +114,7 @@ def _stop_jobs(
113
114
 
114
115
  if not jobs_to_stop:
115
116
  if jobs:
116
- return True, "The selected jobs are currently running."
117
+ return True, "The selected jobs are currently stopped."
117
118
  return False, "No running, paused or restarting jobs to stop."
118
119
 
119
120
  if not action:
@@ -163,6 +164,38 @@ def _stop_jobs(
163
164
  return success, msg
164
165
 
165
166
 
167
+ def _stop_daemons(
168
+ timeout_seconds: Union[int, float, None] = None,
169
+ debug: bool = False,
170
+ **kwargs
171
+ ) -> SuccessTuple:
172
+ """
173
+ Stop the Meerschaum CLI daemon.
174
+ """
175
+ import shutil
176
+ from meerschaum._internal.cli.workers import get_existing_cli_workers
177
+ from meerschaum.config.paths import CLI_RESOURCES_PATH
178
+ workers = get_existing_cli_workers()
179
+
180
+ for worker in workers:
181
+ stop_success, stop_msg = worker.job.stop(timeout_seconds=timeout_seconds, debug=debug)
182
+ if not stop_success:
183
+ return stop_success, stop_msg
184
+
185
+ cleanup_success, cleanup_msg = worker.cleanup(debug=debug)
186
+ if not cleanup_success:
187
+ return cleanup_success, cleanup_msg
188
+
189
+ try:
190
+ if CLI_RESOURCES_PATH.exists():
191
+ shutil.rmtree(CLI_RESOURCES_PATH)
192
+ CLI_RESOURCES_PATH.mkdir(parents=True, exist_ok=True)
193
+ except Exception as e:
194
+ return False, f"Failed to clean up CLI resources directory.\n{e}"
195
+
196
+ return True, "Success"
197
+
198
+
166
199
  ### NOTE: This must be the final statement of the module.
167
200
  ### Any subactions added below these lines will not
168
201
  ### be added to the `help` docstring.
@@ -289,6 +289,7 @@ def _sync_pipes(
289
289
  from meerschaum.utils.formatting import print_pipes_results
290
290
  from meerschaum._internal.static import STATIC_CONFIG
291
291
  from meerschaum.utils.misc import interval_str
292
+ from meerschaum.utils.daemon import running_in_daemon
292
293
 
293
294
  noninteractive_val = os.environ.get(STATIC_CONFIG['environment']['noninteractive'], None)
294
295
  noninteractive = str(noninteractive_val).lower() in ('1', 'true', 'yes')
@@ -301,7 +302,11 @@ def _sync_pipes(
301
302
  cooldown = 2 * (min_seconds + 1)
302
303
  success_pipes, failure_pipes = [], []
303
304
  while run:
304
- _progress = progress() if shell and not noninteractive else None
305
+ _progress = (
306
+ progress()
307
+ if (shell and not noninteractive and not running_in_daemon())
308
+ else None
309
+ )
305
310
  cm = _progress if _progress is not None else contextlib.nullcontext()
306
311
 
307
312
  lap_begin = time.perf_counter()
@@ -21,7 +21,6 @@ from meerschaum.config._paths import API_UVICORN_CONFIG_PATH, API_UVICORN_RESOUR
21
21
  from meerschaum.plugins import _api_plugins
22
22
  from meerschaum.utils.warnings import warn, dprint
23
23
  from meerschaum.utils.threading import RLock
24
- from meerschaum.utils.misc import is_pipe_registered
25
24
  from meerschaum.connectors.parse import parse_instance_keys
26
25
 
27
26
  from meerschaum import __version__ as version
@@ -67,16 +66,13 @@ from meerschaum.api._exceptions import APIPermissionError
67
66
  uvicorn_config_path = API_UVICORN_RESOURCES_PATH / SERVER_ID / 'config.json'
68
67
 
69
68
  uvicorn_config = None
70
- sys_config = get_config('system', 'api')
71
- permissions_config = get_config('system', 'api', 'permissions')
69
+ sys_config = get_config('api')
70
+ permissions_config = get_config('api', 'permissions')
72
71
 
73
72
  def get_uvicorn_config() -> Dict[str, Any]:
74
73
  """Read the Uvicorn configuration JSON and return a dictionary."""
75
74
  global uvicorn_config
76
75
  import json
77
- runtime = os.environ.get(STATIC_CONFIG['environment']['runtime'], None)
78
- if runtime == 'api':
79
- return get_config('system', 'api', 'uvicorn')
80
76
  _uvicorn_config = uvicorn_config
81
77
  with _locks['uvicorn_config']:
82
78
  if uvicorn_config is None:
@@ -85,6 +81,8 @@ def get_uvicorn_config() -> Dict[str, Any]:
85
81
  uvicorn_config = json.load(f)
86
82
  _uvicorn_config = uvicorn_config
87
83
  except Exception:
84
+ import traceback
85
+ traceback.print_exc()
88
86
  _uvicorn_config = sys_config.get('uvicorn', None)
89
87
 
90
88
  if _uvicorn_config is None:
@@ -102,6 +100,10 @@ production = get_uvicorn_config().get('production', False)
102
100
  _include_dash = (not no_dash)
103
101
  _include_webterm = (not no_webterm) and _include_dash
104
102
  docs_enabled = not production or sys_config.get('endpoints', {}).get('docs_in_production', True)
103
+ webterm_port = (
104
+ get_uvicorn_config().get('webterm_port', None)
105
+ or mrsm.get_config('api', 'webterm', 'port')
106
+ )
105
107
 
106
108
  default_instance_keys = None
107
109
  _instance_connectors = defaultdict(lambda: None)
@@ -129,7 +131,7 @@ def get_api_connector(instance_keys: Optional[str] = None):
129
131
  )
130
132
  found_match: bool = False
131
133
  for allowed_keys_pattern in allowed_instance_keys:
132
- if fnmatch(instance_keys, allowed_keys_pattern):
134
+ if fnmatch(str(instance_keys), allowed_keys_pattern):
133
135
  found_match = True
134
136
  break
135
137
  if not found_match:
@@ -141,7 +143,9 @@ def get_api_connector(instance_keys: Optional[str] = None):
141
143
  if _instance_connectors[instance_keys] is None:
142
144
  try:
143
145
  is_valid_connector = True
144
- _instance_connectors[instance_keys] = parse_instance_keys(instance_keys, debug=debug)
146
+ instance_connector = parse_instance_keys(instance_keys, debug=debug)
147
+ instance_connector._cache_connector = get_cache_connector()
148
+ _instance_connectors[instance_keys] = instance_connector
145
149
  except Exception:
146
150
  is_valid_connector = False
147
151
 
@@ -168,7 +172,7 @@ def get_cache_connector(connector_keys: Optional[str] = None):
168
172
  return None
169
173
 
170
174
  connector_keys = connector_keys or get_config(
171
- 'system', 'api', 'cache', 'connector',
175
+ 'api', 'cache', 'connector',
172
176
  warn=False,
173
177
  )
174
178
  if connector_keys is None:
@@ -196,7 +200,11 @@ def pipes(instance_keys: Optional[str] = None, refresh: bool = False) -> PipesDi
196
200
  with _locks['pipes-' + instance_keys]:
197
201
  pipes = _instance_pipes[instance_keys]
198
202
  if pipes is None or refresh:
199
- pipes = _get_pipes(mrsm_instance=instance_keys)
203
+ pipes = _get_pipes(
204
+ mrsm_instance=instance_keys,
205
+ cache=True,
206
+ cache_connector_keys=get_cache_connector(),
207
+ )
200
208
  _instance_pipes[instance_keys] = pipes
201
209
  return pipes
202
210
 
@@ -218,9 +226,23 @@ def get_pipe(
218
226
  detail="Unable to serve any pipes with connector keys `mrsm` over the API.",
219
227
  )
220
228
 
221
- pipe = mrsm.Pipe(connector_keys, metric_key, location_key, mrsm_instance=instance_keys)
222
- if is_pipe_registered(pipe, pipes(instance_keys, refresh=False)):
223
- return pipes(instance_keys, refresh=refresh)[connector_keys][metric_key][location_key]
229
+ pipes_dict = pipes(instance_keys)
230
+ if (
231
+ not refresh
232
+ and connector_keys in pipes_dict
233
+ and metric_key in pipes_dict[connector_keys]
234
+ and location_key in pipes_dict[connector_keys][metric_key]
235
+ ):
236
+ return pipes_dict[connector_keys][metric_key][location_key]
237
+
238
+ pipe = mrsm.Pipe(
239
+ connector_keys,
240
+ metric_key,
241
+ location_key,
242
+ mrsm_instance=instance_keys,
243
+ cache=True,
244
+ cache_connector_keys=get_cache_connector(),
245
+ )
224
246
  return pipe
225
247
 
226
248
 
meerschaum/api/_events.py CHANGED
@@ -8,13 +8,13 @@ Declare FastAPI events in this module (startup, shutdown, etc.).
8
8
 
9
9
  import sys
10
10
  import os
11
- import time
12
11
  from meerschaum.api import (
13
12
  app,
14
13
  get_api_connector,
15
14
  get_cache_connector,
16
15
  get_uvicorn_config,
17
16
  debug,
17
+ webterm_port,
18
18
  no_dash,
19
19
  _include_dash,
20
20
  _include_webterm,
@@ -75,7 +75,7 @@ async def startup():
75
75
  try:
76
76
  if _include_webterm:
77
77
  from meerschaum.api.dash.webterm import start_webterm
78
- start_webterm()
78
+ start_webterm(webterm_port=webterm_port)
79
79
 
80
80
  connected = retry_connect(
81
81
  get_api_connector(),
meerschaum/api/_oauth2.py CHANGED
@@ -21,6 +21,7 @@ from meerschaum.core import User, Token
21
21
  fastapi, starlette = attempt_import('fastapi', 'starlette', lazy=False, check_update=CHECK_UPDATE)
22
22
  fastapi_responses = attempt_import('fastapi.responses', lazy=False, check_update=CHECK_UPDATE)
23
23
  fastapi_login = attempt_import('fastapi_login', check_update=CHECK_UPDATE)
24
+ jose_jwt, jose_exceptions = attempt_import('jose.jwt', 'jose.exceptions', lazy=False, check_update=CHECK_UPDATE)
24
25
  from fastapi import Depends, HTTPException, Request
25
26
  from starlette import status
26
27
 
@@ -91,6 +92,7 @@ async def load_user_or_token(
91
92
  status_code=status.HTTP_401_UNAUTHORIZED,
92
93
  detail="Not authenticated.",
93
94
  )
95
+
94
96
  authorization = authorization.replace('Basic ', '').replace('Bearer ', '')
95
97
  if not authorization.startswith('mrsm-key:'):
96
98
  if not users:
@@ -98,12 +100,15 @@ async def load_user_or_token(
98
100
  status=status.HTTP_401_UNAUTHORIZED,
99
101
  detail="Users not authenticated for this endpoint.",
100
102
  )
103
+
101
104
  return await manager(request)
105
+
102
106
  if not tokens:
103
107
  raise HTTPException(
104
108
  status_code=status.HTTP_401_UNAUTHORIZED,
105
109
  detail="Tokens not authenticated for this endpoint.",
106
110
  )
111
+
107
112
  return get_token_from_authorization(authorization)
108
113
 
109
114
 
@@ -112,6 +117,7 @@ def ScopedAuth(scopes: List[str]):
112
117
  Dependency factory for authenticating with either a user session or a scoped token.
113
118
  """
114
119
  async def _authenticate(
120
+ request: Request,
115
121
  user_or_token: Union[User, Token, None] = Depends(
116
122
  load_user_or_token,
117
123
  ),
@@ -126,12 +132,45 @@ def ScopedAuth(scopes: List[str]):
126
132
  headers={"WWW-Authenticate": "Basic"},
127
133
  )
128
134
 
129
- fresh_scopes = user_or_token.get_scopes(refresh=True, debug=debug)
130
- if '*' in fresh_scopes:
135
+ authorization = request.headers.get('authorization', request.headers.get('Authorization', None))
136
+ is_long_lived = authorization and 'mrsm-key:' in authorization
137
+
138
+ current_scopes = []
139
+ ### For long-lived API tokens, always hit the database.
140
+ if is_long_lived:
141
+ current_scopes = user_or_token.get_scopes(refresh=True, debug=debug)
142
+
143
+ ### For JWTs, trust the scopes in the token.
144
+ else:
145
+ if not authorization:
146
+ # This should be caught by `load_user_or_token` but we can be safe.
147
+ raise HTTPException(
148
+ status_code=status.HTTP_401_UNAUTHORIZED,
149
+ detail="Not authenticated.",
150
+ )
151
+
152
+ scheme, _, token_str = authorization.partition(' ')
153
+ if not token_str or scheme.lower() != 'bearer':
154
+ raise HTTPException(
155
+ status_code=status.HTTP_401_UNAUTHORIZED,
156
+ detail="Unsupported authentication scheme.",
157
+ )
158
+
159
+ try:
160
+ payload = jose_jwt.decode(token_str, SECRET, algorithms=['HS256'])
161
+ current_scopes = payload.get('scopes', [])
162
+ except jose_exceptions.JWTError:
163
+ raise HTTPException(
164
+ status_code=status.HTTP_401_UNAUTHORIZED,
165
+ detail="Invalid access token.",
166
+ headers={"WWW-Authenticate": "Bearer"},
167
+ )
168
+
169
+ if '*' in current_scopes:
131
170
  return user_or_token
132
171
 
133
172
  for scope in scopes:
134
- if scope not in fresh_scopes:
173
+ if scope not in current_scopes:
135
174
  raise HTTPException(
136
175
  status_code=status.HTTP_403_FORBIDDEN,
137
176
  detail=f"Missing required scope: '{scope}'",
@@ -159,4 +198,8 @@ def generate_secret_key() -> bytes:
159
198
 
160
199
 
161
200
  SECRET = generate_secret_key()
162
- manager = LoginManager(SECRET, token_url=endpoints['login'])
201
+ manager = LoginManager(
202
+ SECRET,
203
+ token_url=endpoints['login'],
204
+ scopes=STATIC_CONFIG['tokens']['scopes'],
205
+ )
@@ -1193,3 +1193,32 @@ def toggle_pages_offcanvas(n_clicks: Optional[int], is_open: bool):
1193
1193
  if n_clicks:
1194
1194
  return not is_open, pages_children
1195
1195
  return is_open, pages_children
1196
+
1197
+
1198
+ @dash_app.callback(
1199
+ Output({'type': 'calculate-rowcount-div', 'index': MATCH}, 'children'),
1200
+ Input({'type': 'calculate-rowcount-button', 'index': MATCH}, 'n_clicks'),
1201
+ prevent_initial_call=True,
1202
+ )
1203
+ def calculate_rowcount_button_click(n_clicks: int):
1204
+ """
1205
+ Calculate the rowcount for the pipe.
1206
+ """
1207
+ if not n_clicks:
1208
+ raise PreventUpdate
1209
+
1210
+ triggered = dash.callback_context.triggered
1211
+ if triggered[0]['value'] is None:
1212
+ raise PreventUpdate
1213
+
1214
+ pipe = pipe_from_ctx(triggered, 'n_clicks')
1215
+ if pipe is None:
1216
+ raise PreventUpdate
1217
+
1218
+ try:
1219
+ rowcount = pipe.get_rowcount(debug=debug)
1220
+ return f"{rowcount:,}"
1221
+ except Exception as e:
1222
+ return (
1223
+ alert_from_success_tuple((False, f"Failed to calculate row count: {e}"))
1224
+ )
@@ -13,6 +13,7 @@ import time
13
13
  import traceback
14
14
  from datetime import datetime, timezone
15
15
 
16
+ import meerschaum as mrsm
16
17
  from meerschaum.jobs import get_jobs
17
18
  from meerschaum.utils.typing import Optional, Dict, Any
18
19
  from meerschaum.api import CHECK_UPDATE
@@ -136,12 +137,12 @@ def manage_job_button_click(
136
137
  old_status = job.status
137
138
  try:
138
139
  success, msg = manage_functions[manage_job_action]()
139
- except Exception as e:
140
+ except Exception:
140
141
  success, msg = False, traceback.format_exc()
141
142
 
142
143
  ### Wait for a status change before building the elements.
143
144
  timeout_seconds = 1.0
144
- check_interval_seconds = 0.01
145
+ check_interval_seconds = mrsm.get_config('system', 'cli', 'refresh_seconds')
145
146
  begin = time.perf_counter()
146
147
  while (time.perf_counter() - begin) < timeout_seconds:
147
148
  if job.status != old_status:
@@ -16,11 +16,14 @@ from meerschaum.utils.typing import Optional
16
16
  dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
17
17
  from dash.exceptions import PreventUpdate
18
18
  from dash.dependencies import Input, Output, State
19
+
19
20
  from meerschaum.api.dash import dash_app, debug, pipes, _get_pipes
20
21
  from meerschaum.api.dash.sessions import set_session
21
22
  from meerschaum.api.dash.connectors import get_web_connector
22
23
  from meerschaum.api.routes._login import login
23
24
  from meerschaum.api.dash.components import alert_from_success_tuple
25
+ from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
26
+ from meerschaum._internal.static import STATIC_CONFIG
24
27
  from fastapi_login.exceptions import InvalidCredentialsException
25
28
  from fastapi.exceptions import HTTPException
26
29
  dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
@@ -73,7 +76,13 @@ def login_button_click(
73
76
  raise PreventUpdate
74
77
 
75
78
  try:
76
- _ = login({'username': username, 'password': password})
79
+ form = CustomOAuth2PasswordRequestForm(
80
+ grant_type='password',
81
+ username=username,
82
+ password=password,
83
+ scope=' '.join(STATIC_CONFIG['tokens']['scopes'])
84
+ )
85
+ _ = login(form)
77
86
  session_id = str(uuid.uuid4())
78
87
  session_data = {
79
88
  'session-id': session_id,