meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. meerschaum/__init__.py +6 -1
  2. meerschaum/__main__.py +9 -9
  3. meerschaum/_internal/arguments/__init__.py +1 -1
  4. meerschaum/_internal/arguments/_parse_arguments.py +72 -6
  5. meerschaum/_internal/arguments/_parser.py +45 -15
  6. meerschaum/_internal/docs/index.py +265 -8
  7. meerschaum/_internal/entry.py +167 -37
  8. meerschaum/_internal/shell/Shell.py +290 -99
  9. meerschaum/_internal/shell/updates.py +175 -0
  10. meerschaum/actions/__init__.py +29 -17
  11. meerschaum/actions/api.py +12 -12
  12. meerschaum/actions/attach.py +113 -0
  13. meerschaum/actions/copy.py +68 -41
  14. meerschaum/actions/delete.py +112 -50
  15. meerschaum/actions/edit.py +3 -3
  16. meerschaum/actions/install.py +40 -32
  17. meerschaum/actions/pause.py +44 -27
  18. meerschaum/actions/register.py +19 -5
  19. meerschaum/actions/restart.py +107 -0
  20. meerschaum/actions/show.py +130 -159
  21. meerschaum/actions/start.py +161 -100
  22. meerschaum/actions/stop.py +78 -42
  23. meerschaum/actions/sync.py +3 -3
  24. meerschaum/actions/upgrade.py +28 -36
  25. meerschaum/api/_events.py +25 -1
  26. meerschaum/api/_oauth2.py +2 -0
  27. meerschaum/api/_websockets.py +2 -2
  28. meerschaum/api/dash/callbacks/jobs.py +36 -44
  29. meerschaum/api/dash/jobs.py +89 -78
  30. meerschaum/api/routes/__init__.py +1 -0
  31. meerschaum/api/routes/_actions.py +148 -17
  32. meerschaum/api/routes/_jobs.py +407 -0
  33. meerschaum/api/routes/_pipes.py +25 -25
  34. meerschaum/config/_default.py +1 -0
  35. meerschaum/config/_formatting.py +1 -0
  36. meerschaum/config/_jobs.py +1 -1
  37. meerschaum/config/_paths.py +11 -0
  38. meerschaum/config/_shell.py +84 -67
  39. meerschaum/config/_version.py +1 -1
  40. meerschaum/config/static/__init__.py +18 -0
  41. meerschaum/connectors/Connector.py +13 -7
  42. meerschaum/connectors/__init__.py +28 -15
  43. meerschaum/connectors/api/APIConnector.py +27 -1
  44. meerschaum/connectors/api/_actions.py +71 -6
  45. meerschaum/connectors/api/_jobs.py +368 -0
  46. meerschaum/connectors/api/_misc.py +1 -1
  47. meerschaum/connectors/api/_pipes.py +85 -84
  48. meerschaum/connectors/api/_request.py +13 -9
  49. meerschaum/connectors/parse.py +27 -15
  50. meerschaum/core/Pipe/_bootstrap.py +16 -8
  51. meerschaum/core/Pipe/_sync.py +3 -0
  52. meerschaum/jobs/_Executor.py +69 -0
  53. meerschaum/jobs/_Job.py +899 -0
  54. meerschaum/jobs/__init__.py +396 -0
  55. meerschaum/jobs/systemd.py +694 -0
  56. meerschaum/plugins/__init__.py +97 -12
  57. meerschaum/utils/daemon/Daemon.py +352 -147
  58. meerschaum/utils/daemon/FileDescriptorInterceptor.py +19 -10
  59. meerschaum/utils/daemon/RotatingFile.py +22 -8
  60. meerschaum/utils/daemon/StdinFile.py +121 -0
  61. meerschaum/utils/daemon/__init__.py +42 -27
  62. meerschaum/utils/daemon/_names.py +15 -13
  63. meerschaum/utils/formatting/__init__.py +83 -37
  64. meerschaum/utils/formatting/_jobs.py +146 -55
  65. meerschaum/utils/formatting/_shell.py +6 -0
  66. meerschaum/utils/misc.py +41 -22
  67. meerschaum/utils/packages/__init__.py +21 -15
  68. meerschaum/utils/packages/_packages.py +9 -6
  69. meerschaum/utils/process.py +9 -9
  70. meerschaum/utils/prompt.py +20 -7
  71. meerschaum/utils/schedule.py +21 -15
  72. meerschaum/utils/venv/__init__.py +2 -2
  73. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/METADATA +22 -25
  74. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/RECORD +80 -70
  75. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/WHEEL +1 -1
  76. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/LICENSE +0 -0
  77. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/NOTICE +0 -0
  78. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/entry_points.txt +0 -0
  79. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/top_level.txt +0 -0
  80. {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/zip-safe +0 -0
@@ -11,9 +11,9 @@ import meerschaum as mrsm
11
11
  from meerschaum.utils.typing import SuccessTuple, Union, Sequence, Any, Optional, List, Dict, Tuple
12
12
 
13
13
  def show(
14
- action: Optional[List[str]] = None,
15
- **kw: Any
16
- ) -> SuccessTuple:
14
+ action: Optional[List[str]] = None,
15
+ **kw: Any
16
+ ) -> SuccessTuple:
17
17
  """Show elements of a certain type.
18
18
 
19
19
  Command:
@@ -54,7 +54,7 @@ def _complete_show(
54
54
  """
55
55
  Override the default Meerschaum `complete_` function.
56
56
  """
57
- from meerschaum.actions.start import _complete_start_jobs
57
+ from meerschaum.actions.delete import _complete_delete_jobs
58
58
 
59
59
  if action is None:
60
60
  action = []
@@ -65,10 +65,10 @@ def _complete_show(
65
65
  'config' : _complete_show_config,
66
66
  'package' : _complete_show_packages,
67
67
  'packages' : _complete_show_packages,
68
- 'job' : _complete_start_jobs,
69
- 'jobs' : _complete_start_jobs,
70
- 'log' : _complete_start_jobs,
71
- 'logs' : _complete_start_jobs,
68
+ 'job' : _complete_delete_jobs,
69
+ 'jobs' : _complete_delete_jobs,
70
+ 'log' : _complete_delete_jobs,
71
+ 'logs' : _complete_delete_jobs,
72
72
  }
73
73
 
74
74
  if (
@@ -155,10 +155,10 @@ def _complete_show_config(action: Optional[List[str]] = None, **kw : Any):
155
155
 
156
156
 
157
157
  def _show_pipes(
158
- nopretty: bool = False,
159
- debug: bool = False,
160
- **kw: Any
161
- ) -> SuccessTuple:
158
+ nopretty: bool = False,
159
+ debug: bool = False,
160
+ **kw: Any
161
+ ) -> SuccessTuple:
162
162
  """
163
163
  Print a stylized tree of available Meerschaum pipes.
164
164
  Respects global ANSI and UNICODE settings.
@@ -170,7 +170,7 @@ def _show_pipes(
170
170
  pipes = get_pipes(debug=debug, **kw)
171
171
 
172
172
  if len(pipes) == 0:
173
- return False, "No pipes to show."
173
+ return True, "No pipes to show."
174
174
 
175
175
  if len(flatten_pipes_dict(pipes)) == 1:
176
176
  return flatten_pipes_dict(pipes)[0].show(debug=debug, nopretty=nopretty, **kw)
@@ -552,16 +552,19 @@ def _complete_show_packages(
552
552
 
553
553
  def _show_jobs(
554
554
  action: Optional[List[str]] = None,
555
+ executor_keys: Optional[str] = None,
555
556
  nopretty: bool = False,
557
+ debug: bool = False,
556
558
  **kw: Any
557
559
  ) -> SuccessTuple:
558
560
  """
559
561
  Show the currently running and stopped jobs.
560
562
  """
561
- from meerschaum.utils.daemon import get_filtered_daemons
563
+ from meerschaum.jobs import get_filtered_jobs
562
564
  from meerschaum.utils.formatting._jobs import pprint_jobs
563
- daemons = get_filtered_daemons(action)
564
- if not daemons:
565
+
566
+ jobs = get_filtered_jobs(executor_keys, action, debug=debug)
567
+ if not jobs:
565
568
  if not action and not nopretty:
566
569
  from meerschaum.utils.warnings import info
567
570
  info('No running or stopped jobs.')
@@ -572,13 +575,15 @@ def _show_jobs(
572
575
  " - start api -d\n" +
573
576
  " - start job sync pipes --loop"
574
577
  )
575
- return False, "No jobs to show."
576
- pprint_jobs(daemons, nopretty=nopretty)
578
+ return True, "No jobs to show."
579
+
580
+ pprint_jobs(jobs, nopretty=nopretty)
577
581
  return True, "Success"
578
582
 
579
583
 
580
584
  def _show_logs(
581
585
  action: Optional[List[str]] = None,
586
+ executor_keys: Optional[str] = None,
582
587
  nopretty: bool = False,
583
588
  **kw
584
589
  ) -> SuccessTuple:
@@ -594,179 +599,145 @@ def _show_logs(
594
599
  `show logs myjob myotherjob`
595
600
  """
596
601
  import os, pathlib, random, asyncio
602
+ from functools import partial
597
603
  from datetime import datetime, timezone
598
604
  from meerschaum.utils.packages import attempt_import, import_rich
599
- from meerschaum.utils.daemon import get_filtered_daemons, Daemon
605
+ from meerschaum.jobs import get_filtered_jobs, Job
600
606
  from meerschaum.utils.warnings import warn, info
601
607
  from meerschaum.utils.formatting import get_console, ANSI, UNICODE
602
608
  from meerschaum.utils.misc import tail
603
609
  from meerschaum.config._paths import LOGS_RESOURCES_PATH
604
610
  from meerschaum.config import get_config
611
+ rich = import_rich()
612
+ rich_text = attempt_import('rich.text')
613
+
605
614
  if not ANSI:
606
615
  info = print
607
616
  colors = get_config('jobs', 'logs', 'colors')
608
617
  timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format')
609
618
  follow_timestamp_format = get_config('jobs', 'logs', 'timestamps', 'follow_format')
610
- daemons = get_filtered_daemons(action)
619
+
620
+ jobs = get_filtered_jobs(executor_keys, action)
611
621
  now = datetime.now(timezone.utc)
612
622
  now_str = now.strftime(timestamp_format)
613
623
  now_follow_str = now.strftime(follow_timestamp_format)
614
624
 
615
- def build_buffer_spaces(daemons) -> Dict[str, str]:
625
+ def build_buffer_spaces(_jobs) -> Dict[str, str]:
616
626
  max_len_id = (
617
- max(len(d.daemon_id) for d in daemons) + 1
618
- ) if daemons else 0
627
+ max(len(name) for name in _jobs) + 1
628
+ ) if _jobs else 0
619
629
  buffer_len = max(
620
630
  get_config('jobs', 'logs', 'min_buffer_len'),
621
631
  max_len_id
622
632
  )
623
633
  return {
624
- d.daemon_id: ''.join([' '] * (buffer_len - len(d.daemon_id)))
625
- for d in daemons
634
+ name: ' ' * (buffer_len - len(name))
635
+ for name in _jobs
626
636
  }
627
637
 
628
- def build_job_colors(daemons, _old_job_colors = None) -> Dict[str, str]:
629
- return {d.daemon_id: colors[i % len(colors)] for i, d in enumerate(daemons)}
630
-
631
- buffer_spaces = build_buffer_spaces(daemons)
632
- job_colors = build_job_colors(daemons)
633
-
634
- def get_buffer_spaces(daemon_id):
635
- nonlocal buffer_spaces, daemons
636
- if daemon_id not in buffer_spaces:
637
- d = Daemon(daemon_id=daemon_id)
638
- if d not in daemons:
639
- daemons = get_filtered_daemons(action)
640
- buffer_spaces = build_buffer_spaces(daemons)
641
- return buffer_spaces[daemon_id] or ' '
642
-
643
- def get_job_colors(daemon_id):
644
- nonlocal job_colors, daemons
645
- if daemon_id not in job_colors:
646
- d = Daemon(daemon_id=daemon_id)
647
- if d not in daemons:
648
- daemons = get_filtered_daemons(action)
649
- job_colors = build_job_colors(daemons)
650
- return job_colors[daemon_id]
651
-
652
- def follow_pretty_print():
653
- watchfiles = attempt_import('watchfiles')
654
- rich = import_rich()
655
- rich_text = attempt_import('rich.text')
656
- watch_daemon_ids = {d.daemon_id: d for d in daemons}
657
- info("Watching log files...")
658
-
659
- previous_line_timestamp = None
660
- def print_job_line(daemon, line):
661
- nonlocal previous_line_timestamp
662
- date_prefix_str = line[:len(now_str)]
663
- try:
664
- line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
665
- previous_line_timestamp = line_timestamp
666
- except Exception as e:
667
- line_timestamp = None
668
- if line_timestamp:
669
- line = line[(len(now_str) + 3):]
670
- else:
671
- line_timestamp = previous_line_timestamp
672
-
673
- if len(line) == 0 or line == '\n':
674
- return
675
-
676
- text = rich_text.Text(daemon.daemon_id)
677
- line_prefix = (
678
- get_buffer_spaces(daemon.daemon_id)
679
- + (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
680
- + ' | '
681
- )
682
- text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
683
- if ANSI:
684
- text.stylize(
685
- get_job_colors(daemon.daemon_id),
686
- 0,
687
- len(daemon.daemon_id) + len(line_prefix)
688
- )
689
- get_console().print(text)
690
-
691
-
692
- def print_log_lines(daemon):
693
- for line in daemon.readlines():
694
- print_job_line(daemon, line)
695
-
696
- def seek_back_offset(d) -> bool:
697
- if d.log_offset_path.exists():
698
- d.log_offset_path.unlink()
699
-
700
- latest_subfile_path = d.rotating_log.get_latest_subfile_path()
701
- latest_subfile_index = d.rotating_log.get_latest_subfile_index()
702
-
703
- ### Sometimes the latest file is empty.
704
- if os.stat(latest_subfile_path).st_size == 0 and latest_subfile_index > 0:
705
- latest_subfile_index -= 1
706
- latest_subfile_path = d.rotating_log.get_subfile_path_from_index(
707
- latest_subfile_index
708
- )
709
-
710
- with open(latest_subfile_path, 'r', encoding='utf-8') as f:
711
- latest_lines = f.readlines()
638
+ def build_job_colors(_jobs, _old_job_colors=None) -> Dict[str, str]:
639
+ return {name: colors[i % len(colors)] for i, name in enumerate(_jobs)}
640
+
641
+ buffer_spaces = build_buffer_spaces(jobs)
642
+ job_colors = build_job_colors(jobs)
643
+
644
+ def get_buffer_spaces(name):
645
+ nonlocal buffer_spaces, jobs
646
+ if name not in buffer_spaces:
647
+ if name not in jobs:
648
+ jobs = get_filtered_jobs(executor_keys, action)
649
+ buffer_spaces = build_buffer_spaces(jobs)
650
+ return buffer_spaces[name] or ' '
651
+
652
+ def get_job_colors(name):
653
+ nonlocal job_colors, jobs
654
+ if name not in job_colors:
655
+ if name not in jobs:
656
+ jobs = get_filtered_jobs(executor_keys, action)
657
+ job_colors = build_job_colors(jobs)
658
+ return job_colors[name]
659
+
660
+ previous_line_timestamp = None
661
+ def print_job_line(job, line):
662
+ nonlocal previous_line_timestamp
663
+ date_prefix_str = line[:len(now_str)]
664
+ try:
665
+ line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
666
+ previous_line_timestamp = line_timestamp
667
+ except Exception as e:
668
+ line_timestamp = None
669
+ if line_timestamp:
670
+ line = line[(len(now_str) + 3):]
671
+ else:
672
+ line_timestamp = previous_line_timestamp
712
673
 
713
- lines_to_show = get_config('jobs', 'logs', 'lines_to_show')
714
- positions_to_rewind = len(''.join(latest_lines[(-1 * lines_to_show):]))
715
- backup_index = len(''.join(latest_lines)) - positions_to_rewind
674
+ if len(line) == 0 or line == '\n':
675
+ return
716
676
 
717
- d.rotating_log._cursor = (
718
- latest_subfile_index,
719
- max(0, backup_index)
677
+ text = rich_text.Text(job.name)
678
+ line_prefix = (
679
+ get_buffer_spaces(job.name)
680
+ + (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
681
+ + ' | '
682
+ )
683
+ text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
684
+ if ANSI:
685
+ text.stylize(
686
+ get_job_colors(job.name),
687
+ 0,
688
+ len(job.name) + len(line_prefix)
720
689
  )
721
- d._write_log_offset()
722
- return True
690
+ get_console().print(text)
691
+
692
+ stop_event = asyncio.Event()
693
+ job_tasks = {}
694
+ job_stop_events = {}
695
+
696
+ async def refresh_job_tasks():
697
+ nonlocal job_tasks, jobs
698
+ while not stop_event.is_set():
699
+ jobs = get_filtered_jobs(executor_keys, action)
700
+ for name, job in jobs.items():
701
+ if name not in job_tasks:
702
+ job_stop_events[name] = asyncio.Event()
703
+ job_tasks[name] = asyncio.create_task(
704
+ job.monitor_logs_async(
705
+ partial(print_job_line, job),
706
+ stop_event=job_stop_events[name],
707
+ accept_input=False,
708
+ stop_on_exit=False,
709
+ )
710
+ )
711
+
712
+ for name, task in [(k, v) for k, v in job_tasks.items()]:
713
+ if name not in jobs:
714
+ job_stop_events[name].set()
715
+ task.cancel()
723
716
 
724
- daemons_being_watched = {
725
- d.daemon_id: d
726
- for d in daemons
727
- }
728
- for d in daemons:
729
- seek_back_offset(d)
730
- print_log_lines(d)
731
-
732
- _quit = False
733
- def watch_logs():
734
- for changes in watchfiles.watch(LOGS_RESOURCES_PATH):
735
- if _quit:
736
- return
737
- for change in changes:
738
- file_path_str = change[1]
739
- if '.log' not in file_path_str or '.log.offset' in file_path_str:
740
- continue
741
- file_path = pathlib.Path(file_path_str)
742
- if not file_path.exists():
743
- continue
744
- daemon_id = file_path.name.split('.log')[0]
745
- if daemon_id not in watch_daemon_ids and action:
746
- continue
747
717
  try:
748
- daemon = Daemon(daemon_id=daemon_id)
749
- except Exception as e:
750
- daemon = None
751
- warn(f"Seeing new logs for non-existent job '{daemon_id}'.", stack=False)
752
-
753
- if daemon is not None:
754
- print_log_lines(daemon)
718
+ await task
719
+ except asyncio.CancelledError:
720
+ pass
721
+ finally:
722
+ _ = job_tasks.pop(name, None)
723
+ _ = job_stop_events.pop(name, None)
755
724
 
756
- try:
757
- watch_logs()
758
- except KeyboardInterrupt as ki:
759
- _quit = True
760
-
761
- def print_nopretty_log_text():
762
- for d in daemons:
763
- log_text = d.log_text
764
- print(d.daemon_id)
765
- print(log_text)
725
+ await asyncio.sleep(1)
766
726
 
767
- print_log_text = follow_pretty_print if not nopretty else print_nopretty_log_text
768
- print_log_text()
727
+ async def gather_tasks():
728
+ tasks = [refresh_job_tasks()] + list(job_tasks.values())
729
+ await asyncio.gather(*tasks)
769
730
 
731
+ if not nopretty:
732
+ info("Watching logs...")
733
+ try:
734
+ asyncio.run(gather_tasks())
735
+ except KeyboardInterrupt:
736
+ pass
737
+ else:
738
+ for name, job in jobs.items():
739
+ print(f'\n-*-\nMRSM_JOB: {name}\n-*-')
740
+ print(job.get_logs())
770
741
  return True, "Success"
771
742
 
772
743