meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +6 -1
- meerschaum/__main__.py +9 -9
- meerschaum/_internal/arguments/__init__.py +1 -1
- meerschaum/_internal/arguments/_parse_arguments.py +72 -6
- meerschaum/_internal/arguments/_parser.py +45 -15
- meerschaum/_internal/docs/index.py +265 -8
- meerschaum/_internal/entry.py +167 -37
- meerschaum/_internal/shell/Shell.py +290 -99
- meerschaum/_internal/shell/updates.py +175 -0
- meerschaum/actions/__init__.py +29 -17
- meerschaum/actions/api.py +12 -12
- meerschaum/actions/attach.py +113 -0
- meerschaum/actions/copy.py +68 -41
- meerschaum/actions/delete.py +112 -50
- meerschaum/actions/edit.py +3 -3
- meerschaum/actions/install.py +40 -32
- meerschaum/actions/pause.py +44 -27
- meerschaum/actions/register.py +19 -5
- meerschaum/actions/restart.py +107 -0
- meerschaum/actions/show.py +130 -159
- meerschaum/actions/start.py +161 -100
- meerschaum/actions/stop.py +78 -42
- meerschaum/actions/sync.py +3 -3
- meerschaum/actions/upgrade.py +28 -36
- meerschaum/api/_events.py +25 -1
- meerschaum/api/_oauth2.py +2 -0
- meerschaum/api/_websockets.py +2 -2
- meerschaum/api/dash/callbacks/jobs.py +36 -44
- meerschaum/api/dash/jobs.py +89 -78
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +148 -17
- meerschaum/api/routes/_jobs.py +407 -0
- meerschaum/api/routes/_pipes.py +25 -25
- meerschaum/config/_default.py +1 -0
- meerschaum/config/_formatting.py +1 -0
- meerschaum/config/_jobs.py +1 -1
- meerschaum/config/_paths.py +11 -0
- meerschaum/config/_shell.py +84 -67
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +18 -0
- meerschaum/connectors/Connector.py +13 -7
- meerschaum/connectors/__init__.py +28 -15
- meerschaum/connectors/api/APIConnector.py +27 -1
- meerschaum/connectors/api/_actions.py +71 -6
- meerschaum/connectors/api/_jobs.py +368 -0
- meerschaum/connectors/api/_misc.py +1 -1
- meerschaum/connectors/api/_pipes.py +85 -84
- meerschaum/connectors/api/_request.py +13 -9
- meerschaum/connectors/parse.py +27 -15
- meerschaum/core/Pipe/_bootstrap.py +16 -8
- meerschaum/core/Pipe/_sync.py +3 -0
- meerschaum/jobs/_Executor.py +69 -0
- meerschaum/jobs/_Job.py +899 -0
- meerschaum/jobs/__init__.py +396 -0
- meerschaum/jobs/systemd.py +694 -0
- meerschaum/plugins/__init__.py +97 -12
- meerschaum/utils/daemon/Daemon.py +352 -147
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +19 -10
- meerschaum/utils/daemon/RotatingFile.py +22 -8
- meerschaum/utils/daemon/StdinFile.py +121 -0
- meerschaum/utils/daemon/__init__.py +42 -27
- meerschaum/utils/daemon/_names.py +15 -13
- meerschaum/utils/formatting/__init__.py +83 -37
- meerschaum/utils/formatting/_jobs.py +146 -55
- meerschaum/utils/formatting/_shell.py +6 -0
- meerschaum/utils/misc.py +41 -22
- meerschaum/utils/packages/__init__.py +21 -15
- meerschaum/utils/packages/_packages.py +9 -6
- meerschaum/utils/process.py +9 -9
- meerschaum/utils/prompt.py +20 -7
- meerschaum/utils/schedule.py +21 -15
- meerschaum/utils/venv/__init__.py +2 -2
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/METADATA +22 -25
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/RECORD +80 -70
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.6.dist-info → meerschaum-2.3.0.dist-info}/zip-safe +0 -0
meerschaum/actions/show.py
CHANGED
@@ -11,9 +11,9 @@ import meerschaum as mrsm
|
|
11
11
|
from meerschaum.utils.typing import SuccessTuple, Union, Sequence, Any, Optional, List, Dict, Tuple
|
12
12
|
|
13
13
|
def show(
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
action: Optional[List[str]] = None,
|
15
|
+
**kw: Any
|
16
|
+
) -> SuccessTuple:
|
17
17
|
"""Show elements of a certain type.
|
18
18
|
|
19
19
|
Command:
|
@@ -54,7 +54,7 @@ def _complete_show(
|
|
54
54
|
"""
|
55
55
|
Override the default Meerschaum `complete_` function.
|
56
56
|
"""
|
57
|
-
from meerschaum.actions.
|
57
|
+
from meerschaum.actions.delete import _complete_delete_jobs
|
58
58
|
|
59
59
|
if action is None:
|
60
60
|
action = []
|
@@ -65,10 +65,10 @@ def _complete_show(
|
|
65
65
|
'config' : _complete_show_config,
|
66
66
|
'package' : _complete_show_packages,
|
67
67
|
'packages' : _complete_show_packages,
|
68
|
-
'job' :
|
69
|
-
'jobs' :
|
70
|
-
'log' :
|
71
|
-
'logs' :
|
68
|
+
'job' : _complete_delete_jobs,
|
69
|
+
'jobs' : _complete_delete_jobs,
|
70
|
+
'log' : _complete_delete_jobs,
|
71
|
+
'logs' : _complete_delete_jobs,
|
72
72
|
}
|
73
73
|
|
74
74
|
if (
|
@@ -155,10 +155,10 @@ def _complete_show_config(action: Optional[List[str]] = None, **kw : Any):
|
|
155
155
|
|
156
156
|
|
157
157
|
def _show_pipes(
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
158
|
+
nopretty: bool = False,
|
159
|
+
debug: bool = False,
|
160
|
+
**kw: Any
|
161
|
+
) -> SuccessTuple:
|
162
162
|
"""
|
163
163
|
Print a stylized tree of available Meerschaum pipes.
|
164
164
|
Respects global ANSI and UNICODE settings.
|
@@ -170,7 +170,7 @@ def _show_pipes(
|
|
170
170
|
pipes = get_pipes(debug=debug, **kw)
|
171
171
|
|
172
172
|
if len(pipes) == 0:
|
173
|
-
return
|
173
|
+
return True, "No pipes to show."
|
174
174
|
|
175
175
|
if len(flatten_pipes_dict(pipes)) == 1:
|
176
176
|
return flatten_pipes_dict(pipes)[0].show(debug=debug, nopretty=nopretty, **kw)
|
@@ -552,16 +552,19 @@ def _complete_show_packages(
|
|
552
552
|
|
553
553
|
def _show_jobs(
|
554
554
|
action: Optional[List[str]] = None,
|
555
|
+
executor_keys: Optional[str] = None,
|
555
556
|
nopretty: bool = False,
|
557
|
+
debug: bool = False,
|
556
558
|
**kw: Any
|
557
559
|
) -> SuccessTuple:
|
558
560
|
"""
|
559
561
|
Show the currently running and stopped jobs.
|
560
562
|
"""
|
561
|
-
from meerschaum.
|
563
|
+
from meerschaum.jobs import get_filtered_jobs
|
562
564
|
from meerschaum.utils.formatting._jobs import pprint_jobs
|
563
|
-
|
564
|
-
|
565
|
+
|
566
|
+
jobs = get_filtered_jobs(executor_keys, action, debug=debug)
|
567
|
+
if not jobs:
|
565
568
|
if not action and not nopretty:
|
566
569
|
from meerschaum.utils.warnings import info
|
567
570
|
info('No running or stopped jobs.')
|
@@ -572,13 +575,15 @@ def _show_jobs(
|
|
572
575
|
" - start api -d\n" +
|
573
576
|
" - start job sync pipes --loop"
|
574
577
|
)
|
575
|
-
return
|
576
|
-
|
578
|
+
return True, "No jobs to show."
|
579
|
+
|
580
|
+
pprint_jobs(jobs, nopretty=nopretty)
|
577
581
|
return True, "Success"
|
578
582
|
|
579
583
|
|
580
584
|
def _show_logs(
|
581
585
|
action: Optional[List[str]] = None,
|
586
|
+
executor_keys: Optional[str] = None,
|
582
587
|
nopretty: bool = False,
|
583
588
|
**kw
|
584
589
|
) -> SuccessTuple:
|
@@ -594,179 +599,145 @@ def _show_logs(
|
|
594
599
|
`show logs myjob myotherjob`
|
595
600
|
"""
|
596
601
|
import os, pathlib, random, asyncio
|
602
|
+
from functools import partial
|
597
603
|
from datetime import datetime, timezone
|
598
604
|
from meerschaum.utils.packages import attempt_import, import_rich
|
599
|
-
from meerschaum.
|
605
|
+
from meerschaum.jobs import get_filtered_jobs, Job
|
600
606
|
from meerschaum.utils.warnings import warn, info
|
601
607
|
from meerschaum.utils.formatting import get_console, ANSI, UNICODE
|
602
608
|
from meerschaum.utils.misc import tail
|
603
609
|
from meerschaum.config._paths import LOGS_RESOURCES_PATH
|
604
610
|
from meerschaum.config import get_config
|
611
|
+
rich = import_rich()
|
612
|
+
rich_text = attempt_import('rich.text')
|
613
|
+
|
605
614
|
if not ANSI:
|
606
615
|
info = print
|
607
616
|
colors = get_config('jobs', 'logs', 'colors')
|
608
617
|
timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format')
|
609
618
|
follow_timestamp_format = get_config('jobs', 'logs', 'timestamps', 'follow_format')
|
610
|
-
|
619
|
+
|
620
|
+
jobs = get_filtered_jobs(executor_keys, action)
|
611
621
|
now = datetime.now(timezone.utc)
|
612
622
|
now_str = now.strftime(timestamp_format)
|
613
623
|
now_follow_str = now.strftime(follow_timestamp_format)
|
614
624
|
|
615
|
-
def build_buffer_spaces(
|
625
|
+
def build_buffer_spaces(_jobs) -> Dict[str, str]:
|
616
626
|
max_len_id = (
|
617
|
-
max(len(
|
618
|
-
) if
|
627
|
+
max(len(name) for name in _jobs) + 1
|
628
|
+
) if _jobs else 0
|
619
629
|
buffer_len = max(
|
620
630
|
get_config('jobs', 'logs', 'min_buffer_len'),
|
621
631
|
max_len_id
|
622
632
|
)
|
623
633
|
return {
|
624
|
-
|
625
|
-
for
|
634
|
+
name: ' ' * (buffer_len - len(name))
|
635
|
+
for name in _jobs
|
626
636
|
}
|
627
637
|
|
628
|
-
def build_job_colors(
|
629
|
-
return {
|
630
|
-
|
631
|
-
buffer_spaces = build_buffer_spaces(
|
632
|
-
job_colors = build_job_colors(
|
633
|
-
|
634
|
-
def get_buffer_spaces(
|
635
|
-
nonlocal buffer_spaces,
|
636
|
-
if
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
661
|
-
|
662
|
-
|
663
|
-
try:
|
664
|
-
line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
|
665
|
-
previous_line_timestamp = line_timestamp
|
666
|
-
except Exception as e:
|
667
|
-
line_timestamp = None
|
668
|
-
if line_timestamp:
|
669
|
-
line = line[(len(now_str) + 3):]
|
670
|
-
else:
|
671
|
-
line_timestamp = previous_line_timestamp
|
672
|
-
|
673
|
-
if len(line) == 0 or line == '\n':
|
674
|
-
return
|
675
|
-
|
676
|
-
text = rich_text.Text(daemon.daemon_id)
|
677
|
-
line_prefix = (
|
678
|
-
get_buffer_spaces(daemon.daemon_id)
|
679
|
-
+ (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
|
680
|
-
+ ' | '
|
681
|
-
)
|
682
|
-
text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
|
683
|
-
if ANSI:
|
684
|
-
text.stylize(
|
685
|
-
get_job_colors(daemon.daemon_id),
|
686
|
-
0,
|
687
|
-
len(daemon.daemon_id) + len(line_prefix)
|
688
|
-
)
|
689
|
-
get_console().print(text)
|
690
|
-
|
691
|
-
|
692
|
-
def print_log_lines(daemon):
|
693
|
-
for line in daemon.readlines():
|
694
|
-
print_job_line(daemon, line)
|
695
|
-
|
696
|
-
def seek_back_offset(d) -> bool:
|
697
|
-
if d.log_offset_path.exists():
|
698
|
-
d.log_offset_path.unlink()
|
699
|
-
|
700
|
-
latest_subfile_path = d.rotating_log.get_latest_subfile_path()
|
701
|
-
latest_subfile_index = d.rotating_log.get_latest_subfile_index()
|
702
|
-
|
703
|
-
### Sometimes the latest file is empty.
|
704
|
-
if os.stat(latest_subfile_path).st_size == 0 and latest_subfile_index > 0:
|
705
|
-
latest_subfile_index -= 1
|
706
|
-
latest_subfile_path = d.rotating_log.get_subfile_path_from_index(
|
707
|
-
latest_subfile_index
|
708
|
-
)
|
709
|
-
|
710
|
-
with open(latest_subfile_path, 'r', encoding='utf-8') as f:
|
711
|
-
latest_lines = f.readlines()
|
638
|
+
def build_job_colors(_jobs, _old_job_colors=None) -> Dict[str, str]:
|
639
|
+
return {name: colors[i % len(colors)] for i, name in enumerate(_jobs)}
|
640
|
+
|
641
|
+
buffer_spaces = build_buffer_spaces(jobs)
|
642
|
+
job_colors = build_job_colors(jobs)
|
643
|
+
|
644
|
+
def get_buffer_spaces(name):
|
645
|
+
nonlocal buffer_spaces, jobs
|
646
|
+
if name not in buffer_spaces:
|
647
|
+
if name not in jobs:
|
648
|
+
jobs = get_filtered_jobs(executor_keys, action)
|
649
|
+
buffer_spaces = build_buffer_spaces(jobs)
|
650
|
+
return buffer_spaces[name] or ' '
|
651
|
+
|
652
|
+
def get_job_colors(name):
|
653
|
+
nonlocal job_colors, jobs
|
654
|
+
if name not in job_colors:
|
655
|
+
if name not in jobs:
|
656
|
+
jobs = get_filtered_jobs(executor_keys, action)
|
657
|
+
job_colors = build_job_colors(jobs)
|
658
|
+
return job_colors[name]
|
659
|
+
|
660
|
+
previous_line_timestamp = None
|
661
|
+
def print_job_line(job, line):
|
662
|
+
nonlocal previous_line_timestamp
|
663
|
+
date_prefix_str = line[:len(now_str)]
|
664
|
+
try:
|
665
|
+
line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
|
666
|
+
previous_line_timestamp = line_timestamp
|
667
|
+
except Exception as e:
|
668
|
+
line_timestamp = None
|
669
|
+
if line_timestamp:
|
670
|
+
line = line[(len(now_str) + 3):]
|
671
|
+
else:
|
672
|
+
line_timestamp = previous_line_timestamp
|
712
673
|
|
713
|
-
|
714
|
-
|
715
|
-
backup_index = len(''.join(latest_lines)) - positions_to_rewind
|
674
|
+
if len(line) == 0 or line == '\n':
|
675
|
+
return
|
716
676
|
|
717
|
-
|
718
|
-
|
719
|
-
|
677
|
+
text = rich_text.Text(job.name)
|
678
|
+
line_prefix = (
|
679
|
+
get_buffer_spaces(job.name)
|
680
|
+
+ (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
|
681
|
+
+ ' | '
|
682
|
+
)
|
683
|
+
text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
|
684
|
+
if ANSI:
|
685
|
+
text.stylize(
|
686
|
+
get_job_colors(job.name),
|
687
|
+
0,
|
688
|
+
len(job.name) + len(line_prefix)
|
720
689
|
)
|
721
|
-
|
722
|
-
|
690
|
+
get_console().print(text)
|
691
|
+
|
692
|
+
stop_event = asyncio.Event()
|
693
|
+
job_tasks = {}
|
694
|
+
job_stop_events = {}
|
695
|
+
|
696
|
+
async def refresh_job_tasks():
|
697
|
+
nonlocal job_tasks, jobs
|
698
|
+
while not stop_event.is_set():
|
699
|
+
jobs = get_filtered_jobs(executor_keys, action)
|
700
|
+
for name, job in jobs.items():
|
701
|
+
if name not in job_tasks:
|
702
|
+
job_stop_events[name] = asyncio.Event()
|
703
|
+
job_tasks[name] = asyncio.create_task(
|
704
|
+
job.monitor_logs_async(
|
705
|
+
partial(print_job_line, job),
|
706
|
+
stop_event=job_stop_events[name],
|
707
|
+
accept_input=False,
|
708
|
+
stop_on_exit=False,
|
709
|
+
)
|
710
|
+
)
|
711
|
+
|
712
|
+
for name, task in [(k, v) for k, v in job_tasks.items()]:
|
713
|
+
if name not in jobs:
|
714
|
+
job_stop_events[name].set()
|
715
|
+
task.cancel()
|
723
716
|
|
724
|
-
daemons_being_watched = {
|
725
|
-
d.daemon_id: d
|
726
|
-
for d in daemons
|
727
|
-
}
|
728
|
-
for d in daemons:
|
729
|
-
seek_back_offset(d)
|
730
|
-
print_log_lines(d)
|
731
|
-
|
732
|
-
_quit = False
|
733
|
-
def watch_logs():
|
734
|
-
for changes in watchfiles.watch(LOGS_RESOURCES_PATH):
|
735
|
-
if _quit:
|
736
|
-
return
|
737
|
-
for change in changes:
|
738
|
-
file_path_str = change[1]
|
739
|
-
if '.log' not in file_path_str or '.log.offset' in file_path_str:
|
740
|
-
continue
|
741
|
-
file_path = pathlib.Path(file_path_str)
|
742
|
-
if not file_path.exists():
|
743
|
-
continue
|
744
|
-
daemon_id = file_path.name.split('.log')[0]
|
745
|
-
if daemon_id not in watch_daemon_ids and action:
|
746
|
-
continue
|
747
717
|
try:
|
748
|
-
|
749
|
-
except
|
750
|
-
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
print_log_lines(daemon)
|
718
|
+
await task
|
719
|
+
except asyncio.CancelledError:
|
720
|
+
pass
|
721
|
+
finally:
|
722
|
+
_ = job_tasks.pop(name, None)
|
723
|
+
_ = job_stop_events.pop(name, None)
|
755
724
|
|
756
|
-
|
757
|
-
watch_logs()
|
758
|
-
except KeyboardInterrupt as ki:
|
759
|
-
_quit = True
|
760
|
-
|
761
|
-
def print_nopretty_log_text():
|
762
|
-
for d in daemons:
|
763
|
-
log_text = d.log_text
|
764
|
-
print(d.daemon_id)
|
765
|
-
print(log_text)
|
725
|
+
await asyncio.sleep(1)
|
766
726
|
|
767
|
-
|
768
|
-
|
727
|
+
async def gather_tasks():
|
728
|
+
tasks = [refresh_job_tasks()] + list(job_tasks.values())
|
729
|
+
await asyncio.gather(*tasks)
|
769
730
|
|
731
|
+
if not nopretty:
|
732
|
+
info("Watching logs...")
|
733
|
+
try:
|
734
|
+
asyncio.run(gather_tasks())
|
735
|
+
except KeyboardInterrupt:
|
736
|
+
pass
|
737
|
+
else:
|
738
|
+
for name, job in jobs.items():
|
739
|
+
print(f'\n-*-\nMRSM_JOB: {name}\n-*-')
|
740
|
+
print(job.get_logs())
|
770
741
|
return True, "Success"
|
771
742
|
|
772
743
|
|