meerschaum 2.2.7__py3-none-any.whl → 2.3.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meerschaum/__init__.py +4 -1
- meerschaum/_internal/arguments/_parser.py +44 -15
- meerschaum/_internal/entry.py +22 -1
- meerschaum/_internal/shell/Shell.py +129 -31
- meerschaum/actions/api.py +12 -12
- meerschaum/actions/attach.py +95 -0
- meerschaum/actions/delete.py +35 -26
- meerschaum/actions/show.py +119 -148
- meerschaum/actions/start.py +85 -75
- meerschaum/actions/stop.py +68 -39
- meerschaum/api/_events.py +18 -1
- meerschaum/api/_oauth2.py +2 -0
- meerschaum/api/_websockets.py +2 -2
- meerschaum/api/dash/jobs.py +5 -2
- meerschaum/api/routes/__init__.py +1 -0
- meerschaum/api/routes/_actions.py +122 -44
- meerschaum/api/routes/_jobs.py +340 -0
- meerschaum/api/routes/_pipes.py +5 -5
- meerschaum/config/_default.py +1 -0
- meerschaum/config/_paths.py +1 -0
- meerschaum/config/_shell.py +8 -3
- meerschaum/config/_version.py +1 -1
- meerschaum/config/static/__init__.py +8 -0
- meerschaum/connectors/__init__.py +9 -11
- meerschaum/connectors/api/APIConnector.py +18 -1
- meerschaum/connectors/api/_actions.py +60 -71
- meerschaum/connectors/api/_jobs.py +260 -0
- meerschaum/connectors/parse.py +23 -7
- meerschaum/plugins/__init__.py +89 -5
- meerschaum/utils/daemon/Daemon.py +255 -30
- meerschaum/utils/daemon/FileDescriptorInterceptor.py +5 -5
- meerschaum/utils/daemon/RotatingFile.py +10 -6
- meerschaum/utils/daemon/StdinFile.py +110 -0
- meerschaum/utils/daemon/__init__.py +13 -7
- meerschaum/utils/formatting/__init__.py +2 -1
- meerschaum/utils/formatting/_jobs.py +83 -54
- meerschaum/utils/formatting/_shell.py +6 -0
- meerschaum/utils/jobs/_Job.py +684 -0
- meerschaum/utils/jobs/__init__.py +245 -0
- meerschaum/utils/misc.py +18 -17
- meerschaum/utils/packages/_packages.py +2 -2
- meerschaum/utils/prompt.py +16 -8
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/METADATA +9 -9
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/RECORD +50 -44
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/WHEEL +1 -1
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/LICENSE +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/NOTICE +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/entry_points.txt +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/top_level.txt +0 -0
- {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/zip-safe +0 -0
meerschaum/actions/delete.py
CHANGED
@@ -391,6 +391,7 @@ def _complete_delete_connectors(
|
|
391
391
|
|
392
392
|
def _delete_jobs(
|
393
393
|
action: Optional[List[str]] = None,
|
394
|
+
executor_keys: Optional[str] = None,
|
394
395
|
noask: bool = False,
|
395
396
|
nopretty: bool = False,
|
396
397
|
force: bool = False,
|
@@ -404,8 +405,12 @@ def _delete_jobs(
|
|
404
405
|
If the job is running, ask to kill the job first.
|
405
406
|
|
406
407
|
"""
|
407
|
-
from meerschaum.utils.
|
408
|
-
|
408
|
+
from meerschaum.utils.jobs import (
|
409
|
+
Job,
|
410
|
+
get_running_jobs,
|
411
|
+
get_stopped_jobs,
|
412
|
+
get_filtered_jobs,
|
413
|
+
get_paused_jobs,
|
409
414
|
)
|
410
415
|
from meerschaum.utils.prompt import yes_no
|
411
416
|
from meerschaum.utils.formatting._jobs import pprint_jobs
|
@@ -413,49 +418,53 @@ def _delete_jobs(
|
|
413
418
|
from meerschaum.utils.warnings import warn
|
414
419
|
from meerschaum.utils.misc import items_str
|
415
420
|
from meerschaum.actions import actions
|
416
|
-
|
417
|
-
|
421
|
+
|
422
|
+
jobs = get_filtered_jobs(executor_keys, action, debug=debug)
|
423
|
+
if not jobs:
|
418
424
|
return True, "No jobs to delete; nothing to do."
|
419
425
|
|
420
426
|
_delete_all_jobs = False
|
421
427
|
if not action:
|
422
428
|
if not force:
|
423
|
-
pprint_jobs(
|
429
|
+
pprint_jobs(jobs)
|
424
430
|
if not yes_no(
|
425
431
|
"Delete all jobs? This cannot be undone!",
|
426
432
|
noask=noask, yes=yes, default='n'
|
427
433
|
):
|
428
434
|
return False, "No jobs were deleted."
|
435
|
+
|
429
436
|
_delete_all_jobs = True
|
430
|
-
_running_daemons = get_running_daemons(daemons)
|
431
|
-
_paused_daemons = get_paused_daemons(daemons)
|
432
|
-
_stopped_daemons = get_stopped_daemons(daemons)
|
433
|
-
_to_delete = _stopped_daemons
|
434
437
|
|
435
|
-
|
436
|
-
|
438
|
+
_running_jobs = get_running_jobs(executor_keys, jobs, debug=debug)
|
439
|
+
_paused_jobs = get_paused_jobs(executor_keys, jobs, debug=debug)
|
440
|
+
_stopped_jobs = get_stopped_jobs(executor_keys, jobs, debug=debug)
|
441
|
+
_to_delete = _stopped_jobs
|
442
|
+
|
443
|
+
to_stop_jobs = {**_running_jobs, **_paused_jobs}
|
444
|
+
if to_stop_jobs:
|
437
445
|
clear_screen(debug=debug)
|
438
446
|
if not force:
|
439
|
-
pprint_jobs(
|
447
|
+
pprint_jobs(to_stop_jobs, nopretty=nopretty)
|
440
448
|
if force or yes_no(
|
441
449
|
"Stop these jobs?",
|
442
450
|
default='n', yes=yes, noask=noask
|
443
451
|
):
|
444
452
|
actions['stop'](
|
445
|
-
action
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
453
|
+
action=(['jobs'] + [_name for _name in to_stop_jobs]),
|
454
|
+
executor_keys=executor_keys,
|
455
|
+
nopretty=nopretty,
|
456
|
+
yes=yes,
|
457
|
+
force=force,
|
458
|
+
noask=noask,
|
459
|
+
debug=debug,
|
451
460
|
**kw
|
452
461
|
)
|
453
462
|
### Ensure the running jobs are dead.
|
454
|
-
if
|
463
|
+
if get_running_jobs(executor_keys, jobs, debug=debug):
|
455
464
|
return False, (
|
456
465
|
f"Failed to kill running jobs. Please stop these jobs before deleting."
|
457
466
|
)
|
458
|
-
_to_delete
|
467
|
+
_to_delete.update(to_stop_jobs)
|
459
468
|
|
460
469
|
### User decided not to kill running jobs.
|
461
470
|
else:
|
@@ -473,17 +482,17 @@ def _delete_jobs(
|
|
473
482
|
return False, "No jobs were deleted."
|
474
483
|
|
475
484
|
_deleted = []
|
476
|
-
for
|
477
|
-
|
478
|
-
if
|
479
|
-
warn(f"Failed to delete job '{
|
485
|
+
for name, job in _to_delete.items():
|
486
|
+
delete_success, delete_msg = job.delete()
|
487
|
+
if not delete_success:
|
488
|
+
warn(f"Failed to delete job '{name}'.", stack=False)
|
480
489
|
continue
|
481
|
-
_deleted.append(
|
490
|
+
_deleted.append(name)
|
482
491
|
|
483
492
|
return (
|
484
493
|
len(_deleted) > 0,
|
485
494
|
("Deleted job" + ("s" if len(_deleted) != 1 else '')
|
486
|
-
+ f" {items_str([
|
495
|
+
+ f" {items_str([_name for _name in _deleted])}."),
|
487
496
|
)
|
488
497
|
|
489
498
|
|
meerschaum/actions/show.py
CHANGED
@@ -11,9 +11,9 @@ import meerschaum as mrsm
|
|
11
11
|
from meerschaum.utils.typing import SuccessTuple, Union, Sequence, Any, Optional, List, Dict, Tuple
|
12
12
|
|
13
13
|
def show(
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
action: Optional[List[str]] = None,
|
15
|
+
**kw: Any
|
16
|
+
) -> SuccessTuple:
|
17
17
|
"""Show elements of a certain type.
|
18
18
|
|
19
19
|
Command:
|
@@ -552,16 +552,19 @@ def _complete_show_packages(
|
|
552
552
|
|
553
553
|
def _show_jobs(
|
554
554
|
action: Optional[List[str]] = None,
|
555
|
+
executor_keys: Optional[str] = None,
|
555
556
|
nopretty: bool = False,
|
557
|
+
debug: bool = False,
|
556
558
|
**kw: Any
|
557
559
|
) -> SuccessTuple:
|
558
560
|
"""
|
559
561
|
Show the currently running and stopped jobs.
|
560
562
|
"""
|
561
|
-
from meerschaum.utils.
|
563
|
+
from meerschaum.utils.jobs import get_filtered_jobs
|
562
564
|
from meerschaum.utils.formatting._jobs import pprint_jobs
|
563
|
-
|
564
|
-
|
565
|
+
|
566
|
+
jobs = get_filtered_jobs(executor_keys, action, debug=debug)
|
567
|
+
if not jobs:
|
565
568
|
if not action and not nopretty:
|
566
569
|
from meerschaum.utils.warnings import info
|
567
570
|
info('No running or stopped jobs.')
|
@@ -573,12 +576,14 @@ def _show_jobs(
|
|
573
576
|
" - start job sync pipes --loop"
|
574
577
|
)
|
575
578
|
return False, "No jobs to show."
|
576
|
-
|
579
|
+
|
580
|
+
pprint_jobs(jobs, nopretty=nopretty)
|
577
581
|
return True, "Success"
|
578
582
|
|
579
583
|
|
580
584
|
def _show_logs(
|
581
585
|
action: Optional[List[str]] = None,
|
586
|
+
executor_keys: Optional[str] = None,
|
582
587
|
nopretty: bool = False,
|
583
588
|
**kw
|
584
589
|
) -> SuccessTuple:
|
@@ -594,179 +599,145 @@ def _show_logs(
|
|
594
599
|
`show logs myjob myotherjob`
|
595
600
|
"""
|
596
601
|
import os, pathlib, random, asyncio
|
602
|
+
from functools import partial
|
597
603
|
from datetime import datetime, timezone
|
598
604
|
from meerschaum.utils.packages import attempt_import, import_rich
|
599
|
-
from meerschaum.utils.
|
605
|
+
from meerschaum.utils.jobs import get_filtered_jobs, Job
|
600
606
|
from meerschaum.utils.warnings import warn, info
|
601
607
|
from meerschaum.utils.formatting import get_console, ANSI, UNICODE
|
602
608
|
from meerschaum.utils.misc import tail
|
603
609
|
from meerschaum.config._paths import LOGS_RESOURCES_PATH
|
604
610
|
from meerschaum.config import get_config
|
611
|
+
rich = import_rich()
|
612
|
+
rich_text = attempt_import('rich.text')
|
613
|
+
|
605
614
|
if not ANSI:
|
606
615
|
info = print
|
607
616
|
colors = get_config('jobs', 'logs', 'colors')
|
608
617
|
timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format')
|
609
618
|
follow_timestamp_format = get_config('jobs', 'logs', 'timestamps', 'follow_format')
|
610
|
-
|
619
|
+
|
620
|
+
jobs = get_filtered_jobs(executor_keys, action)
|
611
621
|
now = datetime.now(timezone.utc)
|
612
622
|
now_str = now.strftime(timestamp_format)
|
613
623
|
now_follow_str = now.strftime(follow_timestamp_format)
|
614
624
|
|
615
|
-
def build_buffer_spaces(
|
625
|
+
def build_buffer_spaces(_jobs) -> Dict[str, str]:
|
616
626
|
max_len_id = (
|
617
|
-
max(len(
|
618
|
-
) if
|
627
|
+
max(len(name) for name in _jobs) + 1
|
628
|
+
) if _jobs else 0
|
619
629
|
buffer_len = max(
|
620
630
|
get_config('jobs', 'logs', 'min_buffer_len'),
|
621
631
|
max_len_id
|
622
632
|
)
|
623
633
|
return {
|
624
|
-
|
625
|
-
for
|
634
|
+
name: ' ' * (buffer_len - len(name))
|
635
|
+
for name in _jobs
|
626
636
|
}
|
627
637
|
|
628
|
-
def build_job_colors(
|
629
|
-
return {
|
630
|
-
|
631
|
-
buffer_spaces = build_buffer_spaces(
|
632
|
-
job_colors = build_job_colors(
|
633
|
-
|
634
|
-
def get_buffer_spaces(
|
635
|
-
nonlocal buffer_spaces,
|
636
|
-
if
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
661
|
-
|
662
|
-
|
663
|
-
try:
|
664
|
-
line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
|
665
|
-
previous_line_timestamp = line_timestamp
|
666
|
-
except Exception as e:
|
667
|
-
line_timestamp = None
|
668
|
-
if line_timestamp:
|
669
|
-
line = line[(len(now_str) + 3):]
|
670
|
-
else:
|
671
|
-
line_timestamp = previous_line_timestamp
|
672
|
-
|
673
|
-
if len(line) == 0 or line == '\n':
|
674
|
-
return
|
675
|
-
|
676
|
-
text = rich_text.Text(daemon.daemon_id)
|
677
|
-
line_prefix = (
|
678
|
-
get_buffer_spaces(daemon.daemon_id)
|
679
|
-
+ (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
|
680
|
-
+ ' | '
|
681
|
-
)
|
682
|
-
text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
|
683
|
-
if ANSI:
|
684
|
-
text.stylize(
|
685
|
-
get_job_colors(daemon.daemon_id),
|
686
|
-
0,
|
687
|
-
len(daemon.daemon_id) + len(line_prefix)
|
688
|
-
)
|
689
|
-
get_console().print(text)
|
690
|
-
|
691
|
-
|
692
|
-
def print_log_lines(daemon):
|
693
|
-
for line in daemon.readlines():
|
694
|
-
print_job_line(daemon, line)
|
695
|
-
|
696
|
-
def seek_back_offset(d) -> bool:
|
697
|
-
if d.log_offset_path.exists():
|
698
|
-
d.log_offset_path.unlink()
|
699
|
-
|
700
|
-
latest_subfile_path = d.rotating_log.get_latest_subfile_path()
|
701
|
-
latest_subfile_index = d.rotating_log.get_latest_subfile_index()
|
702
|
-
|
703
|
-
### Sometimes the latest file is empty.
|
704
|
-
if os.stat(latest_subfile_path).st_size == 0 and latest_subfile_index > 0:
|
705
|
-
latest_subfile_index -= 1
|
706
|
-
latest_subfile_path = d.rotating_log.get_subfile_path_from_index(
|
707
|
-
latest_subfile_index
|
708
|
-
)
|
709
|
-
|
710
|
-
with open(latest_subfile_path, 'r', encoding='utf-8') as f:
|
711
|
-
latest_lines = f.readlines()
|
638
|
+
def build_job_colors(_jobs, _old_job_colors=None) -> Dict[str, str]:
|
639
|
+
return {name: colors[i % len(colors)] for i, name in enumerate(_jobs)}
|
640
|
+
|
641
|
+
buffer_spaces = build_buffer_spaces(jobs)
|
642
|
+
job_colors = build_job_colors(jobs)
|
643
|
+
|
644
|
+
def get_buffer_spaces(name):
|
645
|
+
nonlocal buffer_spaces, jobs
|
646
|
+
if name not in buffer_spaces:
|
647
|
+
if name not in jobs:
|
648
|
+
jobs = get_filtered_jobs(executor_keys, action)
|
649
|
+
buffer_spaces = build_buffer_spaces(jobs)
|
650
|
+
return buffer_spaces[name] or ' '
|
651
|
+
|
652
|
+
def get_job_colors(name):
|
653
|
+
nonlocal job_colors, jobs
|
654
|
+
if name not in job_colors:
|
655
|
+
if name not in jobs:
|
656
|
+
jobs = get_filtered_jobs(executor_keys, action)
|
657
|
+
job_colors = build_job_colors(jobs)
|
658
|
+
return job_colors[name]
|
659
|
+
|
660
|
+
previous_line_timestamp = None
|
661
|
+
def print_job_line(job, line):
|
662
|
+
nonlocal previous_line_timestamp
|
663
|
+
date_prefix_str = line[:len(now_str)]
|
664
|
+
try:
|
665
|
+
line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
|
666
|
+
previous_line_timestamp = line_timestamp
|
667
|
+
except Exception as e:
|
668
|
+
line_timestamp = None
|
669
|
+
if line_timestamp:
|
670
|
+
line = line[(len(now_str) + 3):]
|
671
|
+
else:
|
672
|
+
line_timestamp = previous_line_timestamp
|
712
673
|
|
713
|
-
|
714
|
-
|
715
|
-
backup_index = len(''.join(latest_lines)) - positions_to_rewind
|
674
|
+
if len(line) == 0 or line == '\n':
|
675
|
+
return
|
716
676
|
|
717
|
-
|
718
|
-
|
719
|
-
|
677
|
+
text = rich_text.Text(job.name)
|
678
|
+
line_prefix = (
|
679
|
+
get_buffer_spaces(job.name)
|
680
|
+
+ (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
|
681
|
+
+ ' | '
|
682
|
+
)
|
683
|
+
text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
|
684
|
+
if ANSI:
|
685
|
+
text.stylize(
|
686
|
+
get_job_colors(job.name),
|
687
|
+
0,
|
688
|
+
len(job.name) + len(line_prefix)
|
720
689
|
)
|
721
|
-
|
722
|
-
|
690
|
+
get_console().print(text)
|
691
|
+
|
692
|
+
stop_event = asyncio.Event()
|
693
|
+
job_tasks = {}
|
694
|
+
job_stop_events = {}
|
695
|
+
|
696
|
+
async def refresh_job_tasks():
|
697
|
+
nonlocal job_tasks, jobs
|
698
|
+
while not stop_event.is_set():
|
699
|
+
jobs = get_filtered_jobs(executor_keys, action)
|
700
|
+
for name, job in jobs.items():
|
701
|
+
if name not in job_tasks:
|
702
|
+
job_stop_events[name] = asyncio.Event()
|
703
|
+
job_tasks[name] = asyncio.create_task(
|
704
|
+
job.monitor_logs_async(
|
705
|
+
partial(print_job_line, job),
|
706
|
+
stop_event=job_stop_events[name],
|
707
|
+
accept_input=False,
|
708
|
+
stop_on_exit=False,
|
709
|
+
)
|
710
|
+
)
|
711
|
+
|
712
|
+
for name, task in [(k, v) for k, v in job_tasks.items()]:
|
713
|
+
if name not in jobs:
|
714
|
+
job_stop_events[name].set()
|
715
|
+
task.cancel()
|
723
716
|
|
724
|
-
daemons_being_watched = {
|
725
|
-
d.daemon_id: d
|
726
|
-
for d in daemons
|
727
|
-
}
|
728
|
-
for d in daemons:
|
729
|
-
seek_back_offset(d)
|
730
|
-
print_log_lines(d)
|
731
|
-
|
732
|
-
_quit = False
|
733
|
-
def watch_logs():
|
734
|
-
for changes in watchfiles.watch(LOGS_RESOURCES_PATH):
|
735
|
-
if _quit:
|
736
|
-
return
|
737
|
-
for change in changes:
|
738
|
-
file_path_str = change[1]
|
739
|
-
if '.log' not in file_path_str or '.log.offset' in file_path_str:
|
740
|
-
continue
|
741
|
-
file_path = pathlib.Path(file_path_str)
|
742
|
-
if not file_path.exists():
|
743
|
-
continue
|
744
|
-
daemon_id = file_path.name.split('.log')[0]
|
745
|
-
if daemon_id not in watch_daemon_ids and action:
|
746
|
-
continue
|
747
717
|
try:
|
748
|
-
|
749
|
-
except
|
750
|
-
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
print_log_lines(daemon)
|
755
|
-
|
756
|
-
try:
|
757
|
-
watch_logs()
|
758
|
-
except KeyboardInterrupt as ki:
|
759
|
-
_quit = True
|
718
|
+
await task
|
719
|
+
except asyncio.CancelledError:
|
720
|
+
pass
|
721
|
+
finally:
|
722
|
+
_ = job_tasks.pop(name, None)
|
723
|
+
_ = job_stop_events.pop(name, None)
|
760
724
|
|
761
|
-
|
762
|
-
for d in daemons:
|
763
|
-
log_text = d.log_text
|
764
|
-
print(d.daemon_id)
|
765
|
-
print(log_text)
|
725
|
+
await asyncio.sleep(1)
|
766
726
|
|
767
|
-
|
768
|
-
|
727
|
+
async def gather_tasks():
|
728
|
+
tasks = [refresh_job_tasks()] + list(job_tasks.values())
|
729
|
+
await asyncio.gather(*tasks)
|
769
730
|
|
731
|
+
if not nopretty:
|
732
|
+
info("Watching logs...")
|
733
|
+
try:
|
734
|
+
asyncio.run(gather_tasks())
|
735
|
+
except KeyboardInterrupt:
|
736
|
+
pass
|
737
|
+
else:
|
738
|
+
for name, job in jobs.items():
|
739
|
+
print(f'\n-*-\nMRSM_JOB: {name}\n-*-')
|
740
|
+
print(job.get_logs())
|
770
741
|
return True, "Success"
|
771
742
|
|
772
743
|
|