meerschaum 2.2.7__py3-none-any.whl → 2.3.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. meerschaum/__init__.py +4 -1
  2. meerschaum/_internal/arguments/_parser.py +44 -15
  3. meerschaum/_internal/entry.py +22 -1
  4. meerschaum/_internal/shell/Shell.py +129 -31
  5. meerschaum/actions/api.py +12 -12
  6. meerschaum/actions/attach.py +95 -0
  7. meerschaum/actions/delete.py +35 -26
  8. meerschaum/actions/show.py +119 -148
  9. meerschaum/actions/start.py +85 -75
  10. meerschaum/actions/stop.py +68 -39
  11. meerschaum/api/_events.py +18 -1
  12. meerschaum/api/_oauth2.py +2 -0
  13. meerschaum/api/_websockets.py +2 -2
  14. meerschaum/api/dash/jobs.py +5 -2
  15. meerschaum/api/routes/__init__.py +1 -0
  16. meerschaum/api/routes/_actions.py +122 -44
  17. meerschaum/api/routes/_jobs.py +340 -0
  18. meerschaum/api/routes/_pipes.py +5 -5
  19. meerschaum/config/_default.py +1 -0
  20. meerschaum/config/_paths.py +1 -0
  21. meerschaum/config/_shell.py +8 -3
  22. meerschaum/config/_version.py +1 -1
  23. meerschaum/config/static/__init__.py +8 -0
  24. meerschaum/connectors/__init__.py +9 -11
  25. meerschaum/connectors/api/APIConnector.py +18 -1
  26. meerschaum/connectors/api/_actions.py +60 -71
  27. meerschaum/connectors/api/_jobs.py +260 -0
  28. meerschaum/connectors/parse.py +23 -7
  29. meerschaum/plugins/__init__.py +89 -5
  30. meerschaum/utils/daemon/Daemon.py +255 -30
  31. meerschaum/utils/daemon/FileDescriptorInterceptor.py +5 -5
  32. meerschaum/utils/daemon/RotatingFile.py +10 -6
  33. meerschaum/utils/daemon/StdinFile.py +110 -0
  34. meerschaum/utils/daemon/__init__.py +13 -7
  35. meerschaum/utils/formatting/__init__.py +2 -1
  36. meerschaum/utils/formatting/_jobs.py +83 -54
  37. meerschaum/utils/formatting/_shell.py +6 -0
  38. meerschaum/utils/jobs/_Job.py +684 -0
  39. meerschaum/utils/jobs/__init__.py +245 -0
  40. meerschaum/utils/misc.py +18 -17
  41. meerschaum/utils/packages/_packages.py +2 -2
  42. meerschaum/utils/prompt.py +16 -8
  43. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/METADATA +9 -9
  44. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/RECORD +50 -44
  45. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/WHEEL +1 -1
  46. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/LICENSE +0 -0
  47. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/NOTICE +0 -0
  48. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/entry_points.txt +0 -0
  49. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/top_level.txt +0 -0
  50. {meerschaum-2.2.7.dist-info → meerschaum-2.3.0.dev1.dist-info}/zip-safe +0 -0
@@ -391,6 +391,7 @@ def _complete_delete_connectors(
391
391
 
392
392
  def _delete_jobs(
393
393
  action: Optional[List[str]] = None,
394
+ executor_keys: Optional[str] = None,
394
395
  noask: bool = False,
395
396
  nopretty: bool = False,
396
397
  force: bool = False,
@@ -404,8 +405,12 @@ def _delete_jobs(
404
405
  If the job is running, ask to kill the job first.
405
406
 
406
407
  """
407
- from meerschaum.utils.daemon import (
408
- Daemon, get_running_daemons, get_stopped_daemons, get_filtered_daemons, get_paused_daemons
408
+ from meerschaum.utils.jobs import (
409
+ Job,
410
+ get_running_jobs,
411
+ get_stopped_jobs,
412
+ get_filtered_jobs,
413
+ get_paused_jobs,
409
414
  )
410
415
  from meerschaum.utils.prompt import yes_no
411
416
  from meerschaum.utils.formatting._jobs import pprint_jobs
@@ -413,49 +418,53 @@ def _delete_jobs(
413
418
  from meerschaum.utils.warnings import warn
414
419
  from meerschaum.utils.misc import items_str
415
420
  from meerschaum.actions import actions
416
- daemons = get_filtered_daemons(action, warn=(not nopretty))
417
- if not daemons:
421
+
422
+ jobs = get_filtered_jobs(executor_keys, action, debug=debug)
423
+ if not jobs:
418
424
  return True, "No jobs to delete; nothing to do."
419
425
 
420
426
  _delete_all_jobs = False
421
427
  if not action:
422
428
  if not force:
423
- pprint_jobs(daemons)
429
+ pprint_jobs(jobs)
424
430
  if not yes_no(
425
431
  "Delete all jobs? This cannot be undone!",
426
432
  noask=noask, yes=yes, default='n'
427
433
  ):
428
434
  return False, "No jobs were deleted."
435
+
429
436
  _delete_all_jobs = True
430
- _running_daemons = get_running_daemons(daemons)
431
- _paused_daemons = get_paused_daemons(daemons)
432
- _stopped_daemons = get_stopped_daemons(daemons)
433
- _to_delete = _stopped_daemons
434
437
 
435
- to_stop_daemons = _running_daemons + _paused_daemons
436
- if to_stop_daemons:
438
+ _running_jobs = get_running_jobs(executor_keys, jobs, debug=debug)
439
+ _paused_jobs = get_paused_jobs(executor_keys, jobs, debug=debug)
440
+ _stopped_jobs = get_stopped_jobs(executor_keys, jobs, debug=debug)
441
+ _to_delete = _stopped_jobs
442
+
443
+ to_stop_jobs = {**_running_jobs, **_paused_jobs}
444
+ if to_stop_jobs:
437
445
  clear_screen(debug=debug)
438
446
  if not force:
439
- pprint_jobs(to_stop_daemons, nopretty=nopretty)
447
+ pprint_jobs(to_stop_jobs, nopretty=nopretty)
440
448
  if force or yes_no(
441
449
  "Stop these jobs?",
442
450
  default='n', yes=yes, noask=noask
443
451
  ):
444
452
  actions['stop'](
445
- action = (['jobs'] + [d.daemon_id for d in to_stop_daemons]),
446
- nopretty = nopretty,
447
- yes = yes,
448
- force = force,
449
- noask = noask,
450
- debug = debug,
453
+ action=(['jobs'] + [_name for _name in to_stop_jobs]),
454
+ executor_keys=executor_keys,
455
+ nopretty=nopretty,
456
+ yes=yes,
457
+ force=force,
458
+ noask=noask,
459
+ debug=debug,
451
460
  **kw
452
461
  )
453
462
  ### Ensure the running jobs are dead.
454
- if get_running_daemons(daemons):
463
+ if get_running_jobs(executor_keys, jobs, debug=debug):
455
464
  return False, (
456
465
  f"Failed to kill running jobs. Please stop these jobs before deleting."
457
466
  )
458
- _to_delete += to_stop_daemons
467
+ _to_delete.update(to_stop_jobs)
459
468
 
460
469
  ### User decided not to kill running jobs.
461
470
  else:
@@ -473,17 +482,17 @@ def _delete_jobs(
473
482
  return False, "No jobs were deleted."
474
483
 
475
484
  _deleted = []
476
- for d in _to_delete:
477
- d.cleanup()
478
- if d.path.exists() and not nopretty:
479
- warn(f"Failed to delete job '{d.daemon_id}'.", stack=False)
485
+ for name, job in _to_delete.items():
486
+ delete_success, delete_msg = job.delete()
487
+ if not delete_success:
488
+ warn(f"Failed to delete job '{name}'.", stack=False)
480
489
  continue
481
- _deleted.append(d)
490
+ _deleted.append(name)
482
491
 
483
492
  return (
484
493
  len(_deleted) > 0,
485
494
  ("Deleted job" + ("s" if len(_deleted) != 1 else '')
486
- + f" {items_str([d.daemon_id for d in _deleted])}."),
495
+ + f" {items_str([_name for _name in _deleted])}."),
487
496
  )
488
497
 
489
498
 
@@ -11,9 +11,9 @@ import meerschaum as mrsm
11
11
  from meerschaum.utils.typing import SuccessTuple, Union, Sequence, Any, Optional, List, Dict, Tuple
12
12
 
13
13
  def show(
14
- action: Optional[List[str]] = None,
15
- **kw: Any
16
- ) -> SuccessTuple:
14
+ action: Optional[List[str]] = None,
15
+ **kw: Any
16
+ ) -> SuccessTuple:
17
17
  """Show elements of a certain type.
18
18
 
19
19
  Command:
@@ -552,16 +552,19 @@ def _complete_show_packages(
552
552
 
553
553
  def _show_jobs(
554
554
  action: Optional[List[str]] = None,
555
+ executor_keys: Optional[str] = None,
555
556
  nopretty: bool = False,
557
+ debug: bool = False,
556
558
  **kw: Any
557
559
  ) -> SuccessTuple:
558
560
  """
559
561
  Show the currently running and stopped jobs.
560
562
  """
561
- from meerschaum.utils.daemon import get_filtered_daemons
563
+ from meerschaum.utils.jobs import get_filtered_jobs
562
564
  from meerschaum.utils.formatting._jobs import pprint_jobs
563
- daemons = get_filtered_daemons(action)
564
- if not daemons:
565
+
566
+ jobs = get_filtered_jobs(executor_keys, action, debug=debug)
567
+ if not jobs:
565
568
  if not action and not nopretty:
566
569
  from meerschaum.utils.warnings import info
567
570
  info('No running or stopped jobs.')
@@ -573,12 +576,14 @@ def _show_jobs(
573
576
  " - start job sync pipes --loop"
574
577
  )
575
578
  return False, "No jobs to show."
576
- pprint_jobs(daemons, nopretty=nopretty)
579
+
580
+ pprint_jobs(jobs, nopretty=nopretty)
577
581
  return True, "Success"
578
582
 
579
583
 
580
584
  def _show_logs(
581
585
  action: Optional[List[str]] = None,
586
+ executor_keys: Optional[str] = None,
582
587
  nopretty: bool = False,
583
588
  **kw
584
589
  ) -> SuccessTuple:
@@ -594,179 +599,145 @@ def _show_logs(
594
599
  `show logs myjob myotherjob`
595
600
  """
596
601
  import os, pathlib, random, asyncio
602
+ from functools import partial
597
603
  from datetime import datetime, timezone
598
604
  from meerschaum.utils.packages import attempt_import, import_rich
599
- from meerschaum.utils.daemon import get_filtered_daemons, Daemon
605
+ from meerschaum.utils.jobs import get_filtered_jobs, Job
600
606
  from meerschaum.utils.warnings import warn, info
601
607
  from meerschaum.utils.formatting import get_console, ANSI, UNICODE
602
608
  from meerschaum.utils.misc import tail
603
609
  from meerschaum.config._paths import LOGS_RESOURCES_PATH
604
610
  from meerschaum.config import get_config
611
+ rich = import_rich()
612
+ rich_text = attempt_import('rich.text')
613
+
605
614
  if not ANSI:
606
615
  info = print
607
616
  colors = get_config('jobs', 'logs', 'colors')
608
617
  timestamp_format = get_config('jobs', 'logs', 'timestamps', 'format')
609
618
  follow_timestamp_format = get_config('jobs', 'logs', 'timestamps', 'follow_format')
610
- daemons = get_filtered_daemons(action)
619
+
620
+ jobs = get_filtered_jobs(executor_keys, action)
611
621
  now = datetime.now(timezone.utc)
612
622
  now_str = now.strftime(timestamp_format)
613
623
  now_follow_str = now.strftime(follow_timestamp_format)
614
624
 
615
- def build_buffer_spaces(daemons) -> Dict[str, str]:
625
+ def build_buffer_spaces(_jobs) -> Dict[str, str]:
616
626
  max_len_id = (
617
- max(len(d.daemon_id) for d in daemons) + 1
618
- ) if daemons else 0
627
+ max(len(name) for name in _jobs) + 1
628
+ ) if _jobs else 0
619
629
  buffer_len = max(
620
630
  get_config('jobs', 'logs', 'min_buffer_len'),
621
631
  max_len_id
622
632
  )
623
633
  return {
624
- d.daemon_id: ''.join([' '] * (buffer_len - len(d.daemon_id)))
625
- for d in daemons
634
+ name: ' ' * (buffer_len - len(name))
635
+ for name in _jobs
626
636
  }
627
637
 
628
- def build_job_colors(daemons, _old_job_colors = None) -> Dict[str, str]:
629
- return {d.daemon_id: colors[i % len(colors)] for i, d in enumerate(daemons)}
630
-
631
- buffer_spaces = build_buffer_spaces(daemons)
632
- job_colors = build_job_colors(daemons)
633
-
634
- def get_buffer_spaces(daemon_id):
635
- nonlocal buffer_spaces, daemons
636
- if daemon_id not in buffer_spaces:
637
- d = Daemon(daemon_id=daemon_id)
638
- if d not in daemons:
639
- daemons = get_filtered_daemons(action)
640
- buffer_spaces = build_buffer_spaces(daemons)
641
- return buffer_spaces[daemon_id] or ' '
642
-
643
- def get_job_colors(daemon_id):
644
- nonlocal job_colors, daemons
645
- if daemon_id not in job_colors:
646
- d = Daemon(daemon_id=daemon_id)
647
- if d not in daemons:
648
- daemons = get_filtered_daemons(action)
649
- job_colors = build_job_colors(daemons)
650
- return job_colors[daemon_id]
651
-
652
- def follow_pretty_print():
653
- watchfiles = attempt_import('watchfiles')
654
- rich = import_rich()
655
- rich_text = attempt_import('rich.text')
656
- watch_daemon_ids = {d.daemon_id: d for d in daemons}
657
- info("Watching log files...")
658
-
659
- previous_line_timestamp = None
660
- def print_job_line(daemon, line):
661
- nonlocal previous_line_timestamp
662
- date_prefix_str = line[:len(now_str)]
663
- try:
664
- line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
665
- previous_line_timestamp = line_timestamp
666
- except Exception as e:
667
- line_timestamp = None
668
- if line_timestamp:
669
- line = line[(len(now_str) + 3):]
670
- else:
671
- line_timestamp = previous_line_timestamp
672
-
673
- if len(line) == 0 or line == '\n':
674
- return
675
-
676
- text = rich_text.Text(daemon.daemon_id)
677
- line_prefix = (
678
- get_buffer_spaces(daemon.daemon_id)
679
- + (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
680
- + ' | '
681
- )
682
- text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
683
- if ANSI:
684
- text.stylize(
685
- get_job_colors(daemon.daemon_id),
686
- 0,
687
- len(daemon.daemon_id) + len(line_prefix)
688
- )
689
- get_console().print(text)
690
-
691
-
692
- def print_log_lines(daemon):
693
- for line in daemon.readlines():
694
- print_job_line(daemon, line)
695
-
696
- def seek_back_offset(d) -> bool:
697
- if d.log_offset_path.exists():
698
- d.log_offset_path.unlink()
699
-
700
- latest_subfile_path = d.rotating_log.get_latest_subfile_path()
701
- latest_subfile_index = d.rotating_log.get_latest_subfile_index()
702
-
703
- ### Sometimes the latest file is empty.
704
- if os.stat(latest_subfile_path).st_size == 0 and latest_subfile_index > 0:
705
- latest_subfile_index -= 1
706
- latest_subfile_path = d.rotating_log.get_subfile_path_from_index(
707
- latest_subfile_index
708
- )
709
-
710
- with open(latest_subfile_path, 'r', encoding='utf-8') as f:
711
- latest_lines = f.readlines()
638
+ def build_job_colors(_jobs, _old_job_colors=None) -> Dict[str, str]:
639
+ return {name: colors[i % len(colors)] for i, name in enumerate(_jobs)}
640
+
641
+ buffer_spaces = build_buffer_spaces(jobs)
642
+ job_colors = build_job_colors(jobs)
643
+
644
+ def get_buffer_spaces(name):
645
+ nonlocal buffer_spaces, jobs
646
+ if name not in buffer_spaces:
647
+ if name not in jobs:
648
+ jobs = get_filtered_jobs(executor_keys, action)
649
+ buffer_spaces = build_buffer_spaces(jobs)
650
+ return buffer_spaces[name] or ' '
651
+
652
+ def get_job_colors(name):
653
+ nonlocal job_colors, jobs
654
+ if name not in job_colors:
655
+ if name not in jobs:
656
+ jobs = get_filtered_jobs(executor_keys, action)
657
+ job_colors = build_job_colors(jobs)
658
+ return job_colors[name]
659
+
660
+ previous_line_timestamp = None
661
+ def print_job_line(job, line):
662
+ nonlocal previous_line_timestamp
663
+ date_prefix_str = line[:len(now_str)]
664
+ try:
665
+ line_timestamp = datetime.strptime(date_prefix_str, timestamp_format)
666
+ previous_line_timestamp = line_timestamp
667
+ except Exception as e:
668
+ line_timestamp = None
669
+ if line_timestamp:
670
+ line = line[(len(now_str) + 3):]
671
+ else:
672
+ line_timestamp = previous_line_timestamp
712
673
 
713
- lines_to_show = get_config('jobs', 'logs', 'lines_to_show')
714
- positions_to_rewind = len(''.join(latest_lines[(-1 * lines_to_show):]))
715
- backup_index = len(''.join(latest_lines)) - positions_to_rewind
674
+ if len(line) == 0 or line == '\n':
675
+ return
716
676
 
717
- d.rotating_log._cursor = (
718
- latest_subfile_index,
719
- max(0, backup_index)
677
+ text = rich_text.Text(job.name)
678
+ line_prefix = (
679
+ get_buffer_spaces(job.name)
680
+ + (line_timestamp.strftime(follow_timestamp_format) if line_timestamp else '')
681
+ + ' | '
682
+ )
683
+ text.append(line_prefix + (line[:-1] if line[-1] == '\n' else line))
684
+ if ANSI:
685
+ text.stylize(
686
+ get_job_colors(job.name),
687
+ 0,
688
+ len(job.name) + len(line_prefix)
720
689
  )
721
- d._write_log_offset()
722
- return True
690
+ get_console().print(text)
691
+
692
+ stop_event = asyncio.Event()
693
+ job_tasks = {}
694
+ job_stop_events = {}
695
+
696
+ async def refresh_job_tasks():
697
+ nonlocal job_tasks, jobs
698
+ while not stop_event.is_set():
699
+ jobs = get_filtered_jobs(executor_keys, action)
700
+ for name, job in jobs.items():
701
+ if name not in job_tasks:
702
+ job_stop_events[name] = asyncio.Event()
703
+ job_tasks[name] = asyncio.create_task(
704
+ job.monitor_logs_async(
705
+ partial(print_job_line, job),
706
+ stop_event=job_stop_events[name],
707
+ accept_input=False,
708
+ stop_on_exit=False,
709
+ )
710
+ )
711
+
712
+ for name, task in [(k, v) for k, v in job_tasks.items()]:
713
+ if name not in jobs:
714
+ job_stop_events[name].set()
715
+ task.cancel()
723
716
 
724
- daemons_being_watched = {
725
- d.daemon_id: d
726
- for d in daemons
727
- }
728
- for d in daemons:
729
- seek_back_offset(d)
730
- print_log_lines(d)
731
-
732
- _quit = False
733
- def watch_logs():
734
- for changes in watchfiles.watch(LOGS_RESOURCES_PATH):
735
- if _quit:
736
- return
737
- for change in changes:
738
- file_path_str = change[1]
739
- if '.log' not in file_path_str or '.log.offset' in file_path_str:
740
- continue
741
- file_path = pathlib.Path(file_path_str)
742
- if not file_path.exists():
743
- continue
744
- daemon_id = file_path.name.split('.log')[0]
745
- if daemon_id not in watch_daemon_ids and action:
746
- continue
747
717
  try:
748
- daemon = Daemon(daemon_id=daemon_id)
749
- except Exception as e:
750
- daemon = None
751
- warn(f"Seeing new logs for non-existent job '{daemon_id}'.", stack=False)
752
-
753
- if daemon is not None:
754
- print_log_lines(daemon)
755
-
756
- try:
757
- watch_logs()
758
- except KeyboardInterrupt as ki:
759
- _quit = True
718
+ await task
719
+ except asyncio.CancelledError:
720
+ pass
721
+ finally:
722
+ _ = job_tasks.pop(name, None)
723
+ _ = job_stop_events.pop(name, None)
760
724
 
761
- def print_nopretty_log_text():
762
- for d in daemons:
763
- log_text = d.log_text
764
- print(d.daemon_id)
765
- print(log_text)
725
+ await asyncio.sleep(1)
766
726
 
767
- print_log_text = follow_pretty_print if not nopretty else print_nopretty_log_text
768
- print_log_text()
727
+ async def gather_tasks():
728
+ tasks = [refresh_job_tasks()] + list(job_tasks.values())
729
+ await asyncio.gather(*tasks)
769
730
 
731
+ if not nopretty:
732
+ info("Watching logs...")
733
+ try:
734
+ asyncio.run(gather_tasks())
735
+ except KeyboardInterrupt:
736
+ pass
737
+ else:
738
+ for name, job in jobs.items():
739
+ print(f'\n-*-\nMRSM_JOB: {name}\n-*-')
740
+ print(job.get_logs())
770
741
  return True, "Success"
771
742
 
772
743