meerschaum 2.3.0.dev1__py3-none-any.whl → 2.3.0rc1__py3-none-any.whl

meerschaum/{utils/jobs → jobs}/_Job.py

@@ -6,6 +6,8 @@
 Define the Meerschaum abstraction atop daemons.
 """
 
+from __future__ import annotations
+
 import shlex
 import asyncio
 import threading
@@ -18,11 +20,17 @@ from functools import partial
 from datetime import datetime, timezone
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import List, Optional, Union, SuccessTuple, Any, Dict, Callable
+from meerschaum.utils.typing import (
+    List, Optional, Union, SuccessTuple, Any, Dict, Callable, TYPE_CHECKING,
+)
 from meerschaum._internal.entry import entry
 from meerschaum.utils.warnings import warn
 from meerschaum.config.paths import LOGS_RESOURCES_PATH
 from meerschaum.config import get_config
+from meerschaum.config.static import STATIC_CONFIG
+
+if TYPE_CHECKING:
+    from meerschaum.jobs._Executor import Executor
 
 BANNED_CHARS: List[str] = [
     ',', ';', "'", '"',
@@ -52,6 +60,10 @@ class Job:
         sysargs: Union[List[str], str, None] = None,
         executor_keys: Optional[str] = None,
         _properties: Optional[Dict[str, Any]] = None,
+        _rotating_log = None,
+        _stdin_file = None,
+        _status_hook = None,
+        _result_hook = None,
     ):
         """
         Create a new job to manage a `meerschaum.utils.daemon.Daemon`.
@@ -79,19 +91,40 @@ class Job:
         if isinstance(sysargs, str):
             sysargs = shlex.split(sysargs)
 
-        if executor_keys == 'local':
-            executor_keys = None
+        ### NOTE: 'local' and 'systemd' executors are being coalesced.
+        if executor_keys is None:
+            from meerschaum.jobs import get_executor_keys_from_context
+            executor_keys = get_executor_keys_from_context()
+
         self.executor_keys = executor_keys
         self.name = name
         try:
             self._daemon = (
                 Daemon(daemon_id=name)
-                if executor_keys is not None
+                if executor_keys == 'local'
                 else None
             )
         except Exception:
             self._daemon = None
 
+        ### Handle any injected dependencies.
+        if _rotating_log is not None:
+            self._rotating_log = _rotating_log
+            if self._daemon is not None:
+                self._daemon._rotating_log = _rotating_log
+
+        if _stdin_file is not None:
+            self._stdin_file = _stdin_file
+            if self._daemon is not None:
+                self._daemon._stdin_file = _stdin_file
+                self._daemon._blocking_stdin_file_path = _stdin_file.blocking_file_path
+
+        if _status_hook is not None:
+            self._status_hook = _status_hook
+
+        if _result_hook is not None:
+            self._result_hook = _result_hook
+
         self._properties_patch = _properties or {}
 
         daemon_sysargs = (
@@ -113,6 +146,71 @@ class Job:
                 self._properties_patch.update({'restart': True})
                 break
 
+        if '--systemd' in self._sysargs:
+            self._properties_patch.update({'systemd': True})
+
+    @staticmethod
+    def from_pid(pid: int, executor_keys: Optional[str] = None) -> Job:
+        """
+        Build a `Job` from the PID of a running Meerschaum process.
+
+        Parameters
+        ----------
+        pid: int
+            The PID of the process.
+
+        executor_keys: Optional[str], default None
+            The executor keys to assign to the job.
+        """
+        from meerschaum.config.paths import DAEMON_RESOURCES_PATH
+
+        psutil = mrsm.attempt_import('psutil')
+        try:
+            process = psutil.Process(pid)
+        except psutil.NoSuchProcess as e:
+            warn(f"Process with PID {pid} does not exist.", stack=False)
+            raise e
+
+        command_args = process.cmdline()
+        is_daemon = command_args[1] == '-c'
+
+        if is_daemon:
+            daemon_id = command_args[-1].split('daemon_id=')[-1].split(')')[0].replace("'", '')
+            root_dir = process.environ().get(STATIC_CONFIG['environment']['root'], None)
+            if root_dir is None:
+                from meerschaum.config.paths import ROOT_DIR_PATH
+                root_dir = ROOT_DIR_PATH
+            jobs_dir = root_dir / DAEMON_RESOURCES_PATH.name
+            daemon_dir = jobs_dir / daemon_id
+            pid_file = daemon_dir / 'process.pid'
+            properties_path = daemon_dir / 'properties.json'
+            pickle_path = daemon_dir / 'pickle.pkl'
+
+            if pid_file.exists():
+                with open(pid_file, 'r', encoding='utf-8') as f:
+                    daemon_pid = int(f.read())
+                
+                if pid != daemon_pid:
+                    raise EnvironmentError(f"Differing PIDs: {pid=}, {daemon_pid=}")
+            else:
+                raise EnvironmentError(f"Is job '{daemon_id}' running?")
+
+            return Job(daemon_id, executor_keys=executor_keys)
+
+        from meerschaum._internal.arguments._parse_arguments import parse_arguments
+        from meerschaum.utils.daemon import get_new_daemon_name
+
+        mrsm_ix = 0
+        for i, arg in enumerate(command_args):
+            if 'mrsm' in arg or 'meerschaum' in arg.lower():
+                mrsm_ix = i
+                break
+
+        sysargs = command_args[mrsm_ix+1:]
+        kwargs = parse_arguments(sysargs)
+        name = kwargs.get('name', get_new_daemon_name())
+        return Job(name, sysargs, executor_keys=executor_keys)
+
     def start(self, debug: bool = False) -> SuccessTuple:
         """
         Start the job's daemon.
@@ -144,6 +242,8 @@ class Job:
         if self.daemon.status == 'stopped':
             if not self.restart:
                 return True, f"{self} is not running."
+            elif self.stop_time is not None:
+                return True, f"{self} will not restart until manually started."
 
         quit_success, quit_msg = self.daemon.quit(timeout=timeout_seconds)
         if quit_success:
@@ -218,7 +318,8 @@ class Job:
         self,
         callback_function: Callable[[str], None] = partial(print, end=''),
         input_callback_function: Optional[Callable[[], str]] = None,
-        stop_event: Optional[threading.Event] = None,
+        stop_callback_function: Optional[Callable[[SuccessTuple], None]] = None,
+        stop_event: Optional[asyncio.Event] = None,
         stop_on_exit: bool = False,
         strip_timestamps: bool = False,
         accept_input: bool = True,
@@ -237,9 +338,13 @@ class Job:
             If provided, execute this callback when the daemon is blocking on stdin.
             Defaults to `sys.stdin.readline()`.
 
+        stop_callback_function: Optional[Callable[[SuccessTuple]], str], default None
+            If provided, execute this callback when the daemon stops.
+            The job's SuccessTuple will be passed to the callback.
+
         stop_event: Optional[asyncio.Event], default None
             If provided, stop monitoring when this event is set.
-            You may instead raise `meerschaum.utils.jobs.StopMonitoringLogs`
+            You may instead raise `meerschaum.jobs.StopMonitoringLogs`
             from within `callback_function` to stop monitoring.
 
         stop_on_exit: bool, default False
@@ -251,13 +356,29 @@ class Job:
         accept_input: bool, default True
             If `True`, accept input when the daemon blocks on stdin.
         """
+        def default_input_callback_function():
+            return sys.stdin.readline()
+
+        if input_callback_function is None:
+            input_callback_function = default_input_callback_function
+
         if self.executor is not None:
-            self.executor.monitor_logs(self.name, callback_function)
+            self.executor.monitor_logs(
+                self.name,
+                callback_function,
+                input_callback_function=input_callback_function,
+                stop_callback_function=stop_callback_function,
+                stop_on_exit=stop_on_exit,
+                accept_input=accept_input,
+                strip_timestamps=strip_timestamps,
+                debug=debug,
+            )
             return
 
         monitor_logs_coroutine = self.monitor_logs_async(
             callback_function=callback_function,
             input_callback_function=input_callback_function,
+            stop_callback_function=stop_callback_function,
             stop_event=stop_event,
             stop_on_exit=stop_on_exit,
             strip_timestamps=strip_timestamps,
@@ -270,10 +391,14 @@ class Job:
         self,
         callback_function: Callable[[str], None] = partial(print, end='', flush=True),
         input_callback_function: Optional[Callable[[], str]] = None,
+        stop_callback_function: Optional[Callable[[SuccessTuple], None]] = None,
         stop_event: Optional[asyncio.Event] = None,
         stop_on_exit: bool = False,
         strip_timestamps: bool = False,
         accept_input: bool = True,
+        _logs_path: Optional[pathlib.Path] = None,
+        _log = None,
+        _stdin_file = None,
         debug: bool = False,
     ):
         """
@@ -289,9 +414,13 @@ class Job:
             If provided, execute this callback when the daemon is blocking on stdin.
             Defaults to `sys.stdin.readline()`.
 
+        stop_callback_function: Optional[Callable[[SuccessTuple]], str], default None
+            If provided, execute this callback when the daemon stops.
+            The job's SuccessTuple will be passed to the callback.
+
         stop_event: Optional[asyncio.Event], default None
             If provided, stop monitoring when this event is set.
-            You may instead raise `meerschaum.utils.jobs.StopMonitoringLogs`
+            You may instead raise `meerschaum.jobs.StopMonitoringLogs`
             from within `callback_function` to stop monitoring.
 
         stop_on_exit: bool, default False
@@ -314,6 +443,8 @@ class Job:
                 self.name,
                 callback_function,
                 input_callback_function=input_callback_function,
+                stop_callback_function=stop_callback_function,
+                stop_on_exit=stop_on_exit,
                 accept_input=accept_input,
                 debug=debug,
             )
@@ -323,16 +454,18 @@ class Job:
 
         events = {
             'user': stop_event,
-            'stopped': (asyncio.Event() if stop_on_exit else None),
+            'stopped': asyncio.Event(),
         }
         combined_event = asyncio.Event()
         emitted_text = False
+        stdin_file = _stdin_file if _stdin_file is not None else self.daemon.stdin_file
 
         async def check_job_status():
             nonlocal emitted_text
             stopped_event = events.get('stopped', None)
             if stopped_event is None:
                 return
+
             sleep_time = 0.1
             while sleep_time < 60:
                 if self.status == 'stopped':
@@ -340,7 +473,19 @@ class Job:
                         await asyncio.sleep(sleep_time)
                         sleep_time = round(sleep_time * 1.1, 2)
                         continue
-                    events['stopped'].set()
+                    
+                    if stop_callback_function is not None:
+                        try:
+                            if asyncio.iscoroutinefunction(stop_callback_function):
+                                await stop_callback_function(self.result)
+                            else:
+                                stop_callback_function(self.result)
+                        except Exception:
+                            warn(traceback.format_exc())
+
+                    if stop_on_exit:
+                        events['stopped'].set()
+
                     break
                 await asyncio.sleep(0.1)
 
@@ -360,7 +505,7 @@ class Job:
 
                 try:
                     print('', end='', flush=True)
-                    if asyncio.iscoroutinefunction(callback_function):
+                    if asyncio.iscoroutinefunction(input_callback_function):
                         data = await input_callback_function()
                     else:
                         data = input_callback_function()
@@ -368,7 +513,8 @@ class Job:
                     break
                 if not data.endswith('\n'):
                     data += '\n'
-                self.daemon.stdin_file.write(data)
+
+                stdin_file.write(data)
                 await asyncio.sleep(0.1)
 
         async def combine_events():
@@ -396,7 +542,7 @@ class Job:
         check_blocking_on_input_task = asyncio.create_task(check_blocking_on_input())
         combine_events_task = asyncio.create_task(combine_events())
 
-        log = self.daemon.rotating_log
+        log = _log if _log is not None else self.daemon.rotating_log
         lines_to_show = get_config('jobs', 'logs', 'lines_to_show')
 
         async def emit_latest_lines():
@@ -434,7 +580,7 @@ class Job:
 
         watchfiles = mrsm.attempt_import('watchfiles')
         async for changes in watchfiles.awatch(
-            LOGS_RESOURCES_PATH,
+            _logs_path or LOGS_RESOURCES_PATH,
             stop_event=combined_event,
         ):
             for change in changes:
@@ -444,23 +590,7 @@ class Job:
                 if latest_subfile_path != file_path:
                     continue
 
-                lines = log.readlines()
-                for line in lines:
-                    if strip_timestamps:
-                        line = strip_timestamp_from_line(line)
-                    try:
-                        if asyncio.iscoroutinefunction(callback_function):
-                            await callback_function(line)
-                        else:
-                            callback_function(line)
-                        emitted_text = True
-                    except RuntimeError:
-                        return
-                    except StopMonitoringLogs:
-                        return
-                    except Exception:
-                        warn(f"Error in logs callback:\n{traceback.format_exc()}")
-                        return
+                await emit_latest_lines()
 
         await emit_latest_lines()
 
@@ -477,19 +607,16 @@ class Job:
         """
         Write to a job's daemon's `stdin`.
         """
-        if self.executor is not None:
-            pass
-
         self.daemon.stdin_file.write(data)
 
     @property
-    def executor(self) -> Union['APIConnector', None]:
+    def executor(self) -> Union[Executor, None]:
         """
         If the job is remote, return the connector to the remote API instance.
         """
         return (
             mrsm.get_connector(self.executor_keys)
-            if self.executor_keys is not None
+            if self.executor_keys != 'local'
             else None
         )
 
@@ -498,10 +625,11 @@ class Job:
         """
         Return the running status of the job's daemon.
         """
+        if '_status_hook' in self.__dict__:
+            return self._status_hook()
+
         if self.executor is not None:
-            return self.executor.get_job_metadata(
-                self.name
-            ).get('daemon', {}).get('status', 'stopped')
+            return self.executor.get_job_status(self.name)
 
         return self.daemon.status
 
@@ -520,6 +648,9 @@ class Job:
         """
         Return whether to restart a stopped job.
         """
+        if self.executor is not None:
+            return self.executor.get_job_metadata(self.name).get('restart', False)
+
         return self.daemon.properties.get('restart', False)
 
     @property
@@ -530,6 +661,15 @@ class Job:
         if self.is_running():
             return True, f"{self} is running."
 
+        if '_result_hook' in self.__dict__:
+            return self._result_hook()
+
+        if self.executor is not None:
+            return (
+                self.executor.get_job_metadata(self.name)
+                .get('result', (False, "No result available."))
+            )
+
         _result = self.daemon.properties.get('result', None)
         if _result is None:
             return False, "No result available."
@@ -544,7 +684,9 @@ class Job:
         if self._sysargs:
             return self._sysargs
 
-        #  target_args = self.daemon.properties.get('target', {}).get('args', None)
+        if self.executor is not None:
+            return self.executor.get_job_metadata(self.name).get('sysargs', [])
+
         target_args = self.daemon.target_args
         if target_args is None:
             return []
@@ -575,6 +717,12 @@ class Job:
             label=shlex.join(self._sysargs),
             properties=properties,
         )
+        if '_rotating_log' in self.__dict__:
+            self._daemon._rotating_log = self._rotating_log
+
+        if '_stdin_file' in self.__dict__:
+            self._daemon._stdin_file = self._stdin_file
+            self._daemon._blocking_stdin_file_path = self._stdin_file.blocking_file_path
 
         return self._daemon
 
@@ -583,6 +731,16 @@ class Job:
         """
         The datetime when the job began running.
         """
+        if self.executor is not None:
+            began_str = self.executor.get_job_began(name)
+            if began_str is None:
+                return None
+            return (
+                datetime.fromisoformat(began_str)
+                .astimezone(timezone.utc)
+                .replace(tzinfo=None)
+            )
+
         began_str = self.daemon.properties.get('process', {}).get('began', None)
         if began_str is None:
             return None
@@ -594,6 +752,14 @@ class Job:
         """
         The datetime when the job stopped running.
         """
+        if self.executor is not None:
+            ended_str = self.executor.get_job_ended(self.name)
+            return (
+                datetime.fromisoformat(ended_str)
+                .astimezone(timezone.utc)
+                .replace(tzinfo=None)
+            )
+
         ended_str = self.daemon.properties.get('process', {}).get('ended', None)
         if ended_str is None:
             return None

meerschaum/jobs/_LocalExecutor.py

@@ -0,0 +1,88 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Run jobs locally.
+"""
+
+from meerschaum.utils.typing import Dict, Any, List, SuccessTuple, Union
+from meerschaum.jobs import Job, Executor, make_executor
+from meerschaum.utils.daemon import Daemon, get_daemons
+from meerschaum._internal.entry import entry
+
+
+@make_executor
+class LocalExecutor(Executor):
+    """
+    Run jobs locally as Unix daemons.
+    """
+
+    def get_job_daemon(
+        self,
+        name: str,
+        #  sysargs: Opt
+        debug: bool = False,
+    ) -> Union[Daemon, None]:
+        """
+        Return a job's daemon if it exists.
+        """
+        try:
+            daemon = Daemon(name)
+        except Exception:
+            daemon = None
+
+        return daemon
+    
+    def get_daemon_syargs(self, name: str, debug: bool = False) -> Union[List[str], None]:
+        """
+        Return the list of sysargs from the job's daemon.
+        """
+        daemon = self.get_job_daemon(name, debug=debug)
+
+        if daemon is None:
+            return None
+
+        return daemon.properties.get('target', {}).get('args', [None])[0]
+
+    def get_job_exists(self, name: str, debug: bool = False) -> bool:
+        """
+        Return whether a job exists.
+        """
+        daemon = self.get_job_daemon(name, debug=debug)
+        if daemon is None:
+            return False
+    
+    def get_jobs(self) -> Dict[str, Job]:
+        """
+        Return a dictionary of names -> Jobs.
+        """
+
+    def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
+        """
+        Create a new job.
+        """
+
+    def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Start a job.
+        """
+
+    def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Stop a job.
+        """
+
+    def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Pause a job.
+        """
+
+    def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Delete a job.
+        """
+
+    def get_logs(self, name: str, debug: bool = False) -> str:
+        """
+        Return a job's log output.
+        """