meerschaum 2.3.0.dev3__py3-none-any.whl → 2.3.0rc1__py3-none-any.whl

meerschaum/connectors/api/_jobs.py

@@ -13,7 +13,7 @@ from datetime import datetime
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable
-from meerschaum.utils.jobs import Job
+from meerschaum.jobs import Job
 from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.warnings import warn, dprint
 
@@ -98,7 +98,6 @@ def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
     }
     return metadata
 
-
 def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
     """
     Return the daemon properties for a single job.
@@ -106,6 +105,34 @@ def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
     metadata = self.get_job_metadata(name, debug=debug)
     return metadata.get('daemon', {}).get('properties', {})
 
+def get_job_status(self, name: str, debug: bool = False) -> str:
+    """
+    Return the job's status.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    return metadata.get('status', 'stopped')
+
+def get_job_began(self, name: str, debug: bool = False) -> str:
+    """
+    Return a job's `began` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    began_str = properties.get('daemon', {}).get('began', None)
+    if began_str is None:
+        return None
+
+    return began_str
+
+def get_job_ended(self, name: str, debug: bool = False) -> str:
+    """
+    Return a job's `ended` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    ended_str = properties.get('daemon', {}).get('ended', None)
+    if ended_str is None:
+        return None
+
+    return ended_str
 
 def get_job_exists(self, name: str, debug: bool = False) -> bool:
     """
@@ -226,7 +253,7 @@ async def monitor_logs_async(
     """
     Monitor a job's log files and await a callback with the changes.
     """
-    from meerschaum.utils.jobs import StopMonitoringLogs
+    from meerschaum.jobs import StopMonitoringLogs
     from meerschaum.utils.formatting._jobs import strip_timestamp_from_line
 
     websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')

meerschaum/connectors/parse.py

@@ -87,19 +87,17 @@ def parse_connector_keys(
 
 
 def parse_instance_keys(
-        keys: Optional[str],
-        construct: bool = True,
-        as_tuple: bool = False,
-        **kw
-    ):
+    keys: Optional[str],
+    construct: bool = True,
+    as_tuple: bool = False,
+    **kw
+):
     """
     Parse the Meerschaum instance value into a Connector object.
     """
     from meerschaum.utils.warnings import warn
     from meerschaum.config import get_config
 
-    ### TODO Check for valid types? Not sure how to do that if construct = False.
-
     if keys is None:
         keys = get_config('meerschaum', 'instance')
     keys = str(keys)
@@ -120,25 +118,24 @@ def parse_repo_keys(keys: Optional[str] = None, **kw):
 
 
 def parse_executor_keys(keys: Optional[str] = None, **kw):
-    """Parse the executor keys into an APIConnector or None."""
+    """Parse the executor keys into an APIConnector or string."""
+    from meerschaum.jobs import get_executor_keys_from_context
     from meerschaum.config import get_config
     if keys is None:
-        keys = get_config('meerschaum', 'default_executor')
+        keys = get_executor_keys_from_context()
 
     if keys is None or keys == 'local':
         return 'local'
 
     keys = str(keys)
-    if ':' not in keys:
-        keys = 'api:' + keys
-
     return parse_connector_keys(keys, **kw)
 
 
 def is_valid_connector_keys(
     keys: str
 ) -> bool:
-    """Verify a connector_keys string references a valid connector.
+    """
+    Verify a connector_keys string references a valid connector.
     """
     try:
         success = parse_connector_keys(keys, construct=False) is not None

meerschaum/core/Pipe/_bootstrap.py

@@ -88,21 +88,29 @@ def bootstrap(
 
     try:
         if yes_no(
-            f"Would you like to edit the definition for {self}?", yes=yes, noask=noask
+            f"Would you like to edit the definition for {self}?",
+            yes=yes,
+            noask=noask,
+            default='n',
         ):
             edit_tuple = self.edit_definition(debug=debug)
             if not edit_tuple[0]:
                 return edit_tuple
 
-        if yes_no(f"Would you like to try syncing {self} now?", yes=yes, noask=noask):
+        if yes_no(
+            f"Would you like to try syncing {self} now?",
+            yes=yes,
+            noask=noask,
+            default='n',
+        ):
             sync_tuple = actions['sync'](
                 ['pipes'],
-                connector_keys = [self.connector_keys],
-                metric_keys = [self.metric_key],
-                location_keys = [self.location_key],
-                mrsm_instance = str(self.instance_connector),
-                debug = debug,
-                shell = shell,
+                connector_keys=[self.connector_keys],
+                metric_keys=[self.metric_key],
+                location_keys=[self.location_key],
+                mrsm_instance=str(self.instance_connector),
+                debug=debug,
+                shell=shell,
             )
             if not sync_tuple[0]:
                 return sync_tuple

meerschaum/jobs/_Executor.py

@@ -0,0 +1,69 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define the base class for a Job executor.
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+
+from meerschaum.connectors import Connector
+from meerschaum.utils.typing import List, Dict, SuccessTuple, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from meerschaum.jobs import Job
+
+class Executor(Connector):
+    """
+    Define the methods for managing jobs.
+    """
+
+    @abstractmethod
+    def get_job_exists(self, name: str, debug: bool = False) -> bool:
+        """
+        Return whether a job exists.
+        """
+    
+    @abstractmethod
+    def get_jobs(self) -> Dict[str, Job]:
+        """
+        Return a dictionary of names -> Jobs.
+        """
+
+    @abstractmethod
+    def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
+        """
+        Create a new job.
+        """
+
+    @abstractmethod
+    def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Start a job.
+        """
+
+    @abstractmethod
+    def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Stop a job.
+        """
+
+    @abstractmethod
+    def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Pause a job.
+        """
+
+    @abstractmethod
+    def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Delete a job.
+        """
+
+    @abstractmethod
+    def get_logs(self, name: str, debug: bool = False) -> str:
+        """
+        Return a job's log output.
+        """

meerschaum/{utils/jobs → jobs}/_Job.py

@@ -6,6 +6,8 @@
 Define the Meerschaum abstraction atop daemons.
 """
 
+from __future__ import annotations
+
 import shlex
 import asyncio
 import threading
@@ -18,11 +20,17 @@ from functools import partial
 from datetime import datetime, timezone
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import List, Optional, Union, SuccessTuple, Any, Dict, Callable
+from meerschaum.utils.typing import (
+    List, Optional, Union, SuccessTuple, Any, Dict, Callable, TYPE_CHECKING,
+)
 from meerschaum._internal.entry import entry
 from meerschaum.utils.warnings import warn
 from meerschaum.config.paths import LOGS_RESOURCES_PATH
 from meerschaum.config import get_config
+from meerschaum.config.static import STATIC_CONFIG
+
+if TYPE_CHECKING:
+    from meerschaum.jobs._Executor import Executor
 
 BANNED_CHARS: List[str] = [
     ',', ';', "'", '"',
@@ -52,6 +60,10 @@ class Job:
         sysargs: Union[List[str], str, None] = None,
         executor_keys: Optional[str] = None,
         _properties: Optional[Dict[str, Any]] = None,
+        _rotating_log = None,
+        _stdin_file = None,
+        _status_hook = None,
+        _result_hook = None,
     ):
         """
         Create a new job to manage a `meerschaum.utils.daemon.Daemon`.
@@ -79,19 +91,40 @@ class Job:
         if isinstance(sysargs, str):
             sysargs = shlex.split(sysargs)
 
-        if executor_keys == 'local':
-            executor_keys = None
+        ### NOTE: 'local' and 'systemd' executors are being coalesced.
+        if executor_keys is None:
+            from meerschaum.jobs import get_executor_keys_from_context
+            executor_keys = get_executor_keys_from_context()
+
         self.executor_keys = executor_keys
         self.name = name
         try:
             self._daemon = (
                 Daemon(daemon_id=name)
-                if executor_keys is not None
+                if executor_keys == 'local'
                 else None
             )
         except Exception:
             self._daemon = None
 
+        ### Handle any injected dependencies.
+        if _rotating_log is not None:
+            self._rotating_log = _rotating_log
+            if self._daemon is not None:
+                self._daemon._rotating_log = _rotating_log
+
+        if _stdin_file is not None:
+            self._stdin_file = _stdin_file
+            if self._daemon is not None:
+                self._daemon._stdin_file = _stdin_file
+                self._daemon._blocking_stdin_file_path = _stdin_file.blocking_file_path
+
+        if _status_hook is not None:
+            self._status_hook = _status_hook
+
+        if _result_hook is not None:
+            self._result_hook = _result_hook
+
         self._properties_patch = _properties or {}
 
         daemon_sysargs = (
@@ -113,6 +146,71 @@ class Job:
                 self._properties_patch.update({'restart': True})
                 break
 
+        if '--systemd' in self._sysargs:
+            self._properties_patch.update({'systemd': True})
+
+    @staticmethod
+    def from_pid(pid: int, executor_keys: Optional[str] = None) -> Job:
+        """
+        Build a `Job` from the PID of a running Meerschaum process.
+
+        Parameters
+        ----------
+        pid: int
+            The PID of the process.
+
+        executor_keys: Optional[str], default None
+            The executor keys to assign to the job.
+        """
+        from meerschaum.config.paths import DAEMON_RESOURCES_PATH
+
+        psutil = mrsm.attempt_import('psutil')
+        try:
+            process = psutil.Process(pid)
+        except psutil.NoSuchProcess as e:
+            warn(f"Process with PID {pid} does not exist.", stack=False)
+            raise e
+
+        command_args = process.cmdline()
+        is_daemon = command_args[1] == '-c'
+
+        if is_daemon:
+            daemon_id = command_args[-1].split('daemon_id=')[-1].split(')')[0].replace("'", '')
+            root_dir = process.environ().get(STATIC_CONFIG['environment']['root'], None)
+            if root_dir is None:
+                from meerschaum.config.paths import ROOT_DIR_PATH
+                root_dir = ROOT_DIR_PATH
+            jobs_dir = root_dir / DAEMON_RESOURCES_PATH.name
+            daemon_dir = jobs_dir / daemon_id
+            pid_file = daemon_dir / 'process.pid'
+            properties_path = daemon_dir / 'properties.json'
+            pickle_path = daemon_dir / 'pickle.pkl'
+
+            if pid_file.exists():
+                with open(pid_file, 'r', encoding='utf-8') as f:
+                    daemon_pid = int(f.read())
+                
+                if pid != daemon_pid:
+                    raise EnvironmentError(f"Differing PIDs: {pid=}, {daemon_pid=}")
+            else:
+                raise EnvironmentError(f"Is job '{daemon_id}' running?")
+
+            return Job(daemon_id, executor_keys=executor_keys)
+
+        from meerschaum._internal.arguments._parse_arguments import parse_arguments
+        from meerschaum.utils.daemon import get_new_daemon_name
+
+        mrsm_ix = 0
+        for i, arg in enumerate(command_args):
+            if 'mrsm' in arg or 'meerschaum' in arg.lower():
+                mrsm_ix = i
+                break
+
+        sysargs = command_args[mrsm_ix+1:]
+        kwargs = parse_arguments(sysargs)
+        name = kwargs.get('name', get_new_daemon_name())
+        return Job(name, sysargs, executor_keys=executor_keys)
+
     def start(self, debug: bool = False) -> SuccessTuple:
         """
         Start the job's daemon.
@@ -144,6 +242,8 @@ class Job:
         if self.daemon.status == 'stopped':
             if not self.restart:
                 return True, f"{self} is not running."
+            elif self.stop_time is not None:
+                return True, f"{self} will not restart until manually started."
 
         quit_success, quit_msg = self.daemon.quit(timeout=timeout_seconds)
         if quit_success:
@@ -244,7 +344,7 @@ class Job:
 
         stop_event: Optional[asyncio.Event], default None
             If provided, stop monitoring when this event is set.
-            You may instead raise `meerschaum.utils.jobs.StopMonitoringLogs`
+            You may instead raise `meerschaum.jobs.StopMonitoringLogs`
             from within `callback_function` to stop monitoring.
 
         stop_on_exit: bool, default False
@@ -296,6 +396,9 @@ class Job:
         stop_on_exit: bool = False,
         strip_timestamps: bool = False,
         accept_input: bool = True,
+        _logs_path: Optional[pathlib.Path] = None,
+        _log = None,
+        _stdin_file = None,
         debug: bool = False,
     ):
         """
@@ -317,7 +420,7 @@ class Job:
 
         stop_event: Optional[asyncio.Event], default None
             If provided, stop monitoring when this event is set.
-            You may instead raise `meerschaum.utils.jobs.StopMonitoringLogs`
+            You may instead raise `meerschaum.jobs.StopMonitoringLogs`
             from within `callback_function` to stop monitoring.
 
         stop_on_exit: bool, default False
@@ -355,12 +458,14 @@ class Job:
         }
         combined_event = asyncio.Event()
         emitted_text = False
+        stdin_file = _stdin_file if _stdin_file is not None else self.daemon.stdin_file
 
         async def check_job_status():
             nonlocal emitted_text
             stopped_event = events.get('stopped', None)
             if stopped_event is None:
                 return
+
             sleep_time = 0.1
             while sleep_time < 60:
                 if self.status == 'stopped':
@@ -408,7 +513,8 @@ class Job:
                     break
                 if not data.endswith('\n'):
                     data += '\n'
-                self.daemon.stdin_file.write(data)
+
+                stdin_file.write(data)
                 await asyncio.sleep(0.1)
 
         async def combine_events():
@@ -436,7 +542,7 @@ class Job:
         check_blocking_on_input_task = asyncio.create_task(check_blocking_on_input())
         combine_events_task = asyncio.create_task(combine_events())
 
-        log = self.daemon.rotating_log
+        log = _log if _log is not None else self.daemon.rotating_log
         lines_to_show = get_config('jobs', 'logs', 'lines_to_show')
 
         async def emit_latest_lines():
@@ -474,7 +580,7 @@ class Job:
 
         watchfiles = mrsm.attempt_import('watchfiles')
         async for changes in watchfiles.awatch(
-            LOGS_RESOURCES_PATH,
+            _logs_path or LOGS_RESOURCES_PATH,
             stop_event=combined_event,
         ):
             for change in changes:
@@ -485,7 +591,6 @@ class Job:
                     continue
 
                 await emit_latest_lines()
-                await emit_latest_lines()
 
         await emit_latest_lines()
 
@@ -502,20 +607,16 @@ class Job:
         """
         Write to a job's daemon's `stdin`.
         """
-        ### TODO implement remote method?
-        if self.executor is not None:
-            pass
-
         self.daemon.stdin_file.write(data)
 
     @property
-    def executor(self) -> Union['APIConnector', None]:
+    def executor(self) -> Union[Executor, None]:
         """
         If the job is remote, return the connector to the remote API instance.
         """
         return (
             mrsm.get_connector(self.executor_keys)
-            if self.executor_keys is not None
+            if self.executor_keys != 'local'
             else None
         )
 
@@ -524,10 +625,11 @@ class Job:
         """
         Return the running status of the job's daemon.
         """
+        if '_status_hook' in self.__dict__:
+            return self._status_hook()
+
         if self.executor is not None:
-            return self.executor.get_job_metadata(
-                self.name
-            ).get('daemon', {}).get('status', 'stopped')
+            return self.executor.get_job_status(self.name)
 
         return self.daemon.status
 
@@ -546,6 +648,9 @@ class Job:
         """
         Return whether to restart a stopped job.
         """
+        if self.executor is not None:
+            return self.executor.get_job_metadata(self.name).get('restart', False)
+
         return self.daemon.properties.get('restart', False)
 
     @property
@@ -556,6 +661,15 @@ class Job:
         if self.is_running():
             return True, f"{self} is running."
 
+        if '_result_hook' in self.__dict__:
+            return self._result_hook()
+
+        if self.executor is not None:
+            return (
+                self.executor.get_job_metadata(self.name)
+                .get('result', (False, "No result available."))
+            )
+
         _result = self.daemon.properties.get('result', None)
         if _result is None:
             return False, "No result available."
@@ -570,7 +684,9 @@ class Job:
         if self._sysargs:
             return self._sysargs
 
-        #  target_args = self.daemon.properties.get('target', {}).get('args', None)
+        if self.executor is not None:
+            return self.executor.get_job_metadata(self.name).get('sysargs', [])
+
         target_args = self.daemon.target_args
         if target_args is None:
             return []
@@ -601,6 +717,12 @@ class Job:
             label=shlex.join(self._sysargs),
             properties=properties,
         )
+        if '_rotating_log' in self.__dict__:
+            self._daemon._rotating_log = self._rotating_log
+
+        if '_stdin_file' in self.__dict__:
+            self._daemon._stdin_file = self._stdin_file
+            self._daemon._blocking_stdin_file_path = self._stdin_file.blocking_file_path
 
         return self._daemon
 
@@ -609,6 +731,16 @@ class Job:
         """
         The datetime when the job began running.
         """
+        if self.executor is not None:
+            began_str = self.executor.get_job_began(name)
+            if began_str is None:
+                return None
+            return (
+                datetime.fromisoformat(began_str)
+                .astimezone(timezone.utc)
+                .replace(tzinfo=None)
+            )
+
         began_str = self.daemon.properties.get('process', {}).get('began', None)
         if began_str is None:
             return None
@@ -620,6 +752,14 @@ class Job:
         """
         The datetime when the job stopped running.
         """
+        if self.executor is not None:
+            ended_str = self.executor.get_job_ended(self.name)
+            return (
+                datetime.fromisoformat(ended_str)
+                .astimezone(timezone.utc)
+                .replace(tzinfo=None)
+            )
+
         ended_str = self.daemon.properties.get('process', {}).get('ended', None)
         if ended_str is None:
             return None

meerschaum/jobs/_LocalExecutor.py

@@ -0,0 +1,88 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Run jobs locally.
+"""
+
+from meerschaum.utils.typing import Dict, Any, List, SuccessTuple, Union
+from meerschaum.jobs import Job, Executor, make_executor
+from meerschaum.utils.daemon import Daemon, get_daemons
+from meerschaum._internal.entry import entry
+
+
+@make_executor
+class LocalExecutor(Executor):
+    """
+    Run jobs locally as Unix daemons.
+    """
+
+    def get_job_daemon(
+        self,
+        name: str,
+        #  sysargs: Opt
+        debug: bool = False,
+    ) -> Union[Daemon, None]:
+        """
+        Return a job's daemon if it exists.
+        """
+        try:
+            daemon = Daemon(name)
+        except Exception:
+            daemon = None
+
+        return daemon
+    
+    def get_daemon_syargs(self, name: str, debug: bool = False) -> Union[List[str], None]:
+        """
+        Return the list of sysargs from the job's daemon.
+        """
+        daemon = self.get_job_daemon(name, debug=debug)
+
+        if daemon is None:
+            return None
+
+        return daemon.properties.get('target', {}).get('args', [None])[0]
+
+    def get_job_exists(self, name: str, debug: bool = False) -> bool:
+        """
+        Return whether a job exists.
+        """
+        daemon = self.get_job_daemon(name, debug=debug)
+        if daemon is None:
+            return False
+    
+    def get_jobs(self) -> Dict[str, Job]:
+        """
+        Return a dictionary of names -> Jobs.
+        """
+
+    def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
+        """
+        Create a new job.
+        """
+
+    def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Start a job.
+        """
+
+    def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Stop a job.
+        """
+
+    def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Pause a job.
+        """
+
+    def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Delete a job.
+        """
+
+    def get_logs(self, name: str, debug: bool = False) -> str:
+        """
+        Return a job's log output.
+        """