meerschaum 2.3.0.dev3__py3-none-any.whl → 2.3.0rc2__py3-none-any.whl

meerschaum/api/routes/_jobs.py

@@ -19,7 +19,12 @@ from functools import partial
 from fastapi import WebSocket, WebSocketDisconnect
 
 from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Optional, Union
-from meerschaum.utils.jobs import get_jobs as _get_jobs, Job, StopMonitoringLogs
+from meerschaum.jobs import (
+    get_jobs as _get_jobs,
+    Job,
+    StopMonitoringLogs,
+    get_executor_keys_from_context,
+)
 from meerschaum.utils.warnings import warn
 
 from meerschaum.api import (
@@ -35,6 +40,7 @@ from meerschaum.config.static import STATIC_CONFIG
 
 JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
 JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
+EXECUTOR_KEYS: str = get_executor_keys_from_context()
 
 
 @app.get(endpoints['jobs'], tags=['Jobs'])
@@ -46,15 +52,21 @@ def get_jobs(
     """
     Return metadata about the current jobs.
     """
-    jobs = _get_jobs()
+    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
     return {
         name: {
             'sysargs': job.sysargs,
             'result': job.result,
+            'restart': job.restart,
+            'status': job.status,
             'daemon': {
-                'status': job.daemon.status,
-                'pid': job.daemon.pid,
-                'properties': job.daemon.properties,
+                'status': job.daemon.status if job.executor_keys is None else job.status,
+                'pid': job.pid,
+                'properties': (
+                    job.daemon.properties
+                    if job.executor is None
+                    else job.executor.get_job_properties(name)
+                ),
             },
         }
         for name, job in jobs.items()
@@ -71,7 +83,7 @@ def get_job(
     """
     Return metadata for a single job.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -81,10 +93,16 @@ def get_job(
     return {
         'sysargs': job.sysargs,
         'result': job.result,
+        'restart': job.restart,
+        'status': job.status,
         'daemon': {
-            'status': job.daemon.status,
-            'pid': job.daemon.pid,
-            'properties': job.daemon.properties,
+            'status': job.daemon.status if job.executor_keys is None else job.status,
+            'pid': job.pid,
+            'properties': (
+                job.daemon.properties
+                if job.executor is None
+                else job.executor.get_job_properties(job.name)
+            ),
         },
     }
 
@@ -100,7 +118,7 @@ def create_job(
     """
     Create and start a new job.
     """
-    job = Job(name, sysargs)
+    job = Job(name, sysargs, executor_keys=EXECUTOR_KEYS)
     if job.exists():
         raise fastapi.HTTPException(
             status_code=409,
@@ -120,7 +138,7 @@ def delete_job(
     """
     Delete a job.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.delete()
 
 
@@ -134,7 +152,7 @@ def get_job_exists(
     """
     Return whether a job exists.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.exists()
 
 
@@ -149,7 +167,7 @@ def get_logs(
     Return a job's log text.
     To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -169,7 +187,7 @@ def start_job(
     """
     Start a job if stopped.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -188,7 +206,7 @@ def stop_job(
     """
     Stop a job if running.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -207,7 +225,7 @@ def pause_job(
     """
     Pause a job if running.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -226,7 +244,7 @@ def get_stop_time(
     """
     Get the timestamp when the job was manually stopped.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.stop_time
 
 
@@ -240,7 +258,7 @@ def get_is_blocking_on_stdin(
     """
     Return whether a job is blocking on stdin.
     """
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     return job.is_blocking_on_stdin()
 
 
@@ -314,7 +332,7 @@ async def logs_websocket(name: str, websocket: WebSocket):
     Stream logs from a job over a websocket.
     """
     await websocket.accept()
-    job = Job(name)
+    job = Job(name, executor_keys=EXECUTOR_KEYS)
     _job_clients[name].append(websocket)
 
     async def monitor_logs():

meerschaum/config/_default.py

@@ -16,7 +16,7 @@ default_meerschaum_config = {
     'api_instance': 'MRSM{meerschaum:instance}',
     'web_instance': 'MRSM{meerschaum:instance}',
     'default_repository': 'api:mrsm',
-    'default_executor': 'local',
+    #  'default_executor': 'local',
     'connectors': {
         'sql': {
             'default': {},

meerschaum/config/_paths.py

@@ -181,6 +181,11 @@ paths = {
     'LOGS_RESOURCES_PATH'            : ('{ROOT_DIR_PATH}', 'logs'),
     'DAEMON_ERROR_LOG_PATH'          : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
     'CHECK_JOBS_LOCK_PATH'           : ('{INTERNAL_RESOURCES_PATH}', 'check-jobs.lock'),
+    
+    'SYSTEMD_RESOURCES_PATH'         : ('{DOT_CONFIG_DIR_PATH}', 'systemd'),
+    'SYSTEMD_USER_RESOURCES_PATH'    : ('{SYSTEMD_RESOURCES_PATH}', 'user'),
+    'SYSTEMD_ROOT_RESOURCES_PATH'    : ('{ROOT_DIR_PATH}', 'systemd'),
+    'SYSTEMD_LOGS_RESOURCES_PATH'    : ('{SYSTEMD_ROOT_RESOURCES_PATH}', 'logs'),
 }
 
 def set_root(root: Union[Path, str]):

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.3.0.dev3"
+__version__ = "2.3.0rc2"

meerschaum/config/static/__init__.py

@@ -70,6 +70,9 @@ STATIC_CONFIG: Dict[str, Any] = {
         'noask': 'MRSM_NOASK',
         'id': 'MRSM_SERVER_ID',
         'daemon_id': 'MRSM_DAEMON_ID',
+        'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
+        'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
+        'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },
@@ -85,6 +88,8 @@ STATIC_CONFIG: Dict[str, Any] = {
             ),
             'underscore_standin': '<UNDERSCORE>', ### Temporary replacement for parsing.
             'failure_key': '_argparse_exception',
+            'and_key': '+',
+            'escaped_and_key': '++',
         },
         'urls': {
             'get-pip.py': 'https://bootstrap.pypa.io/get-pip.py',

meerschaum/connectors/Connector.py

@@ -21,11 +21,11 @@ class Connector(metaclass=abc.ABCMeta):
     The base connector class to hold connection attributes.
     """
     def __init__(
-            self,
-            type: Optional[str] = None,
-            label: Optional[str] = None,
-            **kw: Any
-        ):
+        self,
+        type: Optional[str] = None,
+        label: Optional[str] = None,
+        **kw: Any
+    ):
         """
         Set the given keyword arguments as attributes.
 
@@ -101,7 +101,7 @@ class Connector(metaclass=abc.ABCMeta):
 
         ### load user config into self._attributes
         if self.type in conn_configs and self.label in conn_configs[self.type]:
-            self._attributes.update(conn_configs[self.type][self.label])
+            self._attributes.update(conn_configs[self.type][self.label] or {})
 
         ### load system config into self._sys_config
         ### (deep copy so future Connectors don't inherit changes)
@@ -200,7 +200,13 @@ class Connector(metaclass=abc.ABCMeta):
         _type = self.__dict__.get('type', None)
         if _type is None:
             import re
-            _type = re.sub(r'connector$', '', self.__class__.__name__.lower())
+            is_executor = self.__class__.__name__.lower().endswith('executor')
+            suffix_regex = (
+                r'connector$'
+                if not is_executor
+                else r'executor$'
+            )
+            _type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
             self.__dict__['type'] = _type
         return _type
 

meerschaum/connectors/__init__.py

@@ -36,9 +36,9 @@ __all__ = (
 ### store connectors partitioned by
 ### type, label for reuse
 connectors: Dict[str, Dict[str, Connector]] = {
-    'api'   : {},
-    'sql'   : {},
-    'plugin': {},
+    'api'    : {},
+    'sql'    : {},
+    'plugin' : {},
 }
 instance_types: List[str] = ['sql', 'api']
 _locks: Dict[str, RLock] = {
@@ -127,10 +127,13 @@ def get_connector(
     global _loaded_plugin_connectors
     if isinstance(type, str) and not label and ':' in type:
         type, label = type.split(':', maxsplit=1)
+
     with _locks['_loaded_plugin_connectors']:
         if not _loaded_plugin_connectors:
             load_plugin_connectors()
+            _load_builtin_custom_connectors()
             _loaded_plugin_connectors = True
+
     if type is None and label is None:
         default_instance_keys = get_config('meerschaum', 'instance', patch=True)
         ### recursive call to get_connector
@@ -274,7 +277,7 @@ def is_connected(keys: str, **kw) -> bool:
         return False
 
 
-def make_connector(cls):
+def make_connector(cls, _is_executor: bool = False):
     """
     Register a class as a `Connector`.
     The `type` will be the lower case of the class name, without the suffix `connector`.
@@ -300,7 +303,12 @@ def make_connector(cls):
     >>> 
     """
     import re
-    typ = re.sub(r'connector$', '', cls.__name__.lower())
+    suffix_regex = (
+        r'connector$'
+        if not _is_executor
+        else r'executor$'
+    )
+    typ = re.sub(suffix_regex, '', cls.__name__.lower())
     with _locks['types']:
         types[typ] = cls
     with _locks['custom_types']:
@@ -363,3 +371,11 @@ def get_connector_plugin(
     )
     plugin = mrsm.Plugin(plugin_name)
     return plugin if plugin.is_installed() else None
+
+
+def _load_builtin_custom_connectors():
+    """
+    Import custom connectors decorated with `@make_connector` or `@make_executor`.
+    """
+    import meerschaum.jobs._SystemdExecutor
+    #  import meerschaum.jobs._LocalExecutor

meerschaum/connectors/api/APIConnector.py

@@ -88,6 +88,9 @@ class APIConnector(Connector):
         monitor_logs,
         monitor_logs_async,
         get_job_is_blocking_on_stdin,
+        get_job_began,
+        get_job_ended,
+        get_job_status,
     )
 
     def __init__(

meerschaum/connectors/api/_jobs.py

@@ -13,7 +13,7 @@ from datetime import datetime
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable
-from meerschaum.utils.jobs import Job
+from meerschaum.jobs import Job
 from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.warnings import warn, dprint
 
@@ -98,7 +98,6 @@ def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
     }
     return metadata
 
-
 def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
     """
     Return the daemon properties for a single job.
@@ -106,6 +105,34 @@ def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
     metadata = self.get_job_metadata(name, debug=debug)
     return metadata.get('daemon', {}).get('properties', {})
 
+def get_job_status(self, name: str, debug: bool = False) -> str:
+    """
+    Return the job's status.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    return metadata.get('status', 'stopped')
+
+def get_job_began(self, name: str, debug: bool = False) -> str:
+    """
+    Return a job's `began` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    began_str = properties.get('daemon', {}).get('began', None)
+    if began_str is None:
+        return None
+
+    return began_str
+
+def get_job_ended(self, name: str, debug: bool = False) -> str:
+    """
+    Return a job's `ended` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    ended_str = properties.get('daemon', {}).get('ended', None)
+    if ended_str is None:
+        return None
+
+    return ended_str
 
 def get_job_exists(self, name: str, debug: bool = False) -> bool:
     """
@@ -226,7 +253,7 @@ async def monitor_logs_async(
     """
     Monitor a job's log files and await a callback with the changes.
     """
-    from meerschaum.utils.jobs import StopMonitoringLogs
+    from meerschaum.jobs import StopMonitoringLogs
     from meerschaum.utils.formatting._jobs import strip_timestamp_from_line
 
     websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')

meerschaum/connectors/parse.py

@@ -87,19 +87,17 @@ def parse_connector_keys(
 
 
 def parse_instance_keys(
-        keys: Optional[str],
-        construct: bool = True,
-        as_tuple: bool = False,
-        **kw
-    ):
+    keys: Optional[str],
+    construct: bool = True,
+    as_tuple: bool = False,
+    **kw
+):
     """
     Parse the Meerschaum instance value into a Connector object.
     """
     from meerschaum.utils.warnings import warn
     from meerschaum.config import get_config
 
-    ### TODO Check for valid types? Not sure how to do that if construct = False.
-
     if keys is None:
         keys = get_config('meerschaum', 'instance')
     keys = str(keys)
@@ -120,25 +118,23 @@ def parse_repo_keys(keys: Optional[str] = None, **kw):
 
 
 def parse_executor_keys(keys: Optional[str] = None, **kw):
-    """Parse the executor keys into an APIConnector or None."""
-    from meerschaum.config import get_config
+    """Parse the executor keys into an APIConnector or string."""
+    from meerschaum.jobs import get_executor_keys_from_context
     if keys is None:
-        keys = get_config('meerschaum', 'default_executor')
+        keys = get_executor_keys_from_context()
 
     if keys is None or keys == 'local':
         return 'local'
 
     keys = str(keys)
-    if ':' not in keys:
-        keys = 'api:' + keys
-
     return parse_connector_keys(keys, **kw)
 
 
 def is_valid_connector_keys(
     keys: str
 ) -> bool:
-    """Verify a connector_keys string references a valid connector.
+    """
+    Verify a connector_keys string references a valid connector.
     """
     try:
         success = parse_connector_keys(keys, construct=False) is not None

meerschaum/core/Pipe/_bootstrap.py

@@ -88,21 +88,29 @@ def bootstrap(
 
     try:
         if yes_no(
-            f"Would you like to edit the definition for {self}?", yes=yes, noask=noask
+            f"Would you like to edit the definition for {self}?",
+            yes=yes,
+            noask=noask,
+            default='n',
         ):
             edit_tuple = self.edit_definition(debug=debug)
             if not edit_tuple[0]:
                 return edit_tuple
 
-        if yes_no(f"Would you like to try syncing {self} now?", yes=yes, noask=noask):
+        if yes_no(
+            f"Would you like to try syncing {self} now?",
+            yes=yes,
+            noask=noask,
+            default='n',
+        ):
             sync_tuple = actions['sync'](
                 ['pipes'],
-                connector_keys = [self.connector_keys],
-                metric_keys = [self.metric_key],
-                location_keys = [self.location_key],
-                mrsm_instance = str(self.instance_connector),
-                debug = debug,
-                shell = shell,
+                connector_keys=[self.connector_keys],
+                metric_keys=[self.metric_key],
+                location_keys=[self.location_key],
+                mrsm_instance=str(self.instance_connector),
+                debug=debug,
+                shell=shell,
             )
             if not sync_tuple[0]:
                 return sync_tuple

meerschaum/jobs/_Executor.py

@@ -0,0 +1,69 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define the base class for a Job executor.
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+
+from meerschaum.connectors import Connector
+from meerschaum.utils.typing import List, Dict, SuccessTuple, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from meerschaum.jobs import Job
+
+class Executor(Connector):
+    """
+    Define the methods for managing jobs.
+    """
+
+    @abstractmethod
+    def get_job_exists(self, name: str, debug: bool = False) -> bool:
+        """
+        Return whether a job exists.
+        """
+    
+    @abstractmethod
+    def get_jobs(self) -> Dict[str, Job]:
+        """
+        Return a dictionary of names -> Jobs.
+        """
+
+    @abstractmethod
+    def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
+        """
+        Create a new job.
+        """
+
+    @abstractmethod
+    def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Start a job.
+        """
+
+    @abstractmethod
+    def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Stop a job.
+        """
+
+    @abstractmethod
+    def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Pause a job.
+        """
+
+    @abstractmethod
+    def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
+        """
+        Delete a job.
+        """
+
+    @abstractmethod
+    def get_logs(self, name: str, debug: bool = False) -> str:
+        """
+        Return a job's log output.
+        """