meerschaum 2.2.7__py3-none-any.whl → 2.3.0__py3-none-any.whl

meerschaum/config/static/__init__.py

@@ -23,6 +23,8 @@ STATIC_CONFIG: Dict[str, Any] = {
             'pipes': '/pipes',
             'metadata': '/metadata',
             'actions': '/actions',
+            'jobs': '/jobs',
+            'logs': '/logs',
             'users': '/users',
             'login': '/login',
             'connectors': '/connectors',
@@ -40,6 +42,11 @@ STATIC_CONFIG: Dict[str, Any] = {
         },
         'webterm_job_name': '_webterm',
         'default_timeout': 600,
+        'jobs': {
+            'stdin_message': 'MRSM_STDIN',
+            'stop_message': 'MRSM_STOP',
+            'metadata_cache_seconds': 5,
+        },
     },
     'sql': {
         'internal_schema': '_mrsm_internal',
@@ -63,6 +70,9 @@ STATIC_CONFIG: Dict[str, Any] = {
         'noask': 'MRSM_NOASK',
         'id': 'MRSM_SERVER_ID',
         'daemon_id': 'MRSM_DAEMON_ID',
+        'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
+        'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
+        'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },
@@ -78,6 +88,10 @@ STATIC_CONFIG: Dict[str, Any] = {
             ),
             'underscore_standin': '<UNDERSCORE>', ### Temporary replacement for parsing.
             'failure_key': '_argparse_exception',
+            'and_key': '+',
+            'escaped_and_key': '++',
+            'pipeline_key': ':',
+            'escaped_pipeline_key': '::',
         },
         'urls': {
             'get-pip.py': 'https://bootstrap.pypa.io/get-pip.py',
@@ -129,6 +143,9 @@ STATIC_CONFIG: Dict[str, Any] = {
         },
         'exists_timeout_seconds': 5.0,
     },
+    'jobs': {
+        'check_restart_seconds': 1.0,
+    },
     'setup': {
         'name': 'meerschaum',
         'formal_name': 'Meerschaum',

meerschaum/connectors/Connector.py

@@ -21,11 +21,11 @@ class Connector(metaclass=abc.ABCMeta):
     The base connector class to hold connection attributes.
     """
     def __init__(
-            self,
-            type: Optional[str] = None,
-            label: Optional[str] = None,
-            **kw: Any
-        ):
+        self,
+        type: Optional[str] = None,
+        label: Optional[str] = None,
+        **kw: Any
+    ):
         """
         Set the given keyword arguments as attributes.
 
@@ -101,7 +101,7 @@ class Connector(metaclass=abc.ABCMeta):
 
         ### load user config into self._attributes
         if self.type in conn_configs and self.label in conn_configs[self.type]:
-            self._attributes.update(conn_configs[self.type][self.label])
+            self._attributes.update(conn_configs[self.type][self.label] or {})
 
         ### load system config into self._sys_config
         ### (deep copy so future Connectors don't inherit changes)
@@ -200,7 +200,13 @@ class Connector(metaclass=abc.ABCMeta):
         _type = self.__dict__.get('type', None)
         if _type is None:
             import re
-            _type = re.sub(r'connector$', '', self.__class__.__name__.lower())
+            is_executor = self.__class__.__name__.lower().endswith('executor')
+            suffix_regex = (
+                r'connector$'
+                if not is_executor
+                else r'executor$'
+            )
+            _type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
             self.__dict__['type'] = _type
         return _type
 

meerschaum/connectors/__init__.py

@@ -36,9 +36,9 @@ __all__ = (
 ### store connectors partitioned by
 ### type, label for reuse
 connectors: Dict[str, Dict[str, Connector]] = {
-    'api'   : {},
-    'sql'   : {},
-    'plugin': {},
+    'api'    : {},
+    'sql'    : {},
+    'plugin' : {},
 }
 instance_types: List[str] = ['sql', 'api']
 _locks: Dict[str, RLock] = {
@@ -70,12 +70,12 @@ _loaded_plugin_connectors: bool = False
 
 
 def get_connector(
-        type: str = None,
-        label: str = None,
-        refresh: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Connector:
+    type: str = None,
+    label: str = None,
+    refresh: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Connector:
     """
     Return existing connector or create new connection and store for reuse.
     
@@ -127,10 +127,13 @@ def get_connector(
     global _loaded_plugin_connectors
     if isinstance(type, str) and not label and ':' in type:
         type, label = type.split(':', maxsplit=1)
+
     with _locks['_loaded_plugin_connectors']:
         if not _loaded_plugin_connectors:
             load_plugin_connectors()
+            _load_builtin_custom_connectors()
             _loaded_plugin_connectors = True
+
     if type is None and label is None:
         default_instance_keys = get_config('meerschaum', 'instance', patch=True)
         ### recursive call to get_connector
@@ -274,9 +277,7 @@ def is_connected(keys: str, **kw) -> bool:
         return False
 
 
-def make_connector(
-        cls,
-    ):
+def make_connector(cls, _is_executor: bool = False):
     """
     Register a class as a `Connector`.
     The `type` will be the lower case of the class name, without the suffix `connector`.
@@ -302,7 +303,12 @@ def make_connector(
     >>> 
     """
     import re
-    typ = re.sub(r'connector$', '', cls.__name__.lower())
+    suffix_regex = (
+        r'connector$'
+        if not _is_executor
+        else r'executor$'
+    )
+    typ = re.sub(suffix_regex, '', cls.__name__.lower())
     with _locks['types']:
         types[typ] = cls
     with _locks['custom_types']:
@@ -338,8 +344,8 @@ def load_plugin_connectors():
 
 
 def get_connector_plugin(
-        connector: Connector,
-    ) -> Union[str, None, mrsm.Plugin]:
+    connector: Connector,
+) -> Union[str, None, mrsm.Plugin]:
     """
     Determine the plugin for a connector.
     This is useful for handling virtual environments for custom instance connectors.
@@ -365,3 +371,10 @@ def get_connector_plugin(
     )
     plugin = mrsm.Plugin(plugin_name)
     return plugin if plugin.is_installed() else None
+
+
+def _load_builtin_custom_connectors():
+    """
+    Import custom connectors decorated with `@make_connector` or `@make_executor`.
+    """
+    import meerschaum.jobs.systemd

meerschaum/connectors/api/APIConnector.py

@@ -33,7 +33,12 @@ class APIConnector(Connector):
         delete,
         wget,
     )
-    from ._actions import get_actions, do_action
+    from ._actions import (
+        get_actions,
+        do_action,
+        do_action_async,
+        do_action_legacy,
+    )
     from ._misc import get_mrsm_version, get_chaining_status
     from ._pipes import (
         register_pipe,
@@ -72,6 +77,27 @@ class APIConnector(Connector):
         get_user_attributes,
     )
     from ._uri import from_uri
+    from ._jobs import (
+        get_jobs,
+        get_job,
+        get_job_metadata,
+        get_job_properties,
+        get_job_exists,
+        delete_job,
+        start_job,
+        create_job,
+        stop_job,
+        pause_job,
+        get_logs,
+        get_job_stop_time,
+        monitor_logs,
+        monitor_logs_async,
+        get_job_is_blocking_on_stdin,
+        get_job_began,
+        get_job_ended,
+        get_job_paused,
+        get_job_status,
+    )
 
     def __init__(
         self,

meerschaum/connectors/api/_actions.py

@@ -7,22 +7,87 @@ Functions to interact with /mrsm/actions
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Optional, List
 
-def get_actions(self) -> list:
+import json
+import asyncio
+from functools import partial
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import SuccessTuple, List, Callable, Optional
+from meerschaum.config.static import STATIC_CONFIG
+
+ACTIONS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['actions']
+
+
+def get_actions(self):
     """Get available actions from the API instance."""
-    from meerschaum.config.static import STATIC_CONFIG
-    return self.get(STATIC_CONFIG['api']['endpoints']['actions'])
+    return self.get(ACTIONS_ENDPOINT)
+
+
+def do_action(self, sysargs: List[str]) -> SuccessTuple:
+    """
+    Execute a Meerschaum action remotely.
+    """
+    return asyncio.run(self.do_action_async(sysargs))
+
+
+async def do_action_async(
+    self,
+    sysargs: List[str],
+    callback_function: Callable[[str], None] = partial(print, end=''),
+) -> SuccessTuple:
+    """
+    Monitor a job's log files and await a callback with the changes.
+    """
+    websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
+    protocol = 'ws' if self.URI.startswith('http://') else 'wss'
+    port = self.port if 'port' in self.__dict__ else ''
+    uri = f"{protocol}://{self.host}:{port}{ACTIONS_ENDPOINT}/ws"
+    if sysargs and sysargs[0] == 'api' and len(sysargs) > 2:
+        sysargs = sysargs[2:]
+
+    sysargs_str = json.dumps(sysargs)
+
+    async with websockets.connect(uri) as websocket:
+        try:
+            await websocket.send(self.token or 'no-login')
+            response = await websocket.recv()
+            init_data = json.loads(response)
+            if not init_data.get('is_authenticated'):
+                return False, "Cannot authenticate with actions endpoint."
 
+            await websocket.send(sysargs_str)
+        except websockets_exceptions.ConnectionClosedOK:
+            return False, "Connection was closed."
 
-def do_action(
+        while True:
+            try:
+                line = await websocket.recv()
+                if asyncio.iscoroutinefunction(callback_function):
+                    await callback_function(line)
+                else:
+                    callback_function(line)
+            except KeyboardInterrupt:
+                await websocket.close()
+                break
+            except websockets_exceptions.ConnectionClosedOK:
+                break
+
+    return True, "Success"
+
+
+def do_action_legacy(
     self,
     action: Optional[List[str]] = None,
     sysargs: Optional[List[str]] = None,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
-    """Execute a Meerschaum action remotely.
+    """
+    NOTE: This method is deprecated.
+    Please use `do_action()` or `do_action_async()`.
+
+    Execute a Meerschaum action remotely.
 
     If `sysargs` are provided, parse those instead.
     Otherwise infer everything from keyword arguments.

meerschaum/connectors/api/_jobs.py

@@ -0,0 +1,368 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Manage jobs via the Meerschaum API.
+"""
+
+import asyncio
+import time
+import json
+from datetime import datetime
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable
+from meerschaum.jobs import Job
+from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.utils.warnings import warn, dprint
+
+JOBS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['jobs']
+LOGS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['logs']
+JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
+JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
+JOB_METADATA_CACHE_SECONDS: int = STATIC_CONFIG['api']['jobs']['metadata_cache_seconds']
+
+
+def get_jobs(self, debug: bool = False) -> Dict[str, Job]:
+    """
+    Return a dictionary of remote jobs.
+    """
+    response = self.get(JOBS_ENDPOINT, debug=debug)
+    if not response:
+        warn(f"Failed to get remote jobs from {self}.")
+        return {}
+    return {
+        name: Job(
+            name,
+            job_meta['sysargs'],
+            executor_keys=str(self),
+            _properties=job_meta['daemon']['properties']
+        )
+        for name, job_meta in response.json().items()
+    }
+
+
+def get_job(self, name: str, debug: bool = False) -> Job:
+    """
+    Return a single Job object.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    if not metadata:
+        raise ValueError(f"Job '{name}' does not exist.")
+
+    return Job(
+        name,
+        metadata['sysargs'],
+        executor_keys=str(self),
+        _properties=metadata['daemon']['properties'],
+    )
+
+
+def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
+    """
+    Return the metadata for a single job.
+    """
+    now = time.perf_counter()
+    _job_metadata_cache = self.__dict__.get('_job_metadata_cache', None)
+    _job_metadata_timestamp = (
+        _job_metadata_cache.get(name, {}).get('timestamp', None)
+    ) if _job_metadata_cache is not None else None
+
+    if (
+        _job_metadata_timestamp is not None
+        and (now - _job_metadata_timestamp) < JOB_METADATA_CACHE_SECONDS
+    ):
+        if debug:
+            dprint(f"Returning cached metadata for job '{name}'.")
+        return _job_metadata_cache[name]['metadata']
+
+    response = self.get(JOBS_ENDPOINT + f"/{name}", debug=debug)
+    if not response:
+        if debug:
+            msg = (
+                response.json()['detail']
+                if 'detail' in response.text
+                else response.text
+            )
+            warn(f"Failed to get metadata for job '{name}':\n{msg}")
+        return {}
+
+    metadata = response.json()
+    if _job_metadata_cache is None:
+        self._job_metadata_cache = {}
+
+    self._job_metadata_cache[name] = {
+        'timestamp': now,
+        'metadata': metadata,
+    }
+    return metadata
+
+def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
+    """
+    Return the daemon properties for a single job.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    return metadata.get('daemon', {}).get('properties', {})
+
+def get_job_status(self, name: str, debug: bool = False) -> str:
+    """
+    Return the job's status.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    return metadata.get('status', 'stopped')
+
+def get_job_began(self, name: str, debug: bool = False) -> Union[str, None]:
+    """
+    Return a job's `began` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    began_str = properties.get('daemon', {}).get('began', None)
+    if began_str is None:
+        return None
+
+    return began_str
+
+def get_job_ended(self, name: str, debug: bool = False) -> Union[str, None]:
+    """
+    Return a job's `ended` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    ended_str = properties.get('daemon', {}).get('ended', None)
+    if ended_str is None:
+        return None
+
+    return ended_str
+
+def get_job_paused(self, name: str, debug: bool = False) -> Union[str, None]:
+    """
+    Return a job's `paused` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    paused_str = properties.get('daemon', {}).get('paused', None)
+    if paused_str is None:
+        return None
+
+    return paused_str
+
+def get_job_exists(self, name: str, debug: bool = False) -> bool:
+    """
+    Return whether a job exists.
+    """
+    response = self.get(JOBS_ENDPOINT + f'/{name}/exists', debug=debug)
+    if not response:
+        warn(f"Failed to determine whether job '{name}' exists.")
+        return False
+
+    return response.json()
+
+
+def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Delete a job.
+    """
+    response = self.delete(JOBS_ENDPOINT + f"/{name}", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Start a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}/start", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
+    """
+    Create a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}", json=sysargs, debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Stop a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}/stop", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Pause a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}/pause", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def get_logs(self, name: str, debug: bool = False) -> str:
+    """
+    Return the logs for a job.
+    """
+    response = self.get(LOGS_ENDPOINT + f"/{name}")
+    if not response:
+        raise ValueError(f"Cannot fetch logs for job '{name}':\n{response.text}")
+
+    return response.json()
+
+
+def get_job_stop_time(self, name: str, debug: bool = False) -> Union[datetime, None]:
+    """
+    Return the job's manual stop time.
+    """
+    response = self.get(JOBS_ENDPOINT + f"/{name}/stop_time")
+    if not response:
+        warn(f"Failed to get stop time for job '{name}':\n{response.text}")
+        return None
+
+    data = response.json()
+    if data is None:
+        return None
+
+    return datetime.fromisoformat(data)
+
+
+async def monitor_logs_async(
+    self,
+    name: str,
+    callback_function: Callable[[Any], Any],
+    input_callback_function: Callable[[], str],
+    stop_callback_function: Callable[[SuccessTuple], str],
+    stop_on_exit: bool = False,
+    strip_timestamps: bool = False,
+    accept_input: bool = True,
+    debug: bool = False,
+):
+    """
+    Monitor a job's log files and await a callback with the changes.
+    """
+    from meerschaum.jobs import StopMonitoringLogs
+    from meerschaum.utils.formatting._jobs import strip_timestamp_from_line
+
+    websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
+    protocol = 'ws' if self.URI.startswith('http://') else 'wss'
+    port = self.port if 'port' in self.__dict__ else ''
+    uri = f"{protocol}://{self.host}:{port}{LOGS_ENDPOINT}/{name}/ws"
+
+    async def _stdin_callback(client):
+        if input_callback_function is None:
+            return
+
+        if asyncio.iscoroutinefunction(input_callback_function):
+            data = await input_callback_function()
+        else:
+            data = input_callback_function()
+
+        await client.send(data)
+
+    async def _stop_callback(client):
+        try:
+            result = tuple(json.loads(await client.recv()))
+        except Exception as e:
+            warn(traceback.format_exc())
+            result = False, str(e)
+
+        if stop_callback_function is not None:
+            if asyncio.iscoroutinefunction(stop_callback_function):
+                await stop_callback_function(result)
+            else:
+                stop_callback_function(result)
+
+        if stop_on_exit:
+            raise StopMonitoringLogs
+
+    message_callbacks = {
+        JOBS_STDIN_MESSAGE: _stdin_callback,
+        JOBS_STOP_MESSAGE: _stop_callback,
+    }
+
+    async with websockets.connect(uri) as websocket:
+        try:
+            await websocket.send(self.token or 'no-login')
+        except websockets_exceptions.ConnectionClosedOK:
+            pass
+
+        while True:
+            try:
+                response = await websocket.recv()
+                callback = message_callbacks.get(response, None)
+                if callback is not None:
+                    await callback(websocket)
+                    continue
+
+                if strip_timestamps:
+                    response = strip_timestamp_from_line(response)
+
+                if asyncio.iscoroutinefunction(callback_function):
+                    await callback_function(response)
+                else:
+                    callback_function(response)
+            except (KeyboardInterrupt, StopMonitoringLogs):
+                await websocket.close()
+                break
+
+
+def monitor_logs(
+    self,
+    name: str,
+    callback_function: Callable[[Any], Any],
+    input_callback_function: Callable[[None], str],
+    stop_callback_function: Callable[[None], str],
+    stop_on_exit: bool = False,
+    strip_timestamps: bool = False,
+    accept_input: bool = True,
+    debug: bool = False,
+):
+    """
+    Monitor a job's log files and execute a callback with the changes.
+    """
+    return asyncio.run(
+        self.monitor_logs_async(
+            name,
+            callback_function,
+            input_callback_function=input_callback_function,
+            stop_callback_function=stop_callback_function,
+            stop_on_exit=stop_on_exit,
+            strip_timestamps=strip_timestamps,
+            accept_input=accept_input,
+            debug=debug
+        )
+    )
+
+def get_job_is_blocking_on_stdin(self, name: str, debug: bool = False) -> bool:
+    """
+    Return whether a remote job is blocking on stdin.
+    """
+    response = self.get(JOBS_ENDPOINT + f'/{name}/is_blocking_on_stdin', debug=debug)
+    if not response:
+        return False
+
+    return response.json()