meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl

meerschaum/config/_paths.py

@@ -123,6 +123,10 @@ paths = {
     'PERMANENT_PATCH_DIR_PATH'       : ('{ROOT_DIR_PATH}', 'permanent_patch_config'),
     'INTERNAL_RESOURCES_PATH'        : ('{ROOT_DIR_PATH}', '.internal'),
 
+    'UPDATES_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'updates'),
+    'UPDATES_CACHE_PATH'             : ('{UPDATES_RESOURCES_PATH}', 'cache.json'),
+    'UPDATES_LOCK_PATH'              : ('{UPDATES_RESOURCES_PATH}', '.updates.lock'),
+
     'STACK_RESOURCES_PATH'           : ('{ROOT_DIR_PATH}', 'stack'),
     'STACK_COMPOSE_FILENAME'         : 'docker-compose.yaml',
     'STACK_COMPOSE_PATH'             : ('{STACK_RESOURCES_PATH}', '{STACK_COMPOSE_FILENAME}'),
@@ -176,6 +180,13 @@ paths = {
     'DAEMON_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'jobs'),
     'LOGS_RESOURCES_PATH'            : ('{ROOT_DIR_PATH}', 'logs'),
     'DAEMON_ERROR_LOG_PATH'          : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
+    'CHECK_JOBS_LOCK_PATH'           : ('{INTERNAL_RESOURCES_PATH}', 'check-jobs.lock'),
+    
+    'SYSTEMD_RESOURCES_PATH'         : ('{DOT_CONFIG_DIR_PATH}', 'systemd'),
+    'SYSTEMD_USER_RESOURCES_PATH'    : ('{SYSTEMD_RESOURCES_PATH}', 'user'),
+    'SYSTEMD_ROOT_RESOURCES_PATH'    : ('{ROOT_DIR_PATH}', 'systemd'),
+    'SYSTEMD_JOBS_RESOURCES_PATH'    : ('{SYSTEMD_ROOT_RESOURCES_PATH}', 'services'),
+    'SYSTEMD_LOGS_RESOURCES_PATH'    : ('{SYSTEMD_ROOT_RESOURCES_PATH}', 'logs'),
 }
 
 def set_root(root: Union[Path, str]):

meerschaum/config/_shell.py

@@ -6,127 +6,144 @@
 Default configuration for the Meerschaum shell.
 """
 
-#  import platform
-#  default_cmd = 'cmd' if platform.system() != 'Windows' else 'cmd2'
 default_cmd = 'cmd'
 
 default_shell_config = {
-    'ansi'             : {
-        'intro'        : {
-            'rich'     : {
-                'style' : "bold bright_blue",
+    'ansi'               : {
+        'intro'          : {
+            'rich'       : {
+                'style'  : "bold bright_blue",
             },
-            'color'    : [
+            'color'      : [
                 'bold',
                 'bright blue',
             ],
         },
-        'close_message': {
-            'rich' : {
-                'style' : 'bright_blue',
+        'close_message'  : {
+            'rich'       : {
+                'style'  : 'bright_blue',
             },
-            'color'    : [
+            'color'      : [
                 'bright blue',
             ],
         },
-        'doc_header': {
-            'rich' : {
-                'style' : 'bright_blue',
+        'doc_header'     : {
+            'rich'       : {
+                'style'  : 'bright_blue',
             },
-            'color'    : [
+            'color'      : [
                 'bright blue',
             ],
         },
-        'undoc_header': {
-            'rich' : {
-                'style' : 'bright_blue',
+        'undoc_header'   : {
+            'rich'       : {
+                'style'  : 'bright_blue',
             },
-            'color'    : [
+            'color'      : [
                 'bright blue',
             ],
         },
-        'ruler': {
-            'rich' : {
-                'style' : 'bold bright_blue',
+        'ruler'          : {
+            'rich'       : {
+                'style'  : 'bold bright_blue',
             },
-            'color'    : [
+            'color'      : [
                 'bold',
                 'bright blue',
             ],
         },
-        'prompt': {
-            'rich' : {
-                'style' : 'green',
+        'prompt'         : {
+            'rich'       : {
+                'style'  : 'green',
             },
-            'color'    : [
+            'color'      : [
                 'green',
             ],
         },
-        'instance' : {
-            'rich' : {
-                'style' : 'cyan',
+        'instance'       : {
+            'rich'       : {
+                'style'  : 'cyan',
             },
-            'color'    : [
+            'color'      : [
                 'cyan',
             ],
         },
-        'repo' : {
-            'rich': {
-                'style': 'magenta',
+        'repo'           : {
+            'rich'       : {
+                'style'  : 'magenta',
             },
-            'color': [
+            'color'      : [
                 'magenta',
             ],
         },
-        'username' : {
-            'rich' : {
-                'style' : 'white',
+        'executor'       : {
+            'rich'       : {
+                'style'  : 'yellow',
             },
-            'color'    : [
+        },
+        'username'       : {
+            'rich'       : {
+                'style'  : 'white',
+            },
+            'color'      : [
                 'white',
             ],
         },
-        'connected' : {
-            'rich' : {
-                'style' : 'green',
+        'connected'      : {
+            'rich'       : {
+                'style'  : 'green',
             },
-            'color'    : [
+            'color'      : [
                 'green',
             ],
         },
-        'disconnected' : {
-            'rich' : {
-                'style' : 'red',
+        'disconnected'   : {
+            'rich'       : {
+                'style'  : 'red',
             },
-            'color'    : [
+            'color'      : [
+                'red',
+            ],
+        },
+        'update_message' : {
+            'rich'       : {
+                'style'  : 'red',
+            },
+            'color'      : [
                 'red',
             ],
         },
     },
-    'ascii'            : {
-        'intro'        : """       ___  ___  __   __   __
+    'ascii'              : {
+        'intro'          : r"""       ___  ___  __   __   __
  |\/| |__  |__  |__) /__` /  ` |__|  /\  |  |  |\/|
- |  | |___ |___ |  \ .__/ \__, |  | /~~\ \__/  |  |\n""",
-        'prompt'       : '\n [ {username}@{instance} ] > ',
-        'ruler'        : '-',
-        'close_message': 'Thank you for using Meerschaum!',
-        'doc_header'   : 'Meerschaum actions (`help <action>` for usage):',
-        'undoc_header' : 'Unimplemented actions:',
+ |  | |___ |___ |  \ .__/ \__, |  | /~~\ \__/  |  |""" + '\n',
+        'prompt'         : '\n [ {username}@{instance} | {executor_keys} ] > ',
+        'ruler'          : '-',
+        'close_message'  : 'Thank you for using Meerschaum!',
+        'doc_header'     : 'Meerschaum actions (`help <action>` for usage):',
+        'undoc_header'   : 'Unimplemented actions:',
+        'update_message' : "Update available!",
     },
-    'unicode'          : {
-        'intro'        : """
+    'unicode'            : {
+        'intro'          : """
  █▄ ▄█ ██▀ ██▀ █▀▄ ▄▀▀ ▄▀▀ █▄█ ▄▀▄ █ █ █▄ ▄█
  █ ▀ █ █▄▄ █▄▄ █▀▄ ▄██ ▀▄▄ █ █ █▀█ ▀▄█ █ ▀ █\n""",
-        'prompt'       : '\n [ {username}@{instance} ] ➤ ',
-        'ruler'        : '─',
-        'close_message': ' MRSM{formatting:emoji:hand} Thank you for using Meerschaum! ',
-        'doc_header'   : 'Meerschaum actions (`help <action>` for usage):',
-        'undoc_header' : 'Unimplemented actions:',
+        'prompt'         : '\n [ {username}@{instance} | {executor_keys} ] ➤ ',
+        'ruler'          : '─',
+        'close_message'  : ' MRSM{formatting:emoji:hand} Thank you for using Meerschaum! ',
+        'doc_header'     : 'Meerschaum actions (`help <action>` for usage):',
+        'undoc_header'   : 'Unimplemented actions:',
+        'update_message' : "MRSM{formatting:emoji:announcement} Update available!",
+    },
+    'timeout'            : 60,
+    'max_history'        : 1000,
+    'clear_screen'       : True,
+    'bottom_toolbar'     : {
+        'enabled'        : True,
     },
-    'timeout'          : 60,
-    'max_history'      : 1000,
-    'clear_screen'     : True,
-    'bottom_toolbar'   : {
-        'enabled'      : True,
+    'updates'            : {
+        'check_remote'   : True,
+        'refresh_minutes': 180,
     },
 }

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.2.6"
+__version__ = "2.3.0"

meerschaum/config/static/__init__.py

@@ -23,6 +23,8 @@ STATIC_CONFIG: Dict[str, Any] = {
             'pipes': '/pipes',
             'metadata': '/metadata',
             'actions': '/actions',
+            'jobs': '/jobs',
+            'logs': '/logs',
             'users': '/users',
             'login': '/login',
             'connectors': '/connectors',
@@ -39,6 +41,12 @@ STATIC_CONFIG: Dict[str, Any] = {
             'token_expires_minutes': 720,
         },
         'webterm_job_name': '_webterm',
+        'default_timeout': 600,
+        'jobs': {
+            'stdin_message': 'MRSM_STDIN',
+            'stop_message': 'MRSM_STOP',
+            'metadata_cache_seconds': 5,
+        },
     },
     'sql': {
         'internal_schema': '_mrsm_internal',
@@ -62,6 +70,9 @@ STATIC_CONFIG: Dict[str, Any] = {
         'noask': 'MRSM_NOASK',
         'id': 'MRSM_SERVER_ID',
         'daemon_id': 'MRSM_DAEMON_ID',
+        'systemd_log_path': 'MRSM_SYSTEMD_LOG_PATH',
+        'systemd_stdin_path': 'MRSM_SYSTEMD_STDIN_PATH',
+        'systemd_result_path': 'MRSM_SYSTEMD_RESULT_PATH',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },
@@ -77,6 +88,10 @@ STATIC_CONFIG: Dict[str, Any] = {
             ),
             'underscore_standin': '<UNDERSCORE>', ### Temporary replacement for parsing.
             'failure_key': '_argparse_exception',
+            'and_key': '+',
+            'escaped_and_key': '++',
+            'pipeline_key': ':',
+            'escaped_pipeline_key': '::',
         },
         'urls': {
             'get-pip.py': 'https://bootstrap.pypa.io/get-pip.py',
@@ -128,6 +143,9 @@ STATIC_CONFIG: Dict[str, Any] = {
         },
         'exists_timeout_seconds': 5.0,
     },
+    'jobs': {
+        'check_restart_seconds': 1.0,
+    },
     'setup': {
         'name': 'meerschaum',
         'formal_name': 'Meerschaum',

meerschaum/connectors/Connector.py

@@ -21,11 +21,11 @@ class Connector(metaclass=abc.ABCMeta):
     The base connector class to hold connection attributes.
     """
     def __init__(
-            self,
-            type: Optional[str] = None,
-            label: Optional[str] = None,
-            **kw: Any
-        ):
+        self,
+        type: Optional[str] = None,
+        label: Optional[str] = None,
+        **kw: Any
+    ):
         """
         Set the given keyword arguments as attributes.
 
@@ -101,7 +101,7 @@ class Connector(metaclass=abc.ABCMeta):
 
         ### load user config into self._attributes
         if self.type in conn_configs and self.label in conn_configs[self.type]:
-            self._attributes.update(conn_configs[self.type][self.label])
+            self._attributes.update(conn_configs[self.type][self.label] or {})
 
         ### load system config into self._sys_config
         ### (deep copy so future Connectors don't inherit changes)
@@ -200,7 +200,13 @@ class Connector(metaclass=abc.ABCMeta):
         _type = self.__dict__.get('type', None)
         if _type is None:
             import re
-            _type = re.sub(r'connector$', '', self.__class__.__name__.lower())
+            is_executor = self.__class__.__name__.lower().endswith('executor')
+            suffix_regex = (
+                r'connector$'
+                if not is_executor
+                else r'executor$'
+            )
+            _type = re.sub(suffix_regex, '', self.__class__.__name__.lower())
             self.__dict__['type'] = _type
         return _type
 

meerschaum/connectors/__init__.py

@@ -36,9 +36,9 @@ __all__ = (
 ### store connectors partitioned by
 ### type, label for reuse
 connectors: Dict[str, Dict[str, Connector]] = {
-    'api'   : {},
-    'sql'   : {},
-    'plugin': {},
+    'api'    : {},
+    'sql'    : {},
+    'plugin' : {},
 }
 instance_types: List[str] = ['sql', 'api']
 _locks: Dict[str, RLock] = {
@@ -70,12 +70,12 @@ _loaded_plugin_connectors: bool = False
 
 
 def get_connector(
-        type: str = None,
-        label: str = None,
-        refresh: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Connector:
+    type: str = None,
+    label: str = None,
+    refresh: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Connector:
     """
     Return existing connector or create new connection and store for reuse.
     
@@ -127,10 +127,13 @@ def get_connector(
     global _loaded_plugin_connectors
     if isinstance(type, str) and not label and ':' in type:
         type, label = type.split(':', maxsplit=1)
+
     with _locks['_loaded_plugin_connectors']:
         if not _loaded_plugin_connectors:
             load_plugin_connectors()
+            _load_builtin_custom_connectors()
             _loaded_plugin_connectors = True
+
     if type is None and label is None:
         default_instance_keys = get_config('meerschaum', 'instance', patch=True)
         ### recursive call to get_connector
@@ -274,9 +277,7 @@ def is_connected(keys: str, **kw) -> bool:
         return False
 
 
-def make_connector(
-        cls,
-    ):
+def make_connector(cls, _is_executor: bool = False):
     """
     Register a class as a `Connector`.
     The `type` will be the lower case of the class name, without the suffix `connector`.
@@ -302,7 +303,12 @@ def make_connector(
     >>> 
     """
     import re
-    typ = re.sub(r'connector$', '', cls.__name__.lower())
+    suffix_regex = (
+        r'connector$'
+        if not _is_executor
+        else r'executor$'
+    )
+    typ = re.sub(suffix_regex, '', cls.__name__.lower())
     with _locks['types']:
         types[typ] = cls
     with _locks['custom_types']:
@@ -338,8 +344,8 @@ def load_plugin_connectors():
 
 
 def get_connector_plugin(
-        connector: Connector,
-    ) -> Union[str, None, mrsm.Plugin]:
+    connector: Connector,
+) -> Union[str, None, mrsm.Plugin]:
     """
     Determine the plugin for a connector.
     This is useful for handling virtual environments for custom instance connectors.
@@ -365,3 +371,10 @@ def get_connector_plugin(
     )
     plugin = mrsm.Plugin(plugin_name)
     return plugin if plugin.is_installed() else None
+
+
+def _load_builtin_custom_connectors():
+    """
+    Import custom connectors decorated with `@make_connector` or `@make_executor`.
+    """
+    import meerschaum.jobs.systemd

meerschaum/connectors/api/APIConnector.py

@@ -33,7 +33,12 @@ class APIConnector(Connector):
         delete,
         wget,
     )
-    from ._actions import get_actions, do_action
+    from ._actions import (
+        get_actions,
+        do_action,
+        do_action_async,
+        do_action_legacy,
+    )
     from ._misc import get_mrsm_version, get_chaining_status
     from ._pipes import (
         register_pipe,
@@ -72,6 +77,27 @@ class APIConnector(Connector):
         get_user_attributes,
     )
     from ._uri import from_uri
+    from ._jobs import (
+        get_jobs,
+        get_job,
+        get_job_metadata,
+        get_job_properties,
+        get_job_exists,
+        delete_job,
+        start_job,
+        create_job,
+        stop_job,
+        pause_job,
+        get_logs,
+        get_job_stop_time,
+        monitor_logs,
+        monitor_logs_async,
+        get_job_is_blocking_on_stdin,
+        get_job_began,
+        get_job_ended,
+        get_job_paused,
+        get_job_status,
+    )
 
     def __init__(
         self,

meerschaum/connectors/api/_actions.py

@@ -7,22 +7,87 @@ Functions to interact with /mrsm/actions
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Optional, List
 
-def get_actions(self) -> list:
+import json
+import asyncio
+from functools import partial
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import SuccessTuple, List, Callable, Optional
+from meerschaum.config.static import STATIC_CONFIG
+
+ACTIONS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['actions']
+
+
+def get_actions(self):
     """Get available actions from the API instance."""
-    from meerschaum.config.static import STATIC_CONFIG
-    return self.get(STATIC_CONFIG['api']['endpoints']['actions'])
+    return self.get(ACTIONS_ENDPOINT)
+
+
+def do_action(self, sysargs: List[str]) -> SuccessTuple:
+    """
+    Execute a Meerschaum action remotely.
+    """
+    return asyncio.run(self.do_action_async(sysargs))
+
+
+async def do_action_async(
+    self,
+    sysargs: List[str],
+    callback_function: Callable[[str], None] = partial(print, end=''),
+) -> SuccessTuple:
+    """
+    Monitor a job's log files and await a callback with the changes.
+    """
+    websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
+    protocol = 'ws' if self.URI.startswith('http://') else 'wss'
+    port = self.port if 'port' in self.__dict__ else ''
+    uri = f"{protocol}://{self.host}:{port}{ACTIONS_ENDPOINT}/ws"
+    if sysargs and sysargs[0] == 'api' and len(sysargs) > 2:
+        sysargs = sysargs[2:]
+
+    sysargs_str = json.dumps(sysargs)
+
+    async with websockets.connect(uri) as websocket:
+        try:
+            await websocket.send(self.token or 'no-login')
+            response = await websocket.recv()
+            init_data = json.loads(response)
+            if not init_data.get('is_authenticated'):
+                return False, "Cannot authenticate with actions endpoint."
 
+            await websocket.send(sysargs_str)
+        except websockets_exceptions.ConnectionClosedOK:
+            return False, "Connection was closed."
 
-def do_action(
+        while True:
+            try:
+                line = await websocket.recv()
+                if asyncio.iscoroutinefunction(callback_function):
+                    await callback_function(line)
+                else:
+                    callback_function(line)
+            except KeyboardInterrupt:
+                await websocket.close()
+                break
+            except websockets_exceptions.ConnectionClosedOK:
+                break
+
+    return True, "Success"
+
+
+def do_action_legacy(
     self,
     action: Optional[List[str]] = None,
     sysargs: Optional[List[str]] = None,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
-    """Execute a Meerschaum action remotely.
+    """
+    NOTE: This method is deprecated.
+    Please use `do_action()` or `do_action_async()`.
+
+    Execute a Meerschaum action remotely.
 
     If `sysargs` are provided, parse those instead.
     Otherwise infer everything from keyword arguments.