meerschaum 2.2.7__py3-none-any.whl → 2.3.0.dev1__py3-none-any.whl

meerschaum/connectors/api/APIConnector.py

@@ -33,7 +33,7 @@ class APIConnector(Connector):
         delete,
         wget,
     )
-    from ._actions import get_actions, do_action
+    from ._actions import get_actions, do_action, do_action_async
     from ._misc import get_mrsm_version, get_chaining_status
     from ._pipes import (
         register_pipe,
@@ -72,6 +72,23 @@ class APIConnector(Connector):
         get_user_attributes,
     )
     from ._uri import from_uri
+    from ._jobs import (
+        get_jobs,
+        get_job,
+        get_job_metadata,
+        get_job_properties,
+        get_job_exists,
+        delete_job,
+        start_job,
+        create_job,
+        stop_job,
+        pause_job,
+        get_logs,
+        get_job_stop_time,
+        monitor_logs,
+        monitor_logs_async,
+        get_job_is_blocking_on_stdin,
+    )
 
     def __init__(
         self,

meerschaum/connectors/api/_actions.py

@@ -7,81 +7,70 @@ Functions to interact with /mrsm/actions
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import SuccessTuple, Optional, List
 
-def get_actions(self) -> list:
+import json
+import asyncio
+from functools import partial
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import SuccessTuple, List, Callable
+from meerschaum.config.static import STATIC_CONFIG
+
+ACTIONS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['actions']
+
+
+def get_actions(self):
     """Get available actions from the API instance."""
-    from meerschaum.config.static import STATIC_CONFIG
-    return self.get(STATIC_CONFIG['api']['endpoints']['actions'])
+    return self.get(ACTIONS_ENDPOINT)
+
+
+def do_action(self, sysargs: List[str]) -> SuccessTuple:
+    """
+    Execute a Meerschaum action remotely.
+    """
+    return asyncio.run(self.do_action_async(sysargs))
 
 
-def do_action(
+async def do_action_async(
     self,
-    action: Optional[List[str]] = None,
-    sysargs: Optional[List[str]] = None,
-    debug: bool = False,
-    **kw
+    sysargs: List[str],
+    callback_function: Callable[[str], None] = partial(print, end=''),
 ) -> SuccessTuple:
-    """Execute a Meerschaum action remotely.
-
-    If `sysargs` are provided, parse those instead.
-    Otherwise infer everything from keyword arguments.
-
-    Examples
-    --------
-    >>> conn = mrsm.get_connector('api:main')
-    >>> conn.do_action(['show', 'pipes'])
-    (True, "Success")
-    >>> conn.do_action(['show', 'arguments'], name='test')
-    (True, "Success")
     """
-    import sys, json
-    from meerschaum.utils.debug import dprint
-    from meerschaum.config.static import STATIC_CONFIG
-    from meerschaum.utils.misc import json_serialize_datetime
-    if action is None:
-        action = []
-
-    if sysargs is not None and action and action[0] == '':
-        from meerschaum._internal.arguments import parse_arguments
-        if debug:
-            dprint(f"Parsing sysargs:\n{sysargs}")
-        json_dict = parse_arguments(sysargs)
-    else:
-        json_dict = kw
-        json_dict['action'] = action
-        if 'noask' not in kw:
-            json_dict['noask'] = True
-        if 'yes' not in kw:
-            json_dict['yes'] = True
-        if debug:
-            json_dict['debug'] = debug
-
-    root_action = json_dict['action'][0]
-    del json_dict['action'][0]
-    r_url = f"{STATIC_CONFIG['api']['endpoints']['actions']}/{root_action}"
-    
-    if debug:
-        from meerschaum.utils.formatting import pprint
-        dprint(f"Sending data to '{self.url + r_url}':")
-        pprint(json_dict, stream=sys.stderr)
-
-    response = self.post(
-        r_url,
-        data = json.dumps(json_dict, default=json_serialize_datetime),
-        debug = debug,
-    )
-    try:
-        response_list = json.loads(response.text)
-        if isinstance(response_list, dict) and 'detail' in response_list:
-            return False, response_list['detail']
-    except Exception as e:
-        print(f"Invalid response: {response}")
-        print(e)
-        return False, response.text
-    if debug:
-        dprint(response)
-    try:
-        return response_list[0], response_list[1]
-    except Exception as e:
-        return False, f"Failed to parse result from action '{root_action}'"
+    Monitor a job's log files and await a callback with the changes.
+    """
+    websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
+    protocol = 'ws' if self.URI.startswith('http://') else 'wss'
+    port = self.port if 'port' in self.__dict__ else ''
+    uri = f"{protocol}://{self.host}:{port}{ACTIONS_ENDPOINT}/ws"
+    if sysargs and sysargs[0] == 'api' and len(sysargs) > 2:
+        sysargs = sysargs[2:]
+
+    sysargs_str = json.dumps(sysargs)
+
+    async with websockets.connect(uri) as websocket:
+        try:
+            await websocket.send(self.token or 'no-login')
+            response = await websocket.recv()
+            init_data = json.loads(response)
+            if not init_data.get('is_authenticated'):
+                return False, "Cannot authenticate with actions endpoint."
+
+            await websocket.send(sysargs_str)
+        except websockets_exceptions.ConnectionClosedOK:
+            return False, "Connection was closed."
+
+        while True:
+            try:
+                line = await websocket.recv()
+                if asyncio.iscoroutinefunction(callback_function):
+                    await callback_function(line)
+                else:
+                    callback_function(line)
+            except KeyboardInterrupt:
+                await websocket.close()
+                break
+            except websockets_exceptions.ConnectionClosedOK:
+                break
+
+    return True, "Success"

meerschaum/connectors/api/_jobs.py

@@ -0,0 +1,260 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Manage jobs via the Meerschaum API.
+"""
+
+import asyncio
+from datetime import datetime
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable
+from meerschaum.utils.jobs import Job
+from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.utils.warnings import warn
+
+JOBS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['jobs']
+LOGS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['logs']
+JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
+
+
+def get_jobs(self, debug: bool = False) -> Dict[str, Job]:
+    """
+    Return a dictionary of remote jobs.
+    """
+    response = self.get(JOBS_ENDPOINT, debug=debug)
+    if not response:
+        warn(f"Failed to get remote jobs from {self}.")
+        return {}
+    return {
+        name: Job(
+            name,
+            job_meta['sysargs'],
+            executor_keys=str(self),
+            _properties=job_meta['daemon']['properties']
+        )
+        for name, job_meta in response.json().items()
+    }
+
+
+def get_job(self, name: str, debug: bool = False) -> Job:
+    """
+    Return a single Job object.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    if not metadata:
+        raise ValueError(f"Job '{name}' does not exist.")
+
+    return Job(
+        name,
+        metadata['sysargs'],
+        executor_keys=str(self),
+        _properties=metadata['daemon']['properties'],
+    )
+
+
+def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
+    """
+    Return the metadata for a single job.
+    """
+    response = self.get(JOBS_ENDPOINT + f"/{name}", debug=debug)
+    if not response:
+        if debug:
+            msg = (
+                response.json()['detail']
+                if 'detail' in response.text
+                else response.text
+            )
+            warn(f"Failed to get metadata for job '{name}':\n{msg}")
+        return {}
+
+    return response.json()
+
+
+def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
+    """
+    Return the daemon properties for a single job.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    return metadata.get('daemon', {}).get('properties', {})
+
+
+def get_job_exists(self, name: str, debug: bool = False) -> bool:
+    """
+    Return whether a job exists.
+    """
+    response = self.get(JOBS_ENDPOINT + f'/{name}/exists', debug=debug)
+    if not response:
+        warn(f"Failed to determine whether job '{name}' exists.")
+        return False
+
+    return response.json()
+
+
+def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Delete a job.
+    """
+    response = self.delete(JOBS_ENDPOINT + f"/{name}", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Start a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}/start", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
+    """
+    Create a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}", json=sysargs, debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Stop a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}/stop", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Pause a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}/pause", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def get_logs(self, name: str, debug: bool = False) -> str:
+    """
+    Return the logs for a job.
+    """
+    response = self.get(LOGS_ENDPOINT + f"/{name}")
+    if not response:
+        raise ValueError(f"Cannot fetch logs for job '{name}':\n{response.text}")
+
+    return response.json()
+
+
+def get_job_stop_time(self, name: str, debug: bool = False) -> Union[datetime, None]:
+    """
+    Return the job's manual stop time.
+    """
+    response = self.get(JOBS_ENDPOINT + f"/{name}/stop_time")
+    if not response:
+        warn(f"Failed to get stop time for job '{name}':\n{response.text}")
+        return None
+
+    data = response.json()
+    if data is None:
+        return None
+
+    return datetime.fromisoformat(data)
+
+
+async def monitor_logs_async(
+    self,
+    name: str,
+    callback_function: Callable[[Any], Any],
+    input_callback_function: Callable[[], str],
+    accept_input: bool = True,
+    debug: bool = False,
+):
+    """
+    Monitor a job's log files and await a callback with the changes.
+    """
+    websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
+    protocol = 'ws' if self.URI.startswith('http://') else 'wss'
+    port = self.port if 'port' in self.__dict__ else ''
+    uri = f"{protocol}://{self.host}:{port}{LOGS_ENDPOINT}/{name}/ws"
+
+    async with websockets.connect(uri) as websocket:
+        try:
+            await websocket.send(self.token or 'no-login')
+        except websockets_exceptions.ConnectionClosedOK:
+            pass
+
+        while True:
+            try:
+                response = await websocket.recv()
+                if response == JOBS_STDIN_MESSAGE:
+                    if asyncio.iscoroutinefunction(input_callback_function):
+                        data = await input_callback_function()
+                    else:
+                        data = input_callback_function()
+
+                    await websocket.send(data)
+                    continue
+
+                if asyncio.iscoroutinefunction(callback_function):
+                    await callback_function(response)
+                else:
+                    callback_function(response)
+            except KeyboardInterrupt:
+                await websocket.close()
+                break
+
+def monitor_logs(
+    self,
+    name: str,
+    callback_function: Callable[[Any], Any],
+    input_callback_function: Callable[[None], str],
+    accept_input: bool = True,
+    debug: bool = False,
+):
+    """
+    Monitor a job's log files and execute a callback with the changes.
+    """
+    return asyncio.run(
+        self.monitor_logs_async(
+            name,
+            callback_function,
+            input_callback_function=input_callback_function,
+            accept_input=accept_input,
+            debug=debug
+        )
+    )
+
+def get_job_is_blocking_on_stdin(self, name: str, debug: bool = False) -> bool:
+    """
+    Return whether a remote job is blocking on stdin.
+    """
+    response = self.get(JOBS_ENDPOINT + f'/{name}/is_blocking_on_stdin', debug=debug)
+    if not response:
+        return False
+
+    return response.json()

meerschaum/connectors/parse.py

@@ -10,11 +10,11 @@ from __future__ import annotations
 from meerschaum.utils.typing import Mapping, Any, SuccessTuple, Union, Optional, Dict, Tuple
 
 def parse_connector_keys(
-        keys: str,
-        construct: bool = True,
-        as_tuple: bool = False,
-        **kw:  Any
-    ) -> (
+    keys: str,
+    construct: bool = True,
+    as_tuple: bool = False,
+    **kw:  Any
+) -> (
         Union[
             meerschaum.connectors.Connector,
             Dict[str, Any],
@@ -119,9 +119,25 @@ def parse_repo_keys(keys: Optional[str] = None, **kw):
     return parse_connector_keys(keys, **kw)
 
 
+def parse_executor_keys(keys: Optional[str] = None, **kw):
+    """Parse the executor keys into an APIConnector or None."""
+    from meerschaum.config import get_config
+    if keys is None:
+        keys = get_config('meerschaum', 'default_executor')
+
+    if keys is None or keys == 'local':
+        return 'local'
+
+    keys = str(keys)
+    if ':' not in keys:
+        keys = 'api:' + keys
+
+    return parse_connector_keys(keys, **kw)
+
+
 def is_valid_connector_keys(
-        keys: str
-    ) -> bool:
+    keys: str
+) -> bool:
     """Verify a connector_keys string references a valid connector.
     """
     try:

meerschaum/plugins/__init__.py

@@ -8,6 +8,7 @@ Expose plugin management APIs from the `meerschaum.plugins` module.
 
 from __future__ import annotations
 import functools
+import meerschaum as mrsm
 from meerschaum.utils.typing import Callable, Any, Union, Optional, Dict, List, Tuple
 from meerschaum.utils.threading import Lock, RLock
 from meerschaum.plugins._Plugin import Plugin
@@ -429,11 +430,11 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
 
 
 def import_plugins(
-        *plugins_to_import: Union[str, List[str], None],
-        warn: bool = True,
-    ) -> Union[
-        'ModuleType', Tuple['ModuleType', None]
-    ]:
+    *plugins_to_import: Union[str, List[str], None],
+    warn: bool = True,
+) -> Union[
+    'ModuleType', Tuple['ModuleType', None]
+]:
     """
     Import the Meerschaum plugins directory.
 
@@ -524,6 +525,89 @@ def import_plugins(
     return imported_plugins
 
 
+def from_plugin_import(plugin_import_name: str, *attrs: str) -> Any:
+    """
+    Emulate the `from module import x` behavior.
+
+    Parameters
+    ----------
+    plugin_import_name: str
+        The import name of the plugin's module.
+        Separate submodules with '.' (e.g. 'compose.utils.pipes')
+
+    attrs: str
+        Names of the attributes to return.
+
+    Returns
+    -------
+    Objects from a plugin's submodule.
+    If multiple objects are provided, return a tuple.
+
+    Examples
+    --------
+    >>> init = from_plugin_import('compose.utils', 'init')
+    >>> with mrsm.Venv('compose'):
+    ...     cf = init()
+    >>> build_parent_pipe, get_defined_pipes = from_plugin_import(
+    ...     'compose.utils.pipes',
+    ...     'build_parent_pipe',
+    ...     'get_defined_pipes',
+    ... )
+    >>> parent_pipe = build_parent_pipe(cf)
+    >>> defined_pipes = get_defined_pipes(cf)
+    """
+    import importlib
+    from meerschaum.config._paths import PLUGINS_RESOURCES_PATH
+    from meerschaum.utils.warnings import warn as _warn
+    if plugin_import_name.startswith('plugins.'):
+        plugin_import_name = plugin_import_name[len('plugins.'):]
+    plugin_import_parts = plugin_import_name.split('.')
+    plugin_root_name = plugin_import_parts[0]
+    plugin = mrsm.Plugin(plugin_root_name)
+
+    submodule_import_name = '.'.join(
+        [PLUGINS_RESOURCES_PATH.stem]
+        + plugin_import_parts
+    )
+    if len(attrs) == 0:
+        raise ValueError(f"Provide which attributes to return from '{submodule_import_name}'.")
+
+    attrs_to_return = []
+    with mrsm.Venv(plugin):
+        if plugin.module is None:
+            return None
+
+        try:
+            submodule = importlib.import_module(submodule_import_name)
+        except ImportError as e:
+            _warn(
+                f"Failed to import plugin '{submodule_import_name}':\n    "
+                + f"{e}\n\nHere's a stacktrace:",
+                stack=False,
+            )
+            from meerschaum.utils.formatting import get_console
+            get_console().print_exception(
+                suppress=[
+                    'meerschaum/plugins/__init__.py',
+                    importlib,
+                    importlib._bootstrap,
+                ]
+            )
+            return None
+
+        for attr in attrs:
+            try:
+                attrs_to_return.append(getattr(submodule, attr))
+            except Exception:
+                _warn(f"Failed to access '{attr}' from '{submodule_import_name}'.")
+                attrs_to_return.append(None)
+        
+        if len(attrs) == 1:
+            return attrs_to_return[0]
+
+        return tuple(attrs_to_return)
+
+
 def load_plugins(debug: bool = False, shell: bool = False) -> None:
     """
     Import Meerschaum plugins and update the actions dictionary.