meerschaum 2.2.6__py3-none-any.whl → 2.3.0__py3-none-any.whl

meerschaum/connectors/api/_jobs.py

@@ -0,0 +1,368 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Manage jobs via the Meerschaum API.
+"""
+
+import asyncio
+import time
+import json
+from datetime import datetime
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Dict, Any, SuccessTuple, List, Union, Callable
+from meerschaum.jobs import Job
+from meerschaum.config.static import STATIC_CONFIG
+from meerschaum.utils.warnings import warn, dprint
+
+JOBS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['jobs']
+LOGS_ENDPOINT: str = STATIC_CONFIG['api']['endpoints']['logs']
+JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
+JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
+JOB_METADATA_CACHE_SECONDS: int = STATIC_CONFIG['api']['jobs']['metadata_cache_seconds']
+
+
+def get_jobs(self, debug: bool = False) -> Dict[str, Job]:
+    """
+    Return a dictionary of remote jobs.
+    """
+    response = self.get(JOBS_ENDPOINT, debug=debug)
+    if not response:
+        warn(f"Failed to get remote jobs from {self}.")
+        return {}
+    return {
+        name: Job(
+            name,
+            job_meta['sysargs'],
+            executor_keys=str(self),
+            _properties=job_meta['daemon']['properties']
+        )
+        for name, job_meta in response.json().items()
+    }
+
+
+def get_job(self, name: str, debug: bool = False) -> Job:
+    """
+    Return a single Job object.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    if not metadata:
+        raise ValueError(f"Job '{name}' does not exist.")
+
+    return Job(
+        name,
+        metadata['sysargs'],
+        executor_keys=str(self),
+        _properties=metadata['daemon']['properties'],
+    )
+
+
+def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
+    """
+    Return the metadata for a single job.
+    """
+    now = time.perf_counter()
+    _job_metadata_cache = self.__dict__.get('_job_metadata_cache', None)
+    _job_metadata_timestamp = (
+        _job_metadata_cache.get(name, {}).get('timestamp', None)
+    ) if _job_metadata_cache is not None else None
+
+    if (
+        _job_metadata_timestamp is not None
+        and (now - _job_metadata_timestamp) < JOB_METADATA_CACHE_SECONDS
+    ):
+        if debug:
+            dprint(f"Returning cached metadata for job '{name}'.")
+        return _job_metadata_cache[name]['metadata']
+
+    response = self.get(JOBS_ENDPOINT + f"/{name}", debug=debug)
+    if not response:
+        if debug:
+            msg = (
+                response.json()['detail']
+                if 'detail' in response.text
+                else response.text
+            )
+            warn(f"Failed to get metadata for job '{name}':\n{msg}")
+        return {}
+
+    metadata = response.json()
+    if _job_metadata_cache is None:
+        self._job_metadata_cache = {}
+
+    self._job_metadata_cache[name] = {
+        'timestamp': now,
+        'metadata': metadata,
+    }
+    return metadata
+
+def get_job_properties(self, name: str, debug: bool = False) -> Dict[str, Any]:
+    """
+    Return the daemon properties for a single job.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    return metadata.get('daemon', {}).get('properties', {})
+
+def get_job_status(self, name: str, debug: bool = False) -> str:
+    """
+    Return the job's status.
+    """
+    metadata = self.get_job_metadata(name, debug=debug)
+    return metadata.get('status', 'stopped')
+
+def get_job_began(self, name: str, debug: bool = False) -> Union[str, None]:
+    """
+    Return a job's `began` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    began_str = properties.get('daemon', {}).get('began', None)
+    if began_str is None:
+        return None
+
+    return began_str
+
+def get_job_ended(self, name: str, debug: bool = False) -> Union[str, None]:
+    """
+    Return a job's `ended` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    ended_str = properties.get('daemon', {}).get('ended', None)
+    if ended_str is None:
+        return None
+
+    return ended_str
+
+def get_job_paused(self, name: str, debug: bool = False) -> Union[str, None]:
+    """
+    Return a job's `paused` timestamp, if it exists.
+    """
+    properties = self.get_job_properties(name, debug=debug)
+    paused_str = properties.get('daemon', {}).get('paused', None)
+    if paused_str is None:
+        return None
+
+    return paused_str
+
+def get_job_exists(self, name: str, debug: bool = False) -> bool:
+    """
+    Return whether a job exists.
+    """
+    response = self.get(JOBS_ENDPOINT + f'/{name}/exists', debug=debug)
+    if not response:
+        warn(f"Failed to determine whether job '{name}' exists.")
+        return False
+
+    return response.json()
+
+
+def delete_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Delete a job.
+    """
+    response = self.delete(JOBS_ENDPOINT + f"/{name}", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def start_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Start a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}/start", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def create_job(self, name: str, sysargs: List[str], debug: bool = False) -> SuccessTuple:
+    """
+    Create a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}", json=sysargs, debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def stop_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Stop a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}/stop", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def pause_job(self, name: str, debug: bool = False) -> SuccessTuple:
+    """
+    Pause a job.
+    """
+    response = self.post(JOBS_ENDPOINT + f"/{name}/pause", debug=debug)
+    if not response:
+        if 'detail' in response.text:
+            return False, response.json()['detail']
+        return False, response.text
+
+    return tuple(response.json())
+
+
+def get_logs(self, name: str, debug: bool = False) -> str:
+    """
+    Return the logs for a job.
+    """
+    response = self.get(LOGS_ENDPOINT + f"/{name}")
+    if not response:
+        raise ValueError(f"Cannot fetch logs for job '{name}':\n{response.text}")
+
+    return response.json()
+
+
+def get_job_stop_time(self, name: str, debug: bool = False) -> Union[datetime, None]:
+    """
+    Return the job's manual stop time.
+    """
+    response = self.get(JOBS_ENDPOINT + f"/{name}/stop_time")
+    if not response:
+        warn(f"Failed to get stop time for job '{name}':\n{response.text}")
+        return None
+
+    data = response.json()
+    if data is None:
+        return None
+
+    return datetime.fromisoformat(data)
+
+
+async def monitor_logs_async(
+    self,
+    name: str,
+    callback_function: Callable[[Any], Any],
+    input_callback_function: Callable[[], str],
+    stop_callback_function: Callable[[SuccessTuple], str],
+    stop_on_exit: bool = False,
+    strip_timestamps: bool = False,
+    accept_input: bool = True,
+    debug: bool = False,
+):
+    """
+    Monitor a job's log files and await a callback with the changes.
+    """
+    from meerschaum.jobs import StopMonitoringLogs
+    from meerschaum.utils.formatting._jobs import strip_timestamp_from_line
+
+    websockets, websockets_exceptions = mrsm.attempt_import('websockets', 'websockets.exceptions')
+    protocol = 'ws' if self.URI.startswith('http://') else 'wss'
+    port = self.port if 'port' in self.__dict__ else ''
+    uri = f"{protocol}://{self.host}:{port}{LOGS_ENDPOINT}/{name}/ws"
+
+    async def _stdin_callback(client):
+        if input_callback_function is None:
+            return
+
+        if asyncio.iscoroutinefunction(input_callback_function):
+            data = await input_callback_function()
+        else:
+            data = input_callback_function()
+
+        await client.send(data)
+
+    async def _stop_callback(client):
+        try:
+            result = tuple(json.loads(await client.recv()))
+        except Exception as e:
+            warn(traceback.format_exc())
+            result = False, str(e)
+
+        if stop_callback_function is not None:
+            if asyncio.iscoroutinefunction(stop_callback_function):
+                await stop_callback_function(result)
+            else:
+                stop_callback_function(result)
+
+        if stop_on_exit:
+            raise StopMonitoringLogs
+
+    message_callbacks = {
+        JOBS_STDIN_MESSAGE: _stdin_callback,
+        JOBS_STOP_MESSAGE: _stop_callback,
+    }
+
+    async with websockets.connect(uri) as websocket:
+        try:
+            await websocket.send(self.token or 'no-login')
+        except websockets_exceptions.ConnectionClosedOK:
+            pass
+
+        while True:
+            try:
+                response = await websocket.recv()
+                callback = message_callbacks.get(response, None)
+                if callback is not None:
+                    await callback(websocket)
+                    continue
+
+                if strip_timestamps:
+                    response = strip_timestamp_from_line(response)
+
+                if asyncio.iscoroutinefunction(callback_function):
+                    await callback_function(response)
+                else:
+                    callback_function(response)
+            except (KeyboardInterrupt, StopMonitoringLogs):
+                await websocket.close()
+                break
+
+
+def monitor_logs(
+    self,
+    name: str,
+    callback_function: Callable[[Any], Any],
+    input_callback_function: Callable[[None], str],
+    stop_callback_function: Callable[[None], str],
+    stop_on_exit: bool = False,
+    strip_timestamps: bool = False,
+    accept_input: bool = True,
+    debug: bool = False,
+):
+    """
+    Monitor a job's log files and execute a callback with the changes.
+    """
+    return asyncio.run(
+        self.monitor_logs_async(
+            name,
+            callback_function,
+            input_callback_function=input_callback_function,
+            stop_callback_function=stop_callback_function,
+            stop_on_exit=stop_on_exit,
+            strip_timestamps=strip_timestamps,
+            accept_input=accept_input,
+            debug=debug
+        )
+    )
+
+def get_job_is_blocking_on_stdin(self, name: str, debug: bool = False) -> bool:
+    """
+    Return whether a remote job is blocking on stdin.
+    """
+    response = self.get(JOBS_ENDPOINT + f'/{name}/is_blocking_on_stdin', debug=debug)
+    if not response:
+        return False
+
+    return response.json()

meerschaum/connectors/api/_misc.py

@@ -17,7 +17,7 @@ def get_mrsm_version(self, **kw) -> Optional[str]:
     try:
         j = self.get(
             STATIC_CONFIG['api']['endpoints']['version'] + '/mrsm',
-            use_token = True,
+            use_token=False,
             **kw
         ).json()
     except Exception as e:

meerschaum/connectors/api/_pipes.py

@@ -11,14 +11,15 @@ import time
 import json
 from io import StringIO
 from datetime import datetime
+
 import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn, error
 from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
 
 def pipe_r_url(
-        pipe: mrsm.Pipe
-    ) -> str:
+    pipe: mrsm.Pipe
+) -> str:
     """Return a relative URL path from a Pipe's keys."""
     from meerschaum.config.static import STATIC_CONFIG
     location_key = pipe.location_key
@@ -30,10 +31,10 @@ def pipe_r_url(
     )
 
 def register_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False
+) -> SuccessTuple:
     """Submit a POST to the API to register a new Pipe object.
     Returns a tuple of (success_bool, response_dict).
     """
@@ -59,11 +60,11 @@ def register_pipe(
 
 
 def edit_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        patch: bool = False,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    patch: bool = False,
+    debug: bool = False,
+) -> SuccessTuple:
     """Submit a PATCH to the API to edit an existing Pipe object.
     Returns a tuple of (success_bool, response_dict).
     """
@@ -89,14 +90,14 @@ def edit_pipe(
 
 
 def fetch_pipes_keys(
-        self,
-        connector_keys: Optional[List[str]] = None,
-        metric_keys: Optional[List[str]] = None,
-        location_keys: Optional[List[str]] = None,
-        tags: Optional[List[str]] = None,
-        params: Optional[Dict[str, Any]] = None,
-        debug: bool = False
-    ) -> Union[List[Tuple[str, str, Union[str, None]]]]:
+    self,
+    connector_keys: Optional[List[str]] = None,
+    metric_keys: Optional[List[str]] = None,
+    location_keys: Optional[List[str]] = None,
+    tags: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False
+) -> Union[List[Tuple[str, str, Union[str, None]]]]:
     """
     Fetch registered Pipes' keys from the API.
     
@@ -158,13 +159,13 @@ def fetch_pipes_keys(
 
 
 def sync_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
-        chunksize: Optional[int] = -1,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
+    chunksize: Optional[int] = -1,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """Sync a DataFrame into a Pipe."""
     from decimal import Decimal
     from meerschaum.utils.debug import dprint
@@ -303,10 +304,10 @@ def sync_pipe(
 
 
 def delete_pipe(
-        self,
-        pipe: Optional[meerschaum.Pipe] = None,
-        debug: bool = None,        
-    ) -> SuccessTuple:
+    self,
+    pipe: Optional[meerschaum.Pipe] = None,
+    debug: bool = None,        
+) -> SuccessTuple:
     """Delete a Pipe and drop its table."""
     if pipe is None:
         error(f"Pipe cannot be None.")
@@ -327,17 +328,17 @@ def delete_pipe(
 
 
 def get_pipe_data(
-        self,
-        pipe: meerschaum.Pipe,
-        select_columns: Optional[List[str]] = None,
-        omit_columns: Optional[List[str]] = None,
-        begin: Union[str, datetime, int, None] = None,
-        end: Union[str, datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        as_chunks: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union[pandas.DataFrame, None]:
+    self,
+    pipe: meerschaum.Pipe,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    as_chunks: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Union[pandas.DataFrame, None]:
     """Fetch data from the API."""
     r_url = pipe_r_url(pipe)
     chunks_list = []
@@ -389,10 +390,10 @@ def get_pipe_data(
 
 
 def get_pipe_id(
-        self,
-        pipe: meerschuam.Pipe,
-        debug: bool = False,
-    ) -> int:
+    self,
+    pipe: meerschuam.Pipe,
+    debug: bool = False,
+) -> int:
     """Get a Pipe's ID from the API."""
     from meerschaum.utils.misc import is_int
     r_url = pipe_r_url(pipe)
@@ -411,10 +412,10 @@ def get_pipe_id(
 
 
 def get_pipe_attributes(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False,
-    ) -> Dict[str, Any]:
+    self,
+    pipe: meerschaum.Pipe,
+    debug: bool = False,
+) -> Dict[str, Any]:
     """Get a Pipe's attributes from the API
 
     Parameters
@@ -437,12 +438,12 @@ def get_pipe_attributes(
 
 
 def get_sync_time(
-        self,
-        pipe: 'meerschaum.Pipe',
-        params: Optional[Dict[str, Any]] = None,
-        newest: bool = True,
-        debug: bool = False,
-    ) -> Union[datetime, int, None]:
+    self,
+    pipe: 'meerschaum.Pipe',
+    params: Optional[Dict[str, Any]] = None,
+    newest: bool = True,
+    debug: bool = False,
+) -> Union[datetime, int, None]:
     """Get a Pipe's most recent datetime value from the API.
 
     Parameters
@@ -492,10 +493,10 @@ def get_sync_time(
 
 
 def pipe_exists(
-        self,
-        pipe: 'meerschaum.Pipe',
-        debug: bool = False
-    ) -> bool:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False
+) -> bool:
     """Check the API to see if a Pipe exists.
 
     Parameters
@@ -523,9 +524,9 @@ def pipe_exists(
 
 
 def create_metadata(
-        self,
-        debug: bool = False
-    ) -> bool:
+    self,
+    debug: bool = False
+) -> bool:
     """Create metadata tables.
 
     Returns
@@ -547,14 +548,14 @@ def create_metadata(
 
 
 def get_pipe_rowcount(
-        self,
-        pipe: 'meerschaum.Pipe',
-        begin: Optional[datetime] = None,
-        end: Optional[datetime] = None,
-        params: Optional[Dict[str, Any]] = None,
-        remote: bool = False,
-        debug: bool = False,
-    ) -> int:
+    self,
+    pipe: mrsm.Pipe,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    params: Optional[Dict[str, Any]] = None,
+    remote: bool = False,
+    debug: bool = False,
+) -> int:
     """Get a pipe's row count from the API.
 
     Parameters
@@ -600,10 +601,10 @@ def get_pipe_rowcount(
 
 
 def drop_pipe(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False
+) -> SuccessTuple:
     """
     Drop a pipe's table but maintain its registration.
 
@@ -644,11 +645,11 @@ def drop_pipe(
 
 
 def clear_pipe(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Delete rows in a pipe's table.
 
@@ -666,7 +667,7 @@ def clear_pipe(
     kw.pop('location_keys', None)
     kw.pop('action', None)
     kw.pop('force', None)
-    return self.do_action(
+    return self.do_action_legacy(
         ['clear', 'pipes'],
         connector_keys = pipe.connector_keys,
         metric_keys = pipe.metric_key,
@@ -678,10 +679,10 @@ def clear_pipe(
 
 
 def get_pipe_columns_types(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False,
-    ) -> Union[Dict[str, str], None]:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Union[Dict[str, str], None]:
     """
     Fetch the columns and types of the pipe's table.