meerschaum 2.3.0rc2__py3-none-any.whl → 2.3.0rc5__py3-none-any.whl

meerschaum/connectors/api/_pipes.py

@@ -11,14 +11,15 @@ import time
 import json
 from io import StringIO
 from datetime import datetime
+
 import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn, error
 from meerschaum.utils.typing import SuccessTuple, Union, Any, Optional, Mapping, List, Dict, Tuple
 
 def pipe_r_url(
-        pipe: mrsm.Pipe
-    ) -> str:
+    pipe: mrsm.Pipe
+) -> str:
     """Return a relative URL path from a Pipe's keys."""
     from meerschaum.config.static import STATIC_CONFIG
     location_key = pipe.location_key
@@ -30,10 +31,10 @@ def pipe_r_url(
     )
 
 def register_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False
+) -> SuccessTuple:
     """Submit a POST to the API to register a new Pipe object.
     Returns a tuple of (success_bool, response_dict).
     """
@@ -59,11 +60,11 @@ def register_pipe(
 
 
 def edit_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        patch: bool = False,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    patch: bool = False,
+    debug: bool = False,
+) -> SuccessTuple:
     """Submit a PATCH to the API to edit an existing Pipe object.
     Returns a tuple of (success_bool, response_dict).
     """
@@ -89,14 +90,14 @@ def edit_pipe(
 
 
 def fetch_pipes_keys(
-        self,
-        connector_keys: Optional[List[str]] = None,
-        metric_keys: Optional[List[str]] = None,
-        location_keys: Optional[List[str]] = None,
-        tags: Optional[List[str]] = None,
-        params: Optional[Dict[str, Any]] = None,
-        debug: bool = False
-    ) -> Union[List[Tuple[str, str, Union[str, None]]]]:
+    self,
+    connector_keys: Optional[List[str]] = None,
+    metric_keys: Optional[List[str]] = None,
+    location_keys: Optional[List[str]] = None,
+    tags: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False
+) -> Union[List[Tuple[str, str, Union[str, None]]]]:
     """
     Fetch registered Pipes' keys from the API.
     
@@ -158,13 +159,13 @@ def fetch_pipes_keys(
 
 
 def sync_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
-        chunksize: Optional[int] = -1,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    df: Optional[Union['pd.DataFrame', Dict[Any, Any], str]] = None,
+    chunksize: Optional[int] = -1,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """Sync a DataFrame into a Pipe."""
     from decimal import Decimal
     from meerschaum.utils.debug import dprint
@@ -303,10 +304,10 @@ def sync_pipe(
 
 
 def delete_pipe(
-        self,
-        pipe: Optional[meerschaum.Pipe] = None,
-        debug: bool = None,        
-    ) -> SuccessTuple:
+    self,
+    pipe: Optional[meerschaum.Pipe] = None,
+    debug: bool = None,        
+) -> SuccessTuple:
     """Delete a Pipe and drop its table."""
     if pipe is None:
         error(f"Pipe cannot be None.")
@@ -327,17 +328,17 @@ def delete_pipe(
 
 
 def get_pipe_data(
-        self,
-        pipe: meerschaum.Pipe,
-        select_columns: Optional[List[str]] = None,
-        omit_columns: Optional[List[str]] = None,
-        begin: Union[str, datetime, int, None] = None,
-        end: Union[str, datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        as_chunks: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union[pandas.DataFrame, None]:
+    self,
+    pipe: meerschaum.Pipe,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    as_chunks: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Union[pandas.DataFrame, None]:
     """Fetch data from the API."""
     r_url = pipe_r_url(pipe)
     chunks_list = []
@@ -389,10 +390,10 @@ def get_pipe_data(
 
 
 def get_pipe_id(
-        self,
-        pipe: meerschuam.Pipe,
-        debug: bool = False,
-    ) -> int:
+    self,
+    pipe: meerschuam.Pipe,
+    debug: bool = False,
+) -> int:
     """Get a Pipe's ID from the API."""
     from meerschaum.utils.misc import is_int
     r_url = pipe_r_url(pipe)
@@ -411,10 +412,10 @@ def get_pipe_id(
 
 
 def get_pipe_attributes(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False,
-    ) -> Dict[str, Any]:
+    self,
+    pipe: meerschaum.Pipe,
+    debug: bool = False,
+) -> Dict[str, Any]:
     """Get a Pipe's attributes from the API
 
     Parameters
@@ -437,12 +438,12 @@ def get_pipe_attributes(
 
 
 def get_sync_time(
-        self,
-        pipe: 'meerschaum.Pipe',
-        params: Optional[Dict[str, Any]] = None,
-        newest: bool = True,
-        debug: bool = False,
-    ) -> Union[datetime, int, None]:
+    self,
+    pipe: 'meerschaum.Pipe',
+    params: Optional[Dict[str, Any]] = None,
+    newest: bool = True,
+    debug: bool = False,
+) -> Union[datetime, int, None]:
     """Get a Pipe's most recent datetime value from the API.
 
     Parameters
@@ -492,10 +493,10 @@ def get_sync_time(
 
 
 def pipe_exists(
-        self,
-        pipe: 'meerschaum.Pipe',
-        debug: bool = False
-    ) -> bool:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False
+) -> bool:
     """Check the API to see if a Pipe exists.
 
     Parameters
@@ -523,9 +524,9 @@ def pipe_exists(
 
 
 def create_metadata(
-        self,
-        debug: bool = False
-    ) -> bool:
+    self,
+    debug: bool = False
+) -> bool:
     """Create metadata tables.
 
     Returns
@@ -547,14 +548,14 @@ def create_metadata(
 
 
 def get_pipe_rowcount(
-        self,
-        pipe: 'meerschaum.Pipe',
-        begin: Optional[datetime] = None,
-        end: Optional[datetime] = None,
-        params: Optional[Dict[str, Any]] = None,
-        remote: bool = False,
-        debug: bool = False,
-    ) -> int:
+    self,
+    pipe: mrsm.Pipe,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    params: Optional[Dict[str, Any]] = None,
+    remote: bool = False,
+    debug: bool = False,
+) -> int:
     """Get a pipe's row count from the API.
 
     Parameters
@@ -600,10 +601,10 @@ def get_pipe_rowcount(
 
 
 def drop_pipe(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False
+) -> SuccessTuple:
     """
     Drop a pipe's table but maintain its registration.
 
@@ -644,11 +645,11 @@ def drop_pipe(
 
 
 def clear_pipe(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Delete rows in a pipe's table.
 
@@ -666,7 +667,7 @@ def clear_pipe(
     kw.pop('location_keys', None)
     kw.pop('action', None)
     kw.pop('force', None)
-    return self.do_action(
+    return self.do_action_legacy(
         ['clear', 'pipes'],
         connector_keys = pipe.connector_keys,
         metric_keys = pipe.metric_key,
@@ -678,10 +679,10 @@ def clear_pipe(
 
 
 def get_pipe_columns_types(
-        self,
-        pipe: meerschaum.Pipe,
-        debug: bool = False,
-    ) -> Union[Dict[str, str], None]:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Union[Dict[str, str], None]:
     """
     Fetch the columns and types of the pipe's table.
 

meerschaum/jobs/_Job.py

@@ -62,8 +62,9 @@ class Job:
         _properties: Optional[Dict[str, Any]] = None,
         _rotating_log = None,
         _stdin_file = None,
-        _status_hook = None,
-        _result_hook = None,
+        _status_hook: Optional[Callable[[], str]] = None,
+        _result_hook: Optional[Callable[[], SuccessTuple]] = None,
+        _externally_managed: bool = False,
     ):
         """
         Create a new job to manage a `meerschaum.utils.daemon.Daemon`.
@@ -133,7 +134,10 @@ class Job:
         if _result_hook is not None:
             self._result_hook = _result_hook
 
+        self._externally_managed = _externally_managed
         self._properties_patch = _properties or {}
+        if _externally_managed:
+            self._properties_patch.update({'externally_managed': _externally_managed})
 
         daemon_sysargs = (
             self._daemon.properties.get('target', {}).get('args', [None])[0]
@@ -154,9 +158,6 @@ class Job:
                 self._properties_patch.update({'restart': True})
                 break
 
-        if '--systemd' in self._sysargs:
-            self._properties_patch.update({'systemd': True})
-
     @staticmethod
     def from_pid(pid: int, executor_keys: Optional[str] = None) -> Job:
         """
@@ -740,7 +741,7 @@ class Job:
         The datetime when the job began running.
         """
         if self.executor is not None:
-            began_str = self.executor.get_job_began(name)
+            began_str = self.executor.get_job_began(self.name)
             if began_str is None:
                 return None
             return (
@@ -762,6 +763,8 @@ class Job:
         """
         if self.executor is not None:
             ended_str = self.executor.get_job_ended(self.name)
+            if ended_str is None:
+                return None
             return (
                 datetime.fromisoformat(ended_str)
                 .astimezone(timezone.utc)
@@ -779,6 +782,16 @@ class Job:
         """
         The datetime when the job was suspended while running.
         """
+        if self.executor is not None:
+            paused_str = self.executor.get_job_paused(self.name)
+            if paused_str is None:
+                return None
+            return (
+                datetime.fromisoformat(paused_str)
+                .astimezone(timezone.utc)
+                .replace(tzinfo=None)
+            )
+
         paused_str = self.daemon.properties.get('process', {}).get('paused', None)
         if paused_str is None:
             return None
@@ -796,11 +809,8 @@ class Job:
         if not self.daemon.stop_path.exists():
             return None
 
-        try:
-            with open(self.daemon.stop_path, 'r', encoding='utf-8') as f:
-                stop_data = json.load(f)
-        except Exception as e:
-            warn(f"Failed to read stop file for {self}:\n{e}")
+        stop_data = self.daemon._read_stop_file()
+        if not stop_data:
             return None
 
         stop_time_str = stop_data.get('stop_time', None)
@@ -815,7 +825,11 @@ class Job:
         """
         Return a bool indicating whether this job should be displayed.
         """
-        return self.name.startswith('_') or self.name.startswith('.')
+        return (
+            self.name.startswith('_')
+            or self.name.startswith('.')
+            or self._is_externally_managed
+        )
 
     def check_restart(self) -> SuccessTuple:
         """
@@ -841,6 +855,33 @@ class Job:
         """
         return shlex.join(self.sysargs).replace(' + ', '\n+ ')
 
+    @property
+    def _externally_managed_file(self) -> pathlib.Path:
+        """
+        Return the path to the externally managed file.
+        """
+        return self.daemon.path / '.externally-managed'
+
+    def _set_externally_managed(self):
+        """
+        Set this job as externally managed.
+        """
+        self._externally_managed = True
+        try:
+            self._externally_managed_file.parent.mkdir(exist_ok=True, parents=True)
+            self._externally_managed_file.touch()
+        except Exception as e:
+            warn(e)
+
+    @property
+    def _is_externally_managed(self) -> bool:
+        """
+        Return whether this job is externally managed.
+        """
+        return self.executor_keys in (None, 'local') and (
+            self._externally_managed or self._externally_managed_file.exists()
+        )
+
     def __str__(self) -> str:
         sysargs = self.sysargs
         sysargs_str = shlex.join(sysargs) if sysargs else ''

meerschaum/jobs/_SystemdExecutor.py

@@ -13,12 +13,14 @@ import asyncio
 import json
 import time
 import traceback
+import shutil
 from datetime import datetime, timezone
 from functools import partial
 
 import meerschaum as mrsm
 from meerschaum.jobs import Job, Executor, make_executor
 from meerschaum.utils.typing import Dict, Any, List, SuccessTuple, Union, Optional, Callable
+from meerschaum.config import get_config
 from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.warnings import warn, dprint
 from meerschaum._internal.arguments._parse_arguments import parse_arguments
@@ -36,10 +38,10 @@ class SystemdExecutor(Executor):
         """
         Return a list of existing jobs, including hidden ones.
         """
-        from meerschaum.config.paths import SYSTEMD_ROOT_RESOURCES_PATH
+        from meerschaum.config.paths import SYSTEMD_USER_RESOURCES_PATH
         return [
             service_name[len('mrsm-'):(-1 * len('.service'))]
-            for service_name in os.listdir(SYSTEMD_ROOT_RESOURCES_PATH)
+            for service_name in os.listdir(SYSTEMD_USER_RESOURCES_PATH)
             if service_name.startswith('mrsm-') and service_name.endswith('.service')
         ]
 
@@ -72,6 +74,13 @@ class SystemdExecutor(Executor):
         """
         return f"mrsm-{name.replace(' ', '-')}.service"
 
+    def get_service_job_path(self, name: str, debug: bool = False) -> pathlib.Path:
+        """
+        Return the path for the job's files under the root directory.
+        """
+        from meerschaum.config.paths import SYSTEMD_JOBS_RESOURCES_PATH
+        return SYSTEMD_JOBS_RESOURCES_PATH / name
+
     def get_service_symlink_file_path(self, name: str, debug: bool = False) -> pathlib.Path:
         """
         Return the path to where to create the service symlink.
@@ -83,8 +92,10 @@ class SystemdExecutor(Executor):
         """
         Return the path to a Job's service file.
         """
-        from meerschaum.config.paths import SYSTEMD_ROOT_RESOURCES_PATH
-        return SYSTEMD_ROOT_RESOURCES_PATH / self.get_service_name(name, debug=debug)
+        return (
+            self.get_service_job_path(name, debug=debug)
+            / self.get_service_name(name, debug=debug)
+        )
 
     def get_service_logs_path(self, name: str, debug: bool = False) -> pathlib.Path:
         """
@@ -93,29 +104,22 @@ class SystemdExecutor(Executor):
         from meerschaum.config.paths import SYSTEMD_LOGS_RESOURCES_PATH
         return SYSTEMD_LOGS_RESOURCES_PATH / (self.get_service_name(name, debug=debug) + '.log')
 
-    def get_service_socket_path(self, name: str, debug: bool = False) -> pathlib.Path:
-        """
-        Return the path to the unit file for the socket (not the socket itself).
-        """
-        from meerschaum.config.paths import SYSTEMD_USER_RESOURCES_PATH
-        return SYSTEMD_USER_RESOURCES_PATH / (
-            self.get_service_name(name, debug=debug).replace('.service', '.socket')
-        )
-
     def get_socket_path(self, name: str, debug: bool = False) -> pathlib.Path:
         """
         Return the path to the FIFO file.
         """
-        from meerschaum.config.paths import SYSTEMD_ROOT_RESOURCES_PATH
-        return SYSTEMD_ROOT_RESOURCES_PATH / (self.get_service_name(name, debug=debug) + '.stdin')
+        return (
+            self.get_service_job_path(name, debug=debug)
+            / (self.get_service_name(name, debug=debug) + '.stdin')
+        )
 
     def get_result_path(self, name: str, debug: bool = False) -> pathlib.Path:
         """
         Return the path to the result file.
         """
-        from meerschaum.config.paths import SYSTEMD_ROOT_RESOURCES_PATH
-        return SYSTEMD_ROOT_RESOURCES_PATH / (
-            self.get_service_name(name, debug=debug) + '.result.json'
+        return (
+            self.get_service_job_path(name, debug=debug)
+            / (self.get_service_name(name, debug=debug) + '.result.json')
         )
 
     def get_service_file_text(self, name: str, sysargs: List[str], debug: bool = False) -> str:
@@ -141,7 +145,8 @@ class SystemdExecutor(Executor):
             STATIC_CONFIG['environment']['systemd_log_path']: service_logs_path.as_posix(),
             STATIC_CONFIG['environment']['systemd_result_path']: result_path.as_posix(),
             STATIC_CONFIG['environment']['systemd_stdin_path']: socket_path.as_posix(),
-
+            'LINES': get_config('jobs', 'terminal', 'lines'),
+            'COLUMNS': get_config('jobs', 'terminal', 'columns'),
         })
         environment_lines = [
             f"Environment={key}={val}"
@@ -189,17 +194,19 @@ class SystemdExecutor(Executor):
         """
         Return the hidden "sister" job to store a job's parameters.
         """
-        hidden_name = f'.systemd-{self.get_service_name(name, debug=debug)}' 
-
-        return Job(
-            hidden_name,
+        job = Job(
+            name,
             sysargs,
             executor_keys='local',
             _rotating_log=self.get_job_rotating_file(name, debug=debug),
             _stdin_file=self.get_job_stdin_file(name, debug=debug),
             _status_hook=partial(self.get_job_status, name),
             _result_hook=partial(self.get_job_result, name),
+            _externally_managed=True,
         )
+        job._set_externally_managed()
+        return job
+
 
     def get_job_metadata(self, name: str, debug: bool = False) -> Dict[str, Any]:
         """
@@ -259,7 +266,10 @@ class SystemdExecutor(Executor):
             return None
 
         psutil = mrsm.attempt_import('psutil')
-        return psutil.Process(pid)
+        try:
+            return psutil.Process(pid)
+        except Exception:
+            return None
 
     def get_job_status(self, name: str, debug: bool = False) -> str:
         """
@@ -271,13 +281,19 @@ class SystemdExecutor(Executor):
             debug=debug,
         )
 
+        if output == 'activating':
+            return 'running'
+
         if output == 'active':
             process = self.get_job_process(name, debug=debug)
             if process is None:
                 return 'stopped'
 
-            if process.status() == 'stopped':
-                return 'paused'
+            try:
+                if process.status() == 'stopped':
+                    return 'paused'
+            except Exception:
+                return 'stopped'
 
             return 'running'
 
@@ -310,7 +326,7 @@ class SystemdExecutor(Executor):
         
         return None
 
-    def get_job_began(self, name: str, debug: bool = False) -> Union[datetime, None]:
+    def get_job_began(self, name: str, debug: bool = False) -> Union[str, None]:
         """
         Return when a job began running.
         """
@@ -326,10 +342,62 @@ class SystemdExecutor(Executor):
         if not output.startswith('ActiveEnterTimestamp'):
             return None
 
+        dt_str = output.split('=')[-1]
+        if not dt_str:
+            return None
+
         dateutil_parser = mrsm.attempt_import('dateutil.parser')
-        dt = dateutil_parser.parse(output.split('=')[-1])
+        try:
+            dt = dateutil_parser.parse(dt_str)
+        except Exception as e:
+            warn(f"Cannot parse '{output}' as a datetime:\n{e}")
+            return None
+
         return dt.astimezone(timezone.utc).isoformat()
 
+    def get_job_ended(self, name: str, debug: bool = False) -> Union[str, None]:
+        """
+        Return when a job began running.
+        """
+        output = self.run_command(
+            [
+                'show',
+                self.get_service_name(name, debug=debug),
+                '--property=InactiveEnterTimestamp'
+            ],
+            as_output=True,
+            debug=debug,
+        )
+        if not output.startswith('InactiveEnterTimestamp'):
+            return None
+
+        dt_str = output.split('=')[-1]
+        if not dt_str:
+            return None
+
+        dateutil_parser = mrsm.attempt_import('dateutil.parser')
+
+        try:
+            dt = dateutil_parser.parse(dt_str)
+        except Exception as e:
+            warn(f"Cannot parse '{output}' as a datetime:\n{e}")
+            return None
+        return dt.astimezone(timezone.utc).isoformat()
+
+    def get_job_paused(self, name: str, debug: bool = False) -> Union[str, None]:
+        """
+        Return when a job was paused.
+        """
+        job = self.get_hidden_job(name, debug=debug)
+        if self.get_job_status(name, debug=debug) != 'paused':
+            return None
+
+        stop_time = job.stop_time
+        if stop_time is None:
+            return None
+
+        return stop_time.isoformat()
+
     def get_job_result(self, name: str, debug: bool = False) -> SuccessTuple:
         """
         Return the job's result SuccessTuple.
@@ -426,6 +494,7 @@ class SystemdExecutor(Executor):
 
         if name not in self._stdin_files:
             socket_path = self.get_socket_path(name, debug=debug)
+            socket_path.parent.mkdir(parents=True, exist_ok=True)
             self._stdin_files[name] = StdinFile(socket_path)
 
         return self._stdin_files[name]
@@ -441,6 +510,9 @@ class SystemdExecutor(Executor):
         socket_stdin = self.get_job_stdin_file(name, debug=debug)
         _ = socket_stdin.file_handler
 
+        ### Init the `externally_managed file`.
+        _ = self.get_hidden_job(name, debug=debug)
+
         with open(service_file_path, 'w+', encoding='utf-8') as f:
             f.write(self.get_service_file_text(name, sysargs, debug=debug))
 
@@ -495,12 +567,13 @@ class SystemdExecutor(Executor):
             debug=debug,
         )
 
+        check_timeout_interval = get_config('jobs', 'check_timeout_interval_seconds')
         loop_start = time.perf_counter()
-        while (time.perf_counter() - loop_start) < 5:
+        while (time.perf_counter() - loop_start) < get_config('jobs', 'timeout_seconds'):
             if self.get_job_status(name, debug=debug) == 'stopped':
                 return True, 'Success'
 
-            time.sleep(0.1)
+            time.sleep(check_timeout_interval)
 
         return self.run_command(
             ['stop', self.get_service_name(name, debug=debug)],
@@ -530,6 +603,14 @@ class SystemdExecutor(Executor):
             debug=debug,
         )
 
+        service_job_path = self.get_service_job_path(name, debug=debug)
+        try:
+            if service_job_path.exists():
+                shutil.rmtree(service_job_path)
+        except Exception as e:
+            warn(e)
+            return False, str(e)
+
         service_logs_path = self.get_service_logs_path(name, debug=debug)
         logs_paths = [
             (SYSTEMD_LOGS_RESOURCES_PATH / name)