meerschaum 2.0.0rc7__py3-none-any.whl → 2.0.0rc9__py3-none-any.whl

meerschaum/core/Pipe/_sync.py

@@ -14,7 +14,16 @@ import threading
 from datetime import datetime, timedelta
 
 from meerschaum.utils.typing import (
-    Union, Optional, Callable, Any, Tuple, SuccessTuple, Mapping, Dict, List, Iterable, Generator,
+    Union,
+    Optional,
+    Callable,
+    Any,
+    Tuple,
+    SuccessTuple,
+    Dict,
+    List,
+    Iterable,
+    Generator,
     Iterator,
 )
 
@@ -29,8 +38,8 @@ def sync(
             List[Dict[str, Any]],
             InferFetch
         ] = InferFetch,
-        begin: Optional[datetime] = None,
-        end: Optional[datetime] = None,
+        begin: Union[datetime, int, str, None] = '',
+        end: Union[datetime, int] = None,
         force: bool = False,
         retries: int = 10,
         min_seconds: int = 1,
@@ -55,28 +64,23 @@ def sync(
     df: Union[None, pd.DataFrame, Dict[str, List[Any]]], default None
         An optional DataFrame to sync into the pipe. Defaults to `None`.
 
-    begin: Optional[datetime], default None
+    begin: Union[datetime, int, str, None], default ''
         Optionally specify the earliest datetime to search for data.
-        Defaults to `None`.
 
-    end: Optional[datetime], default None
+    end: Union[datetime, int, str, None], default None
         Optionally specify the latest datetime to search for data.
-        Defaults to `None`.
 
     force: bool, default False
         If `True`, keep trying to sync untul `retries` attempts.
-        Defaults to `False`.
 
     retries: int, default 10
         If `force`, how many attempts to try syncing before declaring failure.
-        Defaults to `10`.
 
     min_seconds: Union[int, float], default 1
         If `force`, how many seconds to sleep between retries. Defaults to `1`.
 
     check_existing: bool, default True
         If `True`, pull and diff with existing data from the pipe.
-        Defaults to `True`.
 
     blocking: bool, default True
         If `True`, wait for sync to finish and return its result, otherwise
@@ -87,7 +91,6 @@ def sync(
         If provided and the instance connector is thread-safe
         (`pipe.instance_connector.IS_THREAD_SAFE is True`),
         limit concurrent sync to this many threads.
-        Defaults to `None`.
 
     callback: Optional[Callable[[Tuple[bool, str]], Any]], default None
         Callback function which expects a SuccessTuple as input.
@@ -101,7 +104,6 @@ def sync(
         Specify the number of rows to sync per chunk.
         If `-1`, resort to system configuration (default is `900`).
         A `chunksize` of `None` will sync all rows in one transaction.
-        Defaults to `-1`.
 
     sync_chunks: bool, default True
         If possible, sync chunks while fetching them into memory.
@@ -112,7 +114,6 @@ def sync(
     Returns
     -------
     A `SuccessTuple` of success (`bool`) and message (`str`).
-
     """
     from meerschaum.utils.debug import dprint, _checkpoint
     from meerschaum.utils.warnings import warn, error
@@ -183,7 +184,7 @@ def sync(
         ### use that instead.
         ### NOTE: The DataFrame must be omitted for the plugin sync method to apply.
         ### If a DataFrame is provided, continue as expected.
-        if hasattr(df, 'MRSM_INFER_FETCH'):
+        if hasattr(df, 'MRSM_INFER_FETCH'):                   
             try:
                 if p.connector is None:
                     msg = f"{p} does not have a valid connector."
@@ -430,13 +431,19 @@ def sync(
 
 def _determine_begin(
         pipe: meerschaum.Pipe,
-        begin: Optional[datetime] = None,
+        begin: Union[datetime, int, str] = '',
         debug: bool = False,
     ) -> Union[datetime, int, None]:
-    ### Datetime has already been provided.
-    if begin is not None:
+    """
+    Apply the backtrack interval if `--begin` is not provided.
+    """
+    if begin != '':
         return begin
-    return pipe.get_sync_time(debug=debug)
+    sync_time = pipe.get_sync_time(debug=debug)
+    if sync_time is None:
+        return sync_time
+    backtrack_interval = pipe.get_backtrack_interval(debug=debug)
+    return sync_time - backtrack_interval
 
 
 def get_sync_time(

meerschaum/core/Pipe/_verify.py

@@ -62,6 +62,7 @@ def verify(
     A SuccessTuple indicating whether the pipe was successfully resynced.
     """
     from meerschaum.utils.pool import get_pool
+    from meerschaum.utils.misc import interval_str
     workers = self.get_num_workers(workers)
 
     ### Skip configured bounding in parameters
@@ -74,16 +75,16 @@ def verify(
     if bounded is None:
         bounded = bound_time is not None
 
-    if begin is None:
+    if bounded and begin is None:
         begin = (
             bound_time
             if bound_time is not None
             else self.get_sync_time(newest=False, debug=debug)
         )
-    if end is None:
+    if bounded and end is None:
         end = self.get_sync_time(newest=True, debug=debug)
 
-    if bounded:
+    if bounded and end is not None:
         end += (
             timedelta(minutes=1)
             if isinstance(end, datetime)
@@ -93,13 +94,7 @@ def verify(
     sync_less_than_begin = not bounded and begin is None
     sync_greater_than_end = not bounded and end is None
 
-    cannot_determine_bounds = (
-        begin is None
-        or
-        end is None
-        or
-        not self.exists(debug=debug)
-    )
+    cannot_determine_bounds = not self.exists(debug=debug)
 
     if cannot_determine_bounds:
         sync_success, sync_msg = self.sync(
@@ -146,21 +141,48 @@ def verify(
             )
         return True, f"Could not determine chunks between '{begin}' and '{end}'; nothing to do."
 
+    begin_to_print = (
+        begin
+        if begin is not None
+        else (
+            chunk_bounds[0][0]
+            if bounded
+            else chunk_bounds[0][1]
+        )
+    )
+    end_to_print = (
+        end
+        if end is not None
+        else (
+            chunk_bounds[-1][1]
+            if bounded
+            else chunk_bounds[-1][0]
+        )
+    )
+
     info(
         f"Syncing {len(chunk_bounds)} chunk" + ('s' if len(chunk_bounds) != 1 else '')
         + f" ({'un' if not bounded else ''}bounded)"
-        + f" of size '{chunk_interval}'"
-        + f" between '{begin}' and '{end}'."
+        + f" of size '{interval_str(chunk_interval)}'"
+        + f" between '{begin_to_print}' and '{end_to_print}'."
     )
 
     pool = get_pool(workers=workers)
 
+    ### Dictionary of the form bounds -> success_tuple, e.g.:
+    ### {
+    ###    (2023-01-01, 2023-01-02): (True, "Success")
+    ### }
+    bounds_success_tuples = {}
     def process_chunk_bounds(
             chunk_begin_and_end: Tuple[
                 Union[int, datetime],
                 Union[int, datetime]
             ]
         ):
+        if chunk_begin_and_end in bounds_success_tuples:
+            return chunk_begin_and_end, bounds_success_tuples[chunk_begin_and_end]
+
         chunk_begin, chunk_end = chunk_begin_and_end
         return chunk_begin_and_end, self.sync(
             begin = chunk_begin,
@@ -171,14 +193,25 @@ def verify(
             **kwargs
         )
 
-    ### Dictionary of the form bounds -> success_tuple, e.g.:
-    ### {
-    ###    (2023-01-01, 2023-01-02): (True, "Success")
-    ### }
-    bounds_success_tuples = dict(pool.map(process_chunk_bounds, chunk_bounds))
+    ### If we have more than one chunk, attempt to sync the first one and return if its fails.
+    if len(chunk_bounds) > 1:
+        first_chunk_bounds = chunk_bounds[0]
+        (
+            (first_begin, first_end),
+            (first_success, first_msg)
+        ) = process_chunk_bounds(first_chunk_bounds)
+        if not first_success:
+            return (
+                first_success,
+                f"\n{first_begin} - {first_end}\n"
+                + f"Failed to sync first chunk:\n{first_msg}"
+            )
+        bounds_success_tuples[first_chunk_bounds] = (first_success, first_msg)
+
+    bounds_success_tuples.update(dict(pool.map(process_chunk_bounds, chunk_bounds)))
     bounds_success_bools = {bounds: tup[0] for bounds, tup in bounds_success_tuples.items()}
 
-    message_header = f"{begin} - {end}"
+    message_header = f"{begin_to_print} - {end_to_print}"
     if all(bounds_success_bools.values()):
         msg = get_chunks_success_message(bounds_success_tuples, header=message_header)
         if deduplicate:
@@ -195,18 +228,19 @@ def verify(
 
     chunk_bounds_to_resync = [
         bounds
-        for bounds, success in zip(chunk_bounds, chunk_success_bools)
+        for bounds, success in zip(chunk_bounds, bounds_success_bools)
         if not success
     ]
     bounds_to_print = [
         f"{bounds[0]} - {bounds[1]}"
         for bounds in chunk_bounds_to_resync
     ]
-    warn(
-        f"Will resync the following failed chunks:\n    "
-        + '\n    '.join(bounds_to_print),
-        stack = False,
-    )
+    if bounds_to_print:
+        warn(
+            f"Will resync the following failed chunks:\n    "
+            + '\n    '.join(bounds_to_print),
+            stack = False,
+        )
 
     retry_bounds_success_tuples = dict(pool.map(process_chunk_bounds, chunk_bounds_to_resync))
     bounds_success_tuples.update(retry_bounds_success_tuples)
@@ -289,7 +323,8 @@ def get_chunks_success_message(
         ''
         if num_fails == 0
         else (
-            f"\n\nFailed to sync {num_fails} chunks:\n"
+            f"\n\nFailed to sync {num_fails} chunk"
+            + ('s' if num_fails != 1 else '') + ":\n"
             + '\n'.join([
                 f"{fail_begin} - {fail_end}\n{msg}\n"
                 for (fail_begin, fail_end), (_, msg) in fail_chunk_bounds_tuples.items()

meerschaum/plugins/__init__.py

@@ -254,6 +254,9 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
         try:
             if PLUGINS_INTERNAL_LOCK_PATH.exists():
                 PLUGINS_INTERNAL_LOCK_PATH.unlink()
+        ### Sometimes competing threads will delete the lock file at the same time.
+        except FileNotFoundError:
+            pass
         except Exception as e:
             if warn:
                 _warn(f"Error cleaning up lockfile {PLUGINS_INTERNAL_LOCK_PATH}:\n{e}")

meerschaum/utils/daemon/Daemon.py

@@ -168,7 +168,7 @@ class Daemon:
             finally:
                 self._log_refresh_timer.cancel()
                 self.rotating_log.close()
-                if self.pid_path.exists():
+                if self.pid is None and self.pid_path.exists():
                     self.pid_path.unlink()
 
             if keep_daemon_output:
@@ -254,7 +254,7 @@ class Daemon:
         return _launch_success_bool, msg
 
 
-    def kill(self, timeout: Optional[int] = 3) -> SuccessTuple:
+    def kill(self, timeout: Optional[int] = 8) -> SuccessTuple:
         """Forcibly terminate a running daemon.
         Sends a SIGTERM signal to the process.
 
@@ -293,7 +293,7 @@ class Daemon:
         return True, "Success"
 
 
-    def quit(self, timeout: Optional[int] = 3) -> SuccessTuple:
+    def quit(self, timeout: Union[int, float, None] = None) -> SuccessTuple:
         """Gracefully quit a running daemon."""
         if self.status == 'paused':
             return self.kill(timeout)
@@ -305,10 +305,24 @@ class Daemon:
 
     def pause(
             self,
-            timeout: Optional[int] = 3,
-            check_timeout_interval: float = 0.1,
+            timeout: Union[int, float, None] = None,
+            check_timeout_interval: Union[float, int, None] = None,
         ) -> SuccessTuple:
-        """Pause the daemon if it is running."""
+        """
+        Pause the daemon if it is running.
+
+        Parameters
+        ----------
+        timeout: Union[float, int, None], default None
+            The maximum number of seconds to wait for a process to suspend.
+
+        check_timeout_interval: Union[float, int, None], default None
+            The number of seconds to wait between checking if the process is still running.
+
+        Returns
+        -------
+        A `SuccessTuple` indicating whether the `Daemon` process was successfully suspended.
+        """
         if self.process is None:
             return False, f"Daemon '{self.daemon_id}' is not running and cannot be paused."
 
@@ -320,8 +334,18 @@ class Daemon:
         except Exception as e:
             return False, f"Failed to pause daemon '{self.daemon_id}':\n{e}"
 
-        if timeout is None:
-            success = self.process.status() == 'stopped'
+        timeout = self.get_timeout_seconds(timeout)
+        check_timeout_interval = self.get_check_timeout_interval_seconds(
+            check_timeout_interval
+        )
+
+        psutil = attempt_import('psutil')
+
+        if not timeout:
+            try:
+                success = self.process.status() == 'stopped'
+            except psutil.NoSuchProcess as e:
+                success = True
             msg = "Success" if success else f"Failed to suspend daemon '{self.daemon_id}'."
             if success:
                 self._capture_process_timestamp('paused')
@@ -329,9 +353,12 @@ class Daemon:
 
         begin = time.perf_counter()
         while (time.perf_counter() - begin) < timeout:
-            if self.process.status() == 'stopped':
-                self._capture_process_timestamp('paused')
-                return True, "Success"
+            try:
+                if self.process.status() == 'stopped':
+                    self._capture_process_timestamp('paused')
+                    return True, "Success"
+            except psutil.NoSuchProcess as e:
+                return False, f"Process exited unexpectedly. Was it killed?\n{e}"
             time.sleep(check_timeout_interval)
 
         return False, (
@@ -342,10 +369,24 @@ class Daemon:
 
     def resume(
             self,
-            timeout: Optional[int] = 3,
-            check_timeout_interval: float = 0.1,
+            timeout: Union[int, float, None] = None,
+            check_timeout_interval: Union[float, int, None] = None,
         ) -> SuccessTuple:
-        """Resume the daemon if it is paused."""
+        """
+        Resume the daemon if it is paused.
+
+        Parameters
+        ----------
+        timeout: Union[float, int, None], default None
+            The maximum number of seconds to wait for a process to resume.
+
+        check_timeout_interval: Union[float, int, None], default None
+            The number of seconds to wait between checking if the process is still stopped.
+
+        Returns
+        -------
+        A `SuccessTuple` indicating whether the `Daemon` process was successfully resumed.
+        """
         if self.status == 'running':
             return True, f"Daemon '{self.daemon_id}' is already running."
 
@@ -357,7 +398,12 @@ class Daemon:
         except Exception as e:
             return False, f"Failed to resume daemon '{self.daemon_id}':\n{e}"
 
-        if timeout is None:
+        timeout = self.get_timeout_seconds(timeout)
+        check_timeout_interval = self.get_check_timeout_interval_seconds(
+            check_timeout_interval
+        )
+
+        if not timeout:
             success = self.status == 'running'
             msg = "Success" if success else f"Failed to resume daemon '{self.daemon_id}'."
             if success:
@@ -417,8 +463,8 @@ class Daemon:
     def _send_signal(
             self,
             signal_to_send,
-            timeout: Optional[Union[float, int]] = 3,
-            check_timeout_interval: float = 0.1,
+            timeout: Union[float, int, None] = None,
+            check_timeout_interval: Union[float, int, None] = None,
         ) -> SuccessTuple:
         """Send a signal to the daemon process.
 
@@ -427,13 +473,11 @@ class Daemon:
         signal_to_send:
             The signal the send to the daemon, e.g. `signals.SIGINT`.
 
-        timeout:
+        timeout: Union[float, int, None], default None
             The maximum number of seconds to wait for a process to terminate.
-            Defaults to 3.
 
-        check_timeout_interval: float, default 0.1
+        check_timeout_interval: Union[float, int, None], default None
             The number of seconds to wait between checking if the process is still running.
-            Defaults to 0.1.
 
         Returns
         -------
@@ -444,11 +488,17 @@ class Daemon:
         except Exception as e:
             return False, f"Failed to send signal {signal_to_send}:\n{traceback.format_exc()}"
 
-        if timeout is None:
+        timeout = self.get_timeout_seconds(timeout)
+        check_timeout_interval = self.get_check_timeout_interval_seconds(
+            check_timeout_interval
+        )
+
+        if not timeout:
             return True, f"Successfully sent '{signal}' to daemon '{self.daemon_id}'."
+
         begin = time.perf_counter()
         while (time.perf_counter() - begin) < timeout:
-            if not self.pid_path.exists():
+            if not self.status == 'running':
                 return True, "Success"
             time.sleep(check_timeout_interval)
 
@@ -464,8 +514,8 @@ class Daemon:
         raise a `FileExistsError`.
         """
         try:
-            self.path.mkdir(parents=True, exist_ok=False)
-            _already_exists = False
+            self.path.mkdir(parents=True, exist_ok=True)
+            _already_exists = any(os.scandir(self.path))
         except FileExistsError:
             _already_exists = True
 
@@ -506,8 +556,17 @@ class Daemon:
         if self.process is None:
             return 'stopped'
 
-        if self.process.status() == 'stopped':
-            return 'paused'
+        psutil = attempt_import('psutil')
+        try:
+            if self.process.status() == 'stopped':
+                return 'paused'
+        except psutil.NoSuchProcess:
+            if self.pid_path.exists():
+                try:
+                    self.pid_path.unlink()
+                except Exception as e:
+                    pass
+            return 'stopped'
 
         return 'running'
 
@@ -804,6 +863,28 @@ class Daemon:
             self.rotating_log.delete()
 
 
+    def get_timeout_seconds(self, timeout: Union[int, float, None] = None) -> Union[int, float]:
+        """
+        Return the timeout value to use. Use `--timeout-seconds` if provided,
+        else the configured default (8).
+        """
+        if isinstance(timeout, (int, float)):
+            return timeout
+        return get_config('jobs', 'timeout_seconds')
+
+
+    def get_check_timeout_interval_seconds(
+            self,
+            check_timeout_interval: Union[int, float, None] = None,
+        ) -> Union[int, float]:
+        """
+        Return the interval value to check the status of timeouts.
+        """
+        if isinstance(check_timeout_interval, (int, float)):
+            return check_timeout_interval
+        return get_config('jobs', 'check_timeout_interval_seconds')
+
+
     def __getstate__(self):
         """
         Pickle this Daemon.

meerschaum/utils/daemon/__init__.py

@@ -14,6 +14,7 @@ from meerschaum.utils.daemon.Daemon import Daemon
 from meerschaum.utils.daemon.Log import Log
 from meerschaum.utils.daemon.RotatingFile import RotatingFile
 
+
 def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     """Parse sysargs and execute a Meerschaum action as a daemon.
 
@@ -27,7 +28,7 @@ def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
     A SuccessTuple.
     """
     from meerschaum._internal.entry import entry
-    _args = None
+    _args = {}
     if '--name' in sysargs or '--job-name' in sysargs:
         from meerschaum._internal.arguments._parse_arguments import parse_arguments
         _args = parse_arguments(sysargs)
@@ -36,6 +37,38 @@ def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
         label = shlex.join(filtered_sysargs) if sysargs else None
     except Exception as e:
         label = ' '.join(filtered_sysargs) if sysargs else None
+
+    name = _args.get('name', None)
+    daemon = None
+    if name:
+        try:
+            daemon = Daemon(daemon_id=name)
+        except Exception as e:
+            daemon = None
+
+    if daemon is not None:
+        existing_sysargs = daemon.properties['target']['args'][0]
+        existing_kwargs = parse_arguments(existing_sysargs)
+
+        ### Remove sysargs because flags are aliased.
+        _ = _args.pop('daemon', None)
+        _ = _args.pop('sysargs', None)
+        _ = _args.pop('filtered_sysargs', None)
+        debug = _args.pop('debug', None)
+        _args['sub_args'] = sorted(_args.get('sub_args', []))
+        _ = existing_kwargs.pop('daemon', None)
+        _ = existing_kwargs.pop('sysargs', None)
+        _ = existing_kwargs.pop('filtered_sysargs', None)
+        _ = existing_kwargs.pop('debug', None)
+        existing_kwargs['sub_args'] = sorted(existing_kwargs.get('sub_args', []))
+
+        ### Only run if the kwargs equal or no actions are provided.
+        if existing_kwargs == _args or not _args.get('action', []):
+            return daemon.run(
+                debug = debug,
+                allow_dirty_run = True,
+            )
+
     success_tuple = run_daemon(
         entry,
         filtered_sysargs,
@@ -47,6 +80,7 @@ def daemon_entry(sysargs: Optional[List[str]] = None) -> SuccessTuple:
         success_tuple = False, str(success_tuple)
     return success_tuple
 
+
 def daemon_action(**kw) -> SuccessTuple:
     """Execute a Meerschaum action as a daemon."""
     from meerschaum.utils.packages import run_python_package

meerschaum/utils/dataframe.py

@@ -651,6 +651,8 @@ def get_datetime_bound_from_df(
         return best_yet
 
     if 'DataFrame' in str(type(df)):
+        if datetime_column not in df.columns:
+            return None
         return (
             df[datetime_column].min(skipna=True)
             if minimum