meerschaum 2.4.0.dev0__py3-none-any.whl → 2.4.0rc1__py3-none-any.whl

meerschaum/core/Pipe/__init__.py

@@ -135,6 +135,7 @@ class Pipe:
     from ._deduplicate import deduplicate
     from ._bootstrap import bootstrap
     from ._dtypes import enforce_dtypes, infer_dtypes
+    from ._copy import copy_to
 
     def __init__(
         self,

meerschaum/core/Pipe/_clear.py

@@ -7,25 +7,28 @@ Clear pipe data within a bounded or unbounded interval.
 """
 
 from __future__ import annotations
+
+from datetime import datetime
 from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict
 
+
 def clear(
-        self,
-        begin: Optional[datetime.datetime] = None,
-        end: Optional[datetime.datetime] = None,
-        params: Optional[Dict[str, Any]] = None,
-        debug: bool = False,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    self,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Call the Pipe's instance connector's `clear_pipe` method.
 
     Parameters
     ----------
-    begin: Optional[datetime.datetime], default None:
+    begin: Optional[datetime], default None:
         If provided, only remove rows newer than this datetime value.
 
-    end: Optional[datetime.datetime], default None:
+    end: Optional[datetime], default None:
         If provided, only remove rows older than this datetime column (not including end).
 
     params: Optional[Dict[str, Any]], default None
@@ -41,11 +44,11 @@ def clear(
     Examples
     --------
     >>> pipe = mrsm.Pipe('test', 'test', columns={'datetime': 'dt'}, instance='sql:local')
-    >>> pipe.sync({'dt': [datetime.datetime(2020, 1, 1, 0, 0)]})
-    >>> pipe.sync({'dt': [datetime.datetime(2021, 1, 1, 0, 0)]})
-    >>> pipe.sync({'dt': [datetime.datetime(2022, 1, 1, 0, 0)]})
+    >>> pipe.sync({'dt': [datetime(2020, 1, 1, 0, 0)]})
+    >>> pipe.sync({'dt': [datetime(2021, 1, 1, 0, 0)]})
+    >>> pipe.sync({'dt': [datetime(2022, 1, 1, 0, 0)]})
     >>> 
-    >>> pipe.clear(begin=datetime.datetime(2021, 1, 1, 0, 0))
+    >>> pipe.clear(begin=datetime(2021, 1, 1, 0, 0))
     >>> pipe.get_data()
               dt
     0 2020-01-01

meerschaum/core/Pipe/_copy.py

@@ -0,0 +1,106 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define methods for copying pipes.
+"""
+
+from datetime import datetime, timedelta
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Union
+
+
+def copy_to(
+    self,
+    instance_keys: str,
+    sync: bool = True,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
+    """
+    Copy a pipe to another instance.
+
+    Parameters
+    ----------
+    instance_keys: str
+        The instance to which to copy this pipe.
+
+    sync: bool, default True
+        If `True`, sync the source pipe's documents 
+
+    begin: Union[datetime, int, None], default None
+        Beginning datetime value to pass to `Pipe.get_data()`.
+
+    end: Union[datetime, int, None], default None
+        End datetime value to pass to `Pipe.get_data()`.
+
+    params: Optional[Dict[str, Any]], default None
+        Parameters filter to pass to `Pipe.get_data()`.
+
+    chunk_interval: Union[timedelta, int, None], default None
+        The size of chunks to retrieve from `Pipe.get_data()` for syncing.
+
+    kwargs: Any
+        Additional flags to pass to `Pipe.get_data()` and `Pipe.sync()`, e.g. `workers`.
+
+    Returns
+    -------
+    A SuccessTuple indicating success.
+    """
+    if str(instance_keys) == self.instance_keys:
+        return False, f"Cannot copy {self} to instance '{instance_keys}'."
+
+    new_pipe = mrsm.Pipe(
+        self.connector_keys,
+        self.metric_key,
+        self.location_key,
+        parameters=self.parameters.copy(),
+        instance=instance_keys,
+    )
+
+    new_pipe_is_registered = new_pipe.get_id() is not None
+
+    metadata_method = new_pipe.edit if new_pipe_is_registered else new_pipe.register
+    metadata_success, metadata_msg = metadata_method(debug=debug)
+    if not metadata_success:
+        return metadata_success, metadata_msg
+
+    if not self.exists(debug=debug):
+        return True, f"{self} does not exist; nothing to sync."
+
+    original_as_iterator = kwargs.get('as_iterator', None)
+    kwargs['as_iterator'] = True
+
+    chunk_generator = self.get_data(
+        begin=begin,
+        end=end,
+        params=params,
+        chunk_interval=chunk_interval,
+        debug=debug,
+        **kwargs
+    )
+
+    if original_as_iterator is None:
+        _ = kwargs.pop('as_iterator', None)
+    else:
+        kwargs['as_iterator'] = original_as_iterator
+
+    sync_success, sync_msg = new_pipe.sync(
+        chunk_generator,
+        begin=begin,
+        end=end,
+        params=params,
+        debug=debug,
+        **kwargs
+    )
+    msg = (
+        f"Successfully synced {new_pipe}:\n{sync_msg}"
+        if sync_success
+        else f"Failed to sync {new_pipe}:\n{sync_msg}"
+    )
+    return sync_success, msg

meerschaum/core/Pipe/_data.py

@@ -30,6 +30,8 @@ def get_data(
     as_chunks: bool = False,
     as_dask: bool = False,
     chunk_interval: Union[timedelta, int, None] = None,
+    order: Optional[str] = 'asc',
+    limit: Optional[int] = None,
     fresh: bool = False,
     debug: bool = False,
     **kw: Any
@@ -80,6 +82,12 @@ def get_data(
         If `chunk_interval` is a `timedelta` and the `datetime` axis an integer,
         use the number of minutes in the `timedelta`.
 
+    order: Optional[str], default 'asc'
+        If `order` is not `None`, sort the resulting dataframe by indices.
+
+    limit: Optional[int], default None
+        If provided, cap the dataframe to this many rows.
+
     fresh: bool, default True
         If `True`, skip local cache and directly query the instance connector.
         Defaults to `True`.
@@ -113,17 +121,39 @@ def get_data(
 
     as_iterator = as_iterator or as_chunks
 
+    def _sort_df(_df):
+        dt_col = self.columns.get('datetime', None)
+        indices = [] if dt_col not in _df.columns else [dt_col]
+        non_dt_cols = [
+            col
+            for col_ix, col in self.columns.values()
+            if col_ix != 'datetime' and col in _df.columns
+        ]
+        indices.extend(non_dt_cols)
+        _df.sort_values(
+            by=indices,
+            inplace=True,
+            ascending=(str(order).lower() == 'asc')
+        )
+        _df.reset_index(drop=True, inplace=True)
+        if limit is not None and len(_df) > limit:
+            return _df.head(limit)
+        return _df
+
     if as_iterator or as_chunks:
-        return self._get_data_as_iterator(
-            select_columns = select_columns,
-            omit_columns = omit_columns,
-            begin = begin,
-            end = end,
-            params = params,
-            chunk_interval = chunk_interval,
-            fresh = fresh,
-            debug = debug,
+        df = self._get_data_as_iterator(
+            select_columns=select_columns,
+            omit_columns=omit_columns,
+            begin=begin,
+            end=end,
+            params=params,
+            chunk_interval=chunk_interval,
+            limit=limit,
+            order=order,
+            fresh=fresh,
+            debug=debug,
         )
+        return _sort_df(df)
 
     if as_dask:
         from multiprocessing.pool import ThreadPool
@@ -131,22 +161,24 @@ def get_data(
         dask.config.set(pool=dask_pool)
         chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
         bounds = self.get_chunk_bounds(
-            begin = begin,
-            end = end,
-            bounded = False,
-            chunk_interval = chunk_interval,
-            debug = debug,
+            begin=begin,
+            end=end,
+            bounded=False,
+            chunk_interval=chunk_interval,
+            debug=debug,
         )
         dask_chunks = [
             dask.delayed(self.get_data)(
-                select_columns = select_columns,
-                omit_columns = omit_columns,
-                begin = chunk_begin,
-                end = chunk_end,
-                params = params,
-                chunk_interval = chunk_interval,
-                fresh = fresh,
-                debug = debug,
+                select_columns=select_columns,
+                omit_columns=omit_columns,
+                begin=chunk_begin,
+                end=chunk_end,
+                params=params,
+                chunk_interval=chunk_interval,
+                order=order,
+                limit=limit,
+                fresh=fresh,
+                debug=debug,
             )
             for (chunk_begin, chunk_end) in bounds
         ]
@@ -154,18 +186,18 @@ def get_data(
             col: to_pandas_dtype(typ)
             for col, typ in self.dtypes.items()
         }
-        return dd.from_delayed(dask_chunks, meta=dask_meta)
+        return _sort_df(dd.from_delayed(dask_chunks, meta=dask_meta))
 
     if not self.exists(debug=debug):
         return None
-       
+
     if self.cache_pipe is not None:
         if not fresh:
             _sync_cache_tuple = self.cache_pipe.sync(
-                begin = begin,
-                end = end,
-                params = params,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                debug=debug,
                 **kw
             )
             if not _sync_cache_tuple[0]:
@@ -174,27 +206,31 @@ def get_data(
             else: ### Successfully synced cache.
                 return self.enforce_dtypes(
                     self.cache_pipe.get_data(
-                        select_columns = select_columns,
-                        omit_columns = omit_columns,
-                        begin = begin,
-                        end = end,
-                        params = params,
-                        debug = debug,
-                        fresh = True,
+                        select_columns=select_columns,
+                        omit_columns=omit_columns,
+                        begin=begin,
+                        end=end,
+                        params=params,
+                        order=order,
+                        limit=limit,
+                        debug=debug,
+                        fresh=True,
                         **kw
                     ),
-                    debug = debug,
+                    debug=debug,
                 )
 
     with Venv(get_connector_plugin(self.instance_connector)):
         df = self.instance_connector.get_pipe_data(
-            pipe = self,
-            select_columns = select_columns,
-            omit_columns = omit_columns,
-            begin = begin,
-            end = end,
-            params = params,
-            debug = debug,
+            pipe=self,
+            select_columns=select_columns,
+            omit_columns=omit_columns,
+            begin=begin,
+            end=end,
+            params=params,
+            limit=limit,
+            order=order,
+            debug=debug,
             **kw
         )
         if df is None:
@@ -226,7 +262,7 @@ def get_data(
                     + "Consider adding `select_columns` and `omit_columns` support to "
                     + f"'{self.instance_connector.type}' connectors to improve performance."
                 ),
-                stack = False,
+                stack=False,
             )
             _cols_to_select = [col for col in df.columns if col not in cols_to_omit]
             df = df[_cols_to_select]
@@ -237,25 +273,31 @@ def get_data(
                     f"Specified columns {items_str(cols_to_add)} were not found on {self}. "
                     + "Adding these to the DataFrame as null columns."
                 ),
-                stack = False,
+                stack=False,
             )
             df = add_missing_cols_to_df(df, {col: 'string' for col in cols_to_add})
 
-        return self.enforce_dtypes(df, debug=debug)
+        enforced_df = self.enforce_dtypes(df, debug=debug)
+
+        if order:
+            return _sort_df(enforced_df)
+        return enforced_df
 
 
 def _get_data_as_iterator(
-        self,
-        select_columns: Optional[List[str]] = None,
-        omit_columns: Optional[List[str]] = None,
-        begin: Optional[datetime] = None,
-        end: Optional[datetime] = None,
-        params: Optional[Dict[str, Any]] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        fresh: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Iterator['pd.DataFrame']:
+    self,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    params: Optional[Dict[str, Any]] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    limit: Optional[int] = None,
+    order: Optional[str] = 'asc',
+    fresh: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Iterator['pd.DataFrame']:
     """
     Return a pipe's data as a generator.
     """
@@ -305,46 +347,51 @@ def _get_data_as_iterator(
         (min_dt + chunk_interval) > max_dt
     ):
         yield self.get_data(
-            select_columns = select_columns,
-            omit_columns = omit_columns,
-            begin = begin,
-            end = end,
-            params = params,
-            fresh = fresh,
-            debug = debug,
+            select_columns=select_columns,
+            omit_columns=omit_columns,
+            begin=begin,
+            end=end,
+            params=params,
+            limit=limit,
+            order=order,
+            fresh=fresh,
+            debug=debug,
         )
         return
 
     chunk_bounds = self.get_chunk_bounds(
-        begin = min_dt,
-        end = max_dt,
-        chunk_interval = chunk_interval,
-        debug = debug,
+        begin=min_dt,
+        end=max_dt,
+        chunk_interval=chunk_interval,
+        debug=debug,
     )
 
     for chunk_begin, chunk_end in chunk_bounds:
         chunk = self.get_data(
-            select_columns = select_columns,
-            omit_columns = omit_columns,
-            begin = chunk_begin,
-            end = chunk_end,
-            params = params,
-            fresh = fresh,
-            debug = debug,
+            select_columns=select_columns,
+            omit_columns=omit_columns,
+            begin=chunk_begin,
+            end=chunk_end,
+            params=params,
+            limit=limit,
+            order=order,
+            fresh=fresh,
+            debug=debug,
         )
         if len(chunk) > 0:
             yield chunk
 
 
 def get_backtrack_data(
-        self,
-        backtrack_minutes: Optional[int] = None,
-        begin: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        fresh: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Optional['pd.DataFrame']:
+    self,
+    backtrack_minutes: Optional[int] = None,
+    begin: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    limit: Optional[int] = None,
+    fresh: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Optional['pd.DataFrame']:
     """
     Get the most recent data from the instance connector as a Pandas DataFrame.
 
@@ -371,8 +418,10 @@ def get_backtrack_data(
 
     params: Optional[Dict[str, Any]], default None
         The standard Meerschaum `params` query dictionary.
-        
-        
+
+    limit: Optional[int], default None
+        If provided, cap the number of rows to be returned.
+
     fresh: bool, default False
         If `True`, Ignore local cache and pull directly from the instance connector.
         Only comes into effect if a pipe was created with `cache=True`.
@@ -409,28 +458,31 @@ def get_backtrack_data(
             else: ### Successfully synced cache.
                 return self.enforce_dtypes(
                     self.cache_pipe.get_backtrack_data(
-                        fresh = True,
-                        begin = begin,
-                        backtrack_minutes = backtrack_minutes,
-                        params = params,
-                        debug = deubg,
+                        fresh=True,
+                        begin=begin,
+                        backtrack_minutes=backtrack_minutes,
+                        params=params,
+                        limit=limit,
+                        order=kw.get('order', 'desc'),
+                        debug=debug,
                         **kw
                     ),
-                    debug = debug,
+                    debug=debug,
                 )
 
     if hasattr(self.instance_connector, 'get_backtrack_data'):
         with Venv(get_connector_plugin(self.instance_connector)):
             return self.enforce_dtypes(
                 self.instance_connector.get_backtrack_data(
-                    pipe = self,
-                    begin = begin,
-                    backtrack_minutes = backtrack_minutes,
-                    params = params,
-                    debug = debug,
+                    pipe=self,
+                    begin=begin,
+                    backtrack_minutes=backtrack_minutes,
+                    params=params,
+                    limit=limit,
+                    debug=debug,
                     **kw
                 ),
-                debug = debug,
+                debug=debug,
             )
 
     if begin is None:
@@ -445,13 +497,16 @@ def get_backtrack_data(
         begin = begin - backtrack_interval
 
     return self.get_data(
-        begin = begin,
-        params = params,
-        debug = debug,
+        begin=begin,
+        params=params,
+        debug=debug,
+        limit=limit,
+        order=kw.get('order', 'desc'),
         **kw
     )
 
 
+
 def get_rowcount(
         self,
         begin: Optional[datetime] = None,

meerschaum/core/Pipe/_verify.py

@@ -12,17 +12,17 @@ from meerschaum.utils.warnings import warn, info
 from meerschaum.utils.debug import dprint
 
 def verify(
-        self,
-        begin: Union[datetime, int, None] = None,
-        end: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        bounded: Optional[bool] = None,
-        deduplicate: bool = False,
-        workers: Optional[int] = None,
-        debug: bool = False,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    self,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    bounded: Optional[bool] = None,
+    deduplicate: bool = False,
+    workers: Optional[int] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Verify the contents of the pipe by resyncing its interval.
 

meerschaum/jobs/_Job.py

@@ -10,10 +10,7 @@ from __future__ import annotations
 
 import shlex
 import asyncio
-import threading
-import json
 import pathlib
-import os
 import sys
 import traceback
 from functools import partial
@@ -206,13 +203,11 @@ class Job:
             jobs_dir = root_dir / DAEMON_RESOURCES_PATH.name
             daemon_dir = jobs_dir / daemon_id
             pid_file = daemon_dir / 'process.pid'
-            properties_path = daemon_dir / 'properties.json'
-            pickle_path = daemon_dir / 'pickle.pkl'
 
             if pid_file.exists():
                 with open(pid_file, 'r', encoding='utf-8') as f:
                     daemon_pid = int(f.read())
-                
+
                 if pid != daemon_pid:
                     raise EnvironmentError(f"Differing PIDs: {pid=}, {daemon_pid=}")
             else:

meerschaum/jobs/__init__.py

@@ -9,9 +9,9 @@ Higher-level utilities for managing `meerschaum.utils.daemon.Daemon`.
 import pathlib
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Dict, Optional, List, Callable, Any, SuccessTuple
+from meerschaum.utils.typing import Dict, Optional, List, SuccessTuple
 
-from meerschaum.jobs._Job import Job, StopMonitoringLogs
+from meerschaum.jobs._Job import Job
 from meerschaum.jobs._Executor import Executor
 
 __all__ = (
@@ -403,9 +403,14 @@ def get_executor_keys_from_context() -> str:
     if _context_keys is not None:
         return _context_keys
 
+    from meerschaum.config import get_config
     from meerschaum.config.paths import ROOT_DIR_PATH, DEFAULT_ROOT_DIR_PATH
     from meerschaum.utils.misc import is_systemd_available
 
+    configured_executor = get_config('meerschaum', 'executor', warn=False)
+    if configured_executor is not None:
+        return configured_executor
+
     _context_keys = (
         'systemd'
         if is_systemd_available() and ROOT_DIR_PATH == DEFAULT_ROOT_DIR_PATH

meerschaum/utils/dataframe.py

@@ -1187,6 +1187,9 @@ def query_df(
         query_mask = query_mask & mask
 
     original_cols = df.columns
+
+    ### NOTE: We must cast bool columns to `boolean[pyarrow]`
+    ###       to allow for `<NA>` values.
     bool_cols = [
         col
         for col, typ in df.dtypes.items()
@@ -1194,7 +1197,7 @@ def query_df(
     ]
     for col in bool_cols:
         df[col] = df[col].astype('boolean[pyarrow]')
-    df['__mrsm_mask'] = query_mask
+    df['__mrsm_mask'] = query_mask.astype('boolean[pyarrow]')
 
     if inplace:
         df.where(query_mask, other=NA, inplace=True)