meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0__py3-none-any.whl

meerschaum/core/Pipe/_copy.py

@@ -0,0 +1,106 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define methods for copying pipes.
+"""
+
+from datetime import datetime, timedelta
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Union
+
+
+def copy_to(
+    self,
+    instance_keys: str,
+    sync: bool = True,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
+    """
+    Copy a pipe to another instance.
+
+    Parameters
+    ----------
+    instance_keys: str
+        The instance to which to copy this pipe.
+
+    sync: bool, default True
+        If `True`, sync the source pipe's documents 
+
+    begin: Union[datetime, int, None], default None
+        Beginning datetime value to pass to `Pipe.get_data()`.
+
+    end: Union[datetime, int, None], default None
+        End datetime value to pass to `Pipe.get_data()`.
+
+    params: Optional[Dict[str, Any]], default None
+        Parameters filter to pass to `Pipe.get_data()`.
+
+    chunk_interval: Union[timedelta, int, None], default None
+        The size of chunks to retrieve from `Pipe.get_data()` for syncing.
+
+    kwargs: Any
+        Additional flags to pass to `Pipe.get_data()` and `Pipe.sync()`, e.g. `workers`.
+
+    Returns
+    -------
+    A SuccessTuple indicating success.
+    """
+    if str(instance_keys) == self.instance_keys:
+        return False, f"Cannot copy {self} to instance '{instance_keys}'."
+
+    new_pipe = mrsm.Pipe(
+        self.connector_keys,
+        self.metric_key,
+        self.location_key,
+        parameters=self.parameters.copy(),
+        instance=instance_keys,
+    )
+
+    new_pipe_is_registered = new_pipe.get_id() is not None
+
+    metadata_method = new_pipe.edit if new_pipe_is_registered else new_pipe.register
+    metadata_success, metadata_msg = metadata_method(debug=debug)
+    if not metadata_success:
+        return metadata_success, metadata_msg
+
+    if not self.exists(debug=debug):
+        return True, f"{self} does not exist; nothing to sync."
+
+    original_as_iterator = kwargs.get('as_iterator', None)
+    kwargs['as_iterator'] = True
+
+    chunk_generator = self.get_data(
+        begin=begin,
+        end=end,
+        params=params,
+        chunk_interval=chunk_interval,
+        debug=debug,
+        **kwargs
+    )
+
+    if original_as_iterator is None:
+        _ = kwargs.pop('as_iterator', None)
+    else:
+        kwargs['as_iterator'] = original_as_iterator
+
+    sync_success, sync_msg = new_pipe.sync(
+        chunk_generator,
+        begin=begin,
+        end=end,
+        params=params,
+        debug=debug,
+        **kwargs
+    )
+    msg = (
+        f"Successfully synced {new_pipe}:\n{sync_msg}"
+        if sync_success
+        else f"Failed to sync {new_pipe}:\n{sync_msg}"
+    )
+    return sync_success, msg

meerschaum/core/Pipe/_data.py

@@ -30,6 +30,8 @@ def get_data(
     as_chunks: bool = False,
     as_dask: bool = False,
     chunk_interval: Union[timedelta, int, None] = None,
+    order: Optional[str] = 'asc',
+    limit: Optional[int] = None,
     fresh: bool = False,
     debug: bool = False,
     **kw: Any
@@ -80,6 +82,12 @@ def get_data(
         If `chunk_interval` is a `timedelta` and the `datetime` axis an integer,
         use the number of minutes in the `timedelta`.
 
+    order: Optional[str], default 'asc'
+        If `order` is not `None`, sort the resulting dataframe by indices.
+
+    limit: Optional[int], default None
+        If provided, cap the dataframe to this many rows.
+
     fresh: bool, default True
         If `True`, skip local cache and directly query the instance connector.
         Defaults to `True`.
@@ -98,7 +106,7 @@ def get_data(
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.misc import iterate_chunks, items_str
     from meerschaum.utils.dtypes import to_pandas_dtype
-    from meerschaum.utils.dataframe import add_missing_cols_to_df
+    from meerschaum.utils.dataframe import add_missing_cols_to_df, df_is_chunk_generator
     from meerschaum.utils.packages import attempt_import
     dd = attempt_import('dask.dataframe') if as_dask else None
     dask = attempt_import('dask') if as_dask else None
@@ -113,17 +121,48 @@ def get_data(
 
     as_iterator = as_iterator or as_chunks
 
+    def _sort_df(_df):
+        if df_is_chunk_generator(_df):
+            return _df
+        dt_col = self.columns.get('datetime', None)
+        indices = [] if dt_col not in _df.columns else [dt_col]
+        non_dt_cols = [
+            col
+            for col_ix, col in self.columns.items()
+            if col_ix != 'datetime' and col in _df.columns
+        ]
+        indices.extend(non_dt_cols)
+        if 'dask' not in _df.__module__:
+            _df.sort_values(
+                by=indices,
+                inplace=True,
+                ascending=(str(order).lower() == 'asc'),
+            )
+            _df.reset_index(drop=True, inplace=True)
+        else:
+            _df = _df.sort_values(
+                by=indices,
+                ascending=(str(order).lower() == 'asc'),
+            )
+            _df = _df.reset_index(drop=True)
+        if limit is not None and len(_df) > limit:
+            return _df.head(limit)
+        return _df
+
     if as_iterator or as_chunks:
-        return self._get_data_as_iterator(
-            select_columns = select_columns,
-            omit_columns = omit_columns,
-            begin = begin,
-            end = end,
-            params = params,
-            chunk_interval = chunk_interval,
-            fresh = fresh,
-            debug = debug,
+        df = self._get_data_as_iterator(
+            select_columns=select_columns,
+            omit_columns=omit_columns,
+            begin=begin,
+            end=end,
+            params=params,
+            chunk_interval=chunk_interval,
+            limit=limit,
+            order=order,
+            fresh=fresh,
+            debug=debug,
         )
+        return _sort_df(df)
 
     if as_dask:
         from multiprocessing.pool import ThreadPool
@@ -131,22 +170,24 @@ def get_data(
         dask.config.set(pool=dask_pool)
         chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
         bounds = self.get_chunk_bounds(
-            begin = begin,
-            end = end,
-            bounded = False,
-            chunk_interval = chunk_interval,
-            debug = debug,
+            begin=begin,
+            end=end,
+            bounded=False,
+            chunk_interval=chunk_interval,
+            debug=debug,
         )
         dask_chunks = [
             dask.delayed(self.get_data)(
-                select_columns = select_columns,
-                omit_columns = omit_columns,
-                begin = chunk_begin,
-                end = chunk_end,
-                params = params,
-                chunk_interval = chunk_interval,
-                fresh = fresh,
-                debug = debug,
+                select_columns=select_columns,
+                omit_columns=omit_columns,
+                begin=chunk_begin,
+                end=chunk_end,
+                params=params,
+                chunk_interval=chunk_interval,
+                order=order,
+                limit=limit,
+                fresh=fresh,
+                debug=debug,
             )
             for (chunk_begin, chunk_end) in bounds
         ]
@@ -154,18 +195,18 @@ def get_data(
             col: to_pandas_dtype(typ)
             for col, typ in self.dtypes.items()
         }
-        return dd.from_delayed(dask_chunks, meta=dask_meta)
+        return _sort_df(dd.from_delayed(dask_chunks, meta=dask_meta))
 
     if not self.exists(debug=debug):
         return None
-       
+
     if self.cache_pipe is not None:
         if not fresh:
             _sync_cache_tuple = self.cache_pipe.sync(
-                begin = begin,
-                end = end,
-                params = params,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                debug=debug,
                 **kw
             )
             if not _sync_cache_tuple[0]:
@@ -174,27 +215,31 @@ def get_data(
             else: ### Successfully synced cache.
                 return self.enforce_dtypes(
                     self.cache_pipe.get_data(
-                        select_columns = select_columns,
-                        omit_columns = omit_columns,
-                        begin = begin,
-                        end = end,
-                        params = params,
-                        debug = debug,
-                        fresh = True,
+                        select_columns=select_columns,
+                        omit_columns=omit_columns,
+                        begin=begin,
+                        end=end,
+                        params=params,
+                        order=order,
+                        limit=limit,
+                        debug=debug,
+                        fresh=True,
                         **kw
                     ),
-                    debug = debug,
+                    debug=debug,
                 )
 
     with Venv(get_connector_plugin(self.instance_connector)):
         df = self.instance_connector.get_pipe_data(
-            pipe = self,
-            select_columns = select_columns,
-            omit_columns = omit_columns,
-            begin = begin,
-            end = end,
-            params = params,
-            debug = debug,
+            pipe=self,
+            select_columns=select_columns,
+            omit_columns=omit_columns,
+            begin=begin,
+            end=end,
+            params=params,
+            limit=limit,
+            order=order,
+            debug=debug,
             **kw
         )
         if df is None:
@@ -226,7 +271,7 @@ def get_data(
                     + "Consider adding `select_columns` and `omit_columns` support to "
                     + f"'{self.instance_connector.type}' connectors to improve performance."
                 ),
-                stack = False,
+                stack=False,
             )
             _cols_to_select = [col for col in df.columns if col not in cols_to_omit]
             df = df[_cols_to_select]
@@ -237,25 +282,31 @@ def get_data(
                     f"Specified columns {items_str(cols_to_add)} were not found on {self}. "
                     + "Adding these to the DataFrame as null columns."
                 ),
-                stack = False,
+                stack=False,
             )
             df = add_missing_cols_to_df(df, {col: 'string' for col in cols_to_add})
 
-        return self.enforce_dtypes(df, debug=debug)
+        enforced_df = self.enforce_dtypes(df, debug=debug)
+
+        if order:
+            return _sort_df(enforced_df)
+        return enforced_df
 
 
 def _get_data_as_iterator(
-        self,
-        select_columns: Optional[List[str]] = None,
-        omit_columns: Optional[List[str]] = None,
-        begin: Optional[datetime] = None,
-        end: Optional[datetime] = None,
-        params: Optional[Dict[str, Any]] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        fresh: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Iterator['pd.DataFrame']:
+    self,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    params: Optional[Dict[str, Any]] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    limit: Optional[int] = None,
+    order: Optional[str] = 'asc',
+    fresh: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Iterator['pd.DataFrame']:
     """
     Return a pipe's data as a generator.
     """
@@ -305,46 +356,51 @@ def _get_data_as_iterator(
         (min_dt + chunk_interval) > max_dt
     ):
         yield self.get_data(
-            select_columns = select_columns,
-            omit_columns = omit_columns,
-            begin = begin,
-            end = end,
-            params = params,
-            fresh = fresh,
-            debug = debug,
+            select_columns=select_columns,
+            omit_columns=omit_columns,
+            begin=begin,
+            end=end,
+            params=params,
+            limit=limit,
+            order=order,
+            fresh=fresh,
+            debug=debug,
         )
         return
 
     chunk_bounds = self.get_chunk_bounds(
-        begin = min_dt,
-        end = max_dt,
-        chunk_interval = chunk_interval,
-        debug = debug,
+        begin=min_dt,
+        end=max_dt,
+        chunk_interval=chunk_interval,
+        debug=debug,
     )
 
     for chunk_begin, chunk_end in chunk_bounds:
         chunk = self.get_data(
-            select_columns = select_columns,
-            omit_columns = omit_columns,
-            begin = chunk_begin,
-            end = chunk_end,
-            params = params,
-            fresh = fresh,
-            debug = debug,
+            select_columns=select_columns,
+            omit_columns=omit_columns,
+            begin=chunk_begin,
+            end=chunk_end,
+            params=params,
+            limit=limit,
+            order=order,
+            fresh=fresh,
+            debug=debug,
         )
         if len(chunk) > 0:
             yield chunk
 
 
 def get_backtrack_data(
-        self,
-        backtrack_minutes: Optional[int] = None,
-        begin: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        fresh: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Optional['pd.DataFrame']:
+    self,
+    backtrack_minutes: Optional[int] = None,
+    begin: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    limit: Optional[int] = None,
+    fresh: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Optional['pd.DataFrame']:
     """
     Get the most recent data from the instance connector as a Pandas DataFrame.
 
@@ -371,8 +427,10 @@ def get_backtrack_data(
 
     params: Optional[Dict[str, Any]], default None
         The standard Meerschaum `params` query dictionary.
-        
-        
+
+    limit: Optional[int], default None
+        If provided, cap the number of rows to be returned.
+
     fresh: bool, default False
         If `True`, Ignore local cache and pull directly from the instance connector.
         Only comes into effect if a pipe was created with `cache=True`.
@@ -409,28 +467,31 @@ def get_backtrack_data(
             else: ### Successfully synced cache.
                 return self.enforce_dtypes(
                     self.cache_pipe.get_backtrack_data(
-                        fresh = True,
-                        begin = begin,
-                        backtrack_minutes = backtrack_minutes,
-                        params = params,
-                        debug = deubg,
+                        fresh=True,
+                        begin=begin,
+                        backtrack_minutes=backtrack_minutes,
+                        params=params,
+                        limit=limit,
+                        order=kw.get('order', 'desc'),
+                        debug=debug,
                         **kw
                     ),
-                    debug = debug,
+                    debug=debug,
                 )
 
     if hasattr(self.instance_connector, 'get_backtrack_data'):
         with Venv(get_connector_plugin(self.instance_connector)):
             return self.enforce_dtypes(
                 self.instance_connector.get_backtrack_data(
-                    pipe = self,
-                    begin = begin,
-                    backtrack_minutes = backtrack_minutes,
-                    params = params,
-                    debug = debug,
+                    pipe=self,
+                    begin=begin,
+                    backtrack_minutes=backtrack_minutes,
+                    params=params,
+                    limit=limit,
+                    debug=debug,
                     **kw
                 ),
-                debug = debug,
+                debug=debug,
             )
 
     if begin is None:
@@ -445,13 +506,16 @@ def get_backtrack_data(
         begin = begin - backtrack_interval
 
     return self.get_data(
-        begin = begin,
-        params = params,
-        debug = debug,
+        begin=begin,
+        params=params,
+        debug=debug,
+        limit=limit,
+        order=kw.get('order', 'desc'),
         **kw
     )
 
 
+
 def get_rowcount(
         self,
         begin: Optional[datetime] = None,

meerschaum/core/Pipe/_drop.py

@@ -10,10 +10,10 @@ from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Any
 
 def drop(
-        self,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Call the Pipe's instance connector's `drop_pipe()` method.
 

meerschaum/core/Pipe/_dtypes.py

@@ -16,12 +16,12 @@ if TYPE_CHECKING:
     pd = mrsm.attempt_import('pandas')
 
 def enforce_dtypes(
-        self,
-        df: 'pd.DataFrame',
-        chunksize: Optional[int] = -1,
-        safe_copy: bool = True,
-        debug: bool = False,
-    ) -> 'pd.DataFrame':
+    self,
+    df: 'pd.DataFrame',
+    chunksize: Optional[int] = -1,
+    safe_copy: bool = True,
+    debug: bool = False,
+) -> 'pd.DataFrame':
     """
     Cast the input dataframe to the pipe's registered data types.
     If the pipe does not exist and dtypes are not set, return the dataframe.
@@ -35,7 +35,7 @@ def enforce_dtypes(
     if df is None:
         if debug:
             dprint(
-                f"Received None instead of a DataFrame.\n"
+                "Received None instead of a DataFrame.\n"
                 + "    Skipping dtype enforcement..."
             )
         return df
@@ -46,24 +46,24 @@ def enforce_dtypes(
         if isinstance(df, str):
             df = parse_df_datetimes(
                 pd.read_json(StringIO(df)),
-                ignore_cols = [
+                ignore_cols=[
                     col
                     for col, dtype in pipe_dtypes.items()
                     if 'datetime' not in str(dtype)
                 ],
-                chunksize = chunksize,
-                debug = debug,
+                chunksize=chunksize,
+                debug=debug,
             )
         else:
             df = parse_df_datetimes(
                 df,
-                ignore_cols = [
+                ignore_cols=[
                     col
                     for col, dtype in pipe_dtypes.items()
                     if 'datetime' not in str(dtype)
                 ],
-                chunksize = chunksize,
-                debug = debug,
+                chunksize=chunksize,
+                debug=debug,
             )
     except Exception as e:
         warn(f"Unable to cast incoming data as a DataFrame...:\n{e}\n\n{traceback.format_exc()}")
@@ -80,7 +80,7 @@ def enforce_dtypes(
     return _enforce_dtypes(df, pipe_dtypes, safe_copy=safe_copy, debug=debug)
 
 
-def infer_dtypes(self, persist: bool=False, debug: bool=False) -> Dict[str, Any]:
+def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str, Any]:
     """
     If `dtypes` is not set in `meerschaum.Pipe.parameters`,
     infer the data types from the underlying table if it exists.

meerschaum/core/Pipe/_edit.py

@@ -18,12 +18,12 @@ def update(self, *args, **kw) -> SuccessTuple:
 
 
 def edit(
-        self,
-        patch: bool = False,
-        interactive: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    patch: bool = False,
+    interactive: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Edit a Pipe's configuration.
 
@@ -50,11 +50,12 @@ def edit(
     if not interactive:
         with Venv(get_connector_plugin(self.instance_connector)):
             return self.instance_connector.edit_pipe(self, patch=patch, debug=debug, **kw)
+
     from meerschaum.config._paths import PIPES_CACHE_RESOURCES_PATH
     from meerschaum.utils.misc import edit_file
     parameters_filename = str(self) + '.yaml'
     parameters_path = PIPES_CACHE_RESOURCES_PATH / parameters_filename
-    
+
     from meerschaum.utils.yaml import yaml
 
     edit_text = f"Edit the parameters for {self}"
@@ -96,13 +97,13 @@ def edit(
 
 
 def edit_definition(
-        self,
-        yes: bool = False,
-        noask: bool = False,
-        force: bool = False,
-        debug : bool = False,
-        **kw : Any
-    ) -> SuccessTuple:
+    self,
+    yes: bool = False,
+    noask: bool = False,
+    force: bool = False,
+    debug : bool = False,
+    **kw : Any
+) -> SuccessTuple:
     """
     Edit a pipe's definition file and update its configuration.
     **NOTE:** This function is interactive and should not be used in automated scripts!