meerschaum 2.3.6__py3-none-any.whl → 2.4.0.dev1__py3-none-any.whl

meerschaum/core/Pipe/_copy.py

@@ -0,0 +1,106 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define methods for copying pipes.
+"""
+
+from datetime import datetime, timedelta
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Union
+
+
+def copy_to(
+    self,
+    instance_keys: str,
+    sync: bool = True,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
+    """
+    Copy a pipe to another instance.
+
+    Parameters
+    ----------
+    instance_keys: str
+        The instance to which to copy this pipe.
+
+    sync: bool, default True
+        If `True`, sync the source pipe's documents 
+
+    begin: Union[datetime, int, None], default None
+        Beginning datetime value to pass to `Pipe.get_data()`.
+
+    end: Union[datetime, int, None], default None
+        End datetime value to pass to `Pipe.get_data()`.
+
+    params: Optional[Dict[str, Any]], default None
+        Parameters filter to pass to `Pipe.get_data()`.
+
+    chunk_interval: Union[timedelta, int, None], default None
+        The size of chunks to retrieve from `Pipe.get_data()` for syncing.
+
+    kwargs: Any
+        Additional flags to pass to `Pipe.get_data()` and `Pipe.sync()`, e.g. `workers`.
+
+    Returns
+    -------
+    A SuccessTuple indicating success.
+    """
+    if str(instance_keys) == self.instance_keys:
+        return False, f"Cannot copy {self} to instance '{instance_keys}'."
+
+    new_pipe = mrsm.Pipe(
+        self.connector_keys,
+        self.metric_key,
+        self.location_key,
+        parameters=self.parameters.copy(),
+        instance=instance_keys,
+    )
+
+    new_pipe_is_registered = new_pipe.get_id() is not None
+
+    metadata_method = new_pipe.edit if new_pipe_is_registered else new_pipe.register
+    metadata_success, metadata_msg = metadata_method(debug=debug)
+    if not metadata_success:
+        return metadata_success, metadata_msg
+
+    if not self.exists(debug=debug):
+        return True, f"{self} does not exist; nothing to sync."
+
+    original_as_iterator = kwargs.get('as_iterator', None)
+    kwargs['as_iterator'] = True
+
+    chunk_generator = self.get_data(
+        begin=begin,
+        end=end,
+        params=params,
+        chunk_interval=chunk_interval,
+        debug=debug,
+        **kwargs
+    )
+
+    if original_as_iterator is None:
+        _ = kwargs.pop('as_iterator', None)
+    else:
+        kwargs['as_iterator'] = original_as_iterator
+
+    sync_success, sync_msg = new_pipe.sync(
+        chunk_generator,
+        begin=begin,
+        end=end,
+        params=params,
+        debug=debug,
+        **kwargs
+    )
+    msg = (
+        f"Successfully synced {new_pipe}:\n{sync_msg}"
+        if sync_success
+        else f"Failed to sync {new_pipe}:\n{sync_msg}"
+    )
+    return sync_success, msg

meerschaum/core/Pipe/_drop.py

@@ -10,10 +10,10 @@ from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Any
 
 def drop(
-        self,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Call the Pipe's instance connector's `drop_pipe()` method.
 

meerschaum/core/Pipe/_dtypes.py

@@ -16,12 +16,12 @@ if TYPE_CHECKING:
     pd = mrsm.attempt_import('pandas')
 
 def enforce_dtypes(
-        self,
-        df: 'pd.DataFrame',
-        chunksize: Optional[int] = -1,
-        safe_copy: bool = True,
-        debug: bool = False,
-    ) -> 'pd.DataFrame':
+    self,
+    df: 'pd.DataFrame',
+    chunksize: Optional[int] = -1,
+    safe_copy: bool = True,
+    debug: bool = False,
+) -> 'pd.DataFrame':
     """
     Cast the input dataframe to the pipe's registered data types.
     If the pipe does not exist and dtypes are not set, return the dataframe.
@@ -35,7 +35,7 @@ def enforce_dtypes(
     if df is None:
         if debug:
             dprint(
-                f"Received None instead of a DataFrame.\n"
+                "Received None instead of a DataFrame.\n"
                 + "    Skipping dtype enforcement..."
             )
         return df
@@ -46,24 +46,24 @@ def enforce_dtypes(
         if isinstance(df, str):
             df = parse_df_datetimes(
                 pd.read_json(StringIO(df)),
-                ignore_cols = [
+                ignore_cols=[
                     col
                     for col, dtype in pipe_dtypes.items()
                     if 'datetime' not in str(dtype)
                 ],
-                chunksize = chunksize,
-                debug = debug,
+                chunksize=chunksize,
+                debug=debug,
             )
         else:
             df = parse_df_datetimes(
                 df,
-                ignore_cols = [
+                ignore_cols=[
                     col
                     for col, dtype in pipe_dtypes.items()
                     if 'datetime' not in str(dtype)
                 ],
-                chunksize = chunksize,
-                debug = debug,
+                chunksize=chunksize,
+                debug=debug,
             )
     except Exception as e:
         warn(f"Unable to cast incoming data as a DataFrame...:\n{e}\n\n{traceback.format_exc()}")
@@ -80,7 +80,7 @@ def enforce_dtypes(
     return _enforce_dtypes(df, pipe_dtypes, safe_copy=safe_copy, debug=debug)
 
 
-def infer_dtypes(self, persist: bool=False, debug: bool=False) -> Dict[str, Any]:
+def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str, Any]:
     """
     If `dtypes` is not set in `meerschaum.Pipe.parameters`,
     infer the data types from the underlying table if it exists.

meerschaum/core/Pipe/_edit.py

@@ -18,12 +18,12 @@ def update(self, *args, **kw) -> SuccessTuple:
 
 
 def edit(
-        self,
-        patch: bool = False,
-        interactive: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    patch: bool = False,
+    interactive: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Edit a Pipe's configuration.
 
@@ -50,11 +50,12 @@ def edit(
     if not interactive:
         with Venv(get_connector_plugin(self.instance_connector)):
             return self.instance_connector.edit_pipe(self, patch=patch, debug=debug, **kw)
+
     from meerschaum.config._paths import PIPES_CACHE_RESOURCES_PATH
     from meerschaum.utils.misc import edit_file
     parameters_filename = str(self) + '.yaml'
     parameters_path = PIPES_CACHE_RESOURCES_PATH / parameters_filename
-    
+
     from meerschaum.utils.yaml import yaml
 
     edit_text = f"Edit the parameters for {self}"
@@ -96,13 +97,13 @@ def edit(
 
 
 def edit_definition(
-        self,
-        yes: bool = False,
-        noask: bool = False,
-        force: bool = False,
-        debug : bool = False,
-        **kw : Any
-    ) -> SuccessTuple:
+    self,
+    yes: bool = False,
+    noask: bool = False,
+    force: bool = False,
+    debug : bool = False,
+    **kw : Any
+) -> SuccessTuple:
     """
     Edit a pipe's definition file and update its configuration.
     **NOTE:** This function is interactive and should not be used in automated scripts!

meerschaum/core/Pipe/_sync.py

@@ -266,7 +266,6 @@ def sync(
                             **kw
                         )
                     )
-
             except Exception as e:
                 get_console().print_exception(
                     suppress=[
@@ -369,6 +368,11 @@ def sync(
 
         ### Cast to a dataframe and ensure datatypes are what we expect.
         df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+
+        ### Capture `numeric` and `json` columns.
+        self._persist_new_json_columns(df, debug=debug)
+        self._persist_new_numeric_columns(df, debug=debug)
+
         if debug:
             dprint(
                 "DataFrame to sync:\n"
@@ -554,14 +558,15 @@ def exists(
 
 
 def filter_existing(
-        self,
-        df: 'pd.DataFrame',
-        safe_copy: bool = True,
-        date_bound_only: bool = False,
-        chunksize: Optional[int] = -1,
-        debug: bool = False,
-        **kw
-    ) -> Tuple['pd.DataFrame', 'pd.DataFrame', 'pd.DataFrame']:
+    self,
+    df: 'pd.DataFrame',
+    safe_copy: bool = True,
+    date_bound_only: bool = False,
+    include_unchanged_columns: bool = False,
+    chunksize: Optional[int] = -1,
+    debug: bool = False,
+    **kw
+) -> Tuple['pd.DataFrame', 'pd.DataFrame', 'pd.DataFrame']:
     """
     Inspect a dataframe and filter out rows which already exist in the pipe.
 
@@ -569,7 +574,7 @@ def filter_existing(
     ----------
     df: 'pd.DataFrame'
         The dataframe to inspect and filter.
-        
+
     safe_copy: bool, default True
         If `True`, create a copy before comparing and modifying the dataframes.
         Setting to `False` may mutate the DataFrames.
@@ -578,6 +583,10 @@ def filter_existing(
     date_bound_only: bool, default False
         If `True`, only use the datetime index to fetch the sample dataframe.
 
+    include_unchanged_columns: bool, default False
+        If `True`, include the backtrack columns which haven't changed in the update dataframe.
+        This is useful if you can't update individual keys.
+
     chunksize: Optional[int], default -1
         The `chunksize` used when fetching existing data.
 
@@ -605,7 +614,7 @@ def filter_existing(
     from meerschaum.config import get_config
     pd = import_pandas()
     pandas = attempt_import('pandas')
-    if not 'dataframe' in str(type(df)).lower():
+    if 'dataframe' not in str(type(df)).lower():
         df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
     is_dask = 'dask' in df.__module__
     if is_dask:
@@ -615,8 +624,21 @@ def filter_existing(
     else:
         merge = pd.merge
         NA = pd.NA
+
+    def get_empty_df():
+        empty_df = pd.DataFrame([])
+        dtypes = dict(df.dtypes) if df is not None else {}
+        dtypes.update(self.dtypes)
+        pd_dtypes = {
+            col: to_pandas_dtype(str(typ))
+            for col, typ in dtypes.items()
+        }
+        return add_missing_cols_to_df(empty_df, pd_dtypes)
+
     if df is None:
-        return df, df, df
+        empty_df = get_empty_df()
+        return empty_df, empty_df, empty_df
+
     if (df.empty if not is_dask else len(df) == 0):
         return df, df, df
 
@@ -633,7 +655,7 @@ def filter_existing(
             if min_dt_val is not None and 'datetime' in str(dt_type)
             else min_dt_val
         )
-    except Exception as e:
+    except Exception:
         min_dt = None
     if not ('datetime' in str(type(min_dt))) or str(min_dt) == 'NaT':
         if 'int' not in str(type(min_dt)).lower():
@@ -643,7 +665,7 @@ def filter_existing(
         begin = (
             round_time(
                 min_dt,
-                to = 'down'
+                to='down'
             ) - timedelta(minutes=1)
         )
     elif dt_type and 'int' in dt_type.lower():
@@ -661,7 +683,7 @@ def filter_existing(
             if max_dt_val is not None and 'datetime' in str(dt_type)
             else max_dt_val
         )
-    except Exception as e:
+    except Exception:
         import traceback
         traceback.print_exc()
         max_dt = None
@@ -674,14 +696,14 @@ def filter_existing(
         end = (
             round_time(
                 max_dt,
-                to = 'down'
+                to='down'
             ) + timedelta(minutes=1)
         )
     elif dt_type and 'int' in dt_type.lower():
         end = max_dt + 1
 
     if max_dt is not None and min_dt is not None and min_dt > max_dt:
-        warn(f"Detected minimum datetime greater than maximum datetime.")
+        warn("Detected minimum datetime greater than maximum datetime.")
 
     if begin is not None and end is not None and begin > end:
         if isinstance(begin, datetime):
@@ -710,13 +732,18 @@ def filter_existing(
         dprint(f"Looking at data between '{begin}' and '{end}':", **kw)
 
     backtrack_df = self.get_data(
-        begin = begin,
-        end = end,
-        chunksize = chunksize,
-        params = params,
-        debug = debug,
+        begin=begin,
+        end=end,
+        chunksize=chunksize,
+        params=params,
+        debug=debug,
         **kw
     )
+    if backtrack_df is None:
+        if debug:
+            dprint(f"No backtrack data was found for {self}.")
+        return df, get_empty_df(), df
+
     if debug:
         dprint(f"Existing data for {self}:\n" + str(backtrack_df), **kw)
         dprint(f"Existing dtypes for {self}:\n" + str(backtrack_df.dtypes))
@@ -743,18 +770,19 @@ def filter_existing(
         filter_unseen_df(
             backtrack_df,
             df,
-            dtypes = {
+            dtypes={
                 col: to_pandas_dtype(typ)
                 for col, typ in self_dtypes.items()
             },
-            safe_copy = safe_copy,
-            debug = debug
+            safe_copy=safe_copy,
+            debug=debug
         ),
         on_cols_dtypes,
     )
 
     ### Cast dicts or lists to strings so we can merge.
     serializer = functools.partial(json.dumps, sort_keys=True, separators=(',', ':'), default=str)
+
     def deserializer(x):
         return json.loads(x) if isinstance(x, str) else x
 
@@ -767,12 +795,12 @@ def filter_existing(
     casted_cols = set(unhashable_delta_cols + unhashable_backtrack_cols)
 
     joined_df = merge(
-        delta_df.fillna(NA),
-        backtrack_df.fillna(NA),
-        how = 'left',
-        on = on_cols,
-        indicator = True,
-        suffixes = ('', '_old'),
+        delta_df.infer_objects(copy=False).fillna(NA),
+        backtrack_df.infer_objects(copy=False).fillna(NA),
+        how='left',
+        on=on_cols,
+        indicator=True,
+        suffixes=('', '_old'),
     ) if on_cols else delta_df
     for col in casted_cols:
         if col in joined_df.columns:
@@ -782,20 +810,13 @@ def filter_existing(
 
     ### Determine which rows are completely new.
     new_rows_mask = (joined_df['_merge'] == 'left_only') if on_cols else None
-    cols = list(backtrack_df.columns)
+    cols = list(delta_df.columns)
 
     unseen_df = (
-        (
-            joined_df
-            .where(new_rows_mask)
-            .dropna(how='all')[cols]
-            .reset_index(drop=True)
-        ) if not is_dask else (
-            joined_df
-            .where(new_rows_mask)
-            .dropna(how='all')[cols]
-            .reset_index(drop=True)
-        )
+        joined_df
+        .where(new_rows_mask)
+        .dropna(how='all')[cols]
+        .reset_index(drop=True)
     ) if on_cols else delta_df
 
     ### Rows that have already been inserted but values have changed.
@@ -804,20 +825,33 @@ def filter_existing(
         .where(~new_rows_mask)
         .dropna(how='all')[cols]
         .reset_index(drop=True)
-    ) if on_cols else None
+    ) if on_cols else get_empty_df()
+
+    if include_unchanged_columns and on_cols:
+        unchanged_backtrack_cols = [
+            col
+            for col in backtrack_df.columns
+            if col in on_cols or col not in update_df.columns
+        ]
+        update_df = merge(
+            backtrack_df[unchanged_backtrack_cols],
+            update_df,
+            how='inner',
+            on=on_cols,
+        )
 
     return unseen_df, update_df, delta_df
 
 
 @staticmethod
 def _get_chunk_label(
-        chunk: Union[
-            'pd.DataFrame',
-            List[Dict[str, Any]],
-            Dict[str, List[Any]]
-        ],
-        dt_col: str,
-    ) -> str:
+    chunk: Union[
+        'pd.DataFrame',
+        List[Dict[str, Any]],
+        Dict[str, List[Any]]
+    ],
+    dt_col: str,
+) -> str:
     """
     Return the min - max label for the chunk.
     """
@@ -870,3 +904,52 @@ def get_num_workers(self, workers: Optional[int] = None) -> int:
         (desired_workers - current_num_connections),
         1,
     )
+
+
+def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new numeric columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_numeric_cols
+    numeric_cols = get_numeric_cols(df)
+    existing_numeric_cols = [col for col, typ in self.dtypes.items() if typ == 'numeric']
+    new_numeric_cols = [col for col in numeric_cols if col not in existing_numeric_cols]
+    if not new_numeric_cols:
+        return True, "Success"
+
+    dtypes = self.parameters.get('dtypes', {})
+    dtypes.update({col: 'numeric' for col in numeric_cols})
+    self.parameters['dtypes'] = dtypes
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update NUMERIC dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"
+
+
+def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new JSON columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_json_cols
+    json_cols = get_json_cols(df)
+    existing_json_cols = [col for col, typ in self.dtypes.items() if typ == 'json']
+    new_json_cols = [col for col in json_cols if col not in existing_json_cols]
+    if not new_json_cols:
+        return True, "Success"
+
+    dtypes = self.parameters.get('dtypes', {})
+    dtypes.update({col: 'json' for col in json_cols})
+    self.parameters['dtypes'] = dtypes
+
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update JSON dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"

meerschaum/core/Pipe/_verify.py

@@ -12,17 +12,17 @@ from meerschaum.utils.warnings import warn, info
 from meerschaum.utils.debug import dprint
 
 def verify(
-        self,
-        begin: Union[datetime, int, None] = None,
-        end: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        bounded: Optional[bool] = None,
-        deduplicate: bool = False,
-        workers: Optional[int] = None,
-        debug: bool = False,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    self,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    bounded: Optional[bool] = None,
+    deduplicate: bool = False,
+    workers: Optional[int] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Verify the contents of the pipe by resyncing its interval.
 

meerschaum/core/User/_User.py

@@ -11,7 +11,7 @@ import os
 import hashlib
 import hmac
 from binascii import b2a_base64, a2b_base64, Error as _BinAsciiError
-from meerschaum.utils.typing import Optional, Dict, Any, Tuple
+from meerschaum.utils.typing import Optional, Dict, Any, Union
 from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.warnings import warn
 
@@ -19,10 +19,10 @@ from meerschaum.utils.warnings import warn
 __all__ = ('hash_password', 'verify_password', 'User')
 
 def hash_password(
-        password: str,
-        salt: Optional[bytes] = None,
-        rounds: Optional[int] = None,
-    ) -> str:
+    password: str,
+    salt: Optional[bytes] = None,
+    rounds: Optional[int] = None,
+) -> str:
     """
     Return an encoded hash string from the given password.
 
@@ -68,9 +68,9 @@ def hash_password(
 
 
 def verify_password(
-        password: str,
-        password_hash: str,
-    ) -> bool:
+    password: str,
+    password_hash: str,
+) -> bool:
     """
     Return `True` if the password matches the provided hash.
 
@@ -197,26 +197,28 @@ class User:
         return self._attributes
 
     @property
-    def instance_connector(self) -> meerschaum.connectors.Connector:
-        """ """
+    def instance_connector(self) -> 'mrsm.connectors.Connector':
         from meerschaum.connectors.parse import parse_instance_keys
         if '_instance_connector' not in self.__dict__:
             self._instance_connector = parse_instance_keys(self._instance_keys)
         return self._instance_connector
 
     @property
-    def user_id(self) -> int:
+    def user_id(self) -> Union[int, str, None]:
         """NOTE: This causes recursion with the API,
               so don't try to get fancy with read-only attributes.
         """
         return self._user_id
 
     @user_id.setter
-    def user_id(self, user_id):
+    def user_id(self, user_id: Union[int, str, None]):
         self._user_id = user_id
 
     @property
     def password_hash(self):
+        """
+        Return the hash of the user's password.
+        """
         _password_hash = self.__dict__.get('_password_hash', None)
         if _password_hash is not None:
             return _password_hash