meerschaum 2.6.17__py3-none-any.whl → 2.7.0rc1__py3-none-any.whl

meerschaum/core/Pipe/__init__.py

@@ -106,6 +106,7 @@ class Pipe:
         upsert,
         static,
         tzinfo,
+        enforce,
         get_columns,
         get_columns_types,
         get_columns_indices,
@@ -132,6 +133,7 @@ class Pipe:
         _persist_new_json_columns,
         _persist_new_numeric_columns,
         _persist_new_uuid_columns,
+        _persist_new_bytes_columns,
     )
     from ._verify import (
         verify,
@@ -162,12 +164,14 @@ class Pipe:
         upsert: Optional[bool] = None,
         autoincrement: Optional[bool] = None,
         static: Optional[bool] = None,
+        enforce: Optional[bool] = None,
         mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
         cache: bool = False,
         debug: bool = False,
         connector_keys: Optional[str] = None,
         metric_key: Optional[str] = None,
         location_key: Optional[str] = None,
+        instance_keys: Optional[str] = None,
         indexes: Union[Dict[str, str], List[str], None] = None,
     ):
         """
@@ -219,6 +223,10 @@ class Pipe:
         static: Optional[bool], default None
             If `True`, set `static` in the parameters.
 
+        enforce: Optionanl[bool], default None
+            If `False`, skip data type enforcement.
+            Default behavior is `True`.
+
         temporary: bool, default False
             If `True`, prevent instance tables (pipes, users, plugins) from being created.
 
@@ -319,11 +327,13 @@ class Pipe:
         if isinstance(static, bool):
             self._attributes['parameters']['static'] = static
 
+        if isinstance(enforce, bool):
+            self._attributes['parameters']['enforce'] = enforce
+
         ### NOTE: The parameters dictionary is {} by default.
         ###       A Pipe may be registered without parameters, then edited,
         ###       or a Pipe may be registered with parameters set in-memory first.
-        #  from meerschaum.config import get_config
-        _mrsm_instance = mrsm_instance if mrsm_instance is not None else instance
+        _mrsm_instance = mrsm_instance if mrsm_instance is not None else (instance or instance_keys)
         if _mrsm_instance is None:
             _mrsm_instance = get_config('meerschaum', 'instance', patch=True)
 
@@ -341,10 +351,10 @@ class Pipe:
         Return the four keys needed to reconstruct this pipe.
         """
         return {
-            'connector': self.connector_keys,
-            'metric': self.metric_key,
-            'location': self.location_key,
-            'instance': self.instance_keys,
+            'connector_keys': self.connector_keys,
+            'metric_key': self.metric_key,
+            'location_key': self.location_key,
+            'instance_keys': self.instance_keys,
         }
 
     def keys(self) -> List[str]:
@@ -385,7 +395,7 @@ class Pipe:
                 warnings.simplefilter('ignore')
                 try:
                     conn = parse_instance_keys(self.connector_keys)
-                except Exception as e:
+                except Exception:
                     conn = None
             if conn:
                 self._connector = conn
@@ -429,7 +439,7 @@ class Pipe:
             _fetch_patch = {
                 'fetch': ({
                     'definition': (
-                        f"SELECT * FROM "
+                        "SELECT * FROM "
                         + sql_item_name(
                             str(self.target),
                             self.instance_connector.flavor,
@@ -467,7 +477,7 @@ class Pipe:
                 and self.location_key == other.location_key
                 and self.instance_keys == other.instance_keys
             )
-        except Exception as e:
+        except Exception:
             return False
 
     def __hash__(self):
@@ -496,11 +506,11 @@ class Pipe:
         Define the state dictionary (pickling).
         """
         return {
-            'connector': self.connector_keys,
-            'metric': self.metric_key,
-            'location': self.location_key,
+            'connector_keys': self.connector_keys,
+            'metric_key': self.metric_key,
+            'location_key': self.location_key,
             'parameters': self.parameters,
-            'instance': self.instance_keys,
+            'instance_keys': self.instance_keys,
         }
 
     def __setstate__(self, _state: Dict[str, Any]):

meerschaum/core/Pipe/_attributes.py

@@ -289,6 +289,25 @@ def tzinfo(self) -> Union[None, timezone]:
     return None
 
 
+@property
+def enforce(self) -> bool:
+    """
+    Return the `enforce` parameter for the pipe.
+    """
+    if 'enforce' not in self.parameters:
+        self.parameters['enforce'] = True
+
+    return self.parameters['enforce']
+
+
+@enforce.setter
+def enforce(self, _enforce: bool) -> None:
+    """
+    Set the `enforce` parameter for the pipe.
+    """
+    self.parameters['_enforce'] = _enforce
+
+
 def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
     """
     Check if the requested columns are defined.

meerschaum/core/Pipe/_dtypes.py

@@ -41,7 +41,7 @@ def enforce_dtypes(
             )
         return df
 
-    pipe_dtypes = self.dtypes
+    pipe_dtypes = self.dtypes if self.enforce else {}
 
     try:
         if isinstance(df, str):

meerschaum/core/Pipe/_sync.py

@@ -368,10 +368,11 @@ def sync(
         ### Cast to a dataframe and ensure datatypes are what we expect.
         df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
 
-        ### Capture `numeric`, `uuid`, and `json` columns.
+        ### Capture `numeric`, `uuid`, `json`, and `bytes` columns.
         self._persist_new_json_columns(df, debug=debug)
         self._persist_new_numeric_columns(df, debug=debug)
         self._persist_new_uuid_columns(df, debug=debug)
+        self._persist_new_bytes_columns(df, debug=debug)
 
         if debug:
             dprint(
@@ -617,11 +618,13 @@ def filter_existing(
         filter_unseen_df,
         add_missing_cols_to_df,
         get_unhashable_cols,
-        get_numeric_cols,
     )
     from meerschaum.utils.dtypes import (
         to_pandas_dtype,
         none_if_null,
+        to_datetime,
+        are_dtypes_equal,
+        value_is_null,
     )
     from meerschaum.config import get_config
     pd = import_pandas()
@@ -669,29 +672,36 @@ def filter_existing(
     ### begin is the oldest data in the new dataframe
     begin, end = None, None
     dt_col = pipe_columns.get('datetime', None)
+    primary_key = pipe_columns.get('primary', None)
     dt_type = self.dtypes.get(dt_col, 'datetime64[ns, UTC]') if dt_col else None
+
+    if autoincrement and primary_key == dt_col and dt_col not in df.columns:
+        if enforce_dtypes:
+            df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+        return df, get_empty_df(), df
+
     try:
-        min_dt_val = df[dt_col].min(skipna=True) if dt_col else None
+        min_dt_val = df[dt_col].min(skipna=True) if dt_col and dt_col in df.columns else None
         if is_dask and min_dt_val is not None:
             min_dt_val = min_dt_val.compute()
         min_dt = (
-            pandas.to_datetime(min_dt_val).to_pydatetime()
-            if min_dt_val is not None and 'datetime' in str(dt_type)
+            to_datetime(min_dt_val, as_pydatetime=True)
+            if min_dt_val is not None and are_dtypes_equal(dt_type, 'datetime')
             else min_dt_val
         )
     except Exception:
         min_dt = None
-    if not ('datetime' in str(type(min_dt))) or str(min_dt) == 'NaT':
-        if 'int' not in str(type(min_dt)).lower():
+
+    if not are_dtypes_equal('datetime', str(type(min_dt))) or value_is_null(min_dt):
+        if not are_dtypes_equal('int', str(type(min_dt))):
             min_dt = None
 
     if isinstance(min_dt, datetime):
-        begin = (
-            round_time(
-                min_dt,
-                to='down'
-            ) - timedelta(minutes=1)
-        )
+        rounded_min_dt = round_time(min_dt, to='down')
+        try:
+            begin = rounded_min_dt - timedelta(minutes=1)
+        except OverflowError:
+            begin = rounded_min_dt
     elif dt_type and 'int' in dt_type.lower():
         begin = min_dt
     elif dt_col is None:
@@ -699,11 +709,11 @@ def filter_existing(
 
     ### end is the newest data in the new dataframe
     try:
-        max_dt_val = df[dt_col].max(skipna=True) if dt_col else None
+        max_dt_val = df[dt_col].max(skipna=True) if dt_col and dt_col in df.columns else None
         if is_dask and max_dt_val is not None:
             max_dt_val = max_dt_val.compute()
         max_dt = (
-            pandas.to_datetime(max_dt_val).to_pydatetime()
+            to_datetime(max_dt_val, as_pydatetime=True)
             if max_dt_val is not None and 'datetime' in str(dt_type)
             else max_dt_val
         )
@@ -712,8 +722,8 @@ def filter_existing(
         traceback.print_exc()
         max_dt = None
 
-    if ('datetime' not in str(type(max_dt))) or str(min_dt) == 'NaT':
-        if 'int' not in str(type(max_dt)).lower():
+    if not are_dtypes_equal('datetime', str(type(max_dt))) or value_is_null(max_dt):
+        if not are_dtypes_equal('int', str(type(max_dt))):
             max_dt = None
 
     if isinstance(max_dt, datetime):
@@ -723,7 +733,7 @@ def filter_existing(
                 to='down'
             ) + timedelta(minutes=1)
         )
-    elif dt_type and 'int' in dt_type.lower():
+    elif dt_type and 'int' in dt_type.lower() and max_dt is not None:
         end = max_dt + 1
 
     if max_dt is not None and min_dt is not None and min_dt > max_dt:
@@ -738,7 +748,7 @@ def filter_existing(
 
     unique_index_vals = {
         col: df[col].unique()
-        for col in pipe_columns
+        for col in (pipe_columns if not primary_key else [primary_key])
         if col in df.columns and col != dt_col
     } if not date_bound_only else {}
     filter_params_index_limit = get_config('pipes', 'sync', 'filter_params_index_limit')
@@ -777,14 +787,15 @@ def filter_existing(
 
     ### Separate new rows from changed ones.
     on_cols = [
-        col for col_key, col in pipe_columns.items()
+        col
+        for col_key, col in pipe_columns.items()
         if (
             col
             and
             col_key != 'value'
             and col in backtrack_df.columns
         )
-    ]
+    ] if not primary_key else [primary_key]
     self_dtypes = self.dtypes
     on_cols_dtypes = {
         col: to_pandas_dtype(typ)
@@ -1020,3 +1031,32 @@ def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
         return edit_success, edit_msg
 
     return True, "Success"
+
+
+def _persist_new_bytes_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new `bytes` columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_bytes_cols
+    bytes_cols = get_bytes_cols(df)
+    existing_bytes_cols = [col for col, typ in self.dtypes.items() if typ == 'bytes']
+    new_bytes_cols = [col for col in bytes_cols if col not in existing_bytes_cols]
+    if not new_bytes_cols:
+        return True, "Success"
+
+    self._attributes_sync_time = None
+    dt_col = self.columns.get('datetime', None)
+    dtypes = self.parameters.get('dtypes', {})
+    if dt_col not in dtypes:
+        dtypes[dt_col] = 'datetime'
+    dtypes.update({col: 'bytes' for col in bytes_cols})
+    self.parameters['dtypes'] = dtypes
+
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update bytes dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"

meerschaum/core/Pipe/_verify.py

@@ -7,9 +7,10 @@ Verify the contents of a pipe by resyncing its interval.
 """
 
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import SuccessTuple, Any, Optional, Union, Tuple, List, Dict
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, Union, Tuple, Dict
 from meerschaum.utils.warnings import warn, info
-from meerschaum.utils.debug import dprint
 
 
 def verify(
@@ -94,9 +95,6 @@ def verify(
             else 1
         )
 
-    sync_less_than_begin = not bounded and begin is None
-    sync_greater_than_end = not bounded and end is None
-
     cannot_determine_bounds = not self.exists(debug=debug)
 
     if cannot_determine_bounds:
@@ -164,7 +162,7 @@ def verify(
     )
 
     info(
-        f"Syncing {len(chunk_bounds)} chunk" + ('s' if len(chunk_bounds) != 1 else '')
+        f"Verifying {self}:\n    Syncing {len(chunk_bounds)} chunk" + ('s' if len(chunk_bounds) != 1 else '')
         + f" ({'un' if not bounded else ''}bounded)"
         + f" of size '{interval_str(chunk_interval)}'"
         + f" between '{begin_to_print}' and '{end_to_print}'."
@@ -187,7 +185,7 @@ def verify(
             return chunk_begin_and_end, bounds_success_tuples[chunk_begin_and_end]
 
         chunk_begin, chunk_end = chunk_begin_and_end
-        return chunk_begin_and_end, self.sync(
+        chunk_success, chunk_msg = self.sync(
             begin=chunk_begin,
             end=chunk_end,
             params=params,
@@ -195,6 +193,9 @@ def verify(
             debug=debug,
             **kwargs
         )
+        chunk_msg = chunk_msg.strip()
+        mrsm.pprint((chunk_success, chunk_msg))
+        return chunk_begin_and_end, (chunk_success, chunk_msg)
 
     ### If we have more than one chunk, attempt to sync the first one and return if its fails.
     if len(chunk_bounds) > 1:

meerschaum/plugins/_Plugin.py

@@ -255,11 +255,11 @@ class Plugin:
 
 
     def install(
-            self,
-            skip_deps: bool = False,
-            force: bool = False,
-            debug: bool = False,
-        ) -> SuccessTuple:
+        self,
+        skip_deps: bool = False,
+        force: bool = False,
+        debug: bool = False,
+    ) -> SuccessTuple:
         """
         Extract a plugin's tar archive to the plugins directory.
         
@@ -359,7 +359,7 @@ class Plugin:
             is_same_version = new_version and old_version and (
                 packaging_version.parse(old_version) == packaging_version.parse(new_version)
             )
-        except Exception as e:
+        except Exception:
             is_new_version, is_same_version = True, False
 
         ### Determine where to permanently store the new plugin.
@@ -404,7 +404,7 @@ class Plugin:
                         dprint(f"Moving '{src_file}' to '{dst_dir}'...")
                     try:
                         shutil.move(src_file, dst_dir)
-                    except Exception as e:
+                    except Exception:
                         success, msg = False, (
                             f"Failed to install plugin '{self}': " +
                             f"Could not move file '{src_file}' to '{dst_dir}'"
@@ -817,10 +817,10 @@ class Plugin:
 
 
     def install_dependencies(
-            self,
-            force: bool = False,
-            debug: bool = False,
-        ) -> bool:
+        self,
+        force: bool = False,
+        debug: bool = False,
+    ) -> bool:
         """
         If specified, install dependencies.
         
@@ -841,12 +841,9 @@ class Plugin:
         Returns
         -------
         A bool indicating success.
-
         """
         from meerschaum.utils.packages import pip_install, venv_contains_package
-        from meerschaum.utils.debug import dprint
         from meerschaum.utils.warnings import warn, info
-        from meerschaum.connectors.parse import parse_repo_keys
         _deps = self.get_dependencies(debug=debug)
         if not _deps and self.requirements_file_path is None:
             return True

meerschaum/utils/daemon/Daemon.py

@@ -465,6 +465,7 @@ class Daemon:
             self._write_stop_file('kill')
             return True, "Process has already stopped."
 
+        psutil = attempt_import('psutil')
         process = self.process
         try:
             process.terminate()
@@ -473,10 +474,16 @@ class Daemon:
         except Exception as e:
             return False, f"Failed to kill job {self} with exception: {e}"
 
+        try:
+            if process.status():
+                return False, "Failed to stop daemon '{self}' ({process})."
+        except psutil.NoSuchProcess:
+            pass
+
         if self.pid_path.exists():
             try:
                 self.pid_path.unlink()
-            except Exception as e:
+            except Exception:
                 pass
 
         self._write_stop_file('kill')
@@ -534,7 +541,7 @@ class Daemon:
         if not timeout:
             try:
                 success = self.process.status() == 'stopped'
-            except psutil.NoSuchProcess as e:
+            except psutil.NoSuchProcess:
                 success = True
             msg = "Success" if success else f"Failed to suspend daemon '{self.daemon_id}'."
             if success:
@@ -677,11 +684,11 @@ class Daemon:
         raise SystemExit(0)
 
     def _send_signal(
-            self,
-            signal_to_send,
-            timeout: Union[float, int, None] = None,
-            check_timeout_interval: Union[float, int, None] = None,
-        ) -> SuccessTuple:
+        self,
+        signal_to_send,
+        timeout: Union[float, int, None] = None,
+        check_timeout_interval: Union[float, int, None] = None,
+    ) -> SuccessTuple:
         """Send a signal to the daemon process.
 
         Parameters
@@ -709,7 +716,7 @@ class Daemon:
                 )
             
             os.kill(pid, signal_to_send)
-        except Exception as e:
+        except Exception:
             return False, f"Failed to send signal {signal_to_send}:\n{traceback.format_exc()}"
 
         timeout = self.get_timeout_seconds(timeout)
@@ -745,7 +752,7 @@ class Daemon:
         if _already_exists and not allow_dirty_run:
             error(
                 f"Daemon '{self.daemon_id}' already exists. " +
-                f"To allow this daemon to run, do one of the following:\n"
+                "To allow this daemon to run, do one of the following:\n"
                 + "  - Execute `daemon.cleanup()`.\n"
                 + f"  - Delete the directory '{self.path}'.\n"
                 + "  - Pass `allow_dirty_run=True` to `daemon.run()`.\n",
@@ -764,7 +771,7 @@ class Daemon:
         if '_process' not in self.__dict__ or self.__dict__['_process'].pid != int(pid):
             try:
                 self._process = psutil.Process(int(pid))
-            except Exception as e:
+            except Exception:
                 if self.pid_path.exists():
                     self.pid_path.unlink()
                 return None
@@ -788,7 +795,7 @@ class Daemon:
             if self.pid_path.exists():
                 try:
                     self.pid_path.unlink()
-                except Exception as e:
+                except Exception:
                     pass
             return 'stopped'