meerschaum 2.6.16__py3-none-any.whl → 2.7.0__py3-none-any.whl

meerschaum/connectors/valkey/_pipes.py

@@ -46,9 +46,20 @@ def serialize_document(doc: Dict[str, Any]) -> str:
     -------
     A serialized string for the document.
     """
+    from meerschaum.utils.dtypes import serialize_bytes
     return json.dumps(
         doc,
-        default=(lambda x: json_serialize_datetime(x) if hasattr(x, 'tzinfo') else str(x)),
+        default=(
+            lambda x: (
+                json_serialize_datetime(x)
+                if hasattr(x, 'tzinfo')
+                else (
+                    serialize_bytes(x)
+                    if isinstance(x, bytes)
+                    else str(x)
+                )
+            )
+        ),
         separators=(',', ':'),
         sort_keys=True,
     )

meerschaum/core/Pipe/__init__.py

@@ -106,6 +106,7 @@ class Pipe:
         upsert,
         static,
         tzinfo,
+        enforce,
         get_columns,
         get_columns_types,
         get_columns_indices,
@@ -132,6 +133,7 @@ class Pipe:
         _persist_new_json_columns,
         _persist_new_numeric_columns,
         _persist_new_uuid_columns,
+        _persist_new_bytes_columns,
     )
     from ._verify import (
         verify,
@@ -162,12 +164,14 @@ class Pipe:
         upsert: Optional[bool] = None,
         autoincrement: Optional[bool] = None,
         static: Optional[bool] = None,
+        enforce: Optional[bool] = None,
         mrsm_instance: Optional[Union[str, InstanceConnector]] = None,
         cache: bool = False,
         debug: bool = False,
         connector_keys: Optional[str] = None,
         metric_key: Optional[str] = None,
         location_key: Optional[str] = None,
+        instance_keys: Optional[str] = None,
         indexes: Union[Dict[str, str], List[str], None] = None,
     ):
         """
@@ -219,6 +223,10 @@ class Pipe:
         static: Optional[bool], default None
             If `True`, set `static` in the parameters.
 
+        enforce: Optionanl[bool], default None
+            If `False`, skip data type enforcement.
+            Default behavior is `True`.
+
         temporary: bool, default False
             If `True`, prevent instance tables (pipes, users, plugins) from being created.
 
@@ -319,11 +327,13 @@ class Pipe:
         if isinstance(static, bool):
             self._attributes['parameters']['static'] = static
 
+        if isinstance(enforce, bool):
+            self._attributes['parameters']['enforce'] = enforce
+
         ### NOTE: The parameters dictionary is {} by default.
         ###       A Pipe may be registered without parameters, then edited,
         ###       or a Pipe may be registered with parameters set in-memory first.
-        #  from meerschaum.config import get_config
-        _mrsm_instance = mrsm_instance if mrsm_instance is not None else instance
+        _mrsm_instance = mrsm_instance if mrsm_instance is not None else (instance or instance_keys)
         if _mrsm_instance is None:
             _mrsm_instance = get_config('meerschaum', 'instance', patch=True)
 
@@ -341,10 +351,10 @@ class Pipe:
         Return the four keys needed to reconstruct this pipe.
         """
         return {
-            'connector': self.connector_keys,
-            'metric': self.metric_key,
-            'location': self.location_key,
-            'instance': self.instance_keys,
+            'connector_keys': self.connector_keys,
+            'metric_key': self.metric_key,
+            'location_key': self.location_key,
+            'instance_keys': self.instance_keys,
         }
 
     def keys(self) -> List[str]:
@@ -385,7 +395,7 @@ class Pipe:
                 warnings.simplefilter('ignore')
                 try:
                     conn = parse_instance_keys(self.connector_keys)
-                except Exception as e:
+                except Exception:
                     conn = None
             if conn:
                 self._connector = conn
@@ -429,7 +439,7 @@ class Pipe:
             _fetch_patch = {
                 'fetch': ({
                     'definition': (
-                        f"SELECT * FROM "
+                        "SELECT * FROM "
                         + sql_item_name(
                             str(self.target),
                             self.instance_connector.flavor,
@@ -467,7 +477,7 @@ class Pipe:
                 and self.location_key == other.location_key
                 and self.instance_keys == other.instance_keys
             )
-        except Exception as e:
+        except Exception:
             return False
 
     def __hash__(self):
@@ -496,11 +506,11 @@ class Pipe:
         Define the state dictionary (pickling).
         """
         return {
-            'connector': self.connector_keys,
-            'metric': self.metric_key,
-            'location': self.location_key,
+            'connector_keys': self.connector_keys,
+            'metric_key': self.metric_key,
+            'location_key': self.location_key,
             'parameters': self.parameters,
-            'instance': self.instance_keys,
+            'instance_keys': self.instance_keys,
         }
 
     def __setstate__(self, _state: Dict[str, Any]):

meerschaum/core/Pipe/_attributes.py

@@ -200,10 +200,15 @@ def dtypes(self) -> Union[Dict[str, Any], None]:
     If defined, return the `dtypes` dictionary defined in `meerschaum.Pipe.parameters`.
     """
     from meerschaum.config._patch import apply_patch_to_config
+    from meerschaum.utils.dtypes import MRSM_ALIAS_DTYPES
     configured_dtypes = self.parameters.get('dtypes', {})
     remote_dtypes = self.infer_dtypes(persist=False)
     patched_dtypes = apply_patch_to_config(remote_dtypes, configured_dtypes)
-    return patched_dtypes
+    return {
+        col: MRSM_ALIAS_DTYPES.get(typ, typ)
+        for col, typ in patched_dtypes.items()
+        if col and typ
+    }
 
 
 @dtypes.setter
@@ -289,6 +294,25 @@ def tzinfo(self) -> Union[None, timezone]:
     return None
 
 
+@property
+def enforce(self) -> bool:
+    """
+    Return the `enforce` parameter for the pipe.
+    """
+    if 'enforce' not in self.parameters:
+        self.parameters['enforce'] = True
+
+    return self.parameters['enforce']
+
+
+@enforce.setter
+def enforce(self, _enforce: bool) -> None:
+    """
+    Set the `enforce` parameter for the pipe.
+    """
+    self.parameters['_enforce'] = _enforce
+
+
 def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
     """
     Check if the requested columns are defined.

meerschaum/core/Pipe/_dtypes.py

@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     pd = mrsm.attempt_import('pandas')
 
+
 def enforce_dtypes(
     self,
     df: 'pd.DataFrame',
@@ -30,7 +31,7 @@ def enforce_dtypes(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.dataframe import parse_df_datetimes, enforce_dtypes as _enforce_dtypes
-    from meerschaum.utils.dtypes import are_dtypes_equal
+    from meerschaum.utils.dtypes import are_dtypes_equal, MRSM_PD_DTYPES
     from meerschaum.utils.packages import import_pandas
     pd = import_pandas(debug=debug)
     if df is None:
@@ -41,7 +42,11 @@ def enforce_dtypes(
             )
         return df
 
-    pipe_dtypes = self.dtypes
+    pipe_dtypes = self.dtypes if self.enforce else {
+        col: typ
+        for col, typ in self.dtypes.items()
+        if typ in MRSM_PD_DTYPES
+    }
 
     try:
         if isinstance(df, str):
@@ -105,22 +110,16 @@ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str,
     A dictionary of strings containing the pandas data types for this Pipe.
     """
     if not self.exists(debug=debug):
-        dtypes = {}
-        if not self.columns:
-            return {}
-        dt_col = self.columns.get('datetime', None)
-        if dt_col:
-            if not self.parameters.get('dtypes', {}).get(dt_col, None):
-                dtypes[dt_col] = 'datetime64[ns, UTC]'
-        return dtypes
+        return {}
 
     from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
     from meerschaum.utils.dtypes import to_pandas_dtype
-    columns_types = self.get_columns_types(debug=debug)
 
     ### NOTE: get_columns_types() may return either the types as
     ###       PostgreSQL- or Pandas-style.
-    dtypes = {
+    columns_types = self.get_columns_types(debug=debug)
+
+    remote_pd_dtypes = {
         c: (
             get_pd_type_from_db_type(t, allow_custom_dtypes=True)
             if str(t).isupper()
@@ -128,7 +127,15 @@ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str,
         )
         for c, t in columns_types.items()
     } if columns_types else {}
-    if persist:
-        self.dtypes = dtypes
-        self.edit(interactive=False, debug=debug)
-    return dtypes
+    if not persist:
+        return remote_pd_dtypes
+
+    dtypes = self.parameters.get('dtypes', {})
+    dtypes.update({
+        col: typ
+        for col, typ in remote_pd_dtypes.items()
+        if col not in dtypes
+    })
+    self.dtypes = dtypes
+    self.edit(interactive=False, debug=debug)
+    return remote_pd_dtypes

meerschaum/core/Pipe/_sync.py

@@ -161,7 +161,7 @@ def sync(
     self._exists = None
 
     def _sync(
-        p: 'meerschaum.Pipe',
+        p: mrsm.Pipe,
         df: Union[
             'pd.DataFrame',
             Dict[str, List[Any]],
@@ -368,10 +368,11 @@ def sync(
         ### Cast to a dataframe and ensure datatypes are what we expect.
         df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
 
-        ### Capture `numeric`, `uuid`, and `json` columns.
+        ### Capture `numeric`, `uuid`, `json`, and `bytes` columns.
         self._persist_new_json_columns(df, debug=debug)
         self._persist_new_numeric_columns(df, debug=debug)
         self._persist_new_uuid_columns(df, debug=debug)
+        self._persist_new_bytes_columns(df, debug=debug)
 
         if debug:
             dprint(
@@ -617,11 +618,13 @@ def filter_existing(
         filter_unseen_df,
         add_missing_cols_to_df,
         get_unhashable_cols,
-        get_numeric_cols,
     )
     from meerschaum.utils.dtypes import (
         to_pandas_dtype,
         none_if_null,
+        to_datetime,
+        are_dtypes_equal,
+        value_is_null,
     )
     from meerschaum.config import get_config
     pd = import_pandas()
@@ -669,29 +672,36 @@ def filter_existing(
     ### begin is the oldest data in the new dataframe
     begin, end = None, None
     dt_col = pipe_columns.get('datetime', None)
+    primary_key = pipe_columns.get('primary', None)
     dt_type = self.dtypes.get(dt_col, 'datetime64[ns, UTC]') if dt_col else None
+
+    if autoincrement and primary_key == dt_col and dt_col not in df.columns:
+        if enforce_dtypes:
+            df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+        return df, get_empty_df(), df
+
     try:
-        min_dt_val = df[dt_col].min(skipna=True) if dt_col else None
+        min_dt_val = df[dt_col].min(skipna=True) if dt_col and dt_col in df.columns else None
         if is_dask and min_dt_val is not None:
             min_dt_val = min_dt_val.compute()
         min_dt = (
-            pandas.to_datetime(min_dt_val).to_pydatetime()
-            if min_dt_val is not None and 'datetime' in str(dt_type)
+            to_datetime(min_dt_val, as_pydatetime=True)
+            if min_dt_val is not None and are_dtypes_equal(dt_type, 'datetime')
             else min_dt_val
         )
     except Exception:
         min_dt = None
-    if not ('datetime' in str(type(min_dt))) or str(min_dt) == 'NaT':
-        if 'int' not in str(type(min_dt)).lower():
+
+    if not are_dtypes_equal('datetime', str(type(min_dt))) or value_is_null(min_dt):
+        if not are_dtypes_equal('int', str(type(min_dt))):
             min_dt = None
 
     if isinstance(min_dt, datetime):
-        begin = (
-            round_time(
-                min_dt,
-                to='down'
-            ) - timedelta(minutes=1)
-        )
+        rounded_min_dt = round_time(min_dt, to='down')
+        try:
+            begin = rounded_min_dt - timedelta(minutes=1)
+        except OverflowError:
+            begin = rounded_min_dt
     elif dt_type and 'int' in dt_type.lower():
         begin = min_dt
     elif dt_col is None:
@@ -699,11 +709,11 @@ def filter_existing(
 
     ### end is the newest data in the new dataframe
     try:
-        max_dt_val = df[dt_col].max(skipna=True) if dt_col else None
+        max_dt_val = df[dt_col].max(skipna=True) if dt_col and dt_col in df.columns else None
         if is_dask and max_dt_val is not None:
             max_dt_val = max_dt_val.compute()
         max_dt = (
-            pandas.to_datetime(max_dt_val).to_pydatetime()
+            to_datetime(max_dt_val, as_pydatetime=True)
             if max_dt_val is not None and 'datetime' in str(dt_type)
             else max_dt_val
         )
@@ -712,8 +722,8 @@ def filter_existing(
         traceback.print_exc()
         max_dt = None
 
-    if ('datetime' not in str(type(max_dt))) or str(min_dt) == 'NaT':
-        if 'int' not in str(type(max_dt)).lower():
+    if not are_dtypes_equal('datetime', str(type(max_dt))) or value_is_null(max_dt):
+        if not are_dtypes_equal('int', str(type(max_dt))):
             max_dt = None
 
     if isinstance(max_dt, datetime):
@@ -723,7 +733,7 @@ def filter_existing(
                 to='down'
             ) + timedelta(minutes=1)
         )
-    elif dt_type and 'int' in dt_type.lower():
+    elif dt_type and 'int' in dt_type.lower() and max_dt is not None:
         end = max_dt + 1
 
     if max_dt is not None and min_dt is not None and min_dt > max_dt:
@@ -738,7 +748,7 @@ def filter_existing(
 
     unique_index_vals = {
         col: df[col].unique()
-        for col in pipe_columns
+        for col in (pipe_columns if not primary_key else [primary_key])
         if col in df.columns and col != dt_col
     } if not date_bound_only else {}
     filter_params_index_limit = get_config('pipes', 'sync', 'filter_params_index_limit')
@@ -777,14 +787,15 @@ def filter_existing(
 
     ### Separate new rows from changed ones.
     on_cols = [
-        col for col_key, col in pipe_columns.items()
+        col
+        for col_key, col in pipe_columns.items()
         if (
             col
             and
             col_key != 'value'
             and col in backtrack_df.columns
         )
-    ]
+    ] if not primary_key else [primary_key]
     self_dtypes = self.dtypes
     on_cols_dtypes = {
         col: to_pandas_dtype(typ)
@@ -949,10 +960,7 @@ def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
         return True, "Success"
 
     self._attributes_sync_time = None
-    dt_col = self.columns.get('datetime', None)
     dtypes = self.parameters.get('dtypes', {})
-    if dt_col not in dtypes:
-        dtypes[dt_col] = 'datetime'
     dtypes.update({col: 'numeric' for col in numeric_cols})
     self.parameters['dtypes'] = dtypes
     if not self.temporary:
@@ -977,10 +985,7 @@ def _persist_new_uuid_columns(self, df, debug: bool = False) -> SuccessTuple:
         return True, "Success"
 
     self._attributes_sync_time = None
-    dt_col = self.columns.get('datetime', None)
     dtypes = self.parameters.get('dtypes', {})
-    if dt_col not in dtypes:
-        dtypes[dt_col] = 'datetime'
     dtypes.update({col: 'uuid' for col in uuid_cols})
     self.parameters['dtypes'] = dtypes
     if not self.temporary:
@@ -1005,10 +1010,7 @@ def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
         return True, "Success"
 
     self._attributes_sync_time = None
-    dt_col = self.columns.get('datetime', None)
     dtypes = self.parameters.get('dtypes', {})
-    if dt_col not in dtypes:
-        dtypes[dt_col] = 'datetime'
     dtypes.update({col: 'json' for col in json_cols})
     self.parameters['dtypes'] = dtypes
 
@@ -1020,3 +1022,29 @@ def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
         return edit_success, edit_msg
 
     return True, "Success"
+
+
+def _persist_new_bytes_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new `bytes` columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_bytes_cols
+    bytes_cols = get_bytes_cols(df)
+    existing_bytes_cols = [col for col, typ in self.dtypes.items() if typ == 'bytes']
+    new_bytes_cols = [col for col in bytes_cols if col not in existing_bytes_cols]
+    if not new_bytes_cols:
+        return True, "Success"
+
+    self._attributes_sync_time = None
+    dtypes = self.parameters.get('dtypes', {})
+    dtypes.update({col: 'bytes' for col in bytes_cols})
+    self.parameters['dtypes'] = dtypes
+
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update bytes dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"

meerschaum/core/Pipe/_verify.py

@@ -7,9 +7,10 @@ Verify the contents of a pipe by resyncing its interval.
 """
 
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import SuccessTuple, Any, Optional, Union, Tuple, List, Dict
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, Union, Tuple, Dict
 from meerschaum.utils.warnings import warn, info
-from meerschaum.utils.debug import dprint
 
 
 def verify(
@@ -94,9 +95,6 @@ def verify(
             else 1
         )
 
-    sync_less_than_begin = not bounded and begin is None
-    sync_greater_than_end = not bounded and end is None
-
     cannot_determine_bounds = not self.exists(debug=debug)
 
     if cannot_determine_bounds:
@@ -164,7 +162,7 @@ def verify(
     )
 
     info(
-        f"Syncing {len(chunk_bounds)} chunk" + ('s' if len(chunk_bounds) != 1 else '')
+        f"Verifying {self}:\n    Syncing {len(chunk_bounds)} chunk" + ('s' if len(chunk_bounds) != 1 else '')
         + f" ({'un' if not bounded else ''}bounded)"
         + f" of size '{interval_str(chunk_interval)}'"
         + f" between '{begin_to_print}' and '{end_to_print}'."
@@ -187,7 +185,7 @@ def verify(
             return chunk_begin_and_end, bounds_success_tuples[chunk_begin_and_end]
 
         chunk_begin, chunk_end = chunk_begin_and_end
-        return chunk_begin_and_end, self.sync(
+        chunk_success, chunk_msg = self.sync(
             begin=chunk_begin,
             end=chunk_end,
             params=params,
@@ -195,6 +193,9 @@ def verify(
             debug=debug,
             **kwargs
         )
+        chunk_msg = chunk_msg.strip()
+        mrsm.pprint((chunk_success, chunk_msg))
+        return chunk_begin_and_end, (chunk_success, chunk_msg)
 
     ### If we have more than one chunk, attempt to sync the first one and return if its fails.
     if len(chunk_bounds) > 1:

meerschaum/jobs/_Job.py

@@ -200,6 +200,8 @@ class Job:
             if root_dir is None:
                 from meerschaum.config.paths import ROOT_DIR_PATH
                 root_dir = ROOT_DIR_PATH
+            else:
+                root_dir = pathlib.Path(root_dir)
             jobs_dir = root_dir / DAEMON_RESOURCES_PATH.name
             daemon_dir = jobs_dir / daemon_id
             pid_file = daemon_dir / 'process.pid'
@@ -873,7 +875,7 @@ class Job:
         """
         from meerschaum._internal.arguments import compress_pipeline_sysargs
         sysargs = compress_pipeline_sysargs(self.sysargs)
-        return shlex.join(sysargs).replace(' + ', '\n+ ')
+        return shlex.join(sysargs).replace(' + ', '\n+ ').replace(' : ', '\n: ').lstrip().rstrip()
 
     @property
     def _externally_managed_file(self) -> pathlib.Path:
@@ -915,6 +917,7 @@ class Job:
             'PYTHONUNBUFFERED': '1',
             'LINES': str(get_config('jobs', 'terminal', 'lines')),
             'COLUMNS': str(get_config('jobs', 'terminal', 'columns')),
+            STATIC_CONFIG['environment']['noninteractive']: 'true',
         }
         self._env = {**default_env, **_env}
         return self._env

meerschaum/plugins/_Plugin.py

@@ -255,11 +255,11 @@ class Plugin:
 
 
     def install(
-            self,
-            skip_deps: bool = False,
-            force: bool = False,
-            debug: bool = False,
-        ) -> SuccessTuple:
+        self,
+        skip_deps: bool = False,
+        force: bool = False,
+        debug: bool = False,
+    ) -> SuccessTuple:
         """
         Extract a plugin's tar archive to the plugins directory.
         
@@ -359,7 +359,7 @@ class Plugin:
             is_same_version = new_version and old_version and (
                 packaging_version.parse(old_version) == packaging_version.parse(new_version)
             )
-        except Exception as e:
+        except Exception:
             is_new_version, is_same_version = True, False
 
         ### Determine where to permanently store the new plugin.
@@ -404,7 +404,7 @@ class Plugin:
                         dprint(f"Moving '{src_file}' to '{dst_dir}'...")
                     try:
                         shutil.move(src_file, dst_dir)
-                    except Exception as e:
+                    except Exception:
                         success, msg = False, (
                             f"Failed to install plugin '{self}': " +
                             f"Could not move file '{src_file}' to '{dst_dir}'"
@@ -817,10 +817,10 @@ class Plugin:
 
 
     def install_dependencies(
-            self,
-            force: bool = False,
-            debug: bool = False,
-        ) -> bool:
+        self,
+        force: bool = False,
+        debug: bool = False,
+    ) -> bool:
         """
         If specified, install dependencies.
         
@@ -841,12 +841,9 @@ class Plugin:
         Returns
         -------
         A bool indicating success.
-
         """
         from meerschaum.utils.packages import pip_install, venv_contains_package
-        from meerschaum.utils.debug import dprint
         from meerschaum.utils.warnings import warn, info
-        from meerschaum.connectors.parse import parse_repo_keys
         _deps = self.get_dependencies(debug=debug)
         if not _deps and self.requirements_file_path is None:
             return True