meerschaum 2.3.6__py3-none-any.whl → 2.4.0__py3-none-any.whl

meerschaum/core/Pipe/_sync.py

@@ -26,8 +26,6 @@ from meerschaum.utils.typing import (
     SuccessTuple,
     Dict,
     List,
-    Iterable,
-    Generator,
 )
 from meerschaum.utils.warnings import warn, error
 
@@ -266,7 +264,6 @@ def sync(
                             **kw
                         )
                     )
-
             except Exception as e:
                 get_console().print_exception(
                     suppress=[
@@ -369,6 +366,11 @@ def sync(
 
         ### Cast to a dataframe and ensure datatypes are what we expect.
         df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+
+        ### Capture `numeric` and `json` columns.
+        self._persist_new_json_columns(df, debug=debug)
+        self._persist_new_numeric_columns(df, debug=debug)
+
         if debug:
             dprint(
                 "DataFrame to sync:\n"
@@ -554,14 +556,15 @@ def exists(
 
 
 def filter_existing(
-        self,
-        df: 'pd.DataFrame',
-        safe_copy: bool = True,
-        date_bound_only: bool = False,
-        chunksize: Optional[int] = -1,
-        debug: bool = False,
-        **kw
-    ) -> Tuple['pd.DataFrame', 'pd.DataFrame', 'pd.DataFrame']:
+    self,
+    df: 'pd.DataFrame',
+    safe_copy: bool = True,
+    date_bound_only: bool = False,
+    include_unchanged_columns: bool = False,
+    chunksize: Optional[int] = -1,
+    debug: bool = False,
+    **kw
+) -> Tuple['pd.DataFrame', 'pd.DataFrame', 'pd.DataFrame']:
     """
     Inspect a dataframe and filter out rows which already exist in the pipe.
 
@@ -569,7 +572,7 @@ def filter_existing(
     ----------
     df: 'pd.DataFrame'
         The dataframe to inspect and filter.
-        
+
     safe_copy: bool, default True
         If `True`, create a copy before comparing and modifying the dataframes.
         Setting to `False` may mutate the DataFrames.
@@ -578,6 +581,10 @@ def filter_existing(
     date_bound_only: bool, default False
         If `True`, only use the datetime index to fetch the sample dataframe.
 
+    include_unchanged_columns: bool, default False
+        If `True`, include the backtrack columns which haven't changed in the update dataframe.
+        This is useful if you can't update individual keys.
+
     chunksize: Optional[int], default -1
         The `chunksize` used when fetching existing data.
 
@@ -605,7 +612,7 @@ def filter_existing(
     from meerschaum.config import get_config
     pd = import_pandas()
     pandas = attempt_import('pandas')
-    if not 'dataframe' in str(type(df)).lower():
+    if 'dataframe' not in str(type(df)).lower():
         df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
     is_dask = 'dask' in df.__module__
     if is_dask:
@@ -615,8 +622,21 @@ def filter_existing(
     else:
         merge = pd.merge
         NA = pd.NA
+
+    def get_empty_df():
+        empty_df = pd.DataFrame([])
+        dtypes = dict(df.dtypes) if df is not None else {}
+        dtypes.update(self.dtypes)
+        pd_dtypes = {
+            col: to_pandas_dtype(str(typ))
+            for col, typ in dtypes.items()
+        }
+        return add_missing_cols_to_df(empty_df, pd_dtypes)
+
     if df is None:
-        return df, df, df
+        empty_df = get_empty_df()
+        return empty_df, empty_df, empty_df
+
     if (df.empty if not is_dask else len(df) == 0):
         return df, df, df
 
@@ -633,7 +653,7 @@ def filter_existing(
             if min_dt_val is not None and 'datetime' in str(dt_type)
             else min_dt_val
         )
-    except Exception as e:
+    except Exception:
         min_dt = None
     if not ('datetime' in str(type(min_dt))) or str(min_dt) == 'NaT':
         if 'int' not in str(type(min_dt)).lower():
@@ -643,7 +663,7 @@ def filter_existing(
         begin = (
             round_time(
                 min_dt,
-                to = 'down'
+                to='down'
             ) - timedelta(minutes=1)
         )
     elif dt_type and 'int' in dt_type.lower():
@@ -661,7 +681,7 @@ def filter_existing(
             if max_dt_val is not None and 'datetime' in str(dt_type)
             else max_dt_val
         )
-    except Exception as e:
+    except Exception:
         import traceback
         traceback.print_exc()
         max_dt = None
@@ -674,14 +694,14 @@ def filter_existing(
         end = (
             round_time(
                 max_dt,
-                to = 'down'
+                to='down'
             ) + timedelta(minutes=1)
         )
     elif dt_type and 'int' in dt_type.lower():
         end = max_dt + 1
 
     if max_dt is not None and min_dt is not None and min_dt > max_dt:
-        warn(f"Detected minimum datetime greater than maximum datetime.")
+        warn("Detected minimum datetime greater than maximum datetime.")
 
     if begin is not None and end is not None and begin > end:
         if isinstance(begin, datetime):
@@ -710,13 +730,18 @@ def filter_existing(
         dprint(f"Looking at data between '{begin}' and '{end}':", **kw)
 
     backtrack_df = self.get_data(
-        begin = begin,
-        end = end,
-        chunksize = chunksize,
-        params = params,
-        debug = debug,
+        begin=begin,
+        end=end,
+        chunksize=chunksize,
+        params=params,
+        debug=debug,
         **kw
     )
+    if backtrack_df is None:
+        if debug:
+            dprint(f"No backtrack data was found for {self}.")
+        return df, get_empty_df(), df
+
     if debug:
         dprint(f"Existing data for {self}:\n" + str(backtrack_df), **kw)
         dprint(f"Existing dtypes for {self}:\n" + str(backtrack_df.dtypes))
@@ -743,18 +768,19 @@ def filter_existing(
         filter_unseen_df(
             backtrack_df,
             df,
-            dtypes = {
+            dtypes={
                 col: to_pandas_dtype(typ)
                 for col, typ in self_dtypes.items()
             },
-            safe_copy = safe_copy,
-            debug = debug
+            safe_copy=safe_copy,
+            debug=debug
         ),
         on_cols_dtypes,
     )
 
     ### Cast dicts or lists to strings so we can merge.
     serializer = functools.partial(json.dumps, sort_keys=True, separators=(',', ':'), default=str)
+
     def deserializer(x):
         return json.loads(x) if isinstance(x, str) else x
 
@@ -767,12 +793,12 @@ def filter_existing(
     casted_cols = set(unhashable_delta_cols + unhashable_backtrack_cols)
 
     joined_df = merge(
-        delta_df.fillna(NA),
-        backtrack_df.fillna(NA),
-        how = 'left',
-        on = on_cols,
-        indicator = True,
-        suffixes = ('', '_old'),
+        delta_df.infer_objects(copy=False).fillna(NA),
+        backtrack_df.infer_objects(copy=False).fillna(NA),
+        how='left',
+        on=on_cols,
+        indicator=True,
+        suffixes=('', '_old'),
     ) if on_cols else delta_df
     for col in casted_cols:
         if col in joined_df.columns:
@@ -782,20 +808,13 @@ def filter_existing(
 
     ### Determine which rows are completely new.
     new_rows_mask = (joined_df['_merge'] == 'left_only') if on_cols else None
-    cols = list(backtrack_df.columns)
+    cols = list(delta_df.columns)
 
     unseen_df = (
-        (
-            joined_df
-            .where(new_rows_mask)
-            .dropna(how='all')[cols]
-            .reset_index(drop=True)
-        ) if not is_dask else (
-            joined_df
-            .where(new_rows_mask)
-            .dropna(how='all')[cols]
-            .reset_index(drop=True)
-        )
+        joined_df
+        .where(new_rows_mask)
+        .dropna(how='all')[cols]
+        .reset_index(drop=True)
     ) if on_cols else delta_df
 
     ### Rows that have already been inserted but values have changed.
@@ -804,20 +823,33 @@ def filter_existing(
         .where(~new_rows_mask)
         .dropna(how='all')[cols]
         .reset_index(drop=True)
-    ) if on_cols else None
+    ) if on_cols else get_empty_df()
+
+    if include_unchanged_columns and on_cols:
+        unchanged_backtrack_cols = [
+            col
+            for col in backtrack_df.columns
+            if col in on_cols or col not in update_df.columns
+        ]
+        update_df = merge(
+            backtrack_df[unchanged_backtrack_cols],
+            update_df,
+            how='inner',
+            on=on_cols,
+        )
 
     return unseen_df, update_df, delta_df
 
 
 @staticmethod
 def _get_chunk_label(
-        chunk: Union[
-            'pd.DataFrame',
-            List[Dict[str, Any]],
-            Dict[str, List[Any]]
-        ],
-        dt_col: str,
-    ) -> str:
+    chunk: Union[
+        'pd.DataFrame',
+        List[Dict[str, Any]],
+        Dict[str, List[Any]]
+    ],
+    dt_col: str,
+) -> str:
     """
     Return the min - max label for the chunk.
     """
@@ -870,3 +902,52 @@ def get_num_workers(self, workers: Optional[int] = None) -> int:
         (desired_workers - current_num_connections),
         1,
     )
+
+
+def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new numeric columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_numeric_cols
+    numeric_cols = get_numeric_cols(df)
+    existing_numeric_cols = [col for col, typ in self.dtypes.items() if typ == 'numeric']
+    new_numeric_cols = [col for col in numeric_cols if col not in existing_numeric_cols]
+    if not new_numeric_cols:
+        return True, "Success"
+
+    dtypes = self.parameters.get('dtypes', {})
+    dtypes.update({col: 'numeric' for col in numeric_cols})
+    self.parameters['dtypes'] = dtypes
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update NUMERIC dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"
+
+
+def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
+    """
+    Check for new JSON columns and update the parameters.
+    """
+    from meerschaum.utils.dataframe import get_json_cols
+    json_cols = get_json_cols(df)
+    existing_json_cols = [col for col, typ in self.dtypes.items() if typ == 'json']
+    new_json_cols = [col for col in json_cols if col not in existing_json_cols]
+    if not new_json_cols:
+        return True, "Success"
+
+    dtypes = self.parameters.get('dtypes', {})
+    dtypes.update({col: 'json' for col in json_cols})
+    self.parameters['dtypes'] = dtypes
+
+    if not self.temporary:
+        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
+        if not edit_success:
+            warn(f"Unable to update JSON dtypes for {self}:\n{edit_msg}")
+
+        return edit_success, edit_msg
+
+    return True, "Success"

meerschaum/core/Pipe/_verify.py

@@ -12,17 +12,17 @@ from meerschaum.utils.warnings import warn, info
 from meerschaum.utils.debug import dprint
 
 def verify(
-        self,
-        begin: Union[datetime, int, None] = None,
-        end: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        chunk_interval: Union[timedelta, int, None] = None,
-        bounded: Optional[bool] = None,
-        deduplicate: bool = False,
-        workers: Optional[int] = None,
-        debug: bool = False,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    self,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    chunk_interval: Union[timedelta, int, None] = None,
+    bounded: Optional[bool] = None,
+    deduplicate: bool = False,
+    workers: Optional[int] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Verify the contents of the pipe by resyncing its interval.
 

meerschaum/core/User/_User.py

@@ -11,7 +11,7 @@ import os
 import hashlib
 import hmac
 from binascii import b2a_base64, a2b_base64, Error as _BinAsciiError
-from meerschaum.utils.typing import Optional, Dict, Any, Tuple
+from meerschaum.utils.typing import Optional, Dict, Any, Union
 from meerschaum.config.static import STATIC_CONFIG
 from meerschaum.utils.warnings import warn
 
@@ -19,10 +19,10 @@ from meerschaum.utils.warnings import warn
 __all__ = ('hash_password', 'verify_password', 'User')
 
 def hash_password(
-        password: str,
-        salt: Optional[bytes] = None,
-        rounds: Optional[int] = None,
-    ) -> str:
+    password: str,
+    salt: Optional[bytes] = None,
+    rounds: Optional[int] = None,
+) -> str:
     """
     Return an encoded hash string from the given password.
 
@@ -68,9 +68,9 @@ def hash_password(
 
 
 def verify_password(
-        password: str,
-        password_hash: str,
-    ) -> bool:
+    password: str,
+    password_hash: str,
+) -> bool:
     """
     Return `True` if the password matches the provided hash.
 
@@ -197,26 +197,28 @@ class User:
         return self._attributes
 
     @property
-    def instance_connector(self) -> meerschaum.connectors.Connector:
-        """ """
+    def instance_connector(self) -> 'mrsm.connectors.Connector':
         from meerschaum.connectors.parse import parse_instance_keys
         if '_instance_connector' not in self.__dict__:
             self._instance_connector = parse_instance_keys(self._instance_keys)
         return self._instance_connector
 
     @property
-    def user_id(self) -> int:
+    def user_id(self) -> Union[int, str, None]:
         """NOTE: This causes recursion with the API,
               so don't try to get fancy with read-only attributes.
         """
         return self._user_id
 
     @user_id.setter
-    def user_id(self, user_id):
+    def user_id(self, user_id: Union[int, str, None]):
         self._user_id = user_id
 
     @property
     def password_hash(self):
+        """
+        Return the hash of the user's password.
+        """
         _password_hash = self.__dict__.get('_password_hash', None)
         if _password_hash is not None:
             return _password_hash

meerschaum/jobs/_Job.py

@@ -10,10 +10,7 @@ from __future__ import annotations
 
 import shlex
 import asyncio
-import threading
-import json
 import pathlib
-import os
 import sys
 import traceback
 from functools import partial
@@ -206,13 +203,11 @@ class Job:
             jobs_dir = root_dir / DAEMON_RESOURCES_PATH.name
             daemon_dir = jobs_dir / daemon_id
             pid_file = daemon_dir / 'process.pid'
-            properties_path = daemon_dir / 'properties.json'
-            pickle_path = daemon_dir / 'pickle.pkl'
 
             if pid_file.exists():
                 with open(pid_file, 'r', encoding='utf-8') as f:
                     daemon_pid = int(f.read())
-                
+
                 if pid != daemon_pid:
                     raise EnvironmentError(f"Differing PIDs: {pid=}, {daemon_pid=}")
             else:

meerschaum/jobs/__init__.py

@@ -9,9 +9,9 @@ Higher-level utilities for managing `meerschaum.utils.daemon.Daemon`.
 import pathlib
 
 import meerschaum as mrsm
-from meerschaum.utils.typing import Dict, Optional, List, Callable, Any, SuccessTuple
+from meerschaum.utils.typing import Dict, Optional, List, SuccessTuple
 
-from meerschaum.jobs._Job import Job, StopMonitoringLogs
+from meerschaum.jobs._Job import Job
 from meerschaum.jobs._Executor import Executor
 
 __all__ = (
@@ -403,9 +403,14 @@ def get_executor_keys_from_context() -> str:
     if _context_keys is not None:
         return _context_keys
 
+    from meerschaum.config import get_config
     from meerschaum.config.paths import ROOT_DIR_PATH, DEFAULT_ROOT_DIR_PATH
     from meerschaum.utils.misc import is_systemd_available
 
+    configured_executor = get_config('meerschaum', 'executor', warn=False)
+    if configured_executor is not None:
+        return configured_executor
+
     _context_keys = (
         'systemd'
         if is_systemd_available() and ROOT_DIR_PATH == DEFAULT_ROOT_DIR_PATH

meerschaum/plugins/_Plugin.py

@@ -7,8 +7,12 @@ Plugin metadata class
 """
 
 from __future__ import annotations
-import os, pathlib, shutil
-from site import venv
+
+import os
+import pathlib
+import shutil
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import (
     Dict,
     List,
@@ -40,8 +44,8 @@ class Plugin:
         attributes: Optional[Dict[str, Any]] = None,
         archive_path: Optional[pathlib.Path] = None,
         venv_path: Optional[pathlib.Path] = None,
-        repo_connector: Optional['meerschaum.connectors.api.APIConnector'] = None,
-        repo: Union['meerschaum.connectors.api.APIConnector', str, None] = None,
+        repo_connector: Optional['mrsm.connectors.api.APIConnector'] = None,
+        repo: Union['mrsm.connectors.api.APIConnector', str, None] = None,
     ):
         from meerschaum.config.static import STATIC_CONFIG
         sep = STATIC_CONFIG['plugins']['repo_separator']
@@ -470,9 +474,9 @@ class Plugin:
 
 
     def remove_archive(
-            self,        
-            debug: bool = False
-        ) -> SuccessTuple:
+        self,        
+        debug: bool = False
+    ) -> SuccessTuple:
         """Remove a plugin's archive file."""
         if not self.archive_path.exists():
             return True, f"Archive file for plugin '{self}' does not exist."
@@ -484,9 +488,9 @@ class Plugin:
 
 
     def remove_venv(
-            self,        
-            debug: bool = False
-        ) -> SuccessTuple:
+        self,        
+        debug: bool = False
+    ) -> SuccessTuple:
         """Remove a plugin's virtual environment."""
         if not self.venv_path.exists():
             return True, f"Virtual environment for plugin '{self}' does not exist."
@@ -608,9 +612,9 @@ class Plugin:
 
 
     def get_dependencies(
-            self,
-            debug: bool = False,
-        ) -> List[str]:
+        self,
+        debug: bool = False,
+    ) -> List[str]:
         """
         If the Plugin has specified dependencies in a list called `required`, return the list.
         

meerschaum/utils/_get_pipes.py

@@ -16,19 +16,18 @@ from meerschaum.utils.typing import (
 __pdoc__ = {'get_pipes': True, 'fetch_pipes_keys': True}
 
 def get_pipes(
-        connector_keys: Union[str, List[str], None] = None,
-        metric_keys: Union[str, List[str], None] = None,
-        location_keys: Union[str, List[str], None] = None,
-        tags: Optional[List[str]] = None,
-        params: Optional[Dict[str, Any]] = None,
-        mrsm_instance: Union[str, InstanceConnector, None] = None,
-        instance: Union[str, InstanceConnector, None] = None,
-        as_list: bool = False,
-        method: str = 'registered',
-        wait: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union[PipesDict, List[mrsm.Pipe]]:
+    connector_keys: Union[str, List[str], None] = None,
+    metric_keys: Union[str, List[str], None] = None,
+    location_keys: Union[str, List[str], None] = None,
+    tags: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    mrsm_instance: Union[str, InstanceConnector, None] = None,
+    instance: Union[str, InstanceConnector, None] = None,
+    as_list: bool = False,
+    method: str = 'registered',
+    debug: bool = False,
+    **kw: Any
+) -> Union[PipesDict, List[mrsm.Pipe]]:
     """
     Return a dictionary or list of `meerschaum.Pipe` objects.
 
@@ -72,10 +71,6 @@ def get_pipes(
         If `'all'`, create pipes from predefined metrics and locations. Required `connector_keys`.
         **NOTE:** Method `'all'` is not implemented!
 
-    wait: bool, default False
-        Wait for a connection before getting Pipes. Should only be true for cases where the
-        database might not be running (like the API).
-
     **kw: Any:
         Keyword arguments to pass to the `meerschaum.Pipe` constructor.
         
@@ -133,15 +128,14 @@ def get_pipes(
         location_keys = [location_keys]
 
     ### Get SQL or API connector (keys come from `connector.fetch_pipes_keys()`).
-    ### If `wait`, wait until a connection is made
     if mrsm_instance is None:
         mrsm_instance = instance
     if mrsm_instance is None:
         mrsm_instance = get_config('meerschaum', 'instance', patch=True)
     if isinstance(mrsm_instance, str):
         from meerschaum.connectors.parse import parse_instance_keys
-        connector = parse_instance_keys(keys=mrsm_instance, wait=wait, debug=debug)
-    else: ### NOTE: mrsm_instance MUST be a SQL or API connector for this to work
+        connector = parse_instance_keys(keys=mrsm_instance, debug=debug)
+    else:
         from meerschaum.connectors import instance_types
         valid_connector = False
         if hasattr(mrsm_instance, 'type'):