meerschaum 2.1.6__py3-none-any.whl → 2.2.0.dev1__py3-none-any.whl

meerschaum/config/_shell.py

@@ -126,7 +126,6 @@ default_shell_config = {
     'timeout'          : 60,
     'max_history'      : 1000,
     'clear_screen'     : True,
-    'cmd'              : default_cmd,
     'bottom_toolbar'   : {
         'enabled'      : True,
     },

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.1.6"
+__version__ = "2.2.0.dev1"

meerschaum/connectors/api/_plugins.py

@@ -49,6 +49,7 @@ def register_plugin(
 def install_plugin(
         self,
         name: str,
+        skip_deps: bool = False,
         force: bool = False,
         debug: bool = False
     ) -> SuccessTuple:
@@ -78,7 +79,7 @@ def install_plugin(
             success, msg = False, fail_msg
         return success, msg
     plugin = Plugin(name, archive_path=archive_path, repo_connector=self)
-    return plugin.install(force=force, debug=debug)
+    return plugin.install(skip_deps=skip_deps, force=force, debug=debug)
 
 def get_plugins(
         self,

meerschaum/connectors/sql/_create_engine.py

@@ -154,10 +154,10 @@ install_flavor_drivers = {
     'duckdb': ['duckdb', 'duckdb_engine'],
     'mysql': ['pymysql'],
     'mariadb': ['pymysql'],
-    'timescaledb': ['psycopg2'],
-    'postgresql': ['psycopg2'],
-    'citus': ['psycopg2'],
-    'cockroachdb': ['psycopg2', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg2'],
+    'timescaledb': ['psycopg'],
+    'postgresql': ['psycopg'],
+    'citus': ['psycopg'],
+    'cockroachdb': ['psycopg', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg'],
     'mssql': ['pyodbc'],
     'oracle': ['cx_Oracle'],
 }
@@ -165,7 +165,7 @@ require_patching_flavors = {'cockroachdb': [('sqlalchemy-cockroachdb', 'sqlalche
 
 flavor_dialects = {
     'cockroachdb': (
-        'cockroachdb', 'sqlalchemy_cockroachdb.psycopg2', 'CockroachDBDialect_psycopg2'
+        'cockroachdb', 'sqlalchemy_cockroachdb.psycopg', 'CockroachDBDialect_psycopg'
     ),
     'duckdb': ('duckdb', 'duckdb_engine', 'Dialect'),
 }

meerschaum/connectors/sql/_fetch.py

@@ -174,9 +174,6 @@ def get_pipe_metadef(
                 )
 
 
-    if 'order by' in definition.lower() and 'over' not in definition.lower():
-        error("Cannot fetch with an ORDER clause in the definition")
-
     apply_backtrack = begin == '' and check_existing
     backtrack_interval = pipe.get_backtrack_interval(check_existing=check_existing, debug=debug)
     btm = (
@@ -308,9 +305,9 @@ def _simple_fetch_query(pipe, debug: bool=False, **kw) -> str:
     def_name = 'definition'
     definition = get_pipe_query(pipe)
     return (
-        f"WITH {def_name} AS ({definition}) SELECT * FROM {def_name}"
+        f"WITH {def_name} AS (\n{definition}\n) SELECT * FROM {def_name}"
         if pipe.connector.flavor not in ('mysql', 'mariadb')
-        else f"SELECT * FROM ({definition}) AS {def_name}"
+        else f"SELECT * FROM (\n{definition}\n) AS {def_name}"
     )
 
 def _join_fetch_query(
@@ -363,10 +360,10 @@ def _join_fetch_query(
             )
             + f") AS {id_remote_name}, "
             + dateadd_str(
-                flavor=pipe.connector.flavor,
-                begin=_st,
-                datepart='minute',
-                number=pipe.parameters.get('fetch', {}).get('backtrack_minutes', 0)
+                flavor = pipe.connector.flavor,
+                begin = _st,
+                datepart = 'minute',
+                number = pipe.parameters.get('fetch', {}).get('backtrack_minutes', 0)
             ) + " AS " + dt_remote_name + "\nUNION ALL\n"
         )
     _sync_times_q = _sync_times_q[:(-1 * len('UNION ALL\n'))] + ")"
@@ -374,13 +371,13 @@ def _join_fetch_query(
     definition = get_pipe_query(pipe)
     query = (
         f"""
-    WITH definition AS ({definition}){_sync_times_q}
+    WITH definition AS (\n{definition}\n){_sync_times_q}
     SELECT definition.*
     FROM definition"""
         if pipe.connector.flavor not in ('mysql', 'mariadb')
         else (
         f"""
-    SELECT * FROM ({definition}) AS definition"""
+    SELECT * FROM (\n{definition}\n) AS definition"""
         )
     ) + f"""
     LEFT OUTER JOIN {sync_times_remote_name} AS st

meerschaum/connectors/sql/_pipes.py

@@ -1182,7 +1182,12 @@ def sync_pipe(
         dprint("Fetched data:\n" + str(df))
 
     if not isinstance(df, pd.DataFrame):
-        df = pipe.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+        df = pipe.enforce_dtypes(
+            df,
+            chunksize = chunksize,
+            safe_copy = kw.get('safe_copy', False),
+            debug = debug,
+        )
 
     ### if table does not exist, create it with indices
     is_new = False
@@ -1226,6 +1231,7 @@ def sync_pipe(
     upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in update_queries
     if upsert:
         check_existing = False
+    kw['safe_copy'] = kw.get('safe_copy', False)
 
     unseen_df, update_df, delta_df = (
         pipe.filter_existing(

meerschaum/core/Pipe/_dtypes.py

@@ -14,6 +14,7 @@ def enforce_dtypes(
         self,
         df: 'pd.DataFrame',
         chunksize: Optional[int] = -1,
+        safe_copy: bool = True,
         debug: bool = False,
     ) -> 'pd.DataFrame':
     """
@@ -71,7 +72,7 @@ def enforce_dtypes(
             )
         return df
 
-    return _enforce_dtypes(df, pipe_dtypes, debug=debug)
+    return _enforce_dtypes(df, pipe_dtypes, safe_copy=safe_copy, debug=debug)
 
 
 def infer_dtypes(self, persist: bool=False, debug: bool=False) -> Dict[str, Any]:

meerschaum/core/Pipe/_sync.py

@@ -12,6 +12,7 @@ import json
 import time
 import threading
 import multiprocessing
+import functools
 from datetime import datetime, timedelta
 
 from meerschaum.utils.typing import (
@@ -518,6 +519,8 @@ def exists(
 def filter_existing(
         self,
         df: 'pd.DataFrame',
+        safe_copy: bool = True,
+        date_bound_only: bool = False,
         chunksize: Optional[int] = -1,
         debug: bool = False,
         **kw
@@ -530,6 +533,14 @@ def filter_existing(
     df: 'pd.DataFrame'
         The dataframe to inspect and filter.
         
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
+        See `meerschaum.utils.dataframe.filter_unseen_df`.
+
+    date_bound_only: bool, default False
+        If `True`, only use the datetime index to fetch the sample dataframe.
+
     chunksize: Optional[int], default -1
         The `chunksize` used when fetching existing data.
 
@@ -567,7 +578,8 @@ def filter_existing(
     else:
         merge = pd.merge
         NA = pd.NA
-
+    if df is None:
+        return df, df, df
     if (df.empty if not is_dask else len(df) == 0):
         return df, df, df
 
@@ -617,7 +629,7 @@ def filter_existing(
         traceback.print_exc()
         max_dt = None
 
-    if not ('datetime' in str(type(max_dt))) or str(min_dt) == 'NaT':
+    if ('datetime' not in str(type(max_dt))) or str(min_dt) == 'NaT':
         if 'int' not in str(type(max_dt)).lower():
             max_dt = None
 
@@ -645,7 +657,7 @@ def filter_existing(
         col: df[col].unique()
         for col in self.columns
         if col in df.columns and col != dt_col
-    }
+    } if not date_bound_only else {}
     filter_params_index_limit = get_config('pipes', 'sync', 'filter_params_index_limit')
     _ = kw.pop('params', None)
     params = {
@@ -655,7 +667,7 @@ def filter_existing(
         ]
         for col, unique_vals in unique_index_vals.items()
         if len(unique_vals) <= filter_params_index_limit
-    }
+    } if not date_bound_only else {}
 
     if debug:
         dprint(f"Looking at data between '{begin}' and '{end}':", **kw)
@@ -698,18 +710,23 @@ def filter_existing(
                 col: to_pandas_dtype(typ)
                 for col, typ in self_dtypes.items()
             },
+            safe_copy = safe_copy,
             debug = debug
         ),
         on_cols_dtypes,
     )
 
     ### Cast dicts or lists to strings so we can merge.
+    serializer = functools.partial(json.dumps, sort_keys=True, separators=(',', ':'), default=str)
+    def deserializer(x):
+        return json.loads(x) if isinstance(x, str) else x
+
     unhashable_delta_cols = get_unhashable_cols(delta_df)
     unhashable_backtrack_cols = get_unhashable_cols(backtrack_df)
     for col in unhashable_delta_cols:
-        delta_df[col] = delta_df[col].apply(json.dumps)
+        delta_df[col] = delta_df[col].apply(serializer)
     for col in unhashable_backtrack_cols:
-        backtrack_df[col] = backtrack_df[col].apply(json.dumps)
+        backtrack_df[col] = backtrack_df[col].apply(serializer)
     casted_cols = set(unhashable_delta_cols + unhashable_backtrack_cols)
 
     joined_df = merge(
@@ -722,13 +739,9 @@ def filter_existing(
     ) if on_cols else delta_df
     for col in casted_cols:
         if col in joined_df.columns:
-            joined_df[col] = joined_df[col].apply(
-                lambda x: (
-                    json.loads(x)
-                    if isinstance(x, str)
-                    else x
-                )
-            )
+            joined_df[col] = joined_df[col].apply(deserializer)
+        if col in delta_df.columns:
+            delta_df[col] = delta_df[col].apply(deserializer)
 
     ### Determine which rows are completely new.
     new_rows_mask = (joined_df['_merge'] == 'left_only') if on_cols else None

meerschaum/plugins/_Plugin.py

@@ -252,6 +252,7 @@ class Plugin:
 
     def install(
             self,
+            skip_deps: bool = False,
             force: bool = False,
             debug: bool = False,
         ) -> SuccessTuple:
@@ -263,6 +264,9 @@ class Plugin:
 
         Parameters
         ----------
+        skip_deps: bool, default False
+            If `True`, do not install dependencies.
+
         force: bool, default False
             If `True`, continue with installation, even if required packages fail to install.
 
@@ -366,7 +370,11 @@ class Plugin:
                 plugin_installation_dir_path = path
                 break
 
-        success_msg = f"Successfully installed plugin '{self}'."
+        success_msg = (
+            f"Successfully installed plugin '{self}'"
+            + ("\n    (skipped dependencies)" if skip_deps else "")
+            + "."
+        )
         success, abort = None, None
 
         if is_same_version and not force:
@@ -423,7 +431,8 @@ class Plugin:
             return success, msg
 
         ### attempt to install dependencies
-        if not self.install_dependencies(force=force, debug=debug):
+        dependencies_installed = skip_deps or self.install_dependencies(force=force, debug=debug)
+        if not dependencies_installed:
             _ongoing_installations.remove(self.full_name)
             return False, f"Failed to install dependencies for plugin '{self}'."
 

meerschaum/utils/daemon/Daemon.py

@@ -865,21 +865,29 @@ class Daemon:
             error(_write_pickle_success_tuple[1])
 
 
-    def cleanup(self, keep_logs: bool = False) -> None:
-        """Remove a daemon's directory after execution.
+    def cleanup(self, keep_logs: bool = False) -> SuccessTuple:
+        """
+        Remove a daemon's directory after execution.
 
         Parameters
         ----------
         keep_logs: bool, default False
             If `True`, skip deleting the daemon's log files.
+
+        Returns
+        -------
+        A `SuccessTuple` indicating success.
         """
         if self.path.exists():
             try:
                 shutil.rmtree(self.path)
             except Exception as e:
-                warn(e)
+                msg = f"Failed to clean up '{self.daemon_id}':\n{e}"
+                warn(msg)
+                return False, msg
         if not keep_logs:
             self.rotating_log.delete()
+        return True, "Success"
 
 
     def get_timeout_seconds(self, timeout: Union[int, float, None] = None) -> Union[int, float]:

meerschaum/utils/dataframe.py

@@ -7,9 +7,10 @@ Utility functions for working with DataFrames.
 """
 
 from __future__ import annotations
+from datetime import datetime
 from meerschaum.utils.typing import (
     Optional, Dict, Any, List, Hashable, Generator,
-    Iterator, Iterable, Union,
+    Iterator, Iterable, Union, Tuple,
 )
 
 
@@ -71,6 +72,7 @@ def add_missing_cols_to_df(df: 'pd.DataFrame', dtypes: Dict[str, Any]) -> pd.Dat
 def filter_unseen_df(
         old_df: 'pd.DataFrame',
         new_df: 'pd.DataFrame',
+        safe_copy: bool = True,
         dtypes: Optional[Dict[str, Any]] = None,
         debug: bool = False,
     ) -> 'pd.DataFrame':
@@ -84,6 +86,10 @@ def filter_unseen_df(
         
     new_df: 'pd.DataFrame'
         The fetched (source) dataframe. Rows that are contained in `old_df` are removed.
+
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
         
     dtypes: Optional[Dict[str, Any]], default None
         Optionally specify the datatypes of the dataframe.
@@ -111,6 +117,10 @@ def filter_unseen_df(
     if old_df is None:
         return new_df
 
+    if safe_copy:
+        old_df = old_df.copy()
+        new_df = new_df.copy()
+
     import json
     import functools
     import traceback
@@ -118,6 +128,7 @@ def filter_unseen_df(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.packages import import_pandas, attempt_import
     from meerschaum.utils.dtypes import to_pandas_dtype, are_dtypes_equal, attempt_cast_to_numeric
+    from meerschaum.utils.debug import dprint
     pd = import_pandas(debug=debug)
     is_dask = 'dask' in new_df.__module__
     if is_dask:
@@ -243,12 +254,7 @@ def filter_unseen_df(
         indicator = True,
     )
     changed_rows_mask = (joined_df['_merge'] == 'left_only')
-
-    delta_df = joined_df[
-        list(new_df_dtypes.keys())
-    ][
-        changed_rows_mask
-    ].reset_index(drop=True)
+    delta_df = joined_df[list(new_df_dtypes.keys())][changed_rows_mask].reset_index(drop=True)
 
     for json_col in json_cols:
         if json_col not in delta_df.columns:
@@ -535,6 +541,8 @@ def get_numeric_cols(df: 'pd.DataFrame') -> List[str]:
 def enforce_dtypes(
         df: 'pd.DataFrame',
         dtypes: Dict[str, str],
+        safe_copy: bool = True,
+        coerce_numeric: bool = True,
         debug: bool = False,
     ) -> 'pd.DataFrame':
     """
@@ -548,6 +556,14 @@ def enforce_dtypes(
     dtypes: Dict[str, str]
         The data types to attempt to enforce on the DataFrame.
 
+    safe_copy: bool, default True
+        If `True`, create a copy before comparing and modifying the dataframes.
+        Setting to `False` may mutate the DataFrames.
+        See `meerschaum.utils.dataframe.filter_unseen_df`.
+
+    coerce_numeric: bool, default True
+        If `True`, convert float and int collisions to numeric.
+
     debug: bool, default False
         Verbosity toggle.
 
@@ -569,6 +585,8 @@ def enforce_dtypes(
         is_dtype_numeric,
         attempt_cast_to_numeric,
     )
+    if safe_copy:
+        df = df.copy()
     df_dtypes = {c: str(t) for c, t in df.dtypes.items()}
     if len(df_dtypes) == 0:
         if debug:
@@ -674,7 +692,7 @@ def enforce_dtypes(
             explicitly_numeric
             or col in df_numeric_cols
             or (mixed_numeric_types and not explicitly_float)
-        )
+        ) and coerce_numeric
         if cast_to_numeric:
             common_dtypes[col] = attempt_cast_to_numeric
             common_diff_dtypes[col] = attempt_cast_to_numeric
@@ -860,3 +878,160 @@ def get_first_valid_dask_partition(ddf: 'dask.dataframe.DataFrame') -> Union['pd
         if len(pdf) > 0:
             return pdf
     return ddf.compute()
+
+
+def query_df(
+        df: 'pd.DataFrame',
+        params: Optional[Dict[str, Any]] = None,
+        begin: Union[datetime, int, None] = None,
+        end: Union[datetime, int, None] = None,
+        datetime_column: Optional[str] = None,
+        select_columns: Optional[List[str]] = None,
+        omit_columns: Optional[List[str]] = None,
+        inplace: bool = False,
+        reset_index: bool = False,
+        debug: bool = False,
+    ) -> 'pd.DataFrame':
+    """
+    Query the dataframe with the params dictionary.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        The DataFrame to query against.
+
+    params: Optional[Dict[str, Any]], default None
+        The parameters dictionary to use for the query.
+
+    begin: Union[datetime, int, None], default None
+        If `begin` and `datetime_column` are provided, only return rows with a timestamp
+        greater than or equal to this value.
+
+    end: Union[datetime, int, None], default None
+        If `begin` and `datetime_column` are provided, only return rows with a timestamp
+        less than this value.
+
+    datetime_column: Optional[str], default None
+        A `datetime_column` must be provided to use `begin` and `end`.
+
+    select_columns: Optional[List[str]], default None
+        If provided, only return these columns.
+
+    omit_columns: Optional[List[str]], default None
+        If provided, do not include these columns in the result.
+
+    inplace: bool, default False
+        If `True`, modify the DataFrame inplace rather than creating a new DataFrame.
+
+    reset_index: bool, default True
+        If `True`, reset the index in the resulting DataFrame.
+
+    Returns
+    -------
+    A Pandas DataFrame query result.
+    """
+    if not params and not begin and not end:
+        return df
+
+    import json
+    import meerschaum as mrsm
+    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.misc import get_in_ex_params
+    from meerschaum.utils.warnings import warn
+
+    dtypes = {col: str(typ) for col, typ in df.dtypes.items()}
+
+    if begin or end:
+        if not datetime_column or datetime_column not in df.columns:
+            warn(
+                f"The datetime column '{datetime_column}' is not present in the Dataframe, "
+                + "ignoring begin and end...",
+            )
+            begin, end = None, None
+
+    if debug:
+        dprint(f"Querying dataframe:\n{params=} {begin=} {end=} {datetime_column=}")
+
+    in_ex_params = get_in_ex_params(params)
+
+    def serialize(x: Any) -> str:
+        if isinstance(x, (dict, list, tuple)):
+            return json.dumps(x, sort_keys=True, separators=(',', ':'), default=str)
+        if hasattr(x, 'isoformat'):
+            return x.isoformat()
+        return str(x)
+
+    masks = [
+        (
+            (df[datetime_column] >= begin)
+            if begin is not None and datetime_column
+            else True
+        ) & (
+            (df[datetime_column] < end)
+            if end is not None and datetime_column
+            else True
+        )
+    ]
+
+    masks.extend([
+        (
+            (
+                df[col].apply(serialize).isin(
+                    [
+                        serialize(_in_val)
+                        for _in_val in in_vals
+                    ]
+                ) if in_vals else True
+            ) & (
+                ~df[col].apply(serialize).isin(
+                    [
+                        serialize(_ex_val)
+                        for _ex_val in ex_vals
+                    ]
+                ) if ex_vals else True
+            )
+        )
+        for col, (in_vals, ex_vals) in in_ex_params.items()
+        if col in df.columns
+    ])
+    query_mask = masks[0]
+    for mask in masks:
+        query_mask = query_mask & mask
+
+    if inplace:
+        df.where(query_mask, inplace=inplace)
+        df.dropna(how='all', inplace=inplace)
+        result_df = df
+    else:
+        result_df = df.where(query_mask).dropna(how='all')
+
+    if reset_index:
+        result_df.reset_index(drop=True, inplace=True)
+
+    result_df = enforce_dtypes(
+        result_df,
+        dtypes,
+        safe_copy = (not inplace),
+        debug = debug,
+        coerce_numeric = False,
+    )
+
+    if select_columns == ['*']:
+        select_columns = None
+
+    if not select_columns and not omit_columns:
+        return result_df
+
+    if select_columns:
+        for col in list(result_df.columns):
+            if col not in select_columns:
+                del result_df[col]
+        return result_df
+
+    if omit_columns:
+        for col in list(result_df.columns):
+            if col in omit_columns:
+                del result_df[col]
+    if debug:
+        dprint(f"{dtypes=}")
+    return result_df

meerschaum/utils/dtypes/__init__.py

@@ -6,8 +6,10 @@
 Utility functions for working with data types.
 """
 
+import traceback
 from decimal import Decimal, Context, InvalidOperation
 from meerschaum.utils.typing import Dict, Union, Any
+from meerschaum.utils.warnings import warn
 
 MRSM_PD_DTYPES: Dict[str, str] = {
     'json': 'object',
@@ -37,9 +39,7 @@ def to_pandas_dtype(dtype: str) -> str:
         from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
         return get_pd_type_from_db_type(dtype)
 
-    import traceback
     from meerschaum.utils.packages import attempt_import
-    from meerschaum.utils.warnings import warn
     pandas = attempt_import('pandas', lazy=False)
 
     try:
@@ -88,8 +88,12 @@ def are_dtypes_equal(
                 return False
         return True
 
-    if ldtype == rdtype:
-        return True
+    try:
+        if ldtype == rdtype:
+            return True
+    except Exception as e:
+        warn(f"Exception when comparing dtypes, returning False:\n{traceback.format_exc()}")
+        return False
 
     ### Sometimes pandas dtype objects are passed.
     ldtype = str(ldtype)
@@ -177,7 +181,7 @@ def attempt_cast_to_numeric(value: Any) -> Any:
         return value
 
 
-def value_is_null(value: Any) -> Any:
+def value_is_null(value: Any) -> bool:
     """
     Determine if a value is a null-like string.
     """