meerschaum 3.0.0rc1__py3-none-any.whl → 3.0.0rc3__py3-none-any.whl

meerschaum/core/Pipe/_data.py

@@ -29,6 +29,7 @@ def get_data(
     as_iterator: bool = False,
     as_chunks: bool = False,
     as_dask: bool = False,
+    add_missing_columns: bool = False,
     chunk_interval: Union[timedelta, int, None] = None,
     order: Optional[str] = 'asc',
     limit: Optional[int] = None,
@@ -72,6 +73,9 @@ def get_data(
         If `True`, return a `dask.DataFrame`
         (which may be loaded into a Pandas DataFrame with `df.compute()`).
 
+    add_missing_columns: bool, default False
+        If `True`, add any missing columns from `Pipe.dtypes` to the dataframe.
+
     chunk_interval: Union[timedelta, int, None], default None
         If `as_iterator`, then return chunks with `begin` and `end` separated by this interval.
         This may be set under `pipe.parameters['chunk_minutes']`.
@@ -103,12 +107,13 @@ def get_data(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
-    from meerschaum.utils.misc import iterate_chunks, items_str
-    from meerschaum.utils.dtypes import to_pandas_dtype, coerce_timezone
+    from meerschaum.utils.dtypes import to_pandas_dtype
     from meerschaum.utils.dataframe import add_missing_cols_to_df, df_is_chunk_generator
     from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.warnings import dprint
     dd = attempt_import('dask.dataframe') if as_dask else None
     dask = attempt_import('dask') if as_dask else None
+    _ = attempt_import('partd', lazy=False) if as_dask else None
 
     if select_columns == '*':
         select_columns = None
@@ -187,14 +192,17 @@ def get_data(
                 order=order,
                 limit=limit,
                 fresh=fresh,
+                add_missing_columns=True,
                 debug=debug,
             )
             for (chunk_begin, chunk_end) in bounds
         ]
         dask_meta = {
             col: to_pandas_dtype(typ)
-            for col, typ in self.dtypes.items()
+            for col, typ in self.get_dtypes(refresh=True, infer=True, debug=debug).items()
         }
+        if debug:
+            dprint(f"Dask meta:\n{dask_meta}")
         return _sort_df(dd.from_delayed(dask_chunks, meta=dask_meta))
 
     if not self.exists(debug=debug):
@@ -248,6 +256,7 @@ def get_data(
         if not select_columns:
             select_columns = [col for col in df.columns]
 
+        pipe_dtypes = self.get_dtypes(refresh=False, debug=debug)
         cols_to_omit = [
             col
             for col in df.columns
@@ -261,7 +270,11 @@ def get_data(
             col
             for col in select_columns
             if col not in df.columns
-        ]
+        ] + ([
+            col
+            for col in pipe_dtypes
+            if col not in df.columns
+        ] if add_missing_columns else [])
         if cols_to_omit:
             warn(
                 (
@@ -277,16 +290,26 @@ def get_data(
             df = df[_cols_to_select]
 
         if cols_to_add:
-            warn(
-                (
-                    f"Specified columns {items_str(cols_to_add)} were not found on {self}. "
-                    + "Adding these to the DataFrame as null columns."
-                ),
-                stack=False,
+            if not add_missing_columns:
+                from meerschaum.utils.misc import items_str
+                warn(
+                    f"Will add columns {items_str(cols_to_add)} as nulls to dataframe.",
+                    stack=False,
+                )
+
+            df = add_missing_cols_to_df(
+                df,
+                {
+                    col: pipe_dtypes.get(col, 'string')
+                    for col in cols_to_add
+                },
             )
-            df = add_missing_cols_to_df(df, {col: 'string' for col in cols_to_add})
 
-        enforced_df = self.enforce_dtypes(df, debug=debug)
+        enforced_df = self.enforce_dtypes(
+            df,
+            dtypes=pipe_dtypes,
+            debug=debug,
+        )
 
         if order:
             return _sort_df(enforced_df)
@@ -310,7 +333,7 @@ def _get_data_as_iterator(
     """
     Return a pipe's data as a generator.
     """
-    from meerschaum.utils.misc import round_time
+    from meerschaum.utils.dtypes import round_time
     begin, end = self.parse_date_bounds(begin, end)
     if not self.exists(debug=debug):
         return
@@ -623,7 +646,7 @@ def get_chunk_interval(
     if dt_col is None:
         return timedelta(minutes=chunk_minutes)
 
-    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns, UTC]')
+    dt_dtype = self.dtypes.get(dt_col, 'datetime')
     if 'int' in dt_dtype.lower():
         return chunk_minutes
     return timedelta(minutes=chunk_minutes)
@@ -687,11 +710,26 @@ def get_chunk_bounds(
         elif are_dtypes_equal(str(type(end)), 'int'):
             end += 1
             consolidate_end_chunk = True
+
     if begin is None and end is None:
         return [(None, None)]
 
     begin, end = self.parse_date_bounds(begin, end)
 
+    if begin and end:
+        if begin >= end:
+            return (
+                [(begin, begin)]
+                if bounded
+                else [(begin, None)]
+            )
+        if end <= begin:
+            return (
+                [(end, end)]
+                if bounded
+                else [(None, begin)]
+            )
+
     ### Set the chunk interval under `pipe.parameters['verify']['chunk_minutes']`.
     chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     
@@ -798,7 +836,7 @@ def parse_date_bounds(self, *dt_vals: Union[datetime, int, None]) -> Union[
     Given a date bound (begin, end), coerce a timezone if necessary.
     """
     from meerschaum.utils.misc import is_int
-    from meerschaum.utils.dtypes import coerce_timezone
+    from meerschaum.utils.dtypes import coerce_timezone, MRSM_PD_DTYPES
     from meerschaum.utils.warnings import warn
     dateutil_parser = mrsm.attempt_import('dateutil.parser')
 
@@ -823,9 +861,9 @@ def parse_date_bounds(self, *dt_vals: Union[datetime, int, None]) -> Union[
                 return None
 
         dt_col = self.columns.get('datetime', None)
-        dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
+        dt_typ = str(self.dtypes.get(dt_col, 'datetime'))
         if dt_typ == 'datetime':
-            dt_typ = 'datetime64[ns, UTC]'
+            dt_typ = MRSM_PD_DTYPES['datetime']
         return coerce_timezone(dt_val, strip_utc=('utc' not in dt_typ.lower()))
 
     bounds = tuple(_parse_date_bound(dt_val) for dt_val in dt_vals)

meerschaum/core/Pipe/_dtypes.py

@@ -22,6 +22,7 @@ def enforce_dtypes(
     chunksize: Optional[int] = -1,
     enforce: bool = True,
     safe_copy: bool = True,
+    dtypes: Optional[Dict[str, str]] = None,
     debug: bool = False,
 ) -> 'pd.DataFrame':
     """
@@ -50,7 +51,8 @@ def enforce_dtypes(
     if not self.enforce:
         enforce = False
 
-    pipe_dtypes = self.dtypes if enforce else {}
+    explicit_dtypes = self.get_dtypes(infer=False, debug=debug) if enforce else {}
+    pipe_dtypes = self.get_dtypes(infer=True, debug=debug) if not dtypes else dtypes
 
     try:
         if isinstance(df, str):
@@ -103,14 +105,21 @@ def enforce_dtypes(
     return _enforce_dtypes(
         df,
         pipe_dtypes,
+        explicit_dtypes=explicit_dtypes,
         safe_copy=safe_copy,
         strip_timezone=(self.tzinfo is None),
+        coerce_numeric=self.mixed_numerics,
         coerce_timezone=enforce,
         debug=debug,
     )
 
 
-def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str, Any]:
+def infer_dtypes(
+    self,
+    persist: bool = False,
+    refresh: bool = False,
+    debug: bool = False,
+) -> Dict[str, Any]:
     """
     If `dtypes` is not set in `meerschaum.Pipe.parameters`,
     infer the data types from the underlying table if it exists.
@@ -119,6 +128,11 @@ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str,
     ----------
     persist: bool, default False
         If `True`, persist the inferred data types to `meerschaum.Pipe.parameters`.
+        NOTE: Use with caution! Generally `dtypes` is meant to be user-configurable only.
+
+    refresh: bool, default False
+        If `True`, retrieve the latest columns-types for the pipe.
+        See `Pipe.get_columns.types()`.
 
     Returns
     -------
@@ -132,7 +146,7 @@ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str,
 
     ### NOTE: get_columns_types() may return either the types as
     ###       PostgreSQL- or Pandas-style.
-    columns_types = self.get_columns_types(debug=debug)
+    columns_types = self.get_columns_types(refresh=refresh, debug=debug)
 
     remote_pd_dtypes = {
         c: (
@@ -145,7 +159,8 @@ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str,
     if not persist:
         return remote_pd_dtypes
 
-    dtypes = self.parameters.get('dtypes', {})
+    parameters = self.get_parameters(refresh=refresh, debug=debug)
+    dtypes = parameters.get('dtypes', {})
     dtypes.update({
         col: typ
         for col, typ in remote_pd_dtypes.items()

meerschaum/core/Pipe/_edit.py

@@ -47,6 +47,8 @@ def edit(
     if self.temporary:
         return False, "Cannot edit pipes created with `temporary=True` (read-only)."
 
+    self._invalidate_cache(hard=True, debug=debug)
+
     if hasattr(self, '_symlinks'):
         from meerschaum.utils.misc import get_val_from_dict_path, set_val_in_dict_path
         for path, vals in self._symlinks.items():

meerschaum/core/Pipe/_fetch.py

@@ -127,7 +127,7 @@ def get_backtrack_interval(
     if dt_col is None:
         return backtrack_interval
 
-    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns, UTC]')
+    dt_dtype = self.dtypes.get(dt_col, 'datetime')
     if 'int' in dt_dtype.lower():
         return backtrack_minutes