meerschaum 2.9.5__py3-none-any.whl → 3.0.0rc2__py3-none-any.whl

meerschaum/core/Pipe/_bootstrap.py

@@ -7,6 +7,8 @@ Attempt to create a pipe's requirements in one method.
 """
 
 from __future__ import annotations
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Dict, Any
 
 
@@ -207,28 +209,56 @@ def _ask_for_columns(pipe, debug: bool=False) -> Dict[str, str]:
     """
     Prompt the user for the column names.
     """
-    from meerschaum.utils.warnings import info, warn
-    from meerschaum.utils.prompt import prompt
+    import json
+    from meerschaum.utils.warnings import info
+    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum.utils.formatting import get_console
+    from meerschaum.utils.formatting._shell import clear_screen
+    from meerschaum.utils.misc import to_snake_case
+    from meerschaum.config import get_config
+    rich_json = mrsm.attempt_import('rich.json')
 
-    info(f"Please enter column names for {pipe}:")
-    while True:
-        try:
-            datetime_name = prompt(f"Datetime column (empty to omit):", icon=False)
-        except KeyboardInterrupt:
-            return False, f"Cancelled bootstrapping {pipe}."
-        if datetime_name == '':
-            datetime_name = None
-
-        try:
-            id_name = prompt(f"ID column (empty to omit):", icon=False)
-        except KeyboardInterrupt:
-            return False, f"Cancelled bootstrapping {pipe}."
-        if id_name == '':
-            id_name = None
-
-        break
-
-    return {
-        'datetime': datetime_name,
-        'id': id_name,
-    }
+    do_clear = get_config('shell', 'clear_screen')
+
+    cols = {}
+
+    info(f"Please enter index columns for {pipe}:")
+    try:
+        datetime_name = prompt("Datetime column (empty to omit):", icon=False)
+    except KeyboardInterrupt:
+        datetime_name = None
+
+    if datetime_name:
+        cols['datetime'] = datetime_name
+
+    try:
+        id_name = prompt("ID column (empty to omit):", icon=False)
+    except KeyboardInterrupt:
+        id_name = None
+
+    if id_name:
+        cols['id'] = id_name
+
+    if yes_no("Add more columns?"):
+        while True:
+            if do_clear:
+                clear_screen(debug=debug)
+
+            cols_text = json.dumps(cols, indent=4)
+            info("Current index columns:")
+            get_console().print(rich_json.JSON(cols_text))
+
+            col_name = prompt("Enter index column (empty to stop):")
+            if not col_name:
+                break
+
+            if col_name in cols.values():
+                continue
+
+            col_ix = to_snake_case(col_name)
+            if col_ix in cols:
+                col_ix = col_ix + '_'
+
+            cols[col_ix] = col_name
+
+    return cols

meerschaum/core/Pipe/_data.py

@@ -29,6 +29,7 @@ def get_data(
     as_iterator: bool = False,
     as_chunks: bool = False,
     as_dask: bool = False,
+    add_missing_columns: bool = False,
     chunk_interval: Union[timedelta, int, None] = None,
     order: Optional[str] = 'asc',
     limit: Optional[int] = None,
@@ -72,6 +73,9 @@ def get_data(
         If `True`, return a `dask.DataFrame`
         (which may be loaded into a Pandas DataFrame with `df.compute()`).
 
+    add_missing_columns: bool, default False
+        If `True`, add any missing columns from `Pipe.dtypes` to the dataframe.
+
     chunk_interval: Union[timedelta, int, None], default None
         If `as_iterator`, then return chunks with `begin` and `end` separated by this interval.
         This may be set under `pipe.parameters['chunk_minutes']`.
@@ -103,13 +107,13 @@ def get_data(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
-    from meerschaum.utils.misc import iterate_chunks, items_str
-    from meerschaum.utils.dtypes import to_pandas_dtype, coerce_timezone
+    from meerschaum.utils.dtypes import to_pandas_dtype
     from meerschaum.utils.dataframe import add_missing_cols_to_df, df_is_chunk_generator
     from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.warnings import dprint
     dd = attempt_import('dask.dataframe') if as_dask else None
     dask = attempt_import('dask') if as_dask else None
-    dateutil_parser = attempt_import('dateutil.parser')
+    _ = attempt_import('partd', lazy=False) if as_dask else None
 
     if select_columns == '*':
         select_columns = None
@@ -188,14 +192,17 @@ def get_data(
                 order=order,
                 limit=limit,
                 fresh=fresh,
+                add_missing_columns=True,
                 debug=debug,
             )
             for (chunk_begin, chunk_end) in bounds
         ]
         dask_meta = {
             col: to_pandas_dtype(typ)
-            for col, typ in self.dtypes.items()
+            for col, typ in self.get_dtypes(refresh=True, infer=True, debug=debug).items()
         }
+        if debug:
+            dprint(f"Dask meta:\n{dask_meta}")
         return _sort_df(dd.from_delayed(dask_chunks, meta=dask_meta))
 
     if not self.exists(debug=debug):
@@ -249,6 +256,7 @@ def get_data(
         if not select_columns:
             select_columns = [col for col in df.columns]
 
+        pipe_dtypes = self.get_dtypes(refresh=False, debug=debug)
         cols_to_omit = [
             col
             for col in df.columns
@@ -262,7 +270,11 @@ def get_data(
             col
             for col in select_columns
             if col not in df.columns
-        ]
+        ] + ([
+            col
+            for col in pipe_dtypes
+            if col not in df.columns
+        ] if add_missing_columns else [])
         if cols_to_omit:
             warn(
                 (
@@ -278,16 +290,26 @@ def get_data(
             df = df[_cols_to_select]
 
         if cols_to_add:
-            warn(
-                (
-                    f"Specified columns {items_str(cols_to_add)} were not found on {self}. "
-                    + "Adding these to the DataFrame as null columns."
-                ),
-                stack=False,
+            if not add_missing_columns:
+                from meerschaum.utils.misc import items_str
+                warn(
+                    f"Will add columns {items_str(cols_to_add)} as nulls to dataframe.",
+                    stack=False,
+                )
+
+            df = add_missing_cols_to_df(
+                df,
+                {
+                    col: pipe_dtypes.get(col, 'string')
+                    for col in cols_to_add
+                },
             )
-            df = add_missing_cols_to_df(df, {col: 'string' for col in cols_to_add})
 
-        enforced_df = self.enforce_dtypes(df, debug=debug)
+        enforced_df = self.enforce_dtypes(
+            df,
+            dtypes=pipe_dtypes,
+            debug=debug,
+        )
 
         if order:
             return _sort_df(enforced_df)
@@ -311,7 +333,7 @@ def _get_data_as_iterator(
     """
     Return a pipe's data as a generator.
     """
-    from meerschaum.utils.misc import round_time
+    from meerschaum.utils.dtypes import round_time
     begin, end = self.parse_date_bounds(begin, end)
     if not self.exists(debug=debug):
         return
@@ -624,7 +646,7 @@ def get_chunk_interval(
     if dt_col is None:
         return timedelta(minutes=chunk_minutes)
 
-    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns, UTC]')
+    dt_dtype = self.dtypes.get(dt_col, 'datetime')
     if 'int' in dt_dtype.lower():
         return chunk_minutes
     return timedelta(minutes=chunk_minutes)
@@ -688,11 +710,26 @@ def get_chunk_bounds(
         elif are_dtypes_equal(str(type(end)), 'int'):
             end += 1
             consolidate_end_chunk = True
+
     if begin is None and end is None:
         return [(None, None)]
 
     begin, end = self.parse_date_bounds(begin, end)
 
+    if begin and end:
+        if begin >= end:
+            return (
+                [(begin, begin)]
+                if bounded
+                else [(begin, None)]
+            )
+        if end <= begin:
+            return (
+                [(end, end)]
+                if bounded
+                else [(None, begin)]
+            )
+
     ### Set the chunk interval under `pipe.parameters['verify']['chunk_minutes']`.
     chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     
@@ -799,7 +836,7 @@ def parse_date_bounds(self, *dt_vals: Union[datetime, int, None]) -> Union[
     Given a date bound (begin, end), coerce a timezone if necessary.
     """
     from meerschaum.utils.misc import is_int
-    from meerschaum.utils.dtypes import coerce_timezone
+    from meerschaum.utils.dtypes import coerce_timezone, MRSM_PD_DTYPES
     from meerschaum.utils.warnings import warn
     dateutil_parser = mrsm.attempt_import('dateutil.parser')
 
@@ -824,12 +861,53 @@ def parse_date_bounds(self, *dt_vals: Union[datetime, int, None]) -> Union[
                 return None
 
         dt_col = self.columns.get('datetime', None)
-        dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
+        dt_typ = str(self.dtypes.get(dt_col, 'datetime'))
         if dt_typ == 'datetime':
-            dt_typ = 'datetime64[ns, UTC]'
+            dt_typ = MRSM_PD_DTYPES['datetime']
         return coerce_timezone(dt_val, strip_utc=('utc' not in dt_typ.lower()))
 
     bounds = tuple(_parse_date_bound(dt_val) for dt_val in dt_vals)
     if len(bounds) == 1:
         return bounds[0]
     return bounds
+
+
+def get_doc(self, **kwargs) -> Union[Dict[str, Any], None]:
+    """
+    Convenience function to return a single row as a dictionary (or `None`) from `Pipe.get_data().
+    Keywords arguments are passed to `Pipe.get_data()`.
+    """
+    from meerschaum.utils.warnings import warn
+    kwargs['limit'] = 1
+    try:
+        result_df = self.get_data(**kwargs)
+        if result_df is None or len(result_df) == 0:
+            return None
+        return result_df.reset_index(drop=True).iloc[0].to_dict()
+    except Exception as e:
+        warn(f"Failed to read value from {self}:\n{e}", stack=False)
+        return None
+
+def get_value(
+    self,
+    column: str,
+    params: Optional[Dict[str, Any]] = None,
+    **kwargs: Any
+) -> Any:
+    """
+    Convenience function to return a single value (or `None`) from `Pipe.get_data()`.
+    Keywords arguments are passed to `Pipe.get_data()`.
+    """
+    from meerschaum.utils.warnings import warn
+    kwargs['select_columns'] = [column]
+    kwargs['limit'] = 1
+    try:
+        result_df = self.get_data(params=params, **kwargs)
+        if result_df is None or len(result_df) == 0:
+            return None
+        if column not in result_df.columns:
+            raise ValueError(f"Column '{column}' was not included in the result set.")
+        return result_df[column][0]
+    except Exception as e:
+        warn(f"Failed to read value from {self}:\n{e}", stack=False)
+        return None

meerschaum/core/Pipe/_dtypes.py

@@ -22,6 +22,7 @@ def enforce_dtypes(
     chunksize: Optional[int] = -1,
     enforce: bool = True,
     safe_copy: bool = True,
+    dtypes: Optional[Dict[str, str]] = None,
     debug: bool = False,
 ) -> 'pd.DataFrame':
     """
@@ -31,7 +32,11 @@ def enforce_dtypes(
     import traceback
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.dataframe import parse_df_datetimes, enforce_dtypes as _enforce_dtypes
+    from meerschaum.utils.dataframe import (
+        parse_df_datetimes,
+        enforce_dtypes as _enforce_dtypes,
+        parse_simple_lines,
+    )
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.packages import import_pandas
     pd = import_pandas(debug=debug)
@@ -45,23 +50,35 @@ def enforce_dtypes(
 
     if not self.enforce:
         enforce = False
-    pipe_dtypes = self.dtypes if enforce else {}
+
+    explicit_dtypes = self.get_dtypes(infer=False, debug=debug) if enforce else {}
+    pipe_dtypes = self.get_dtypes(infer=True, debug=debug) if not dtypes else dtypes
 
     try:
         if isinstance(df, str):
-            df = parse_df_datetimes(
-                pd.read_json(StringIO(df)),
-                ignore_cols=[
-                    col
-                    for col, dtype in pipe_dtypes.items()
-                    if (not enforce or not are_dtypes_equal(dtype, 'datetime'))
-                ],
-                ignore_all=(not enforce),
-                strip_timezone=(self.tzinfo is None),
-                chunksize=chunksize,
-                debug=debug,
-            )
-        elif isinstance(df, (dict, list)):
+            if df.strip() and df.strip()[0] not in ('{', '['):
+                df = parse_df_datetimes(
+                    parse_simple_lines(df),
+                    ignore_cols=[
+                        col
+                        for col, dtype in pipe_dtypes.items()
+                        if (not enforce or not are_dtypes_equal(dtype, 'datetime'))
+                    ],
+                )
+            else:
+                df = parse_df_datetimes(
+                    pd.read_json(StringIO(df)),
+                    ignore_cols=[
+                        col
+                        for col, dtype in pipe_dtypes.items()
+                        if (not enforce or not are_dtypes_equal(dtype, 'datetime'))
+                    ],
+                    ignore_all=(not enforce),
+                    strip_timezone=(self.tzinfo is None),
+                    chunksize=chunksize,
+                    debug=debug,
+                )
+        elif isinstance(df, (dict, list, tuple)):
             df = parse_df_datetimes(
                 df,
                 ignore_cols=[
@@ -88,14 +105,21 @@ def enforce_dtypes(
     return _enforce_dtypes(
         df,
         pipe_dtypes,
+        explicit_dtypes=explicit_dtypes,
         safe_copy=safe_copy,
         strip_timezone=(self.tzinfo is None),
+        coerce_numeric=self.mixed_numerics,
         coerce_timezone=enforce,
         debug=debug,
     )
 
 
-def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str, Any]:
+def infer_dtypes(
+    self,
+    persist: bool = False,
+    refresh: bool = False,
+    debug: bool = False,
+) -> Dict[str, Any]:
     """
     If `dtypes` is not set in `meerschaum.Pipe.parameters`,
     infer the data types from the underlying table if it exists.
@@ -104,6 +128,11 @@ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str,
     ----------
     persist: bool, default False
         If `True`, persist the inferred data types to `meerschaum.Pipe.parameters`.
+        NOTE: Use with caution! Generally `dtypes` is meant to be user-configurable only.
+
+    refresh: bool, default False
+        If `True`, retrieve the latest columns-types for the pipe.
+        See `Pipe.get_columns.types()`.
 
     Returns
     -------
@@ -117,7 +146,7 @@ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str,
 
     ### NOTE: get_columns_types() may return either the types as
     ###       PostgreSQL- or Pandas-style.
-    columns_types = self.get_columns_types(debug=debug)
+    columns_types = self.get_columns_types(refresh=refresh, debug=debug)
 
     remote_pd_dtypes = {
         c: (
@@ -130,7 +159,8 @@ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str,
     if not persist:
         return remote_pd_dtypes
 
-    dtypes = self.parameters.get('dtypes', {})
+    parameters = self.get_parameters(refresh=refresh, debug=debug)
+    dtypes = parameters.get('dtypes', {})
     dtypes.update({
         col: typ
         for col, typ in remote_pd_dtypes.items()

meerschaum/core/Pipe/_edit.py

@@ -3,7 +3,7 @@
 # vim:fenc=utf-8
 
 """
-Edit a Pipe's parameters here.
+Edit a Pipe's parameters.
 """
 
 from __future__ import annotations
@@ -47,6 +47,15 @@ def edit(
     if self.temporary:
         return False, "Cannot edit pipes created with `temporary=True` (read-only)."
 
+    self._invalidate_cache(hard=True, debug=debug)
+
+    if hasattr(self, '_symlinks'):
+        from meerschaum.utils.misc import get_val_from_dict_path, set_val_in_dict_path
+        for path, vals in self._symlinks.items():
+            current_val = get_val_from_dict_path(self.parameters, path)
+            if current_val == vals['substituted']:
+                set_val_in_dict_path(self.parameters, path, vals['original'])
+
     if not interactive:
         with Venv(get_connector_plugin(self.instance_connector)):
             return self.instance_connector.edit_pipe(self, patch=patch, debug=debug, **kw)
@@ -65,7 +74,8 @@ def edit(
     from meerschaum.config import get_config
     parameters = dict(get_config('pipes', 'parameters', patch=True))
     from meerschaum.config._patch import apply_patch_to_config
-    parameters = apply_patch_to_config(parameters, self.parameters)
+    raw_parameters = self.attributes.get('parameters', {})
+    parameters = apply_patch_to_config(parameters, raw_parameters)
 
     ### write parameters to yaml file
     with open(parameters_path, 'w+') as f:
@@ -194,7 +204,7 @@ def edit_definition(
         return True, "Success"
 
     def _edit_sql():
-        import pathlib, os, textwrap
+        import textwrap
         from meerschaum.config._paths import PIPES_CACHE_RESOURCES_PATH
         from meerschaum.utils.misc import edit_file
         definition_filename = str(self) + '.sql'
@@ -214,7 +224,7 @@ def edit_definition(
 
         edit_file(definition_path)
         try:
-            with open(definition_path, 'r') as f:
+            with open(definition_path, 'r', encoding='utf-8') as f:
                 file_definition = f.read()
         except Exception as e:
             return False, f"Failed reading file '{definition_path}':\n" + str(e)

meerschaum/core/Pipe/_fetch.py

@@ -127,7 +127,7 @@ def get_backtrack_interval(
     if dt_col is None:
         return backtrack_interval
 
-    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns, UTC]')
+    dt_dtype = self.dtypes.get(dt_col, 'datetime')
     if 'int' in dt_dtype.lower():
         return backtrack_minutes
 

meerschaum/core/Pipe/_show.py

@@ -9,11 +9,11 @@ Show information about a Pipe
 from meerschaum.utils.typing import SuccessTuple
 
 def show(
-        self,
-        nopretty: bool = False,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    nopretty: bool = False,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Show attributes of a Pipe.