meerschaum 2.9.4__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

meerschaum/core/Pipe/__init__.py

@@ -50,12 +50,16 @@ with correct credentials, as well as a network connection and valid permissions.
 """
 
 from __future__ import annotations
+
 import sys
 import copy
-from meerschaum.utils.typing import Optional, Dict, Any, Union, InstanceConnector, List
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Optional, Dict, Any, Union, List, InstanceConnector
 from meerschaum.utils.formatting._pipes import pipe_repr
 from meerschaum.config import get_config
 
+
 class Pipe:
     """
     Access Meerschaum pipes via Pipe objects.
@@ -89,6 +93,9 @@ class Pipe:
         get_data,
         get_backtrack_data,
         get_rowcount,
+        get_data,
+        get_doc,
+        get_value,
         _get_data_as_iterator,
         get_chunk_interval,
         get_chunk_bounds,
@@ -104,6 +111,7 @@ class Pipe:
         indexes,
         dtypes,
         autoincrement,
+        autotime,
         upsert,
         static,
         tzinfo,
@@ -113,6 +121,8 @@ class Pipe:
         get_columns_types,
         get_columns_indices,
         get_indices,
+        get_parameters,
+        update_parameters,
         tags,
         get_id,
         id,
@@ -168,6 +178,7 @@ class Pipe:
         temporary: bool = False,
         upsert: Optional[bool] = None,
         autoincrement: Optional[bool] = None,
+        autotime: Optional[bool] = None,
         static: Optional[bool] = None,
         enforce: Optional[bool] = None,
         null_indices: Optional[bool] = None,
@@ -226,6 +237,9 @@ class Pipe:
         autoincrement: Optional[bool], default None
             If `True`, set `autoincrement` in the parameters.
 
+        autotime: Optional[bool], default None
+            If `True`, set `autotime` in the parameters.
+
         static: Optional[bool], default None
             If `True`, set `static` in the parameters.
 
@@ -264,7 +278,7 @@ class Pipe:
         if location in ('[None]', 'None'):
             location = None
 
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
         negation_prefix = STATIC_CONFIG['system']['fetch_pipes_keys']['negation_prefix']
         for k in (connector, metric, location, *(tags or [])):
             if str(k).startswith(negation_prefix):
@@ -291,11 +305,13 @@ class Pipe:
                 warn(f"The provided parameters are of invalid type '{type(parameters)}'.")
             self._attributes['parameters'] = {}
 
-        columns = columns or self._attributes.get('parameters', {}).get('columns', {})
-        if isinstance(columns, list):
+        columns = columns or self._attributes.get('parameters', {}).get('columns', None)
+        if isinstance(columns, (list, tuple)):
             columns = {str(col): str(col) for col in columns}
         if isinstance(columns, dict):
             self._attributes['parameters']['columns'] = columns
+        elif isinstance(columns, str) and 'Pipe(' in columns:
+            pass
         elif columns is not None:
             warn(f"The provided columns are of invalid type '{type(columns)}'.")
 
@@ -334,6 +350,9 @@ class Pipe:
         if isinstance(autoincrement, bool):
             self._attributes['parameters']['autoincrement'] = autoincrement
 
+        if isinstance(autotime, bool):
+            self._attributes['parameters']['autotime'] = autotime
+
         if isinstance(static, bool):
             self._attributes['parameters']['static'] = static
 
@@ -383,9 +402,7 @@ class Pipe:
     @property
     def instance_connector(self) -> Union[InstanceConnector, None]:
         """
-        The connector to where this pipe resides.
-        May either be of type `meerschaum.connectors.sql.SQLConnector` or
-        `meerschaum.connectors.api.APIConnector`.
+        The instance connector on which this pipe resides.
         """
         if '_instance_connector' not in self.__dict__:
             from meerschaum.connectors.parse import parse_instance_keys
@@ -397,7 +414,7 @@ class Pipe:
         return self._instance_connector
 
     @property
-    def connector(self) -> Union[meerschaum.connectors.Connector, None]:
+    def connector(self) -> Union['Connector', None]:
         """
         The connector to the data source.
         """
@@ -417,7 +434,7 @@ class Pipe:
         return self._connector
 
     @property
-    def cache_connector(self) -> Union[meerschaum.connectors.sql.SQLConnector, None]:
+    def cache_connector(self) -> Union['Connector', None]:
         """
         If the pipe was created with `cache=True`, return the connector to the pipe's
         SQLite database for caching.
@@ -438,7 +455,7 @@ class Pipe:
         return self._cache_connector
 
     @property
-    def cache_pipe(self) -> Union['meerschaum.Pipe', None]:
+    def cache_pipe(self) -> Union[mrsm.Pipe, None]:
         """
         If the pipe was created with `cache=True`, return another `meerschaum.Pipe` used to
         manage the local data.
@@ -470,10 +487,10 @@ class Pipe:
                 self.instance_keys,
                 (self.connector_keys + '_' + self.metric_key + '_cache'),
                 self.location_key,
-                mrsm_instance = self.cache_connector,
-                parameters = _parameters,
-                cache = False,
-                temporary = True,
+                mrsm_instance=self.cache_connector,
+                parameters=_parameters,
+                cache=False,
+                temporary=True,
             )
 
         return self._cache_pipe

meerschaum/core/Pipe/_attributes.py

@@ -8,6 +8,7 @@ Fetch and manipulate Pipes' attributes
 
 from __future__ import annotations
 
+import uuid
 from datetime import timezone
 
 import meerschaum as mrsm
@@ -48,21 +49,74 @@ def attributes(self) -> Dict[str, Any]:
     return self._attributes
 
 
+def get_parameters(
+    self,
+    apply_symlinks: bool = True,
+    _visited: 'Optional[set[mrsm.Pipe]]' = None,
+) -> Dict[str, Any]:
+    """
+    Return the `parameters` dictionary of the pipe.
+    """
+    from meerschaum.config._patch import apply_patch_to_config
+    from meerschaum.utils.warnings import warn
+    from meerschaum.config._read_config import search_and_substitute_config
+
+    if _visited is None:
+        _visited = {self}
+
+    raw_parameters = self.attributes.get('parameters', {})
+    ref_keys = raw_parameters.get('reference')
+    if not apply_symlinks:
+        return raw_parameters
+
+    if ref_keys:
+        try:
+            ref_pipe = mrsm.Pipe(**ref_keys)
+            if ref_pipe in _visited:
+                warn(f"Circular reference detected in {self}: chain involves {ref_pipe}.")
+                return search_and_substitute_config(raw_parameters)
+
+            _visited.add(ref_pipe)
+            base_params = ref_pipe.get_parameters(_visited=_visited)
+        except Exception as e:
+            warn(f"Failed to resolve reference pipe for {self}: {e}")
+            base_params = {}
+
+        params_to_apply = {k: v for k, v in raw_parameters.items() if k != 'reference'}
+        parameters = apply_patch_to_config(base_params, params_to_apply)
+    else:
+        parameters = raw_parameters
+
+    from meerschaum.utils.pipes import replace_pipes_syntax
+    self._symlinks = {}
+
+    def recursive_replace(obj: Any, path: tuple) -> Any:
+        if isinstance(obj, dict):
+            return {k: recursive_replace(v, path + (k,)) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [recursive_replace(elem, path + (i,)) for i, elem in enumerate(obj)]
+        if isinstance(obj, str):
+            substituted_val = replace_pipes_syntax(obj)
+            if substituted_val != obj:
+                self._symlinks[path] = {
+                    'original': obj,
+                    'substituted': substituted_val,
+                }
+            return substituted_val
+        return obj
+
+    return search_and_substitute_config(recursive_replace(parameters, tuple()))
+
+
 @property
 def parameters(self) -> Optional[Dict[str, Any]]:
     """
     Return the parameters dictionary of the pipe.
     """
-    if 'parameters' not in self.attributes:
-        self.attributes['parameters'] = {}
-    _parameters = self.attributes['parameters']
-    dt_col = _parameters.get('columns', {}).get('datetime', None)
-    dt_typ = _parameters.get('dtypes', {}).get(dt_col, None) if dt_col else None
-    if dt_col and not dt_typ:
-        if 'dtypes' not in _parameters:
-            self.attributes['parameters']['dtypes'] = {}
-        self.attributes['parameters']['dtypes'][dt_col] = 'datetime'
-    return self.attributes['parameters']
+    if (_parameters := self.__dict__.get('_parameters', None)) is not None:
+        return _parameters
+    self._parameters = self.get_parameters()
+    return self._parameters
 
 
 @parameters.setter
@@ -71,7 +125,9 @@ def parameters(self, parameters: Dict[str, Any]) -> None:
     Set the parameters dictionary of the in-memory pipe.
     Call `meerschaum.Pipe.edit()` to persist changes.
     """
-    self.attributes['parameters'] = parameters
+    self._attributes['parameters'] = parameters
+    if '_parameters' in self.__dict__:
+        del self.__dict__['_parameters']
 
 
 @property
@@ -79,12 +135,9 @@ def columns(self) -> Union[Dict[str, str], None]:
     """
     Return the `columns` dictionary defined in `meerschaum.Pipe.parameters`.
     """
-    if 'columns' not in self.parameters:
-        self.parameters['columns'] = {}
-    cols = self.parameters['columns']
+    cols = self.parameters.get('columns', {})
     if not isinstance(cols, dict):
-        cols = {}
-        self.parameters['columns'] = cols
+        return {}
     return {col_ix: col for col_ix, col in cols.items() if col}
 
 
@@ -99,7 +152,7 @@ def columns(self, _columns: Union[Dict[str, str], List[str]]) -> None:
     if not isinstance(_columns, dict):
         warn(f"{self}.columns must be a dictionary, received {type(_columns)}.")
         return
-    self.parameters['columns'] = _columns
+    self.update_parameters({'columns': _columns}, persist=False)
 
 
 @property
@@ -158,7 +211,7 @@ def indices(self, _indices: Union[Dict[str, Union[str, List[str]]], List[str]])
         if 'indexes' in self.parameters
         else 'indices'
     )
-    self.parameters[indices_key] = _indices
+    self.update_parameters({indices_key: _indices}, persist=False)
 
 
 @indexes.setter
@@ -174,24 +227,22 @@ def tags(self) -> Union[List[str], None]:
     """
     If defined, return the `tags` list defined in `meerschaum.Pipe.parameters`.
     """
-    if 'tags' not in self.parameters:
-        self.parameters['tags'] = []
-    return self.parameters['tags']
+    return self.parameters.get('tags', [])
 
 
 @tags.setter
-def tags(self, _tags: List[str, str]) -> None:
+def tags(self, _tags: List[str]) -> None:
     """
     Override the tags list of the in-memory pipe.
     Call `meerschaum.Pipe.edit` to persist changes.
     """
     from meerschaum.utils.warnings import error
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     negation_prefix = STATIC_CONFIG['system']['fetch_pipes_keys']['negation_prefix']
     for t in _tags:
         if t.startswith(negation_prefix):
             error(f"Tags cannot begin with '{negation_prefix}'.")
-    self.parameters['tags'] = _tags
+    self.update_parameters({'tags': _tags}, persist=False)
 
 
 @property
@@ -204,11 +255,18 @@ def dtypes(self) -> Union[Dict[str, Any], None]:
     configured_dtypes = self.parameters.get('dtypes', {})
     remote_dtypes = self.infer_dtypes(persist=False)
     patched_dtypes = apply_patch_to_config(remote_dtypes, configured_dtypes)
-    return {
+    dt_col = self.columns.get('datetime', None)
+    primary_col = self.columns.get('primary', None)
+    _dtypes = {
         col: MRSM_ALIAS_DTYPES.get(typ, typ)
         for col, typ in patched_dtypes.items()
         if col and typ
     }
+    if dt_col and dt_col not in configured_dtypes:
+        _dtypes[dt_col] = 'datetime'
+    if primary_col and self.autoincrement and primary_col not in _dtypes:
+        _dtypes[primary_col] = 'int'
+    return _dtypes
 
 
 @dtypes.setter
@@ -217,7 +275,7 @@ def dtypes(self, _dtypes: Dict[str, Any]) -> None:
     Override the dtypes dictionary of the in-memory pipe.
     Call `meerschaum.Pipe.edit()` to persist changes.
     """
-    self.parameters['dtypes'] = _dtypes
+    self.update_parameters({'dtypes': _dtypes}, persist=False)
 
 
 @property
@@ -225,9 +283,7 @@ def upsert(self) -> bool:
     """
     Return whether `upsert` is set for the pipe.
     """
-    if 'upsert' not in self.parameters:
-        self.parameters['upsert'] = False
-    return self.parameters['upsert']
+    return self.parameters.get('upsert', False)
 
 
 @upsert.setter
@@ -235,7 +291,7 @@ def upsert(self, _upsert: bool) -> None:
     """
     Set the `upsert` parameter for the pipe.
     """
-    self.parameters['upsert'] = _upsert
+    self.update_parameters({'upsert': _upsert}, persist=False)
 
 
 @property
@@ -243,9 +299,7 @@ def static(self) -> bool:
     """
     Return whether `static` is set for the pipe.
     """
-    if 'static' not in self.parameters:
-        self.parameters['static'] = False
-    return self.parameters['static']
+    return self.parameters.get('static', False)
 
 
 @static.setter
@@ -253,7 +307,7 @@ def static(self, _static: bool) -> None:
     """
     Set the `static` parameter for the pipe.
     """
-    self.parameters['static'] = _static
+    self.update_parameters({'static': _static}, persist=False)
 
 
 @property
@@ -261,10 +315,7 @@ def autoincrement(self) -> bool:
     """
     Return the `autoincrement` parameter for the pipe.
     """
-    if 'autoincrement' not in self.parameters:
-        self.parameters['autoincrement'] = False
-
-    return self.parameters['autoincrement']
+    return self.parameters.get('autoincrement', False)
 
 
 @autoincrement.setter
@@ -272,7 +323,23 @@ def autoincrement(self, _autoincrement: bool) -> None:
     """
     Set the `autoincrement` parameter for the pipe.
     """
-    self.parameters['autoincrement'] = _autoincrement
+    self.update_parameters({'autoincrement': _autoincrement}, persist=False)
+
+
+@property
+def autotime(self) -> bool:
+    """
+    Return the `autotime` parameter for the pipe.
+    """
+    return self.parameters.get('autotime', False)
+
+
+@autotime.setter
+def autotime(self, _autotime: bool) -> None:
+    """
+    Set the `autotime` parameter for the pipe.
+    """
+    self.update_parameters({'autotime': _autotime}, persist=False)
 
 
 @property
@@ -299,10 +366,7 @@ def enforce(self) -> bool:
     """
     Return the `enforce` parameter for the pipe.
     """
-    if 'enforce' not in self.parameters:
-        self.parameters['enforce'] = True
-
-    return self.parameters['enforce']
+    return self.parameters.get('enforce', True)
 
 
 @enforce.setter
@@ -310,7 +374,7 @@ def enforce(self, _enforce: bool) -> None:
     """
     Set the `enforce` parameter for the pipe.
     """
-    self.parameters['enforce'] = _enforce
+    self.update_parameters({'enforce': _enforce}, persist=False)
 
 
 @property
@@ -318,10 +382,7 @@ def null_indices(self) -> bool:
     """
     Return the `null_indices` parameter for the pipe.
     """
-    if 'null_indices' not in self.parameters:
-        self.parameters['null_indices'] = True
-
-    return self.parameters['null_indices']
+    return self.parameters.get('null_indices', True)
 
 
 @null_indices.setter
@@ -329,7 +390,7 @@ def null_indices(self, _null_indices: bool) -> None:
     """
     Set the `null_indices` parameter for the pipe.
     """
-    self.parameters['null_indices'] = _null_indices
+    self.update_parameters({'null_indices': _null_indices}, persist=False)
 
 
 def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]:
@@ -357,7 +418,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
     >>> pipe.get_columns('value', error=True)
     Exception:  🛑 Missing 'value' column for Pipe('test', 'test').
     """
-    from meerschaum.utils.warnings import error as _error, warn
+    from meerschaum.utils.warnings import error as _error
     if not args:
         args = tuple(self.columns.keys())
     col_names = []
@@ -367,7 +428,7 @@ def get_columns(self, *args: str, error: bool = False) -> Union[str, Tuple[str]]
             col_name = self.columns[col]
             if col_name is None and error:
                 _error(f"Please define the name of the '{col}' column for {self}.")
-        except Exception as e:
+        except Exception:
             col_name = None
         if col_name is None and error:
             _error(f"Missing '{col}'" + f" column for {self}.")
@@ -409,7 +470,7 @@ def get_columns_types(
     """
     import time
     from meerschaum.connectors import get_connector_plugin
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.warnings import dprint
 
     now = time.perf_counter()
@@ -454,7 +515,7 @@ def get_columns_indices(
     """
     import time
     from meerschaum.connectors import get_connector_plugin
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.warnings import dprint
 
     now = time.perf_counter()
@@ -509,7 +570,7 @@ def get_id(self, **kw: Any) -> Union[int, None]:
 
 
 @property
-def id(self) -> Union[int, None]:
+def id(self) -> Union[int, str, uuid.UUID, None]:
     """
     Fetch and cache a pipe's ID.
     """
@@ -539,7 +600,7 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
         dprint('Attempting to determine the value column...')
     try:
         val_name = self.get_columns('value')
-    except Exception as e:
+    except Exception:
         val_name = None
     if val_name is not None:
         if debug:
@@ -553,11 +614,11 @@ def get_val_column(self, debug: bool = False) -> Union[str, None]:
         return None
     try:
         dt_name = self.get_columns('datetime', error=False)
-    except Exception as e:
+    except Exception:
         dt_name = None
     try:
         id_name = self.get_columns('id', errors=False)
-    except Exception as e:
+    except Exception:
         id_name = None
 
     if debug:
@@ -714,7 +775,7 @@ def target(self, _target: str) -> None:
     Override the target of the in-memory pipe.
     Call `meerschaum.Pipe.edit` to persist changes.
     """
-    self.parameters['target'] = _target
+    self.update_parameters({'target': _target}, persist=False)
 
 
 def guess_datetime(self) -> Union[str, None]:
@@ -755,3 +816,40 @@ def get_indices(self) -> Dict[str, str]:
             result = {}
     
     return result
+
+
+def update_parameters(
+    self,
+    parameters_patch: Dict[str, Any],
+    persist: bool = True,
+    debug: bool = False,
+) -> mrsm.SuccessTuple:
+    """
+    Apply a patch to a pipe's `parameters` dictionary.
+
+    Parameters
+    ----------
+    parameters_patch: Dict[str, Any]
+        The patch to be applied to `Pipe.parameters`.
+
+    persist: bool, default True
+        If `True`, call `Pipe.edit()` to persist the new parameters.
+    """
+    from meerschaum.config import apply_patch_to_config
+    if '_parameters' in self.__dict__:
+        del self.__dict__['_parameters']
+    if 'parameters' not in self._attributes:
+        self._attributes['parameters'] = {}
+
+    self._attributes['parameters'] = apply_patch_to_config(
+        self._attributes['parameters'],
+        parameters_patch,
+    )
+
+    if self.temporary:
+        persist = False
+
+    if not persist:
+        return True, "Success"
+
+    return self.edit(debug=debug)

meerschaum/core/Pipe/_bootstrap.py

@@ -7,6 +7,8 @@ Attempt to create a pipe's requirements in one method.
 """
 
 from __future__ import annotations
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Dict, Any
 
 
@@ -207,28 +209,56 @@ def _ask_for_columns(pipe, debug: bool=False) -> Dict[str, str]:
     """
     Prompt the user for the column names.
     """
-    from meerschaum.utils.warnings import info, warn
-    from meerschaum.utils.prompt import prompt
+    import json
+    from meerschaum.utils.warnings import info
+    from meerschaum.utils.prompt import prompt, yes_no
+    from meerschaum.utils.formatting import get_console
+    from meerschaum.utils.formatting._shell import clear_screen
+    from meerschaum.utils.misc import to_snake_case
+    from meerschaum.config import get_config
+    rich_json = mrsm.attempt_import('rich.json')
 
-    info(f"Please enter column names for {pipe}:")
-    while True:
-        try:
-            datetime_name = prompt(f"Datetime column (empty to omit):", icon=False)
-        except KeyboardInterrupt:
-            return False, f"Cancelled bootstrapping {pipe}."
-        if datetime_name == '':
-            datetime_name = None
-
-        try:
-            id_name = prompt(f"ID column (empty to omit):", icon=False)
-        except KeyboardInterrupt:
-            return False, f"Cancelled bootstrapping {pipe}."
-        if id_name == '':
-            id_name = None
-
-        break
-
-    return {
-        'datetime': datetime_name,
-        'id': id_name,
-    }
+    do_clear = get_config('shell', 'clear_screen')
+
+    cols = {}
+
+    info(f"Please enter index columns for {pipe}:")
+    try:
+        datetime_name = prompt("Datetime column (empty to omit):", icon=False)
+    except KeyboardInterrupt:
+        datetime_name = None
+
+    if datetime_name:
+        cols['datetime'] = datetime_name
+
+    try:
+        id_name = prompt("ID column (empty to omit):", icon=False)
+    except KeyboardInterrupt:
+        id_name = None
+
+    if id_name:
+        cols['id'] = id_name
+
+    if yes_no("Add more columns?"):
+        while True:
+            if do_clear:
+                clear_screen(debug=debug)
+
+            cols_text = json.dumps(cols, indent=4)
+            info("Current index columns:")
+            get_console().print(rich_json.JSON(cols_text))
+
+            col_name = prompt("Enter index column (empty to stop):")
+            if not col_name:
+                break
+
+            if col_name in cols.values():
+                continue
+
+            col_ix = to_snake_case(col_name)
+            if col_ix in cols:
+                col_ix = col_ix + '_'
+
+            cols[col_ix] = col_name
+
+    return cols