meerschaum 2.9.4__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

meerschaum/core/Pipe/_data.py

@@ -109,7 +109,6 @@ def get_data(
     from meerschaum.utils.packages import attempt_import
     dd = attempt_import('dask.dataframe') if as_dask else None
     dask = attempt_import('dask') if as_dask else None
-    dateutil_parser = attempt_import('dateutil.parser')
 
     if select_columns == '*':
         select_columns = None
@@ -833,3 +832,44 @@ def parse_date_bounds(self, *dt_vals: Union[datetime, int, None]) -> Union[
     if len(bounds) == 1:
         return bounds[0]
     return bounds
+
+
+def get_doc(self, **kwargs) -> Union[Dict[str, Any], None]:
+    """
+    Convenience function to return a single row as a dictionary (or `None`) from `Pipe.get_data().
+    Keywords arguments are passed to `Pipe.get_data()`.
+    """
+    from meerschaum.utils.warnings import warn
+    kwargs['limit'] = 1
+    try:
+        result_df = self.get_data(**kwargs)
+        if result_df is None or len(result_df) == 0:
+            return None
+        return result_df.reset_index(drop=True).iloc[0].to_dict()
+    except Exception as e:
+        warn(f"Failed to read value from {self}:\n{e}", stack=False)
+        return None
+
+def get_value(
+    self,
+    column: str,
+    params: Optional[Dict[str, Any]] = None,
+    **kwargs: Any
+) -> Any:
+    """
+    Convenience function to return a single value (or `None`) from `Pipe.get_data()`.
+    Keywords arguments are passed to `Pipe.get_data()`.
+    """
+    from meerschaum.utils.warnings import warn
+    kwargs['select_columns'] = [column]
+    kwargs['limit'] = 1
+    try:
+        result_df = self.get_data(params=params, **kwargs)
+        if result_df is None or len(result_df) == 0:
+            return None
+        if column not in result_df.columns:
+            raise ValueError(f"Column '{column}' was not included in the result set.")
+        return result_df[column][0]
+    except Exception as e:
+        warn(f"Failed to read value from {self}:\n{e}", stack=False)
+        return None

meerschaum/core/Pipe/_dtypes.py

@@ -31,7 +31,11 @@ def enforce_dtypes(
     import traceback
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
-    from meerschaum.utils.dataframe import parse_df_datetimes, enforce_dtypes as _enforce_dtypes
+    from meerschaum.utils.dataframe import (
+        parse_df_datetimes,
+        enforce_dtypes as _enforce_dtypes,
+        parse_simple_lines,
+    )
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.packages import import_pandas
     pd = import_pandas(debug=debug)
@@ -45,23 +49,34 @@ def enforce_dtypes(
 
     if not self.enforce:
         enforce = False
+
     pipe_dtypes = self.dtypes if enforce else {}
 
     try:
         if isinstance(df, str):
-            df = parse_df_datetimes(
-                pd.read_json(StringIO(df)),
-                ignore_cols=[
-                    col
-                    for col, dtype in pipe_dtypes.items()
-                    if (not enforce or not are_dtypes_equal(dtype, 'datetime'))
-                ],
-                ignore_all=(not enforce),
-                strip_timezone=(self.tzinfo is None),
-                chunksize=chunksize,
-                debug=debug,
-            )
-        elif isinstance(df, (dict, list)):
+            if df.strip() and df.strip()[0] not in ('{', '['):
+                df = parse_df_datetimes(
+                    parse_simple_lines(df),
+                    ignore_cols=[
+                        col
+                        for col, dtype in pipe_dtypes.items()
+                        if (not enforce or not are_dtypes_equal(dtype, 'datetime'))
+                    ],
+                )
+            else:
+                df = parse_df_datetimes(
+                    pd.read_json(StringIO(df)),
+                    ignore_cols=[
+                        col
+                        for col, dtype in pipe_dtypes.items()
+                        if (not enforce or not are_dtypes_equal(dtype, 'datetime'))
+                    ],
+                    ignore_all=(not enforce),
+                    strip_timezone=(self.tzinfo is None),
+                    chunksize=chunksize,
+                    debug=debug,
+                )
+        elif isinstance(df, (dict, list, tuple)):
             df = parse_df_datetimes(
                 df,
                 ignore_cols=[

meerschaum/core/Pipe/_edit.py

@@ -3,7 +3,7 @@
 # vim:fenc=utf-8
 
 """
-Edit a Pipe's parameters here.
+Edit a Pipe's parameters.
 """
 
 from __future__ import annotations
@@ -47,6 +47,13 @@ def edit(
     if self.temporary:
         return False, "Cannot edit pipes created with `temporary=True` (read-only)."
 
+    if hasattr(self, '_symlinks'):
+        from meerschaum.utils.misc import get_val_from_dict_path, set_val_in_dict_path
+        for path, vals in self._symlinks.items():
+            current_val = get_val_from_dict_path(self.parameters, path)
+            if current_val == vals['substituted']:
+                set_val_in_dict_path(self.parameters, path, vals['original'])
+
     if not interactive:
         with Venv(get_connector_plugin(self.instance_connector)):
             return self.instance_connector.edit_pipe(self, patch=patch, debug=debug, **kw)
@@ -65,7 +72,8 @@ def edit(
     from meerschaum.config import get_config
     parameters = dict(get_config('pipes', 'parameters', patch=True))
     from meerschaum.config._patch import apply_patch_to_config
-    parameters = apply_patch_to_config(parameters, self.parameters)
+    raw_parameters = self.attributes.get('parameters', {})
+    parameters = apply_patch_to_config(parameters, raw_parameters)
 
     ### write parameters to yaml file
     with open(parameters_path, 'w+') as f:
@@ -194,7 +202,7 @@ def edit_definition(
         return True, "Success"
 
     def _edit_sql():
-        import pathlib, os, textwrap
+        import textwrap
         from meerschaum.config._paths import PIPES_CACHE_RESOURCES_PATH
         from meerschaum.utils.misc import edit_file
         definition_filename = str(self) + '.sql'
@@ -214,7 +222,7 @@ def edit_definition(
 
         edit_file(definition_path)
         try:
-            with open(definition_path, 'r') as f:
+            with open(definition_path, 'r', encoding='utf-8') as f:
                 file_definition = f.read()
         except Exception as e:
             return False, f"Failed reading file '{definition_path}':\n" + str(e)

meerschaum/core/Pipe/_show.py

@@ -9,11 +9,11 @@ Show information about a Pipe
 from meerschaum.utils.typing import SuccessTuple
 
 def show(
-        self,
-        nopretty: bool = False,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    nopretty: bool = False,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Show attributes of a Pipe.
 

meerschaum/core/Pipe/_sync.py

@@ -13,7 +13,7 @@ import time
 import threading
 import multiprocessing
 import functools
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
 
 import meerschaum as mrsm
@@ -42,6 +42,7 @@ def sync(
         pd.DataFrame,
         Dict[str, List[Any]],
         List[Dict[str, Any]],
+        str,
         InferFetch
     ] = InferFetch,
     begin: Union[datetime, int, str, None] = '',
@@ -71,6 +72,7 @@ def sync(
     ----------
     df: Union[None, pd.DataFrame, Dict[str, List[Any]]], default None
         An optional DataFrame to sync into the pipe. Defaults to `None`.
+        If `df` is a string, it will be parsed via `meerschaum.utils.dataframe.parse_simple_lines()`.
 
     begin: Union[datetime, int, str, None], default ''
         Optionally specify the earliest datetime to search for data.
@@ -134,6 +136,7 @@ def sync(
     from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords, filter_arguments
     from meerschaum.utils.pool import get_pool
     from meerschaum.config import get_config
+    from meerschaum.utils.dtypes import are_dtypes_equal
 
     if (callback is not None or error_callback is not None) and blocking:
         warn("Callback functions are only executed when blocking = False. Ignoring...")
@@ -170,6 +173,7 @@ def sync(
             'pd.DataFrame',
             Dict[str, List[Any]],
             List[Dict[str, Any]],
+            str,
             InferFetch
         ] = InferFetch,
     ) -> SuccessTuple:
@@ -189,6 +193,10 @@ def sync(
                     p._exists = None
                     return register_success, register_msg
 
+        if isinstance(df, str):
+            from meerschaum.utils.dataframe import parse_simple_lines
+            df = parse_simple_lines(df)
+
         ### If connector is a plugin with a `sync()` method, return that instead.
         ### If the plugin does not have a `sync()` method but does have a `fetch()` method,
         ### use that instead.
@@ -209,7 +217,7 @@ def sync(
                 p._exists = None
                 return False, f"Unable to create the connector for {p}."
 
-            ### Sync in place if this is a SQL pipe.
+            ### Sync in place if possible.
             if (
                 str(self.connector) == str(self.instance_connector)
                 and 
@@ -398,6 +406,21 @@ def sync(
             enforce=enforce_dtypes,
             debug=debug,
         )
+        if p.autotime:
+            dt_col = p.columns.get('datetime', 'ts')
+            dt_typ = p.dtypes.get(dt_col, 'datetime') if dt_col else 'datetime'
+            if dt_col and hasattr(df, 'columns') and dt_col not in df.columns:
+                now = datetime.now(timezone.utc)
+                now_val = (
+                    int(now.timestamp() * 1000)
+                    if are_dtypes_equal(dt_typ, 'int')
+                    else now
+                )
+                if debug:
+                    dprint(f"Adding current timestamp to dataframe synced to {p}: {now_val}")
+
+                df[dt_col] = now_val
+                kw['check_existing'] = False
 
         ### Capture `numeric`, `uuid`, `json`, and `bytes` columns.
         self._persist_new_json_columns(df, debug=debug)
@@ -447,9 +470,9 @@ def sync(
         if self.cache_pipe is not None:
             if debug:
                 dprint("Caching retrieved dataframe.", **kw)
-                _sync_cache_tuple = self.cache_pipe.sync(df, debug=debug, **kw)
-                if not _sync_cache_tuple[0]:
-                    warn(f"Failed to sync local cache for {self}.")
+            _sync_cache_tuple = self.cache_pipe.sync(df, debug=debug, **kw)
+            if not _sync_cache_tuple[0]:
+                warn(f"Failed to sync local cache for {self}.")
 
         self._exists = None
         return return_tuple
@@ -691,7 +714,9 @@ def filter_existing(
         NA = pd.NA
 
     primary_key = self.columns.get('primary', None)
+    dt_col = self.columns.get('datetime', None)
     autoincrement = self.parameters.get('autoincrement', False)
+    autotime = self.parameters.get('autotime', False)
     pipe_columns = self.columns.copy()
 
     if primary_key and autoincrement and df is not None and primary_key in df.columns:
@@ -702,6 +727,14 @@ def filter_existing(
             del df[primary_key]
             _ = self.columns.pop(primary_key, None)
 
+    if dt_col and autotime and df is not None and dt_col in df.columns:
+        if safe_copy:
+            df = df.copy()
+            safe_copy = False
+        if df[dt_col].isnull().all():
+            del df[dt_col]
+            _ = self.columns.pop(dt_col, None)
+
     def get_empty_df():
         empty_df = pd.DataFrame([])
         dtypes = dict(df.dtypes) if df is not None else {}
@@ -730,6 +763,11 @@ def filter_existing(
             df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
         return df, get_empty_df(), df
 
+    if autotime and dt_col and dt_col not in df.columns:
+        if enforce_dtypes:
+            df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+        return df, get_empty_df(), df
+
     try:
         min_dt_val = df[dt_col].min(skipna=True) if dt_col and dt_col in df.columns else None
         if is_dask and min_dt_val is not None:
@@ -1013,15 +1051,7 @@ def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
     dtypes.update({col: 'numeric' for col in new_numeric_cols})
-    self.parameters['dtypes'] = dtypes
-    if not self.temporary:
-        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
-        if not edit_success:
-            warn(f"Unable to update NUMERIC dtypes for {self}:\n{edit_msg}")
-
-        return edit_success, edit_msg
-
-    return True, "Success"
+    return self.update_parameters({'dtypes': dtypes}, debug=debug)
 
 
 def _persist_new_uuid_columns(self, df, debug: bool = False) -> SuccessTuple:
@@ -1038,15 +1068,7 @@ def _persist_new_uuid_columns(self, df, debug: bool = False) -> SuccessTuple:
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
     dtypes.update({col: 'uuid' for col in new_uuid_cols})
-    self.parameters['dtypes'] = dtypes
-    if not self.temporary:
-        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
-        if not edit_success:
-            warn(f"Unable to update UUID dtypes for {self}:\n{edit_msg}")
-
-        return edit_success, edit_msg
-
-    return True, "Success"
+    return self.update_parameters({'dtypes': dtypes}, debug=debug)
 
 
 def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
@@ -1063,16 +1085,7 @@ def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
     dtypes.update({col: 'json' for col in new_json_cols})
-    self.parameters['dtypes'] = dtypes
-
-    if not self.temporary:
-        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
-        if not edit_success:
-            warn(f"Unable to update JSON dtypes for {self}:\n{edit_msg}")
-
-        return edit_success, edit_msg
-
-    return True, "Success"
+    return self.update_parameters({'dtypes': dtypes}, debug=debug)
 
 
 def _persist_new_bytes_columns(self, df, debug: bool = False) -> SuccessTuple:
@@ -1089,16 +1102,7 @@ def _persist_new_bytes_columns(self, df, debug: bool = False) -> SuccessTuple:
     self._attributes_sync_time = None
     dtypes = self.parameters.get('dtypes', {})
     dtypes.update({col: 'bytes' for col in new_bytes_cols})
-    self.parameters['dtypes'] = dtypes
-
-    if not self.temporary:
-        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
-        if not edit_success:
-            warn(f"Unable to update bytes dtypes for {self}:\n{edit_msg}")
-
-        return edit_success, edit_msg
-
-    return True, "Success"
+    return self.update_parameters({'dtypes': dtypes}, debug=debug)
 
 
 def _persist_new_geometry_columns(self, df, debug: bool = False) -> SuccessTuple:
@@ -1146,13 +1150,4 @@ def _persist_new_geometry_columns(self, df, debug: bool = False) -> SuccessTuple
         new_cols_types[col] = f"{new_dtype}{modifier}"
 
     dtypes.update(new_cols_types)
-    self.parameters['dtypes'] = dtypes
-
-    if not self.temporary:
-        edit_success, edit_msg = self.edit(interactive=False, debug=debug)
-        if not edit_success:
-            warn(f"Unable to update bytes dtypes for {self}:\n{edit_msg}")
-
-        return edit_success, edit_msg
-
-    return True, "Success"
+    return self.update_parameters({'dtypes': dtypes})

meerschaum/core/Pipe/_verify.py

@@ -12,7 +12,7 @@ import time
 import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Any, Optional, Union, Tuple, Dict
 from meerschaum.utils.warnings import warn, info
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 
 def verify(

meerschaum/{plugins → core/Plugin}/_Plugin.py

@@ -22,19 +22,13 @@ from meerschaum.utils.typing import (
     Union,
 )
 from meerschaum.utils.warnings import error, warn
-from meerschaum.config import get_config
-from meerschaum.config._paths import (
-    PLUGINS_RESOURCES_PATH,
-    PLUGINS_ARCHIVES_RESOURCES_PATH,
-    PLUGINS_TEMP_RESOURCES_PATH,
-    VIRTENV_RESOURCES_PATH,
-    PLUGINS_DIR_PATHS,
-)
 _tmpversion = None
 _ongoing_installations = set()
 
+
 class Plugin:
     """Handle packaging of Meerschaum plugins."""
+
     def __init__(
         self,
         name: str,
@@ -47,7 +41,8 @@ class Plugin:
         repo_connector: Optional['mrsm.connectors.api.APIConnector'] = None,
         repo: Union['mrsm.connectors.api.APIConnector', str, None] = None,
     ):
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
+        from meerschaum.config.paths import PLUGINS_ARCHIVES_RESOURCES_PATH, VIRTENV_RESOURCES_PATH
         sep = STATIC_CONFIG['plugins']['repo_separator']
         _repo = None
         if sep in name:
@@ -130,6 +125,8 @@ class Plugin:
         if self.__dict__.get('_module', None) is not None:
             return self.module.__file__
 
+        from meerschaum.config.paths import PLUGINS_RESOURCES_PATH
+
         potential_dir = PLUGINS_RESOURCES_PATH / self.name
         if (
             potential_dir.exists()
@@ -295,6 +292,7 @@ class Plugin:
         from meerschaum.utils.packages import attempt_import, determine_version, reload_meerschaum
         from meerschaum.utils.venv import init_venv
         from meerschaum.utils.misc import safely_extract_tar
+        from meerschaum.config.paths import PLUGINS_TEMP_RESOURCES_PATH, PLUGINS_DIR_PATHS
         old_cwd = os.getcwd()
         old_version = ''
         new_version = ''
@@ -722,7 +720,7 @@ class Plugin:
         """
         from meerschaum.utils.warnings import warn
         from meerschaum.config import get_config
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
         from meerschaum.connectors.parse import is_valid_connector_keys
         plugins = []
         _deps = self.get_dependencies(debug=debug)
@@ -953,7 +951,7 @@ class Plugin:
         """
         Include the repo keys with the plugin's name.
         """
-        from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum._internal.static import STATIC_CONFIG
         sep = STATIC_CONFIG['plugins']['repo_separator']
         return self.name + sep + str(self.repo_connector)
 

meerschaum/core/Plugin/__init__.py

@@ -6,4 +6,4 @@
 Import the Plugin class.
 """
 
-from meerschaum.plugins import Plugin
+from meerschaum.core.Plugin._Plugin import Plugin