meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0.dev0__py3-none-any.whl

meerschaum/connectors/sql/_pipes.py

@@ -15,10 +15,10 @@ from meerschaum.utils.warnings import warn, error, info
 from meerschaum.utils.debug import dprint
 
 def register_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> SuccessTuple:
     """
     Register a new pipe.
     A pipe's attributes must be set before registering.
@@ -140,14 +140,14 @@ def edit_pipe(
 
 
 def fetch_pipes_keys(
-        self,
-        connector_keys: Optional[List[str]] = None,
-        metric_keys: Optional[List[str]] = None,
-        location_keys: Optional[List[str]] = None,
-        tags: Optional[List[str]] = None,
-        params: Optional[Dict[str, Any]] = None,
-        debug: bool = False
-    ) -> Optional[List[Tuple[str, str, Optional[str]]]]:
+    self,
+    connector_keys: Optional[List[str]] = None,
+    metric_keys: Optional[List[str]] = None,
+    location_keys: Optional[List[str]] = None,
+    tags: Optional[List[str]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False
+) -> Optional[List[Tuple[str, str, Optional[str]]]]:
     """
     Return a list of tuples corresponding to the parameters provided.
 
@@ -634,20 +634,20 @@ def delete_pipe(
 
 
 def get_pipe_data(
-        self,
-        pipe: mrsm.Pipe,
-        select_columns: Optional[List[str]] = None,
-        omit_columns: Optional[List[str]] = None,
-        begin: Union[datetime, str, None] = None,
-        end: Union[datetime, str, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        order: str = 'asc',
-        limit: Optional[int] = None,
-        begin_add_minutes: int = 0,
-        end_add_minutes: int = 0,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union[pd.DataFrame, None]:
+    self,
+    pipe: mrsm.Pipe,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[datetime, str, None] = None,
+    end: Union[datetime, str, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    order: str = 'asc',
+    limit: Optional[int] = None,
+    begin_add_minutes: int = 0,
+    end_add_minutes: int = 0,
+    debug: bool = False,
+    **kw: Any
+) -> Union[pd.DataFrame, None]:
     """
     Access a pipe's data from the SQL instance.
 
@@ -746,19 +746,19 @@ def get_pipe_data(
     }
     query = self.get_pipe_data_query(
         pipe,
-        select_columns = select_columns,
-        omit_columns = omit_columns,
-        begin = begin,
-        end = end,
-        params = params,
-        order = order,
-        limit = limit,
-        begin_add_minutes = begin_add_minutes,
-        end_add_minutes = end_add_minutes,
-        debug = debug,
+        select_columns=select_columns,
+        omit_columns=omit_columns,
+        begin=begin,
+        end=end,
+        params=params,
+        order=order,
+        limit=limit,
+        begin_add_minutes=begin_add_minutes,
+        end_add_minutes=end_add_minutes,
+        debug=debug,
         **kw
     )
-    
+
     if is_dask:
         index_col = pipe.columns.get('datetime', None)
         kw['index_col'] = index_col
@@ -769,11 +769,11 @@ def get_pipe_data(
         if typ == 'numeric' and col in dtypes
     ]
     kw['coerce_float'] = kw.get('coerce_float', (len(numeric_columns) == 0))
-    
+
     df = self.read(
         query,
-        dtype = dtypes,
-        debug = debug,
+        dtype=dtypes,
+        debug=debug,
         **kw
     )
     for col in numeric_columns:
@@ -782,28 +782,25 @@ def get_pipe_data(
         df[col] = df[col].apply(attempt_cast_to_numeric)
 
     if self.flavor == 'sqlite':
+        ignore_dt_cols = [
+            col
+            for col, dtype in pipe.dtypes.items()
+            if 'datetime' not in str(dtype)
+        ]
         ### NOTE: We have to consume the iterator here to ensure that datetimes are parsed correctly
         df = (
             parse_df_datetimes(
                 df,
-                ignore_cols = [
-                    col
-                    for col, dtype in pipe.dtypes.items()
-                    if 'datetime' not in str(dtype)
-                ],
-                chunksize = kw.get('chunksize', None),
-                debug = debug,
+                ignore_cols=ignore_dt_cols,
+                chunksize=kw.get('chunksize', None),
+                debug=debug,
             ) if isinstance(df, pd.DataFrame) else (
                 [
                     parse_df_datetimes(
                         c,
-                        ignore_cols = [
-                            col
-                            for col, dtype in pipe.dtypes.items()
-                            if 'datetime' not in str(dtype)
-                        ],
+                        ignore_cols=ignore_dt_cols,
                         chunksize = kw.get('chunksize', None),
-                        debug = debug,
+                        debug=debug,
                     )
                     for c in df
                 ]
@@ -817,21 +814,21 @@ def get_pipe_data(
 
 
 def get_pipe_data_query(
-        self,
-        pipe: mrsm.Pipe,
-        select_columns: Optional[List[str]] = None,
-        omit_columns: Optional[List[str]] = None,
-        begin: Union[datetime, int, str, None] = None,
-        end: Union[datetime, int, str, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        order: str = 'asc',
-        limit: Optional[int] = None,
-        begin_add_minutes: int = 0,
-        end_add_minutes: int = 0,
-        replace_nulls: Optional[str] = None,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union[str, None]:
+    self,
+    pipe: mrsm.Pipe,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[datetime, int, str, None] = None,
+    end: Union[datetime, int, str, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    order: str = 'asc',
+    limit: Optional[int] = None,
+    begin_add_minutes: int = 0,
+    end_add_minutes: int = 0,
+    replace_nulls: Optional[str] = None,
+    debug: bool = False,
+    **kw: Any
+) -> Union[str, None]:
     """
     Return the `SELECT` query for retrieving a pipe's data from its instance.
 
@@ -1107,18 +1104,18 @@ def get_pipe_attributes(
 
 
 def sync_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        df: Union[pd.DataFrame, str, Dict[Any, Any], None] = None,
-        begin: Optional[datetime] = None,
-        end: Optional[datetime] = None,
-        chunksize: Optional[int] = -1,
-        check_existing: bool = True,
-        blocking: bool = True,
-        debug: bool = False,
-        _check_temporary_tables: bool = True,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    df: Union[pd.DataFrame, str, Dict[Any, Any], None] = None,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    chunksize: Optional[int] = -1,
+    check_existing: bool = True,
+    blocking: bool = True,
+    debug: bool = False,
+    _check_temporary_tables: bool = True,
+    **kw: Any
+) -> SuccessTuple:
     """
     Sync a pipe using a database connection.
 
@@ -1191,9 +1188,9 @@ def sync_pipe(
     if not isinstance(df, pd.DataFrame):
         df = pipe.enforce_dtypes(
             df,
-            chunksize = chunksize,
-            safe_copy = kw.get('safe_copy', False),
-            debug = debug,
+            chunksize=chunksize,
+            safe_copy=kw.get('safe_copy', False),
+            debug=debug,
         )
 
     ### if table does not exist, create it with indices
@@ -1243,8 +1240,8 @@ def sync_pipe(
     unseen_df, update_df, delta_df = (
         pipe.filter_existing(
             df,
-            chunksize = chunksize,
-            debug = debug,
+            chunksize=chunksize,
+            debug=debug,
             **kw
         ) if check_existing else (df, None, df)
     )
@@ -1263,31 +1260,6 @@ def sync_pipe(
     if 'name' in kw:
         kw.pop('name')
 
-    ### Account for first-time syncs of JSON columns.
-    unseen_json_cols = get_json_cols(unseen_df)
-    update_json_cols = get_json_cols(update_df) if update_df is not None else []
-    json_cols = list(set(unseen_json_cols + update_json_cols))
-    existing_json_cols = [col for col, typ in pipe.dtypes.items() if typ == 'json']
-    new_json_cols = [col for col in json_cols if col not in existing_json_cols]
-    if new_json_cols:
-        pipe.dtypes.update({col: 'json' for col in json_cols})
-        if not pipe.temporary:
-            edit_success, edit_msg = pipe.edit(interactive=False, debug=debug)
-            if not edit_success:
-                warn(f"Unable to update JSON dtypes for {pipe}:\n{edit_msg}")
-
-    unseen_numeric_cols = get_numeric_cols(unseen_df)
-    update_numeric_cols = get_numeric_cols(update_df) if update_df is not None else []
-    numeric_cols = list(set(unseen_numeric_cols + update_numeric_cols))
-    existing_numeric_cols = [col for col, typ in pipe.dtypes.items() if typ == 'numeric']
-    new_numeric_cols = [col for col in numeric_cols if col not in existing_numeric_cols]
-    if new_numeric_cols:
-        pipe.dtypes.update({col: 'numeric' for col in numeric_cols})
-        if not pipe.temporary:
-            edit_success, edit_msg = pipe.edit(interactive=False, debug=debug)
-            if not edit_success:
-                warn(f"Unable to update NUMERIC dtypes for {pipe}:\n{edit_msg}")
-
     ### Insert new data into Pipe's table.
     unseen_kw = copy.deepcopy(kw)
     unseen_kw.update({
@@ -1326,16 +1298,16 @@ def sync_pipe(
         self._log_temporary_tables_creation(temp_target, create=(not pipe.temporary), debug=debug)
         temp_pipe = Pipe(
             pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
-            instance = pipe.instance_keys,
-            columns = {
+            instance=pipe.instance_keys,
+            columns={
                 ix_key: ix
                 for ix_key, ix in pipe.columns.items()
                 if ix and ix in update_df.columns
             },
-            dtypes = pipe.dtypes,
-            target = temp_target,
-            temporary = True,
-            parameters = {
+            dtypes=pipe.dtypes,
+            target=temp_target,
+            temporary=True,
+            parameters={
                 'schema': self.internal_schema,
                 'hypertable': False,
             },
@@ -1352,20 +1324,20 @@ def sync_pipe(
             temp_target, 
             self,
             join_cols,
-            upsert = upsert,
-            schema = self.get_pipe_schema(pipe),
-            patch_schema = self.internal_schema,
-            datetime_col = pipe.columns.get('datetime', None),
-            debug = debug,
+            upsert=upsert,
+            schema=self.get_pipe_schema(pipe),
+            patch_schema=self.internal_schema,
+            datetime_col=pipe.columns.get('datetime', None),
+            debug=debug,
         )
         update_success = all(
             self.exec_queries(update_queries, break_on_error=True, rollback=True, debug=debug)
         )
         self._log_temporary_tables_creation(
             temp_target,
-            ready_to_drop = True,
-            create = (not pipe.temporary),
-            debug = debug,
+            ready_to_drop=True,
+            create=(not pipe.temporary),
+            debug=debug,
         )
         if not update_success:
             warn(f"Failed to apply update to {pipe}.")
@@ -1406,16 +1378,16 @@ def sync_pipe(
 
 
 def sync_pipe_inplace(
-        self,
-        pipe: 'mrsm.Pipe',
-        params: Optional[Dict[str, Any]] = None,
-        begin: Optional[datetime] = None,
-        end: Optional[datetime] = None,
-        chunksize: Optional[int] = -1,
-        check_existing: bool = True,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    pipe: 'mrsm.Pipe',
+    params: Optional[Dict[str, Any]] = None,
+    begin: Optional[datetime] = None,
+    end: Optional[datetime] = None,
+    chunksize: Optional[int] = -1,
+    check_existing: bool = True,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     If a pipe's connector is the same as its instance connector,
     it's more efficient to sync the pipe in-place rather than reading data into Pandas.
@@ -1455,13 +1427,13 @@ def sync_pipe_inplace(
     """
     if self.flavor == 'duckdb':
         return pipe.sync(
-            params = params,
-            begin = begin,
-            end = end,
-            chunksize = chunksize,
-            check_existing = check_existing,
-            debug = debug,
-            _inplace = False,
+            params=params,
+            begin=begin,
+            end=end,
+            chunksize=chunksize,
+            check_existing=check_existing,
+            debug=debug,
+            _inplace=False,
             **kw
         )
     from meerschaum.utils.sql import (
@@ -2062,14 +2034,14 @@ def pipe_exists(
 
 
 def get_pipe_rowcount(
-        self,
-        pipe: mrsm.Pipe,
-        begin: Union[datetime, int, None] = None,
-        end: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        remote: bool = False,
-        debug: bool = False
-    ) -> Union[int, None]:
+    self,
+    pipe: mrsm.Pipe,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    remote: bool = False,
+    debug: bool = False
+) -> Union[int, None]:
     """
     Get the rowcount for a pipe in accordance with given parameters.
 
@@ -2411,11 +2383,11 @@ def get_pipe_columns_types(
 
 
 def get_add_columns_queries(
-        self,
-        pipe: mrsm.Pipe,
-        df: Union[pd.DataFrame, Dict[str, str]],
-        debug: bool = False,
-    ) -> List[str]:
+    self,
+    pipe: mrsm.Pipe,
+    df: Union[pd.DataFrame, Dict[str, str]],
+    debug: bool = False,
+) -> List[str]:
     """
     Add new null columns of the correct type to a table from a dataframe.
 

meerschaum/connectors/sql/_plugins.py

@@ -7,15 +7,19 @@ Functions for managing plugins registration via the SQL connector
 """
 
 from __future__ import annotations
+
+import json
+
+import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Any, List, SuccessTuple, Dict
 
 def register_plugin(
-        self,
-        plugin: 'meerschaum.core.Plugin',
-        force: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    plugin: 'mrsm.core.Plugin',
+    force: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """Register a new plugin to the plugins table."""
     from meerschaum.utils.warnings import warn, error
     from meerschaum.utils.packages import attempt_import
@@ -46,14 +50,13 @@ def register_plugin(
                 f"must be greater than existing version '{old_version}'."
             )
 
-    import json
     bind_variables = {
-        'plugin_name' : plugin.name,
-        'version' : plugin.version,
-        'attributes' : (
+        'plugin_name': plugin.name,
+        'version': plugin.version,
+        'attributes': (
             json.dumps(plugin.attributes) if self.flavor not in json_flavors else plugin.attributes
         ),
-        'user_id' : plugin.user_id,
+        'user_id': plugin.user_id,
     }
 
     if old_id is None:
@@ -71,10 +74,10 @@ def register_plugin(
     return True, f"Successfully registered plugin '{plugin}'."
 
 def get_plugin_id(
-        self,
-        plugin: 'meerschaum.core.Plugin',
-        debug: bool = False
-    ) -> Optional[int]:
+    self,
+    plugin: 'mrsm.core.Plugin',
+    debug: bool = False
+) -> Optional[int]:
     """
     Return a plugin's ID.
     """
@@ -96,10 +99,10 @@ def get_plugin_id(
         return None
 
 def get_plugin_version(
-        self,
-        plugin: 'meerschaum.core.Plugin',
-        debug: bool = False
-    ) -> Optional[str]:
+    self,
+    plugin: 'mrsm.core.Plugin',
+    debug: bool = False
+) -> Optional[str]:
     """
     Return a plugin's version.
     """
@@ -112,10 +115,10 @@ def get_plugin_version(
     return self.value(query, debug=debug)
 
 def get_plugin_user_id(
-        self,
-        plugin: 'meerschaum.core.Plugin',
-        debug: bool = False
-    ) -> Optional[int]:
+    self,
+    plugin: 'mrsm.core.Plugin',
+    debug: bool = False
+) -> Optional[int]:
     """
     Return a plugin's user ID.
     """
@@ -137,10 +140,10 @@ def get_plugin_user_id(
         return None
 
 def get_plugin_username(
-        self,
-        plugin: 'meerschaum.core.Plugin',
-        debug: bool = False
-    ) -> Optional[str]:
+    self,
+    plugin: 'mrsm.core.Plugin',
+    debug: bool = False
+) -> Optional[str]:
     """
     Return the username of a plugin's owner.
     """
@@ -163,16 +166,15 @@ def get_plugin_username(
 
 
 def get_plugin_attributes(
-        self,
-        plugin: 'meerschaum.core.Plugin',
-        debug: bool = False
-    ) -> Dict[str, Any]:
+    self,
+    plugin: 'mrsm.core.Plugin',
+    debug: bool = False
+) -> Dict[str, Any]:
     """
     Return the attributes of a plugin.
     """
     ### ensure plugins table exists
     from meerschaum.connectors.sql.tables import get_tables
-    import json
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
     sqlalchemy = attempt_import('sqlalchemy')
@@ -191,12 +193,12 @@ def get_plugin_attributes(
     return _attr
 
 def get_plugins(
-        self,
-        user_id: Optional[int] = None,
-        search_term: Optional[str] = None,
-        debug: bool = False,
-        **kw: Any
-    ) -> List[str]:
+    self,
+    user_id: Optional[int] = None,
+    search_term: Optional[str] = None,
+    debug: bool = False,
+    **kw: Any
+) -> List[str]:
     """
     Return a list of all registered plugins.
 
@@ -238,11 +240,11 @@ def get_plugins(
 
 
 def delete_plugin(
-        self,
-        plugin: 'meerschaum.core.Plugin',
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    plugin: 'mrsm.core.Plugin',
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """Delete a plugin from the plugins table."""
     from meerschaum.utils.warnings import warn, error
     from meerschaum.utils.packages import attempt_import