meerschaum 2.4.7__tar.gz → 2.4.8__tar.gz

{meerschaum-2.4.7/meerschaum.egg-info → meerschaum-2.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.4.7
+Version: 2.4.8
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -127,7 +127,8 @@ Provides-Extra: sql
 Requires-Dist: numpy>=1.18.5; extra == "sql"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "sql"
 Requires-Dist: pyarrow>=16.1.0; extra == "sql"
-Requires-Dist: dask[dataframe]>=2024.5.1; extra == "sql"
+Requires-Dist: dask[complete]>=2024.5.1; extra == "sql"
+Requires-Dist: partd>=1.4.2; extra == "sql"
 Requires-Dist: pytz; extra == "sql"
 Requires-Dist: joblib>=0.17.0; extra == "sql"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "sql"
@@ -186,7 +187,8 @@ Requires-Dist: valkey>=6.0.0; extra == "api"
 Requires-Dist: numpy>=1.18.5; extra == "api"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "api"
 Requires-Dist: pyarrow>=16.1.0; extra == "api"
-Requires-Dist: dask[dataframe]>=2024.5.1; extra == "api"
+Requires-Dist: dask[complete]>=2024.5.1; extra == "api"
+Requires-Dist: partd>=1.4.2; extra == "api"
 Requires-Dist: pytz; extra == "api"
 Requires-Dist: joblib>=0.17.0; extra == "api"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "api"
@@ -292,7 +294,8 @@ Requires-Dist: pycparser>=2.21.0; extra == "full"
 Requires-Dist: numpy>=1.18.5; extra == "full"
 Requires-Dist: pandas[parquet]>=2.0.1; extra == "full"
 Requires-Dist: pyarrow>=16.1.0; extra == "full"
-Requires-Dist: dask[dataframe]>=2024.5.1; extra == "full"
+Requires-Dist: dask[complete]>=2024.5.1; extra == "full"
+Requires-Dist: partd>=1.4.2; extra == "full"
 Requires-Dist: pytz; extra == "full"
 Requires-Dist: joblib>=0.17.0; extra == "full"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "full"

{meerschaum-2.4.7 → meerschaum-2.4.8}/meerschaum/actions/sql.py

@@ -59,7 +59,7 @@ def sql(
         - `sql local exec "INSERT INTO table (id) VALUES (1)"
             Execute the above query on `sql:local`.
     """
-    from meerschaum.utils.dataframe impor to_json
+    from meerschaum.utils.dataframe import to_json
 
     if action is None:
         action = []

{meerschaum-2.4.7 → meerschaum-2.4.8}/meerschaum/api/routes/_pipes.py

@@ -408,8 +408,8 @@ def get_pipe_data(
             _params = None
     if not isinstance(_params, dict):
         raise fastapi.HTTPException(
-            status_code = 409,
-            detail = "Params must be a valid JSON-encoded dictionary.",
+            status_code=409,
+            detail="Params must be a valid JSON-encoded dictionary.",
         )
 
     _select_columns = []
@@ -422,8 +422,8 @@ def get_pipe_data(
             _select_columns = None
     if not isinstance(_select_columns, list):
         raise fastapi.HTTPException(
-            status_code = 409,
-            detail = "Selected columns must be a JSON-encoded list."
+            status_code=409,
+            detail="Selected columns must be a JSON-encoded list."
         )
 
     _omit_columns = []
@@ -436,35 +436,35 @@ def get_pipe_data(
             _omit_columns = None
     if _omit_columns is None:
         raise fastapi.HTTPException(
-            status_code = 409,
-            detail = "Omitted columns must be a JSON-encoded list.",
+            status_code=409,
+            detail="Omitted columns must be a JSON-encoded list.",
         )
 
     pipe = get_pipe(connector_keys, metric_key, location_key)
     if not is_pipe_registered(pipe, pipes(refresh=True)):
         raise fastapi.HTTPException(
-            status_code = 409,
-            detail = "Pipe must be registered with the datetime column specified."
+            status_code=409,
+            detail="Pipe must be registered with the datetime column specified."
         )
 
     if pipe.target in ('users', 'plugins', 'pipes'):
         raise fastapi.HTTPException(
-            status_code = 409,
-            detail = f"Cannot retrieve data from protected table '{pipe.target}'.",
+            status_code=409,
+            detail=f"Cannot retrieve data from protected table '{pipe.target}'.",
         )
 
     df = pipe.get_data(
-        select_columns = _select_columns,
-        omit_columns = _omit_columns,
-        begin = begin,
-        end = end,
-        params = _params,
-        debug = debug,
+        select_columns=_select_columns,
+        omit_columns=_omit_columns,
+        begin=begin,
+        end=end,
+        params=_params,
+        debug=debug,
     )
     if df is None:
         raise fastapi.HTTPException(
-            status_code = 400,
-            detail = f"Could not fetch data with the given parameters.",
+            status_code=400,
+            detail="Could not fetch data with the given parameters.",
         )
 
     ### NaN cannot be JSON-serialized.
@@ -482,16 +482,16 @@ def get_pipe_data(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv', tags=['Pipes'])
 def get_pipe_csv(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        begin: Union[str, int, None] = None,
-        end: Union[str, int, None] = None,
-        params: Optional[str] = None,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> str:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    params: Optional[str] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> str:
     """
     Get a Pipe's data as a CSV file. Optionally set query boundaries.
     """
@@ -518,8 +518,8 @@ def get_pipe_csv(
 
     if not isinstance(_params, dict):
         raise fastapi.HTTPException(
-            status_code = 409,
-            detail = "Params must be a valid JSON-encoded dictionary.",
+            status_code=409,
+            detail="Params must be a valid JSON-encoded dictionary.",
         )
 
     p = get_pipe(connector_keys, metric_key, location_key)
@@ -529,7 +529,7 @@ def get_pipe_csv(
             detail = "Pipe must be registered with the datetime column specified."
         )
 
-    dt_col = pipe.columns.get('datetime', None)
+    dt_col = p.columns.get('datetime', None)
     if dt_col:
         if begin is None:
             begin = p.get_sync_time(round_down=False, newest=False)
@@ -552,13 +552,13 @@ def get_pipe_csv(
 
 @app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id', tags=['Pipes'])
 def get_pipe_id(
-        connector_keys: str,
-        metric_key: str,
-        location_key: str,
-        curr_user = (
-            fastapi.Depends(manager) if not no_auth else None
-        ),
-    ) -> int:
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+) -> int:
     """
     Get a Pipe's ID.
     """

{meerschaum-2.4.7 → meerschaum-2.4.8}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.7"
+__version__ = "2.4.8"

{meerschaum-2.4.7 → meerschaum-2.4.8}/meerschaum/connectors/api/_pipes.py

@@ -346,14 +346,14 @@ def get_pipe_data(
         try:
             response = self.get(
                 r_url + "/data",
-                params = {
+                params={
                     'select_columns': json.dumps(select_columns),
                     'omit_columns': json.dumps(omit_columns),
                     'begin': begin,
                     'end': end,
-                    'params': json.dumps(params)
+                    'params': json.dumps(params, default=str)
                 },
-                debug = debug
+                debug=debug
             )
             if not response.ok:
                 return None

{meerschaum-2.4.7 → meerschaum-2.4.8}/meerschaum/connectors/sql/_SQLConnector.py

@@ -272,7 +272,7 @@ class SQLConnector(Connector):
         """
         Return whether this connector may be multithreaded.
         """
-        if self.flavor == 'duckdb':
+        if self.flavor in ('duckdb', 'oracle'):
             return False
         if self.flavor == 'sqlite':
             return ':memory:' not in self.URI

{meerschaum-2.4.7 → meerschaum-2.4.8}/meerschaum/connectors/sql/_instance.py

@@ -15,12 +15,12 @@ from meerschaum.utils.warnings import warn
 
 _in_memory_temp_tables: Dict[str, bool] = {}
 def _log_temporary_tables_creation(
-        self,
-        tables: Union[str, List[str]],
-        ready_to_drop: bool = False,
-        create: bool = True,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    tables: Union[str, List[str]],
+    ready_to_drop: bool = False,
+    create: bool = True,
+    debug: bool = False,
+) -> SuccessTuple:
     """
     Log a temporary table's creation for later deletion.
     """
@@ -58,15 +58,15 @@ def _log_temporary_tables_creation(
 
 
 def _drop_temporary_table(
-        self,
-        table: str,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    table: str,
+    debug: bool = False,
+) -> SuccessTuple:
     """
     Drop a temporary table and clear it from the internal table.
     """
-    from meerschaum.utils.sql import sql_item_name, table_exists, SKIP_IF_EXISTS_FLAVORS
-    if_exists = "IF EXISTS" if self.flavor not in SKIP_IF_EXISTS_FLAVORS else ""
+    from meerschaum.utils.sql import sql_item_name, table_exists, DROP_IF_EXISTS_FLAVORS
+    if_exists = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
     if not if_exists:
         if not table_exists(table, self, self.internal_schema, debug=debug):
             return True, "Success"

{meerschaum-2.4.7 → meerschaum-2.4.8}/meerschaum/connectors/sql/_pipes.py

@@ -459,6 +459,11 @@ def get_create_index_queries(
                 + 'if_not_exists => true, '
                 + "migrate_data => true);"
             )
+        elif self.flavor == 'mssql':
+            dt_query = (
+                f"CREATE CLUSTERED INDEX {_datetime_index_name} "
+                f"ON {_pipe_name} ({_datetime_name})"
+            )
         else: ### mssql, sqlite, etc.
             dt_query = (
                 f"CREATE INDEX {_datetime_index_name} "
@@ -563,7 +568,12 @@ def get_drop_index_queries(
         return {}
     if not pipe.exists(debug=debug):
         return {}
-    from meerschaum.utils.sql import sql_item_name, table_exists, hypertable_queries
+    from meerschaum.utils.sql import (
+        sql_item_name,
+        table_exists,
+        hypertable_queries,
+        DROP_IF_EXISTS_FLAVORS,
+    )
     drop_queries = {}
     schema = self.get_pipe_schema(pipe)
     schema_prefix = (schema + '_') if schema else ''
@@ -580,16 +590,17 @@ def get_drop_index_queries(
         is_hypertable_query = hypertable_queries[self.flavor].format(table_name=pipe_name)
         is_hypertable = self.value(is_hypertable_query, silent=True, debug=debug) is not None
 
+    if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
     if is_hypertable:
         nuke_queries = []
         temp_table = '_' + pipe.target + '_temp_migration'
         temp_table_name = sql_item_name(temp_table, self.flavor, self.get_pipe_schema(pipe))
 
         if table_exists(temp_table, self, schema=self.get_pipe_schema(pipe), debug=debug):
-            nuke_queries.append(f"DROP TABLE {temp_table_name}")
+            nuke_queries.append(f"DROP TABLE {if_exists_str} {temp_table_name}")
         nuke_queries += [
             f"SELECT * INTO {temp_table_name} FROM {pipe_name}",
-            f"DROP TABLE {pipe_name}",
+            f"DROP TABLE {if_exists_str} {pipe_name}",
             f"ALTER TABLE {temp_table_name} RENAME TO {pipe_name_no_schema}",
         ]
         nuke_ix_keys = ('datetime', 'id')
@@ -811,7 +822,7 @@ def get_pipe_data(
                     parse_df_datetimes(
                         c,
                         ignore_cols=ignore_dt_cols,
-                        chunksize = kw.get('chunksize', None),
+                        chunksize=kw.get('chunksize', None),
                         debug=debug,
                     )
                     for c in df
@@ -1017,7 +1028,7 @@ def get_pipe_data_query(
             if _dt and _dt in existing_cols:
                 order_by += dt + ' ' + order + ','
             for key, quoted_col_name in quoted_indices.items():
-                if key == 'datetime':
+                if dt == quoted_col_name:
                     continue
                 order_by += ' ' + quoted_col_name + ' ' + order + ','
             order_by = order_by[:-1]
@@ -1034,7 +1045,7 @@ def get_pipe_data_query(
             )
         else:
             query += f"\nLIMIT {limit}"
-    
+
     if debug:
         to_print = (
             []
@@ -1315,7 +1326,7 @@ def sync_pipe(
         ) if dt_col else None
 
         transact_id = generate_password(3)
-        temp_target = '-' + transact_id + '_' + pipe.target
+        temp_target = '##' + transact_id + '_' + pipe.target
         self._log_temporary_tables_creation(temp_target, create=(not pipe.temporary), debug=debug)
         temp_pipe = Pipe(
             pipe.connector_keys.replace(':', '_') + '_', pipe.metric_key, pipe.location_key,
@@ -1721,7 +1732,7 @@ def sync_pipe_inplace(
 
     delta_cols_types = get_table_cols_types(
         temp_tables['delta'],
-        connectable = connectable,
+        connectable=connectable,
         flavor=self.flavor,
         schema=internal_schema,
         database=database,
@@ -1779,7 +1790,7 @@ def sync_pipe_inplace(
     create_joined_success, create_joined_msg = session_execute(
         session,
         create_joined_query,
-        debug = debug,
+        debug=debug,
     ) if on_cols and not upsert else (True, "Success")
     if not create_joined_success:
         _ = clean_up_temp_tables()
@@ -1790,14 +1801,14 @@ def sync_pipe_inplace(
         + (', '.join([
             (
                 "CASE\n    WHEN " + sql_item_name(c + '_delta', self.flavor, None)
-                + " != " + get_null_replacement(typ, self.flavor) 
+                + " != " + get_null_replacement(typ, self.flavor)
                 + " THEN " + sql_item_name(c + '_delta', self.flavor, None)
                 + "\n    ELSE NULL\nEND "
                 + " AS " + sql_item_name(c, self.flavor, None)
             ) for c, typ in delta_cols.items()
         ]))
         + f"\nFROM {temp_table_names['joined']}\n"
-        + f"WHERE "
+        + "WHERE "
         + '\nAND\n'.join([
             (
                 sql_item_name(c + '_backtrack', self.flavor, None) + ' IS NULL'
@@ -1813,8 +1824,8 @@ def sync_pipe_inplace(
     (create_unseen_success, create_unseen_msg), create_unseen_results = session_execute(
         session,
         create_unseen_query,
-        with_results = True,
-        debug = debug
+        with_results=True,
+        debug=debug
     ) if not upsert else (True, "Success"), None
     if not create_unseen_success:
         _ = clean_up_temp_tables()
@@ -1832,7 +1843,7 @@ def sync_pipe_inplace(
             ) for c, typ in delta_cols.items()
         ]))
         + f"\nFROM {temp_table_names['joined']}\n"
-        + f"WHERE "
+        + "WHERE "
         + '\nOR\n'.join([
             (
                 sql_item_name(c + '_backtrack', self.flavor, None) + ' IS NOT NULL'
@@ -1849,8 +1860,8 @@ def sync_pipe_inplace(
     (create_update_success, create_update_msg), create_update_results = session_execute(
         session,
         create_update_query,
-        with_results = True,
-        debug = debug,
+        with_results=True,
+        debug=debug,
     ) if on_cols and not upsert else ((True, "Success"), [])
     apply_update_queries = (
         get_update_queries(
@@ -1858,12 +1869,12 @@ def sync_pipe_inplace(
             temp_tables['update'],
             session,
             on_cols,
-            upsert = upsert,
-            schema = self.get_pipe_schema(pipe),
-            patch_schema = internal_schema,
-            datetime_col = pipe.columns.get('datetime', None),
-            flavor = self.flavor,
-            debug = debug
+            upsert=upsert,
+            schema=self.get_pipe_schema(pipe),
+            patch_schema=internal_schema,
+            datetime_col=pipe.columns.get('datetime', None),
+            flavor=self.flavor,
+            debug=debug,
         )
         if on_cols else []
     )
@@ -1883,8 +1894,8 @@ def sync_pipe_inplace(
     (apply_unseen_success, apply_unseen_msg), apply_unseen_results = session_execute(
         session,
         apply_unseen_queries,
-        with_results = True,
-        debug = debug,
+        with_results=True,
+        debug=debug,
     ) if not upsert else (True, "Success"), None
     if not apply_unseen_success:
         _ = clean_up_temp_tables()
@@ -1894,8 +1905,8 @@ def sync_pipe_inplace(
     (apply_update_success, apply_update_msg), apply_update_results = session_execute(
         session,
         apply_update_queries,
-        with_results = True,
-        debug = debug,
+        with_results=True,
+        debug=debug,
     )
     if not apply_update_success:
         _ = clean_up_temp_tables()
@@ -2198,7 +2209,7 @@ def get_pipe_rowcount(
                     else 'WHERE'
                 )
             )
-        
+
     result = self.value(query, debug=debug, silent=True)
     try:
         return int(result)
@@ -2207,11 +2218,11 @@ def get_pipe_rowcount(
 
 
 def drop_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Drop a pipe's tables but maintain its registration.
 
@@ -2219,30 +2230,36 @@ def drop_pipe(
     ----------
     pipe: mrsm.Pipe
         The pipe to drop.
-        
+
+    Returns
+    -------
+    A `SuccessTuple` indicated success.
     """
-    from meerschaum.utils.sql import table_exists, sql_item_name
+    from meerschaum.utils.sql import table_exists, sql_item_name, DROP_IF_EXISTS_FLAVORS
     success = True
     target = pipe.target
     target_name = (
         sql_item_name(target, self.flavor, self.get_pipe_schema(pipe))
     )
     if table_exists(target, self, debug=debug):
-        success = self.exec(f"DROP TABLE {target_name}", silent=True, debug=debug) is not None
+        if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
+        success = self.exec(
+            f"DROP TABLE {if_exists_str} {target_name}", silent=True, debug=debug
+        ) is not None
 
     msg = "Success" if success else f"Failed to drop {pipe}."
     return success, msg
 
 
 def clear_pipe(
-        self,
-        pipe: mrsm.Pipe,
-        begin: Union[datetime, int, None] = None,
-        end: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    self,
+    pipe: mrsm.Pipe,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Delete a pipe's data within a bounded or unbounded interval without dropping the table.
 
@@ -2535,7 +2552,7 @@ def get_alter_columns_queries(
     """
     if not pipe.exists(debug=debug):
         return []
-    from meerschaum.utils.sql import sql_item_name
+    from meerschaum.utils.sql import sql_item_name, DROP_IF_EXISTS_FLAVORS
     from meerschaum.utils.dataframe import get_numeric_cols
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
@@ -2691,7 +2708,9 @@ def get_alter_columns_queries(
             f"\nFROM {sql_item_name(temp_table_name, self.flavor, self.get_pipe_schema(pipe))}"
         )
 
-        drop_query = "DROP TABLE " + sql_item_name(
+        if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
+
+        drop_query = f"DROP TABLE {if_exists_str}" + sql_item_name(
             temp_table_name, self.flavor, self.get_pipe_schema(pipe)
         )
         return [
@@ -2882,6 +2901,7 @@ def deduplicate_pipe(
         NO_CTE_FLAVORS,
         get_rename_table_queries,
         NO_SELECT_INTO_FLAVORS,
+        DROP_IF_EXISTS_FLAVORS,
         get_create_table_query,
         format_cte_subquery,
         get_null_replacement,
@@ -3012,6 +3032,7 @@ def deduplicate_pipe(
     ) + f"""
     ORDER BY {index_list_str_ordered}
     """
+    if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
     alter_queries = flatten_list([
         get_rename_table_queries(
             pipe.target, temp_old_table, self.flavor, schema=self.get_pipe_schema(pipe)
@@ -3020,7 +3041,7 @@ def deduplicate_pipe(
             dedup_table, pipe.target, self.flavor, schema=self.get_pipe_schema(pipe)
         ),
         f"""
-        DROP TABLE {temp_old_table_name}
+        DROP TABLE {if_exists_str} {temp_old_table_name}
         """,
     ])
 
@@ -3030,9 +3051,9 @@ def deduplicate_pipe(
 
     results = self.exec_queries(
         alter_queries,
-        break_on_error = True,
-        rollback = True,
-        debug = debug,
+        break_on_error=True,
+        rollback=True,
+        debug=debug,
     )
 
     fail_query = None

{meerschaum-2.4.7 → meerschaum-2.4.8}/meerschaum/connectors/sql/_sql.py

@@ -753,6 +753,7 @@ def to_sql(
         table_exists,
         json_flavors,
         truncate_item_name,
+        DROP_IF_EXISTS_FLAVORS,
     )
     from meerschaum.utils.dataframe import get_json_cols, get_numeric_cols, get_uuid_cols
     from meerschaum.utils.dtypes import are_dtypes_equal, quantize_decimal
@@ -827,12 +828,13 @@ def to_sql(
             'parallel': True,
         })
 
+    if_exists_str = "IF EXISTS" if self.flavor in DROP_IF_EXISTS_FLAVORS else ""
     if self.flavor == 'oracle':
         ### For some reason 'replace' doesn't work properly in pandas,
         ### so try dropping first.
         if if_exists == 'replace' and table_exists(name, self, schema=schema, debug=debug):
             success = self.exec(
-                "DROP TABLE " + sql_item_name(name, 'oracle', schema)
+                f"DROP TABLE {if_exists_str}" + sql_item_name(name, 'oracle', schema)
             ) is not None
             if not success:
                 warn(f"Unable to drop {name}")

{meerschaum-2.4.7 → meerschaum-2.4.8}/meerschaum/core/Pipe/_sync.py

@@ -295,7 +295,7 @@ def sync(
 
         ### CHECKPOINT: Retrieved the DataFrame.
         _checkpoint(**kw)
-        
+
         ### Allow for dataframe generators or iterables.
         if df_is_chunk_generator(df):
             kw['workers'] = p.get_num_workers(kw.get('workers', None))