meerschaum 2.6.5__tar.gz → 2.6.7__tar.gz

{meerschaum-2.6.5/meerschaum.egg-info → meerschaum-2.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.6.5
+Version: 2.6.7
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -101,6 +101,7 @@ Requires-Dist: pytest>=6.2.2; extra == "dev-tools"
 Requires-Dist: pytest-xdist>=3.2.1; extra == "dev-tools"
 Requires-Dist: heartrate>=0.2.1; extra == "dev-tools"
 Requires-Dist: build>=1.2.1; extra == "dev-tools"
+Requires-Dist: attrs>=24.2.0; extra == "dev-tools"
 Provides-Extra: setup
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.1.2; extra == "docs"

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/_internal/docs/index.py

@@ -741,10 +741,11 @@ def init_dash(dash_app):
       <li><code>meerschaum.utils.sql.get_null_replacement()</code></li>
       <li><code>meerschaum.utils.sql.get_db_version()</code></li>
       <li><code>meerschaum.utils.sql.get_rename_table_queries()</code></li>
-      <li><code>meerschaum.utils.sql.get_create_table_query()</code></li>
+      <li><code>meerschaum.utils.sql.get_create_table_queries()</code></li>
       <li><code>meerschaum.utils.sql.wrap_query_with_cte()</code></li>
       <li><code>meerschaum.utils.sql.format_cte_subquery()</code></li>
       <li><code>meerschaum.utils.sql.session_execute()</code></li>
+      <li><code>meerschaum.utils.sql.get_reset_autoincrement_queries()</code></li>
     </ul>
   </details>
   </ul>

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.6.5"
+__version__ = "2.6.7"

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/connectors/sql/_SQLConnector.py

@@ -11,7 +11,7 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Any, Union
 
 from meerschaum.connectors import Connector
-from meerschaum.utils.warnings import error
+from meerschaum.utils.warnings import error, warn
 
 
 class SQLConnector(Connector):
@@ -212,7 +212,6 @@ class SQLConnector(Connector):
 
         if connect:
             if not self.test_connection(debug=debug):
-                from meerschaum.utils.warnings import warn
                 warn(f"Failed to connect with connector '{self}'!", stack=False)
 
     @property
@@ -230,8 +229,11 @@ class SQLConnector(Connector):
 
     @property
     def engine(self):
-        import os, threading
-        ### build the sqlalchemy engine
+        """
+        Return the SQLAlchemy engine connected to the configured database.
+        """
+        import os
+        import threading
         if '_engine' not in self.__dict__:
             self._engine, self._engine_str = self.create_engine(include_uri=True)
 
@@ -242,13 +244,14 @@ class SQLConnector(Connector):
         if not same_process:
             self._pid = os.getpid()
             self._thread = threading.current_thread()
-            from meerschaum.utils.warnings import warn
-            warn(f"Different PID detected. Disposing of connections...")
+            warn("Different PID detected. Disposing of connections...")
             self._engine.dispose()
 
         ### handle different threads
         if not same_thread:
-            pass
+            if self.flavor == 'duckdb':
+                warn("Different thread detected.")
+                self._engine.dispose()
 
         return self._engine
 

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/connectors/sql/_pipes.py

@@ -1349,9 +1349,9 @@ def create_pipe_table_from_df(
             for col, typ in df.dtypes.items()
         },
         **{
-            col: 'int'
+            col: str(df.dtypes.get(col, 'int'))
             for col_ix, col in pipe.columns.items()
-            if col_ix != 'primary'
+            if col and col_ix != 'primary'
         },
         **{
             col: 'uuid'
@@ -1696,7 +1696,9 @@ def sync_pipe(
             )
             for col, typ in update_df.dtypes.items()
         }
-        temp_pipe.__dict__['_columns_types_timestamp'] = time.perf_counter()
+        now_ts = time.perf_counter()
+        temp_pipe.__dict__['_columns_types_timestamp'] = now_ts
+        temp_pipe.__dict__['_skip_check_indices'] = True
         temp_success, temp_msg = temp_pipe.sync(update_df, check_existing=False, debug=debug)
         if not temp_success:
             return temp_success, temp_msg
@@ -1867,6 +1869,7 @@ def sync_pipe_inplace(
     )
     pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
     upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
+    static = pipe.parameters.get('static', False)
     database = getattr(self, 'database', self.parse_uri(self.URI).get('database', None))
     primary_key = pipe.columns.get('primary', None)
     autoincrement = pipe.parameters.get('autoincrement', False)
@@ -1944,7 +1947,7 @@ def sync_pipe_inplace(
         schema=internal_schema,
         database=database,
         debug=debug,
-    )
+    ) if not static else pipe.get_columns_types(debug=debug)
     if not new_cols_types:
         return False, f"Failed to get new columns for {pipe}."
 
@@ -2049,7 +2052,7 @@ def sync_pipe_inplace(
         schema=internal_schema,
         database=database,
         debug=debug,
-    ) if not upsert else new_cols_types
+    ) if not (upsert or static) else new_cols_types
 
     common_cols = [col for col in new_cols if col in backtrack_cols_types]
     on_cols = {
@@ -2123,7 +2126,7 @@ def sync_pipe_inplace(
         schema=internal_schema,
         database=database,
         debug=debug,
-    ) if not upsert else new_cols_types
+    ) if not (upsert or static) else new_cols_types
 
     ### This is a weird bug on SQLite.
     ### Sometimes the backtrack dtypes are all empty strings.
@@ -2812,12 +2815,13 @@ def get_pipe_columns_indices(
     pipe: mrsm.Pipe
         The pipe to be queried against.
 
-
     Returns
     -------
     A dictionary mapping columns names to lists of dictionaries.
     The dictionaries in the lists contain the name and type of the indices.
     """
+    if pipe.__dict__.get('_skip_check_indices', False):
+        return {}
     from meerschaum.utils.sql import get_table_cols_indices
     return get_table_cols_indices(
         pipe.target,

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/connectors/valkey/_pipes.py

@@ -305,6 +305,14 @@ def drop_pipe(
     -------
     A `SuccessTuple` indicating success.
     """
+    for chunk_begin, chunk_end in pipe.get_chunk_bounds(debug=debug):
+        clear_chunk_success, clear_chunk_msg = pipe.clear(
+            begin=chunk_begin,
+            end=chunk_end,
+            debug=debug,
+        )
+        if not clear_chunk_success:
+            return clear_chunk_success, clear_chunk_msg
     try:
         self.drop_table(pipe.target, debug=debug)
     except Exception as e:
@@ -547,7 +555,7 @@ def sync_pipe(
                 new_dtypes[col] = 'string'
                 df[col] = df[col].astype('string')
 
-    if new_dtypes and not static:
+    if new_dtypes and (not static or not valkey_dtypes):
         valkey_dtypes.update(new_dtypes)
         if 'valkey' not in pipe.parameters:
             pipe.parameters['valkey'] = {}
@@ -560,19 +568,19 @@ def sync_pipe(
     unseen_df, update_df, delta_df = (
         pipe.filter_existing(df, include_unchanged_columns=True, debug=debug)
         if check_existing and not upsert
-        else (df, None, df)
+        else (None, df, df)
     )
     num_insert = len(unseen_df) if unseen_df is not None else 0
     num_update = len(update_df) if update_df is not None else 0
     msg = (
         f"Inserted {num_insert}, updated {num_update} rows."
         if not upsert
-        else f"Upserted {num_insert} rows."
+        else f"Upserted {num_update} rows."
     )
     if len(delta_df) == 0:
         return True, msg
 
-    unseen_docs = unseen_df.to_dict(orient='records')
+    unseen_docs = unseen_df.to_dict(orient='records') if unseen_df is not None else []
     unseen_indices_docs = _serialize_indices_docs(unseen_docs)
     unseen_ix_vals = {
         get_document_key(doc, indices, table_name): serialize_document(doc)
@@ -599,11 +607,53 @@ def sync_pipe(
         get_document_key(doc, indices, table_name): doc
         for doc in update_docs
     }
-    for key, doc in update_ix_docs.items():
+    existing_docs_data = {
+        key: self.get(key)
+        for key in update_ix_docs
+    } if pipe.exists(debug=debug) else {}
+    existing_docs = {
+        key: json.loads(data)
+        for key, data in existing_docs_data.items()
+        if data
+    }
+    new_update_docs = {
+        key: doc
+        for key, doc in update_ix_docs.items()
+        if key not in existing_docs
+    }
+    new_ix_vals = {
+        get_document_key(doc, indices, table_name): serialize_document(doc)
+        for doc in new_update_docs.values()
+    }
+    for key, val in new_ix_vals.items():
         try:
-            old_doc = json.loads(self.get(key))
-            old_doc.update(doc)
-            self.set(key, serialize_document(old_doc))
+            self.set(key, val)
+        except Exception as e:
+            return False, f"Failed to set keys for {pipe}:\n{e}"
+
+    old_update_docs = {
+        key: {
+            **existing_docs[key],
+            **doc
+        }
+        for key, doc in update_ix_docs.items()
+        if key in existing_docs
+    }
+    new_indices_docs = _serialize_indices_docs([doc for doc in new_update_docs.values()])
+    try:
+        if new_indices_docs:
+            self.push_docs(
+                new_indices_docs,
+                pipe.target,
+                datetime_column=dt_col,
+                debug=debug,
+            )
+    except Exception as e:
+        return False, f"Failed to upsert '{pipe.target}':\n{e}"
+
+    for key, doc in old_update_docs.items():
+        try:
+            self.set(key, serialize_document(doc))
         except Exception as e:
             return False, f"Failed to set keys for {pipe}:\n{e}"
 
@@ -667,9 +717,6 @@ def clear_pipe(
     -------
     A `SuccessTuple` indicating success.
     """
-    if begin is None and end is None and params is None:
-        return self.drop_pipe(pipe, debug=debug)
-
     dt_col = pipe.columns.get('datetime', None)
 
     existing_df = pipe.get_data(

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/core/Pipe/_attributes.py

@@ -85,7 +85,7 @@ def columns(self) -> Union[Dict[str, str], None]:
     if not isinstance(cols, dict):
         cols = {}
         self.parameters['columns'] = cols
-    return cols
+    return {col_ix: col for col_ix, col in cols.items() if col}
 
 
 @columns.setter
@@ -127,11 +127,11 @@ def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
     ) + [
         col
         for col_ix, col in _columns.items()
-        if col_ix != 'datetime'
+        if col and col_ix != 'datetime'
     ]))
     return {
         **({'unique': unique_cols} if len(unique_cols) > 1 else {}),
-        **_columns,
+        **{col_ix: col for col_ix, col in _columns.items() if col},
         **_indices
     }
 
@@ -371,8 +371,7 @@ def get_columns_types(
 
     now = time.perf_counter()
     cache_seconds = STATIC_CONFIG['pipes']['static_schema_cache_seconds']
-    static = self.parameters.get('static', False)
-    if not static:
+    if not self.static:
         refresh = True
     if refresh:
         _ = self.__dict__.pop('_columns_types_timestamp', None)
@@ -416,7 +415,11 @@ def get_columns_indices(
     from meerschaum.utils.warnings import dprint
 
     now = time.perf_counter()
-    exists_timeout_seconds = STATIC_CONFIG['pipes']['exists_timeout_seconds']
+    cache_seconds = (
+        STATIC_CONFIG['pipes']['static_schema_cache_seconds']
+        if self.static
+        else STATIC_CONFIG['pipes']['exists_timeout_seconds']
+    )
     if refresh:
         _ = self.__dict__.pop('_columns_indices_timestamp', None)
         _ = self.__dict__.pop('_columns_indices', None)
@@ -425,7 +428,7 @@ def get_columns_indices(
         columns_indices_timestamp = self.__dict__.get('_columns_indices_timestamp', None)
         if columns_indices_timestamp is not None:
             delta = now - columns_indices_timestamp
-            if delta < exists_timeout_seconds:
+            if delta < cache_seconds:
                 if debug:
                     dprint(
                         f"Returning cached `columns_indices` for {self} "

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/core/Pipe/_drop.py

@@ -9,6 +9,7 @@ Drop a Pipe's table but keep its registration
 from __future__ import annotations
 from meerschaum.utils.typing import SuccessTuple, Any
 
+
 def drop(
     self,
     debug: bool = False,
@@ -39,4 +40,8 @@ def drop(
 
     with Venv(get_connector_plugin(self.instance_connector)):
         result = self.instance_connector.drop_pipe(self, debug=debug, **kw)
+
+    _ = self.__dict__.pop('_exists', None)
+    _ = self.__dict__.pop('_exists_timestamp', None)
+
     return result

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/utils/dataframe.py

@@ -234,10 +234,13 @@ def filter_unseen_df(
     cast_dt_cols = True
     try:
         for col, typ in dt_dtypes.items():
+            strip_utc = (
+                (dtypes or {}).get(col, 'datetime') == 'datetime64[ns]'
+            )
             if col in old_df.columns:
-                old_df[col] = coerce_timezone(old_df[col])
+                old_df[col] = coerce_timezone(old_df[col], strip_utc=strip_utc)
             if col in new_df.columns:
-                new_df[col] = coerce_timezone(new_df[col])
+                new_df[col] = coerce_timezone(new_df[col], strip_utc=strip_utc)
         cast_dt_cols = False
     except Exception as e:
         warn(f"Could not cast datetime columns:\n{e}")
@@ -1284,7 +1287,8 @@ def query_df(
                     if debug:
                         dprint(f"Casting column '{datetime_column}' to UTC...")
                     df[datetime_column] = coerce_timezone(df[datetime_column], strip_utc=False)
-                dprint(f"Using datetime bounds:\n{begin=}\n{end=}")
+                if debug:
+                    dprint(f"Using datetime bounds:\n{begin=}\n{end=}")
 
     in_ex_params = get_in_ex_params(params)
 

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/utils/dtypes/__init__.py

@@ -270,9 +270,9 @@ def coerce_timezone(
         pandas = mrsm.attempt_import('pandas')
         dd = mrsm.attempt_import('dask.dataframe') if is_dask else None
         dt_series = (
-            pandas.to_datetime(dt, utc=True)
+            pandas.to_datetime(dt, utc=True, format='ISO8601')
             if dd is None
-            else dd.to_datetime(dt, utc=True)
+            else dd.to_datetime(dt, utc=True, format='ISO8601')
         )
         if strip_utc:
             dt_series = dt_series.apply(lambda x: x.replace(tzinfo=None))

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/utils/packages/__init__.py

@@ -1467,7 +1467,6 @@ def import_pandas(
     """
     Quality-of-life function to attempt to import the configured version of `pandas`.
     """
-    import sys
     pandas_module_name = pandas_name()
     global emitted_pandas_warning
 
@@ -1482,11 +1481,11 @@ def import_pandas(
                         + f"'{pandas_module_name}'"
                         + "\n   Features may not work as expected."
                     ),
-                    stack = False,
+                    stack=False,
                 )
 
     pytz = attempt_import('pytz', debug=debug, lazy=False, **kw)
-    pandas = attempt_import('pandas', debug=debug, lazy=False, **kw)
+    pandas, pyarrow = attempt_import('pandas', 'pyarrow', debug=debug, lazy=False, **kw)
     pd = attempt_import(pandas_module_name, debug=debug, lazy=lazy, **kw)
     return pd
 

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/utils/packages/_packages.py

@@ -103,6 +103,7 @@ packages: Dict[str, Dict[str, str]] = {
         'pytest_xdist'               : 'pytest-xdist>=3.2.1',
         'heartrate'                  : 'heartrate>=0.2.1',
         'build'                      : 'build>=1.2.1',
+        'attrs'                      : 'attrs>=24.2.0',
     },
     'setup': {
     },

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum/utils/sql.py

@@ -87,9 +87,11 @@ update_queries = {
         WHERE {date_bounds_subquery}
     """,
     'mysql-upsert': """
-        REPLACE INTO {target_table_name} ({patch_cols_str})
+        INSERT {ignore}INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
+        {on_duplicate_key_update}
+            {cols_equal_values}
     """,
     'mariadb': """
         UPDATE {target_table_name} AS f
@@ -99,9 +101,11 @@ update_queries = {
         WHERE {date_bounds_subquery}
     """,
     'mariadb-upsert': """
-        REPLACE INTO {target_table_name} ({patch_cols_str})
+        INSERT {ignore}INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
         FROM {patch_table_name}
+        {on_duplicate_key_update}
+            {cols_equal_values}
     """,
     'mssql': """
         MERGE {target_table_name} f
@@ -1578,6 +1582,19 @@ def get_update_queries(
         f"     UPDATE {sets_subquery('', 'p.')}"
     )
 
+    cols_equal_values = '\n,'.join(
+        [
+            f"{sql_item_name(c_name, flavor)} = VALUES({sql_item_name(c_name, flavor)})"
+            for c_name, c_type in value_cols
+        ]
+    )
+    on_duplicate_key_update = (
+        "ON DUPLICATE KEY UPDATE"
+        if value_cols
+        else ""
+    )
+    ignore = "IGNORE " if not value_cols else ""
+
     return [
         base_query.format(
             sets_subquery_none=sets_subquery('', 'p.'),
@@ -1594,6 +1611,9 @@ def get_update_queries(
             coalesce_join_cols_str=coalesce_join_cols_str,
             update_or_nothing=update_or_nothing,
             when_matched_update_sets_subquery_none=when_matched_update_sets_subquery_none,
+            cols_equal_values=cols_equal_values,
+            on_duplicate_key_update=on_duplicate_key_update,
+            ignore=ignore,
         )
         for base_query in base_queries
     ]
@@ -2179,7 +2199,25 @@ def get_reset_autoincrement_queries(
     debug: bool = False,
 ) -> List[str]:
     """
-    Return a list of queries to reset a table's auto-increment counter.
+    Return a list of queries to reset a table's auto-increment counter to the next largest value.
+
+    Parameters
+    ----------
+    table: str
+        The name of the table on which the auto-incrementing column exists.
+
+    column: str
+        The name of the auto-incrementing column.
+
+    connector: mrsm.connectors.SQLConnector
+        The SQLConnector to the database on which the table exists.
+
+    schema: Optional[str], default None
+        The schema of the table. Defaults to `connector.schema`.
+
+    Returns
+    -------
+    A list of queries to be executed to reset the auto-incrementing column.
     """
     if not table_exists(table, connector, schema=schema, debug=debug):
         return []

{meerschaum-2.6.5 → meerschaum-2.6.7/meerschaum.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.6.5
+Version: 2.6.7
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -101,6 +101,7 @@ Requires-Dist: pytest>=6.2.2; extra == "dev-tools"
 Requires-Dist: pytest-xdist>=3.2.1; extra == "dev-tools"
 Requires-Dist: heartrate>=0.2.1; extra == "dev-tools"
 Requires-Dist: build>=1.2.1; extra == "dev-tools"
+Requires-Dist: attrs>=24.2.0; extra == "dev-tools"
 Provides-Extra: setup
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.1.2; extra == "docs"

{meerschaum-2.6.5 → meerschaum-2.6.7}/meerschaum.egg-info/requires.txt

@@ -125,6 +125,7 @@ pytest>=6.2.2
 pytest-xdist>=3.2.1
 heartrate>=0.2.1
 build>=1.2.1
+attrs>=24.2.0
 
 [docs]
 mkdocs>=1.1.2

{meerschaum-2.6.5 → meerschaum-2.6.7}/tests/test_deduplicate.py

@@ -6,28 +6,22 @@
 Test pipe deduplication syncs.
 """
 
-import pytest
 import json
 from datetime import datetime, timedelta
+
+import pytest
+
+import meerschaum as mrsm
+
 from tests import debug
-from tests.pipes import all_pipes, stress_pipes, remote_pipes
 from tests.connectors import conns, get_flavors
-from tests.test_users import test_register_user
-import meerschaum as mrsm
-from meerschaum import Pipe
-from meerschaum.actions import actions
 
-@pytest.fixture(autouse=True)
-def run_before_and_after(flavor: str):
-    test_register_user(flavor)
-    yield
 
 @pytest.mark.parametrize("flavor", get_flavors())
 def test_deduplicate_default(flavor: str):
     """
     Test that verification will fill any backtracked docs.
     """
-    import pandas as pd
     from meerschaum.utils.sql import sql_item_name
     from meerschaum.utils.dataframe import parse_df_datetimes
     conn = conns[flavor]
@@ -70,16 +64,16 @@ def test_deduplicate_without_instance_method(flavor: str):
     """
     Test that verification will fill any backtracked docs.
     """
-    import pandas as pd
-    from meerschaum.utils.sql import sql_item_name
     from meerschaum.utils.dataframe import parse_df_datetimes
     conn = conns[flavor]
     if conn.type != 'sql':
         return
+    pipe = mrsm.Pipe('test', 'deduplicate', 'chunked', instance=conn)
+    pipe.delete()
     pipe = mrsm.Pipe(
         'test', 'deduplicate', 'chunked',
-        instance = conn,
-        columns = {
+        instance=conn,
+        columns={
             'datetime': 'datetime',
             'id': 'id',
         }

{meerschaum-2.6.5 → meerschaum-2.6.7}/tests/test_pipe_data.py

@@ -10,16 +10,6 @@ from meerschaum import Pipe
 
 from tests.connectors import conns, get_flavors
 from tests import debug
-from tests.test_users import test_register_user as _test_register_user
-
-
-@pytest.fixture(autouse=True)
-def run_before_and_after(flavor: str):
-    """
-    Ensure the test user is registered before running tests.
-    """
-    _test_register_user(flavor)
-    yield
 
 
 @pytest.mark.parametrize("flavor", get_flavors())