meerschaum 1.5.11__tar.gz → 1.5.18__tar.gz

{meerschaum-1.5.11/meerschaum.egg-info → meerschaum-1.5.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 1.5.11
+Version: 1.5.18
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-1.5.11 → meerschaum-1.5.18}/meerschaum/_internal/arguments/_parser.py

@@ -234,11 +234,6 @@ groups['sync'].add_argument(
         "Specify the chunksize for syncing and retrieving data. Defaults to 900."
     ),
 )
-groups['sync'].add_argument(
-    '--sync-chunks', action='store_true',
-    help="Sync chunks while fetching data instead of waiting until all have arrived. " +
-    "Similar to --async. WARNING! This can be very dangerous when used with --async.",
-)
 groups['sync'].add_argument(
     '--skip-check-existing', '--allow-duplicates', action='store_true',
     help = (

{meerschaum-1.5.11 → meerschaum-1.5.18}/meerschaum/config/_default.py

@@ -59,7 +59,7 @@ default_system_config = {
             'pandas'           : 'pandas',
         },
         'sql' : {
-            'chunksize'        : 1000000,
+            'chunksize'        : 100000,
             'poolclass'        : 'sqlalchemy.pool.QueuePool',
             'create_engine'    : {
                 'method'       : 'multi',

{meerschaum-1.5.11 → meerschaum-1.5.18}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "1.5.11"
+__version__ = "1.5.18"

{meerschaum-1.5.11 → meerschaum-1.5.18}/meerschaum/connectors/api/APIConnector.py

@@ -22,6 +22,7 @@ class APIConnector(Connector):
     """
 
     IS_INSTANCE: bool = True
+    IS_THREAD_SAFE: bool = False
 
     from ._delete import delete
     from ._post import post

{meerschaum-1.5.11 → meerschaum-1.5.18}/meerschaum/connectors/api/_pipes.py

@@ -336,7 +336,15 @@ def get_pipe_data(
     except Exception as e:
         warn(str(e))
         return None
-    df = parse_df_datetimes(df, debug=debug)
+    df = parse_df_datetimes(
+        df,
+        ignore_cols = [
+            col
+            for col, dtype in pipe.dtypes.items()
+            if 'datetime' not in str(dtype)
+        ],
+        debug = debug,
+    )
     return df
 
 

{meerschaum-1.5.11 → meerschaum-1.5.18}/meerschaum/connectors/sql/SQLConnector.py

@@ -224,15 +224,28 @@ class SQLConnector(Connector):
         return self._engine
 
     @property
-    def DATABASE_URL(self):
+    def DATABASE_URL(self) -> str:
+        """
+        Return the URI connection string (alias for `SQLConnector.URI`.
+        """
         _ = self.engine
         return str(self._engine_str)
 
     @property
-    def URI(self):
+    def URI(self) -> str:
+        """
+        Return the URI connection string.
+        """
         _ = self.engine
         return str(self._engine_str)
 
+    @property
+    def IS_THREAD_SAFE(self) -> str:
+        """
+        Return whether this connector may be multithreaded.
+        """
+        return self.flavor not in ('duckdb',)
+
     @property
     def metadata(self):
         from meerschaum.utils.packages import attempt_import

{meerschaum-1.5.11 → meerschaum-1.5.18}/meerschaum/connectors/sql/_fetch.py

@@ -19,9 +19,10 @@ def fetch(
         end: Union[datetime.datetime, str, None] = None,
         chunk_hook: Optional[Callable[[pd.DataFrame], Any]] = None,
         chunksize: Optional[int] = -1,
+        workers: Optional[int] = None,
         debug: bool = False,
         **kw: Any
-    ) -> Union['pd.DataFrame', None]:
+    ) -> Union['pd.DataFrame', List[Any], None]:
     """Execute the SQL definition and return a Pandas DataFrame.
 
     Parameters
@@ -46,13 +47,24 @@ def fetch(
         The latest datetime to search for data.
         If `end` is `None`, do not bound 
 
+    chunk_hook: Callable[[pd.DataFrame], Any], default None
+        A function to pass to `SQLConnector.read()` that accepts a Pandas DataFrame.
+
+    chunksize: Optional[int], default -1
+        How many rows to load into memory at once (when `chunk_hook` is provided).
+        Otherwise the entire result set is loaded into memory.
+
+    workers: Optional[int], default None
+        How many threads to use when consuming the generator (when `chunk_hook is provided).
+        Defaults to the number of cores.
+
     debug: bool, default False
         Verbosity toggle.
        
     Returns
     -------
     A pandas DataFrame or `None`.
-
+    If `chunk_hook` is not None, return a list of the hook function's results.
     """
     meta_def = self.get_pipe_metadef(
         pipe,
@@ -61,12 +73,32 @@ def fetch(
         debug = debug,
         **kw
     )
-    df = self.read(meta_def, chunk_hook=chunk_hook, chunksize=chunksize, debug=debug)
+    as_hook_results = chunk_hook is not None
+    chunks = self.read(
+        meta_def,
+        chunk_hook = chunk_hook,
+        as_hook_results = as_hook_results,
+        chunksize = chunksize,
+        workers = workers,
+        debug = debug,
+    )
     ### if sqlite, parse for datetimes
-    if self.flavor == 'sqlite':
+    if not as_hook_results and self.flavor == 'sqlite':
         from meerschaum.utils.misc import parse_df_datetimes
-        df = parse_df_datetimes(df, debug=debug)
-    return df
+        ignore_cols = [
+            col
+            for col, dtype in pipe.dtypes.items()
+            if 'datetime' not in str(dtype)
+        ]
+        return (
+            parse_df_datetimes(
+                chunk,
+                ignore_cols = ignore_cols,
+                debug = debug,
+            )
+            for chunk in chunks
+        )
+    return chunks
 
 
 def get_pipe_metadef(

{meerschaum-1.5.11 → meerschaum-1.5.18}/meerschaum/connectors/sql/_pipes.py

@@ -25,7 +25,7 @@ def register_pipe(
 
     ### ensure pipes table exists
     from meerschaum.connectors.sql.tables import get_tables
-    pipes = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
+    pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
 
     if pipe.get_id(debug=debug) is not None:
         return False, f"{pipe} is already registered."
@@ -61,7 +61,7 @@ def register_pipe(
             else parameters
         ),
     }
-    query = sqlalchemy.insert(pipes).values(**values)
+    query = sqlalchemy.insert(pipes_tbl).values(**values)
     result = self.exec(query, debug=debug)
     if result is None:
         return False, f"Failed to register {pipe}."
@@ -111,7 +111,7 @@ def edit_pipe(
 
     ### ensure pipes table exists
     from meerschaum.connectors.sql.tables import get_tables
-    pipes = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
+    pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
 
     import json
     sqlalchemy = attempt_import('sqlalchemy')
@@ -123,8 +123,8 @@ def edit_pipe(
             else parameters
         ),
     }
-    q = sqlalchemy.update(pipes).values(**values).where(
-        pipes.c.pipe_id == pipe.id
+    q = sqlalchemy.update(pipes_tbl).values(**values).where(
+        pipes_tbl.c.pipe_id == pipe.id
     )
 
     result = self.exec(q, debug=debug)
@@ -214,7 +214,7 @@ def fetch_pipes_keys(
         return []
 
     from meerschaum.connectors.sql.tables import get_tables
-    pipes = get_tables(mrsm_instance=self, create=False, debug=debug)['pipes']
+    pipes_tbl = get_tables(mrsm_instance=self, create=False, debug=debug)['pipes']
 
     _params = {}
     for k, v in parameters.items():
@@ -226,30 +226,30 @@ def fetch_pipes_keys(
     ### If a param begins with '_', negate it instead.
     _where = [
         (
-            (pipes.c[key] == val) if not str(val).startswith(negation_prefix)
-            else (pipes.c[key] != key)
+            (pipes_tbl.c[key] == val) if not str(val).startswith(negation_prefix)
+            else (pipes_tbl.c[key] != key)
         ) for key, val in _params.items()
-            if not isinstance(val, (list, tuple)) and key in pipes.c
+            if not isinstance(val, (list, tuple)) and key in pipes_tbl.c
     ]
     select_cols = (
-        [pipes.c.connector_keys, pipes.c.metric_key, pipes.c.location_key]
-        + ([pipes.c.parameters] if tags else [])
+        [pipes_tbl.c.connector_keys, pipes_tbl.c.metric_key, pipes_tbl.c.location_key]
+        + ([pipes_tbl.c.parameters] if tags else [])
     )
 
     q = sqlalchemy.select(*select_cols).where(sqlalchemy.and_(True, *_where))
 
     ### Parse IN params and add OR IS NULL if None in list.
     for c, vals in cols.items():
-        if not isinstance(vals, (list, tuple)) or not vals or not c in pipes.c:
+        if not isinstance(vals, (list, tuple)) or not vals or not c in pipes_tbl.c:
             continue
         _in_vals, _ex_vals = separate_negation_values(vals)
         ### Include params (positive)
         q = (
-            q.where(pipes.c[c].in_(_in_vals)) if None not in _in_vals
-            else q.where(sqlalchemy.or_(pipes.c[c].in_(_in_vals), pipes.c[c].is_(None)))
+            q.where(pipes_tbl.c[c].in_(_in_vals)) if None not in _in_vals
+            else q.where(sqlalchemy.or_(pipes_tbl.c[c].in_(_in_vals), pipes_tbl.c[c].is_(None)))
         ) if _in_vals else q
         ### Exclude params (negative)
-        q = q.where(pipes.c[c].not_in(_ex_vals)) if _ex_vals else q
+        q = q.where(pipes_tbl.c[c].not_in(_ex_vals)) if _ex_vals else q
 
     ### Finally, parse tags.
     _in_tags, _ex_tags = separate_negation_values(tags)
@@ -257,7 +257,7 @@ def fetch_pipes_keys(
     for nt in _in_tags:
         ors.append(
             sqlalchemy.cast(
-                pipes.c['parameters'],
+                pipes_tbl.c['parameters'],
                 sqlalchemy.String,
             ).like(f'%"tags":%"{nt}"%')
         )
@@ -266,17 +266,17 @@ def fetch_pipes_keys(
     for xt in _ex_tags:
         ors.append(
             sqlalchemy.cast(
-                pipes.c['parameters'],
+                pipes_tbl.c['parameters'],
                 sqlalchemy.String,
             ).not_like(f'%"tags":%"{xt}"%')
         )
     q = q.where(sqlalchemy.and_(sqlalchemy.or_(*ors).self_group())) if ors else q
-    loc_asc = sqlalchemy.asc(pipes.c['location_key'])
+    loc_asc = sqlalchemy.asc(pipes_tbl.c['location_key'])
     if self.flavor not in OMIT_NULLSFIRST_FLAVORS:
         loc_asc = sqlalchemy.nullsfirst(loc_asc)
     q = q.order_by(
-        sqlalchemy.asc(pipes.c['connector_keys']),
-        sqlalchemy.asc(pipes.c['metric_key']),
+        sqlalchemy.asc(pipes_tbl.c['connector_keys']),
+        sqlalchemy.asc(pipes_tbl.c['metric_key']),
         loc_asc,
     )
 
@@ -565,9 +565,9 @@ def delete_pipe(
 
     ### ensure pipes table exists
     from meerschaum.connectors.sql.tables import get_tables
-    pipes = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
+    pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
 
-    q = sqlalchemy.delete(pipes).where(pipes.c.pipe_id == pipe.id)
+    q = sqlalchemy.delete(pipes_tbl).where(pipes_tbl.c.pipe_id == pipe.id)
     if not self.exec(q, debug=debug):
         return False, f"Failed to delete registration for {pipe}."
 
@@ -733,9 +733,30 @@ def get_pipe_data(
         from meerschaum.utils.misc import parse_df_datetimes
         from meerschaum.utils.packages import import_pandas
         pd = import_pandas()
-        ### NOTE: We have to consume the iterator here to ensure that datatimes are parsed correctly
-        df = parse_df_datetimes(df, debug=debug) if isinstance(df, pd.DataFrame) else (
-            [parse_df_datetimes(c, debug=debug) for c in df]
+        ### NOTE: We have to consume the iterator here to ensure that datetimes are parsed correctly
+        df = (
+            parse_df_datetimes(
+                df,
+                ignore_cols = [
+                    col
+                    for col, dtype in pipe.dtypes.items()
+                    if 'datetime' not in str(dtype)
+                ],
+                debug = debug,
+            ) if isinstance(df, pd.DataFrame) else (
+                [
+                    parse_df_datetimes(
+                        c,
+                        ignore_cols = [
+                            col
+                            for col, dtype in pipe.dtypes.items()
+                            if 'datetime' not in str(dtype)
+                        ],
+                        debug = debug,
+                    )
+                    for c in df
+                ]
+            )
         )
         for col, typ in dtypes.items():
             if typ != 'json':
@@ -947,15 +968,15 @@ def get_pipe_id(
     import json
     sqlalchemy = attempt_import('sqlalchemy')
     from meerschaum.connectors.sql.tables import get_tables
-    pipes = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
+    pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
 
-    query = sqlalchemy.select(pipes.c.pipe_id).where(
-        pipes.c.connector_keys == pipe.connector_keys
+    query = sqlalchemy.select(pipes_tbl.c.pipe_id).where(
+        pipes_tbl.c.connector_keys == pipe.connector_keys
     ).where(
-        pipes.c.metric_key == pipe.metric_key
+        pipes_tbl.c.metric_key == pipe.metric_key
     ).where(
-        (pipes.c.location_key == pipe.location_key) if pipe.location_key is not None
-        else pipes.c.location_key.is_(None)
+        (pipes_tbl.c.location_key == pipe.location_key) if pipe.location_key is not None
+        else pipes_tbl.c.location_key.is_(None)
     )
     _id = self.value(query, debug=debug, silent=pipe.temporary)
     if _id is not None:
@@ -980,10 +1001,10 @@ def get_pipe_attributes(
     if pipe.get_id(debug=debug) is None:
         return {}
 
-    pipes = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
+    pipes_tbl = get_tables(mrsm_instance=self, create=(not pipe.temporary), debug=debug)['pipes']
 
     try:
-        q = sqlalchemy.select(pipes).where(pipes.c.pipe_id == pipe.id)
+        q = sqlalchemy.select(pipes_tbl).where(pipes_tbl.c.pipe_id == pipe.id)
         if debug:
             dprint(q)
         attributes = (
@@ -1113,14 +1134,13 @@ def sync_pipe(
         if alter_cols_queries:
             if not self.exec_queries(alter_cols_queries, debug=debug):
                 warn(f"Failed to alter columns for {pipe}.")
-
+            else:
+                pipe.infer_dtypes(persist=True)
 
     unseen_df, update_df, delta_df = (
         pipe.filter_existing(
             df,
             chunksize = chunksize,
-            begin = begin,
-            end = end,
             debug = debug,
             **kw
         ) if check_existing else (df, None, df)
@@ -1373,6 +1393,8 @@ def sync_pipe_inplace(
     if alter_cols_queries:
         if not self.exec_queries(alter_cols_queries, debug=debug):
             warn(f"Failed to alter columns for {pipe}.")
+        else:
+            pipe.infer_dtypes(persist=True)
 
     if not check_existing:
         new_count = self.value(f"SELECT COUNT(*) FROM {new_table_name}", debug=debug)
@@ -1396,6 +1418,8 @@ def sync_pipe_inplace(
         pipe,
         begin = begin,
         end = end,
+        begin_add_minutes = -1,
+        end_add_minutes = 1,
         params = params,
         debug = debug,
         order = None,
@@ -1403,7 +1427,7 @@ def sync_pipe_inplace(
 
     create_backtrack_query = (
         (
-            f"WITH backtrack_def AS ({backtrack_def})\n"
+            f"WITH backtrack_def AS (\n{backtrack_def}\n)\n"
             + f"SELECT *\nINTO {backtrack_table_name}\nFROM backtrack_def"
         ) if self.flavor not in ('sqlite', 'oracle', 'mysql', 'mariadb', 'duckdb')
         else (
@@ -2314,7 +2338,12 @@ def get_add_columns_queries(
     if not pipe.exists(debug=debug):
         return []
     import copy
-    from meerschaum.utils.sql import get_pd_type, get_db_type, sql_item_name
+    from meerschaum.utils.sql import (
+        get_pd_type,
+        get_db_type,
+        sql_item_name,
+        SINGLE_ALTER_TABLE_FLAVORS,
+    )
     from meerschaum.utils.misc import flatten_list
     table_obj = self.get_pipe_table(pipe, debug=debug)
     df_cols_types = (
@@ -2346,13 +2375,25 @@ def get_add_columns_queries(
         ) for col in new_cols
     }
 
-    query = "ALTER TABLE " + sql_item_name(pipe.target, self.flavor)
+    alter_table_query = "ALTER TABLE " + sql_item_name(pipe.target, self.flavor)
+    queries = []
     for col, typ in new_cols_types.items():
-        query += "\nADD " + sql_item_name(col, self.flavor) + " " + typ + ","
-    query = query[:-1]
+        add_col_query = "\nADD " + sql_item_name(col, self.flavor) + " " + typ + ","
+
+        if self.flavor in SINGLE_ALTER_TABLE_FLAVORS:
+            queries.append(alter_table_query + add_col_query[:-1])
+        else:
+            alter_table_query += add_col_query
+
+    ### For most flavors, only one query is required.
+    ### This covers SQLite which requires one query per column.
+    if not queries:
+        queries.append(alter_table_query[:-1])
+
     if self.flavor != 'duckdb':
-        return [query]
+        return queries
 
+    ### NOTE: For DuckDB, we must drop and rebuild the indices.
     drop_index_queries = list(flatten_list(
         [q for ix, q in self.get_drop_index_queries(pipe, debug=debug).items()]
     ))
@@ -2360,7 +2401,7 @@ def get_add_columns_queries(
         [q for ix, q in self.get_create_index_queries(pipe, debug=debug).items()]
     ))
 
-    return drop_index_queries + [query] + create_index_queries
+    return drop_index_queries + queries + create_index_queries
 
 
 def get_alter_columns_queries(
@@ -2385,13 +2426,13 @@ def get_alter_columns_queries(
     -------
     A list of the `ALTER TABLE` SQL query or queries to be executed on the provided connector.
     """
-    if self.flavor == 'sqlite':
-        return []
     if not pipe.exists(debug=debug):
         return []
     from meerschaum.utils.sql import get_pd_type, get_db_type, sql_item_name
-    from meerschaum.utils.misc import flatten_list
+    from meerschaum.utils.misc import flatten_list, generate_password
     table_obj = self.get_pipe_table(pipe, debug=debug)
+    target = pipe.target
+    session_id = generate_password(3)
     df_cols_types = (
         {col: str(typ) for col, typ in df.dtypes.items()}
         if not isinstance(df, dict) else df
@@ -2411,15 +2452,59 @@ def get_alter_columns_queries(
         for col in altered_cols
     }
 
+    if self.flavor == 'sqlite':
+        temp_table_name = '_' + session_id + '_' + target
+        rename_query = (
+            "ALTER TABLE "
+            + sql_item_name(target, self.flavor)
+            + " RENAME TO "
+            + sql_item_name(temp_table_name, self.flavor)
+        )
+        create_query = (
+            "CREATE TABLE "
+            + sql_item_name(target, self.flavor)
+            + " (\n"
+        )
+        for col_name, col_obj in table_obj.columns.items():
+            create_query += (
+                sql_item_name(col_name, self.flavor)
+                + " "
+                + (str(col_obj.type) if col_name not in altered_cols else text_type)
+                + ",\n"
+            )
+        create_query = create_query[:-2] + "\n)"
+
+        insert_query = (
+            "INSERT INTO "
+            + sql_item_name(target, self.flavor)
+            + "\nSELECT\n"
+        )
+        for col_name, col_obj in table_obj.columns.items():
+            new_col_str = (
+                sql_item_name(col_name, self.flavor)
+                if col_name not in altered_cols
+                else f"CAST({sql_item_name(col_name, self.flavor)} AS {text_type})"
+            )
+            insert_query += new_col_str + ",\n"
+        insert_query = insert_query[:-2] + f"\nFROM {sql_item_name(temp_table_name, self.flavor)}"
+
+        drop_query = f"DROP TABLE {sql_item_name(temp_table_name, self.flavor)}"
+        return [
+            rename_query,
+            create_query,
+            insert_query,
+            drop_query,
+        ]
+
     queries = []
     if self.flavor == 'oracle':
-        add_query = "ALTER TABLE " + sql_item_name(pipe.target, self.flavor)
+        add_query = "ALTER TABLE " + sql_item_name(target, self.flavor)
         for col, typ in altered_cols_types.items():
             add_query += "\nADD " + sql_item_name(col + '_temp', self.flavor) + " " + typ + ","
         add_query = add_query[:-1]
         queries.append(add_query)
 
-        populate_temp_query = "UPDATE " + sql_item_name(pipe.target, self.flavor)
+        populate_temp_query = "UPDATE " + sql_item_name(target, self.flavor)
         for col, typ in altered_cols_types.items():
             populate_temp_query += (
                 "\nSET " + sql_item_name(col + '_temp', self.flavor)
@@ -2428,7 +2513,7 @@ def get_alter_columns_queries(
         populate_temp_query = populate_temp_query[:-1]
         queries.append(populate_temp_query)
 
-        set_old_cols_to_null_query = "UPDATE " + sql_item_name(pipe.target, self.flavor)
+        set_old_cols_to_null_query = "UPDATE " + sql_item_name(target, self.flavor)
         for col, typ in altered_cols_types.items():
             set_old_cols_to_null_query += (
                 "\nSET " + sql_item_name(col, self.flavor)
@@ -2437,7 +2522,7 @@ def get_alter_columns_queries(
         set_old_cols_to_null_query = set_old_cols_to_null_query[:-1]
         queries.append(set_old_cols_to_null_query)
 
-        alter_type_query = "ALTER TABLE " + sql_item_name(pipe.target, self.flavor)
+        alter_type_query = "ALTER TABLE " + sql_item_name(target, self.flavor)
         for col, typ in altered_cols_types.items():
             alter_type_query += (
                 "\nMODIFY " + sql_item_name(col, self.flavor) + ' '
@@ -2446,7 +2531,7 @@ def get_alter_columns_queries(
         alter_type_query = alter_type_query[:-1]
         queries.append(alter_type_query)
 
-        set_old_to_temp_query = "UPDATE " + sql_item_name(pipe.target, self.flavor)
+        set_old_to_temp_query = "UPDATE " + sql_item_name(target, self.flavor)
         for col, typ in altered_cols_types.items():
             set_old_to_temp_query += (
                 "\nSET " + sql_item_name(col, self.flavor)
@@ -2455,7 +2540,7 @@ def get_alter_columns_queries(
         set_old_to_temp_query = set_old_to_temp_query[:-1]
         queries.append(set_old_to_temp_query)
 
-        drop_temp_query = "ALTER TABLE " + sql_item_name(pipe.target, self.flavor)
+        drop_temp_query = "ALTER TABLE " + sql_item_name(target, self.flavor)
         for col, typ in altered_cols_types.items():
             drop_temp_query += (
                 "\nDROP COLUMN " + sql_item_name(col + '_temp', self.flavor) + ','
@@ -2466,7 +2551,7 @@ def get_alter_columns_queries(
         return queries
 
 
-    query = "ALTER TABLE " + sql_item_name(pipe.target, self.flavor)
+    query = "ALTER TABLE " + sql_item_name(target, self.flavor)
     for col, typ in altered_cols_types.items():
         alter_col_prefix = (
             'ALTER' if self.flavor not in ('mysql', 'mariadb', 'oracle')