meerschaum 2.7.2__py3-none-any.whl → 2.7.4__py3-none-any.whl

meerschaum/_internal/arguments/_parse_arguments.py

@@ -240,6 +240,8 @@ def parse_synonyms(
         args_dict['instance'] = args_dict['mrsm_instance']
     if args_dict.get('skip_check_existing', None):
         args_dict['check_existing'] = False
+    if args_dict.get('skip_enforce_dtypes', None):
+        args_dict['enforce_dtypes'] = False
     if args_dict.get('venv', None) in ('None', '[None]'):
         args_dict['venv'] = None
     chunk_minutes = args_dict.get('chunk_minutes', None)

meerschaum/_internal/arguments/_parser.py

@@ -87,7 +87,7 @@ def parse_datetime(dt_str: str) -> Union[datetime, int, str]:
                 dt = round_time(dt, round_delta)
         else:
             dt = dateutil_parser.parse(dt_str)
-    except Exception as e:
+    except Exception:
         dt = None
     if dt is None:
         from meerschaum.utils.warnings import error
@@ -121,7 +121,7 @@ def parse_help(sysargs: Union[List[str], Dict[str, Any]]) -> None:
     """Parse the `--help` flag to determine which help message to print."""
     from meerschaum._internal.arguments._parse_arguments import parse_arguments, parse_line
     from meerschaum.actions import actions, get_subactions
-    import importlib, inspect, textwrap
+    import textwrap
     from copy import deepcopy
     if isinstance(sysargs, list):
         args = parse_arguments(sysargs)
@@ -140,14 +140,14 @@ def parse_help(sysargs: Union[List[str], Dict[str, Any]]) -> None:
     if len(args['action']) > 1:
         try:
             subaction = get_subactions(args['action'][0])[args['action'][1]]
-        except Exception as e:
+        except Exception:
             subaction = None
         if subaction is not None:
             return print(textwrap.dedent(subaction.__doc__))
 
     try:
         doc = actions[args['action'][0]].__doc__
-    except Exception as e:
+    except Exception:
         doc = None
     if doc is None:
         doc = "No help available for '" + f"{args['action'][0]}" + "'."
@@ -309,7 +309,7 @@ groups['sync'].add_argument(
 )
 groups['sync'].add_argument(
     '--chunksize', type=int, help=(
-        "Specify the database chunksize. Defaults to 10,000."
+        "Specify the database chunksize. Defaults to 100,000."
     ),
 )
 groups['sync'].add_argument(
@@ -336,19 +336,25 @@ groups['sync'].add_argument(
 )
 groups['sync'].add_argument(
     '--bounded', '--bound', action="store_true",
-    help = (
+    help=(
         "When verifying, do not sync outside "
-        + "the existing oldest and newest datetime bounds."
+        "the existing oldest and newest datetime bounds."
     )
 )
 groups['sync'].add_argument(
     '--skip-check-existing', '--allow-duplicates', action='store_true',
-    help = (
-        "Skip checking for duplicate rows when syncing. " +
-        "This drastically improves performance when all rows to be synced are unique. " +
-        "For example, this setting is highly recommended for use with IoT devices."
+    help=(
+        "Skip checking for duplicate rows when syncing. "
+        "This improves performance when all rows to be synced are unique. "
     )
 )
+groups['sync'].add_argument(
+    '--skip-enforce-dtypes', action='store_true',
+    help=(
+        "Skip enforcing incoming data to a pipe's dtypes. "
+        "This improves performance when all rows are expected to already be of the correct type."
+    ),
+)
 groups['sync'].add_argument(
     '--cache', action='store_true',
     help = (

meerschaum/actions/clear.py

@@ -137,7 +137,7 @@ def _ask_with_rowcounts(
     )
     total_num_rows = sum([rc for p, rc in pipes_rowcounts.items()])
     question = (
-        f"Are you sure you want to delete {total_num_rows} rows across {len(pipes)} pipe"
+        f"Are you sure you want to delete {total_num_rows:,} rows across {len(pipes)} pipe"
         + ('s' if len(pipes) != 1 else '')
         + " in the following range?\n"
     )

meerschaum/actions/edit.py

@@ -498,7 +498,7 @@ def _edit_jobs(
         num_edited += 1
 
     msg = (
-        "Successfully edit job"
+        "Successfully edited job"
         + ('s' if len(jobs) != 1 else '')
         + ' '
         + items_str(list(jobs.keys()))

meerschaum/actions/start.py

@@ -183,7 +183,7 @@ def _start_jobs(
         ### Cannot find dameon_id
         else:
             msg = (
-                f"Unknown job" + ('s' if len(action) != 1 else '') + ' '
+                "Unknown job" + ('s' if len(action) != 1 else '') + ' '
                 + items_str(action, and_str='or') + '.'
             )
             return False, msg
@@ -216,7 +216,7 @@ def _start_jobs(
         return job.start(debug=debug), name
 
     def _run_existing_job(name: str):
-        job = Job(name, executor_keys=executor_keys)
+        job = jobs.get(name, Job(name, executor_keys=executor_keys))
         return job.start(debug=debug), name
 
     if not names:

meerschaum/actions/verify.py

@@ -10,9 +10,9 @@ from __future__ import annotations
 from meerschaum.utils.typing import Union, Any, Sequence, SuccessTuple, Optional, Tuple, List
 
 def verify(
-        action: Optional[List[str]] = None,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Verify the states of pipes, packages, and more.
     """
@@ -36,19 +36,17 @@ def _verify_pipes(**kwargs) -> SuccessTuple:
 
 
 def _verify_packages(
-        debug: bool = False,
-        venv: Optional[str] = 'mrsm',
-        **kw
-    ) -> SuccessTuple:
+    debug: bool = False,
+    venv: Optional[str] = 'mrsm',
+    **kw
+) -> SuccessTuple:
     """
     Verify the versions of packages.
     """
     from meerschaum.utils.packages import (
-        attempt_import, need_update, all_packages, is_installed, venv_contains_package,
+        attempt_import, all_packages, is_installed, venv_contains_package,
         _monkey_patch_get_distribution, manually_import_module,
     )
-    from meerschaum.utils.formatting import pprint
-    from meerschaum.utils.debug import dprint
 
     venv_packages, base_packages, miss_packages = [], [], []
 
@@ -80,10 +78,10 @@ def _verify_packages(
 
 
 def _verify_venvs(
-        action: Optional[List[str]],
-        debug: bool = False,
-        **kw
-    ) -> SuccessTuple:
+    action: Optional[List[str]],
+    debug: bool = False,
+    **kw
+) -> SuccessTuple:
     """
     Verify your virtual environments.
     """
@@ -94,15 +92,14 @@ def _verify_venvs(
 
 
 def _verify_plugins(
-        action: Optional[List[str]] = None,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Verify that all of the available plugins are able to be imported as expected.
     """
-    from meerschaum.utils.formatting import print_options, UNICODE, print_tuple
-    from meerschaum.plugins import import_plugins, get_plugins_names, Plugin
-    from meerschaum.config import get_config
+    from meerschaum.utils.formatting import print_options, print_tuple
+    from meerschaum.plugins import get_plugins_names, Plugin
     from meerschaum.utils.misc import items_str
 
     plugins_names_to_verify = action or get_plugins_names()
@@ -135,7 +132,7 @@ def _verify_plugins(
         f"Successfully imported {len(plugins_names_to_verify)} plugins."
         if success
         else (
-            f"Failed to import plugin"
+            "Failed to import plugin"
             + ('s' if len(failed_to_import) != 1 else '')
             + f" {items_str(failed_to_import)}."
         )

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.7.2"
+__version__ = "2.7.4"

meerschaum/connectors/sql/_fetch.py

@@ -9,6 +9,7 @@ Implement the Connector fetch() method
 from __future__ import annotations
 
 from datetime import datetime, timedelta
+
 import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Union, Callable, Any, List, Dict
 
@@ -144,37 +145,37 @@ def get_pipe_metadef(
     -------
     A pipe's meta definition fetch query string.
     """
-    from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import warn, error
+    from meerschaum.utils.warnings import warn
     from meerschaum.utils.sql import sql_item_name, dateadd_str, build_where
+    from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     from meerschaum.utils.misc import is_int
     from meerschaum.config import get_config
 
-    definition = get_pipe_query(pipe)
-
-    if not pipe.columns.get('datetime', None):
-        _dt = pipe.guess_datetime()
-        dt_name = sql_item_name(_dt, self.flavor, None) if _dt else None
+    dt_col = pipe.columns.get('datetime', None)
+    if not dt_col:
+        dt_col = pipe.guess_datetime()
+        dt_name = sql_item_name(dt_col, self.flavor, None) if dt_col else None
         is_guess = True
     else:
-        _dt = pipe.get_columns('datetime')
-        dt_name = sql_item_name(_dt, self.flavor, None)
+        dt_name = sql_item_name(dt_col, self.flavor, None)
         is_guess = False
+    dt_typ = pipe.dtypes.get(dt_col, 'datetime') if dt_col else None
+    db_dt_typ = get_db_type_from_pd_type(dt_typ, self.flavor) if dt_typ else None
 
     if begin not in (None, '') or end is not None:
         if is_guess:
-            if _dt is None:
+            if dt_col is None:
                 warn(
                     f"Unable to determine a datetime column for {pipe}."
                     + "\n    Ignoring begin and end...",
-                    stack = False,
+                    stack=False,
                 )
                 begin, end = '', None
             else:
                 warn(
                     f"A datetime wasn't specified for {pipe}.\n"
-                    + f"    Using column \"{_dt}\" for datetime bounds...",
-                    stack = False
+                    + f"    Using column \"{dt_col}\" for datetime bounds...",
+                    stack=False
                 )
 
     apply_backtrack = begin == '' and check_existing
@@ -200,6 +201,7 @@ def get_pipe_metadef(
                 datepart='minute',
                 number=((-1 * btm) if apply_backtrack else 0),
                 begin=begin,
+                db_type=db_dt_typ,
             )
             if begin
             else None
@@ -210,11 +212,13 @@ def get_pipe_metadef(
                 datepart='minute',
                 number=0,
                 begin=end,
+                db_type=db_dt_typ,
             )
             if end
             else None
         )
 
+    definition_name = sql_item_name('definition', self.flavor, None)
     meta_def = (
         _simple_fetch_query(pipe, self.flavor) if (
             (not (pipe.columns or {}).get('id', None))
@@ -225,26 +229,26 @@ def get_pipe_metadef(
     has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
     if dt_name and (begin_da or end_da):
         definition_dt_name = (
-            dateadd_str(self.flavor, 'minute', 0, f"definition.{dt_name}")
+            dateadd_str(self.flavor, 'minute', 0, f"{definition_name}.{dt_name}", db_type=db_dt_typ)
             if not is_int((begin_da or end_da))
-            else f"definition.{dt_name}"
+            else f"{definition_name}.{dt_name}"
         )
         meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
         has_where = True
         if begin_da:
-            meta_def += f"{definition_dt_name} >= {begin_da}"
+            meta_def += f"\n    {definition_dt_name}\n    >=\n    {begin_da}\n"
         if begin_da and end_da:
-            meta_def += " AND "
+            meta_def += "    AND"
         if end_da:
-            meta_def += f"{definition_dt_name} < {end_da}"
+            meta_def += f"\n    {definition_dt_name}\n    <\n    {end_da}\n"
 
     if params is not None:
         params_where = build_where(params, self, with_where=False)
-        meta_def += "\n" + ("AND" if has_where else "WHERE") + " "
+        meta_def += "\n    " + ("AND" if has_where else "WHERE") + "    "
         has_where = True
         meta_def += params_where
 
-    return meta_def
+    return meta_def.rstrip()
 
 
 def get_pipe_query(pipe: mrsm.Pipe, warn: bool = True) -> Union[str, None]:
@@ -256,7 +260,11 @@ def get_pipe_query(pipe: mrsm.Pipe, warn: bool = True) -> Union[str, None]:
     - query
     - sql
     """
+    import re
+    import textwrap
     from meerschaum.utils.warnings import warn as _warn
+    from meerschaum.utils.misc import parse_arguments_str
+    from meerschaum.utils.sql import sql_item_name
     if pipe.parameters.get('fetch', {}).get('definition', None):
         definition = pipe.parameters['fetch']['definition']
     elif pipe.parameters.get('definition', None):
@@ -272,7 +280,23 @@ def get_pipe_query(pipe: mrsm.Pipe, warn: bool = True) -> Union[str, None]:
                 + "    Set the key `query` in `pipe.parameters` to a valid SQL query."
             )
         return None
-    return definition
+
+    def replace_pipe_match(pipe_match):
+        try:
+            args_str = pipe_match.group(1)
+            args, kwargs = parse_arguments_str(args_str)
+            pipe = mrsm.Pipe(*args, **kwargs)
+        except Exception as e:
+            if warn:
+                _warn(f"Failed to parse pipe from SQL definition:\n{e}")
+            raise e
+
+        target = pipe.target
+        schema = pipe.instance_connector.get_pipe_schema(pipe)
+        return sql_item_name(target, pipe.instance_connector.flavor, schema)
+
+    definition = re.sub(r'\{\{Pipe\((.*?)\)\}\}', replace_pipe_match, definition)
+    return textwrap.dedent(definition.lstrip().rstrip())
 
 
 def set_pipe_query(pipe: mrsm.Pipe, query: str) -> None:
@@ -331,11 +355,7 @@ def _join_fetch_query(
     pipe_instance_name = sql_item_name(
         pipe.target, pipe.instance_connector.flavor, pipe.instance_connector.schema
     )
-    #  pipe_remote_name = sql_item_name(pipe.target, pipe.connector.flavor)
     sync_times_table = pipe.target + "_sync_times"
-    sync_times_instance_name = sql_item_name(
-        sync_times_table, pipe.instance_connector.flavor, None
-    )
     sync_times_remote_name = sql_item_name(
         sync_times_table, pipe.connector.flavor, None
     )
@@ -393,4 +413,3 @@ def _join_fetch_query(
     WHERE definition.{dt_remote_name} > st.{dt_remote_name}
     """ + (f"  OR st.{id_remote_name} IS NULL" if new_ids else "")
     return query
-

meerschaum/connectors/sql/_instance.py

@@ -96,15 +96,15 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
         sqlalchemy.select(temp_tables_table.c.table)
         .where(temp_tables_table.c.ready_to_drop.is_not(None))
     )
-    tables_to_drop = [
+    tables_to_drop = {
         table
         for table, ready_to_drop in _in_memory_temp_tables.items()
         if ready_to_drop
-    ]
+    }
     if not tables_to_drop:
         df = self.read(query, silent=True, debug=debug)
         tables_to_drop = (
-            list(df['table'])
+            set(df['table'])
             if df is not None
             else []
         )
@@ -126,7 +126,7 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
             sqlalchemy.delete(temp_tables_table)
             .where(temp_tables_table.c.table.in_(dropped_tables))
         )
-        delete_result = self.exec(delete_query, silent=True, debug=debug)
+        _ = self.exec(delete_query, silent=True, debug=debug)
 
     success = len(failed_tables) == 0
     msg = (