meerschaum 2.7.2__py3-none-any.whl → 2.7.4__py3-none-any.whl

meerschaum/core/Pipe/_sync.py

@@ -50,6 +50,7 @@ def sync(
     retries: int = 10,
     min_seconds: int = 1,
     check_existing: bool = True,
+    enforce_dtypes: bool = True,
     blocking: bool = True,
     workers: Optional[int] = None,
     callback: Optional[Callable[[Tuple[bool, str]], Any]] = None,
@@ -89,6 +90,10 @@ def sync(
     check_existing: bool, default True
         If `True`, pull and diff with existing data from the pipe.
 
+    enforce_dtypes: bool, default True
+        If `True`, enforce dtypes on incoming data.
+        Set this to `False` if the incoming rows are expected to be of the correct dtypes.
+
     blocking: bool, default True
         If `True`, wait for sync to finish and return its result, otherwise
         asyncronously sync (oxymoron?) and return success. Defaults to `True`.
@@ -123,8 +128,6 @@ def sync(
     A `SuccessTuple` of success (`bool`) and message (`str`).
     """
     from meerschaum.utils.debug import dprint, _checkpoint
-    from meerschaum.connectors import custom_types
-    from meerschaum.plugins import Plugin
     from meerschaum.utils.formatting import get_console
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
@@ -366,7 +369,12 @@ def sync(
             return success, msg
 
         ### Cast to a dataframe and ensure datatypes are what we expect.
-        df = self.enforce_dtypes(df, chunksize=chunksize, debug=debug)
+        df = self.enforce_dtypes(
+            df,
+            chunksize=chunksize,
+            enforce=enforce_dtypes,
+            debug=debug,
+        )
 
         ### Capture `numeric`, `uuid`, `json`, and `bytes` columns.
         self._persist_new_json_columns(df, debug=debug)

meerschaum/core/Pipe/_verify.py

@@ -162,7 +162,8 @@ def verify(
     )
 
     info(
-        f"Verifying {self}:\n    Syncing {len(chunk_bounds)} chunk" + ('s' if len(chunk_bounds) != 1 else '')
+        f"Verifying {self}:\n    Syncing {len(chunk_bounds)} chunk"
+        + ('s' if len(chunk_bounds) != 1 else '')
         + f" ({'un' if not bounded else ''}bounded)"
         + f" of size '{interval_str(chunk_interval)}'"
         + f" between '{begin_to_print}' and '{end_to_print}'."
@@ -194,6 +195,9 @@ def verify(
             **kwargs
         )
         chunk_msg = chunk_msg.strip()
+        if ' - ' not in chunk_msg:
+            chunk_label = f"{chunk_begin} - {chunk_end}"
+            chunk_msg = f'{chunk_label}\n{chunk_msg}'
         mrsm.pprint((chunk_success, chunk_msg))
         return chunk_begin_and_end, (chunk_success, chunk_msg)
 
@@ -323,16 +327,16 @@ def get_chunks_success_message(
     header = (header + "\n") if header else ""
     stats_msg = items_str(
         (
-            ([f'inserted {num_inserted}'] if num_inserted else [])
-            + ([f'updated {num_updated}'] if num_updated else [])
-            + ([f'upserted {num_upserted}'] if num_upserted else [])
+            ([f'inserted {num_inserted:,}'] if num_inserted else [])
+            + ([f'updated {num_updated:,}'] if num_updated else [])
+            + ([f'upserted {num_upserted:,}'] if num_upserted else [])
         ) or ['synced 0'],
         quotes=False,
         and_=False,
     )
 
     success_msg = (
-        f"Successfully synced {len(chunk_success_tuples)} chunk"
+        f"Successfully synced {len(chunk_success_tuples):,} chunk"
         + ('s' if len(chunk_success_tuples) != 1 else '')
         + '\n(' + stats_msg
         + ' rows in total).'

meerschaum/jobs/__init__.py

@@ -6,8 +6,6 @@
 Higher-level utilities for managing `meerschaum.utils.daemon.Daemon`.
 """
 
-import pathlib
-
 import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Optional, List, SuccessTuple
 
@@ -342,7 +340,7 @@ def check_restart_jobs(
 
 def _check_restart_jobs_against_lock(*args, **kwargs):
     from meerschaum.config.paths import CHECK_JOBS_LOCK_PATH
-    fasteners = mrsm.attempt_import('fasteners')
+    fasteners = mrsm.attempt_import('fasteners', lazy=False)
     lock = fasteners.InterProcessLock(CHECK_JOBS_LOCK_PATH)
     with lock:
         check_restart_jobs(*args, **kwargs)

meerschaum/utils/daemon/Daemon.py

@@ -13,7 +13,6 @@ import pathlib
 import json
 import shutil
 import signal
-import sys
 import time
 import traceback
 from functools import partial
@@ -301,6 +300,7 @@ class Daemon:
             os.environ['LINES'], os.environ['COLUMNS'] = str(int(lines)), str(int(columns))
             with self._daemon_context:
                 sys.stdin = self.stdin_file
+                _ = os.environ.pop(STATIC_CONFIG['environment']['systemd_stdin_path'], None)
                 os.environ[STATIC_CONFIG['environment']['daemon_id']] = self.daemon_id
                 os.environ['PYTHONUNBUFFERED'] = '1'
 

meerschaum/utils/daemon/StdinFile.py

@@ -104,7 +104,10 @@ class StdinFile(io.TextIOBase):
         if self._file_handler is not None:
             self.sel.unregister(self._file_handler)
             self._file_handler.close()
-            os.close(self._fd)
+            try:
+                os.close(self._fd)
+            except OSError:
+                pass
             self._file_handler = None
             self._fd = None
 

meerschaum/utils/dataframe.py

@@ -390,6 +390,7 @@ def parse_df_datetimes(
     strip_timezone: bool = False,
     chunksize: Optional[int] = None,
     dtype_backend: str = 'numpy_nullable',
+    ignore_all: bool = False,
     debug: bool = False,
 ) -> 'pd.DataFrame':
     """
@@ -414,6 +415,9 @@ def parse_df_datetimes(
         use this as the datatypes backend.
         Accepted values are 'numpy_nullable' and 'pyarrow'.
 
+    ignore_all: bool, default False
+        If `True`, do not attempt to cast any columns to datetimes.
+
     debug: bool, default False
         Verbosity toggle.
 
@@ -504,7 +508,11 @@ def parse_df_datetimes(
             if 'datetime' in str(dtype)
         ]
     )
-    cols_to_inspect = [col for col in pdf.columns if col not in ignore_cols]
+    cols_to_inspect = [
+        col
+        for col in pdf.columns
+        if col not in ignore_cols
+    ] if not ignore_all else []
 
     if len(cols_to_inspect) == 0:
         if debug:
@@ -1263,7 +1271,7 @@ def df_from_literal(
         import ast
         try:
             val = ast.literal_eval(literal)
-        except Exception as e:
+        except Exception:
             warn(
                 "Failed to parse value from string:\n" + f"{literal}" +
                 "\n\nWill cast as a string instead."\

meerschaum/utils/dtypes/sql.py

@@ -7,7 +7,7 @@ Utility functions for working with SQL data types.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Dict, Union, Tuple, List
+from meerschaum.utils.typing import Dict, Union, Tuple
 
 NUMERIC_PRECISION_FLAVORS: Dict[str, Tuple[int, int]] = {
     'mariadb': (38, 20),

meerschaum/utils/formatting/__init__.py

@@ -57,18 +57,7 @@ def colored_fallback(*args, **kw):
     return ' '.join(args)
 
 def translate_rich_to_termcolor(*colors) -> tuple:
-    """Translate between rich and more_termcolor terminology.
-    This is probably prone to breaking.
-
-    Parameters
-    ----------
-    *colors :
-        
-
-    Returns
-    -------
-
-    """
+    """Translate between rich and more_termcolor terminology."""
     _colors = []
     for c in colors:
         _c_list = []
@@ -131,7 +120,7 @@ def _init():
     try:
         colorama.init(autoreset=False)
         success = True
-    except Exception as e:
+    except Exception:
         import traceback
         traceback.print_exc()
         _attrs['ANSI'], _attrs['UNICODE'], _attrs['CHARSET'] = False, False, 'ascii'
@@ -219,7 +208,7 @@ def get_console():
     rich_console = attempt_import('rich.console')
     try:
         console = rich_console.Console(force_terminal=True, color_system='truecolor')
-    except Exception as e:
+    except Exception:
         console = None
     return console
 
@@ -307,7 +296,6 @@ def format_success_tuple(
     calm: bool, default False
         If `True`, use the default emoji and color scheme.
     """
-    from meerschaum.config.static import STATIC_CONFIG
     _init()
     try:
         status = 'success' if tup[0] else 'failure'
@@ -381,12 +369,9 @@ def print_options(
         If `True`, print the option's number in the list (1 index).
 
     """
-    import os
     from meerschaum.utils.packages import import_rich
-    from meerschaum.utils.formatting import make_header, highlight_pipes
-    from meerschaum.actions import actions as _actions
-    from meerschaum.utils.misc import get_cols_lines, string_width, iterate_chunks
-
+    from meerschaum.utils.formatting import highlight_pipes
+    from meerschaum.utils.misc import get_cols_lines, string_width
 
     if options is None:
         options = {}
@@ -429,15 +414,10 @@ def print_options(
                 continue
             break
 
-    from meerschaum.utils.formatting import pprint, get_console
     from meerschaum.utils.packages import attempt_import
-    rich_columns = attempt_import('rich.columns')
-    rich_panel = attempt_import('rich.panel')
     rich_table = attempt_import('rich.table')
     Text = attempt_import('rich.text').Text
     box = attempt_import('rich.box')
-    Panel = rich_panel.Panel
-    Columns = rich_columns.Columns
     Table = rich_table.Table
 
     if _header is not None:

meerschaum/utils/formatting/_pipes.py

@@ -482,9 +482,9 @@ def print_pipes_results(
 
 
 def extract_stats_from_message(
-        message: str,
-        stat_keys: Optional[List[str]] = None,
-    ) -> Dict[str, int]:
+    message: str,
+    stat_keys: Optional[List[str]] = None,
+) -> Dict[str, int]:
     """
     Given a sync message, return the insert, update, upsert stats from within.
 
@@ -511,9 +511,9 @@ def extract_stats_from_message(
 
 
 def extract_stats_from_line(
-        line: str,
-        stat_keys: List[str],
-    ) -> Dict[str, int]:
+    line: str,
+    stat_keys: List[str],
+) -> Dict[str, int]:
     """
     Return the insert, update, upsert stats from a single line.
     """
@@ -524,6 +524,9 @@ def extract_stats_from_line(
         if search_key not in line.lower():
             continue
 
+        ### the count may be formatted with commas
+        line = line.replace(',', '')
+
         ### stat_text starts with the digits we want.
         try:
             stat_text = line.lower().split(search_key + ' ')[1]