meerschaum 2.0.0rc6__py3-none-any.whl → 2.0.0rc8__py3-none-any.whl

meerschaum/core/Pipe/_deduplicate.py

@@ -23,6 +23,7 @@ def deduplicate(
         end: Union[datetime, int, None] = None,
         params: Optional[Dict[str, Any]] = None,
         chunk_interval: Union[datetime, int, None] = None,
+        bounded: Optional[bool] = None,
         workers: Optional[int] = None,
         debug: bool = False,
         **kwargs: Any
@@ -46,6 +47,9 @@ def deduplicate(
         If provided, use this for the chunk bounds.
         Defaults to the value set in `pipe.parameters['chunk_minutes']` (1440).
 
+    bounded: Optional[bool], default None
+        Only check outside the oldest and newest sync times if bounded is explicitly `False`.
+
     workers: Optional[int], default None
         If the instance connector is thread-safe, limit concurrenct syncs to this many threads.
 
@@ -61,6 +65,7 @@ def deduplicate(
     A `SuccessTuple` corresponding to whether all of the chunks were successfully deduplicated.
     """
     from meerschaum.utils.warnings import warn, info
+    from meerschaum.utils.misc import interval_str, items_str
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.pool import get_pool
@@ -70,6 +75,7 @@ def deduplicate(
             begin = begin,
             end = end,
             params = params,
+            bounded = bounded,
             debug = debug,
             **kwargs
         )
@@ -86,17 +92,44 @@ def deduplicate(
                 begin = begin,
                 end = end,
                 params = params,
+                bounded = bounded,
                 debug = debug,
                 **kwargs
             )
 
+    ### Only unbound if explicitly False.
+    if bounded is None:
+        bounded = True
+    chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
+
+    bound_time = self.get_bound_time(debug=debug)
+    if bounded and begin is None:
+        begin = (
+            bound_time
+            if bound_time is not None
+            else self.get_sync_time(newest=False, debug=debug)
+        )
+    if bounded and end is None:
+        end = self.get_sync_time(newest=True, debug=debug)
+
+    if bounded and end is not None:
+        end += (
+            timedelta(minutes=1)
+            if isinstance(end, datetime)
+            else 1
+        )
+
     chunk_bounds = self.get_chunk_bounds(
-        bounded = True,
+        bounded = bounded,
+        begin = begin,
+        end = end,
         chunk_interval = chunk_interval,
         debug = debug,
     )
 
     indices = [col for col in self.columns.values() if col]
+    if not indices:
+        return False, f"Cannot deduplicate without index columns."
     dt_col = self.columns.get('datetime', None)
 
     def process_chunk_bounds(bounds) -> Tuple[
@@ -115,6 +148,8 @@ def deduplicate(
             params = params,
             debug = debug,
         )
+        if chunk_df is None:
+            return bounds, (True, "")
         existing_chunk_len = len(chunk_df)
         deduped_chunk_df = chunk_df.drop_duplicates(keep='last')
         deduped_chunk_len = len(deduped_chunk_df)
@@ -131,11 +166,19 @@ def deduplicate(
             params = params,
             debug = debug,
         )
-        if len(full_chunk) == 0:
+        if full_chunk is None or len(full_chunk) == 0:
             return bounds, (True, f"{chunk_msg_header}\nChunk is empty, skipping...")
 
         chunk_indices = [ix for ix in indices if ix in full_chunk.columns]
-        full_chunk = full_chunk.drop_duplicates(subset=chunk_indices, keep='last')
+        if not chunk_indices:
+            return bounds, (False, f"None of {items_str(indices)} were present in chunk.")
+        try:
+            full_chunk = full_chunk.drop_duplicates(subset=chunk_indices, keep='last')
+        except Exception as e:
+            return (
+                bounds,
+                (False, f"Failed to deduplicate chunk on {items_str(chunk_indices)}:\n({e})")
+            )
 
         clear_success, clear_msg = self.clear(
             begin = chunk_begin,
@@ -172,10 +215,17 @@ def deduplicate(
             True, (
                 chunk_msg_header + "\n"
                 + chunk_msg_body + ("\n" if chunk_msg_body else '')
-                + f"Chunk succesfully deduplicated to {chunk_rowcount} rows."
+                + f"Deduplicated chunk from {existing_chunk_len} to {chunk_rowcount} rows."
             )
         )
 
+    info(
+        f"Deduplicating {len(chunk_bounds)} chunk"
+        + ('s' if len(chunk_bounds) != 1 else '')
+        + f" ({'un' if not bounded else ''}bounded)"
+        + f" of size '{interval_str(chunk_interval)}'"
+        + f" on {self}."
+    )
     bounds_success_tuples = dict(pool.map(process_chunk_bounds, chunk_bounds))
     bounds_successes = {
         bounds: success_tuple
@@ -195,8 +245,8 @@ def deduplicate(
             (
                 f"Failed to deduplicate {len(bounds_failures)} chunk"
                 + ('s' if len(bounds_failures) != 1 else '')
-                + ":\n"
-                + "\n".join([msg for _, (_, msg) in bounds_failures.items()])
+                + ".\n"
+                + "\n".join([msg for _, (_, msg) in bounds_failures.items() if msg])
             )
         )
 
@@ -208,7 +258,7 @@ def deduplicate(
                 f"Successfully deduplicated {len(bounds_successes)} chunk"
                 + ('s' if len(bounds_successes) != 1 else '')
                 + ".\n"
-                + "\n".join([msg for _, (_, msg) in bounds_successes.items()])
+                + "\n".join([msg for _, (_, msg) in bounds_successes.items() if msg])
             ).rstrip('\n')
         )
 
@@ -232,8 +282,8 @@ def deduplicate(
             (
                 f"Successfully deduplicated {len(bounds_successes)} chunk"
                 + ('s' if len(bounds_successes) != 1 else '')
-                + f" ({len(retry_bounds_successes)} retried):\n"
-                + "\n".join([msg for _, (_, msg) in bounds_successes.items()])
+                + f"({len(retry_bounds_successes)} retried):\n"
+                + "\n".join([msg for _, (_, msg) in bounds_successes.items() if msg])
             ).rstrip('\n')
         )
 
@@ -242,7 +292,7 @@ def deduplicate(
         (
             f"Failed to deduplicate {len(bounds_failures)} chunk"
             + ('s' if len(retry_bounds_failures) != 1 else '')
-            + ":\n"
-            + "\n".join([msg for _, (_, msg) in retry_bounds_failures.items()])
+            + ".\n"
+            + "\n".join([msg for _, (_, msg) in retry_bounds_failures.items() if msg])
         ).rstrip('\n')
     )

meerschaum/core/Pipe/_dtypes.py

@@ -7,6 +7,7 @@ Enforce data types for a pipe's underlying table.
 """
 
 from __future__ import annotations
+from io import StringIO
 from meerschaum.utils.typing import Dict, Any, Optional
 
 def enforce_dtypes(
@@ -38,7 +39,7 @@ def enforce_dtypes(
     try:
         if isinstance(df, str):
             df = parse_df_datetimes(
-                pd.read_json(df),
+                pd.read_json(StringIO(df)),
                 ignore_cols = [
                     col
                     for col, dtype in pipe_dtypes.items()

meerschaum/core/Pipe/_verify.py

@@ -17,7 +17,7 @@ def verify(
         end: Union[datetime, int, None] = None,
         params: Optional[Dict[str, Any]] = None,
         chunk_interval: Union[timedelta, int, None] = None,
-        bounded: bool = False,
+        bounded: Optional[bool] = None,
         deduplicate: bool = False,
         workers: Optional[int] = None,
         debug: bool = False,
@@ -38,10 +38,11 @@ def verify(
         If provided, use this as the size of the chunk boundaries.
         Default to the value set in `pipe.parameters['chunk_minutes']` (1440).
 
-    bounded: bool, default False
-        If `True`, do not verify older than the oldest existing datetime 
-        or newer than the newest existing datetime (i.e. `begin=pipe.get_sync_time(newest=False)`
-        and `end=pipe.get_sync_time() + timedelta(minutes=1)`.
+    bounded: Optional[bool], default None
+        If `True`, do not verify older than the oldest sync time or newer than the newest.
+        If `False`, verify unbounded syncs outside of the new and old sync times.
+        The default behavior (`None`) is to bound only if a bound interval is set
+        (e.g. `pipe.parameters['verify']['bound_days']`).
 
     deduplicate: bool, default False
         If `True`, deduplicate the pipe's table after the verification syncs.
@@ -61,29 +62,39 @@ def verify(
     A SuccessTuple indicating whether the pipe was successfully resynced.
     """
     from meerschaum.utils.pool import get_pool
+    from meerschaum.utils.misc import interval_str
     workers = self.get_num_workers(workers)
-    sync_less_than_begin = not bounded and begin is None
-    sync_greater_than_end = not bounded and end is None
 
-    if begin is None:
-        begin = self.get_sync_time(newest=False, debug=debug)
-    if end is None:
+    ### Skip configured bounding in parameters
+    ### if `bounded` is explicitly `False`.
+    bound_time = (
+        self.get_bound_time(debug=debug)
+        if bounded is not False
+        else None
+    )
+    if bounded is None:
+        bounded = bound_time is not None
+
+    if bounded and begin is None:
+        begin = (
+            bound_time
+            if bound_time is not None
+            else self.get_sync_time(newest=False, debug=debug)
+        )
+    if bounded and end is None:
         end = self.get_sync_time(newest=True, debug=debug)
 
-    if bounded:
+    if bounded and end is not None:
         end += (
             timedelta(minutes=1)
             if isinstance(end, datetime)
             else 1
         )
 
-    cannot_determine_bounds = (
-        begin is None
-        or
-        end is None
-        or
-        not self.exists(debug=debug)
-    )
+    sync_less_than_begin = not bounded and begin is None
+    sync_greater_than_end = not bounded and end is None
+
+    cannot_determine_bounds = not self.exists(debug=debug)
 
     if cannot_determine_bounds:
         sync_success, sync_msg = self.sync(
@@ -108,9 +119,10 @@ def verify(
         return sync_success, sync_msg
 
 
-    if not chunk_interval:
-        chunk_interval = self.get_chunk_interval(debug=debug)
+    chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     chunk_bounds = self.get_chunk_bounds(
+        begin = begin,
+        end = end,
         chunk_interval = chunk_interval,
         bounded = bounded,
         debug = debug,
@@ -129,20 +141,48 @@ def verify(
             )
         return True, f"Could not determine chunks between '{begin}' and '{end}'; nothing to do."
 
+    begin_to_print = (
+        begin
+        if begin is not None
+        else (
+            chunk_bounds[0][0]
+            if bounded
+            else chunk_bounds[0][1]
+        )
+    )
+    end_to_print = (
+        end
+        if end is not None
+        else (
+            chunk_bounds[-1][1]
+            if bounded
+            else chunk_bounds[-1][0]
+        )
+    )
+
     info(
         f"Syncing {len(chunk_bounds)} chunk" + ('s' if len(chunk_bounds) != 1 else '')
-        + f" of size '{chunk_interval}'"
-        + f" between '{begin}' and '{end}'."
+        + f" ({'un' if not bounded else ''}bounded)"
+        + f" of size '{interval_str(chunk_interval)}'"
+        + f" between '{begin_to_print}' and '{end_to_print}'."
     )
 
     pool = get_pool(workers=workers)
 
+    ### Dictionary of the form bounds -> success_tuple, e.g.:
+    ### {
+    ###    (2023-01-01, 2023-01-02): (True, "Success")
+    ### }
+    bounds_success_tuples = {}
     def process_chunk_bounds(
             chunk_begin_and_end: Tuple[
                 Union[int, datetime],
                 Union[int, datetime]
             ]
         ):
+        if chunk_begin_and_end in bounds_success_tuples:
+            return chunk_begin_and_end, bounds_success_tuples[chunk_begin_and_end]
+
         chunk_begin, chunk_end = chunk_begin_and_end
         return chunk_begin_and_end, self.sync(
             begin = chunk_begin,
@@ -153,11 +193,22 @@ def verify(
             **kwargs
         )
 
-    ### Dictionary of the form bounds -> success_tuple, e.g.:
-    ### {
-    ###    (2023-01-01, 2023-01-02): (True, "Success")
-    ### }
-    bounds_success_tuples = dict(pool.map(process_chunk_bounds, chunk_bounds))
+    ### If we have more than one chunk, attempt to sync the first one and return if its fails.
+    if len(chunk_bounds) > 1:
+        first_chunk_bounds = chunk_bounds[0]
+        (
+            (first_begin, first_end),
+            (first_success, first_msg)
+        ) = process_chunk_bounds(first_chunk_bounds)
+        if not first_success:
+            return (
+                first_success,
+                f"\n{first_begin} - {first_end}\n"
+                + f"Failed to sync first chunk:\n{first_msg}"
+            )
+        bounds_success_tuples[first_chunk_bounds] = (first_success, first_msg)
+
+    bounds_success_tuples.update(dict(pool.map(process_chunk_bounds, chunk_bounds)))
     bounds_success_bools = {bounds: tup[0] for bounds, tup in bounds_success_tuples.items()}
 
     message_header = f"{begin} - {end}"
@@ -177,18 +228,19 @@ def verify(
 
     chunk_bounds_to_resync = [
         bounds
-        for bounds, success in zip(chunk_bounds, chunk_success_bools)
+        for bounds, success in zip(chunk_bounds, bounds_success_bools)
         if not success
     ]
     bounds_to_print = [
         f"{bounds[0]} - {bounds[1]}"
         for bounds in chunk_bounds_to_resync
     ]
-    warn(
-        f"Will resync the following failed chunks:\n    "
-        + '\n    '.join(bounds_to_print),
-        stack = False,
-    )
+    if bounds_to_print:
+        warn(
+            f"Will resync the following failed chunks:\n    "
+            + '\n    '.join(bounds_to_print),
+            stack = False,
+        )
 
     retry_bounds_success_tuples = dict(pool.map(process_chunk_bounds, chunk_bounds_to_resync))
     bounds_success_tuples.update(retry_bounds_success_tuples)
@@ -271,7 +323,8 @@ def get_chunks_success_message(
         ''
         if num_fails == 0
         else (
-            f"\n\nFailed to sync {num_fails} chunks:\n"
+            f"\n\nFailed to sync {num_fails} chunk"
+            + ('s' if num_fails != 1 else '') + ":\n"
             + '\n'.join([
                 f"{fail_begin} - {fail_end}\n{msg}\n"
                 for (fail_begin, fail_end), (_, msg) in fail_chunk_bounds_tuples.items()
@@ -280,3 +333,90 @@ def get_chunks_success_message(
     )
 
     return header + success_msg + fail_msg
+
+
+def get_bound_interval(self, debug: bool = False) -> Union[timedelta, int, None]:
+    """
+    Return the interval used to determine the bound time (limit for verification syncs).
+    If the datetime axis is an integer, just return its value.
+
+    Below are the supported keys for the bound interval:
+
+        - `pipe.parameters['verify']['bound_minutes']`
+        - `pipe.parameters['verify']['bound_hours']`
+        - `pipe.parameters['verify']['bound_days']`
+        - `pipe.parameters['verify']['bound_weeks']`
+        - `pipe.parameters['verify']['bound_years']`
+        - `pipe.parameters['verify']['bound_seconds']`
+
+    If multiple keys are present, the first on this priority list will be used.
+
+    Returns
+    -------
+    A `timedelta` or `int` value to be used to determine the bound time.
+    """
+    verify_params = self.parameters.get('verify', {})
+    prefix = 'bound_'
+    suffixes_to_check = ('minutes', 'hours', 'days', 'weeks', 'years', 'seconds')
+    keys_to_search = {
+        key: val
+        for key, val in verify_params.items()
+        if key.startswith(prefix)
+    }
+    bound_time_key, bound_time_value = None, None
+    for key, value in keys_to_search.items():
+        for suffix in suffixes_to_check:
+            if key == prefix + suffix:
+                bound_time_key = key
+                bound_time_value = value
+                break
+        if bound_time_key is not None:
+            break
+
+    if bound_time_value is None:
+        return bound_time_value
+
+    dt_col = self.columns.get('datetime', None)
+    if not dt_col:
+        return bound_time_value
+
+    dt_typ = self.dtypes.get(dt_col, 'datetime64[ns]')
+    if 'int' in dt_typ.lower():
+        return int(bound_time_value)
+
+    interval_type = bound_time_key.replace(prefix, '')
+    return timedelta(**{interval_type: bound_time_value})
+
+
+def get_bound_time(self, debug: bool = False) -> Union[datetime, int, None]:
+    """
+    The bound time is the limit at which long-running verification syncs should stop.
+    A value of `None` means verification syncs should be unbounded.
+
+    Like deriving a backtrack time from `pipe.get_sync_time()`,
+    the bound time is the sync time minus a large window (e.g. 366 days).
+
+    Unbound verification syncs (i.e. `bound_time is None`)
+    if the oldest sync time is less than the bound interval.
+
+    Returns
+    -------
+    A `datetime` or `int` corresponding to the
+    `begin` bound for verification and deduplication syncs.
+    """ 
+    bound_interval = self.get_bound_interval(debug=debug)
+    if bound_interval is None:
+        return None
+
+    sync_time = self.get_sync_time(debug=debug)
+    if sync_time is None:
+        return None
+
+    bound_time = sync_time - bound_interval
+    oldest_sync_time = self.get_sync_time(newest=False, debug=debug)
+
+    return (
+        bound_time
+        if bound_time > oldest_sync_time
+        else None
+    )

meerschaum/plugins/__init__.py

@@ -254,6 +254,9 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
         try:
             if PLUGINS_INTERNAL_LOCK_PATH.exists():
                 PLUGINS_INTERNAL_LOCK_PATH.unlink()
+        ### Sometimes competing threads will delete the lock file at the same time.
+        except FileNotFoundError:
+            pass
         except Exception as e:
             if warn:
                 _warn(f"Error cleaning up lockfile {PLUGINS_INTERNAL_LOCK_PATH}:\n{e}")