meerschaum 2.6.0.dev1__py3-none-any.whl → 2.6.2__py3-none-any.whl

meerschaum/core/Pipe/_data.py

@@ -120,26 +120,9 @@ def get_data(
     if isinstance(omit_columns, str):
         omit_columns = [omit_columns]
 
+    begin, end = self.parse_date_bounds(begin, end)
     as_iterator = as_iterator or as_chunks
     dt_col = self.columns.get('datetime', None)
-    dt_typ = self.dtypes.get(dt_col, 'datetime64[ns, UTC]')
-    dt_is_utc = 'utc' in dt_typ.lower()
-    if isinstance(begin, str):
-        try:
-            begin = dateutil_parser.parse(begin)
-        except Exception as e:
-            warn(f"Failed to parse '{begin}' as datetime:\n{e}")
-            begin = None
-    if isinstance(end, str):
-        try:
-            end = dateutil_parser.parse(end)
-        except Exception as e:
-            warn(f"Failed to parse '{end}' as datetime:\n{e}")
-            end = None
-    if isinstance(begin, datetime):
-        begin = coerce_timezone(begin, strip_utc=(not dt_is_utc))
-    if isinstance(end, datetime):
-        end = coerce_timezone(end, strip_utc=(not dt_is_utc))
 
     def _sort_df(_df):
         if df_is_chunk_generator(_df):
@@ -330,16 +313,8 @@ def _get_data_as_iterator(
     Return a pipe's data as a generator.
     """
     from meerschaum.utils.misc import round_time
-    parse_begin = isinstance(begin, str)
-    parse_end = isinstance(end, str)
-    if parse_begin or parse_end:
-        from meerschaum.utils.packages import attempt_import
-        dateutil_parser = attempt_import('dateutil.parser')
-    if parse_begin:
-        begin = dateutil_parser.parse(begin)
-    if parse_end:
-        end = dateutil_parser.parse(end)
-
+    from meerschaum.utils.dtypes import coerce_timezone
+    begin, end = self.parse_date_bounds(begin, end)
     if not self.exists(debug=debug):
         return
 
@@ -351,11 +326,15 @@ def _get_data_as_iterator(
         if begin is not None
         else self.get_sync_time(round_down=False, newest=False, params=params, debug=debug)
     ) if dt_col else None
+    if isinstance(min_dt, datetime):
+        min_dt = coerce_timezone(min_dt)
     max_dt = (
         end
         if end is not None
         else self.get_sync_time(round_down=False, newest=True, params=params, debug=debug)
     ) if dt_col else None
+    if isinstance(max_dt, datetime):
+        max_dt = coerce_timezone(max_dt)
 
     ### We want to search just past the maximum value.
     if end is None:
@@ -469,6 +448,8 @@ def get_backtrack_data(
     if not self.exists(debug=debug):
         return None
 
+    begin = self.parse_date_bounds(begin)
+
     backtrack_interval = self.get_backtrack_interval(debug=debug)
     if backtrack_minutes is None:
         backtrack_minutes = (
@@ -569,6 +550,7 @@ def get_rowcount(
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
 
+    begin, end = self.parse_date_bounds(begin, end)
     connector = self.instance_connector if not remote else self.connector
     try:
         with Venv(get_connector_plugin(connector)):
@@ -683,6 +665,8 @@ def get_chunk_bounds(
     if begin is None and end is None:
         return [(None, None)]
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     ### Set the chunk interval under `pipe.parameters['verify']['chunk_minutes']`.
     chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     
@@ -714,3 +698,48 @@ def get_chunk_bounds(
         chunk_bounds = chunk_bounds + [(end, None)]
 
     return chunk_bounds
+
+
+def parse_date_bounds(self, *dt_vals: Union[datetime, int, None]) -> Union[
+    datetime,
+    int,
+    str,
+    None,
+    Tuple[Union[datetime, int, str, None]]
+]:
+    """
+    Given a date bound (begin, end), coerce a timezone if necessary.
+    """
+    from meerschaum.utils.misc import is_int
+    from meerschaum.utils.dtypes import coerce_timezone
+    from meerschaum.utils.warnings import warn
+    dateutil_parser = mrsm.attempt_import('dateutil.parser')
+
+    def _parse_date_bound(dt_val):
+        if dt_val is None:
+            return None
+
+        if isinstance(dt_val, int):
+            return dt_val
+
+        if dt_val == '':
+            return ''
+
+        if is_int(dt_val):
+            return int(dt_val)
+
+        if isinstance(dt_val, str):
+            try:
+                dt_val = dateutil_parser.parse(dt_val)
+            except Exception as e:
+                warn(f"Could not parse '{dt_val}' as datetime:\n{e}")
+                return None
+
+        dt_col = self.columns.get('datetime', None)
+        dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
+        return coerce_timezone(dt_val, strip_utc=('utc' not in dt_typ.lower()))
+
+    bounds = tuple(_parse_date_bound(dt_val) for dt_val in dt_vals)
+    if len(bounds) == 1:
+        return bounds[0]
+    return bounds

meerschaum/core/Pipe/_deduplicate.py

@@ -65,14 +65,16 @@ def deduplicate(
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.pool import get_pool
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     if self.cache_pipe is not None:
         success, msg = self.cache_pipe.deduplicate(
-            begin = begin,
-            end = end,
-            params = params,
-            bounded = bounded,
-            debug = debug,
-            _use_instance_method = _use_instance_method,
+            begin=begin,
+            end=end,
+            params=params,
+            bounded=bounded,
+            debug=debug,
+            _use_instance_method=_use_instance_method,
             **kwargs
         )
         if not success:
@@ -86,11 +88,11 @@ def deduplicate(
             if hasattr(self.instance_connector, 'deduplicate_pipe'):
                 return self.instance_connector.deduplicate_pipe(
                     self,
-                    begin = begin,
-                    end = end,
-                    params = params,
-                    bounded = bounded,
-                    debug = debug,
+                    begin=begin,
+                    end=end,
+                    params=params,
+                    bounded=bounded,
+                    debug=debug,
                     **kwargs
                 )
 
@@ -117,33 +119,33 @@ def deduplicate(
         )
 
     chunk_bounds = self.get_chunk_bounds(
-        bounded = bounded,
-        begin = begin,
-        end = end,
-        chunk_interval = chunk_interval,
-        debug = debug,
+        bounded=bounded,
+        begin=begin,
+        end=end,
+        chunk_interval=chunk_interval,
+        debug=debug,
     )
 
     indices = [col for col in self.columns.values() if col]
     if not indices:
-        return False, f"Cannot deduplicate without index columns."
+        return False, "Cannot deduplicate without index columns."
     dt_col = self.columns.get('datetime', None)
 
     def process_chunk_bounds(bounds) -> Tuple[
-            Tuple[
-                Union[datetime, int, None],
-                Union[datetime, int, None]
-            ],
-            SuccessTuple
-        ]:
+        Tuple[
+            Union[datetime, int, None],
+            Union[datetime, int, None]
+        ],
+        SuccessTuple
+    ]:
         ### Only selecting the index values here to keep bandwidth down.
         chunk_begin, chunk_end = bounds
         chunk_df = self.get_data(
-            select_columns = indices, 
-            begin = chunk_begin,
-            end = chunk_end,
-            params = params,
-            debug = debug,
+            select_columns=indices, 
+            begin=chunk_begin,
+            end=chunk_end,
+            params=params,
+            debug=debug,
         )
         if chunk_df is None:
             return bounds, (True, "")

meerschaum/core/Pipe/_dtypes.py

@@ -30,6 +30,7 @@ def enforce_dtypes(
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.dataframe import parse_df_datetimes, enforce_dtypes as _enforce_dtypes
+    from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.packages import import_pandas
     pd = import_pandas(debug=debug)
     if df is None:
@@ -51,6 +52,7 @@ def enforce_dtypes(
                     for col, dtype in pipe_dtypes.items()
                     if 'datetime' not in str(dtype)
                 ],
+                strip_timezone=(self.tzinfo is None),
                 chunksize=chunksize,
                 debug=debug,
             )
@@ -60,8 +62,9 @@ def enforce_dtypes(
                 ignore_cols=[
                     col
                     for col, dtype in pipe_dtypes.items()
-                    if 'datetime' not in str(dtype)
+                    if not are_dtypes_equal(str(dtype), 'datetime')
                 ],
+                strip_timezone=(self.tzinfo is None),
                 chunksize=chunksize,
                 debug=debug,
             )
@@ -77,7 +80,14 @@ def enforce_dtypes(
             )
         return df
 
-    return _enforce_dtypes(df, pipe_dtypes, safe_copy=safe_copy, debug=debug)
+    return _enforce_dtypes(
+        df,
+        pipe_dtypes,
+        safe_copy=safe_copy,
+        strip_timezone=(self.tzinfo is None),
+        coerce_timezone=True,
+        debug=debug,
+    )
 
 
 def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str, Any]:

meerschaum/core/Pipe/_fetch.py

@@ -18,14 +18,14 @@ if TYPE_CHECKING:
     pd = mrsm.attempt_import('pandas')
 
 def fetch(
-        self,
-        begin: Union[datetime, str, None] = '',
-        end: Optional[datetime] = None,
-        check_existing: bool = True,
-        sync_chunks: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union['pd.DataFrame', Iterator['pd.DataFrame'], None]:
+    self,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, None] = None,
+    check_existing: bool = True,
+    sync_chunks: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Union['pd.DataFrame', Iterator['pd.DataFrame'], None]:
     """
     Fetch a Pipe's latest data from its connector.
 
@@ -76,6 +76,8 @@ def fetch(
                 chunk_message = '\n' + chunk_label + '\n' + chunk_message
             return chunk_success, chunk_message
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     with mrsm.Venv(get_connector_plugin(self.connector)):
         _args, _kwargs = filter_arguments(
             self.connector.fetch,
@@ -164,6 +166,6 @@ def _determine_begin(
         backtrack_interval = timedelta(minutes=backtrack_interval)
     try:
         return sync_time - backtrack_interval
-    except Exception as e:
+    except Exception:
         warn(f"Unable to substract backtrack interval {backtrack_interval} from {sync_time}.")
     return sync_time

meerschaum/core/Pipe/_sync.py

@@ -141,6 +141,7 @@ def sync(
         chunksize = None
         sync_chunks = False
 
+    begin, end = self.parse_date_bounds(begin, end)
     kw.update({
         'begin': begin,
         'end': end,
@@ -460,7 +461,7 @@ def get_sync_time(
     apply_backtrack_interval: bool = False,
     round_down: bool = False,
     debug: bool = False
-) -> Union['datetime', None]:
+) -> Union['datetime', int, None]:
     """
     Get the most recent datetime value for a Pipe.
 
@@ -485,7 +486,7 @@ def get_sync_time(
 
     Returns
     -------
-    A `datetime` object if the pipe exists, otherwise `None`.
+    A `datetime` or int, if the pipe exists, otherwise `None`.
 
     """
     from meerschaum.utils.venv import Venv
@@ -510,13 +511,13 @@ def get_sync_time(
         except Exception as e:
             warn(f"Failed to apply backtrack interval:\n{e}")
 
-    return sync_time
+    return self.parse_date_bounds(sync_time)
 
 
 def exists(
-        self,
-        debug : bool = False
-    ) -> bool:
+    self,
+    debug: bool = False
+) -> bool:
     """
     See if a Pipe's table exists.
 
@@ -549,7 +550,11 @@ def exists(
                 return _exists
 
     with Venv(get_connector_plugin(self.instance_connector)):
-        _exists = self.instance_connector.pipe_exists(pipe=self, debug=debug)
+        _exists = (
+            self.instance_connector.pipe_exists(pipe=self, debug=debug)
+            if hasattr(self.instance_connector, 'pipe_exists')
+            else False
+        )
 
     self.__dict__['_exists'] = _exists
     self.__dict__['_exists_timestamp'] = now
@@ -928,7 +933,11 @@ def _persist_new_numeric_columns(self, df, debug: bool = False) -> SuccessTuple:
     if not new_numeric_cols:
         return True, "Success"
 
+    self._attributes_sync_time = None
+    dt_col = self.columns.get('datetime', None)
     dtypes = self.parameters.get('dtypes', {})
+    if dt_col not in dtypes:
+        dtypes[dt_col] = 'datetime'
     dtypes.update({col: 'numeric' for col in numeric_cols})
     self.parameters['dtypes'] = dtypes
     if not self.temporary:
@@ -952,7 +961,11 @@ def _persist_new_uuid_columns(self, df, debug: bool = False) -> SuccessTuple:
     if not new_uuid_cols:
         return True, "Success"
 
+    self._attributes_sync_time = None
+    dt_col = self.columns.get('datetime', None)
     dtypes = self.parameters.get('dtypes', {})
+    if dt_col not in dtypes:
+        dtypes[dt_col] = 'datetime'
     dtypes.update({col: 'uuid' for col in uuid_cols})
     self.parameters['dtypes'] = dtypes
     if not self.temporary:
@@ -976,7 +989,11 @@ def _persist_new_json_columns(self, df, debug: bool = False) -> SuccessTuple:
     if not new_json_cols:
         return True, "Success"
 
+    self._attributes_sync_time = None
+    dt_col = self.columns.get('datetime', None)
     dtypes = self.parameters.get('dtypes', {})
+    if dt_col not in dtypes:
+        dtypes[dt_col] = 'datetime'
     dtypes.update({col: 'json' for col in json_cols})
     self.parameters['dtypes'] = dtypes
 

meerschaum/core/Pipe/_verify.py

@@ -11,6 +11,7 @@ from meerschaum.utils.typing import SuccessTuple, Any, Optional, Union, Tuple, L
 from meerschaum.utils.warnings import warn, info
 from meerschaum.utils.debug import dprint
 
+
 def verify(
     self,
     begin: Union[datetime, int, None] = None,
@@ -84,6 +85,8 @@ def verify(
     if bounded and end is None:
         end = self.get_sync_time(newest=True, debug=debug)
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     if bounded and end is not None:
         end += (
             timedelta(minutes=1)
@@ -98,45 +101,45 @@ def verify(
 
     if cannot_determine_bounds:
         sync_success, sync_msg = self.sync(
-            begin = begin,
-            end = end,
-            params = params,
-            workers = workers,
-            debug = debug,
+            begin=begin,
+            end=end,
+            params=params,
+            workers=workers,
+            debug=debug,
             **kwargs
         )
         if not sync_success:
             return sync_success, sync_msg
+
         if deduplicate:
             return self.deduplicate(
-                begin = begin,
-                end = end,
-                params = params,
-                workers = workers,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                workers=workers,
+                debug=debug,
                 **kwargs
             )
         return sync_success, sync_msg
 
-
     chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     chunk_bounds = self.get_chunk_bounds(
-        begin = begin,
-        end = end,
-        chunk_interval = chunk_interval,
-        bounded = bounded,
-        debug = debug,
+        begin=begin,
+        end=end,
+        chunk_interval=chunk_interval,
+        bounded=bounded,
+        debug=debug,
     )
 
     ### Consider it a success if no chunks need to be verified.
     if not chunk_bounds:
         if deduplicate:
             return self.deduplicate(
-                begin = begin,
-                end = end,
-                params = params,
-                workers = workers,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                workers=workers,
+                debug=debug,
                 **kwargs
             )
         return True, f"Could not determine chunks between '{begin}' and '{end}'; nothing to do."
@@ -175,21 +178,21 @@ def verify(
     ### }
     bounds_success_tuples = {}
     def process_chunk_bounds(
-            chunk_begin_and_end: Tuple[
-                Union[int, datetime],
-                Union[int, datetime]
-            ]
-        ):
+        chunk_begin_and_end: Tuple[
+            Union[int, datetime],
+            Union[int, datetime]
+        ]
+    ):
         if chunk_begin_and_end in bounds_success_tuples:
             return chunk_begin_and_end, bounds_success_tuples[chunk_begin_and_end]
 
         chunk_begin, chunk_end = chunk_begin_and_end
         return chunk_begin_and_end, self.sync(
-            begin = chunk_begin,
-            end = chunk_end,
-            params = params,
-            workers = workers,
-            debug = debug,
+            begin=chunk_begin,
+            end=chunk_end,
+            params=params,
+            workers=workers,
+            debug=debug,
             **kwargs
         )
 
@@ -216,11 +219,11 @@ def verify(
         msg = get_chunks_success_message(bounds_success_tuples, header=message_header)
         if deduplicate:
             deduplicate_success, deduplicate_msg = self.deduplicate(
-                begin = begin,
-                end = end,
-                params = params,
-                workers = workers,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                workers=workers,
+                debug=debug,
                 **kwargs
             )
             return deduplicate_success, msg + '\n\n' + deduplicate_msg
@@ -239,7 +242,7 @@ def verify(
         warn(
             f"Will resync the following failed chunks:\n    "
             + '\n    '.join(bounds_to_print),
-            stack = False,
+            stack=False,
         )
 
     retry_bounds_success_tuples = dict(pool.map(process_chunk_bounds, chunk_bounds_to_resync))
@@ -256,11 +259,11 @@ def verify(
         )
         if deduplicate:
             deduplicate_success, deduplicate_msg = self.deduplicate(
-                begin = begin,
-                end = end,
-                params = params,
-                workers = workers,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                workers=workers,
+                debug=debug,
                 **kwargs
             )
             return deduplicate_success, message + '\n\n' + deduplicate_msg
@@ -269,11 +272,11 @@ def verify(
     message = get_chunks_success_message(bounds_success_tuples, header=message_header)
     if deduplicate:
         deduplicate_success, deduplicate_msg = self.deduplicate(
-            begin = begin,
-            end = end,
-            params = params,
-            workers = workers,
-            debug = debug,
+            begin=begin,
+            end=end,
+            params=params,
+            workers=workers,
+            debug=debug,
             **kwargs
         )
         return deduplicate_success, message + '\n\n' + deduplicate_msg
@@ -417,7 +420,7 @@ def get_bound_time(self, debug: bool = False) -> Union[datetime, int, None]:
     -------
     A `datetime` or `int` corresponding to the
     `begin` bound for verification and deduplication syncs.
-    """ 
+    """
     bound_interval = self.get_bound_interval(debug=debug)
     if bound_interval is None:
         return None

meerschaum/utils/dataframe.py

@@ -235,9 +235,9 @@ def filter_unseen_df(
     try:
         for col, typ in dt_dtypes.items():
             if col in old_df.columns:
-                old_df[col] = coerce_timezone(pd.to_datetime(old_df[col], utc=True))
+                old_df[col] = coerce_timezone(old_df[col])
             if col in new_df.columns:
-                new_df[col] = coerce_timezone(pd.to_datetime(new_df[col], utc=True))
+                new_df[col] = coerce_timezone(new_df[col])
         cast_dt_cols = False
     except Exception as e:
         warn(f"Could not cast datetime columns:\n{e}")
@@ -365,7 +365,7 @@ def filter_unseen_df(
 def parse_df_datetimes(
     df: 'pd.DataFrame',
     ignore_cols: Optional[Iterable[str]] = None,
-    strip_timezone: bool = True,
+    strip_timezone: bool = False,
     chunksize: Optional[int] = None,
     dtype_backend: str = 'numpy_nullable',
     debug: bool = False,
@@ -381,7 +381,7 @@ def parse_df_datetimes(
     ignore_cols: Optional[Iterable[str]], default None
         If provided, do not attempt to coerce these columns as datetimes.
 
-    strip_timezone: bool, default True
+    strip_timezone: bool, default False
         If `True`, remove the UTC `tzinfo` property.
 
     chunksize: Optional[int], default None
@@ -486,7 +486,7 @@ def parse_df_datetimes(
     if len(cols_to_inspect) == 0:
         if debug:
             dprint(f"All columns are ignored, skipping datetime detection...")
-        return df
+        return df.fillna(pandas.NA)
 
     ### apply regex to columns to determine which are ISO datetimes
     iso_dt_regex = r'\d{4}-\d{2}-\d{2}.\d{2}\:\d{2}\:\d+'
@@ -499,7 +499,7 @@ def parse_df_datetimes(
     if not datetime_cols:
         if debug:
             dprint("No columns detected as datetimes, returning...")
-        return df
+        return df.fillna(pandas.NA)
 
     if debug:
         dprint("Converting columns to datetimes: " + str(datetime_cols))
@@ -537,7 +537,7 @@ def parse_df_datetimes(
                     + f"{traceback.format_exc()}"
                 )
 
-    return df
+    return df.fillna(pandas.NA)
 
 
 def get_unhashable_cols(df: 'pd.DataFrame') -> List[str]:
@@ -689,6 +689,7 @@ def enforce_dtypes(
     safe_copy: bool = True,
     coerce_numeric: bool = True,
     coerce_timezone: bool = True,
+    strip_timezone: bool = False,
     debug: bool = False,
 ) -> 'pd.DataFrame':
     """
@@ -713,6 +714,10 @@ def enforce_dtypes(
     coerce_timezone: bool, default True
         If `True`, convert datetimes to UTC.
 
+    strip_timezone: bool, default False
+        If `coerce_timezone` and `strip_timezone` are `True`,
+        remove timezone information from datetimes.
+
     debug: bool, default False
         Verbosity toggle.
 
@@ -731,6 +736,8 @@ def enforce_dtypes(
         attempt_cast_to_uuid,
         coerce_timezone as _coerce_timezone,
     )
+    pandas = mrsm.attempt_import('pandas')
+    is_dask = 'dask' in df.__module__
     if safe_copy:
         df = df.copy()
     if len(df.columns) == 0:
@@ -814,7 +821,8 @@ def enforce_dtypes(
         if debug:
             dprint(f"Checking for datetime conversion: {datetime_cols}")
         for col in datetime_cols:
-            df[col] = _coerce_timezone(df[col])
+            if col in df.columns:
+                df[col] = _coerce_timezone(df[col], strip_utc=strip_timezone)
 
     df_dtypes = {c: str(t) for c, t in df.dtypes.items()}
     if are_dtypes_equal(df_dtypes, pipe_pandas_dtypes):