meerschaum 2.5.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

meerschaum/core/Pipe/_copy.py

@@ -55,6 +55,8 @@ def copy_to(
     if str(instance_keys) == self.instance_keys:
         return False, f"Cannot copy {self} to instance '{instance_keys}'."
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     new_pipe = mrsm.Pipe(
         self.connector_keys,
         self.metric_key,

meerschaum/core/Pipe/_data.py

@@ -23,8 +23,8 @@ def get_data(
     self,
     select_columns: Optional[List[str]] = None,
     omit_columns: Optional[List[str]] = None,
-    begin: Union[datetime, int, None] = None,
-    end: Union[datetime, int, None] = None,
+    begin: Union[datetime, int, str, None] = None,
+    end: Union[datetime, int, str, None] = None,
     params: Optional[Dict[str, Any]] = None,
     as_iterator: bool = False,
     as_chunks: bool = False,
@@ -48,12 +48,12 @@ def get_data(
     omit_columns: Optional[List[str]], default None
         If provided, remove these columns from the selection.
 
-    begin: Union[datetime, int, None], default None
+    begin: Union[datetime, int, str, None], default None
         Lower bound datetime to begin searching for data (inclusive).
         Translates to a `WHERE` clause like `WHERE datetime >= begin`.
         Defaults to `None`.
 
-    end: Union[datetime, int, None], default None
+    end: Union[datetime, int, str, None], default None
         Upper bound datetime to stop searching for data (inclusive).
         Translates to a `WHERE` clause like `WHERE datetime < end`.
         Defaults to `None`.
@@ -105,11 +105,12 @@ def get_data(
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.misc import iterate_chunks, items_str
-    from meerschaum.utils.dtypes import to_pandas_dtype
+    from meerschaum.utils.dtypes import to_pandas_dtype, coerce_timezone
     from meerschaum.utils.dataframe import add_missing_cols_to_df, df_is_chunk_generator
     from meerschaum.utils.packages import attempt_import
     dd = attempt_import('dask.dataframe') if as_dask else None
     dask = attempt_import('dask') if as_dask else None
+    dateutil_parser = attempt_import('dateutil.parser')
 
     if select_columns == '*':
         select_columns = None
@@ -119,12 +120,13 @@ def get_data(
     if isinstance(omit_columns, str):
         omit_columns = [omit_columns]
 
+    begin, end = self.parse_date_bounds(begin, end)
     as_iterator = as_iterator or as_chunks
+    dt_col = self.columns.get('datetime', None)
 
     def _sort_df(_df):
         if df_is_chunk_generator(_df):
             return _df
-        dt_col = self.columns.get('datetime', None)
         indices = [] if dt_col not in _df.columns else [dt_col]
         non_dt_cols = [
             col
@@ -311,16 +313,8 @@ def _get_data_as_iterator(
     Return a pipe's data as a generator.
     """
     from meerschaum.utils.misc import round_time
-    parse_begin = isinstance(begin, str)
-    parse_end = isinstance(end, str)
-    if parse_begin or parse_end:
-        from meerschaum.utils.packages import attempt_import
-        dateutil_parser = attempt_import('dateutil.parser')
-    if parse_begin:
-        begin = dateutil_parser.parse(begin)
-    if parse_end:
-        end = dateutil_parser.parse(end)
-
+    from meerschaum.utils.dtypes import coerce_timezone
+    begin, end = self.parse_date_bounds(begin, end)
     if not self.exists(debug=debug):
         return
 
@@ -332,11 +326,15 @@ def _get_data_as_iterator(
         if begin is not None
         else self.get_sync_time(round_down=False, newest=False, params=params, debug=debug)
     ) if dt_col else None
+    if isinstance(min_dt, datetime):
+        min_dt = coerce_timezone(min_dt)
     max_dt = (
         end
         if end is not None
         else self.get_sync_time(round_down=False, newest=True, params=params, debug=debug)
     ) if dt_col else None
+    if isinstance(max_dt, datetime):
+        max_dt = coerce_timezone(max_dt)
 
     ### We want to search just past the maximum value.
     if end is None:
@@ -450,6 +448,8 @@ def get_backtrack_data(
     if not self.exists(debug=debug):
         return None
 
+    begin = self.parse_date_bounds(begin)
+
     backtrack_interval = self.get_backtrack_interval(debug=debug)
     if backtrack_minutes is None:
         backtrack_minutes = (
@@ -550,6 +550,7 @@ def get_rowcount(
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
 
+    begin, end = self.parse_date_bounds(begin, end)
     connector = self.instance_connector if not remote else self.connector
     try:
         with Venv(get_connector_plugin(connector)):
@@ -607,7 +608,7 @@ def get_chunk_interval(
     if dt_col is None:
         return timedelta(minutes=chunk_minutes)
 
-    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns]')
+    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns, UTC]')
     if 'int' in dt_dtype.lower():
         return chunk_minutes
     return timedelta(minutes=chunk_minutes)
@@ -664,6 +665,8 @@ def get_chunk_bounds(
     if begin is None and end is None:
         return [(None, None)]
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     ### Set the chunk interval under `pipe.parameters['verify']['chunk_minutes']`.
     chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     
@@ -695,3 +698,48 @@ def get_chunk_bounds(
         chunk_bounds = chunk_bounds + [(end, None)]
 
     return chunk_bounds
+
+
+def parse_date_bounds(self, *dt_vals: Union[datetime, int, None]) -> Union[
+    datetime,
+    int,
+    str,
+    None,
+    Tuple[Union[datetime, int, str, None]]
+]:
+    """
+    Given a date bound (begin, end), coerce a timezone if necessary.
+    """
+    from meerschaum.utils.misc import is_int
+    from meerschaum.utils.dtypes import coerce_timezone
+    from meerschaum.utils.warnings import warn
+    dateutil_parser = mrsm.attempt_import('dateutil.parser')
+
+    def _parse_date_bound(dt_val):
+        if dt_val is None:
+            return None
+
+        if isinstance(dt_val, int):
+            return dt_val
+
+        if dt_val == '':
+            return ''
+
+        if is_int(dt_val):
+            return int(dt_val)
+
+        if isinstance(dt_val, str):
+            try:
+                dt_val = dateutil_parser.parse(dt_val)
+            except Exception as e:
+                warn(f"Could not parse '{dt_val}' as datetime:\n{e}")
+                return None
+
+        dt_col = self.columns.get('datetime', None)
+        dt_typ = str(self.dtypes.get(dt_col, 'datetime64[ns, UTC]'))
+        return coerce_timezone(dt_val, strip_utc=('utc' not in dt_typ.lower()))
+
+    bounds = tuple(_parse_date_bound(dt_val) for dt_val in dt_vals)
+    if len(bounds) == 1:
+        return bounds[0]
+    return bounds

meerschaum/core/Pipe/_deduplicate.py

@@ -65,14 +65,16 @@ def deduplicate(
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.pool import get_pool
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     if self.cache_pipe is not None:
         success, msg = self.cache_pipe.deduplicate(
-            begin = begin,
-            end = end,
-            params = params,
-            bounded = bounded,
-            debug = debug,
-            _use_instance_method = _use_instance_method,
+            begin=begin,
+            end=end,
+            params=params,
+            bounded=bounded,
+            debug=debug,
+            _use_instance_method=_use_instance_method,
             **kwargs
         )
         if not success:
@@ -86,11 +88,11 @@ def deduplicate(
             if hasattr(self.instance_connector, 'deduplicate_pipe'):
                 return self.instance_connector.deduplicate_pipe(
                     self,
-                    begin = begin,
-                    end = end,
-                    params = params,
-                    bounded = bounded,
-                    debug = debug,
+                    begin=begin,
+                    end=end,
+                    params=params,
+                    bounded=bounded,
+                    debug=debug,
                     **kwargs
                 )
 
@@ -117,33 +119,33 @@ def deduplicate(
         )
 
     chunk_bounds = self.get_chunk_bounds(
-        bounded = bounded,
-        begin = begin,
-        end = end,
-        chunk_interval = chunk_interval,
-        debug = debug,
+        bounded=bounded,
+        begin=begin,
+        end=end,
+        chunk_interval=chunk_interval,
+        debug=debug,
     )
 
     indices = [col for col in self.columns.values() if col]
     if not indices:
-        return False, f"Cannot deduplicate without index columns."
+        return False, "Cannot deduplicate without index columns."
     dt_col = self.columns.get('datetime', None)
 
     def process_chunk_bounds(bounds) -> Tuple[
-            Tuple[
-                Union[datetime, int, None],
-                Union[datetime, int, None]
-            ],
-            SuccessTuple
-        ]:
+        Tuple[
+            Union[datetime, int, None],
+            Union[datetime, int, None]
+        ],
+        SuccessTuple
+    ]:
         ### Only selecting the index values here to keep bandwidth down.
         chunk_begin, chunk_end = bounds
         chunk_df = self.get_data(
-            select_columns = indices, 
-            begin = chunk_begin,
-            end = chunk_end,
-            params = params,
-            debug = debug,
+            select_columns=indices, 
+            begin=chunk_begin,
+            end=chunk_end,
+            params=params,
+            debug=debug,
         )
         if chunk_df is None:
             return bounds, (True, "")

meerschaum/core/Pipe/_dtypes.py

@@ -101,18 +101,18 @@ def infer_dtypes(self, persist: bool = False, debug: bool = False) -> Dict[str,
         dt_col = self.columns.get('datetime', None)
         if dt_col:
             if not self.parameters.get('dtypes', {}).get(dt_col, None):
-                dtypes[dt_col] = 'datetime64[ns]'
+                dtypes[dt_col] = 'datetime64[ns, UTC]'
         return dtypes
 
-    from meerschaum.utils.sql import get_pd_type
-    from meerschaum.utils.misc import to_pandas_dtype
+    from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
+    from meerschaum.utils.dtypes import to_pandas_dtype
     columns_types = self.get_columns_types(debug=debug)
 
     ### NOTE: get_columns_types() may return either the types as
     ###       PostgreSQL- or Pandas-style.
     dtypes = {
         c: (
-            get_pd_type(t, allow_custom_dtypes=True)
+            get_pd_type_from_db_type(t, allow_custom_dtypes=True)
             if str(t).isupper()
             else to_pandas_dtype(t)
         )

meerschaum/core/Pipe/_fetch.py

@@ -18,14 +18,14 @@ if TYPE_CHECKING:
     pd = mrsm.attempt_import('pandas')
 
 def fetch(
-        self,
-        begin: Union[datetime, str, None] = '',
-        end: Optional[datetime] = None,
-        check_existing: bool = True,
-        sync_chunks: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union['pd.DataFrame', Iterator['pd.DataFrame'], None]:
+    self,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, None] = None,
+    check_existing: bool = True,
+    sync_chunks: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Union['pd.DataFrame', Iterator['pd.DataFrame'], None]:
     """
     Fetch a Pipe's latest data from its connector.
 
@@ -76,6 +76,8 @@ def fetch(
                 chunk_message = '\n' + chunk_label + '\n' + chunk_message
             return chunk_success, chunk_message
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     with mrsm.Venv(get_connector_plugin(self.connector)):
         _args, _kwargs = filter_arguments(
             self.connector.fetch,
@@ -125,7 +127,7 @@ def get_backtrack_interval(
     if dt_col is None:
         return backtrack_interval
 
-    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns]')
+    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns, UTC]')
     if 'int' in dt_dtype.lower():
         return backtrack_minutes
 
@@ -164,6 +166,6 @@ def _determine_begin(
         backtrack_interval = timedelta(minutes=backtrack_interval)
     try:
         return sync_time - backtrack_interval
-    except Exception as e:
+    except Exception:
         warn(f"Unable to substract backtrack interval {backtrack_interval} from {sync_time}.")
     return sync_time

meerschaum/core/Pipe/_sync.py

@@ -141,6 +141,7 @@ def sync(
         chunksize = None
         sync_chunks = False
 
+    begin, end = self.parse_date_bounds(begin, end)
     kw.update({
         'begin': begin,
         'end': end,
@@ -460,7 +461,7 @@ def get_sync_time(
     apply_backtrack_interval: bool = False,
     round_down: bool = False,
     debug: bool = False
-) -> Union['datetime', None]:
+) -> Union['datetime', int, None]:
     """
     Get the most recent datetime value for a Pipe.
 
@@ -485,7 +486,7 @@ def get_sync_time(
 
     Returns
     -------
-    A `datetime` object if the pipe exists, otherwise `None`.
+    A `datetime` or int, if the pipe exists, otherwise `None`.
 
     """
     from meerschaum.utils.venv import Venv
@@ -510,13 +511,13 @@ def get_sync_time(
         except Exception as e:
             warn(f"Failed to apply backtrack interval:\n{e}")
 
-    return sync_time
+    return self.parse_date_bounds(sync_time)
 
 
 def exists(
-        self,
-        debug : bool = False
-    ) -> bool:
+    self,
+    debug: bool = False
+) -> bool:
     """
     See if a Pipe's table exists.
 
@@ -549,7 +550,11 @@ def exists(
                 return _exists
 
     with Venv(get_connector_plugin(self.instance_connector)):
-        _exists = self.instance_connector.pipe_exists(pipe=self, debug=debug)
+        _exists = (
+            self.instance_connector.pipe_exists(pipe=self, debug=debug)
+            if hasattr(self.instance_connector, 'pipe_exists')
+            else False
+        )
 
     self.__dict__['_exists'] = _exists
     self.__dict__['_exists_timestamp'] = now
@@ -624,6 +629,18 @@ def filter_existing(
         merge = pd.merge
         NA = pd.NA
 
+    primary_key = self.columns.get('primary', None)
+    autoincrement = self.parameters.get('autoincrement', False)
+    pipe_columns = self.columns.copy()
+
+    if primary_key and autoincrement and df is not None and primary_key in df.columns:
+        if safe_copy:
+            df = df.copy()
+            safe_copy = False
+        if df[primary_key].isnull().all():
+            del df[primary_key]
+            _ = self.columns.pop(primary_key, None)
+
     def get_empty_df():
         empty_df = pd.DataFrame([])
         dtypes = dict(df.dtypes) if df is not None else {}
@@ -643,8 +660,8 @@ def filter_existing(
 
     ### begin is the oldest data in the new dataframe
     begin, end = None, None
-    dt_col = self.columns.get('datetime', None)
-    dt_type = self.dtypes.get(dt_col, 'datetime64[ns]') if dt_col else None
+    dt_col = pipe_columns.get('datetime', None)
+    dt_type = self.dtypes.get(dt_col, 'datetime64[ns, UTC]') if dt_col else None
     try:
         min_dt_val = df[dt_col].min(skipna=True) if dt_col else None
         if is_dask and min_dt_val is not None:
@@ -713,7 +730,7 @@ def filter_existing(
 
     unique_index_vals = {
         col: df[col].unique()
-        for col in self.columns
+        for col in pipe_columns
         if col in df.columns and col != dt_col
     } if not date_bound_only else {}
     filter_params_index_limit = get_config('pipes', 'sync', 'filter_params_index_limit')
@@ -749,7 +766,7 @@ def filter_existing(
 
     ### Separate new rows from changed ones.
     on_cols = [
-        col for col_key, col in self.columns.items()
+        col for col_key, col in pipe_columns.items()
         if (
             col
             and

meerschaum/core/Pipe/_verify.py

@@ -11,6 +11,7 @@ from meerschaum.utils.typing import SuccessTuple, Any, Optional, Union, Tuple, L
 from meerschaum.utils.warnings import warn, info
 from meerschaum.utils.debug import dprint
 
+
 def verify(
     self,
     begin: Union[datetime, int, None] = None,
@@ -84,6 +85,8 @@ def verify(
     if bounded and end is None:
         end = self.get_sync_time(newest=True, debug=debug)
 
+    begin, end = self.parse_date_bounds(begin, end)
+
     if bounded and end is not None:
         end += (
             timedelta(minutes=1)
@@ -98,45 +101,45 @@ def verify(
 
     if cannot_determine_bounds:
         sync_success, sync_msg = self.sync(
-            begin = begin,
-            end = end,
-            params = params,
-            workers = workers,
-            debug = debug,
+            begin=begin,
+            end=end,
+            params=params,
+            workers=workers,
+            debug=debug,
             **kwargs
         )
         if not sync_success:
             return sync_success, sync_msg
+
         if deduplicate:
             return self.deduplicate(
-                begin = begin,
-                end = end,
-                params = params,
-                workers = workers,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                workers=workers,
+                debug=debug,
                 **kwargs
             )
         return sync_success, sync_msg
 
-
     chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     chunk_bounds = self.get_chunk_bounds(
-        begin = begin,
-        end = end,
-        chunk_interval = chunk_interval,
-        bounded = bounded,
-        debug = debug,
+        begin=begin,
+        end=end,
+        chunk_interval=chunk_interval,
+        bounded=bounded,
+        debug=debug,
     )
 
     ### Consider it a success if no chunks need to be verified.
     if not chunk_bounds:
         if deduplicate:
             return self.deduplicate(
-                begin = begin,
-                end = end,
-                params = params,
-                workers = workers,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                workers=workers,
+                debug=debug,
                 **kwargs
             )
         return True, f"Could not determine chunks between '{begin}' and '{end}'; nothing to do."
@@ -175,21 +178,21 @@ def verify(
     ### }
     bounds_success_tuples = {}
     def process_chunk_bounds(
-            chunk_begin_and_end: Tuple[
-                Union[int, datetime],
-                Union[int, datetime]
-            ]
-        ):
+        chunk_begin_and_end: Tuple[
+            Union[int, datetime],
+            Union[int, datetime]
+        ]
+    ):
         if chunk_begin_and_end in bounds_success_tuples:
             return chunk_begin_and_end, bounds_success_tuples[chunk_begin_and_end]
 
         chunk_begin, chunk_end = chunk_begin_and_end
         return chunk_begin_and_end, self.sync(
-            begin = chunk_begin,
-            end = chunk_end,
-            params = params,
-            workers = workers,
-            debug = debug,
+            begin=chunk_begin,
+            end=chunk_end,
+            params=params,
+            workers=workers,
+            debug=debug,
             **kwargs
         )
 
@@ -216,11 +219,11 @@ def verify(
         msg = get_chunks_success_message(bounds_success_tuples, header=message_header)
         if deduplicate:
             deduplicate_success, deduplicate_msg = self.deduplicate(
-                begin = begin,
-                end = end,
-                params = params,
-                workers = workers,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                workers=workers,
+                debug=debug,
                 **kwargs
             )
             return deduplicate_success, msg + '\n\n' + deduplicate_msg
@@ -239,7 +242,7 @@ def verify(
         warn(
             f"Will resync the following failed chunks:\n    "
             + '\n    '.join(bounds_to_print),
-            stack = False,
+            stack=False,
         )
 
     retry_bounds_success_tuples = dict(pool.map(process_chunk_bounds, chunk_bounds_to_resync))
@@ -256,11 +259,11 @@ def verify(
         )
         if deduplicate:
             deduplicate_success, deduplicate_msg = self.deduplicate(
-                begin = begin,
-                end = end,
-                params = params,
-                workers = workers,
-                debug = debug,
+                begin=begin,
+                end=end,
+                params=params,
+                workers=workers,
+                debug=debug,
                 **kwargs
             )
             return deduplicate_success, message + '\n\n' + deduplicate_msg
@@ -269,11 +272,11 @@ def verify(
     message = get_chunks_success_message(bounds_success_tuples, header=message_header)
     if deduplicate:
         deduplicate_success, deduplicate_msg = self.deduplicate(
-            begin = begin,
-            end = end,
-            params = params,
-            workers = workers,
-            debug = debug,
+            begin=begin,
+            end=end,
+            params=params,
+            workers=workers,
+            debug=debug,
             **kwargs
         )
         return deduplicate_success, message + '\n\n' + deduplicate_msg
@@ -394,7 +397,7 @@ def get_bound_interval(self, debug: bool = False) -> Union[timedelta, int, None]
     if not dt_col:
         return bound_time_value
 
-    dt_typ = self.dtypes.get(dt_col, 'datetime64[ns]')
+    dt_typ = self.dtypes.get(dt_col, 'datetime64[ns, UTC]')
     if 'int' in dt_typ.lower():
         return int(bound_time_value)
 
@@ -417,7 +420,7 @@ def get_bound_time(self, debug: bool = False) -> Union[datetime, int, None]:
     -------
     A `datetime` or `int` corresponding to the
     `begin` bound for verification and deduplication syncs.
-    """ 
+    """
     bound_interval = self.get_bound_interval(debug=debug)
     if bound_interval is None:
         return None