meerschaum 2.0.0rc8__tar.gz → 2.0.0rc9__tar.gz

{meerschaum-2.0.0rc8/meerschaum.egg-info → meerschaum-2.0.0rc9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.0.0rc8
+Version: 2.0.0rc9
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/config/_default.py

@@ -112,12 +112,12 @@ default_pipes_config = {
             'datetime': None,
             'id': None,
         },
-        'chunk_minutes': 1440,
         'fetch': {
             'backtrack_minutes': 1440,
         },
         'verify': {
             'bound_days': 366,
+            'chunk_minutes': 1440,
         },
     },
     'attributes': {

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.0.0rc8"
+__version__ = "2.0.0rc9"

meerschaum-2.0.0rc9/meerschaum/connectors/api/_fetch.py

@@ -0,0 +1,68 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Fetch Pipe data via the API connector
+"""
+
+from __future__ import annotations
+from datetime import datetime
+import copy
+import meerschaum as mrsm
+from meerschaum.utils.typing import Any, Optional, Dict, Iterator, Union
+
+def fetch(
+        self,
+        pipe: mrsm.Pipe,
+        begin: Union[datetime, str, int] = '',
+        end: Union[datetime, int] = None,
+        params: Optional[Dict, Any] = None,
+        debug: bool = False,
+        **kw: Any
+    ) -> Iterator['pd.DataFrame']:
+    """Get the Pipe data from the remote Pipe."""
+    from meerschaum.utils.debug import dprint
+    from meerschaum.utils.warnings import warn, error
+    from meerschaum.config._patch import apply_patch_to_config
+
+    fetch_params = pipe.parameters.get('fetch', {})
+    if not fetch_params:
+        warn(f"Missing 'fetch' parameters for {pipe}.", stack=False)
+        return None
+
+    pipe_meta = fetch_params.get('pipe', {})
+    ### Legacy: check for `connector_keys`, etc. at the root.
+    if not pipe_meta:
+        ck, mk, lk = (
+            fetch_params.get('connector_keys', None),
+            fetch_params.get('metric_key', None),
+            fetch_params.get('location_key', None),
+        )
+        if not ck or not mk:
+            warn(f"Missing `fetch:pipe` keys for {pipe}.", stack=False)
+            return None
+
+        pipe_meta.update({
+            'connector': ck,
+            'metric': mk,
+            'location': lk,
+        })
+
+    pipe_meta['instance'] = self
+    source_pipe = mrsm.Pipe(**pipe_meta)
+
+    _params = copy.deepcopy(params) if params is not None else {}
+    _params = apply_patch_to_config(_params, fetch_params.get('params', {}))
+    select_columns = fetch_params.get('select_columns', [])
+    omit_columns = fetch_params.get('omit_columns', [])
+
+    return source_pipe.get_data(
+        select_columns = select_columns,
+        omit_columns = omit_columns,
+        begin = begin,
+        end = end,
+        params = _params,
+        debug = debug,
+        as_iterator = True,
+    )

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/connectors/api/_pipes.py

@@ -164,9 +164,7 @@ def sync_pipe(
         debug: bool = False,
         **kw: Any
     ) -> SuccessTuple:
-    """Append a pandas DataFrame to a Pipe.
-    If Pipe does not exist, it is registered with supplied metadata.
-    """
+    """Sync a DataFrame into a Pipe."""
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.misc import json_serialize_datetime
     from meerschaum.config import get_config
@@ -186,14 +184,14 @@ def sync_pipe(
 
     df = json.loads(df) if isinstance(df, str) else df
 
-    ### TODO Make separate chunksize for API?
-    _chunksize : Optional[int] = (1 if chunksize is None else (
+    _chunksize: Optional[int] = (1 if chunksize is None else (
         get_config('system', 'connectors', 'sql', 'chunksize') if chunksize == -1
         else chunksize
     ))
-    keys : list = list(df.keys())
+    keys: list = list(df.keys())
     chunks = []
     if hasattr(df, 'index'):
+        df = df.reset_index(drop=True)
         rowcount = len(df)
         chunks = [df.iloc[i] for i in more_itertools.chunked(df.index, _chunksize)]
     elif isinstance(df, dict):

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/connectors/sql/_pipes.py

@@ -1438,12 +1438,11 @@ def sync_pipe_inplace(
     drop_backtrack_query = f"DROP TABLE {backtrack_table_name}"
     if table_exists(backtrack_table_raw, self, debug=debug):
         backtrack_queries.append(drop_backtrack_query)
-    btm = max(self.get_pipe_backtrack_minutes(pipe), 1)
     backtrack_def = self.get_pipe_data_query(
         pipe,
         begin = begin,
         end = end,
-        begin_add_minutes = (-1 * btm),
+        begin_add_minutes = 0,
         end_add_minutes = 1,
         params = params,
         debug = debug,

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/core/Pipe/__init__.py

@@ -81,7 +81,10 @@ class Pipe:
     ```
     """
 
-    from ._fetch import fetch
+    from ._fetch import (
+        fetch,
+        get_backtrack_interval,
+    )
     from ._data import (
         get_data,
         get_backtrack_data,
@@ -279,15 +282,26 @@ class Pipe:
 
     @property
     def meta(self):
-        """Simulate the MetaPipe model without importing FastAPI."""
-        if '_meta' not in self.__dict__:
-            self._meta = {
-                'connector_keys' : self.connector_keys,
-                'metric_key'     : self.metric_key,
-                'location_key'   : self.location_key,
-                'instance'       : self.instance_keys,
-            }
-        return self._meta
+        """
+        Return the four keys needed to reconstruct this pipe.
+        """
+        return {
+            'connector_keys': self.connector_keys,
+            'metric_key'    : self.metric_key,
+            'location_key'  : self.location_key,
+            'instance'      : self.instance_keys,
+        }
+
+
+    def keys(self) -> List[str]:
+        """
+        Return the ordered keys for this pipe.
+        """
+        return {
+            key: val
+            for key, val in self.meta.items()
+            if key != 'instance'
+        }
 
 
     @property
@@ -436,3 +450,10 @@ class Pipe:
         metric_key = _state.pop('metric_key')
         location_key = _state.pop('location_key')
         self.__init__(connector_keys, metric_key, location_key, **_state)
+
+
+    def __getitem__(self, *args, **kwargs) -> Any:
+        """
+        Index the pipe's attributes.
+        """
+        return self.attributes.__getitem__(*args, **kwargs)

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/core/Pipe/_data.py

@@ -325,7 +325,7 @@ def _get_data_as_iterator(
 def get_backtrack_data(
         self,
         backtrack_minutes: int = 0,
-        begin: Optional[datetime] = None,
+        begin: Union[datetime, int, None] = None,
         params: Optional[Dict[str, Any]] = None,
         fresh: bool = False,
         debug: bool = False,
@@ -338,7 +338,7 @@ def get_backtrack_data(
     ----------
     backtrack_minutes: int, default 0
         How many minutes from `begin` to select from.
-        Defaults to 0. This may return a few rows due to a rounding quirk.
+        If 0 (default), use `pipe.parameters['fetch']['backtrack_minutes']`.
 
     begin: Optional[datetime], default None
         The starting point to search for data.
@@ -370,7 +370,6 @@ def get_backtrack_data(
     -------
     A `pd.DataFrame` for the pipe's data corresponding to the provided parameters. Backtrack data
     is a convenient way to get a pipe's data "backtracked" from the most recent datetime.
-
     """
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
@@ -379,6 +378,14 @@ def get_backtrack_data(
     if not self.exists(debug=debug):
         return None
 
+    backtrack_interval = self.get_backtrack_interval(debug=debug)
+    if backtrack_minutes == 0:
+        backtrack_minutes = (
+            (backtrack_interval.total_seconds() * 60)
+            if isinstance(backtrack_interval, timedelta)
+            else backtrack_interval
+        )
+
     if self.cache_pipe is not None:
         if not fresh:
             _sync_cache_tuple = self.cache_pipe.sync(begin=begin, params=params, debug=debug, **kw)
@@ -438,7 +445,7 @@ def get_rowcount(
         params: Optional[Dict[str, Any]] = None,
         remote: bool = False,
         debug: bool = False
-    ) -> Union[int, None]:
+    ) -> int:
     """
     Get a Pipe's instance or remote rowcount.
 
@@ -460,8 +467,7 @@ def get_rowcount(
     Returns
     -------
     An `int` of the number of rows in the pipe corresponding to the provided parameters.
-    `None` is returned if the pipe does not exist.
-
+    Returned 0 if the pipe does not exist.
     """
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
@@ -470,7 +476,7 @@ def get_rowcount(
     connector = self.instance_connector if not remote else self.connector
     try:
         with Venv(get_connector_plugin(connector)):
-            return connector.get_pipe_rowcount(
+            rowcount = connector.get_pipe_rowcount(
                 self,
                 begin = begin,
                 end = end,
@@ -478,12 +484,15 @@ def get_rowcount(
                 remote = remote,
                 debug = debug,
             )
+            if rowcount is None:
+                return 0
+            return rowcount
     except AttributeError as e:
         warn(e)
         if remote:
-            return None
+            return 0
     warn(f"Failed to get a rowcount for {self}.")
-    return None
+    return 0
 
 
 def get_chunk_interval(
@@ -505,8 +514,8 @@ def get_chunk_interval(
     -------
     The chunk interval (`timedelta` or `int`) to use with this pipe's `datetime` axis.
     """
-    default_chunk_minutes = get_config('pipes', 'parameters', 'chunk_minutes')
-    configured_chunk_minutes = self.parameters.get('chunk_minutes', None)
+    default_chunk_minutes = get_config('pipes', 'parameters', 'verify', 'chunk_minutes')
+    configured_chunk_minutes = self.parameters.get('verify', {}).get('chunk_minutes', None)
     chunk_minutes = (
         (configured_chunk_minutes or default_chunk_minutes)
         if chunk_interval is None
@@ -559,7 +568,8 @@ def get_chunk_bounds(
 
     chunk_interval: Union[timedelta, int, None], default None
         If provided, use this interval for the size of chunk boundaries.
-        The default value for this pipe may be set under `pipe.parameters['chunk_minutes']`.
+        The default value for this pipe may be set
+        under `pipe.parameters['verify']['chunk_minutes']`.
 
     debug: bool, default False
         Verbosity toggle.
@@ -578,7 +588,7 @@ def get_chunk_bounds(
     if begin is None and end is None:
         return [(None, None)]
 
-    ### Set the chunk interval under `pipe.parameters['chunk_minutes']`.
+    ### Set the chunk interval under `pipe.parameters['verify']['chunk_minutes']`.
     chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     
     ### Build a list of tuples containing the chunk boundaries

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/core/Pipe/_deduplicate.py

@@ -173,7 +173,12 @@ def deduplicate(
         if not chunk_indices:
             return bounds, (False, f"None of {items_str(indices)} were present in chunk.")
         try:
-            full_chunk = full_chunk.drop_duplicates(subset=chunk_indices, keep='last')
+            full_chunk = full_chunk.drop_duplicates(
+                subset = chunk_indices,
+                keep = 'last'
+            ).reset_index(
+                drop = True,
+            )
         except Exception as e:
             return (
                 bounds,

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/core/Pipe/_fetch.py

@@ -7,7 +7,9 @@ Functions for fetching new data into the Pipe
 """
 
 from __future__ import annotations
+from datetime import timedelta
 from meerschaum.utils.typing import Optional, Any
+from meerschaum.config import get_config
 
 def fetch(
         self,
@@ -98,3 +100,30 @@ def fetch(
         connector_plugin.deactivate_venv(debug=debug)
 
     return df
+
+
+def get_backtrack_interval(self, debug: bool = False) -> Union[timedelta, int]:
+    """
+    Get the chunk interval to use for this pipe.
+
+    Returns
+    -------
+    The backtrack interval (`timedelta` or `int`) to use with this pipe's `datetime` axis.
+    """
+    default_backtrack_minutes = get_config('pipes', 'parameters', 'fetch', 'backtrack_minutes')
+    configured_backtrack_minutes = self.parameters.get('fetch', {}).get('backtrack_minutes', None)
+    backtrack_minutes = (
+        configured_backtrack_minutes
+        if configured_backtrack_minutes is not None
+        else default_backtrack_minutes
+    )
+
+    dt_col = self.columns.get('datetime', None)
+    if dt_col is None:
+        return timedelta(minutes=backtrack_minutes)
+
+    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns]')
+    if 'datetime' in dt_dtype.lower():
+        return timedelta(minutes=backtrack_minutes)
+
+    return backtrack_minutes

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/core/Pipe/_sync.py

@@ -14,7 +14,16 @@ import threading
 from datetime import datetime, timedelta
 
 from meerschaum.utils.typing import (
-    Union, Optional, Callable, Any, Tuple, SuccessTuple, Mapping, Dict, List, Iterable, Generator,
+    Union,
+    Optional,
+    Callable,
+    Any,
+    Tuple,
+    SuccessTuple,
+    Dict,
+    List,
+    Iterable,
+    Generator,
     Iterator,
 )
 
@@ -29,8 +38,8 @@ def sync(
             List[Dict[str, Any]],
             InferFetch
         ] = InferFetch,
-        begin: Optional[datetime] = None,
-        end: Optional[datetime] = None,
+        begin: Union[datetime, int, str, None] = '',
+        end: Union[datetime, int] = None,
         force: bool = False,
         retries: int = 10,
         min_seconds: int = 1,
@@ -55,28 +64,23 @@ def sync(
     df: Union[None, pd.DataFrame, Dict[str, List[Any]]], default None
         An optional DataFrame to sync into the pipe. Defaults to `None`.
 
-    begin: Optional[datetime], default None
+    begin: Union[datetime, int, str, None], default ''
         Optionally specify the earliest datetime to search for data.
-        Defaults to `None`.
 
-    end: Optional[datetime], default None
+    end: Union[datetime, int, str, None], default None
         Optionally specify the latest datetime to search for data.
-        Defaults to `None`.
 
     force: bool, default False
         If `True`, keep trying to sync untul `retries` attempts.
-        Defaults to `False`.
 
     retries: int, default 10
         If `force`, how many attempts to try syncing before declaring failure.
-        Defaults to `10`.
 
     min_seconds: Union[int, float], default 1
         If `force`, how many seconds to sleep between retries. Defaults to `1`.
 
     check_existing: bool, default True
         If `True`, pull and diff with existing data from the pipe.
-        Defaults to `True`.
 
     blocking: bool, default True
         If `True`, wait for sync to finish and return its result, otherwise
@@ -87,7 +91,6 @@ def sync(
         If provided and the instance connector is thread-safe
         (`pipe.instance_connector.IS_THREAD_SAFE is True`),
         limit concurrent sync to this many threads.
-        Defaults to `None`.
 
     callback: Optional[Callable[[Tuple[bool, str]], Any]], default None
         Callback function which expects a SuccessTuple as input.
@@ -101,7 +104,6 @@ def sync(
         Specify the number of rows to sync per chunk.
         If `-1`, resort to system configuration (default is `900`).
         A `chunksize` of `None` will sync all rows in one transaction.
-        Defaults to `-1`.
 
     sync_chunks: bool, default True
         If possible, sync chunks while fetching them into memory.
@@ -112,7 +114,6 @@ def sync(
     Returns
     -------
     A `SuccessTuple` of success (`bool`) and message (`str`).
-
     """
     from meerschaum.utils.debug import dprint, _checkpoint
     from meerschaum.utils.warnings import warn, error
@@ -183,7 +184,7 @@ def sync(
         ### use that instead.
         ### NOTE: The DataFrame must be omitted for the plugin sync method to apply.
         ### If a DataFrame is provided, continue as expected.
-        if hasattr(df, 'MRSM_INFER_FETCH'):
+        if hasattr(df, 'MRSM_INFER_FETCH'):                   
             try:
                 if p.connector is None:
                     msg = f"{p} does not have a valid connector."
@@ -430,13 +431,19 @@ def sync(
 
 def _determine_begin(
         pipe: meerschaum.Pipe,
-        begin: Optional[datetime] = None,
+        begin: Union[datetime, int, str] = '',
         debug: bool = False,
     ) -> Union[datetime, int, None]:
-    ### Datetime has already been provided.
-    if begin is not None:
+    """
+    Apply the backtrack interval if `--begin` is not provided.
+    """
+    if begin != '':
         return begin
-    return pipe.get_sync_time(debug=debug)
+    sync_time = pipe.get_sync_time(debug=debug)
+    if sync_time is None:
+        return sync_time
+    backtrack_interval = pipe.get_backtrack_interval(debug=debug)
+    return sync_time - backtrack_interval
 
 
 def get_sync_time(

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/core/Pipe/_verify.py

@@ -211,7 +211,7 @@ def verify(
     bounds_success_tuples.update(dict(pool.map(process_chunk_bounds, chunk_bounds)))
     bounds_success_bools = {bounds: tup[0] for bounds, tup in bounds_success_tuples.items()}
 
-    message_header = f"{begin} - {end}"
+    message_header = f"{begin_to_print} - {end_to_print}"
     if all(bounds_success_bools.values()):
         msg = get_chunks_success_message(bounds_success_tuples, header=message_header)
         if deduplicate:

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/utils/dataframe.py

@@ -651,6 +651,8 @@ def get_datetime_bound_from_df(
         return best_yet
 
     if 'DataFrame' in str(type(df)):
+        if datetime_column not in df.columns:
+            return None
         return (
             df[datetime_column].min(skipna=True)
             if minimum

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9}/meerschaum/utils/misc.py

@@ -500,7 +500,7 @@ def timed_input(
         return input(prompt)
     except TimeoutExpired:
         return None
-    except EOFError:
+    except (EOFError, RuntimeError):
         try:
             print(prompt)
             time.sleep(seconds)

{meerschaum-2.0.0rc8 → meerschaum-2.0.0rc9/meerschaum.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.0.0rc8
+Version: 2.0.0rc9
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares

meerschaum-2.0.0rc8/meerschaum/connectors/api/_fetch.py

@@ -1,66 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-# vim:fenc=utf-8
-
-"""
-Fetch Pipe data via the API connector
-"""
-
-from __future__ import annotations
-import datetime
-import copy
-from meerschaum.utils.typing import Any, Optional, Dict
-
-def fetch(
-        self,
-        pipe: meerschaum.Pipe,
-        begin: Optional[datetime.datetime, str] = '',
-        end: Optional[datetime.datetime] = None,
-        params: Optional[Dict, Any] = None,
-        debug: bool = False,
-        **kw: Any
-    ) -> pandas.DataFrame:
-    """Get the Pipe data from the remote Pipe."""
-    from meerschaum.utils.debug import dprint
-    from meerschaum.utils.warnings import warn, error
-    from meerschaum.config.static import _static_config
-    from meerschaum.config._patch import apply_patch_to_config
-
-    if 'fetch' not in pipe.parameters:
-        warn(f"Missing 'fetch' parameters for Pipe '{pipe}'.", stack=False)
-        return None
-
-    instructions = pipe.parameters['fetch']
-
-    if 'connector_keys' not in instructions:
-        warn(f"Missing connector_keys in fetch parameters for Pipe '{pipe}'", stack=False)
-        return None
-    remote_connector_keys = instructions.get('connector_keys', None)
-    if 'metric_key' not in instructions:
-        warn(f"Missing metric_key in fetch parameters for Pipe '{pipe}'", stack=False)
-        return None
-    remote_metric_key = instructions.get('metric_key', None)
-    remote_location_key = instructions.get('location_key', None)
-    if begin is None:
-        begin = pipe.get_sync_time(debug=debug)
-
-    _params = copy.deepcopy(params) if params is not None else {}
-    _params = apply_patch_to_config(_params, instructions.get('params', {}))
-
-    from meerschaum import Pipe
-    p = Pipe(
-        remote_connector_keys,
-        remote_metric_key,
-        remote_location_key,
-        mrsm_instance = self
-    )
-    begin = (
-        begin if not (isinstance(begin, str) and begin == '')
-        else pipe.get_sync_time(debug=debug)
-    )
-
-    return p.get_data(
-        begin=begin, end=end,
-        params=_params,
-        debug=debug
-    )