meerschaum 2.0.0rc7__py3-none-any.whl → 2.0.0rc9__py3-none-any.whl

meerschaum/connectors/api/_fetch.py

@@ -7,60 +7,62 @@ Fetch Pipe data via the API connector
 """
 
 from __future__ import annotations
-import datetime
+from datetime import datetime
 import copy
-from meerschaum.utils.typing import Any, Optional, Dict
+import meerschaum as mrsm
+from meerschaum.utils.typing import Any, Optional, Dict, Iterator, Union
 
 def fetch(
         self,
-        pipe: meerschaum.Pipe,
-        begin: Optional[datetime.datetime, str] = '',
-        end: Optional[datetime.datetime] = None,
+        pipe: mrsm.Pipe,
+        begin: Union[datetime, str, int] = '',
+        end: Union[datetime, int] = None,
         params: Optional[Dict, Any] = None,
         debug: bool = False,
         **kw: Any
-    ) -> pandas.DataFrame:
+    ) -> Iterator['pd.DataFrame']:
     """Get the Pipe data from the remote Pipe."""
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn, error
-    from meerschaum.config.static import _static_config
     from meerschaum.config._patch import apply_patch_to_config
 
-    if 'fetch' not in pipe.parameters:
-        warn(f"Missing 'fetch' parameters for Pipe '{pipe}'.", stack=False)
+    fetch_params = pipe.parameters.get('fetch', {})
+    if not fetch_params:
+        warn(f"Missing 'fetch' parameters for {pipe}.", stack=False)
         return None
 
-    instructions = pipe.parameters['fetch']
+    pipe_meta = fetch_params.get('pipe', {})
+    ### Legacy: check for `connector_keys`, etc. at the root.
+    if not pipe_meta:
+        ck, mk, lk = (
+            fetch_params.get('connector_keys', None),
+            fetch_params.get('metric_key', None),
+            fetch_params.get('location_key', None),
+        )
+        if not ck or not mk:
+            warn(f"Missing `fetch:pipe` keys for {pipe}.", stack=False)
+            return None
 
-    if 'connector_keys' not in instructions:
-        warn(f"Missing connector_keys in fetch parameters for Pipe '{pipe}'", stack=False)
-        return None
-    remote_connector_keys = instructions.get('connector_keys', None)
-    if 'metric_key' not in instructions:
-        warn(f"Missing metric_key in fetch parameters for Pipe '{pipe}'", stack=False)
-        return None
-    remote_metric_key = instructions.get('metric_key', None)
-    remote_location_key = instructions.get('location_key', None)
-    if begin is None:
-        begin = pipe.sync_time
+        pipe_meta.update({
+            'connector': ck,
+            'metric': mk,
+            'location': lk,
+        })
 
-    _params = copy.deepcopy(params) if params is not None else {}
-    _params = apply_patch_to_config(_params, instructions.get('params', {}))
+    pipe_meta['instance'] = self
+    source_pipe = mrsm.Pipe(**pipe_meta)
 
-    from meerschaum import Pipe
-    p = Pipe(
-        remote_connector_keys,
-        remote_metric_key,
-        remote_location_key,
-        mrsm_instance = self
-    )
-    begin = (
-        begin if not (isinstance(begin, str) and begin == '')
-        else pipe.get_sync_time(debug=debug)
-    )
+    _params = copy.deepcopy(params) if params is not None else {}
+    _params = apply_patch_to_config(_params, fetch_params.get('params', {}))
+    select_columns = fetch_params.get('select_columns', [])
+    omit_columns = fetch_params.get('omit_columns', [])
 
-    return p.get_data(
-        begin=begin, end=end,
-        params=_params,
-        debug=debug
+    return source_pipe.get_data(
+        select_columns = select_columns,
+        omit_columns = omit_columns,
+        begin = begin,
+        end = end,
+        params = _params,
+        debug = debug,
+        as_iterator = True,
     )

meerschaum/connectors/api/_pipes.py

@@ -7,6 +7,9 @@ Register or fetch Pipes from the API
 """
 
 from __future__ import annotations
+import time
+import json
+from io import StringIO
 from datetime import datetime
 from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn, error
@@ -123,7 +126,6 @@ def fetch_pipes_keys(
     A list of tuples containing pipes' keys.
     """
     from meerschaum.config.static import STATIC_CONFIG
-    import json
     if connector_keys is None:
         connector_keys = []
     if metric_keys is None:
@@ -162,14 +164,11 @@ def sync_pipe(
         debug: bool = False,
         **kw: Any
     ) -> SuccessTuple:
-    """Append a pandas DataFrame to a Pipe.
-    If Pipe does not exist, it is registered with supplied metadata.
-    """
+    """Sync a DataFrame into a Pipe."""
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.misc import json_serialize_datetime
     from meerschaum.config import get_config
     from meerschaum.utils.packages import attempt_import
-    import json, time
     begin = time.time()
     more_itertools = attempt_import('more_itertools')
     if df is None:
@@ -185,14 +184,14 @@ def sync_pipe(
 
     df = json.loads(df) if isinstance(df, str) else df
 
-    ### TODO Make separate chunksize for API?
-    _chunksize : Optional[int] = (1 if chunksize is None else (
+    _chunksize: Optional[int] = (1 if chunksize is None else (
         get_config('system', 'connectors', 'sql', 'chunksize') if chunksize == -1
         else chunksize
     ))
-    keys : list = list(df.keys())
+    keys: list = list(df.keys())
     chunks = []
     if hasattr(df, 'index'):
+        df = df.reset_index(drop=True)
         rowcount = len(df)
         chunks = [df.iloc[i] for i in more_itertools.chunked(df.index, _chunksize)]
     elif isinstance(df, dict):
@@ -310,7 +309,6 @@ def get_pipe_data(
         **kw: Any
     ) -> Union[pandas.DataFrame, None]:
     """Fetch data from the API."""
-    import json
     r_url = pipe_r_url(pipe)
     chunks_list = []
     while True:
@@ -340,7 +338,7 @@ def get_pipe_data(
     from meerschaum.utils.dataframe import parse_df_datetimes
     pd = import_pandas()
     try:
-        df = pd.read_json(response.text)
+        df = pd.read_json(StringIO(response.text))
     except Exception as e:
         warn(f"Failed to parse response for {pipe}:\n{e}")
         return None
@@ -367,7 +365,6 @@ def get_backtrack_data(
         **kw: Any,
     ) -> pandas.DataFrame:
     """Get a Pipe's backtrack data from the API."""
-    import json
     r_url = pipe_r_url(pipe)
     try:
         response = self.get(
@@ -389,12 +386,12 @@ def get_backtrack_data(
         dprint(response.text)
     pd = import_pandas()
     try:
-        df = pd.read_json(response.text)
+        df = pd.read_json(StringIO(response.text))
     except Exception as e:
         warn(f"Failed to read response into a dataframe:\n{e}")
         return None
 
-    df = parse_df_datetimes(pd.read_json(response.text), debug=debug)
+    df = parse_df_datetimes(pd.read_json(StringIO(response.text)), debug=debug)
     return df
 
 def get_pipe_id(
@@ -438,7 +435,6 @@ def get_pipe_attributes(
     """
     r_url = pipe_r_url(pipe)
     response = self.get(r_url + '/attributes', debug=debug)
-    import json
     try:
         return json.loads(response.text)
     except Exception as e:
@@ -474,7 +470,6 @@ def get_sync_time(
     """
     from meerschaum.utils.misc import is_int
     from meerschaum.utils.warnings import warn
-    import datetime, json
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/sync_time',
@@ -545,7 +540,6 @@ def create_metadata(
     """
     from meerschaum.utils.debug import dprint
     from meerschaum.config.static import STATIC_CONFIG
-    import json
     r_url = STATIC_CONFIG['api']['endpoints']['metadata']
     response = self.post(r_url, debug=debug)
     if debug:
@@ -590,7 +584,6 @@ def get_pipe_rowcount(
     The number of rows in the pipe's table, bound the given parameters.
     If the table does not exist, return 0.
     """
-    import json
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + "/rowcount",

meerschaum/connectors/sql/_fetch.py

@@ -148,7 +148,7 @@ def get_pipe_metadef(
         dt_name = sql_item_name(_dt, self.flavor)
         is_guess = False
 
-    if begin is not None or end is not None:
+    if begin not in (None, '') or end is not None:
         if is_guess:
             if _dt is None:
                 warn(
@@ -168,20 +168,38 @@ def get_pipe_metadef(
     if 'order by' in definition.lower() and 'over' not in definition.lower():
         error("Cannot fetch with an ORDER clause in the definition")
 
+    apply_backtrack = begin == ''
     begin = (
-        begin if not (isinstance(begin, str) and begin == '')
-        else pipe.get_sync_time(debug=debug)
+        pipe.get_sync_time(debug=debug)
+        if begin == ''
+        else begin
     )
-        
+
+    if begin and end and begin >= end:
+        begin = None
+    
     da = None
     if dt_name:
-        ### default: do not backtrack
-        begin_da = dateadd_str(
-            flavor=self.flavor, datepart='minute', number=(-1 * btm), begin=begin,
-        ) if begin else None
-        end_da = dateadd_str(
-            flavor=self.flavor, datepart='minute', number=1, begin=end,
-        ) if end else None
+        begin_da = (
+            dateadd_str(
+                flavor = self.flavor,
+                datepart = 'minute',
+                number = ((-1 * btm) if apply_backtrack else 0), 
+                begin = begin,
+            )
+            if begin
+            else None
+        )
+        end_da = (
+            dateadd_str(
+                flavor = self.flavor,
+                datepart = 'minute',
+                number = 0,
+                begin = end,
+            )
+            if end
+            else None
+        )
 
     meta_def = (
         _simple_fetch_query(pipe) if (

meerschaum/connectors/sql/_pipes.py

@@ -1438,12 +1438,11 @@ def sync_pipe_inplace(
     drop_backtrack_query = f"DROP TABLE {backtrack_table_name}"
     if table_exists(backtrack_table_raw, self, debug=debug):
         backtrack_queries.append(drop_backtrack_query)
-    btm = max(self.get_pipe_backtrack_minutes(pipe), 1)
     backtrack_def = self.get_pipe_data_query(
         pipe,
         begin = begin,
         end = end,
-        begin_add_minutes = (-1 * btm),
+        begin_add_minutes = 0,
         end_add_minutes = 1,
         params = params,
         debug = debug,

meerschaum/core/Pipe/__init__.py

@@ -81,7 +81,10 @@ class Pipe:
     ```
     """
 
-    from ._fetch import fetch
+    from ._fetch import (
+        fetch,
+        get_backtrack_interval,
+    )
     from ._data import (
         get_data,
         get_backtrack_data,
@@ -279,15 +282,26 @@ class Pipe:
 
     @property
     def meta(self):
-        """Simulate the MetaPipe model without importing FastAPI."""
-        if '_meta' not in self.__dict__:
-            self._meta = {
-                'connector_keys' : self.connector_keys,
-                'metric_key'     : self.metric_key,
-                'location_key'   : self.location_key,
-                'instance'       : self.instance_keys,
-            }
-        return self._meta
+        """
+        Return the four keys needed to reconstruct this pipe.
+        """
+        return {
+            'connector_keys': self.connector_keys,
+            'metric_key'    : self.metric_key,
+            'location_key'  : self.location_key,
+            'instance'      : self.instance_keys,
+        }
+
+
+    def keys(self) -> List[str]:
+        """
+        Return the ordered keys for this pipe.
+        """
+        return {
+            key: val
+            for key, val in self.meta.items()
+            if key != 'instance'
+        }
 
 
     @property
@@ -436,3 +450,10 @@ class Pipe:
         metric_key = _state.pop('metric_key')
         location_key = _state.pop('location_key')
         self.__init__(connector_keys, metric_key, location_key, **_state)
+
+
+    def __getitem__(self, *args, **kwargs) -> Any:
+        """
+        Index the pipe's attributes.
+        """
+        return self.attributes.__getitem__(*args, **kwargs)

meerschaum/core/Pipe/_data.py

@@ -325,7 +325,7 @@ def _get_data_as_iterator(
 def get_backtrack_data(
         self,
         backtrack_minutes: int = 0,
-        begin: Optional[datetime] = None,
+        begin: Union[datetime, int, None] = None,
         params: Optional[Dict[str, Any]] = None,
         fresh: bool = False,
         debug: bool = False,
@@ -338,7 +338,7 @@ def get_backtrack_data(
     ----------
     backtrack_minutes: int, default 0
         How many minutes from `begin` to select from.
-        Defaults to 0. This may return a few rows due to a rounding quirk.
+        If 0 (default), use `pipe.parameters['fetch']['backtrack_minutes']`.
 
     begin: Optional[datetime], default None
         The starting point to search for data.
@@ -370,7 +370,6 @@ def get_backtrack_data(
     -------
     A `pd.DataFrame` for the pipe's data corresponding to the provided parameters. Backtrack data
     is a convenient way to get a pipe's data "backtracked" from the most recent datetime.
-
     """
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
@@ -379,6 +378,14 @@ def get_backtrack_data(
     if not self.exists(debug=debug):
         return None
 
+    backtrack_interval = self.get_backtrack_interval(debug=debug)
+    if backtrack_minutes == 0:
+        backtrack_minutes = (
+            (backtrack_interval.total_seconds() * 60)
+            if isinstance(backtrack_interval, timedelta)
+            else backtrack_interval
+        )
+
     if self.cache_pipe is not None:
         if not fresh:
             _sync_cache_tuple = self.cache_pipe.sync(begin=begin, params=params, debug=debug, **kw)
@@ -438,7 +445,7 @@ def get_rowcount(
         params: Optional[Dict[str, Any]] = None,
         remote: bool = False,
         debug: bool = False
-    ) -> Union[int, None]:
+    ) -> int:
     """
     Get a Pipe's instance or remote rowcount.
 
@@ -460,8 +467,7 @@ def get_rowcount(
     Returns
     -------
     An `int` of the number of rows in the pipe corresponding to the provided parameters.
-    `None` is returned if the pipe does not exist.
-
+    Returned 0 if the pipe does not exist.
     """
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.venv import Venv
@@ -470,7 +476,7 @@ def get_rowcount(
     connector = self.instance_connector if not remote else self.connector
     try:
         with Venv(get_connector_plugin(connector)):
-            return connector.get_pipe_rowcount(
+            rowcount = connector.get_pipe_rowcount(
                 self,
                 begin = begin,
                 end = end,
@@ -478,12 +484,15 @@ def get_rowcount(
                 remote = remote,
                 debug = debug,
             )
+            if rowcount is None:
+                return 0
+            return rowcount
     except AttributeError as e:
         warn(e)
         if remote:
-            return None
+            return 0
     warn(f"Failed to get a rowcount for {self}.")
-    return None
+    return 0
 
 
 def get_chunk_interval(
@@ -505,8 +514,8 @@ def get_chunk_interval(
     -------
     The chunk interval (`timedelta` or `int`) to use with this pipe's `datetime` axis.
     """
-    default_chunk_minutes = get_config('pipes', 'parameters', 'chunk_minutes')
-    configured_chunk_minutes = self.parameters.get('chunk_minutes', None)
+    default_chunk_minutes = get_config('pipes', 'parameters', 'verify', 'chunk_minutes')
+    configured_chunk_minutes = self.parameters.get('verify', {}).get('chunk_minutes', None)
     chunk_minutes = (
         (configured_chunk_minutes or default_chunk_minutes)
         if chunk_interval is None
@@ -559,7 +568,8 @@ def get_chunk_bounds(
 
     chunk_interval: Union[timedelta, int, None], default None
         If provided, use this interval for the size of chunk boundaries.
-        The default value for this pipe may be set under `pipe.parameters['chunk_minutes']`.
+        The default value for this pipe may be set
+        under `pipe.parameters['verify']['chunk_minutes']`.
 
     debug: bool, default False
         Verbosity toggle.
@@ -578,7 +588,7 @@ def get_chunk_bounds(
     if begin is None and end is None:
         return [(None, None)]
 
-    ### Set the chunk interval under `pipe.parameters['chunk_minutes']`.
+    ### Set the chunk interval under `pipe.parameters['verify']['chunk_minutes']`.
     chunk_interval = self.get_chunk_interval(chunk_interval, debug=debug)
     
     ### Build a list of tuples containing the chunk boundaries

meerschaum/core/Pipe/_deduplicate.py

@@ -65,6 +65,7 @@ def deduplicate(
     A `SuccessTuple` corresponding to whether all of the chunks were successfully deduplicated.
     """
     from meerschaum.utils.warnings import warn, info
+    from meerschaum.utils.misc import interval_str, items_str
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
     from meerschaum.utils.pool import get_pool
@@ -74,6 +75,7 @@ def deduplicate(
             begin = begin,
             end = end,
             params = params,
+            bounded = bounded,
             debug = debug,
             **kwargs
         )
@@ -90,6 +92,7 @@ def deduplicate(
                 begin = begin,
                 end = end,
                 params = params,
+                bounded = bounded,
                 debug = debug,
                 **kwargs
             )
@@ -104,8 +107,18 @@ def deduplicate(
         begin = (
             bound_time
             if bound_time is not None
-            else self.get_sync_time(debug=debug)
+            else self.get_sync_time(newest=False, debug=debug)
         )
+    if bounded and end is None:
+        end = self.get_sync_time(newest=True, debug=debug)
+
+    if bounded and end is not None:
+        end += (
+            timedelta(minutes=1)
+            if isinstance(end, datetime)
+            else 1
+        )
+
     chunk_bounds = self.get_chunk_bounds(
         bounded = bounded,
         begin = begin,
@@ -115,6 +128,8 @@ def deduplicate(
     )
 
     indices = [col for col in self.columns.values() if col]
+    if not indices:
+        return False, f"Cannot deduplicate without index columns."
     dt_col = self.columns.get('datetime', None)
 
     def process_chunk_bounds(bounds) -> Tuple[
@@ -155,7 +170,20 @@ def deduplicate(
             return bounds, (True, f"{chunk_msg_header}\nChunk is empty, skipping...")
 
         chunk_indices = [ix for ix in indices if ix in full_chunk.columns]
-        full_chunk = full_chunk.drop_duplicates(subset=chunk_indices, keep='last')
+        if not chunk_indices:
+            return bounds, (False, f"None of {items_str(indices)} were present in chunk.")
+        try:
+            full_chunk = full_chunk.drop_duplicates(
+                subset = chunk_indices,
+                keep = 'last'
+            ).reset_index(
+                drop = True,
+            )
+        except Exception as e:
+            return (
+                bounds,
+                (False, f"Failed to deduplicate chunk on {items_str(chunk_indices)}:\n({e})")
+            )
 
         clear_success, clear_msg = self.clear(
             begin = chunk_begin,
@@ -192,19 +220,16 @@ def deduplicate(
             True, (
                 chunk_msg_header + "\n"
                 + chunk_msg_body + ("\n" if chunk_msg_body else '')
-                + f"Chunk succesfully deduplicated to {chunk_rowcount} rows."
+                + f"Deduplicated chunk from {existing_chunk_len} to {chunk_rowcount} rows."
             )
         )
 
-    _start = chunk_bounds[0][(0 if bounded else 1)]
-    _end = chunk_bounds[-1][(0 if not bounded else 1)]
-    message_header = f"{_start} - {_end}"
     info(
         f"Deduplicating {len(chunk_bounds)} chunk"
         + ('s' if len(chunk_bounds) != 1 else '')
         + f" ({'un' if not bounded else ''}bounded)"
-        + f" of size '{chunk_interval}'"
-        + f" from '{_start}' to '{_end}'..."
+        + f" of size '{interval_str(chunk_interval)}'"
+        + f" on {self}."
     )
     bounds_success_tuples = dict(pool.map(process_chunk_bounds, chunk_bounds))
     bounds_successes = {
@@ -223,11 +248,10 @@ def deduplicate(
         return (
             False,
             (
-                message_header + "\n"
-                + f"Failed to deduplicate {len(bounds_failures)} chunk"
+                f"Failed to deduplicate {len(bounds_failures)} chunk"
                 + ('s' if len(bounds_failures) != 1 else '')
-                + ":\n"
-                + "\n".join([msg for _, (_, msg) in bounds_failures.items()])
+                + ".\n"
+                + "\n".join([msg for _, (_, msg) in bounds_failures.items() if msg])
             )
         )
 
@@ -236,11 +260,10 @@ def deduplicate(
         return (
             True,
             (
-                message_header + "\n"
-                + f"Successfully deduplicated {len(bounds_successes)} chunk"
+                f"Successfully deduplicated {len(bounds_successes)} chunk"
                 + ('s' if len(bounds_successes) != 1 else '')
                 + ".\n"
-                + "\n".join([msg for _, (_, msg) in bounds_successes.items()])
+                + "\n".join([msg for _, (_, msg) in bounds_successes.items() if msg])
             ).rstrip('\n')
         )
 
@@ -262,21 +285,19 @@ def deduplicate(
         return (
             True,
             (
-                message_header + "\n"
-                + f"Successfully deduplicated {len(bounds_successes)} chunk"
+                f"Successfully deduplicated {len(bounds_successes)} chunk"
                 + ('s' if len(bounds_successes) != 1 else '')
-                + f" ({len(retry_bounds_successes)} retried):\n"
-                + "\n".join([msg for _, (_, msg) in bounds_successes.items()])
+                + f"({len(retry_bounds_successes)} retried):\n"
+                + "\n".join([msg for _, (_, msg) in bounds_successes.items() if msg])
             ).rstrip('\n')
         )
 
     return (
         False,
         (
-            message_header + "\n"
-            + f"Failed to deduplicate {len(bounds_failures)} chunk"
+            f"Failed to deduplicate {len(bounds_failures)} chunk"
             + ('s' if len(retry_bounds_failures) != 1 else '')
-            + ":\n"
-            + "\n".join([msg for _, (_, msg) in retry_bounds_failures.items()])
+            + ".\n"
+            + "\n".join([msg for _, (_, msg) in retry_bounds_failures.items() if msg])
         ).rstrip('\n')
     )

meerschaum/core/Pipe/_dtypes.py

@@ -7,6 +7,7 @@ Enforce data types for a pipe's underlying table.
 """
 
 from __future__ import annotations
+from io import StringIO
 from meerschaum.utils.typing import Dict, Any, Optional
 
 def enforce_dtypes(
@@ -38,7 +39,7 @@ def enforce_dtypes(
     try:
         if isinstance(df, str):
             df = parse_df_datetimes(
-                pd.read_json(df),
+                pd.read_json(StringIO(df)),
                 ignore_cols = [
                     col
                     for col, dtype in pipe_dtypes.items()

meerschaum/core/Pipe/_fetch.py

@@ -7,7 +7,9 @@ Functions for fetching new data into the Pipe
 """
 
 from __future__ import annotations
+from datetime import timedelta
 from meerschaum.utils.typing import Optional, Any
+from meerschaum.config import get_config
 
 def fetch(
         self,
@@ -98,3 +100,30 @@ def fetch(
         connector_plugin.deactivate_venv(debug=debug)
 
     return df
+
+
+def get_backtrack_interval(self, debug: bool = False) -> Union[timedelta, int]:
+    """
+    Get the chunk interval to use for this pipe.
+
+    Returns
+    -------
+    The backtrack interval (`timedelta` or `int`) to use with this pipe's `datetime` axis.
+    """
+    default_backtrack_minutes = get_config('pipes', 'parameters', 'fetch', 'backtrack_minutes')
+    configured_backtrack_minutes = self.parameters.get('fetch', {}).get('backtrack_minutes', None)
+    backtrack_minutes = (
+        configured_backtrack_minutes
+        if configured_backtrack_minutes is not None
+        else default_backtrack_minutes
+    )
+
+    dt_col = self.columns.get('datetime', None)
+    if dt_col is None:
+        return timedelta(minutes=backtrack_minutes)
+
+    dt_dtype = self.dtypes.get(dt_col, 'datetime64[ns]')
+    if 'datetime' in dt_dtype.lower():
+        return timedelta(minutes=backtrack_minutes)
+
+    return backtrack_minutes