meerschaum 2.8.4__py3-none-any.whl → 2.9.0.dev1__py3-none-any.whl

meerschaum/api/routes/_pipes.py

@@ -27,8 +27,10 @@ from meerschaum.api import (
     debug,
     no_auth,
 )
+from meerschaum.api._chunks import generate_chunks_cursor_token
 from meerschaum.utils.packages import attempt_import
 from meerschaum.utils.dataframe import to_json
+from meerschaum.utils.dtypes import are_dtypes_equal, json_serialize_value
 from meerschaum.utils.misc import (
     is_pipe_registered,
     is_int,
@@ -38,14 +40,16 @@ from meerschaum.connectors.sql.tables import get_tables
 
 fastapi_responses = attempt_import('fastapi.responses', lazy=False)
 StreamingResponse = fastapi_responses.StreamingResponse
-dateutil_parser = attempt_import('dateutil.parser', lazy=False)
 pipes_endpoint = endpoints['pipes']
 pd = attempt_import('pandas', lazy=False)
 
 MAX_RESPONSE_ROW_LIMIT: int = mrsm.get_config('system', 'api', 'data', 'max_response_row_limit')
 
 
-@app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register', tags=['Pipes'])
+@app.post(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register',
+    tags=['Pipes: Attributes'],
+)
 def register_pipe(
     connector_keys: str,
     metric_key: str,
@@ -80,18 +84,23 @@ def register_pipe(
     return results
 
 
-@app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete', tags=['Pipes'])
-def delete_pipe(
+@app.patch(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit',
+    tags=['Pipes: Attributes'],
+)
+def edit_pipe(
     connector_keys: str,
     metric_key: str,
     location_key: str,
+    parameters: dict,
     instance_keys: Optional[str] = None,
+    patch: bool = False,
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
 ):
     """
-    Delete a Pipe (without dropping its table).
+    Edit an existing pipe's parameters.
     """
     allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
     if not allow_actions:
@@ -107,54 +116,27 @@ def delete_pipe(
         raise fastapi.HTTPException(
             status_code=409, detail=f"{pipe} is not registered."
         )
-    results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
-    pipes(instance_keys, refresh=True)
-    return results
-
-
-@app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop', tags=['Pipes'])
-def drop_pipe(
-    connector_keys: str,
-    metric_key: str,
-    location_key: str,
-    instance_keys: Optional[str] = None,
-    curr_user = (
-        fastapi.Depends(manager) if not no_auth else None
-    ),
-):
-    """
-    Drop a pipes' underlying target table.
-    """
-    allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
-    if not allow_actions:
-        return False, (
-            "The administrator for this server has not allowed actions.\n\n"
-            "Please contact the system administrator, or if you are running this server, "
-            "open the configuration file with `edit config system` and search for 'permissions'."
-            " Under the keys `api:permissions:actions`, " +
-            "you can toggle non-admin actions."
-        )
-    pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
-    results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
+    pipe.parameters = parameters
+    results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
     pipes(instance_keys, refresh=True)
     return results
 
 
-
-@app.patch(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit', tags=['Pipes'])
-def edit_pipe(
+@app.delete(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete',
+    tags=['Pipes: Attributes'],
+)
+def delete_pipe(
     connector_keys: str,
     metric_key: str,
     location_key: str,
-    parameters: dict,
     instance_keys: Optional[str] = None,
-    patch: bool = False,
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
 ):
     """
-    Edit an existing pipe's parameters.
+    Delete a Pipe (without dropping its table).
     """
     allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
     if not allow_actions:
@@ -170,13 +152,12 @@ def edit_pipe(
         raise fastapi.HTTPException(
             status_code=409, detail=f"{pipe} is not registered."
         )
-    pipe.parameters = parameters
-    results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
+    results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
     pipes(instance_keys, refresh=True)
     return results
 
 
-@app.get(pipes_endpoint + '/keys', tags=['Pipes'])
+@app.get(pipes_endpoint + '/keys', tags=['Pipes: Attributes'])
 async def fetch_pipes_keys(
     connector_keys: str = "[]",
     metric_keys: str = "[]",
@@ -201,7 +182,7 @@ async def fetch_pipes_keys(
     return keys
 
 
-@app.get(pipes_endpoint, tags=['Pipes'])
+@app.get(pipes_endpoint, tags=['Pipes: Attributes'])
 async def get_pipes(
     connector_keys: str = "",
     metric_keys: str = "",
@@ -225,7 +206,7 @@ async def get_pipes(
     return replace_pipes_in_dict(_get_pipes(**kw), lambda p: p.attributes)
 
 
-@app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes'])
+@app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes: Attributes'])
 async def get_pipes_by_connector(
     connector_keys: str,
     instance_keys: Optional[str] = None,
@@ -243,7 +224,7 @@ async def get_pipes_by_connector(
     return replace_pipes_in_dict(pipes(instance_keys)[connector_keys], lambda p: p.attributes)
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes'])
+@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes: Attributes'])
 async def get_pipes_by_connector_and_metric(
     connector_keys: str,
     metric_key: str,
@@ -271,7 +252,10 @@ async def get_pipes_by_connector_and_metric(
     )
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}',
+    tags=['Pipes: Attributes'],
+)
 async def get_pipe_by_connector_and_metric_and_location(
     connector_keys: str,
     metric_key: str,
@@ -302,7 +286,10 @@ async def get_pipe_by_connector_and_metric_and_location(
     return pipes(instance_keys)[connector_keys][metric_key][location_key].attributes
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time',
+    tags=['Pipes: Data'],
+)
 def get_sync_time(
     connector_keys: str,
     metric_key: str,
@@ -333,7 +320,10 @@ def get_sync_time(
     return sync_time
 
 
-@app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
+@app.post(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
+    tags=['Pipes: Data'],
+)
 def sync_pipe(
     connector_keys: str,
     metric_key: str,
@@ -377,7 +367,10 @@ def sync_pipe(
     return list((success, msg))
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
+    tags=['Pipes: Data'],
+)
 def get_pipe_data(
     connector_keys: str,
     metric_key: str,
@@ -389,6 +382,9 @@ def get_pipe_data(
     end: Union[str, int, None] = None,
     params: Optional[str] = None,
     limit: int = MAX_RESPONSE_ROW_LIMIT,
+    order: str = 'asc', 
+    as_chunks: bool = False,
+    chunk_interval: Optional[int] = None,
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
@@ -398,11 +394,16 @@ def get_pipe_data(
     See [`Pipe.get_data()`](https://docs.meerschaum.io/meerschaum.html#Pipe.get_data).
 
     Note that `select_columns`, `omit_columns`, and `params` are JSON-encoded strings.
+
+    Parameters
+    ----------
+    instance_keys: Optional[str], default None
+        The connector key to the instance on which the pipe is registered.
+        Defaults to the configured value for `meerschaum:api_instance`.
+
+    as_chunks: bool, default False
+        If `True`, return a chunk token to be consumed by the `/chunks` endpoint.
     """
-    if is_int(begin):
-        begin = int(begin)
-    if is_int(end):
-        end = int(end)
     if limit > MAX_RESPONSE_ROW_LIMIT:
         raise fastapi.HTTPException(
             status_code=413,
@@ -455,18 +456,38 @@ def get_pipe_data(
         )
 
     pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    begin, end = pipe.parse_date_bounds(begin, end)
     if not is_pipe_registered(pipe, pipes(instance_keys, refresh=True)):
         raise fastapi.HTTPException(
             status_code=409,
             detail="Pipe must be registered with the datetime column specified."
         )
 
-    if pipe.target in ('users', 'plugins', 'pipes'):
+    if pipe.target in ('mrsm_users', 'mrsm_plugins', 'mrsm_pipes'):
         raise fastapi.HTTPException(
             status_code=409,
             detail=f"Cannot retrieve data from protected table '{pipe.target}'.",
         )
 
+    if as_chunks:
+        chunks_cursor_token = generate_chunks_cursor_token(
+            pipe,
+            select_columns=_select_columns,
+            omit_columns=_omit_columns,
+            begin=begin,
+            end=end,
+            params=_params,
+            limit=limit,
+            order=order,
+            debug=debug,
+        )
+        return fastapi.Response(
+            json.dumps({
+                'chunks_cursor': chunks_cursor,
+            }),
+            media_type='application/json',
+        )
+
     df = pipe.get_data(
         select_columns=_select_columns,
         omit_columns=_omit_columns,
@@ -474,6 +495,7 @@ def get_pipe_data(
         end=end,
         params=_params,
         limit=min(limit, MAX_RESPONSE_ROW_LIMIT),
+        order=order,
         debug=debug,
     )
     if df is None:
@@ -489,7 +511,100 @@ def get_pipe_data(
     )
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk_bounds',
+    tags=['Pipes: Data'],
+)
+def get_pipe_chunk_bounds(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    instance_keys: Optional[str] = None,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    bounded: bool = True,
+    chunk_interval_minutes: Union[int, None] = None,
+) -> List[List[Union[str, int, None]]]:
+    """
+    Return a list of request boundaries between `begin` and `end` (or the pipe's sync times).
+    Optionally specify the interval between chunk bounds
+    (defaults to the pipe's configured chunk interval).
+    """
+    pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    begin, end = pipe.parse_date_bounds(begin, end)
+    dt_col = pipe.columns.get('datetime', None)
+    dt_typ = pipe.dtypes.get(dt_col, 'datetime')
+    chunk_interval = None if chunk_interval_minutes is None else (
+        chunk_interval_minutes
+        if are_dtypes_equal(dt_typ, 'int')
+        else timedelta(minutes=chunk_interval_minutes)
+    )
+
+    chunk_bounds = pipe.get_chunk_bounds(
+        begin=begin,
+        end=end,
+        bounded=bounded,
+        chunk_interval=chunk_interval,
+        debug=debug,
+    )
+
+    return fastapi.Response(
+        json.dumps(chunk_bounds, default=json_serialize_value),
+        media_type='application/json',
+    )
+
+
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk/{chunk_token}',
+    tags=['Pipes: Data'],
+)
+def get_pipe_chunk(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    chunk_token: str
+) -> Dict[str, Any]:
+    """
+    Consume a chunk token, returning the dataframe.
+    """
+
+
+@app.delete(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop',
+    tags=['Pipes: Data'],
+)
+def drop_pipe(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    instance_keys: Optional[str] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
+    """
+    Drop a pipes' underlying target table.
+    """
+    allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
+    if not allow_actions:
+        return False, (
+            "The administrator for this server has not allowed actions.\n\n"
+            "Please contact the system administrator, or if you are running this server, "
+            "open the configuration file with `edit config system` and search for 'permissions'."
+            " Under the keys `api:permissions:actions`, " +
+            "you can toggle non-admin actions."
+        )
+    pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
+    pipes(instance_keys, refresh=True)
+    return results
+
+
+
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv',
+    tags=['Pipes: Data'],
+)
 def get_pipe_csv(
     connector_keys: str,
     metric_key: str,
@@ -505,18 +620,6 @@ def get_pipe_csv(
     """
     Get a pipe's data as a CSV file. Optionally set query boundaries.
     """
-    if begin is not None:
-        begin = (
-            int(begin)
-            if is_int(begin)
-            else dateutil_parser.parse(begin)
-        )
-    if end is not None:
-        end = (
-            int(end)
-            if is_int(end)
-            else dateutil_parser.parse(end)
-        )
 
     _params = {}
     if params == 'null':
@@ -540,6 +643,7 @@ def get_pipe_csv(
             detail="Pipe must be registered."
         )
 
+    begin, end = pipe.parse_date_bounds(begin, end)
     dt_col = pipe.columns.get('datetime', None)
     if dt_col:
         if begin is None:
@@ -567,7 +671,10 @@ def get_pipe_csv(
     return response
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id',
+    tags=['Pipes: Attributes'],
+)
 def get_pipe_id(
     connector_keys: str,
     metric_key: str,
@@ -588,7 +695,7 @@ def get_pipe_id(
 
 @app.get(
     pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/attributes',
-    tags=['Pipes']
+    tags=['Pipes: Attributes'],
 )
 def get_pipe_attributes(
     connector_keys: str,
@@ -609,7 +716,10 @@ def get_pipe_attributes(
     ).attributes
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists',
+    tags=['Pipes: Data'],
+)
 def get_pipe_exists(
     connector_keys: str,
     metric_key: str,
@@ -623,7 +733,7 @@ def get_pipe_exists(
     return get_pipe(connector_keys, metric_key, location_key, instance_keys).exists(debug=debug)
 
 
-@app.post(endpoints['metadata'], tags=['Pipes'])
+@app.post(endpoints['metadata'], tags=['Misc'])
 def create_metadata(
     instance_keys: Optional[str] = None,
     curr_user = (
@@ -641,7 +751,10 @@ def create_metadata(
     return True
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount',
+    tags=['Pipes: Data'],
+)
 def get_pipe_rowcount(
     connector_keys: str,
     metric_key: str,
@@ -692,7 +805,7 @@ def get_pipe_rowcount(
 
 @app.get(
     pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/types',
-    tags=['Pipes']
+    tags=['Pipes: Data'],
 )
 def get_pipe_columns_types(
     connector_keys: str,
@@ -720,7 +833,7 @@ def get_pipe_columns_types(
 
 @app.get(
     pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/indices',
-    tags=['Pipes']
+    tags=['Pipes: Data'],
 )
 def get_pipe_columns_indices(
     connector_keys: str,
@@ -774,7 +887,7 @@ def get_pipe_columns_indices(
 
 @app.get(
     pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/indices/names',
-    tags=['Pipes']
+    tags=['Pipes: Data']
 )
 def get_pipe_index_names(
     connector_keys: str,
@@ -784,7 +897,7 @@ def get_pipe_index_names(
     curr_user=(
         fastapi.Depends(manager) if not no_auth else None
     ),
-) -> Dict[str, List[Dict[str, str]]]:
+) -> Dict[str, Union[str, Dict[str, str], List[Dict[str, str]]]]:
     """
     Return a dictionary of index keys and index names.
 
@@ -795,4 +908,4 @@ def get_pipe_index_names(
         metric_key,
         location_key,
         instance_keys,
-    ).get_indices(debug=debug)
+    ).get_indices()

meerschaum/config/_default.py

@@ -113,6 +113,9 @@ default_system_config = {
         },
         'data': {
             'max_response_row_limit': 100_000,
+            'chunks': {
+                'ttl_seconds': 1800,
+            },
         },
         'endpoints': {
             'docs_in_production': True,

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.8.4"
+__version__ = "2.9.0.dev1"

meerschaum/connectors/api/_APIConnector.py

@@ -6,8 +6,10 @@
 Interact with Meerschaum APIs. May be chained together (see 'meerschaum:api_instance' in your config.yaml).
 """
 
+from __future__ import annotations
+
 from datetime import datetime, timedelta, timezone
-from meerschaum.utils.typing import Optional, List
+from meerschaum.utils.typing import Optional, List, Union
 from meerschaum.connectors import Connector
 from meerschaum.utils.warnings import warn, error
 from meerschaum.utils.packages import attempt_import
@@ -43,6 +45,7 @@ class APIConnector(Connector):
     )
     from ._misc import get_mrsm_version, get_chaining_status
     from ._pipes import (
+        get_pipe_instance_keys,
         register_pipe,
         fetch_pipes_keys,
         edit_pipe,
@@ -143,6 +146,7 @@ class APIConnector(Connector):
         self._token = None
         self._expires = None
         self._session = None
+        self._instance_keys = self.__dict__.get('instance_keys', None)
 
 
     @property
@@ -195,3 +199,10 @@ class APIConnector(Connector):
                 warn(msg, stack=False)
                 self._emitted_warning = True
         return self._token
+
+    @property
+    def instance_keys(self) -> Union[str, None]:
+        """
+        Return the instance keys to be sent alongside pipe requests.
+        """
+        return self._instance_keys

meerschaum/connectors/api/_pipes.py

@@ -31,6 +31,14 @@ def pipe_r_url(
     )
 
 
+def get_pipe_instance_keys(self, pipe: mrsm.Pipe) -> Union[str, None]:
+    """
+    Return the configured instance keys for a pipe if set,
+    else fall back to the default `instance_keys` for this `APIConnector`.
+    """
+    return pipe.parameters.get('instance_keys', self.instance_keys)
+
+
 def register_pipe(
     self,
     pipe: mrsm.Pipe,
@@ -40,13 +48,12 @@ def register_pipe(
     Returns a tuple of (success_bool, response_dict).
     """
     from meerschaum.utils.debug import dprint
-    ### NOTE: if `parameters` is supplied in the Pipe constructor,
-    ###       then `pipe.parameters` will exist and not be fetched from the database.
     r_url = pipe_r_url(pipe)
     response = self.post(
         r_url + '/register',
-        json = pipe.parameters,
-        debug = debug,
+        json=pipe._attributes.get('parameters', {}),
+        params={'instance_keys': self.get_pipe_instance_keys(pipe)},
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -79,9 +86,9 @@ def edit_pipe(
     r_url = pipe_r_url(pipe)
     response = self.patch(
         r_url + '/edit',
-        params = {'patch': patch,},
-        json = pipe.parameters,
-        debug = debug,
+        params={'patch': patch, 'instance_keys': self.get_pipe_instance_keys(pipe)},
+        json=pipe.parameters,
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -149,12 +156,13 @@ def fetch_pipes_keys(
     try:
         j = self.get(
             r_url,
-            params = {
+            params={
                 'connector_keys': json.dumps(connector_keys),
                 'metric_keys': json.dumps(metric_keys),
                 'location_keys': json.dumps(location_keys),
                 'tags': json.dumps(tags),
                 'params': json.dumps(params),
+                'instance_keys': self.instance_keys,
             },
             debug=debug
         ).json()
@@ -250,8 +258,10 @@ def sync_pipe(
         chunks = (df[i] for i in more_itertools.chunked(df, _chunksize))
 
     ### Send columns in case the user has defined them locally.
+    request_params = kw.copy()
     if pipe.columns:
-        kw['columns'] = json.dumps(pipe.columns)
+        request_params['columns'] = json.dumps(pipe.columns)
+    request_params['instance_keys'] = self.get_pipe_instance_keys(pipe)
     r_url = pipe_r_url(pipe) + '/data'
 
     rowcount = 0
@@ -268,10 +278,9 @@ def sync_pipe(
         try:
             response = self.post(
                 r_url,
-                ### handles check_existing
-                params = kw,
-                data = json_str,
-                debug = debug
+                params=request_params,
+                data=json_str,
+                debug=debug,
             )
         except Exception as e:
             msg = f"Failed to post a chunk to {pipe}:\n{e}"
@@ -323,7 +332,8 @@ def delete_pipe(
     r_url = pipe_r_url(pipe)
     response = self.delete(
         r_url + '/delete',
-        debug = debug,
+        params={'instance_keys': self.get_pipe_instance_keys(pipe)},
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -361,7 +371,9 @@ def get_pipe_data(
                     'omit_columns': json.dumps(omit_columns),
                     'begin': begin,
                     'end': end,
-                    'params': json.dumps(params, default=str)
+                    'params': json.dumps(params, default=str),
+                    'instance': self.get_pipe_instance_keys(pipe),
+                    'as_chunks': as_chunks,
                 },
                 debug=debug
             )