meerschaum 2.8.3__py3-none-any.whl → 2.9.0__py3-none-any.whl

meerschaum/api/routes/_pipes.py

@@ -27,8 +27,10 @@ from meerschaum.api import (
     debug,
     no_auth,
 )
+from meerschaum.api._chunks import generate_chunks_cursor_token
 from meerschaum.utils.packages import attempt_import
 from meerschaum.utils.dataframe import to_json
+from meerschaum.utils.dtypes import are_dtypes_equal, json_serialize_value
 from meerschaum.utils.misc import (
     is_pipe_registered,
     is_int,
@@ -38,14 +40,16 @@ from meerschaum.connectors.sql.tables import get_tables
 
 fastapi_responses = attempt_import('fastapi.responses', lazy=False)
 StreamingResponse = fastapi_responses.StreamingResponse
-dateutil_parser = attempt_import('dateutil.parser', lazy=False)
 pipes_endpoint = endpoints['pipes']
 pd = attempt_import('pandas', lazy=False)
 
 MAX_RESPONSE_ROW_LIMIT: int = mrsm.get_config('system', 'api', 'data', 'max_response_row_limit')
 
 
-@app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register', tags=['Pipes'])
+@app.post(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/register',
+    tags=['Pipes: Attributes'],
+)
 def register_pipe(
     connector_keys: str,
     metric_key: str,
@@ -80,18 +84,23 @@ def register_pipe(
     return results
 
 
-@app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete', tags=['Pipes'])
-def delete_pipe(
+@app.patch(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit',
+    tags=['Pipes: Attributes'],
+)
+def edit_pipe(
     connector_keys: str,
     metric_key: str,
     location_key: str,
+    parameters: dict,
     instance_keys: Optional[str] = None,
+    patch: bool = False,
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
 ):
     """
-    Delete a Pipe (without dropping its table).
+    Edit an existing pipe's parameters.
     """
     allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
     if not allow_actions:
@@ -107,54 +116,27 @@ def delete_pipe(
         raise fastapi.HTTPException(
             status_code=409, detail=f"{pipe} is not registered."
         )
-    results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
-    pipes(instance_keys, refresh=True)
-    return results
-
-
-@app.delete(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop', tags=['Pipes'])
-def drop_pipe(
-    connector_keys: str,
-    metric_key: str,
-    location_key: str,
-    instance_keys: Optional[str] = None,
-    curr_user = (
-        fastapi.Depends(manager) if not no_auth else None
-    ),
-):
-    """
-    Drop a pipes' underlying target table.
-    """
-    allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
-    if not allow_actions:
-        return False, (
-            "The administrator for this server has not allowed actions.\n\n"
-            "Please contact the system administrator, or if you are running this server, "
-            "open the configuration file with `edit config system` and search for 'permissions'."
-            " Under the keys `api:permissions:actions`, " +
-            "you can toggle non-admin actions."
-        )
-    pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
-    results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
+    pipe.parameters = parameters
+    results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
     pipes(instance_keys, refresh=True)
     return results
 
 
-
-@app.patch(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/edit', tags=['Pipes'])
-def edit_pipe(
+@app.delete(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/delete',
+    tags=['Pipes: Attributes'],
+)
+def delete_pipe(
     connector_keys: str,
     metric_key: str,
     location_key: str,
-    parameters: dict,
     instance_keys: Optional[str] = None,
-    patch: bool = False,
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
 ):
     """
-    Edit an existing pipe's parameters.
+    Delete a Pipe (without dropping its table).
     """
     allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
     if not allow_actions:
@@ -170,13 +152,12 @@ def edit_pipe(
         raise fastapi.HTTPException(
             status_code=409, detail=f"{pipe} is not registered."
         )
-    pipe.parameters = parameters
-    results = get_api_connector(instance_keys).edit_pipe(pipe, patch=patch, debug=debug)
+    results = get_api_connector(instance_keys).delete_pipe(pipe, debug=debug)
     pipes(instance_keys, refresh=True)
     return results
 
 
-@app.get(pipes_endpoint + '/keys', tags=['Pipes'])
+@app.get(pipes_endpoint + '/keys', tags=['Pipes: Attributes'])
 async def fetch_pipes_keys(
     connector_keys: str = "[]",
     metric_keys: str = "[]",
@@ -201,7 +182,7 @@ async def fetch_pipes_keys(
     return keys
 
 
-@app.get(pipes_endpoint, tags=['Pipes'])
+@app.get(pipes_endpoint, tags=['Pipes: Attributes'])
 async def get_pipes(
     connector_keys: str = "",
     metric_keys: str = "",
@@ -225,7 +206,7 @@ async def get_pipes(
     return replace_pipes_in_dict(_get_pipes(**kw), lambda p: p.attributes)
 
 
-@app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes'])
+@app.get(pipes_endpoint + '/{connector_keys}', tags=['Pipes: Attributes'])
 async def get_pipes_by_connector(
     connector_keys: str,
     instance_keys: Optional[str] = None,
@@ -243,7 +224,7 @@ async def get_pipes_by_connector(
     return replace_pipes_in_dict(pipes(instance_keys)[connector_keys], lambda p: p.attributes)
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes'])
+@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}', tags=['Pipes: Attributes'])
 async def get_pipes_by_connector_and_metric(
     connector_keys: str,
     metric_key: str,
@@ -271,8 +252,11 @@ async def get_pipes_by_connector_and_metric(
     )
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}', tags=['Pipes'])
-async def get_pipes_by_connector_and_metric_and_location(
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}',
+    tags=['Pipes: Attributes'],
+)
+async def get_pipe_by_connector_and_metric_and_location(
     connector_keys: str,
     metric_key: str,
     location_key: str,
@@ -299,10 +283,13 @@ async def get_pipes_by_connector_and_metric_and_location(
             detail=f"location_key '{location_key}' not found."
         )
 
-    return str(pipes(instance_keys)[connector_keys][metric_key][location_key])
+    return pipes(instance_keys)[connector_keys][metric_key][location_key].attributes
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/sync_time',
+    tags=['Pipes: Data'],
+)
 def get_sync_time(
     connector_keys: str,
     metric_key: str,
@@ -333,7 +320,10 @@ def get_sync_time(
     return sync_time
 
 
-@app.post(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
+@app.post(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
+    tags=['Pipes: Data'],
+)
 def sync_pipe(
     connector_keys: str,
     metric_key: str,
@@ -377,7 +367,10 @@ def sync_pipe(
     return list((success, msg))
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/data',
+    tags=['Pipes: Data'],
+)
 def get_pipe_data(
     connector_keys: str,
     metric_key: str,
@@ -389,6 +382,11 @@ def get_pipe_data(
     end: Union[str, int, None] = None,
     params: Optional[str] = None,
     limit: int = MAX_RESPONSE_ROW_LIMIT,
+    order: str = 'asc', 
+    date_format: str = 'iso',
+    date_unit: str = 'us',
+    double_precision: int = 15,
+    geometry_format: str = 'wkb_hex',
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
@@ -398,11 +396,28 @@ def get_pipe_data(
     See [`Pipe.get_data()`](https://docs.meerschaum.io/meerschaum.html#Pipe.get_data).
 
     Note that `select_columns`, `omit_columns`, and `params` are JSON-encoded strings.
+
+    Parameters
+    ----------
+    instance_keys: Optional[str], default None
+        The connector key to the instance on which the pipe is registered.
+        Defaults to the configured value for `meerschaum:api_instance`.
+
+    date_format: str, default 'iso'
+        Serialzation format for datetime values.
+        Accepted values are `'iso`' (ISO8601) and `'epoch'` (epoch milliseconds).
+
+    date_unit: str, default 'us'
+        Timestamp precision for serialization. Accepted values are `'s'` (seconds),
+        `'ms'` (milliseconds), `'us'` (microseconds), and `'ns'`.
+
+    double_precision: int, default 15
+        The number of decimal places to use when encoding floating point values (maximum `15`).
+
+    geometry_format: str, default 'wkb_hex'
+        The serialization format for geometry data.
+        Accepted values are `geojson`, `wkb_hex`, and `wkt`.
     """
-    if is_int(begin):
-        begin = int(begin)
-    if is_int(end):
-        end = int(end)
     if limit > MAX_RESPONSE_ROW_LIMIT:
         raise fastapi.HTTPException(
             status_code=413,
@@ -455,13 +470,14 @@ def get_pipe_data(
         )
 
     pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    begin, end = pipe.parse_date_bounds(begin, end)
     if not is_pipe_registered(pipe, pipes(instance_keys, refresh=True)):
         raise fastapi.HTTPException(
             status_code=409,
             detail="Pipe must be registered with the datetime column specified."
         )
 
-    if pipe.target in ('users', 'plugins', 'pipes'):
+    if pipe.target in ('mrsm_users', 'mrsm_plugins', 'mrsm_pipes'):
         raise fastapi.HTTPException(
             status_code=409,
             detail=f"Cannot retrieve data from protected table '{pipe.target}'.",
@@ -474,6 +490,7 @@ def get_pipe_data(
         end=end,
         params=_params,
         limit=min(limit, MAX_RESPONSE_ROW_LIMIT),
+        order=order,
         debug=debug,
     )
     if df is None:
@@ -482,14 +499,152 @@ def get_pipe_data(
             detail="Could not fetch data with the given parameters.",
         )
 
-    json_content = to_json(df)
+    json_content = to_json(
+        df,
+        date_format=date_format,
+        date_unit=date_unit,
+        geometry_format=geometry_format,
+        double_precision=double_precision,
+    )
     return fastapi.Response(
         json_content,
         media_type='application/json',
     )
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk_bounds',
+    tags=['Pipes: Data'],
+)
+def get_pipe_chunk_bounds(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    instance_keys: Optional[str] = None,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    bounded: bool = True,
+    chunk_interval_minutes: Union[int, None] = None,
+) -> List[List[Union[str, int, None]]]:
+    """
+    Return a list of request boundaries between `begin` and `end` (or the pipe's sync times).
+    Optionally specify the interval between chunk bounds
+    (defaults to the pipe's configured chunk interval).
+    """
+    pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    begin, end = pipe.parse_date_bounds(begin, end)
+    dt_col = pipe.columns.get('datetime', None)
+    dt_typ = pipe.dtypes.get(dt_col, 'datetime')
+    chunk_interval = None if chunk_interval_minutes is None else (
+        chunk_interval_minutes
+        if are_dtypes_equal(dt_typ, 'int')
+        else timedelta(minutes=chunk_interval_minutes)
+    )
+
+    chunk_bounds = pipe.get_chunk_bounds(
+        begin=begin,
+        end=end,
+        bounded=bounded,
+        chunk_interval=chunk_interval,
+        debug=debug,
+    )
+
+    return fastapi.Response(
+        json.dumps(chunk_bounds, default=json_serialize_value),
+        media_type='application/json',
+    )
+
+
+@app.delete(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop',
+    tags=['Pipes: Data'],
+)
+def drop_pipe(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    instance_keys: Optional[str] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
+    """
+    Drop a pipe's target table.
+    """
+    allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
+    if not allow_actions:
+        return False, (
+            "The administrator for this server has not allowed actions.\n\n"
+            "Please contact the system administrator, or if you are running this server, "
+            "open the configuration file with `edit config system` and search for 'permissions'."
+            " Under the keys `api:permissions:actions`, " +
+            "you can toggle non-admin actions."
+        )
+    pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
+    pipes(instance_keys, refresh=True)
+    return results
+
+
+@app.delete(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/clear',
+    tags=['Pipes: Data'],
+)
+def clear_pipe(
+    connector_keys: str,
+    metric_key: str,
+    location_key: str,
+    instance_keys: Optional[str] = None,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    params: Optional[str] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
+    """
+    Delete rows from a pipe's target table.
+    """
+    _params = {}
+    if params == 'null':
+        params = None
+    if params is not None:
+        try:
+            _params = json.loads(params)
+        except Exception:
+            _params = None
+    if not isinstance(_params, dict):
+        raise fastapi.HTTPException(
+            status_code=409,
+            detail="Params must be a valid JSON-encoded dictionary.",
+        )
+
+    allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
+    if not allow_actions:
+        return False, (
+            "The administrator for this server has not allowed actions.\n\n"
+            "Please contact the system administrator, or if you are running this server, "
+            "open the configuration file with `edit config system` and search for 'permissions'."
+            " Under the keys `api:permissions:actions`, " +
+            "you can toggle non-admin actions."
+        )
+    pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    begin, end = pipe.parse_date_bounds(begin, end)
+    results = get_api_connector(instance_keys=instance_keys).clear_pipe(
+        pipe,
+        begin=begin,
+        end=end,
+        params=_params,
+        debug=debug,
+    )
+    pipes(instance_keys, refresh=True)
+    return results
+
+
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv',
+    tags=['Pipes: Data'],
+)
 def get_pipe_csv(
     connector_keys: str,
     metric_key: str,
@@ -505,24 +660,11 @@ def get_pipe_csv(
     """
     Get a pipe's data as a CSV file. Optionally set query boundaries.
     """
-    if begin is not None:
-        begin = (
-            int(begin)
-            if is_int(begin)
-            else dateutil_parser.parse(begin)
-        )
-    if end is not None:
-        end = (
-            int(end)
-            if is_int(end)
-            else dateutil_parser.parse(end)
-        )
 
     _params = {}
     if params == 'null':
         params = None
     if params is not None:
-        import json
         try:
             _params = json.loads(params)
         except Exception:
@@ -541,6 +683,7 @@ def get_pipe_csv(
             detail="Pipe must be registered."
         )
 
+    begin, end = pipe.parse_date_bounds(begin, end)
     dt_col = pipe.columns.get('datetime', None)
     if dt_col:
         if begin is None:
@@ -568,7 +711,10 @@ def get_pipe_csv(
     return response
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/id',
+    tags=['Pipes: Attributes'],
+)
 def get_pipe_id(
     connector_keys: str,
     metric_key: str,
@@ -589,7 +735,7 @@ def get_pipe_id(
 
 @app.get(
     pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/attributes',
-    tags=['Pipes']
+    tags=['Pipes: Attributes'],
 )
 def get_pipe_attributes(
     connector_keys: str,
@@ -610,7 +756,10 @@ def get_pipe_attributes(
     ).attributes
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/exists',
+    tags=['Pipes: Data'],
+)
 def get_pipe_exists(
     connector_keys: str,
     metric_key: str,
@@ -624,7 +773,7 @@ def get_pipe_exists(
     return get_pipe(connector_keys, metric_key, location_key, instance_keys).exists(debug=debug)
 
 
-@app.post(endpoints['metadata'], tags=['Pipes'])
+@app.post(endpoints['metadata'], tags=['Misc'])
 def create_metadata(
     instance_keys: Optional[str] = None,
     curr_user = (
@@ -642,7 +791,10 @@ def create_metadata(
     return True
 
 
-@app.get(pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount', tags=['Pipes'])
+@app.get(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/rowcount',
+    tags=['Pipes: Data'],
+)
 def get_pipe_rowcount(
     connector_keys: str,
     metric_key: str,
@@ -693,7 +845,7 @@ def get_pipe_rowcount(
 
 @app.get(
     pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/types',
-    tags=['Pipes']
+    tags=['Pipes: Data'],
 )
 def get_pipe_columns_types(
     connector_keys: str,
@@ -721,7 +873,7 @@ def get_pipe_columns_types(
 
 @app.get(
     pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/columns/indices',
-    tags=['Pipes']
+    tags=['Pipes: Data'],
 )
 def get_pipe_columns_indices(
     connector_keys: str,
@@ -775,7 +927,7 @@ def get_pipe_columns_indices(
 
 @app.get(
     pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/indices/names',
-    tags=['Pipes']
+    tags=['Pipes: Data']
 )
 def get_pipe_index_names(
     connector_keys: str,
@@ -785,7 +937,7 @@ def get_pipe_index_names(
     curr_user=(
         fastapi.Depends(manager) if not no_auth else None
     ),
-) -> Dict[str, List[Dict[str, str]]]:
+) -> Dict[str, Union[str, Dict[str, str], List[Dict[str, str]]]]:
     """
     Return a dictionary of index keys and index names.
 
@@ -796,4 +948,4 @@ def get_pipe_index_names(
         metric_key,
         location_key,
         instance_keys,
-    ).get_indices(debug=debug)
+    ).get_indices()

meerschaum/config/_default.py

@@ -70,6 +70,7 @@ default_system_config = {
         'sql': {
             'bulk_insert': {
                 'postgresql': True,
+                'postgis': True,
                 'citus': True,
                 'timescaledb': True,
                 'mssql': True,
@@ -113,6 +114,9 @@ default_system_config = {
         },
         'data': {
             'max_response_row_limit': 100_000,
+            'chunks': {
+                'ttl_seconds': 1800,
+            },
         },
         'endpoints': {
             'docs_in_production': True,

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.8.3"
+__version__ = "2.9.0"

meerschaum/connectors/__init__.py

@@ -34,6 +34,7 @@ __all__ = (
     "api",
     "sql",
     "valkey",
+    "parse",
 )
 
 ### store connectors partitioned by

meerschaum/connectors/api/_APIConnector.py

@@ -6,8 +6,10 @@
 Interact with Meerschaum APIs. May be chained together (see 'meerschaum:api_instance' in your config.yaml).
 """
 
+from __future__ import annotations
+
 from datetime import datetime, timedelta, timezone
-from meerschaum.utils.typing import Optional, List
+from meerschaum.utils.typing import Optional, List, Union
 from meerschaum.connectors import Connector
 from meerschaum.utils.warnings import warn, error
 from meerschaum.utils.packages import attempt_import
@@ -43,6 +45,7 @@ class APIConnector(Connector):
     )
     from ._misc import get_mrsm_version, get_chaining_status
     from ._pipes import (
+        get_pipe_instance_keys,
         register_pipe,
         fetch_pipes_keys,
         edit_pipe,
@@ -143,6 +146,7 @@ class APIConnector(Connector):
         self._token = None
         self._expires = None
         self._session = None
+        self._instance_keys = self.__dict__.get('instance_keys', None)
 
 
     @property
@@ -195,3 +199,10 @@ class APIConnector(Connector):
                 warn(msg, stack=False)
                 self._emitted_warning = True
         return self._token
+
+    @property
+    def instance_keys(self) -> Union[str, None]:
+        """
+        Return the instance keys to be sent alongside pipe requests.
+        """
+        return self._instance_keys