meerschaum 2.9.0rc3__py3-none-any.whl → 2.9.2__py3-none-any.whl

meerschaum/api/resources/templates/termpage.html

@@ -55,6 +55,8 @@ window.addEventListener(
         if (typeof lk === "string"){
             quote_str = lk.includes(" ") ? "'" : "";
             location_keys_str += " " + quote_str + lk + quote_str;
+        } else if (lk === null) {
+            location_keys_str += " None";
         }
     }
     if (location_keys_str === " -l"){

meerschaum/api/routes/_pipes.py

@@ -387,8 +387,6 @@ def get_pipe_data(
     date_unit: str = 'us',
     double_precision: int = 15,
     geometry_format: str = 'wkb_hex',
-    as_chunks: bool = False,
-    chunk_interval: Optional[int] = None,
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
@@ -414,14 +412,11 @@ def get_pipe_data(
         `'ms'` (milliseconds), `'us'` (microseconds), and `'ns'`.
 
     double_precision: int, default 15
-        The number of decimal places to use when encoding floating point values (maximum 15).
+        The number of decimal places to use when encoding floating point values (maximum `15`).
 
     geometry_format: str, default 'wkb_hex'
         The serialization format for geometry data.
         Accepted values are `geojson`, `wkb_hex`, and `wkt`.
-
-    as_chunks: bool, default False
-        If `True`, return a chunk token to be consumed by the `/chunks` endpoint.
     """
     if limit > MAX_RESPONSE_ROW_LIMIT:
         raise fastapi.HTTPException(
@@ -488,25 +483,6 @@ def get_pipe_data(
             detail=f"Cannot retrieve data from protected table '{pipe.target}'.",
         )
 
-    if as_chunks:
-        chunks_cursor_token = generate_chunks_cursor_token(
-            pipe,
-            select_columns=_select_columns,
-            omit_columns=_omit_columns,
-            begin=begin,
-            end=end,
-            params=_params,
-            limit=limit,
-            order=order,
-            debug=debug,
-        )
-        return fastapi.Response(
-            json.dumps({
-                'chunks_cursor': chunks_cursor_token,
-            }),
-            media_type='application/json',
-        )
-
     df = pipe.get_data(
         select_columns=_select_columns,
         omit_columns=_omit_columns,
@@ -579,37 +555,70 @@ def get_pipe_chunk_bounds(
     )
 
 
-@app.get(
-    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/chunk/{chunk_token}',
+@app.delete(
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop',
     tags=['Pipes: Data'],
 )
-def get_pipe_chunk(
+def drop_pipe(
     connector_keys: str,
     metric_key: str,
     location_key: str,
-    chunk_token: str
-) -> Dict[str, Any]:
+    instance_keys: Optional[str] = None,
+    curr_user = (
+        fastapi.Depends(manager) if not no_auth else None
+    ),
+):
     """
-    Consume a chunk token, returning the dataframe.
+    Drop a pipe's target table.
     """
+    allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
+    if not allow_actions:
+        return False, (
+            "The administrator for this server has not allowed actions.\n\n"
+            "Please contact the system administrator, or if you are running this server, "
+            "open the configuration file with `edit config system` and search for 'permissions'."
+            " Under the keys `api:permissions:actions`, " +
+            "you can toggle non-admin actions."
+        )
+    pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
+    pipes(instance_keys, refresh=True)
+    return results
 
 
 @app.delete(
-    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/drop',
+    pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/clear',
     tags=['Pipes: Data'],
 )
-def drop_pipe(
+def clear_pipe(
     connector_keys: str,
     metric_key: str,
     location_key: str,
     instance_keys: Optional[str] = None,
+    begin: Union[str, int, None] = None,
+    end: Union[str, int, None] = None,
+    params: Optional[str] = None,
     curr_user = (
         fastapi.Depends(manager) if not no_auth else None
     ),
 ):
     """
-    Drop a pipes' underlying target table.
+    Delete rows from a pipe's target table.
     """
+    _params = {}
+    if params == 'null':
+        params = None
+    if params is not None:
+        try:
+            _params = json.loads(params)
+        except Exception:
+            _params = None
+    if not isinstance(_params, dict):
+        raise fastapi.HTTPException(
+            status_code=409,
+            detail="Params must be a valid JSON-encoded dictionary.",
+        )
+
     allow_actions = mrsm.get_config('system', 'api', 'permissions', 'actions', 'non_admin')
     if not allow_actions:
         return False, (
@@ -619,13 +628,19 @@ def drop_pipe(
             " Under the keys `api:permissions:actions`, " +
             "you can toggle non-admin actions."
         )
-    pipe_object = get_pipe(connector_keys, metric_key, location_key, instance_keys)
-    results = get_api_connector(instance_keys=instance_keys).drop_pipe(pipe_object, debug=debug)
+    pipe = get_pipe(connector_keys, metric_key, location_key, instance_keys)
+    begin, end = pipe.parse_date_bounds(begin, end)
+    results = get_api_connector(instance_keys=instance_keys).clear_pipe(
+        pipe,
+        begin=begin,
+        end=end,
+        params=_params,
+        debug=debug,
+    )
     pipes(instance_keys, refresh=True)
     return results
 
 
-
 @app.get(
     pipes_endpoint + '/{connector_keys}/{metric_key}/{location_key}/csv',
     tags=['Pipes: Data'],

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.9.0rc3"
+__version__ = "2.9.2"

meerschaum/connectors/__init__.py

@@ -34,6 +34,7 @@ __all__ = (
     "api",
     "sql",
     "valkey",
+    "parse",
 )
 
 ### store connectors partitioned by

meerschaum/connectors/api/_pipes.py

@@ -414,19 +414,24 @@ def get_pipe_id(
     self,
     pipe: mrsm.Pipe,
     debug: bool = False,
-) -> int:
+) -> Union[int, str, None]:
     """Get a Pipe's ID from the API."""
     from meerschaum.utils.misc import is_int
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/id',
-        debug = debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     if debug:
         dprint(f"Got pipe ID: {response.text}")
     try:
         if is_int(response.text):
             return int(response.text)
+        if response.text and response.text[0] != '{':
+            return response.text
     except Exception as e:
         warn(f"Failed to get the ID for {pipe}:\n{e}")
     return None
@@ -450,7 +455,13 @@ def get_pipe_attributes(
     If the pipe does not exist, return an empty dictionary.
     """
     r_url = pipe_r_url(pipe)
-    response = self.get(r_url + '/attributes', debug=debug)
+    response = self.get(
+        r_url + '/attributes',
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug
+    )
     try:
         return json.loads(response.text)
     except Exception as e:
@@ -489,9 +500,13 @@ def get_sync_time(
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/sync_time',
-        json = params,
-        params = {'newest': newest, 'debug': debug},
-        debug = debug,
+        json=params,
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+            'newest': newest,
+            'debug': debug,
+        },
+        debug=debug,
     )
     if not response:
         warn(f"Failed to get the sync time for {pipe}:\n" + response.text)
@@ -532,7 +547,13 @@ def pipe_exists(
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn
     r_url = pipe_r_url(pipe)
-    response = self.get(r_url + '/exists', debug=debug)
+    response = self.get(
+        r_url + '/exists',
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
+    )
     if not response:
         warn(f"Failed to check if {pipe} exists:\n{response.text}")
         return False
@@ -570,8 +591,8 @@ def create_metadata(
 def get_pipe_rowcount(
     self,
     pipe: mrsm.Pipe,
-    begin: Optional[datetime] = None,
-    end: Optional[datetime] = None,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
     params: Optional[Dict[str, Any]] = None,
     remote: bool = False,
     debug: bool = False,
@@ -583,10 +604,10 @@ def get_pipe_rowcount(
     pipe: 'meerschaum.Pipe':
         The pipe whose row count we are counting.
         
-    begin: Optional[datetime], default None
+    begin: Union[str, datetime, int, None], default None
         If provided, bound the count by this datetime.
 
-    end: Optional[datetime]
+    end: Union[str, datetime, int, None], default None
         If provided, bound the count by this datetime.
 
     params: Optional[Dict[str, Any]], default None
@@ -608,6 +629,7 @@ def get_pipe_rowcount(
             'begin': begin,
             'end': end,
             'remote': remote,
+            'instance': self.get_pipe_instance_keys(pipe),
         },
         debug = debug
     )
@@ -645,7 +667,10 @@ def drop_pipe(
     r_url = pipe_r_url(pipe)
     response = self.delete(
         r_url + '/drop',
-        debug = debug,
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -668,6 +693,9 @@ def drop_pipe(
 def clear_pipe(
     self,
     pipe: mrsm.Pipe,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -683,20 +711,33 @@ def clear_pipe(
     -------
     A success tuple.
     """
-    kw.pop('metric_keys', None)
-    kw.pop('connector_keys', None)
-    kw.pop('location_keys', None)
-    kw.pop('action', None)
-    kw.pop('force', None)
-    return self.do_action_legacy(
-        ['clear', 'pipes'],
-        connector_keys=pipe.connector_keys,
-        metric_keys=pipe.metric_key,
-        location_keys=pipe.location_key,
-        force=True,
+    r_url = pipe_r_url(pipe)
+    response = self.delete(
+        r_url + '/clear',
+        params={
+            'begin': begin,
+            'end': end,
+            'params': json.dumps(params),
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
         debug=debug,
-        **kw
     )
+    if debug:
+        dprint(response.text)
+
+    try:
+        data = response.json()
+    except Exception as e:
+        return False, f"Failed to clear {pipe} with constraints {begin=}, {end=}, {params=}."
+
+    if isinstance(data, list):
+        response_tuple = data[0], data[1]
+    elif 'detail' in response.json():
+        response_tuple = response.__bool__(), data['detail']
+    else:
+        response_tuple = response.__bool__(), response.text
+
+    return response_tuple
 
 
 def get_pipe_columns_types(
@@ -728,7 +769,10 @@ def get_pipe_columns_types(
     r_url = pipe_r_url(pipe) + '/columns/types'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
@@ -760,7 +804,10 @@ def get_pipe_columns_indices(
     r_url = pipe_r_url(pipe) + '/columns/indices'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
@@ -779,14 +826,16 @@ def get_pipe_index_names(self, pipe: mrsm.Pipe, debug: bool = False) -> Dict[str
     r_url = pipe_r_url(pipe) + '/indices/names'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
         warn(j['detail'])
-        return None
+        return {}
     if not isinstance(j, dict):
         warn(response.text)
-        return None
+        return {}
     return j
-

meerschaum/connectors/sql/_pipes.py

@@ -499,6 +499,7 @@ def get_create_index_queries(
         get_rename_table_queries,
         COALESCE_UNIQUE_INDEX_FLAVORS,
     )
+    from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
         get_db_type_from_pd_type,
         get_pd_type_from_db_type,
@@ -793,8 +794,19 @@ def get_create_index_queries(
         cols_names = [sql_item_name(col, self.flavor, None) for col in cols if col]
         if not cols_names:
             continue
+
         cols_names_str = ", ".join(cols_names)
-        index_queries[ix_key] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({cols_names_str})"]
+        index_query_params_clause = f" ({cols_names_str})"
+        if self.flavor == 'postgis':
+            for col in cols:
+                col_typ = existing_cols_pd_types.get(cols[0], 'object')
+                if col_typ != 'object' and are_dtypes_equal(col_typ, 'geometry'):
+                    index_query_params_clause = f" USING GIST ({cols_names_str})"
+                    break
+
+        index_queries[ix_key] = [
+            f"CREATE INDEX {ix_name} ON {_pipe_name}{index_query_params_clause}"
+        ]
 
     indices_cols_str = ', '.join(
         list({
@@ -1544,7 +1556,11 @@ def create_pipe_table_from_df(
         get_datetime_cols,
         get_bytes_cols,
     )
-    from meerschaum.utils.sql import get_create_table_queries, sql_item_name
+    from meerschaum.utils.sql import (
+        get_create_table_queries,
+        sql_item_name,
+        get_create_schema_if_not_exists_queries,
+    )
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     primary_key = pipe.columns.get('primary', None)
     primary_key_typ = (
@@ -1601,15 +1617,21 @@ def create_pipe_table_from_df(
     if autoincrement:
         _ = new_dtypes.pop(primary_key, None)
 
+    schema = self.get_pipe_schema(pipe)
     create_table_queries = get_create_table_queries(
         new_dtypes,
         pipe.target,
         self.flavor,
-        schema=self.get_pipe_schema(pipe),
+        schema=schema,
         primary_key=primary_key,
         primary_key_db_type=primary_key_db_type,
         datetime_column=dt_col,
     )
+    if schema:
+        create_table_queries = (
+            get_create_schema_if_not_exists_queries(schema, self.flavor)
+            + create_table_queries
+        )
     success = all(
         self.exec_queries(create_table_queries, break_on_error=True, rollback=True, debug=debug)
     )
@@ -2085,6 +2107,7 @@ def sync_pipe_inplace(
         get_update_queries,
         get_null_replacement,
         get_create_table_queries,
+        get_create_schema_if_not_exists_queries,
         get_table_cols_types,
         session_execute,
         dateadd_str,
@@ -2164,18 +2187,28 @@ def sync_pipe_inplace(
             warn(drop_stale_msg)
         return drop_stale_success, drop_stale_msg
 
-    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import(
+        'sqlalchemy',
+        'sqlalchemy.orm',
+    )
     if not pipe.exists(debug=debug):
+        schema = self.get_pipe_schema(pipe)
         create_pipe_queries = get_create_table_queries(
             metadef,
             pipe.target,
             self.flavor,
-            schema=self.get_pipe_schema(pipe),
+            schema=schema,
             primary_key=primary_key,
             primary_key_db_type=primary_key_db_type,
             autoincrement=autoincrement,
             datetime_column=dt_col,
         )
+        if schema:
+            create_pipe_queries = (
+                get_create_schema_if_not_exists_queries(schema, self.flavor)
+                + create_pipe_queries
+            )
+
         results = self.exec_queries(create_pipe_queries, debug=debug)
         if not all(results):
             _ = clean_up_temp_tables()

meerschaum/connectors/valkey/_pipes.py

@@ -448,6 +448,7 @@ def get_pipe_data(
             debug=debug,
         )
     ]
+    print(f"{ix_docs=}")
     try:
         docs_strings = [
             self.get(

meerschaum/core/Pipe/_data.py

@@ -679,12 +679,15 @@ def get_chunk_bounds(
     include_greater_than_end = not bounded and end is None
     if begin is None:
         begin = self.get_sync_time(newest=False, debug=debug)
+    consolidate_end_chunk = False
     if end is None:
         end = self.get_sync_time(newest=True, debug=debug)
         if end is not None and hasattr(end, 'tzinfo'):
             end += timedelta(minutes=1)
+            consolidate_end_chunk = True
         elif are_dtypes_equal(str(type(end)), 'int'):
             end += 1
+            consolidate_end_chunk = True
     if begin is None and end is None:
         return [(None, None)]
 
@@ -707,9 +710,15 @@ def get_chunk_bounds(
         num_chunks += 1
         if num_chunks >= max_chunks:
             raise ValueError(
-                f"Too many chunks of size '{interval_str(chunk_interval)}' between '{begin}' and '{end}'."
+                f"Too many chunks of size '{interval_str(chunk_interval)}' "
+                f"between '{begin}' and '{end}'."
             )
 
+    if num_chunks > 1 and consolidate_end_chunk:
+        last_bounds, second_last_bounds = chunk_bounds[-1], chunk_bounds[-2]
+        chunk_bounds = chunk_bounds[:-2]
+        chunk_bounds.append((second_last_bounds[0], last_bounds[1]))
+
     ### The chunk interval might be too large.
     if not chunk_bounds and end >= begin:
         chunk_bounds = [(begin, end)]

meerschaum/core/Pipe/_verify.py

@@ -282,6 +282,7 @@ def verify(
             )
         bounds_success_tuples[first_chunk_bounds] = (first_success, first_msg)
         info(f"Completed first chunk for {self}:\n    {first_label}\n")
+        chunk_bounds = chunk_bounds[1:]
 
     pool = get_pool(workers=workers)
     batches = self.get_chunk_bounds_batches(chunk_bounds, batchsize=batchsize, workers=workers)

meerschaum/plugins/__init__.py

@@ -91,8 +91,8 @@ def make_action(
 
 
 def pre_sync_hook(
-        function: Callable[[Any], Any],
-    ) -> Callable[[Any], Any]:
+    function: Callable[[Any], Any],
+) -> Callable[[Any], Any]:
     """
     Register a function as a sync hook to be executed right before sync.
     
@@ -169,6 +169,7 @@ def web_page(
     page: Union[str, None, Callable[[Any], Any]] = None,
     login_required: bool = True,
     skip_navbar: bool = False,
+    page_group: Optional[str] = None,
     **kwargs
 ) -> Any:
     """
@@ -188,7 +189,7 @@ def web_page(
     page_str = None
 
     def _decorator(_func: Callable[[Any], Any]) -> Callable[[Any], Any]:
-        nonlocal page_str
+        nonlocal page_str, page_group
 
         @functools.wraps(_func)
         def wrapper(*_args, **_kwargs):
@@ -198,10 +199,31 @@ def web_page(
             page_str = _func.__name__
 
         page_str = page_str.lstrip('/').rstrip('/').strip()
-        _plugin_endpoints_to_pages[page_str] = {
+        page_key = (
+            ' '.join(
+                [
+                    word.capitalize()
+                    for word in (
+                        page_str.replace('/dash', '').lstrip('/').rstrip('/').strip()
+                        .replace('-', ' ').replace('_', ' ').split(' ')
+                    )
+                ]
+            )
+        )
+ 
+        package_name = _func.__globals__['__name__']
+        plugin_name = (
+            package_name.split('.')[1]
+            if package_name.startswith('plugins.') else None
+        )
+        page_group = page_group or plugin_name
+        if page_group not in _plugin_endpoints_to_pages:
+            _plugin_endpoints_to_pages[page_group] = {}
+        _plugin_endpoints_to_pages[page_group][page_str] = {
             'function': _func,
             'login_required': login_required,
             'skip_navbar': skip_navbar,
+            'page_key': page_key,
         }
         return wrapper
 

meerschaum/utils/dataframe.py

@@ -922,7 +922,12 @@ def get_geometry_cols(
     for col in geo_cols:
         try:
             sample_geo_series = gpd.GeoSeries(df[col], crs=None)
-            geometry_types = {geom.geom_type for geom in sample_geo_series}
+            geometry_types = {
+                geom.geom_type
+                for geom in sample_geo_series
+                if hasattr(geom, 'geom_type')
+            }
+            geometry_has_z = any(getattr(geom, 'has_z', False) for geom in sample_geo_series)
             srid = (
                 (
                     sample_geo_series.crs.sub_crs_list[0].to_epsg()
@@ -933,6 +938,8 @@ def get_geometry_cols(
                 else 0
             )
             geometry_type = list(geometry_types)[0] if len(geometry_types) == 1 else 'geometry'
+            if geometry_type != 'geometry' and geometry_has_z:
+                geometry_type = geometry_type + 'Z'
         except Exception:
             srid = 0
             geometry_type = 'geometry'