meerschaum 2.9.0rc3__py3-none-any.whl → 2.9.1__py3-none-any.whl

meerschaum/connectors/api/_pipes.py

@@ -414,19 +414,24 @@ def get_pipe_id(
     self,
     pipe: mrsm.Pipe,
     debug: bool = False,
-) -> int:
+) -> Union[int, str, None]:
     """Get a Pipe's ID from the API."""
     from meerschaum.utils.misc import is_int
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/id',
-        debug = debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     if debug:
         dprint(f"Got pipe ID: {response.text}")
     try:
         if is_int(response.text):
             return int(response.text)
+        if response.text and response.text[0] != '{':
+            return response.text
     except Exception as e:
         warn(f"Failed to get the ID for {pipe}:\n{e}")
     return None
@@ -450,7 +455,13 @@ def get_pipe_attributes(
     If the pipe does not exist, return an empty dictionary.
     """
     r_url = pipe_r_url(pipe)
-    response = self.get(r_url + '/attributes', debug=debug)
+    response = self.get(
+        r_url + '/attributes',
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug
+    )
     try:
         return json.loads(response.text)
     except Exception as e:
@@ -489,9 +500,13 @@ def get_sync_time(
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/sync_time',
-        json = params,
-        params = {'newest': newest, 'debug': debug},
-        debug = debug,
+        json=params,
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+            'newest': newest,
+            'debug': debug,
+        },
+        debug=debug,
     )
     if not response:
         warn(f"Failed to get the sync time for {pipe}:\n" + response.text)
@@ -532,7 +547,13 @@ def pipe_exists(
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn
     r_url = pipe_r_url(pipe)
-    response = self.get(r_url + '/exists', debug=debug)
+    response = self.get(
+        r_url + '/exists',
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
+    )
     if not response:
         warn(f"Failed to check if {pipe} exists:\n{response.text}")
         return False
@@ -570,8 +591,8 @@ def create_metadata(
 def get_pipe_rowcount(
     self,
     pipe: mrsm.Pipe,
-    begin: Optional[datetime] = None,
-    end: Optional[datetime] = None,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
     params: Optional[Dict[str, Any]] = None,
     remote: bool = False,
     debug: bool = False,
@@ -583,10 +604,10 @@ def get_pipe_rowcount(
     pipe: 'meerschaum.Pipe':
         The pipe whose row count we are counting.
         
-    begin: Optional[datetime], default None
+    begin: Union[str, datetime, int, None], default None
         If provided, bound the count by this datetime.
 
-    end: Optional[datetime]
+    end: Union[str, datetime, int, None], default None
         If provided, bound the count by this datetime.
 
     params: Optional[Dict[str, Any]], default None
@@ -608,6 +629,7 @@ def get_pipe_rowcount(
             'begin': begin,
             'end': end,
             'remote': remote,
+            'instance': self.get_pipe_instance_keys(pipe),
         },
         debug = debug
     )
@@ -645,7 +667,10 @@ def drop_pipe(
     r_url = pipe_r_url(pipe)
     response = self.delete(
         r_url + '/drop',
-        debug = debug,
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -668,6 +693,9 @@ def drop_pipe(
 def clear_pipe(
     self,
     pipe: mrsm.Pipe,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -683,20 +711,33 @@ def clear_pipe(
     -------
     A success tuple.
     """
-    kw.pop('metric_keys', None)
-    kw.pop('connector_keys', None)
-    kw.pop('location_keys', None)
-    kw.pop('action', None)
-    kw.pop('force', None)
-    return self.do_action_legacy(
-        ['clear', 'pipes'],
-        connector_keys=pipe.connector_keys,
-        metric_keys=pipe.metric_key,
-        location_keys=pipe.location_key,
-        force=True,
+    r_url = pipe_r_url(pipe)
+    response = self.delete(
+        r_url + '/clear',
+        params={
+            'begin': begin,
+            'end': end,
+            'params': json.dumps(params),
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
         debug=debug,
-        **kw
     )
+    if debug:
+        dprint(response.text)
+
+    try:
+        data = response.json()
+    except Exception as e:
+        return False, f"Failed to clear {pipe} with constraints {begin=}, {end=}, {params=}."
+
+    if isinstance(data, list):
+        response_tuple = data[0], data[1]
+    elif 'detail' in response.json():
+        response_tuple = response.__bool__(), data['detail']
+    else:
+        response_tuple = response.__bool__(), response.text
+
+    return response_tuple
 
 
 def get_pipe_columns_types(
@@ -728,7 +769,10 @@ def get_pipe_columns_types(
     r_url = pipe_r_url(pipe) + '/columns/types'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
@@ -760,7 +804,10 @@ def get_pipe_columns_indices(
     r_url = pipe_r_url(pipe) + '/columns/indices'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
@@ -779,14 +826,16 @@ def get_pipe_index_names(self, pipe: mrsm.Pipe, debug: bool = False) -> Dict[str
     r_url = pipe_r_url(pipe) + '/indices/names'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
         warn(j['detail'])
-        return None
+        return {}
     if not isinstance(j, dict):
         warn(response.text)
-        return None
+        return {}
     return j
-

meerschaum/connectors/sql/_pipes.py

@@ -499,6 +499,7 @@ def get_create_index_queries(
         get_rename_table_queries,
         COALESCE_UNIQUE_INDEX_FLAVORS,
     )
+    from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
         get_db_type_from_pd_type,
         get_pd_type_from_db_type,
@@ -793,8 +794,19 @@ def get_create_index_queries(
         cols_names = [sql_item_name(col, self.flavor, None) for col in cols if col]
         if not cols_names:
             continue
+
         cols_names_str = ", ".join(cols_names)
-        index_queries[ix_key] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({cols_names_str})"]
+        index_query_params_clause = f" ({cols_names_str})"
+        if self.flavor == 'postgis':
+            for col in cols:
+                col_typ = existing_cols_pd_types.get(cols[0], 'object')
+                if col_typ != 'object' and are_dtypes_equal(col_typ, 'geometry'):
+                    index_query_params_clause = f" USING GIST ({cols_names_str})"
+                    break
+
+        index_queries[ix_key] = [
+            f"CREATE INDEX {ix_name} ON {_pipe_name}{index_query_params_clause}"
+        ]
 
     indices_cols_str = ', '.join(
         list({
@@ -1544,7 +1556,11 @@ def create_pipe_table_from_df(
         get_datetime_cols,
         get_bytes_cols,
     )
-    from meerschaum.utils.sql import get_create_table_queries, sql_item_name
+    from meerschaum.utils.sql import (
+        get_create_table_queries,
+        sql_item_name,
+        get_create_schema_if_not_exists_queries,
+    )
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
     primary_key = pipe.columns.get('primary', None)
     primary_key_typ = (
@@ -1601,15 +1617,21 @@ def create_pipe_table_from_df(
     if autoincrement:
         _ = new_dtypes.pop(primary_key, None)
 
+    schema = self.get_pipe_schema(pipe)
     create_table_queries = get_create_table_queries(
         new_dtypes,
         pipe.target,
         self.flavor,
-        schema=self.get_pipe_schema(pipe),
+        schema=schema,
         primary_key=primary_key,
         primary_key_db_type=primary_key_db_type,
         datetime_column=dt_col,
     )
+    if schema:
+        create_table_queries = (
+            get_create_schema_if_not_exists_queries(schema, self.flavor)
+            + create_table_queries
+        )
     success = all(
         self.exec_queries(create_table_queries, break_on_error=True, rollback=True, debug=debug)
     )
@@ -2085,6 +2107,7 @@ def sync_pipe_inplace(
         get_update_queries,
         get_null_replacement,
         get_create_table_queries,
+        get_create_schema_if_not_exists_queries,
         get_table_cols_types,
         session_execute,
         dateadd_str,
@@ -2164,18 +2187,28 @@ def sync_pipe_inplace(
             warn(drop_stale_msg)
         return drop_stale_success, drop_stale_msg
 
-    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import(
+        'sqlalchemy',
+        'sqlalchemy.orm',
+    )
     if not pipe.exists(debug=debug):
+        schema = self.get_pipe_schema(pipe)
         create_pipe_queries = get_create_table_queries(
             metadef,
             pipe.target,
             self.flavor,
-            schema=self.get_pipe_schema(pipe),
+            schema=schema,
             primary_key=primary_key,
             primary_key_db_type=primary_key_db_type,
             autoincrement=autoincrement,
             datetime_column=dt_col,
         )
+        if schema:
+            create_pipe_queries = (
+                get_create_schema_if_not_exists_queries(schema, self.flavor)
+                + create_pipe_queries
+            )
+
         results = self.exec_queries(create_pipe_queries, debug=debug)
         if not all(results):
             _ = clean_up_temp_tables()

meerschaum/connectors/valkey/_pipes.py

@@ -448,6 +448,7 @@ def get_pipe_data(
             debug=debug,
         )
     ]
+    print(f"{ix_docs=}")
     try:
         docs_strings = [
             self.get(

meerschaum/plugins/__init__.py

@@ -91,8 +91,8 @@ def make_action(
 
 
 def pre_sync_hook(
-        function: Callable[[Any], Any],
-    ) -> Callable[[Any], Any]:
+    function: Callable[[Any], Any],
+) -> Callable[[Any], Any]:
     """
     Register a function as a sync hook to be executed right before sync.
     
@@ -169,6 +169,7 @@ def web_page(
     page: Union[str, None, Callable[[Any], Any]] = None,
     login_required: bool = True,
     skip_navbar: bool = False,
+    page_group: Optional[str] = None,
     **kwargs
 ) -> Any:
     """
@@ -188,7 +189,7 @@ def web_page(
     page_str = None
 
     def _decorator(_func: Callable[[Any], Any]) -> Callable[[Any], Any]:
-        nonlocal page_str
+        nonlocal page_str, page_group
 
         @functools.wraps(_func)
         def wrapper(*_args, **_kwargs):
@@ -198,10 +199,31 @@ def web_page(
             page_str = _func.__name__
 
         page_str = page_str.lstrip('/').rstrip('/').strip()
-        _plugin_endpoints_to_pages[page_str] = {
+        page_key = (
+            ' '.join(
+                [
+                    word.capitalize()
+                    for word in (
+                        page_str.replace('/dash', '').lstrip('/').rstrip('/').strip()
+                        .replace('-', ' ').replace('_', ' ').split(' ')
+                    )
+                ]
+            )
+        )
+ 
+        package_name = _func.__globals__['__name__']
+        plugin_name = (
+            package_name.split('.')[1]
+            if package_name.startswith('plugins.') else None
+        )
+        page_group = page_group or plugin_name
+        if page_group not in _plugin_endpoints_to_pages:
+            _plugin_endpoints_to_pages[page_group] = {}
+        _plugin_endpoints_to_pages[page_group][page_str] = {
             'function': _func,
             'login_required': login_required,
             'skip_navbar': skip_navbar,
+            'page_key': page_key,
         }
         return wrapper
 

meerschaum/utils/dataframe.py

@@ -922,7 +922,12 @@ def get_geometry_cols(
     for col in geo_cols:
         try:
             sample_geo_series = gpd.GeoSeries(df[col], crs=None)
-            geometry_types = {geom.geom_type for geom in sample_geo_series}
+            geometry_types = {
+                geom.geom_type
+                for geom in sample_geo_series
+                if hasattr(geom, 'geom_type')
+            }
+            geometry_has_z = any(getattr(geom, 'has_z', False) for geom in sample_geo_series)
             srid = (
                 (
                     sample_geo_series.crs.sub_crs_list[0].to_epsg()
@@ -933,6 +938,8 @@ def get_geometry_cols(
                 else 0
             )
             geometry_type = list(geometry_types)[0] if len(geometry_types) == 1 else 'geometry'
+            if geometry_type != 'geometry' and geometry_has_z:
+                geometry_type = geometry_type + 'Z'
         except Exception:
             srid = 0
             geometry_type = 'geometry'

meerschaum/utils/dtypes/__init__.py

@@ -745,19 +745,18 @@ def get_geometry_type_srid(
     if not modifier:
         return default_type, default_srid
 
-    shapely_geometry_base = mrsm.attempt_import('shapely.geometry.base')
-    geometry_types = {
-        typ.lower(): typ
-        for typ in shapely_geometry_base.GEOMETRY_TYPES
-    }
-
-    parts = [part.lower().replace('srid=', '').replace('type=', '').strip() for part in modifier.split(',')]
+    parts = [
+        part.split('=')[-1].strip()
+        for part in modifier.split(',')
+    ]
     parts_casted = [
         (
             int(part)
             if is_int(part)
             else part
-        ) for part in parts]
+        )
+        for part in parts
+    ]
 
     srid = default_srid
     geometry_type = default_type
@@ -767,9 +766,9 @@ def get_geometry_type_srid(
             srid = part
             break
 
-    for part in parts:
-        if part.lower() in geometry_types:
-            geometry_type = geometry_types.get(part)
+    for part in parts_casted:
+        if isinstance(part, str):
+            geometry_type = part
             break
 
     return geometry_type, srid

meerschaum/utils/misc.py

@@ -175,7 +175,7 @@ def string_to_dict(
     import ast
     params_dict = {}
     for param in params_string.split(","):
-        _keys = param.split(":")
+        _keys = param.split(":", maxsplit=1)
         keys = _keys[:-1]
         try:
             val = ast.literal_eval(_keys[-1])

meerschaum/utils/packages/_packages.py

@@ -125,7 +125,6 @@ packages: Dict[str, Dict[str, str]] = {
         'jinja2'                     : 'jinja2==3.0.3',
     },
     'gui': {
-        'toga'                       : 'toga>=0.3.0-dev29',
         'webview'                    : 'pywebview>=3.6.3',
         'pycparser'                  : 'pycparser>=2.21.0',
     },

meerschaum/utils/sql.py

@@ -2587,3 +2587,37 @@ def get_postgis_geo_columns_types(
         for col, (func, typ, srid) in cols_type_tuples.items()
     }
     return geometry_cols_types
+
+
+def get_create_schema_if_not_exists_queries(
+    schema: str,
+    flavor: str,
+) -> List[str]:
+    """
+    Return the queries to create a schema if it does not yet exist.
+    For databases which do not support schemas, an empty list will be returned.
+    """
+    if not schema:
+        return []
+    
+    if flavor in NO_SCHEMA_FLAVORS:
+        return []
+
+    clean(schema)
+
+    if flavor == 'mssql':
+        return [
+            (
+                f"IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{schema}')\n"
+                "BEGIN\n"
+                f"    EXEC('CREATE SCHEMA {sql_item_name(schema, flavor)}');\n"
+                "END;"
+            )
+        ]
+
+    if flavor == 'oracle':
+        return []
+
+    return [
+        f"CREATE SCHEMA IF NOT EXISTS {sql_item_name(schema, flavor)};"
+    ]

{meerschaum-2.9.0rc3.dist-info → meerschaum-2.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: meerschaum
-Version: 2.9.0rc3
+Version: 2.9.1
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -118,7 +118,6 @@ Requires-Dist: mkdocs-linkcheck>=1.0.6; extra == "docs"
 Requires-Dist: mkdocs-redirects>=1.0.4; extra == "docs"
 Requires-Dist: jinja2==3.0.3; extra == "docs"
 Provides-Extra: gui
-Requires-Dist: toga>=0.3.0-dev29; extra == "gui"
 Requires-Dist: pywebview>=3.6.3; extra == "gui"
 Requires-Dist: pycparser>=2.21.0; extra == "gui"
 Provides-Extra: extras
@@ -301,7 +300,6 @@ Requires-Dist: duckdb-engine>=0.13.0; extra == "full"
 Requires-Dist: pyproj>=3.7.1; extra == "full"
 Requires-Dist: geopandas>=1.0.1; extra == "full"
 Requires-Dist: shapely>=2.0.7; extra == "full"
-Requires-Dist: toga>=0.3.0-dev29; extra == "full"
 Requires-Dist: pywebview>=3.6.3; extra == "full"
 Requires-Dist: pycparser>=2.21.0; extra == "full"
 Requires-Dist: numpy>=1.18.5; extra == "full"