meerschaum 2.8.4__py3-none-any.whl → 2.9.0rc1__py3-none-any.whl

meerschaum/connectors/api/_plugins.py

@@ -7,21 +7,25 @@ Manage plugins via the API connector
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Union, Any, Optional, SuccessTuple, Mapping, Sequence
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Union, Any, Optional, SuccessTuple, List, Dict
+
 
 def plugin_r_url(
-        plugin: Union[meerschaum.core.Plugin, str]
-    ) -> str:
+    plugin: Union[mrsm.core.Plugin, str],
+) -> str:
     """Generate a relative URL path from a Plugin."""
     from meerschaum.config.static import STATIC_CONFIG
     return f"{STATIC_CONFIG['api']['endpoints']['plugins']}/{plugin}"
 
+
 def register_plugin(
-        self,
-        plugin: meerschaum.core.Plugin,
-        make_archive: bool = True,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    plugin: mrsm.core.Plugin,
+    make_archive: bool = True,
+    debug: bool = False,
+) -> SuccessTuple:
     """Register a plugin and upload its archive."""
     import json
     archive_path = plugin.make_tar(debug=debug) if make_archive else plugin.archive_path
@@ -34,27 +38,30 @@ def register_plugin(
     r_url = plugin_r_url(plugin)
     try:
         response = self.post(r_url, files=files, params=metadata, debug=debug)
-    except Exception as e:
+    except Exception:
         return False, f"Failed to register plugin '{plugin}'."
     finally:
         file_pointer.close()
 
     try:
         success, msg = json.loads(response.text)
-    except Exception as e:
+    except Exception:
         return False, response.text
 
     return success, msg
 
+
 def install_plugin(
-        self,
-        name: str,
-        skip_deps: bool = False,
-        force: bool = False,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    name: str,
+    skip_deps: bool = False,
+    force: bool = False,
+    debug: bool = False
+) -> SuccessTuple:
     """Download and attempt to install a plugin from the API."""
-    import os, pathlib, json
+    import os
+    import pathlib
+    import json
     from meerschaum.core import Plugin
     from meerschaum.config._paths import PLUGINS_TEMP_RESOURCES_PATH
     from meerschaum.utils.debug import dprint
@@ -75,41 +82,39 @@ def install_plugin(
                 success, msg = tuple(j)
             elif isinstance(j, dict) and 'detail' in j:
                 success, msg = False, fail_msg
-        except Exception as e:
+        except Exception:
             success, msg = False, fail_msg
         return success, msg
     plugin = Plugin(name, archive_path=archive_path, repo_connector=self)
     return plugin.install(skip_deps=skip_deps, force=force, debug=debug)
 
+
 def get_plugins(
-        self,
-        user_id : Optional[int] = None,
-        search_term : Optional[str] = None,
-        debug : bool = False
-    ) -> Sequence[str]:
+    self,
+    user_id: Optional[int] = None,
+    search_term: Optional[str] = None,
+    debug: bool = False
+) -> List[str]:
     """Return a list of registered plugin names.
 
     Parameters
     ----------
-    user_id :
+    user_id: Optional[int], default None
         If specified, return all plugins from a certain user.
-    user_id : Optional[int] :
-         (Default value = None)
-    search_term : Optional[str] :
-         (Default value = None)
-    debug : bool :
-         (Default value = False)
+
+    search_term: Optional[str], default None
+        If specified, return plugins beginning with this string.
 
     Returns
     -------
-
+    A list of plugin names.
     """
     import json
-    from meerschaum.utils.warnings import warn, error
+    from meerschaum.utils.warnings import error
     from meerschaum.config.static import STATIC_CONFIG
     response = self.get(
         STATIC_CONFIG['api']['endpoints']['plugins'],
-        params = {'user_id' : user_id, 'search_term' : search_term},
+        params = {'user_id': user_id, 'search_term': search_term},
         use_token = True,
         debug = debug
     )
@@ -120,11 +125,12 @@ def get_plugins(
         error(response.text)
     return plugins
 
+
 def get_plugin_attributes(
-        self,
-        plugin: meerschaum.core.Plugin,
-        debug: bool = False
-    ) -> Mapping[str, Any]:
+    self,
+    plugin: mrsm.core.Plugin,
+    debug: bool = False
+) -> Dict[str, Any]:
     """
     Return a plugin's attributes.
     """
@@ -136,7 +142,7 @@ def get_plugin_attributes(
     if isinstance(attributes, str) and attributes and attributes[0] == '{':
         try:
             attributes = json.loads(attributes)
-        except Exception as e:
+        except Exception:
             pass
     if not isinstance(attributes, dict):
         error(response.text)
@@ -145,23 +151,23 @@ def get_plugin_attributes(
         return {}
     return attributes
 
+
 def delete_plugin(
-        self,
-        plugin: meerschaum.core.Plugin,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    plugin: mrsm.core.Plugin,
+    debug: bool = False
+) -> SuccessTuple:
     """Delete a plugin from an API repository."""
     import json
     r_url = plugin_r_url(plugin)
     try:
         response = self.delete(r_url, debug=debug)
-    except Exception as e:
+    except Exception:
         return False, f"Failed to delete plugin '{plugin}'."
 
     try:
         success, msg = json.loads(response.text)
-    except Exception as e:
+    except Exception:
         return False, response.text
 
     return success, msg
-

meerschaum/connectors/api/_request.py

@@ -95,7 +95,7 @@ def make_request(
     return self.session.request(
         method.upper(),
         request_url,
-        headers = headers,
+        headers=headers,
         **kwargs
     )
 

meerschaum/connectors/parse.py

@@ -140,7 +140,6 @@ def is_valid_connector_keys(
     """
     try:
         success = parse_connector_keys(keys, construct=False) is not None
-    except Exception as e:
+    except Exception:
         success = False
     return success
-

meerschaum/connectors/sql/_SQLConnector.py

@@ -151,6 +151,9 @@ class SQLConnector(Connector):
             if uri.startswith('timescaledb://'):
                 uri = uri.replace('timescaledb://', 'postgresql+psycopg://', 1)
                 flavor = 'timescaledb'
+            if uri.startswith('postgis://'):
+                uri = uri.replace('postgis://', 'postgresql+psycopg://', 1)
+                flavor = 'postgis'
             kw['uri'] = uri
             from_uri_params = self.from_uri(kw['uri'], as_dict=True)
             label = label or from_uri_params.get('label', None)

meerschaum/connectors/sql/_cli.py

@@ -15,6 +15,7 @@ from meerschaum.utils.typing import SuccessTuple
 
 flavor_clis = {
     'postgresql'  : 'pgcli',
+    'postgis'  : 'pgcli',
     'timescaledb' : 'pgcli',
     'cockroachdb' : 'pgcli',
     'citus'       : 'pgcli',

meerschaum/connectors/sql/_create_engine.py

@@ -7,6 +7,7 @@ This module contains the logic that builds the sqlalchemy engine string.
 """
 
 import traceback
+import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
 
 ### determine driver and requirements from flavor
@@ -47,6 +48,16 @@ flavor_configs = {
             'port': 5432,
         },
     },
+    'postgis': {
+        'engine': 'postgresql+psycopg',
+        'create_engine': default_create_engine_args,
+        'omit_create_engine': {'method',},
+        'to_sql': {},
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 5432,
+        },
+    },
     'citus': {
         'engine': 'postgresql+psycopg',
         'create_engine': default_create_engine_args,
@@ -162,6 +173,7 @@ install_flavor_drivers = {
     'mariadb': ['pymysql'],
     'timescaledb': ['psycopg'],
     'postgresql': ['psycopg'],
+    'postgis': ['psycopg', 'geoalchemy'],
     'citus': ['psycopg'],
     'cockroachdb': ['psycopg', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg'],
     'mssql': ['pyodbc'],
@@ -198,8 +210,7 @@ def create_engine(
             warn=False,
         )
         if self.flavor == 'mssql':
-            pyodbc = attempt_import('pyodbc', debug=debug, lazy=False, warn=False)
-            pyodbc.pooling = False
+            _init_mssql_sqlalchemy()
     if self.flavor in require_patching_flavors:
         from meerschaum.utils.packages import determine_version, _monkey_patch_get_distribution
         import pathlib
@@ -257,8 +268,8 @@ def create_engine(
 
         ### Sometimes the timescaledb:// flavor can slip in.
         if _uri and self.flavor in _uri:
-            if self.flavor == 'timescaledb':
-                engine_str = engine_str.replace(f'{self.flavor}', 'postgresql', 1)
+            if self.flavor in ('timescaledb', 'postgis'):
+                engine_str = engine_str.replace(self.flavor, 'postgresql', 1)
             elif _uri.startswith('postgresql://'):
                 engine_str = engine_str.replace('postgresql://', 'postgresql+psycopg2://')
 
@@ -313,3 +324,39 @@ def create_engine(
     if include_uri:
         return engine, engine_str
     return engine
+
+
+def _init_mssql_sqlalchemy():
+    """
+    When first instantiating a SQLAlchemy connection to MSSQL,
+    monkey-patch `pyodbc` handling in SQLAlchemy.
+    """
+    pyodbc, sqlalchemy_dialects_mssql_pyodbc = mrsm.attempt_import(
+        'pyodbc',
+        'sqlalchemy.dialects.mssql.pyodbc',
+        lazy=False,
+        warn=False,
+    )
+    pyodbc.pooling = False
+
+    MSDialect_pyodbc = sqlalchemy_dialects_mssql_pyodbc.MSDialect_pyodbc
+
+    def _handle_geometry(val):
+        from binascii import hexlify
+        hex_str = f"0x{hexlify(val).decode().upper()}"
+        return hex_str
+
+    def custom_on_connect(self):
+        super_ = super(MSDialect_pyodbc, self).on_connect()
+
+        def _on_connect(conn):
+            if super_ is not None:
+                super_(conn)
+
+            self._setup_timestampoffset_type(conn)
+            conn.add_output_converter(-151, _handle_geometry)
+
+        return _on_connect
+
+    ### TODO: Parse proprietary MSSQL geometry bytes into WKB.
+    #  MSDialect_pyodbc.on_connect = custom_on_connect

meerschaum/connectors/sql/_pipes.py

@@ -731,7 +731,7 @@ def get_create_index_queries(
                         ) + f"{primary_key_name})"
                     ),
                 ])
-            elif self.flavor in ('citus', 'postgresql', 'duckdb'):
+            elif self.flavor in ('citus', 'postgresql', 'duckdb', 'postgis'):
                 primary_queries.extend([
                     (
                         f"ALTER TABLE {_pipe_name}\n"
@@ -1052,6 +1052,7 @@ def get_pipe_data(
         attempt_cast_to_numeric,
         attempt_cast_to_uuid,
         attempt_cast_to_bytes,
+        attempt_cast_to_geometry,
         are_dtypes_equal,
     )
     from meerschaum.utils.dtypes.sql import get_pd_type_from_db_type
@@ -1138,6 +1139,11 @@ def get_pipe_data(
         for col, typ in pipe.dtypes.items()
         if typ == 'bytes' and col in dtypes
     ]
+    geometry_columns = [
+        col
+        for col, typ in pipe.dtypes.items()
+        if typ.startswith('geometry') and col in dtypes
+    ]
 
     kw['coerce_float'] = kw.get('coerce_float', (len(numeric_columns) == 0))
 
@@ -1162,6 +1168,11 @@ def get_pipe_data(
             continue
         df[col] = df[col].apply(attempt_cast_to_bytes)
 
+    for col in geometry_columns:
+        if col not in df.columns:
+            continue
+        df[col] = df[col].apply(attempt_cast_to_geometry)
+
     if self.flavor == 'sqlite':
         ignore_dt_cols = [
             col
@@ -1511,7 +1522,7 @@ def get_pipe_attributes(
             if isinstance(parameters, str) and parameters[0] == '{':
                 parameters = json.loads(parameters)
             attributes['parameters'] = parameters
-        except Exception as e:
+        except Exception:
             attributes['parameters'] = {}
 
     return attributes

meerschaum/connectors/sql/_sql.py

@@ -17,7 +17,7 @@ from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn
 
 ### database flavors that can use bulk insert
-_bulk_flavors = {'postgresql', 'timescaledb', 'citus', 'mssql'}
+_bulk_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'mssql'}
 ### flavors that do not support chunks
 _disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000}
@@ -798,6 +798,7 @@ def to_sql(
         get_numeric_cols,
         get_uuid_cols,
         get_bytes_cols,
+        get_geometry_cols,
     )
     from meerschaum.utils.dtypes import (
         are_dtypes_equal,
@@ -805,7 +806,9 @@ def to_sql(
         encode_bytes_for_bytea,
         serialize_bytes,
         serialize_decimal,
+        serialize_geometry,
         json_serialize_value,
+        get_geometry_type_srid,
     )
     from meerschaum.utils.dtypes.sql import (
         PD_TO_SQLALCHEMY_DTYPES_FLAVORS,
@@ -822,6 +825,7 @@ def to_sql(
 
     bytes_cols = get_bytes_cols(df)
     numeric_cols = get_numeric_cols(df)
+    geometry_cols = get_geometry_cols(df)
     ### NOTE: This excludes non-numeric serialized Decimals (e.g. SQLite).
     numeric_cols_dtypes = {
         col: typ
@@ -830,7 +834,6 @@ def to_sql(
             col in df.columns
             and 'numeric' in str(typ).lower()
         )
-        
     }
     numeric_cols.extend([col for col in numeric_cols_dtypes if col not in numeric_cols])
     numeric_cols_precisions_scales = {
@@ -841,6 +844,22 @@ def to_sql(
         )
         for col, typ in numeric_cols_dtypes.items()
     }
+    geometry_cols_dtypes = {
+        col: typ
+        for col, typ in kw.get('dtype', {}).items()
+        if (
+            col in df.columns
+            and 'geometry' in str(typ).lower() or 'geography' in str(typ).lower()
+        )
+    }
+    geometry_cols.extend([col for col in geometry_cols_dtypes if col not in geometry_cols])
+    geometry_cols_types_srids = {
+        col: (typ.geometry_type, typ.srid)
+        if hasattr(typ, 'srid')
+        else get_geometry_type_srid()
+        for col, typ in geometry_cols_dtypes.items()
+    }
+
     cols_pd_types = {
         col: get_pd_type_from_db_type(str(typ))
         for col, typ in kw.get('dtype', {}).items()
@@ -856,8 +875,9 @@ def to_sql(
     }
 
     enable_bulk_insert = mrsm.get_config(
-        'system', 'connectors', 'sql', 'bulk_insert'
-    ).get(self.flavor, False)
+        'system', 'connectors', 'sql', 'bulk_insert', self.flavor,
+        warn=False,
+    ) or False
     stats = {'target': name}
     ### resort to defaults if None
     copied = False
@@ -901,6 +921,17 @@ def to_sql(
             )
         )
 
+    for col in geometry_cols:
+        geometry_type, srid = geometry_cols_types_srids.get(col, get_geometry_type_srid())
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            df[col] = df[col].apply(
+                functools.partial(
+                    serialize_geometry,
+                    as_wkt=(self.flavor == 'mssql')
+                )
+            )
+
     stats['method'] = method.__name__ if hasattr(method, '__name__') else str(method)
 
     default_chunksize = self._sys_config.get('chunksize', None)

meerschaum/core/Pipe/_data.py

@@ -88,9 +88,8 @@ def get_data(
     limit: Optional[int], default None
         If provided, cap the dataframe to this many rows.
 
-    fresh: bool, default True
+    fresh: bool, default False
         If `True`, skip local cache and directly query the instance connector.
-        Defaults to `True`.
 
     debug: bool, default False
         Verbosity toggle.

meerschaum/plugins/_Plugin.py

@@ -450,7 +450,7 @@ class Plugin:
                 success, msg = False, (
                     f"Failed to run post-install setup for plugin '{self}'." + '\n' +
                     f"Check `setup()` in '{self.__file__}' for more information " +
-                    f"(no error message provided)."
+                    "(no error message provided)."
                 )
             else:
                 success, msg = True, success_msg
@@ -458,7 +458,7 @@ class Plugin:
             success = True
             msg = (
                 f"Post-install for plugin '{self}' returned None. " +
-                f"Assuming plugin successfully installed."
+                "Assuming plugin successfully installed."
             )
             warn(msg)
         else:
@@ -469,7 +469,7 @@ class Plugin:
             )
 
         _ongoing_installations.remove(self.full_name)
-        module = self.module
+        _ = self.module
         return success, msg
 
 
@@ -716,13 +716,14 @@ class Plugin:
         return required
 
 
-    def get_required_plugins(self, debug: bool=False) -> List[meerschaum.plugins.Plugin]:
+    def get_required_plugins(self, debug: bool=False) -> List[mrsm.plugins.Plugin]:
         """
         Return a list of required Plugin objects.
         """
         from meerschaum.utils.warnings import warn
         from meerschaum.config import get_config
         from meerschaum.config.static import STATIC_CONFIG
+        from meerschaum.connectors.parse import is_valid_connector_keys
         plugins = []
         _deps = self.get_dependencies(debug=debug)
         sep = STATIC_CONFIG['plugins']['repo_separator']
@@ -731,11 +732,13 @@ class Plugin:
             if _d.startswith('plugin:') and len(_d) > len('plugin:')
         ]
         default_repo_keys = get_config('meerschaum', 'default_repository')
+        skipped_repo_keys = set()
+
         for _plugin_name in plugin_names:
             if sep in _plugin_name:
                 try:
                     _plugin_name, _repo_keys = _plugin_name.split(sep)
-                except Exception as e:
+                except Exception:
                     _repo_keys = default_repo_keys
                     warn(
                         f"Invalid repo keys for required plugin '{_plugin_name}'.\n    "
@@ -744,7 +747,20 @@ class Plugin:
                     )
             else:
                 _repo_keys = default_repo_keys
+
+            if _repo_keys in skipped_repo_keys:
+                continue
+
+            if not is_valid_connector_keys(_repo_keys):
+                warn(
+                    f"Invalid connector '{_repo_keys}'.\n"
+                    f"    Skipping required plugins from repository '{_repo_keys}'",
+                    stack=False,
+                )
+                continue
+
             plugins.append(Plugin(_plugin_name, repo=_repo_keys))
+
         return plugins
 
 

meerschaum/plugins/__init__.py

@@ -166,10 +166,11 @@ def post_sync_hook(
 
 _plugin_endpoints_to_pages = {}
 def web_page(
-        page: Union[str, None, Callable[[Any], Any]] = None,
-        login_required: bool = True,
-        **kwargs
-    ) -> Any:
+    page: Union[str, None, Callable[[Any], Any]] = None,
+    login_required: bool = True,
+    skip_navbar: bool = False,
+    **kwargs
+) -> Any:
     """
     Quickly add pages to the dash application.
 
@@ -200,6 +201,7 @@ def web_page(
         _plugin_endpoints_to_pages[page_str] = {
             'function': _func,
             'login_required': login_required,
+            'skip_navbar': skip_navbar,
         }
         return wrapper