meerschaum 2.8.4__py3-none-any.whl → 2.9.0__py3-none-any.whl

meerschaum/connectors/api/_pipes.py

@@ -31,6 +31,14 @@ def pipe_r_url(
     )
 
 
+def get_pipe_instance_keys(self, pipe: mrsm.Pipe) -> Union[str, None]:
+    """
+    Return the configured instance keys for a pipe if set,
+    else fall back to the default `instance_keys` for this `APIConnector`.
+    """
+    return pipe.parameters.get('instance_keys', self.instance_keys)
+
+
 def register_pipe(
     self,
     pipe: mrsm.Pipe,
@@ -40,13 +48,12 @@ def register_pipe(
     Returns a tuple of (success_bool, response_dict).
     """
     from meerschaum.utils.debug import dprint
-    ### NOTE: if `parameters` is supplied in the Pipe constructor,
-    ###       then `pipe.parameters` will exist and not be fetched from the database.
     r_url = pipe_r_url(pipe)
     response = self.post(
         r_url + '/register',
-        json = pipe.parameters,
-        debug = debug,
+        json=pipe._attributes.get('parameters', {}),
+        params={'instance_keys': self.get_pipe_instance_keys(pipe)},
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -79,9 +86,9 @@ def edit_pipe(
     r_url = pipe_r_url(pipe)
     response = self.patch(
         r_url + '/edit',
-        params = {'patch': patch,},
-        json = pipe.parameters,
-        debug = debug,
+        params={'patch': patch, 'instance_keys': self.get_pipe_instance_keys(pipe)},
+        json=pipe.parameters,
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -149,12 +156,13 @@ def fetch_pipes_keys(
     try:
         j = self.get(
             r_url,
-            params = {
+            params={
                 'connector_keys': json.dumps(connector_keys),
                 'metric_keys': json.dumps(metric_keys),
                 'location_keys': json.dumps(location_keys),
                 'tags': json.dumps(tags),
                 'params': json.dumps(params),
+                'instance_keys': self.instance_keys,
             },
             debug=debug
         ).json()
@@ -250,8 +258,10 @@ def sync_pipe(
         chunks = (df[i] for i in more_itertools.chunked(df, _chunksize))
 
     ### Send columns in case the user has defined them locally.
+    request_params = kw.copy()
     if pipe.columns:
-        kw['columns'] = json.dumps(pipe.columns)
+        request_params['columns'] = json.dumps(pipe.columns)
+    request_params['instance_keys'] = self.get_pipe_instance_keys(pipe)
     r_url = pipe_r_url(pipe) + '/data'
 
     rowcount = 0
@@ -268,10 +278,9 @@ def sync_pipe(
         try:
             response = self.post(
                 r_url,
-                ### handles check_existing
-                params = kw,
-                data = json_str,
-                debug = debug
+                params=request_params,
+                data=json_str,
+                debug=debug,
             )
         except Exception as e:
             msg = f"Failed to post a chunk to {pipe}:\n{e}"
@@ -323,7 +332,8 @@ def delete_pipe(
     r_url = pipe_r_url(pipe)
     response = self.delete(
         r_url + '/delete',
-        debug = debug,
+        params={'instance_keys': self.get_pipe_instance_keys(pipe)},
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -361,7 +371,9 @@ def get_pipe_data(
                     'omit_columns': json.dumps(omit_columns),
                     'begin': begin,
                     'end': end,
-                    'params': json.dumps(params, default=str)
+                    'params': json.dumps(params, default=str),
+                    'instance': self.get_pipe_instance_keys(pipe),
+                    'as_chunks': as_chunks,
                 },
                 debug=debug
             )
@@ -402,19 +414,24 @@ def get_pipe_id(
     self,
     pipe: mrsm.Pipe,
     debug: bool = False,
-) -> int:
+) -> Union[int, str, None]:
     """Get a Pipe's ID from the API."""
     from meerschaum.utils.misc import is_int
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/id',
-        debug = debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     if debug:
         dprint(f"Got pipe ID: {response.text}")
     try:
         if is_int(response.text):
             return int(response.text)
+        if response.text and response.text[0] != '{':
+            return response.text
     except Exception as e:
         warn(f"Failed to get the ID for {pipe}:\n{e}")
     return None
@@ -438,7 +455,13 @@ def get_pipe_attributes(
     If the pipe does not exist, return an empty dictionary.
     """
     r_url = pipe_r_url(pipe)
-    response = self.get(r_url + '/attributes', debug=debug)
+    response = self.get(
+        r_url + '/attributes',
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug
+    )
     try:
         return json.loads(response.text)
     except Exception as e:
@@ -477,9 +500,13 @@ def get_sync_time(
     r_url = pipe_r_url(pipe)
     response = self.get(
         r_url + '/sync_time',
-        json = params,
-        params = {'newest': newest, 'debug': debug},
-        debug = debug,
+        json=params,
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+            'newest': newest,
+            'debug': debug,
+        },
+        debug=debug,
     )
     if not response:
         warn(f"Failed to get the sync time for {pipe}:\n" + response.text)
@@ -520,7 +547,13 @@ def pipe_exists(
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn
     r_url = pipe_r_url(pipe)
-    response = self.get(r_url + '/exists', debug=debug)
+    response = self.get(
+        r_url + '/exists',
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
+    )
     if not response:
         warn(f"Failed to check if {pipe} exists:\n{response.text}")
         return False
@@ -558,8 +591,8 @@ def create_metadata(
 def get_pipe_rowcount(
     self,
     pipe: mrsm.Pipe,
-    begin: Optional[datetime] = None,
-    end: Optional[datetime] = None,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
     params: Optional[Dict[str, Any]] = None,
     remote: bool = False,
     debug: bool = False,
@@ -571,10 +604,10 @@ def get_pipe_rowcount(
     pipe: 'meerschaum.Pipe':
         The pipe whose row count we are counting.
         
-    begin: Optional[datetime], default None
+    begin: Union[str, datetime, int, None], default None
         If provided, bound the count by this datetime.
 
-    end: Optional[datetime]
+    end: Union[str, datetime, int, None], default None
         If provided, bound the count by this datetime.
 
     params: Optional[Dict[str, Any]], default None
@@ -596,6 +629,7 @@ def get_pipe_rowcount(
             'begin': begin,
             'end': end,
             'remote': remote,
+            'instance': self.get_pipe_instance_keys(pipe),
         },
         debug = debug
     )
@@ -633,7 +667,10 @@ def drop_pipe(
     r_url = pipe_r_url(pipe)
     response = self.delete(
         r_url + '/drop',
-        debug = debug,
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     if debug:
         dprint(response.text)
@@ -656,6 +693,9 @@ def drop_pipe(
 def clear_pipe(
     self,
     pipe: mrsm.Pipe,
+    begin: Union[str, datetime, int, None] = None,
+    end: Union[str, datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
     debug: bool = False,
     **kw
 ) -> SuccessTuple:
@@ -671,20 +711,33 @@ def clear_pipe(
     -------
     A success tuple.
     """
-    kw.pop('metric_keys', None)
-    kw.pop('connector_keys', None)
-    kw.pop('location_keys', None)
-    kw.pop('action', None)
-    kw.pop('force', None)
-    return self.do_action_legacy(
-        ['clear', 'pipes'],
-        connector_keys=pipe.connector_keys,
-        metric_keys=pipe.metric_key,
-        location_keys=pipe.location_key,
-        force=True,
+    r_url = pipe_r_url(pipe)
+    response = self.delete(
+        r_url + '/clear',
+        params={
+            'begin': begin,
+            'end': end,
+            'params': json.dumps(params),
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
         debug=debug,
-        **kw
     )
+    if debug:
+        dprint(response.text)
+
+    try:
+        data = response.json()
+    except Exception as e:
+        return False, f"Failed to clear {pipe} with constraints {begin=}, {end=}, {params=}."
+
+    if isinstance(data, list):
+        response_tuple = data[0], data[1]
+    elif 'detail' in response.json():
+        response_tuple = response.__bool__(), data['detail']
+    else:
+        response_tuple = response.__bool__(), response.text
+
+    return response_tuple
 
 
 def get_pipe_columns_types(
@@ -716,7 +769,10 @@ def get_pipe_columns_types(
     r_url = pipe_r_url(pipe) + '/columns/types'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
@@ -748,7 +804,10 @@ def get_pipe_columns_indices(
     r_url = pipe_r_url(pipe) + '/columns/indices'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
@@ -767,14 +826,16 @@ def get_pipe_index_names(self, pipe: mrsm.Pipe, debug: bool = False) -> Dict[str
     r_url = pipe_r_url(pipe) + '/indices/names'
     response = self.get(
         r_url,
-        debug=debug
+        params={
+            'instance': self.get_pipe_instance_keys(pipe),
+        },
+        debug=debug,
     )
     j = response.json()
     if isinstance(j, dict) and 'detail' in j and len(j.keys()) == 1:
         warn(j['detail'])
-        return None
+        return {}
     if not isinstance(j, dict):
         warn(response.text)
-        return None
+        return {}
     return j
-

meerschaum/connectors/api/_plugins.py

@@ -7,21 +7,25 @@ Manage plugins via the API connector
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import Union, Any, Optional, SuccessTuple, Mapping, Sequence
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import Union, Any, Optional, SuccessTuple, List, Dict
+
 
 def plugin_r_url(
-        plugin: Union[meerschaum.core.Plugin, str]
-    ) -> str:
+    plugin: Union[mrsm.core.Plugin, str],
+) -> str:
     """Generate a relative URL path from a Plugin."""
     from meerschaum.config.static import STATIC_CONFIG
     return f"{STATIC_CONFIG['api']['endpoints']['plugins']}/{plugin}"
 
+
 def register_plugin(
-        self,
-        plugin: meerschaum.core.Plugin,
-        make_archive: bool = True,
-        debug: bool = False,
-    ) -> SuccessTuple:
+    self,
+    plugin: mrsm.core.Plugin,
+    make_archive: bool = True,
+    debug: bool = False,
+) -> SuccessTuple:
     """Register a plugin and upload its archive."""
     import json
     archive_path = plugin.make_tar(debug=debug) if make_archive else plugin.archive_path
@@ -34,27 +38,30 @@ def register_plugin(
     r_url = plugin_r_url(plugin)
     try:
         response = self.post(r_url, files=files, params=metadata, debug=debug)
-    except Exception as e:
+    except Exception:
         return False, f"Failed to register plugin '{plugin}'."
     finally:
         file_pointer.close()
 
     try:
         success, msg = json.loads(response.text)
-    except Exception as e:
+    except Exception:
         return False, response.text
 
     return success, msg
 
+
 def install_plugin(
-        self,
-        name: str,
-        skip_deps: bool = False,
-        force: bool = False,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    name: str,
+    skip_deps: bool = False,
+    force: bool = False,
+    debug: bool = False
+) -> SuccessTuple:
     """Download and attempt to install a plugin from the API."""
-    import os, pathlib, json
+    import os
+    import pathlib
+    import json
     from meerschaum.core import Plugin
     from meerschaum.config._paths import PLUGINS_TEMP_RESOURCES_PATH
     from meerschaum.utils.debug import dprint
@@ -75,41 +82,39 @@ def install_plugin(
                 success, msg = tuple(j)
             elif isinstance(j, dict) and 'detail' in j:
                 success, msg = False, fail_msg
-        except Exception as e:
+        except Exception:
             success, msg = False, fail_msg
         return success, msg
     plugin = Plugin(name, archive_path=archive_path, repo_connector=self)
     return plugin.install(skip_deps=skip_deps, force=force, debug=debug)
 
+
 def get_plugins(
-        self,
-        user_id : Optional[int] = None,
-        search_term : Optional[str] = None,
-        debug : bool = False
-    ) -> Sequence[str]:
+    self,
+    user_id: Optional[int] = None,
+    search_term: Optional[str] = None,
+    debug: bool = False
+) -> List[str]:
     """Return a list of registered plugin names.
 
     Parameters
     ----------
-    user_id :
+    user_id: Optional[int], default None
         If specified, return all plugins from a certain user.
-    user_id : Optional[int] :
-         (Default value = None)
-    search_term : Optional[str] :
-         (Default value = None)
-    debug : bool :
-         (Default value = False)
+
+    search_term: Optional[str], default None
+        If specified, return plugins beginning with this string.
 
     Returns
     -------
-
+    A list of plugin names.
     """
     import json
-    from meerschaum.utils.warnings import warn, error
+    from meerschaum.utils.warnings import error
     from meerschaum.config.static import STATIC_CONFIG
     response = self.get(
         STATIC_CONFIG['api']['endpoints']['plugins'],
-        params = {'user_id' : user_id, 'search_term' : search_term},
+        params = {'user_id': user_id, 'search_term': search_term},
         use_token = True,
         debug = debug
     )
@@ -120,11 +125,12 @@ def get_plugins(
         error(response.text)
     return plugins
 
+
 def get_plugin_attributes(
-        self,
-        plugin: meerschaum.core.Plugin,
-        debug: bool = False
-    ) -> Mapping[str, Any]:
+    self,
+    plugin: mrsm.core.Plugin,
+    debug: bool = False
+) -> Dict[str, Any]:
     """
     Return a plugin's attributes.
     """
@@ -136,7 +142,7 @@ def get_plugin_attributes(
     if isinstance(attributes, str) and attributes and attributes[0] == '{':
         try:
             attributes = json.loads(attributes)
-        except Exception as e:
+        except Exception:
             pass
     if not isinstance(attributes, dict):
         error(response.text)
@@ -145,23 +151,23 @@ def get_plugin_attributes(
         return {}
     return attributes
 
+
 def delete_plugin(
-        self,
-        plugin: meerschaum.core.Plugin,
-        debug: bool = False
-    ) -> SuccessTuple:
+    self,
+    plugin: mrsm.core.Plugin,
+    debug: bool = False
+) -> SuccessTuple:
     """Delete a plugin from an API repository."""
     import json
     r_url = plugin_r_url(plugin)
     try:
         response = self.delete(r_url, debug=debug)
-    except Exception as e:
+    except Exception:
         return False, f"Failed to delete plugin '{plugin}'."
 
     try:
         success, msg = json.loads(response.text)
-    except Exception as e:
+    except Exception:
         return False, response.text
 
     return success, msg
-

meerschaum/connectors/api/_request.py

@@ -95,7 +95,7 @@ def make_request(
     return self.session.request(
         method.upper(),
         request_url,
-        headers = headers,
+        headers=headers,
         **kwargs
     )
 

meerschaum/connectors/parse.py

@@ -140,7 +140,6 @@ def is_valid_connector_keys(
     """
     try:
         success = parse_connector_keys(keys, construct=False) is not None
-    except Exception as e:
+    except Exception:
         success = False
     return success
-

meerschaum/connectors/sql/_SQLConnector.py

@@ -151,6 +151,9 @@ class SQLConnector(Connector):
             if uri.startswith('timescaledb://'):
                 uri = uri.replace('timescaledb://', 'postgresql+psycopg://', 1)
                 flavor = 'timescaledb'
+            if uri.startswith('postgis://'):
+                uri = uri.replace('postgis://', 'postgresql+psycopg://', 1)
+                flavor = 'postgis'
             kw['uri'] = uri
             from_uri_params = self.from_uri(kw['uri'], as_dict=True)
             label = label or from_uri_params.get('label', None)

meerschaum/connectors/sql/_cli.py

@@ -15,6 +15,7 @@ from meerschaum.utils.typing import SuccessTuple
 
 flavor_clis = {
     'postgresql'  : 'pgcli',
+    'postgis'  : 'pgcli',
     'timescaledb' : 'pgcli',
     'cockroachdb' : 'pgcli',
     'citus'       : 'pgcli',

meerschaum/connectors/sql/_create_engine.py

@@ -7,6 +7,7 @@ This module contains the logic that builds the sqlalchemy engine string.
 """
 
 import traceback
+import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
 
 ### determine driver and requirements from flavor
@@ -47,6 +48,16 @@ flavor_configs = {
             'port': 5432,
         },
     },
+    'postgis': {
+        'engine': 'postgresql+psycopg',
+        'create_engine': default_create_engine_args,
+        'omit_create_engine': {'method',},
+        'to_sql': {},
+        'requirements': default_requirements,
+        'defaults': {
+            'port': 5432,
+        },
+    },
     'citus': {
         'engine': 'postgresql+psycopg',
         'create_engine': default_create_engine_args,
@@ -162,6 +173,7 @@ install_flavor_drivers = {
     'mariadb': ['pymysql'],
     'timescaledb': ['psycopg'],
     'postgresql': ['psycopg'],
+    'postgis': ['psycopg', 'geoalchemy'],
     'citus': ['psycopg'],
     'cockroachdb': ['psycopg', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg'],
     'mssql': ['pyodbc'],
@@ -198,8 +210,7 @@ def create_engine(
             warn=False,
         )
         if self.flavor == 'mssql':
-            pyodbc = attempt_import('pyodbc', debug=debug, lazy=False, warn=False)
-            pyodbc.pooling = False
+            _init_mssql_sqlalchemy()
     if self.flavor in require_patching_flavors:
         from meerschaum.utils.packages import determine_version, _monkey_patch_get_distribution
         import pathlib
@@ -257,8 +268,8 @@ def create_engine(
 
         ### Sometimes the timescaledb:// flavor can slip in.
         if _uri and self.flavor in _uri:
-            if self.flavor == 'timescaledb':
-                engine_str = engine_str.replace(f'{self.flavor}', 'postgresql', 1)
+            if self.flavor in ('timescaledb', 'postgis'):
+                engine_str = engine_str.replace(self.flavor, 'postgresql', 1)
             elif _uri.startswith('postgresql://'):
                 engine_str = engine_str.replace('postgresql://', 'postgresql+psycopg2://')
 
@@ -313,3 +324,39 @@ def create_engine(
     if include_uri:
         return engine, engine_str
     return engine
+
+
+def _init_mssql_sqlalchemy():
+    """
+    When first instantiating a SQLAlchemy connection to MSSQL,
+    monkey-patch `pyodbc` handling in SQLAlchemy.
+    """
+    pyodbc, sqlalchemy_dialects_mssql_pyodbc = mrsm.attempt_import(
+        'pyodbc',
+        'sqlalchemy.dialects.mssql.pyodbc',
+        lazy=False,
+        warn=False,
+    )
+    pyodbc.pooling = False
+
+    MSDialect_pyodbc = sqlalchemy_dialects_mssql_pyodbc.MSDialect_pyodbc
+
+    def _handle_geometry(val):
+        from binascii import hexlify
+        hex_str = f"0x{hexlify(val).decode().upper()}"
+        return hex_str
+
+    def custom_on_connect(self):
+        super_ = super(MSDialect_pyodbc, self).on_connect()
+
+        def _on_connect(conn):
+            if super_ is not None:
+                super_(conn)
+
+            self._setup_timestampoffset_type(conn)
+            conn.add_output_converter(-151, _handle_geometry)
+
+        return _on_connect
+
+    ### TODO: Parse proprietary MSSQL geometry bytes into WKB.
+    #  MSDialect_pyodbc.on_connect = custom_on_connect