meerschaum 2.9.4__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

meerschaum/connectors/sql/_create_engine.py

@@ -7,171 +7,23 @@ This module contains the logic that builds the sqlalchemy engine string.
 """
 
 import traceback
+
 import meerschaum as mrsm
 from meerschaum.utils.debug import dprint
+from meerschaum._internal.static import STATIC_CONFIG
 
-### determine driver and requirements from flavor
-default_requirements = {
-    'username',
-    'password',
-    'host',
-    'database',
-}
 
 ### NOTE: These are defined in the `system.json` config file and so this dictionary's values
 ### will all be overwritten if applicable.
-default_create_engine_args = {
-    #  'method': 'multi',
-    'pool_size': 5,
-    'max_overflow': 10,
-    'pool_recycle': 3600,
-    'connect_args': {},
-}
-flavor_configs = {
-    'timescaledb': {
-        'engine': 'postgresql+psycopg',
-        'create_engine': default_create_engine_args,
-        'omit_create_engine': {'method',},
-        'to_sql': {},
-        'requirements': default_requirements,
-        'defaults': {
-            'port': 5432,
-        },
-    },
-    'postgresql': {
-        'engine': 'postgresql+psycopg',
-        'create_engine': default_create_engine_args,
-        'omit_create_engine': {'method',},
-        'to_sql': {},
-        'requirements': default_requirements,
-        'defaults': {
-            'port': 5432,
-        },
-    },
-    'postgis': {
-        'engine': 'postgresql+psycopg',
-        'create_engine': default_create_engine_args,
-        'omit_create_engine': {'method',},
-        'to_sql': {},
-        'requirements': default_requirements,
-        'defaults': {
-            'port': 5432,
-        },
-    },
-    'citus': {
-        'engine': 'postgresql+psycopg',
-        'create_engine': default_create_engine_args,
-        'omit_create_engine': {'method',},
-        'to_sql': {},
-        'requirements': default_requirements,
-        'defaults': {
-            'port': 5432,
-        },
-    },
-    'mssql': {
-        'engine': 'mssql+pyodbc',
-        'create_engine': {
-            'fast_executemany': True,
-            'use_insertmanyvalues': False,
-            'isolation_level': 'AUTOCOMMIT',
-            'use_setinputsizes': False,
-            'pool_pre_ping': True,
-            'ignore_no_transaction_on_rollback': True,
-        },
-        'omit_create_engine': {'method',},
-        'to_sql': {
-            'method': None,
-        },
-        'requirements': default_requirements,
-        'defaults': {
-            'port': 1433,
-            'options': (
-                "driver=ODBC Driver 18 for SQL Server"
-                "&UseFMTONLY=Yes"
-                "&TrustServerCertificate=yes"
-                "&Encrypt=no"
-                "&MARS_Connection=yes"
-            ),
-        },
-    },
-    'mysql': {
-        'engine': 'mysql+pymysql',
-        'create_engine': default_create_engine_args,
-        'omit_create_engine': {'method',},
-        'to_sql': {
-            'method': 'multi',
-        },
-        'requirements': default_requirements,
-        'defaults': {
-            'port': 3306,
-        },
-    },
-    'mariadb': {
-        'engine': 'mysql+pymysql',
-        'create_engine': default_create_engine_args,
-        'omit_create_engine': {'method',},
-        'to_sql': {
-            'method': 'multi',
-        },
-        'requirements': default_requirements,
-        'defaults': {
-            'port': 3306,
-        },
-    },
-    'oracle': {
-        'engine': 'oracle+oracledb',
-        'create_engine': default_create_engine_args,
-        'omit_create_engine': {'method',},
-        'to_sql': {
-            'method': None,
-        },
-        'requirements': default_requirements,
-        'defaults': {
-            'port': 1521,
-        },
-    },
-    'sqlite': {
-        'engine': 'sqlite',
-        'create_engine': default_create_engine_args,
-        'omit_create_engine': {'method',},
-        'to_sql': {
-            'method': 'multi',
-        },
-        'requirements': {'database'},
-        'defaults': {},
-    },
-    'duckdb': {
-        'engine': 'duckdb',
-        'create_engine': {},
-        'omit_create_engine': {'ALL',},
-        'to_sql': {
-            'method': 'multi',
-        },
-        'requirements': '',
-        'defaults': {},
-    },
-    'cockroachdb': {
-        'engine': 'cockroachdb',
-        'omit_create_engine': {'method',},
-        'create_engine': default_create_engine_args,
-        'to_sql': {
-            'method': 'multi',
-        },
-        'requirements': {'host'},
-        'defaults': {
-            'port': 26257,
-            'database': 'defaultdb',
-            'username': 'root',
-            'password': 'admin',
-        },
-    },
-}
+
+flavor_configs = STATIC_CONFIG['sql']['create_engine_flavors']
 install_flavor_drivers = {
     'sqlite': ['aiosqlite'],
     'duckdb': ['duckdb', 'duckdb_engine'],
     'mysql': ['pymysql'],
     'mariadb': ['pymysql'],
     'timescaledb': ['psycopg'],
+    'timescaledb-ha': ['psycopg', 'geoalchemy'],
     'postgresql': ['psycopg'],
     'postgis': ['psycopg', 'geoalchemy'],
     'citus': ['psycopg'],
@@ -268,7 +120,7 @@ def create_engine(
 
         ### Sometimes the timescaledb:// flavor can slip in.
         if _uri and self.flavor in _uri:
-            if self.flavor in ('timescaledb', 'postgis'):
+            if self.flavor in ('timescaledb', 'timescaledb-ha', 'postgis'):
                 engine_str = engine_str.replace(self.flavor, 'postgresql', 1)
             elif _uri.startswith('postgresql://'):
                 engine_str = engine_str.replace('postgresql://', 'postgresql+psycopg2://')

meerschaum/connectors/sql/_fetch.py

@@ -230,11 +230,9 @@ def get_pipe_query(pipe: mrsm.Pipe, warn: bool = True) -> Union[str, None]:
     - query
     - sql
     """
-    import re
     import textwrap
     from meerschaum.utils.warnings import warn as _warn
-    from meerschaum.utils.misc import parse_arguments_str
-    from meerschaum.utils.sql import sql_item_name
+    from meerschaum.utils.pipes import replace_pipes_syntax
     if pipe.parameters.get('fetch', {}).get('definition', None):
         definition = pipe.parameters['fetch']['definition']
     elif pipe.parameters.get('definition', None):
@@ -251,21 +249,7 @@ def get_pipe_query(pipe: mrsm.Pipe, warn: bool = True) -> Union[str, None]:
             )
         return None
 
-    def replace_pipe_match(pipe_match):
-        try:
-            args_str = pipe_match.group(1)
-            args, kwargs = parse_arguments_str(args_str)
-            pipe = mrsm.Pipe(*args, **kwargs)
-        except Exception as e:
-            if warn:
-                _warn(f"Failed to parse pipe from SQL definition:\n{e}")
-            raise e
-
-        target = pipe.target
-        schema = pipe.instance_connector.get_pipe_schema(pipe)
-        return sql_item_name(target, pipe.instance_connector.flavor, schema)
-
-    definition = re.sub(r'\{\{Pipe\((.*?)\)\}\}', replace_pipe_match, definition)
+    definition = replace_pipes_syntax(definition)
     return textwrap.dedent(definition.lstrip().rstrip())
 
 

meerschaum/connectors/sql/_pipes.py

@@ -45,7 +45,7 @@ def register_pipe(
     ###    (which shouldn't be able to be registered anyway but that's an issue for later).
     parameters = None
     try:
-        parameters = pipe.parameters
+        parameters = pipe.get_parameters(apply_symlinks=False)
     except Exception as e:
         if debug:
             dprint(str(e))
@@ -76,7 +76,7 @@ def register_pipe(
 
 def edit_pipe(
     self,
-    pipe : mrsm.Pipe = None,
+    pipe: mrsm.Pipe,
     patch: bool = False,
     debug: bool = False,
     **kw : Any
@@ -108,10 +108,10 @@ def edit_pipe(
         original_parameters = Pipe(
             pipe.connector_keys, pipe.metric_key, pipe.location_key,
             mrsm_instance=pipe.instance_keys
-        ).parameters
+        ).get_parameters(apply_symlinks=False)
         parameters = apply_patch_to_config(
             original_parameters,
-            pipe.parameters
+            pipe._attributes['parameters']
         )
 
     ### ensure pipes table exists
@@ -174,7 +174,7 @@ def fetch_pipes_keys(
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.misc import separate_negation_values
     from meerschaum.utils.sql import OMIT_NULLSFIRST_FLAVORS, table_exists
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     import json
     from copy import deepcopy
     sqlalchemy, sqlalchemy_sql_functions = attempt_import(
@@ -603,7 +603,10 @@ def get_create_index_queries(
     ### create datetime index
     dt_query = None
     if _datetime is not None:
-        if self.flavor == 'timescaledb' and pipe.parameters.get('hypertable', True):
+        if (
+            self.flavor in ('timescaledb', 'timescaledb-ha')
+            and pipe.parameters.get('hypertable', True)
+        ):
             _id_count = (
                 get_distinct_col_count(_id, f"SELECT {_id_name} FROM {_pipe_name}", self)
                 if (_id is not None and _create_space_partition) else None
@@ -719,7 +722,7 @@ def get_create_index_queries(
                         f"ADD CONSTRAINT {primary_key_constraint_name} PRIMARY KEY ({primary_key_name})"
                     )
                 ])
-            elif self.flavor == 'timescaledb':
+            elif self.flavor in ('timescaledb', 'timescaledb-ha'):
                 primary_queries.extend([
                     (
                         f"ALTER TABLE {_pipe_name}\n"
@@ -758,7 +761,7 @@ def get_create_index_queries(
 
     ### create id index
     if _id_name is not None:
-        if self.flavor == 'timescaledb':
+        if self.flavor in ('timescaledb', 'timescaledb-ha'):
             ### Already created indices via create_hypertable.
             id_query = (
                 None if (_id is not None and _create_space_partition)
@@ -797,7 +800,7 @@ def get_create_index_queries(
 
         cols_names_str = ", ".join(cols_names)
         index_query_params_clause = f" ({cols_names_str})"
-        if self.flavor == 'postgis':
+        if self.flavor in ('postgis', 'timescaledb-ha'):
             for col in cols:
                 col_typ = existing_cols_pd_types.get(cols[0], 'object')
                 if col_typ != 'object' and are_dtypes_equal(col_typ, 'geometry'):
@@ -1073,14 +1076,14 @@ def get_pipe_data(
 
     cols_types = pipe.get_columns_types(debug=debug) if pipe.enforce else {}
     dtypes = {
+        **{
+            col: get_pd_type_from_db_type(typ)
+            for col, typ in cols_types.items()
+        },
         **{
             p_col: to_pandas_dtype(p_typ)
             for p_col, p_typ in pipe.dtypes.items()
         },
-        **{
-            col: get_pd_type_from_db_type(typ)
-            for col, typ in cols_types.items()
-        }
     } if pipe.enforce else {}
     if dtypes:
         if self.flavor == 'sqlite':
@@ -1419,7 +1422,7 @@ def get_pipe_data_query(
             if k in existing_cols or skip_existing_cols_check
         }
         if valid_params:
-            where += build_where(valid_params, self).replace(
+            where += '    ' + build_where(valid_params, self).lstrip().replace(
                 'WHERE', ('    AND' if is_dt_bound else "    ")
             )
 
@@ -1648,8 +1651,8 @@ def sync_pipe(
     self,
     pipe: mrsm.Pipe,
     df: Union[pd.DataFrame, str, Dict[Any, Any], None] = None,
-    begin: Optional[datetime] = None,
-    end: Optional[datetime] = None,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
     chunksize: Optional[int] = -1,
     check_existing: bool = True,
     blocking: bool = True,
@@ -1669,11 +1672,11 @@ def sync_pipe(
         An optional DataFrame or equivalent to sync into the pipe.
         Defaults to `None`.
 
-    begin: Optional[datetime], default None
+    begin: Union[datetime, int, None], default None
         Optionally specify the earliest datetime to search for data.
         Defaults to `None`.
 
-    end: Optional[datetime], default None
+    end: Union[datetime, int, None], default None
         Optionally specify the latest datetime to search for data.
         Defaults to `None`.
 
@@ -1833,10 +1836,12 @@ def sync_pipe(
         )
     )
     if autoincrement and autoincrement not in pipe.parameters:
-        pipe.parameters['autoincrement'] = autoincrement
-        edit_success, edit_msg = pipe.edit(debug=debug)
-        if not edit_success:
-            return edit_success, edit_msg
+        update_success, update_msg = pipe.update_parameters(
+            {'autoincrement': autoincrement},
+            debug=debug,
+        )
+        if not update_success:
+            return update_success, update_msg
 
     def _check_pk(_df_to_clear):
         if _df_to_clear is None:
@@ -1969,7 +1974,11 @@ def sync_pipe(
             if col and col in existing_cols
         ] if not primary_key or self.flavor == 'oracle' else (
             [dt_col, primary_key]
-            if self.flavor == 'timescaledb' and dt_col and dt_col in update_df.columns
+            if (
+                self.flavor in ('timescaledb', 'timescaledb-ha')
+                and dt_col
+                and dt_col in update_df.columns
+            )
             else [primary_key]
         )
         update_queries = get_update_queries(
@@ -2833,7 +2842,6 @@ def get_pipe_rowcount(
             error(msg)
             return None
 
-
     flavor = self.flavor if not remote else pipe.connector.flavor
     conn = self if not remote else pipe.connector
     _pipe_name = sql_item_name(pipe.target, flavor, self.get_pipe_schema(pipe))
@@ -3412,12 +3420,13 @@ def get_alter_columns_queries(
 
     if numeric_cols:
         pipe.dtypes.update({col: 'numeric' for col in numeric_cols})
-        edit_success, edit_msg = pipe.edit(debug=debug)
-        if not edit_success:
-            warn(
-                f"Failed to update dtypes for numeric columns {items_str(numeric_cols)}:\n"
-                + f"{edit_msg}"
-            )
+        if not pipe.temporary:
+            edit_success, edit_msg = pipe.edit(debug=debug)
+            if not edit_success:
+                warn(
+                    f"Failed to update dtypes for numeric columns {items_str(numeric_cols)}:\n"
+                    + f"{edit_msg}"
+                )
     else:
         numeric_cols.extend([col for col, typ in pipe.dtypes.items() if typ.startswith('numeric')])
 
@@ -3881,13 +3890,15 @@ def get_pipe_schema(self, pipe: mrsm.Pipe) -> Union[str, None]:
     -------
     A schema string or `None` if nothing is configured.
     """
+    if self.flavor == 'sqlite':
+        return self.schema
     return pipe.parameters.get('schema', self.schema)
 
 
 @staticmethod
 def get_temporary_target(
     target: str,
-    transact_id: Optional[str, None] = None,
+    transact_id: Optional[str] = None,
     label: Optional[str] = None,
     separator: Optional[str] = None,
 ) -> str:

meerschaum/connectors/sql/_plugins.py

@@ -13,6 +13,35 @@ import json
 import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Any, List, SuccessTuple, Dict
 
+
+def get_plugins_pipe(self) -> mrsm.Pipe:
+    """
+    Return the internal metadata plugins pipe.
+    """
+    users_pipe = self.get_users_pipe()
+    user_id_dtype = users_pipe.dtypes.get('user_id', 'int')
+    return mrsm.Pipe(
+        'mrsm', 'plugins',
+        instance=self,
+        temporary=True,
+        static=True,
+        null_indices=False,
+        columns={
+            'primary': 'plugin_id',
+            'user_id': 'user_id',    
+        },
+        dtypes={
+            'plugin_name': 'string',
+            'user_id': user_id_dtype,
+            'attributes': 'json',
+            'version': 'string',
+        },
+        indices={
+            'unique': 'plugin_name',
+        },
+    )
+
+
 def register_plugin(
     self,
     plugin: 'mrsm.core.Plugin',

meerschaum/connectors/sql/_sql.py

@@ -17,7 +17,14 @@ from meerschaum.utils.debug import dprint
 from meerschaum.utils.warnings import warn
 
 ### database flavors that can use bulk insert
-_bulk_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'mssql'}
+_bulk_flavors = {
+    'postgresql',
+    'postgis',
+    'timescaledb',
+    'timescaledb-ha',
+    'citus',
+    'mssql',
+}
 ### flavors that do not support chunks
 _disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000}
@@ -928,7 +935,7 @@ def to_sql(
             df[col] = df[col].apply(
                 functools.partial(
                     serialize_geometry,
-                    as_wkt=(self.flavor == 'mssql')
+                    geometry_format=('wkt' if self.flavor == 'mssql' else 'wkb_hex'),
                 )
             )
 

meerschaum/connectors/sql/_users.py

@@ -12,6 +12,32 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, Optional, Any, Dict, List, Union
 
 
+def get_users_pipe(self) -> mrsm.Pipe:
+    """
+    Return the internal metadata pipe for users management.
+    """
+    return mrsm.Pipe(
+        'mrsm', 'users',
+        temporary=True,
+        static=True,
+        null_indices=False,
+        enforce=False,
+        autoincrement=True,
+        columns={
+            'primary': 'user_id',
+        },
+        dtypes={
+            'user_id': 'int',
+            'username': 'string',
+            'attributes': 'json',
+            'user_type': 'string',
+        },
+        indices={
+            'unique': 'username',
+        },
+    )
+
+
 def register_user(
     self,
     user: mrsm.core.User,
@@ -64,7 +90,7 @@ def register_user(
 
 def valid_username(username: str) -> SuccessTuple:
     """Verify that a given username is valid."""
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     fail_reasons = []
 
     min_length = STATIC_CONFIG['users']['min_username_length']
@@ -104,6 +130,7 @@ def edit_user(
 ) -> SuccessTuple:
     """Update an existing user's metadata."""
     from meerschaum.utils.packages import attempt_import
+    from meerschaum.utils.sql import json_flavors
     sqlalchemy = attempt_import('sqlalchemy', lazy=False)
     from meerschaum.connectors.sql.tables import get_tables
     users_tbl = get_tables(mrsm_instance=self, debug=debug)['users']
@@ -131,7 +158,7 @@ def edit_user(
         bind_variables['email'] = user.email
     if user.attributes is not None and user.attributes != {}:
         bind_variables['attributes'] = (
-            json.dumps(user.attributes) if self.flavor in ('duckdb',)
+            json.dumps(user.attributes) if self.flavor not in json_flavors
             else user.attributes
         )
     if user.type != '':

meerschaum/connectors/sql/tables/__init__.py

@@ -231,7 +231,7 @@ def create_schemas(
     """
     Create the internal Meerschaum schema on the database.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.sql import sql_item_name, NO_SCHEMA_FLAVORS, SKIP_IF_EXISTS_FLAVORS
     if conn.flavor in NO_SCHEMA_FLAVORS:

meerschaum/connectors/valkey/_ValkeyConnector.py

@@ -10,19 +10,18 @@ import json
 from datetime import datetime, timezone
 
 import meerschaum as mrsm
-from meerschaum.connectors import Connector, make_connector
+from meerschaum.connectors import InstanceConnector, make_connector
 from meerschaum.utils.typing import List, Dict, Any, Optional, Union
 from meerschaum.utils.warnings import dprint
 
 
 @make_connector
-class ValkeyConnector(Connector):
+class ValkeyConnector(InstanceConnector):
     """
     Manage a Valkey instance.
 
     Build a `ValkeyConnector` from connection attributes or a URI string.
     """
-    IS_INSTANCE: bool = True
     REQUIRED_ATTRIBUTES: List[str] = ['host']
     OPTIONAL_ATTRIBUTES: List[str] = [
         'port', 'username', 'password', 'db', 'socket_timeout',
@@ -80,7 +79,6 @@ class ValkeyConnector(Connector):
         get_plugin_user_id,
         get_plugin_username,
         get_plugin_attributes,
-        get_plugins,
         delete_plugin,
     )
 

meerschaum/connectors/valkey/_pipes.py

@@ -13,7 +13,7 @@ from meerschaum.utils.typing import SuccessTuple, Any, Union, Optional, Dict, Li
 from meerschaum.utils.misc import string_to_dict
 from meerschaum.utils.dtypes import json_serialize_value
 from meerschaum.utils.warnings import warn, dprint
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 PIPES_TABLE: str = 'mrsm_pipes'
 PIPES_COUNTER: str = 'mrsm_pipes:counter'
@@ -276,7 +276,7 @@ def edit_pipe(
         return False, f"{pipe} is not registered."
 
     parameters_key = get_pipe_parameters_key(pipe)
-    parameters_str = json.dumps(pipe.parameters, separators=(',', ':'))
+    parameters_str = json.dumps(pipe.get_parameters(apply_symlinks=False), separators=(',', ':'))
     self.set(parameters_key, parameters_str)
     return True, "Success"
 
@@ -337,7 +337,7 @@ def drop_pipe(
     if 'valkey' not in pipe.parameters:
         return True, "Success"
 
-    pipe.parameters['valkey']['dtypes'] = {}
+    pipe._attributes['parameters']['valkey']['dtypes'] = {}
     if not pipe.temporary:
         edit_success, edit_msg = pipe.edit(debug=debug)
         if not edit_success:
@@ -577,13 +577,12 @@ def sync_pipe(
 
     if new_dtypes and (not static or not valkey_dtypes):
         valkey_dtypes.update(new_dtypes)
-        if 'valkey' not in pipe.parameters:
-            pipe.parameters['valkey'] = {}
-        pipe.parameters['valkey']['dtypes'] = valkey_dtypes
-        if not pipe.temporary:
-            edit_success, edit_msg = pipe.edit(debug=debug)
-            if not edit_success:
-                return edit_success, edit_msg
+        update_success, update_msg = pipe.update_parameters(
+            {'valkey': {'dtypes': valkey_dtypes}},
+            debug=debug,
+        )
+        if not update_success:
+            return False, update_msg
 
     unseen_df, update_df, delta_df = (
         pipe.filter_existing(df, include_unchanged_columns=True, debug=debug)

meerschaum/connectors/valkey/_plugins.py

@@ -127,7 +127,8 @@ def get_plugin_id(
     """
     Return a plugin's ID.
     """
-    return plugin.name
+    user_id = self.get_plugin_user_id(plugin, debug=debug)
+    return plugin.name if user_id is not None else None
 
 
 def get_plugin_version(
@@ -199,31 +200,6 @@ def get_plugin_attributes(
         return {}
 
 
-def get_plugins(
-    self,
-    user_id: Optional[int] = None,
-    search_term: Optional[str] = None,
-    debug: bool = False,
-    **kw: Any
-) -> List[str]:
-    """
-    Return a list of plugin names.
-    """
-    plugins_pipe = self.get_plugins_pipe()
-    params = {}
-    if user_id:
-        params['user_id'] = user_id
-
-    df = plugins_pipe.get_data(['plugin_name'], params=params, debug=debug)
-    docs = df.to_dict(orient='records')
-
-    return [
-        doc['plugin_name']
-        for doc in docs
-        if (plugin_name := doc['plugin_name']).startswith(search_term or '')
-    ]
-
-
 def delete_plugin(
     self,
     plugin: 'mrsm.core.Plugin',