meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/utils/sql.py

@@ -41,13 +41,33 @@ version_queries = {
 }
 SKIP_IF_EXISTS_FLAVORS = {'mssql', 'oracle'}
 DROP_IF_EXISTS_FLAVORS = {
-    'timescaledb', 'postgresql', 'postgis', 'citus', 'mssql', 'mysql', 'mariadb', 'sqlite',
+    'timescaledb',
+    'timescaledb-ha',
+    'postgresql',
+    'postgis',
+    'citus',
+    'mssql',
+    'mysql',
+    'mariadb',
+    'sqlite',
 }
 DROP_INDEX_IF_EXISTS_FLAVORS = {
-    'mssql', 'timescaledb', 'postgresql', 'postgis', 'sqlite', 'citus',
+    'mssql',
+    'timescaledb',
+    'timescaledb-ha',
+    'postgresql',
+    'postgis',
+    'sqlite',
+    'citus',
 }
 SKIP_AUTO_INCREMENT_FLAVORS = {'citus', 'duckdb'}
-COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'postgis', 'citus'}
+COALESCE_UNIQUE_INDEX_FLAVORS = {
+    'timescaledb',
+    'timescaledb-ha',
+    'postgresql',
+    'postgis',
+    'citus',
+}
 UPDATE_QUERIES = {
     'default': """
         UPDATE {target_table_name} AS f
@@ -67,6 +87,12 @@ UPDATE_QUERIES = {
         FROM {patch_table_name}
         ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
+    'timescaledb-ha-upsert': """
+        INSERT INTO {target_table_name} ({patch_cols_str})
+        SELECT {patch_cols_str}
+        FROM {patch_table_name}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
+    """,
     'postgresql-upsert': """
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
@@ -233,7 +259,7 @@ columns_types_queries = {
             DATA_TYPE AS [type],
             NUMERIC_PRECISION AS [numeric_precision],
             NUMERIC_SCALE AS [numeric_scale]
-        FROM {db_prefix}INFORMATION_SCHEMA.COLUMNS
+        FROM {db_prefix}INFORMATION_SCHEMA.COLUMNS WITH (NOLOCK)
         WHERE TABLE_NAME IN (
             '{table}',
             '{table_trunc}'
@@ -286,6 +312,7 @@ columns_types_queries = {
 }
 hypertable_queries = {
     'timescaledb': 'SELECT hypertable_size(\'{table_name}\')',
+    'timescaledb-ha': 'SELECT hypertable_size(\'{table_name}\')',
     'citus': 'SELECT citus_table_size(\'{table_name}\')',
 }
 columns_indices_queries = {
@@ -377,18 +404,18 @@ columns_indices_queries = {
                 ELSE CAST(0 AS BIT)
             END AS [clustered]
         FROM
-            sys.schemas s
-        INNER JOIN sys.tables t
+            sys.schemas s WITH (NOLOCK)
+        INNER JOIN sys.tables t WITH (NOLOCK)
             ON s.schema_id = t.schema_id
-        INNER JOIN sys.indexes i
+        INNER JOIN sys.indexes i WITH (NOLOCK)
             ON t.object_id = i.object_id
-        INNER JOIN sys.index_columns ic
+        INNER JOIN sys.index_columns ic WITH (NOLOCK)
             ON i.object_id = ic.object_id
             AND i.index_id = ic.index_id
-        INNER JOIN sys.columns c
+        INNER JOIN sys.columns c WITH (NOLOCK)
             ON ic.object_id = c.object_id
             AND ic.column_id = c.column_id
-        LEFT JOIN sys.key_constraints kc
+        LEFT JOIN sys.key_constraints kc WITH (NOLOCK)
             ON kc.parent_object_id = i.object_id
             AND kc.type = 'PK'
             AND kc.name = i.name
@@ -483,49 +510,83 @@ reset_autoincrement_queries: Dict[str, Union[str, List[str]]] = {
     ),
 }
 table_wrappers = {
-    'default'    : ('"', '"'),
-    'timescaledb': ('"', '"'),
-    'citus'      : ('"', '"'),
-    'duckdb'     : ('"', '"'),
-    'postgresql' : ('"', '"'),
-    'postgis'    : ('"', '"'),
-    'sqlite'     : ('"', '"'),
-    'mysql'      : ('`', '`'),
-    'mariadb'    : ('`', '`'),
-    'mssql'      : ('[', ']'),
-    'cockroachdb': ('"', '"'),
-    'oracle'     : ('"', '"'),
+    'default'       : ('"', '"'),
+    'timescaledb'   : ('"', '"'),
+    'timescaledb-ha': ('"', '"'),
+    'citus'         : ('"', '"'),
+    'duckdb'        : ('"', '"'),
+    'postgresql'    : ('"', '"'),
+    'postgis'       : ('"', '"'),
+    'sqlite'        : ('"', '"'),
+    'mysql'         : ('`', '`'),
+    'mariadb'       : ('`', '`'),
+    'mssql'         : ('[', ']'),
+    'cockroachdb'   : ('"', '"'),
+    'oracle'        : ('"', '"'),
 }
 max_name_lens = {
-    'default'    : 64,
-    'mssql'      : 128,
-    'oracle'     : 30,
-    'postgresql' : 64,
-    'postgis'    : 64,
-    'timescaledb': 64,
-    'citus'      : 64,
-    'cockroachdb': 64,
-    'sqlite'     : 1024, ### Probably more, but 1024 seems more than reasonable.
-    'mysql'      : 64,
-    'mariadb'    : 64,
+    'default'       : 64,
+    'mssql'         : 128,
+    'oracle'        : 30,
+    'postgresql'    : 64,
+    'postgis'       : 64,
+    'timescaledb'   : 64,
+    'timescaledb-ha': 64,
+    'citus'         : 64,
+    'cockroachdb'   : 64,
+    'sqlite'        : 1024, ### Probably more, but 1024 seems more than reasonable.
+    'mysql'         : 64,
+    'mariadb'       : 64,
+}
+json_flavors = {
+    'postgresql',
+    'postgis',
+    'timescaledb',
+    'timescaledb-ha',
+    'citus',
+    'cockroachdb',
+}
+NO_SCHEMA_FLAVORS = {
+    'oracle',
+    'sqlite',
+    'mysql',
+    'mariadb',
+    'duckdb',
 }
-json_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'cockroachdb'}
-NO_SCHEMA_FLAVORS = {'oracle', 'sqlite', 'mysql', 'mariadb', 'duckdb'}
 DEFAULT_SCHEMA_FLAVORS = {
     'postgresql': 'public',
     'postgis': 'public',
     'timescaledb': 'public',
+    'timescaledb-ha': 'public',
     'citus': 'public',
     'cockroachdb': 'public',
     'mysql': 'mysql',
     'mariadb': 'mysql',
     'mssql': 'dbo',
 }
-OMIT_NULLSFIRST_FLAVORS = {'mariadb', 'mysql', 'mssql'}
+OMIT_NULLSFIRST_FLAVORS = {
+    'mariadb',
+    'mysql',
+    'mssql',
+}
 
-SINGLE_ALTER_TABLE_FLAVORS = {'duckdb', 'sqlite', 'mssql', 'oracle'}
-NO_CTE_FLAVORS = {'mysql', 'mariadb'}
-NO_SELECT_INTO_FLAVORS = {'sqlite', 'oracle', 'mysql', 'mariadb', 'duckdb'}
+SINGLE_ALTER_TABLE_FLAVORS = {
+    'duckdb',
+    'sqlite',
+    'mssql',
+    'oracle',
+}
+NO_CTE_FLAVORS = {
+    'mysql',
+    'mariadb',
+}
+NO_SELECT_INTO_FLAVORS = {
+    'sqlite',
+    'oracle',
+    'mysql',
+    'mariadb',
+    'duckdb',
+}
 
 
 def clean(substring: str) -> None:
@@ -560,6 +621,7 @@ def dateadd_str(
         - `'postgresql'`
         - `'postgis'`
         - `'timescaledb'`
+        - `'timescaledb-ha'`
         - `'citus'`
         - `'cockroachdb'`
         - `'duckdb'`
@@ -663,7 +725,14 @@ def dateadd_str(
     )
 
     da = ""
-    if flavor in ('postgresql', 'postgis', 'timescaledb', 'cockroachdb', 'citus'):
+    if flavor in (
+        'postgresql',
+        'postgis',
+        'timescaledb',
+        'timescaledb-ha',
+        'cockroachdb',
+        'citus',
+    ):
         begin = (
             f"CAST({begin} AS {db_type})" if begin != 'now'
             else f"CAST(NOW() AT TIME ZONE 'utc' AS {db_type})"
@@ -831,7 +900,7 @@ def sql_item_name(item: str, flavor: str, schema: Optional[str] = None) -> str:
         ### NOTE: System-reserved words must be quoted.
         if truncated_item.lower() in (
             'float', 'varchar', 'nvarchar', 'clob',
-            'boolean', 'integer', 'table', 'row',
+            'boolean', 'integer', 'table', 'row', 'date',
         ):
             wrappers = ('"', '"')
         else:
@@ -969,7 +1038,7 @@ def build_where(
     ```
     """
     import json
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.dtypes import value_is_null, none_if_null
     negation_prefix = STATIC_CONFIG['system']['fetch_pipes_keys']['negation_prefix']
@@ -1517,6 +1586,8 @@ def get_update_queries(
     datetime_col: Optional[str] = None,
     schema: Optional[str] = None,
     patch_schema: Optional[str] = None,
+    target_cols_types: Optional[Dict[str, str]] = None,
+    patch_cols_types: Optional[Dict[str, str]] = None,
     identity_insert: bool = False,
     null_indices: bool = True,
     cast_columns: bool = True,
@@ -1557,6 +1628,14 @@ def get_update_queries(
         If provided, use this schema when quoting the patch table.
         Defaults to `schema`.
 
+    target_cols_types: Optional[Dict[str, Any]], default None
+        If provided, use these as the columns-types dictionary for the target table.
+        Default will infer from the database context.
+
+    patch_cols_types: Optional[Dict[str, Any]], default None
+        If provided, use these as the columns-types dictionary for the target table.
+        Default will infer from the database context.
+
     identity_insert: bool, default False
         If `True`, include `SET IDENTITY_INSERT` queries before and after the update queries.
         Only applies for MSSQL upserts.
@@ -1603,14 +1682,14 @@ def get_update_queries(
         flavor=flavor,
         schema=schema,
         debug=debug,
-    )
+    ) if not target_cols_types else target_cols_types
     patch_table_columns = get_table_cols_types(
         patch,
         connectable,
         flavor=flavor,
         schema=patch_schema,
         debug=debug,
-    )
+    ) if not patch_cols_types else patch_cols_types
 
     patch_cols_str = ', '.join(
         [
@@ -1838,10 +1917,9 @@ def get_null_replacement(typ: str, flavor: str) -> str:
     A value which may stand in place of NULL for this type.
     `'None'` is returned if a value cannot be determined.
     """
-    from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES
     if 'geometry' in typ.lower():
-        return '010100000000008058346FCDC100008058346FCDC1'
+        return "'010100000000008058346FCDC100008058346FCDC1'"
     if 'int' in typ.lower() or typ.lower() in ('numeric', 'number'):
         return '-987654321'
     if 'bool' in typ.lower() or typ.lower() == 'bit':
@@ -2113,7 +2191,11 @@ def _get_create_table_query_from_dtypes(
             )
         elif flavor == 'oracle':
             query += f"\n    {col_name} {col_db_type} {auto_increment_str} PRIMARY KEY,"
-        elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
+        elif (
+            flavor in ('timescaledb', 'timescaledb-ha')
+            and datetime_column
+            and datetime_column != primary_key
+        ):
             query += f"\n    {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
         elif flavor == 'mssql':
             query += f"\n    {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
@@ -2126,7 +2208,7 @@ def _get_create_table_query_from_dtypes(
         col_name = sql_item_name(col, schema=None, flavor=flavor)
         query += f"\n    {col_name} {db_type},"
     if (
-        flavor == 'timescaledb'
+        flavor in ('timescaledb', 'timescaledb-ha')
         and datetime_column
         and primary_key
         and datetime_column != primary_key
@@ -2249,7 +2331,11 @@ def _get_create_table_query_from_cte(
                 "ADD PRIMARY KEY ({primary_key_name})"
             ),
         ]
-    elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
+    elif (
+        flavor in ('timescaledb', 'timescaledb-ha')
+        and datetime_column
+        and datetime_column != primary_key
+    ):
         create_table_queries = [
             (
                 "SELECT *\n"

meerschaum/utils/threading.py

@@ -10,6 +10,9 @@ from __future__ import annotations
 from meerschaum.utils.typing import Optional
 
 import threading
+import ctypes
+import signal
+
 Lock = threading.Lock
 RLock = threading.RLock
 Event = threading.Event
@@ -54,6 +57,45 @@ class Thread(threading.Thread):
         """Set the return to the result of the target."""
         self._return = self._target(*self._args, **self._kwargs)
 
+    def send_signal(self, signalnum):
+        """
+        Send a signal to the thread.
+        """
+        if not self.is_alive():
+            return
+
+        if signalnum == signal.SIGINT:
+            self.raise_exception(KeyboardInterrupt())
+        elif signalnum == signal.SIGTERM:
+            self.raise_exception(SystemExit())
+        else:
+            signal.pthread_kill(self.ident, signalnum)
+
+    def raise_exception(self, exc: BaseException):
+        """
+        Raise an exception in the thread.
+
+        This uses a CPython-specific implementation and is not guaranteed to be stable.
+        It may also be deprecated in future Python versions.
+        """
+        if not self.is_alive():
+            return
+
+        if not hasattr(ctypes.pythonapi, 'PyThreadState_SetAsyncExc'):
+            return
+
+        exc_class = exc if isinstance(exc, type) else type(exc)
+
+        ident = self.ident
+        if ident is None:
+            return
+
+        ret = ctypes.pythonapi.PyThreadState_SetAsyncExc(
+            ctypes.c_ulong(ident),
+            ctypes.py_object(exc_class)
+        )
+        if ret > 1:
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(ident, 0)
 
 class Worker(threading.Thread):
     """Wrapper for `threading.Thread` for working with `queue.Queue` objects."""

meerschaum/utils/typing.py

@@ -76,10 +76,7 @@ import collections.abc
 collections.Iterable = collections.abc.Iterable
 
 SuccessTuple = Tuple[bool, str]
-InstanceConnector = Union[
-    'meerschaum.connectors.sql.SQLConnector',
-    'meerschaum.connectors.api.APIConnector'
-]
+InstanceConnector = 'meerschaum.connectors.InstanceConnector'
 PipesDict = Dict[
     str, Dict[                           ### connector_keys : metrics
         str, Dict[                       ### metric_key     : locations

meerschaum/utils/venv/_Venv.py

@@ -35,13 +35,13 @@ class Venv:
 
     def __init__(
         self,
-        venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
+        venv: Union[str, 'meerschaum.core.Plugin', None] = 'mrsm',
         debug: bool = False,
     ) -> None:
         from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
         ### For some weird threading issue,
         ### we can't use `isinstance` here.
-        if 'meerschaum.plugins._Plugin' in str(type(venv)):
+        if '_Plugin' in str(type(venv)):
             self._venv = venv.name
             self._activate = venv.activate_venv
             self._deactivate = venv.deactivate_venv

meerschaum/utils/venv/__init__.py

@@ -8,6 +8,9 @@ Manage virtual environments.
 
 from __future__ import annotations
 
+import sys
+import pathlib
+
 from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple
 from meerschaum.utils.threading import RLock, get_ident
 
@@ -229,7 +232,6 @@ def verify_venv(
     """
     Verify that the virtual environment matches the expected state.
     """
-    import pathlib
     import platform
     import os
     import shutil
@@ -382,11 +384,10 @@ def init_venv(
     import sys
     import platform
     import os
-    import pathlib
     import shutil
     import time
 
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.config._paths import (
         VIRTENV_RESOURCES_PATH,
         VENVS_CACHE_RESOURCES_PATH,
@@ -659,6 +660,7 @@ def venv_exec(
         cmd_list,
         stdout=stdout,
         stderr=stderr,
+        stdin=sys.stdin,
         env=env,
         **group_kwargs
     )
@@ -682,7 +684,7 @@ def venv_target_path(
     venv: Union[str, None],
     allow_nonexistent: bool = False,
     debug: bool = False,
-) -> 'pathlib.Path':
+) -> pathlib.Path:
     """
     Return a virtual environment's site-package path.
 
@@ -702,10 +704,9 @@ def venv_target_path(
     import os
     import sys
     import platform
-    import pathlib
     import site
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
 
     ### Check sys.path for a user-writable site-packages directory.
     if venv is None:
@@ -839,7 +840,6 @@ def get_module_venv(module) -> Union[str, None]:
     -------
     The name of a venv or `None`.
     """
-    import pathlib
     from meerschaum.config.paths import VIRTENV_RESOURCES_PATH
     module_path = pathlib.Path(module.__file__).resolve()
     try:

meerschaum/utils/warnings.py

@@ -100,11 +100,11 @@ def warn(*args, stacklevel=2, stack=True, color: bool = True, **kw) -> None:
 
 
 def exception_with_traceback(
-        message: str,
-        exception_class = Exception, 
-        stacklevel = 1,
-        tb_type = 'single'
-    ):
+    message: str,
+    exception_class = Exception, 
+    stacklevel = 1,
+    tb_type = 'single'
+):
     """Traceback construction help found here:
     https://stackoverflow.com/questions/27138440/how-to-create-a-traceback-object
     """
@@ -165,12 +165,16 @@ def error(
     Raise an exception with supressed traceback.
     """
     from meerschaum.utils.formatting import (
-        CHARSET, ANSI, get_console, fill_ansi, highlight_pipes, _init
+        CHARSET,
+        ANSI,
+        get_console,
+        fill_ansi,
+        highlight_pipes,
+        _init,
     )
     from meerschaum.utils.packages import import_rich, attempt_import
     from meerschaum.config import get_config
-    import types, inspect
-    rich = import_rich()
+    _ = import_rich()
     rich_traceback = attempt_import('rich.traceback')
     error_config = get_config('formatting', 'errors', patch=True)
     message = ' ' + error_config[CHARSET]['icon'] + ' ' + str(message)
@@ -186,23 +190,25 @@ def error(
             exception_class, exception, exception.__traceback__
         )
         rtb = rich_traceback.Traceback(trace)
-    except Exception as e:
+    except Exception:
         trace, rtb = None, None
     if trace is None or rtb is None:
         nopretty = True
     if not nopretty and stack:
         if get_console() is not None:
             get_console().print(rtb)
-    frame = sys._getframe(len(inspect.stack()) - 1)
     if raise_:
         raise color_exception
 
 
 def info(message: str, icon: bool = True, **kw):
     """Print an informative message."""
-    from meerschaum.utils.packages import import_rich, attempt_import
     from meerschaum.utils.formatting import (
-        CHARSET, ANSI, highlight_pipes, fill_ansi, _init
+        CHARSET,
+        ANSI,
+        highlight_pipes,
+        fill_ansi,
+        _init,
     )
     from meerschaum.config import get_config
     info_config = get_config('formatting', 'info', patch=True)
@@ -215,4 +221,4 @@ def info(message: str, icon: bool = True, **kw):
         message = fill_ansi(lines[0], **info_config['ansi']['rich']) + (
             '\n' + '\n'.join(lines[1:]) if len(lines) > 1 else ''
         )
-    print(message)
+    print(message, flush=True)