meerschaum 2.9.4__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

meerschaum/utils/pipes.py

@@ -0,0 +1,181 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define utilities for working with pipes.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Callable
+import re
+import json
+import ast
+import copy
+import uuid
+
+from meerschaum.utils.typing import PipesDict, Optional, Any
+import meerschaum as mrsm
+
+
+def evaluate_pipe_access_chain(access_chain: str, pipe: mrsm.Pipe):
+    """
+    Safely evaluate the access chain on a Pipe.
+    """
+    expr = f"pipe{access_chain}"
+    tree = ast.parse(expr, mode='eval')
+
+    def _eval(node, context):
+        if isinstance(node, ast.Expression):
+            return _eval(node.body, context)
+
+        elif isinstance(node, ast.Name):
+            if node.id == "pipe":
+                return context
+            raise ValueError(f"Unknown variable: {node.id}")
+
+        elif isinstance(node, ast.Attribute):
+            value = _eval(node.value, context)
+            return getattr(value, node.attr)
+
+        elif isinstance(node, ast.Subscript):
+            value = _eval(node.value, context)
+            key = _eval(node.slice, context) if isinstance(node.slice, ast.Index) else _eval(node.slice, context)
+            return value[key]
+
+        elif isinstance(node, ast.Constant):  # Python 3.8+
+            return node.value
+
+        elif isinstance(node, ast.Str):  # Older Python
+            return node.s
+
+        elif isinstance(node, ast.Index):  # Older Python AST style
+            return _eval(node.value, context)
+
+        else:
+            raise TypeError(f"Unsupported AST node: {ast.dump(node)}")
+
+    return _eval(tree, pipe)
+
+
+
+def _evaluate_pipe_access_chain_from_match(pipe_match: re.Match) -> Any:
+    """
+    Helper function to evaluate a pipe from a regex match object.
+    """
+    from meerschaum.utils.warnings import warn
+    from meerschaum.utils.misc import parse_arguments_str
+    from meerschaum.utils.sql import sql_item_name
+    try:
+        args_str = pipe_match.group(1)
+        access_chain = pipe_match.group(2)
+        args, kwargs = parse_arguments_str(args_str)
+        pipe = mrsm.Pipe(*args, **kwargs)
+    except Exception as e:
+        warn(f"Failed to parse pipe from template string:\n{e}")
+        raise e
+
+    if not access_chain:
+        target = pipe.target
+        schema = (
+            pipe.instance_connector.get_pipe_schema(pipe)
+            if hasattr(pipe.instance_connector, 'get_pipe_schema')
+            else None
+        )
+        return (
+            sql_item_name(target, pipe.instance_connector.flavor, schema)
+            if pipe.instance_connector.type == 'sql'
+            else pipe.target
+        )
+
+    return evaluate_pipe_access_chain(access_chain, pipe)
+
+
+def replace_pipes_syntax(text: str) -> Any:
+    """
+    Parse a string containing the `{{ Pipe() }}` syntax.
+    """
+    from meerschaum.utils.warnings import warn
+    from meerschaum.utils.sql import sql_item_name
+    from meerschaum.utils.dtypes import json_serialize_value
+    from meerschaum.utils.misc import parse_arguments_str
+    pattern = r'\{\{\s*(?:mrsm\.)?Pipe\((.*?)\)((?:\.[\w]+|\[[^\]]+\])*)\s*\}\}'
+
+    matches = list(re.finditer(pattern, text))
+    if not matches:
+        return text
+
+    placeholders = {}
+    for match in matches:
+        placeholder = f"__mrsm_pipe_placeholder_{uuid.uuid4().hex}__"
+        placeholders[placeholder] = match
+
+    substituted_text = text
+    for placeholder, match in placeholders.items():
+        substituted_text = substituted_text.replace(match.group(0), placeholder)
+
+    resolved_values = {}
+    for placeholder, match in placeholders.items():
+        try:
+            resolved_values[placeholder] = _evaluate_pipe_access_chain_from_match(match)
+        except Exception as e:
+            warn(f"Failed to resolve pipe syntax '{match.group(0)}': {e}")
+            resolved_values[placeholder] = match.group(0)
+
+    if len(matches) == 1:
+        match = matches[0]
+        placeholder = list(placeholders.keys())[0]
+        if text.strip() == match.group(0):
+            return resolved_values[placeholder]
+
+    final_text = substituted_text
+    for placeholder, value in resolved_values.items():
+        if isinstance(value, (dict, list, bool, int, float)) or value is None:
+            final_text = final_text.replace(placeholder, json.dumps(value, default=json_serialize_value))
+        else:
+            final_text = final_text.replace(placeholder, str(value))
+
+    return final_text
+
+
+def replace_pipes_in_dict(
+    pipes: Optional[PipesDict] = None,
+    func: Callable[[Any], Any] = str,
+    debug: bool = False,
+    **kw
+) -> PipesDict:
+    """
+    Replace the Pipes in a Pipes dict with the result of another function.
+
+    Parameters
+    ----------
+    pipes: Optional[PipesDict], default None
+        The pipes dict to be processed.
+
+    func: Callable[[Any], Any], default str
+        The function to be applied to every pipe.
+        Defaults to the string constructor.
+
+    debug: bool, default False
+        Verbosity toggle.
+    
+
+    Returns
+    -------
+    A dictionary where every pipe is replaced with the output of a function.
+
+    """
+    def change_dict(d : Dict[Any, Any], func : 'function') -> None:
+        for k, v in d.items():
+            if isinstance(v, dict):
+                change_dict(v, func)
+            else:
+                d[k] = func(v)
+
+    if pipes is None:
+        from meerschaum import get_pipes
+        pipes = get_pipes(debug=debug, **kw)
+
+    result = copy.deepcopy(pipes)
+    change_dict(result, func)
+    return result

meerschaum/utils/process.py

@@ -18,7 +18,7 @@ import platform
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Union, Optional, Any, Callable, Dict, Tuple
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 _child_processes = []
 def signal_handler(sig, frame):

meerschaum/utils/prompt.py

@@ -399,6 +399,8 @@ def choose(
             noask = noask,
             **kw
         )
+        if not answer:
+            continue
         ### Split along the delimiter.
         _answers = [answer] if not multiple else [a for a in answer.split(delimiter)]
 
@@ -564,7 +566,7 @@ def check_noask(noask: bool = False) -> bool:
     """
     Flip `noask` to `True` if `MRSM_NOASK` is set.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     NOASK = STATIC_CONFIG['environment']['noask']
     if noask:
         return True

meerschaum/utils/schedule.py

@@ -14,6 +14,7 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import Callable, Any, Optional, List, Dict
 from meerschaum.utils.warnings import warn, error
 
+
 STARTING_KEYWORD: str = 'starting'
 INTERVAL_UNITS: List[str] = ['months', 'weeks', 'days', 'hours', 'minutes', 'seconds', 'years']
 FREQUENCY_ALIASES: Dict[str, str] = {

meerschaum/utils/sql.py

@@ -41,13 +41,33 @@ version_queries = {
 }
 SKIP_IF_EXISTS_FLAVORS = {'mssql', 'oracle'}
 DROP_IF_EXISTS_FLAVORS = {
-    'timescaledb', 'postgresql', 'postgis', 'citus', 'mssql', 'mysql', 'mariadb', 'sqlite',
+    'timescaledb',
+    'timescaledb-ha',
+    'postgresql',
+    'postgis',
+    'citus',
+    'mssql',
+    'mysql',
+    'mariadb',
+    'sqlite',
 }
 DROP_INDEX_IF_EXISTS_FLAVORS = {
-    'mssql', 'timescaledb', 'postgresql', 'postgis', 'sqlite', 'citus',
+    'mssql',
+    'timescaledb',
+    'timescaledb-ha',
+    'postgresql',
+    'postgis',
+    'sqlite',
+    'citus',
 }
 SKIP_AUTO_INCREMENT_FLAVORS = {'citus', 'duckdb'}
-COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'postgis', 'citus'}
+COALESCE_UNIQUE_INDEX_FLAVORS = {
+    'timescaledb',
+    'timescaledb-ha',
+    'postgresql',
+    'postgis',
+    'citus',
+}
 UPDATE_QUERIES = {
     'default': """
         UPDATE {target_table_name} AS f
@@ -67,6 +87,12 @@ UPDATE_QUERIES = {
         FROM {patch_table_name}
         ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
+    'timescaledb-ha-upsert': """
+        INSERT INTO {target_table_name} ({patch_cols_str})
+        SELECT {patch_cols_str}
+        FROM {patch_table_name}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
+    """,
     'postgresql-upsert': """
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
@@ -286,6 +312,7 @@ columns_types_queries = {
 }
 hypertable_queries = {
     'timescaledb': 'SELECT hypertable_size(\'{table_name}\')',
+    'timescaledb-ha': 'SELECT hypertable_size(\'{table_name}\')',
     'citus': 'SELECT citus_table_size(\'{table_name}\')',
 }
 columns_indices_queries = {
@@ -483,49 +510,83 @@ reset_autoincrement_queries: Dict[str, Union[str, List[str]]] = {
     ),
 }
 table_wrappers = {
-    'default'    : ('"', '"'),
-    'timescaledb': ('"', '"'),
-    'citus'      : ('"', '"'),
-    'duckdb'     : ('"', '"'),
-    'postgresql' : ('"', '"'),
-    'postgis'    : ('"', '"'),
-    'sqlite'     : ('"', '"'),
-    'mysql'      : ('`', '`'),
-    'mariadb'    : ('`', '`'),
-    'mssql'      : ('[', ']'),
-    'cockroachdb': ('"', '"'),
-    'oracle'     : ('"', '"'),
+    'default'       : ('"', '"'),
+    'timescaledb'   : ('"', '"'),
+    'timescaledb-ha': ('"', '"'),
+    'citus'         : ('"', '"'),
+    'duckdb'        : ('"', '"'),
+    'postgresql'    : ('"', '"'),
+    'postgis'       : ('"', '"'),
+    'sqlite'        : ('"', '"'),
+    'mysql'         : ('`', '`'),
+    'mariadb'       : ('`', '`'),
+    'mssql'         : ('[', ']'),
+    'cockroachdb'   : ('"', '"'),
+    'oracle'        : ('"', '"'),
 }
 max_name_lens = {
-    'default'    : 64,
-    'mssql'      : 128,
-    'oracle'     : 30,
-    'postgresql' : 64,
-    'postgis'    : 64,
-    'timescaledb': 64,
-    'citus'      : 64,
-    'cockroachdb': 64,
-    'sqlite'     : 1024, ### Probably more, but 1024 seems more than reasonable.
-    'mysql'      : 64,
-    'mariadb'    : 64,
+    'default'       : 64,
+    'mssql'         : 128,
+    'oracle'        : 30,
+    'postgresql'    : 64,
+    'postgis'       : 64,
+    'timescaledb'   : 64,
+    'timescaledb-ha': 64,
+    'citus'         : 64,
+    'cockroachdb'   : 64,
+    'sqlite'        : 1024, ### Probably more, but 1024 seems more than reasonable.
+    'mysql'         : 64,
+    'mariadb'       : 64,
+}
+json_flavors = {
+    'postgresql',
+    'postgis',
+    'timescaledb',
+    'timescaledb-ha',
+    'citus',
+    'cockroachdb',
+}
+NO_SCHEMA_FLAVORS = {
+    'oracle',
+    'sqlite',
+    'mysql',
+    'mariadb',
+    'duckdb',
 }
-json_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'cockroachdb'}
-NO_SCHEMA_FLAVORS = {'oracle', 'sqlite', 'mysql', 'mariadb', 'duckdb'}
 DEFAULT_SCHEMA_FLAVORS = {
     'postgresql': 'public',
     'postgis': 'public',
     'timescaledb': 'public',
+    'timescaledb-ha': 'public',
     'citus': 'public',
     'cockroachdb': 'public',
     'mysql': 'mysql',
     'mariadb': 'mysql',
     'mssql': 'dbo',
 }
-OMIT_NULLSFIRST_FLAVORS = {'mariadb', 'mysql', 'mssql'}
+OMIT_NULLSFIRST_FLAVORS = {
+    'mariadb',
+    'mysql',
+    'mssql',
+}
 
-SINGLE_ALTER_TABLE_FLAVORS = {'duckdb', 'sqlite', 'mssql', 'oracle'}
-NO_CTE_FLAVORS = {'mysql', 'mariadb'}
-NO_SELECT_INTO_FLAVORS = {'sqlite', 'oracle', 'mysql', 'mariadb', 'duckdb'}
+SINGLE_ALTER_TABLE_FLAVORS = {
+    'duckdb',
+    'sqlite',
+    'mssql',
+    'oracle',
+}
+NO_CTE_FLAVORS = {
+    'mysql',
+    'mariadb',
+}
+NO_SELECT_INTO_FLAVORS = {
+    'sqlite',
+    'oracle',
+    'mysql',
+    'mariadb',
+    'duckdb',
+}
 
 
 def clean(substring: str) -> None:
@@ -560,6 +621,7 @@ def dateadd_str(
         - `'postgresql'`
         - `'postgis'`
         - `'timescaledb'`
+        - `'timescaledb-ha'`
         - `'citus'`
         - `'cockroachdb'`
         - `'duckdb'`
@@ -663,7 +725,14 @@ def dateadd_str(
     )
 
     da = ""
-    if flavor in ('postgresql', 'postgis', 'timescaledb', 'cockroachdb', 'citus'):
+    if flavor in (
+        'postgresql',
+        'postgis',
+        'timescaledb',
+        'timescaledb-ha',
+        'cockroachdb',
+        'citus',
+    ):
         begin = (
             f"CAST({begin} AS {db_type})" if begin != 'now'
             else f"CAST(NOW() AT TIME ZONE 'utc' AS {db_type})"
@@ -969,7 +1038,7 @@ def build_where(
     ```
     """
     import json
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
     from meerschaum.utils.dtypes import value_is_null, none_if_null
     negation_prefix = STATIC_CONFIG['system']['fetch_pipes_keys']['negation_prefix']
@@ -1838,10 +1907,9 @@ def get_null_replacement(typ: str, flavor: str) -> str:
     A value which may stand in place of NULL for this type.
     `'None'` is returned if a value cannot be determined.
     """
-    from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES
     if 'geometry' in typ.lower():
-        return '010100000000008058346FCDC100008058346FCDC1'
+        return "'010100000000008058346FCDC100008058346FCDC1'"
     if 'int' in typ.lower() or typ.lower() in ('numeric', 'number'):
         return '-987654321'
     if 'bool' in typ.lower() or typ.lower() == 'bit':
@@ -2113,7 +2181,11 @@ def _get_create_table_query_from_dtypes(
             )
         elif flavor == 'oracle':
             query += f"\n    {col_name} {col_db_type} {auto_increment_str} PRIMARY KEY,"
-        elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
+        elif (
+            flavor in ('timescaledb', 'timescaledb-ha')
+            and datetime_column
+            and datetime_column != primary_key
+        ):
             query += f"\n    {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
         elif flavor == 'mssql':
             query += f"\n    {col_name} {col_db_type}{auto_increment_str} NOT NULL,"
@@ -2126,7 +2198,7 @@ def _get_create_table_query_from_dtypes(
         col_name = sql_item_name(col, schema=None, flavor=flavor)
         query += f"\n    {col_name} {db_type},"
     if (
-        flavor == 'timescaledb'
+        flavor in ('timescaledb', 'timescaledb-ha')
         and datetime_column
         and primary_key
         and datetime_column != primary_key
@@ -2249,7 +2321,11 @@ def _get_create_table_query_from_cte(
                 "ADD PRIMARY KEY ({primary_key_name})"
             ),
         ]
-    elif flavor == 'timescaledb' and datetime_column and datetime_column != primary_key:
+    elif (
+        flavor in ('timescaledb', 'timescaledb-ha')
+        and datetime_column
+        and datetime_column != primary_key
+    ):
         create_table_queries = [
             (
                 "SELECT *\n"

meerschaum/utils/typing.py

@@ -76,10 +76,7 @@ import collections.abc
 collections.Iterable = collections.abc.Iterable
 
 SuccessTuple = Tuple[bool, str]
-InstanceConnector = Union[
-    'meerschaum.connectors.sql.SQLConnector',
-    'meerschaum.connectors.api.APIConnector'
-]
+InstanceConnector = 'meerschaum.connectors.InstanceConnector'
 PipesDict = Dict[
     str, Dict[                           ### connector_keys : metrics
         str, Dict[                       ### metric_key     : locations

meerschaum/utils/venv/_Venv.py

@@ -35,13 +35,13 @@ class Venv:
 
     def __init__(
         self,
-        venv: Union[str, 'meerschaum.plugins.Plugin', None] = 'mrsm',
+        venv: Union[str, 'meerschaum.core.Plugin', None] = 'mrsm',
         debug: bool = False,
     ) -> None:
         from meerschaum.utils.venv import activate_venv, deactivate_venv, active_venvs
         ### For some weird threading issue,
         ### we can't use `isinstance` here.
-        if 'meerschaum.plugins._Plugin' in str(type(venv)):
+        if '_Plugin' in str(type(venv)):
             self._venv = venv.name
             self._activate = venv.activate_venv
             self._deactivate = venv.deactivate_venv

meerschaum/utils/venv/__init__.py

@@ -8,6 +8,8 @@ Manage virtual environments.
 
 from __future__ import annotations
 
+import pathlib
+
 from meerschaum.utils.typing import Optional, Union, Dict, List, Tuple
 from meerschaum.utils.threading import RLock, get_ident
 
@@ -229,7 +231,6 @@ def verify_venv(
     """
     Verify that the virtual environment matches the expected state.
     """
-    import pathlib
     import platform
     import os
     import shutil
@@ -382,11 +383,10 @@ def init_venv(
     import sys
     import platform
     import os
-    import pathlib
     import shutil
     import time
 
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.config._paths import (
         VIRTENV_RESOURCES_PATH,
         VENVS_CACHE_RESOURCES_PATH,
@@ -682,7 +682,7 @@ def venv_target_path(
     venv: Union[str, None],
     allow_nonexistent: bool = False,
     debug: bool = False,
-) -> 'pathlib.Path':
+) -> pathlib.Path:
     """
     Return a virtual environment's site-package path.
 
@@ -702,10 +702,9 @@ def venv_target_path(
     import os
     import sys
     import platform
-    import pathlib
     import site
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
 
     ### Check sys.path for a user-writable site-packages directory.
     if venv is None:
@@ -839,7 +838,6 @@ def get_module_venv(module) -> Union[str, None]:
     -------
     The name of a venv or `None`.
     """
-    import pathlib
     from meerschaum.config.paths import VIRTENV_RESOURCES_PATH
     module_path = pathlib.Path(module.__file__).resolve()
     try: