meerschaum 2.9.5__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/utils/packages/__init__.py

@@ -75,7 +75,9 @@ def get_module_path(
 
     venv_target_candidate_paths = [vtp]
     if venv is None:
-        site_user_packages_dirs = [pathlib.Path(site.getusersitepackages())]
+        site_user_packages_dirs = [
+            pathlib.Path(site.getusersitepackages())
+        ] if not inside_venv() else []
         site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()]
 
         paths_to_add = [
@@ -134,17 +136,17 @@ def get_module_path(
 
 
 def manually_import_module(
-        import_name: str,
-        venv: Optional[str] = 'mrsm',
-        check_update: bool = True,
-        check_pypi: bool = False,
-        install: bool = True,
-        split: bool = True,
-        warn: bool = True,
-        color: bool = True,
-        debug: bool = False,
-        use_sys_modules: bool = True,
-    ) -> Union['ModuleType', None]:
+    import_name: str,
+    venv: Optional[str] = 'mrsm',
+    check_update: bool = True,
+    check_pypi: bool = False,
+    install: bool = True,
+    split: bool = True,
+    warn: bool = True,
+    color: bool = True,
+    debug: bool = False,
+    use_sys_modules: bool = True,
+) -> Union['ModuleType', None]:
     """
     Manually import a module from a virtual environment (or the base environment).
 
@@ -755,7 +757,7 @@ def get_pip(
     import subprocess
     from meerschaum.utils.misc import wget
     from meerschaum.config._paths import CACHE_RESOURCES_PATH
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     url = STATIC_CONFIG['system']['urls']['get-pip.py']
     dest = CACHE_RESOURCES_PATH / 'get-pip.py'
     try:
@@ -837,7 +839,7 @@ def pip_install(
 
     """
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
     if args is None:
         args = ['--upgrade'] if not _uninstall else []
@@ -1177,7 +1179,8 @@ def run_python_package(
     Either a return code integer or a `subprocess.Popen` object
     (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
     """
-    import sys, platform
+    import sys
+    import platform
     import subprocess
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
     from meerschaum.utils.process import run_process
@@ -1204,7 +1207,7 @@ def run_python_package(
             capture_output=capture_output,
             **kw
         )
-    except Exception as e:
+    except Exception:
         msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
         warn(msg, color=False)
         stdout, stderr = (
@@ -1216,6 +1219,7 @@ def run_python_package(
             command,
             stdout=stdout,
             stderr=stderr,
+            stdin=sys.stdin,
             env=env_dict,
         )
         to_return = proc if as_proc else proc.wait()
@@ -1500,15 +1504,12 @@ def import_pandas(
 def import_rich(
     lazy: bool = True,
     debug: bool = False,
-    **kw : Any
+    **kw: Any
 ) -> 'ModuleType':
     """
     Quality of life function for importing `rich`.
     """
     from meerschaum.utils.formatting import ANSI, UNICODE
-    if not ANSI and not UNICODE:
-        return None
-
     ## need typing_extensions for `from rich import box`
     typing_extensions = attempt_import(
         'typing_extensions', lazy=False, debug=debug
@@ -1773,6 +1774,10 @@ def is_installed(
     allow_outside_venv: bool, default True
         If `True`, search outside of the specified virtual environment
         if the package cannot be found.
+
+    Returns
+    -------
+    A bool indicating whether a package may be imported.
     """
     if debug:
         from meerschaum.utils.debug import dprint
@@ -1864,26 +1869,6 @@ def ensure_readline() -> 'ModuleType':
     sys.modules['readline'] = readline
     return readline
 
-_pkg_resources_get_distribution = None
-_custom_distributions = {}
-def _monkey_patch_get_distribution(_dist: str, _version: str) -> None:
-    """
-    Monkey patch `pkg_resources.get_distribution` to allow for importing `flask_compress`.
-    """
-    import pkg_resources
-    from collections import namedtuple
-    global _pkg_resources_get_distribution
-    with _locks['_pkg_resources_get_distribution']:
-        _pkg_resources_get_distribution = pkg_resources.get_distribution
-    _custom_distributions[_dist] = _version
-    _Dist = namedtuple('_Dist', ['version'])
-    def _get_distribution(dist):
-        """Hack for flask-compress."""
-        if dist in _custom_distributions:
-            return _Dist(_custom_distributions[dist])
-        return _pkg_resources_get_distribution(dist)
-    pkg_resources.get_distribution = _get_distribution
-
 
 def _get_pip_os_env(color: bool = True):
     """

meerschaum/utils/packages/_packages.py

@@ -13,8 +13,7 @@ packages dictionary is structured in the following schema:
     }
 """
 
-from __future__ import annotations
-from meerschaum.utils.typing import Dict
+from typing import Dict
 
 _MRSM_PACKAGE_ARCHIVES_PREFIX: str = "https://meerschaum.io/files/archives/wheels/"
 
@@ -54,23 +53,25 @@ packages: Dict[str, Dict[str, str]] = {
         'virtualenv'                 : 'virtualenv>=20.1.0',
         'attrs'                      : 'attrs>=24.2.0',
         'uv'                         : 'uv>=0.2.11',
+        'pydantic'                   : 'pydantic>=2.11.7',
+        'annotated-types'            : 'annotated-types>=0.7.0',
     },
     '_internal'                      : {
         'apscheduler'                : (
                                        f"{_MRSM_PACKAGE_ARCHIVES_PREFIX}"
-                                       "APScheduler-4.0.0a5.post87+mrsm-py3-none-any.whl>=4.0.0a5"
+                                       "apscheduler-4.0.0a6.post8+mrsm-py3-none-any.whl>=4.0.0a6"
         ),
-        'dataclass_wizard'           : 'dataclass-wizard>=0.28.0',
+        'dataclass_wizard'           : 'dataclass-wizard>=0.35.0',
     },
     'jobs': {
-        'dill'                       : 'dill>=0.3.3',
-        'daemon'                     : 'python-daemon>=0.2.3',
-        'watchfiles'                 : 'watchfiles>=0.21.0',
-        'psutil'                     : 'psutil>=5.8.0',
+        'dill'                       : 'dill>=0.4.0',
+        'daemon'                     : 'python-daemon>=3.1.2',
+        'watchfiles'                 : 'watchfiles>=1.1.0',
+        'psutil'                     : 'psutil>=7.0.0',
     },
     'drivers': {
         'cryptography'               : 'cryptography>=38.0.1',
-        'psycopg'                    : 'psycopg[binary]>=3.2.1',
+        'psycopg'                    : 'psycopg[binary]>=3.2.9',
         'pymysql'                    : 'PyMySQL>=0.9.0',
         'aiomysql'                   : 'aiomysql>=0.0.21',
         'sqlalchemy_cockroachdb'     : 'sqlalchemy-cockroachdb>=2.0.0',
@@ -122,7 +123,7 @@ packages: Dict[str, Dict[str, str]] = {
         'mkdocs_section_index'       : 'mkdocs-section-index>=0.3.3',
         'mkdocs_linkcheck'           : 'mkdocs-linkcheck>=1.0.6',
         'mkdocs_redirects'           : 'mkdocs-redirects>=1.0.4',
-        'jinja2'                     : 'jinja2==3.0.3',
+        'jinja2'                     : 'jinja2>=3.1.6',
     },
     'gui': {
         'webview'                    : 'pywebview>=3.6.3',
@@ -137,43 +138,44 @@ packages: Dict[str, Dict[str, str]] = {
     },
 }
 packages['sql'] = {
-    'numpy'                          : 'numpy>=1.18.5',
-    'pandas'                         : 'pandas[parquet]>=2.0.1',
-    'pyarrow'                        : 'pyarrow>=16.1.0',
+    'numpy'                          : 'numpy>=2.3.1',
+    'pandas'                         : 'pandas[parquet]>=2.3.1',
+    'pyarrow'                        : 'pyarrow>=20.0.0',
     'dask'                           : 'dask[complete]>=2024.12.1',
     'partd'                          : 'partd>=1.4.2',
     'pytz'                           : 'pytz',
-    'joblib'                         : 'joblib>=0.17.0',
-    'sqlalchemy'                     : 'SQLAlchemy>=2.0.5',
+    'joblib'                         : 'joblib>=1.5.1',
+    'sqlalchemy'                     : 'SQLAlchemy>=2.0.41',
     'geoalchemy'                     : 'GeoAlchemy2>=0.17.1',
-    'databases'                      : 'databases>=0.4.0',
-    'aiosqlite'                      : 'aiosqlite>=0.16.0',
-    'asyncpg'                        : 'asyncpg>=0.21.0',
+    'databases'                      : 'databases>=0.9.0',
+    'aiosqlite'                      : 'aiosqlite>=0.21.0',
+    'asyncpg'                        : 'asyncpg>=0.30.0',
 }
 packages['sql'].update(packages['drivers'])
 packages['sql'].update(packages['core'])
 packages['sql'].update(packages['gis'])
 packages['dash'] = {
-    'flask_compress'                 : 'Flask-Compress>=1.10.1',
-    'dash'                           : 'dash>=2.6.2',
+    'flask_compress'                 : 'Flask-Compress>=1.17.0',
+    'dash'                           : 'dash>=3.1.1',
     'dash_bootstrap_components'      : 'dash-bootstrap-components>=1.7.1',
     'dash_ace'                       : 'dash-ace>=0.2.1',
-    'dash_extensions'                : 'dash-extensions>=1.0.4',
-    'dash_daq'                       : 'dash-daq>=0.5.0',
-    'terminado'                      : 'terminado>=0.12.1',
-    'tornado'                        : 'tornado>=6.1.0',
+    'dash_extensions'                : 'dash-extensions>=2.0.4',
+    'dash_daq'                       : 'dash-daq>=0.6.0',
+    'terminado'                      : 'terminado>=0.18.1',
+    'tornado'                        : 'tornado>=6.5.1',
 }
 packages['api'] = {
-    'uvicorn'                        : 'uvicorn[standard]>=0.29.0',
-    'gunicorn'                       : 'gunicorn>=22.0.0',
-    'dotenv'                         : 'python-dotenv>=0.20.0',
-    'websockets'                     : 'websockets>=11.0.3',
-    'fastapi'                        : 'fastapi>=0.111.0',
-    'fastapi_login'                  : 'fastapi-login>=1.7.2',
-    'multipart'                      : 'python-multipart>=0.0.9',
-    'httpx'                          : 'httpx>=0.27.2',
-    'httpcore'                       : 'httpcore>=1.0.6',
-    'valkey'                         : 'valkey>=6.0.0',
+    'uvicorn'                        : 'uvicorn[standard]>=0.35.0',
+    'gunicorn'                       : 'gunicorn>=23.0.0',
+    'dotenv'                         : 'python-dotenv>=1.1.1',
+    'websockets'                     : 'websockets>=15.0.1',
+    'fastapi'                        : 'fastapi>=0.116.0',
+    'fastapi_login'                  : 'fastapi-login>=1.10.3',
+    'multipart'                      : 'python-multipart>=0.0.20',
+    'httpx'                          : 'httpx>=0.28.1',
+    'httpcore'                       : 'httpcore>=1.0.9',
+    'valkey'                         : 'valkey>=6.1.0',
+    'jose'                           : 'python-jose>=3.5.0',
 }
 packages['api'].update(packages['sql'])
 packages['api'].update(packages['formatting'])
@@ -216,3 +218,9 @@ for group, import_names in packages.items():
     for import_name, install_name in import_names.items():
         _full[import_name] = install_name
 packages['full'] = _full
+
+extras = {
+    group: list(import_names_install_names_map.values())
+    for group, import_names_install_names_map in packages.items()
+    if not group.startswith('_')
+}

meerschaum/utils/pipes.py

@@ -0,0 +1,213 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define utilities for working with pipes.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Callable
+import re
+import json
+import ast
+import copy
+import uuid
+
+from meerschaum.utils.typing import PipesDict, Optional
+import meerschaum as mrsm
+
+
+def evaluate_pipe_access_chain(access_chain: str, pipe: mrsm.Pipe):
+    """
+    Safely evaluate the access chain on a Pipe.
+    """
+    expr = f"pipe{access_chain}"
+    tree = ast.parse(expr, mode='eval')
+
+    def _eval(node, context):
+        if isinstance(node, ast.Expression):
+            return _eval(node.body, context)
+
+        elif isinstance(node, ast.Name):
+            if node.id == "pipe":
+                return context
+            raise ValueError(f"Unknown variable: {node.id}")
+
+        elif isinstance(node, ast.Attribute):
+            value = _eval(node.value, context)
+            return getattr(value, node.attr)
+
+        elif isinstance(node, ast.Subscript):
+            value = _eval(node.value, context)
+            key = _eval(node.slice, context) if isinstance(node.slice, ast.Index) else _eval(node.slice, context)
+            return value[key]
+
+        elif isinstance(node, ast.Constant):  # Python 3.8+
+            return node.value
+
+        elif isinstance(node, ast.Str):  # Older Python
+            return node.s
+
+        elif isinstance(node, ast.Index):  # Older Python AST style
+            return _eval(node.value, context)
+
+        else:
+            raise TypeError(f"Unsupported AST node: {ast.dump(node)}")
+
+    return _eval(tree, pipe)
+
+
+
+def _evaluate_pipe_access_chain_from_match(pipe_match: re.Match) -> Any:
+    """
+    Helper function to evaluate a pipe from a regex match object.
+    """
+    from meerschaum.utils.warnings import warn
+    from meerschaum.utils.misc import parse_arguments_str
+    from meerschaum.utils.sql import sql_item_name
+    try:
+        args_str = pipe_match.group(1)
+        access_chain = pipe_match.group(2)
+        args, kwargs = parse_arguments_str(args_str)
+        pipe = mrsm.Pipe(*args, **kwargs)
+    except Exception as e:
+        warn(f"Failed to parse pipe from template string:\n{e}")
+        raise e
+
+    if not access_chain:
+        target = pipe.target
+        schema = (
+            pipe.instance_connector.get_pipe_schema(pipe)
+            if hasattr(pipe.instance_connector, 'get_pipe_schema')
+            else None
+        )
+        return (
+            sql_item_name(target, pipe.instance_connector.flavor, schema)
+            if pipe.instance_connector.type == 'sql'
+            else pipe.target
+        )
+
+    return evaluate_pipe_access_chain(access_chain, pipe)
+
+
+def replace_pipes_syntax(text: str) -> Any:
+    """
+    Parse a string containing the `{{ Pipe() }}` syntax.
+    """
+    from meerschaum.utils.warnings import warn
+    from meerschaum.utils.dtypes import json_serialize_value
+    pattern = r'\{\{\s*(?:mrsm\.)?Pipe\((.*?)\)((?:\.[\w]+|\[[^\]]+\])*)\s*\}\}'
+
+    matches = list(re.finditer(pattern, text))
+    if not matches:
+        return text
+
+    placeholders = {}
+    for match in matches:
+        placeholder = f"__mrsm_pipe_placeholder_{uuid.uuid4().hex}__"
+        placeholders[placeholder] = match
+
+    substituted_text = text
+    for placeholder, match in placeholders.items():
+        substituted_text = substituted_text.replace(match.group(0), placeholder)
+
+    resolved_values = {}
+    for placeholder, match in placeholders.items():
+        try:
+            resolved_values[placeholder] = _evaluate_pipe_access_chain_from_match(match)
+        except Exception as e:
+            warn(f"Failed to resolve pipe syntax '{match.group(0)}': {e}")
+            resolved_values[placeholder] = match.group(0)
+
+    if len(matches) == 1:
+        match = matches[0]
+        placeholder = list(placeholders.keys())[0]
+        if text.strip() == match.group(0):
+            return resolved_values[placeholder]
+
+    final_text = substituted_text
+    for placeholder, value in resolved_values.items():
+        if isinstance(value, (dict, list, bool, int, float)) or value is None:
+            final_text = final_text.replace(placeholder, json.dumps(value, default=json_serialize_value))
+        else:
+            final_text = final_text.replace(placeholder, str(value))
+
+    return final_text
+
+
+def replace_pipes_in_dict(
+    pipes: Optional[PipesDict] = None,
+    func: Callable[[Any], Any] = str,
+    debug: bool = False,
+    **kw
+) -> PipesDict:
+    """
+    Replace the Pipes in a Pipes dict with the result of another function.
+
+    Parameters
+    ----------
+    pipes: Optional[PipesDict], default None
+        The pipes dict to be processed.
+
+    func: Callable[[Any], Any], default str
+        The function to be applied to every pipe.
+        Defaults to the string constructor.
+
+    debug: bool, default False
+        Verbosity toggle.
+
+    Returns
+    -------
+    A dictionary where every pipe is replaced with the output of a function.
+
+    """
+    def change_dict(d: Dict[Any, Any]) -> None:
+        for k, v in d.items():
+            if isinstance(v, dict):
+                change_dict(v)
+            elif isinstance(v, list):
+                d[k] = [func(i) for i in v]
+            elif isinstance(v, tuple):
+                d[k] = tuple([func(i) for i in v])
+            else:
+                d[k] = func(v)
+
+    if pipes is None:
+        from meerschaum import get_pipes
+        pipes = get_pipes(debug=debug, **kw)
+
+    result = copy.deepcopy(pipes)
+    change_dict(result)
+    return result
+
+
+def is_pipe_registered(
+    pipe: mrsm.Pipe,
+    pipes: PipesDict,
+    debug: bool = False
+) -> bool:
+    """
+    Check if a Pipe is inside the pipes dictionary.
+
+    Parameters
+    ----------
+    pipe: meerschaum.Pipe
+        The pipe to see if it's in the dictionary.
+
+    pipes: PipesDict
+        The dictionary to search inside.
+
+    debug: bool, default False
+        Verbosity toggle.
+
+    Returns
+    -------
+    A bool indicating whether the pipe is inside the dictionary.
+    """
+    from meerschaum.utils.debug import dprint
+    ck, mk, lk = pipe.connector_keys, pipe.metric_key, pipe.location_key
+    if debug:
+        dprint(f'{ck}, {mk}, {lk}')
+        dprint(f'{pipe}, {pipes}')
+    return ck in pipes and mk in pipes[ck] and lk in pipes[ck][mk]

meerschaum/utils/process.py

@@ -18,7 +18,7 @@ import platform
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Union, Optional, Any, Callable, Dict, Tuple
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 _child_processes = []
 def signal_handler(sig, frame):
@@ -178,7 +178,7 @@ def run_process(
             # make us tty's foreground again
             try:
                 os.tcsetpgrp(sys.stdin.fileno(), old_pgrp)
-            except Exception as e:
+            except Exception:
                 pass
             # now restore the handler
             signal.signal(signal.SIGTTOU, hdlr)