meerschaum 2.9.5__py3-none-any.whl → 3.0.0rc2__py3-none-any.whl

meerschaum/utils/packages/__init__.py

@@ -75,7 +75,9 @@ def get_module_path(
 
     venv_target_candidate_paths = [vtp]
     if venv is None:
-        site_user_packages_dirs = [pathlib.Path(site.getusersitepackages())]
+        site_user_packages_dirs = [
+            pathlib.Path(site.getusersitepackages())
+        ] if not inside_venv() else []
         site_packages_dirs = [pathlib.Path(path) for path in site.getsitepackages()]
 
         paths_to_add = [
@@ -755,7 +757,7 @@ def get_pip(
     import subprocess
     from meerschaum.utils.misc import wget
     from meerschaum.config._paths import CACHE_RESOURCES_PATH
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     url = STATIC_CONFIG['system']['urls']['get-pip.py']
     dest = CACHE_RESOURCES_PATH / 'get-pip.py'
     try:
@@ -837,7 +839,7 @@ def pip_install(
 
     """
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     from meerschaum.utils.warnings import warn
     if args is None:
         args = ['--upgrade'] if not _uninstall else []
@@ -1500,15 +1502,12 @@ def import_pandas(
 def import_rich(
     lazy: bool = True,
     debug: bool = False,
-    **kw : Any
+    **kw: Any
 ) -> 'ModuleType':
     """
     Quality of life function for importing `rich`.
     """
     from meerschaum.utils.formatting import ANSI, UNICODE
-    if not ANSI and not UNICODE:
-        return None
-
     ## need typing_extensions for `from rich import box`
     typing_extensions = attempt_import(
         'typing_extensions', lazy=False, debug=debug
@@ -1773,6 +1772,10 @@ def is_installed(
     allow_outside_venv: bool, default True
         If `True`, search outside of the specified virtual environment
         if the package cannot be found.
+
+    Returns
+    -------
+    A bool indicating whether a package may be imported.
     """
     if debug:
         from meerschaum.utils.debug import dprint
@@ -1864,26 +1867,6 @@ def ensure_readline() -> 'ModuleType':
     sys.modules['readline'] = readline
     return readline
 
-_pkg_resources_get_distribution = None
-_custom_distributions = {}
-def _monkey_patch_get_distribution(_dist: str, _version: str) -> None:
-    """
-    Monkey patch `pkg_resources.get_distribution` to allow for importing `flask_compress`.
-    """
-    import pkg_resources
-    from collections import namedtuple
-    global _pkg_resources_get_distribution
-    with _locks['_pkg_resources_get_distribution']:
-        _pkg_resources_get_distribution = pkg_resources.get_distribution
-    _custom_distributions[_dist] = _version
-    _Dist = namedtuple('_Dist', ['version'])
-    def _get_distribution(dist):
-        """Hack for flask-compress."""
-        if dist in _custom_distributions:
-            return _Dist(_custom_distributions[dist])
-        return _pkg_resources_get_distribution(dist)
-    pkg_resources.get_distribution = _get_distribution
-
 
 def _get_pip_os_env(color: bool = True):
     """

meerschaum/utils/packages/_packages.py

@@ -13,8 +13,7 @@ packages dictionary is structured in the following schema:
     }
 """
 
-from __future__ import annotations
-from meerschaum.utils.typing import Dict
+from typing import Dict
 
 _MRSM_PACKAGE_ARCHIVES_PREFIX: str = "https://meerschaum.io/files/archives/wheels/"
 
@@ -54,23 +53,25 @@ packages: Dict[str, Dict[str, str]] = {
         'virtualenv'                 : 'virtualenv>=20.1.0',
         'attrs'                      : 'attrs>=24.2.0',
         'uv'                         : 'uv>=0.2.11',
+        'pydantic'                   : 'pydantic>=2.11.7',
+        'annotated-types'            : 'annotated-types>=0.7.0',
     },
     '_internal'                      : {
         'apscheduler'                : (
                                        f"{_MRSM_PACKAGE_ARCHIVES_PREFIX}"
-                                       "APScheduler-4.0.0a5.post87+mrsm-py3-none-any.whl>=4.0.0a5"
+                                       "apscheduler-4.0.0a6.post8+mrsm-py3-none-any.whl>=4.0.0a6"
         ),
-        'dataclass_wizard'           : 'dataclass-wizard>=0.28.0',
+        'dataclass_wizard'           : 'dataclass-wizard>=0.35.0',
     },
     'jobs': {
-        'dill'                       : 'dill>=0.3.3',
-        'daemon'                     : 'python-daemon>=0.2.3',
-        'watchfiles'                 : 'watchfiles>=0.21.0',
-        'psutil'                     : 'psutil>=5.8.0',
+        'dill'                       : 'dill>=0.4.0',
+        'daemon'                     : 'python-daemon>=3.1.2',
+        'watchfiles'                 : 'watchfiles>=1.1.0',
+        'psutil'                     : 'psutil>=7.0.0',
     },
     'drivers': {
         'cryptography'               : 'cryptography>=38.0.1',
-        'psycopg'                    : 'psycopg[binary]>=3.2.1',
+        'psycopg'                    : 'psycopg[binary]>=3.2.9',
         'pymysql'                    : 'PyMySQL>=0.9.0',
         'aiomysql'                   : 'aiomysql>=0.0.21',
         'sqlalchemy_cockroachdb'     : 'sqlalchemy-cockroachdb>=2.0.0',
@@ -122,7 +123,7 @@ packages: Dict[str, Dict[str, str]] = {
         'mkdocs_section_index'       : 'mkdocs-section-index>=0.3.3',
         'mkdocs_linkcheck'           : 'mkdocs-linkcheck>=1.0.6',
         'mkdocs_redirects'           : 'mkdocs-redirects>=1.0.4',
-        'jinja2'                     : 'jinja2==3.0.3',
+        'jinja2'                     : 'jinja2>=3.1.6',
     },
     'gui': {
         'webview'                    : 'pywebview>=3.6.3',
@@ -137,43 +138,43 @@ packages: Dict[str, Dict[str, str]] = {
     },
 }
 packages['sql'] = {
-    'numpy'                          : 'numpy>=1.18.5',
-    'pandas'                         : 'pandas[parquet]>=2.0.1',
-    'pyarrow'                        : 'pyarrow>=16.1.0',
+    'numpy'                          : 'numpy>=2.3.1',
+    'pandas'                         : 'pandas[parquet]>=2.3.1',
+    'pyarrow'                        : 'pyarrow>=20.0.0',
     'dask'                           : 'dask[complete]>=2024.12.1',
     'partd'                          : 'partd>=1.4.2',
     'pytz'                           : 'pytz',
-    'joblib'                         : 'joblib>=0.17.0',
-    'sqlalchemy'                     : 'SQLAlchemy>=2.0.5',
+    'joblib'                         : 'joblib>=1.5.1',
+    'sqlalchemy'                     : 'SQLAlchemy>=2.0.41',
     'geoalchemy'                     : 'GeoAlchemy2>=0.17.1',
-    'databases'                      : 'databases>=0.4.0',
-    'aiosqlite'                      : 'aiosqlite>=0.16.0',
-    'asyncpg'                        : 'asyncpg>=0.21.0',
+    'databases'                      : 'databases>=0.9.0',
+    'aiosqlite'                      : 'aiosqlite>=0.21.0',
+    'asyncpg'                        : 'asyncpg>=0.30.0',
 }
 packages['sql'].update(packages['drivers'])
 packages['sql'].update(packages['core'])
 packages['sql'].update(packages['gis'])
 packages['dash'] = {
-    'flask_compress'                 : 'Flask-Compress>=1.10.1',
-    'dash'                           : 'dash>=2.6.2',
+    'flask_compress'                 : 'Flask-Compress>=1.17.0',
+    'dash'                           : 'dash>=3.1.1',
     'dash_bootstrap_components'      : 'dash-bootstrap-components>=1.7.1',
     'dash_ace'                       : 'dash-ace>=0.2.1',
-    'dash_extensions'                : 'dash-extensions>=1.0.4',
-    'dash_daq'                       : 'dash-daq>=0.5.0',
-    'terminado'                      : 'terminado>=0.12.1',
-    'tornado'                        : 'tornado>=6.1.0',
+    'dash_extensions'                : 'dash-extensions>=2.0.4',
+    'dash_daq'                       : 'dash-daq>=0.6.0',
+    'terminado'                      : 'terminado>=0.18.1',
+    'tornado'                        : 'tornado>=6.5.1',
 }
 packages['api'] = {
-    'uvicorn'                        : 'uvicorn[standard]>=0.29.0',
-    'gunicorn'                       : 'gunicorn>=22.0.0',
-    'dotenv'                         : 'python-dotenv>=0.20.0',
-    'websockets'                     : 'websockets>=11.0.3',
-    'fastapi'                        : 'fastapi>=0.111.0',
-    'fastapi_login'                  : 'fastapi-login>=1.7.2',
-    'multipart'                      : 'python-multipart>=0.0.9',
-    'httpx'                          : 'httpx>=0.27.2',
-    'httpcore'                       : 'httpcore>=1.0.6',
-    'valkey'                         : 'valkey>=6.0.0',
+    'uvicorn'                        : 'uvicorn[standard]>=0.35.0',
+    'gunicorn'                       : 'gunicorn>=23.0.0',
+    'dotenv'                         : 'python-dotenv>=1.1.1',
+    'websockets'                     : 'websockets>=15.0.1',
+    'fastapi'                        : 'fastapi>=0.116.0',
+    'fastapi_login'                  : 'fastapi-login>=1.10.3',
+    'multipart'                      : 'python-multipart>=0.0.20',
+    'httpx'                          : 'httpx>=0.28.1',
+    'httpcore'                       : 'httpcore>=1.0.9',
+    'valkey'                         : 'valkey>=6.1.0',
 }
 packages['api'].update(packages['sql'])
 packages['api'].update(packages['formatting'])
@@ -216,3 +217,9 @@ for group, import_names in packages.items():
     for import_name, install_name in import_names.items():
         _full[import_name] = install_name
 packages['full'] = _full
+
+extras = {
+    group: list(import_names_install_names_map.values())
+    for group, import_names_install_names_map in packages.items()
+    if not group.startswith('_')
+}

meerschaum/utils/pipes.py

@@ -0,0 +1,181 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define utilities for working with pipes.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Callable
+import re
+import json
+import ast
+import copy
+import uuid
+
+from meerschaum.utils.typing import PipesDict, Optional, Any
+import meerschaum as mrsm
+
+
+def evaluate_pipe_access_chain(access_chain: str, pipe: mrsm.Pipe):
+    """
+    Safely evaluate the access chain on a Pipe.
+    """
+    expr = f"pipe{access_chain}"
+    tree = ast.parse(expr, mode='eval')
+
+    def _eval(node, context):
+        if isinstance(node, ast.Expression):
+            return _eval(node.body, context)
+
+        elif isinstance(node, ast.Name):
+            if node.id == "pipe":
+                return context
+            raise ValueError(f"Unknown variable: {node.id}")
+
+        elif isinstance(node, ast.Attribute):
+            value = _eval(node.value, context)
+            return getattr(value, node.attr)
+
+        elif isinstance(node, ast.Subscript):
+            value = _eval(node.value, context)
+            key = _eval(node.slice, context) if isinstance(node.slice, ast.Index) else _eval(node.slice, context)
+            return value[key]
+
+        elif isinstance(node, ast.Constant):  # Python 3.8+
+            return node.value
+
+        elif isinstance(node, ast.Str):  # Older Python
+            return node.s
+
+        elif isinstance(node, ast.Index):  # Older Python AST style
+            return _eval(node.value, context)
+
+        else:
+            raise TypeError(f"Unsupported AST node: {ast.dump(node)}")
+
+    return _eval(tree, pipe)
+
+
+
+def _evaluate_pipe_access_chain_from_match(pipe_match: re.Match) -> Any:
+    """
+    Helper function to evaluate a pipe from a regex match object.
+    """
+    from meerschaum.utils.warnings import warn
+    from meerschaum.utils.misc import parse_arguments_str
+    from meerschaum.utils.sql import sql_item_name
+    try:
+        args_str = pipe_match.group(1)
+        access_chain = pipe_match.group(2)
+        args, kwargs = parse_arguments_str(args_str)
+        pipe = mrsm.Pipe(*args, **kwargs)
+    except Exception as e:
+        warn(f"Failed to parse pipe from template string:\n{e}")
+        raise e
+
+    if not access_chain:
+        target = pipe.target
+        schema = (
+            pipe.instance_connector.get_pipe_schema(pipe)
+            if hasattr(pipe.instance_connector, 'get_pipe_schema')
+            else None
+        )
+        return (
+            sql_item_name(target, pipe.instance_connector.flavor, schema)
+            if pipe.instance_connector.type == 'sql'
+            else pipe.target
+        )
+
+    return evaluate_pipe_access_chain(access_chain, pipe)
+
+
+def replace_pipes_syntax(text: str) -> Any:
+    """
+    Parse a string containing the `{{ Pipe() }}` syntax.
+    """
+    from meerschaum.utils.warnings import warn
+    from meerschaum.utils.sql import sql_item_name
+    from meerschaum.utils.dtypes import json_serialize_value
+    from meerschaum.utils.misc import parse_arguments_str
+    pattern = r'\{\{\s*(?:mrsm\.)?Pipe\((.*?)\)((?:\.[\w]+|\[[^\]]+\])*)\s*\}\}'
+
+    matches = list(re.finditer(pattern, text))
+    if not matches:
+        return text
+
+    placeholders = {}
+    for match in matches:
+        placeholder = f"__mrsm_pipe_placeholder_{uuid.uuid4().hex}__"
+        placeholders[placeholder] = match
+
+    substituted_text = text
+    for placeholder, match in placeholders.items():
+        substituted_text = substituted_text.replace(match.group(0), placeholder)
+
+    resolved_values = {}
+    for placeholder, match in placeholders.items():
+        try:
+            resolved_values[placeholder] = _evaluate_pipe_access_chain_from_match(match)
+        except Exception as e:
+            warn(f"Failed to resolve pipe syntax '{match.group(0)}': {e}")
+            resolved_values[placeholder] = match.group(0)
+
+    if len(matches) == 1:
+        match = matches[0]
+        placeholder = list(placeholders.keys())[0]
+        if text.strip() == match.group(0):
+            return resolved_values[placeholder]
+
+    final_text = substituted_text
+    for placeholder, value in resolved_values.items():
+        if isinstance(value, (dict, list, bool, int, float)) or value is None:
+            final_text = final_text.replace(placeholder, json.dumps(value, default=json_serialize_value))
+        else:
+            final_text = final_text.replace(placeholder, str(value))
+
+    return final_text
+
+
+def replace_pipes_in_dict(
+    pipes: Optional[PipesDict] = None,
+    func: Callable[[Any], Any] = str,
+    debug: bool = False,
+    **kw
+) -> PipesDict:
+    """
+    Replace the Pipes in a Pipes dict with the result of another function.
+
+    Parameters
+    ----------
+    pipes: Optional[PipesDict], default None
+        The pipes dict to be processed.
+
+    func: Callable[[Any], Any], default str
+        The function to be applied to every pipe.
+        Defaults to the string constructor.
+
+    debug: bool, default False
+        Verbosity toggle.
+    
+
+    Returns
+    -------
+    A dictionary where every pipe is replaced with the output of a function.
+
+    """
+    def change_dict(d : Dict[Any, Any], func : 'function') -> None:
+        for k, v in d.items():
+            if isinstance(v, dict):
+                change_dict(v, func)
+            else:
+                d[k] = func(v)
+
+    if pipes is None:
+        from meerschaum import get_pipes
+        pipes = get_pipes(debug=debug, **kw)
+
+    result = copy.deepcopy(pipes)
+    change_dict(result, func)
+    return result

meerschaum/utils/process.py

@@ -18,7 +18,7 @@ import platform
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Union, Optional, Any, Callable, Dict, Tuple
-from meerschaum.config.static import STATIC_CONFIG
+from meerschaum._internal.static import STATIC_CONFIG
 
 _child_processes = []
 def signal_handler(sig, frame):

meerschaum/utils/prompt.py

@@ -399,6 +399,8 @@ def choose(
             noask = noask,
             **kw
         )
+        if not answer:
+            continue
         ### Split along the delimiter.
         _answers = [answer] if not multiple else [a for a in answer.split(delimiter)]
 
@@ -564,7 +566,7 @@ def check_noask(noask: bool = False) -> bool:
     """
     Flip `noask` to `True` if `MRSM_NOASK` is set.
     """
-    from meerschaum.config.static import STATIC_CONFIG
+    from meerschaum._internal.static import STATIC_CONFIG
     NOASK = STATIC_CONFIG['environment']['noask']
     if noask:
         return True

meerschaum/utils/schedule.py

@@ -14,6 +14,7 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import Callable, Any, Optional, List, Dict
 from meerschaum.utils.warnings import warn, error
 
+
 STARTING_KEYWORD: str = 'starting'
 INTERVAL_UNITS: List[str] = ['months', 'weeks', 'days', 'hours', 'minutes', 'seconds', 'years']
 FREQUENCY_ALIASES: Dict[str, str] = {
@@ -292,7 +293,7 @@ def parse_start_time(schedule: str, now: Optional[datetime] = None) -> datetime:
     >>> parse_start_time('hourly starting 00:30')
     datetime.datetime(2024, 5, 13, 0, 30, tzinfo=datetime.timezone.utc)
     """
-    from meerschaum.utils.misc import round_time
+    from meerschaum.utils.dtypes import round_time
     dateutil_parser = mrsm.attempt_import('dateutil.parser')
     starting_parts = schedule.split(STARTING_KEYWORD)
     starting_str = ('now' if len(starting_parts) == 1 else starting_parts[-1]).strip()