meerschaum 3.0.0rc3__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/utils/packages/__init__.py

@@ -136,17 +136,17 @@ def get_module_path(
 
 
 def manually_import_module(
-        import_name: str,
-        venv: Optional[str] = 'mrsm',
-        check_update: bool = True,
-        check_pypi: bool = False,
-        install: bool = True,
-        split: bool = True,
-        warn: bool = True,
-        color: bool = True,
-        debug: bool = False,
-        use_sys_modules: bool = True,
-    ) -> Union['ModuleType', None]:
+    import_name: str,
+    venv: Optional[str] = 'mrsm',
+    check_update: bool = True,
+    check_pypi: bool = False,
+    install: bool = True,
+    split: bool = True,
+    warn: bool = True,
+    color: bool = True,
+    debug: bool = False,
+    use_sys_modules: bool = True,
+) -> Union['ModuleType', None]:
     """
     Manually import a module from a virtual environment (or the base environment).
 
@@ -1179,7 +1179,8 @@ def run_python_package(
     Either a return code integer or a `subprocess.Popen` object
     (or `None` if a `KeyboardInterrupt` occurs and as_proc is `True`).
     """
-    import sys, platform
+    import sys
+    import platform
     import subprocess
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
     from meerschaum.utils.process import run_process
@@ -1206,7 +1207,7 @@ def run_python_package(
             capture_output=capture_output,
             **kw
         )
-    except Exception as e:
+    except Exception:
         msg = f"Failed to execute {command}, will try again:\n{traceback.format_exc()}"
         warn(msg, color=False)
         stdout, stderr = (
@@ -1218,6 +1219,7 @@ def run_python_package(
             command,
             stdout=stdout,
             stderr=stderr,
+            stdin=sys.stdin,
             env=env_dict,
         )
         to_return = proc if as_proc else proc.wait()

meerschaum/utils/packages/_packages.py

@@ -175,6 +175,7 @@ packages['api'] = {
     'httpx'                          : 'httpx>=0.28.1',
     'httpcore'                       : 'httpcore>=1.0.9',
     'valkey'                         : 'valkey>=6.1.0',
+    'jose'                           : 'python-jose>=3.5.0',
 }
 packages['api'].update(packages['sql'])
 packages['api'].update(packages['formatting'])

meerschaum/utils/pipes.py

@@ -14,7 +14,7 @@ import ast
 import copy
 import uuid
 
-from meerschaum.utils.typing import PipesDict, Optional, Any
+from meerschaum.utils.typing import PipesDict, Optional
 import meerschaum as mrsm
 
 
@@ -96,9 +96,7 @@ def replace_pipes_syntax(text: str) -> Any:
     Parse a string containing the `{{ Pipe() }}` syntax.
     """
     from meerschaum.utils.warnings import warn
-    from meerschaum.utils.sql import sql_item_name
     from meerschaum.utils.dtypes import json_serialize_value
-    from meerschaum.utils.misc import parse_arguments_str
     pattern = r'\{\{\s*(?:mrsm\.)?Pipe\((.*?)\)((?:\.[\w]+|\[[^\]]+\])*)\s*\}\}'
 
     matches = list(re.finditer(pattern, text))
@@ -158,17 +156,20 @@ def replace_pipes_in_dict(
 
     debug: bool, default False
         Verbosity toggle.
-    
 
     Returns
     -------
     A dictionary where every pipe is replaced with the output of a function.
 
     """
-    def change_dict(d : Dict[Any, Any], func : 'function') -> None:
+    def change_dict(d: Dict[Any, Any]) -> None:
         for k, v in d.items():
             if isinstance(v, dict):
-                change_dict(v, func)
+                change_dict(v)
+            elif isinstance(v, list):
+                d[k] = [func(i) for i in v]
+            elif isinstance(v, tuple):
+                d[k] = tuple([func(i) for i in v])
             else:
                 d[k] = func(v)
 
@@ -177,5 +178,36 @@ def replace_pipes_in_dict(
         pipes = get_pipes(debug=debug, **kw)
 
     result = copy.deepcopy(pipes)
-    change_dict(result, func)
+    change_dict(result)
     return result
+
+
+def is_pipe_registered(
+    pipe: mrsm.Pipe,
+    pipes: PipesDict,
+    debug: bool = False
+) -> bool:
+    """
+    Check if a Pipe is inside the pipes dictionary.
+
+    Parameters
+    ----------
+    pipe: meerschaum.Pipe
+        The pipe to see if it's in the dictionary.
+
+    pipes: PipesDict
+        The dictionary to search inside.
+
+    debug: bool, default False
+        Verbosity toggle.
+
+    Returns
+    -------
+    A bool indicating whether the pipe is inside the dictionary.
+    """
+    from meerschaum.utils.debug import dprint
+    ck, mk, lk = pipe.connector_keys, pipe.metric_key, pipe.location_key
+    if debug:
+        dprint(f'{ck}, {mk}, {lk}')
+        dprint(f'{pipe}, {pipes}')
+    return ck in pipes and mk in pipes[ck] and lk in pipes[ck][mk]

meerschaum/utils/process.py

@@ -178,7 +178,7 @@ def run_process(
             # make us tty's foreground again
             try:
                 os.tcsetpgrp(sys.stdin.fileno(), old_pgrp)
-            except Exception as e:
+            except Exception:
                 pass
             # now restore the handler
             signal.signal(signal.SIGTTOU, hdlr)

meerschaum/utils/prompt.py

@@ -14,7 +14,8 @@ from meerschaum.utils.typing import Any, Union, Optional, Tuple, List
 
 
 def prompt(
-    question: str,
+    question: str = '',
+    *,
     icon: bool = True,
     default: Union[str, Tuple[str, str], None] = None,
     default_editable: Optional[str] = None,
@@ -22,6 +23,7 @@ def prompt(
     is_password: bool = False,
     wrap_lines: bool = True,
     noask: bool = False,
+    silent: bool = False,
     **kw: Any
 ) -> str:
     """
@@ -59,6 +61,9 @@ def prompt(
     noask: bool, default False
         If `True`, only print the question and return the default answer.
 
+    silent: bool, default False
+        If `True` do not print anything to the screen, but still block for input.
+
     Returns
     -------
     A `str` of the input provided by the user.
@@ -67,11 +72,24 @@ def prompt(
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.formatting import ANSI, CHARSET, highlight_pipes, fill_ansi
     from meerschaum.config import get_config
-    from meerschaum.utils.misc import filter_keywords
-    from meerschaum.utils.daemon import running_in_daemon
+    from meerschaum.utils.misc import filter_keywords, remove_ansi
+    from meerschaum.utils.daemon import running_in_daemon, get_current_daemon
+
+    original_kwargs = {
+        'question': question,
+        'icon': icon,
+        'default': default,
+        'default_editable': default_editable,
+        'detect_password': detect_password,
+        'is_password': is_password,
+        'wrap_lines': wrap_lines,
+        'noask': noask,
+        'silent': silent,
+        **kw
+    }
+
     noask = check_noask(noask)
-    if not noask:
-        prompt_toolkit = attempt_import('prompt_toolkit')
+    prompt_toolkit = attempt_import('prompt_toolkit')
     question_config = get_config('formatting', 'question', patch=True)
 
     ### if a default is provided, append it to the question.
@@ -103,27 +121,49 @@ def prompt(
     question += first_line
     if len(other_lines) > 0:
         question += '\n' + other_lines
-    question += ' '
+
+    if not remove_ansi(question).endswith(' '):
+        question += ' '
+
+    prompt_kwargs = {
+        'message': prompt_toolkit.formatted_text.ANSI(question) if not silent else '',
+        'wrap_lines': wrap_lines,
+        'default': default_editable or '',
+        **filter_keywords(prompt_toolkit.prompt, **kw)
+    }
+
+    printed_question = False
 
     if not running_in_daemon():
-        answer = (
-            prompt_toolkit.prompt(
-                prompt_toolkit.formatted_text.ANSI(question),
-                wrap_lines=wrap_lines,
-                default=default_editable or '',
-                **filter_keywords(prompt_toolkit.prompt, **kw)
-            ) if not noask else ''
-        )
+        answer = prompt_toolkit.prompt(**prompt_kwargs) if not noask else ''
+        printed_question = True
     else:
-        print(question, end='\n', flush=True)
+        import json
+        daemon = get_current_daemon()
+        print('', end='', flush=True)
+        wrote_file = False
+        try:
+            with open(daemon.prompt_kwargs_file_path, 'w+', encoding='utf-8') as f:
+                json.dump(original_kwargs, f, separators=(',', ':'))
+            wrote_file = True
+        except Exception:
+            pass
+
+        if not silent and not wrote_file:
+            print(question, end='', flush=True)
+            printed_question = True
+
         try:
             answer = input() if not noask else ''
         except EOFError:
             answer = ''
-    if noask:
-        print(question)
+
+    if noask and not silent and not printed_question:
+        print(question, flush=True)
+
     if answer == '' and default is not None:
         return default_answer
+
     return answer
 
 
@@ -207,7 +247,7 @@ def yes_no(
 
 def choose(
     question: str,
-    choices: List[Union[str, Tuple[str, str]]],
+    choices: Union[List[str], List[Tuple[str, str]]],
     default: Union[str, List[str], None] = None,
     numeric: bool = True,
     multiple: bool = False,
@@ -270,8 +310,9 @@ def choose(
     noask = check_noask(noask)
 
     ### Handle empty choices.
-    if len(choices) == 0:
-        _warn(f"No available choices. Returning default value '{default}'.", stacklevel=3)
+    if not choices:
+        if warn:
+            _warn(f"No available choices. Returning default value '{default}'.", stacklevel=3)
         return default
 
     ### If the default case is to include multiple answers, allow for multiple inputs.
@@ -279,18 +320,25 @@ def choose(
         multiple = True
 
     choices_indices = {}
-    for i, c in enumerate(choices):
+    for i, c in enumerate(choices, start=1):
         if isinstance(c, tuple):
             i, c = c
         choices_indices[i] = c
 
+    choices_values_indices = {v: k for k, v in choices_indices.items()}
+    ordered_keys = list(choices_indices.keys())
+    numeric_map = {str(i): key for i, key in enumerate(ordered_keys, 1)}
+
     def _enforce_default(d):
-        if d is not None and d not in choices and d not in choices_indices and warn:
-            _warn(
-                f"Default choice '{default}' is not contained in the choices {choices}. "
-                + "Setting numeric = False.",
-                stacklevel = 3
-            )
+        if d is None:
+            return True
+        if d not in choices_values_indices and d not in choices_indices:
+            if warn:
+                _warn(
+                    f"Default choice '{d}' is not contained in the choices. "
+                    + "Setting numeric = False.",
+                    stacklevel=3
+                )
             return False
         return True
 
@@ -300,163 +348,141 @@ def choose(
             numeric = False
             break
 
-    _default = default
-    _choices = list(choices_indices.values())
+    _choices = (
+        [str(k) for k in choices_indices] if numeric
+        else list(choices_indices.values())
+    )
     if multiple:
         question += f"\n    Enter your choices, separated by '{delimiter}'.\n"
 
     altered_choices = {}
-    altered_indices = {}
-    altered_default_indices = {}
-    delim_replacement = '_' if delimiter != '_' else '-'
-    can_strip_start_spaces, can_strip_end_spaces = True, True
-    for i, c in choices_indices.items():
-        if isinstance(c, tuple):
-            key, c = c
-        if can_strip_start_spaces and c.startswith(' '):
-            can_strip_start_spaces = False
-        if can_strip_end_spaces and c.endswith(' '):
-            can_strip_end_spaces = False
-
-    if multiple:
-        ### Check if the defaults have the delimiter.
-        for i, d in enumerate(default if isinstance(default, list) else [default]):
-            if d is None or delimiter not in d:
-                continue
-            new_d = d.replace(delimiter, delim_replacement)
-            altered_choices[new_d] = d
-            altered_default_indices[i] = new_d
-        for i, new_d in altered_default_indices.items():
-            if not isinstance(default, list):
-                default = new_d
-                break
-            default[i] = new_d
-
+    if multiple and not numeric:
+        delim_replacement = '_' if delimiter != '_' else '-'
         ### Check if the choices have the delimiter.
         for i, c in choices_indices.items():
-            if delimiter in c and not numeric and warn:
+            if delimiter not in c:
+                continue
+            if warn:
                 _warn(
                     f"The delimiter '{delimiter}' is contained within choice '{c}'.\n"
                     + f"Replacing the string '{delimiter}' with '{delim_replacement}' in "
                     + "the choice for correctly parsing input (will be replaced upon returning the prompt).",
-                    stacklevel = 3,
+                    stacklevel=3,
                 )
-                new_c = c.replace(delimiter, delim_replacement)
-                altered_choices[new_c] = c
-                altered_indices[i] = new_c
-        for i, new_c in altered_indices.items():
+            new_c = c.replace(delimiter, delim_replacement)
+            altered_choices[new_c] = c
             choices_indices[i] = new_c
-        default = delimiter.join(default) if isinstance(default, list) else default
 
     question_options = []
+    default_tuple = None
     if numeric:
-        _choices = [str(i + 1) for i, c in enumerate(choices)]
-        _default = ''
+        _default_prompt_str = ''
         if default is not None:
-            for d in (default.split(delimiter) if multiple else [default]):
-                if d not in choices and d in choices_indices:
-                    d_index = d
-                    d_value = choices_indices[d]
-                    for _i, _option in enumerate(choices):
-                        if (
-                            isinstance(_option, tuple) and (
-                                _option[1] == d_value
-                                or
-                                _option[0] == d_index
-                            )
-                        ) or d_index == _i:
-                            d = _option
-
-                _d = str(choices.index(d) + 1)
-                _default += _d + delimiter
-        _default = _default[:-1 * len(delimiter)]
-        #  question += '\n'
+            default_list = default if isinstance(default, list) else [default]
+            if multiple and isinstance(default, str):
+                default_list = default.split(delimiter)
+
+            _default_indices = []
+            for d in default_list:
+                key = None
+                if d in choices_values_indices:  # is a value
+                    key = choices_values_indices[d]
+                elif d in choices_indices:  # is an index
+                    key = d
+
+                if key in ordered_keys:
+                    _default_indices.append(str(ordered_keys.index(key) + 1))
+
+            _default_prompt_str = delimiter.join(_default_indices)
+        
         choices_digits = len(str(len(choices)))
-        for i, c in enumerate(choices_indices.values()):
+        for choice_ix, c in enumerate(choices_indices.values(), start=1):
             question_options.append(
-                f"  {i + 1}. "
-                + (" " * (choices_digits - len(str(i + 1))))
+                f"  {choice_ix}. "
+                + (" " * (choices_digits - len(str(choice_ix))))
                 + f"{c}\n"
             )
-        default_tuple = (_default, default) if default is not None else None
+        default_tuple = (_default_prompt_str, default) if default is not None else None
     else:
         default_tuple = default
-        #  question += '\n'
         for c in choices_indices.values():
-            question_options.append(f"{c}\n")
+            question_options.append(f"  • {c}\n")
 
     if 'completer' not in kw:
-        WordCompleter = attempt_import('prompt_toolkit.completion').WordCompleter
-        kw['completer'] = WordCompleter(choices_indices.values(), sentence=True)
+        WordCompleter = attempt_import('prompt_toolkit.completion', lazy=False).WordCompleter
+        kw['completer'] = WordCompleter(
+            [str(v) for v in choices_indices.values()] + [str(i) for i in choices_indices],
+            sentence=True,
+        )
 
-    valid = False
-    while not valid:
+    answers = []
+    while not answers:
         print_options(question_options, header='')
         answer = prompt(
             question,
-            icon = icon,
-            default = default_tuple,
-            noask = noask,
+            icon=icon,
+            default=default_tuple,
+            noask=noask,
             **kw
         )
+        if not answer and default is not None:
+            answer = default if isinstance(default, str) else delimiter.join(default)
+
         if not answer:
+            if warn:
+                _warn("Please pick a valid choice.", stack=False)
             continue
-        ### Split along the delimiter.
-        _answers = [answer] if not multiple else [a for a in answer.split(delimiter)]
-
-        ### Remove trailing spaces if possible.
-        _answers = [(_a.rstrip(' ') if can_strip_end_spaces else _a) for _a in _answers]
-
-        ### Remove leading spaces if possible.
-        _answers = [(_a.lstrip(' ') if can_strip_start_spaces else _a) for _a in _answers]
-
-        ### Remove empty strings.
-        _answers = [_a for _a in _answers if _a]
-
-        if multiple and len(_answers) == 0:
-            _answers = default_tuple if isinstance(default_tuple, list) else [default_tuple]
-        answers = [altered_choices.get(a, a) for a in _answers]
-
-        valid = (len(answers) > 1 or not (len(answers) == 1 and answers[0] is None))
-        for a in answers:
-            if (
-                a not in {_original for _new, _original in altered_choices.items()}
-                and a not in _choices
-                and a != default
-                and not noask
-            ):
-                valid = False
-                break
-        if valid:
-            break
-        if warn:
-            _warn("Please pick a valid choice.", stack=False)
+
+        _answers = [answer] if not multiple else [a.strip() for a in answer.split(delimiter)]
+        _answers = [a for a in _answers if a]
+
+        if numeric:
+            _raw_answers = list(_answers)
+            _answers = []
+            for _a in _raw_answers:
+                if _a in choices_values_indices:
+                    _answers.append(str(choices_values_indices[_a]))
+                elif _a in numeric_map:
+                    _answers.append(str(numeric_map[_a]))
+                else:
+                    _answers.append(_a)
+
+        _processed_answers = [altered_choices.get(a, a) for a in _answers]
+
+        valid_answers = []
+        for a in _processed_answers:
+            if a in _choices:
+                valid_answers.append(a)
+
+        if len(valid_answers) != len(_processed_answers):
+            if warn:
+                _warn("Please pick a valid choice.", stack=False)
+            continue
+        answers = valid_answers
+
+    def get_key(key_str):
+        try:
+            return int(key_str)
+        except (ValueError, TypeError):
+            return key_str
 
     if not multiple:
+        answer = answers[0]
         if not numeric:
-            return answer
-        try:
-            _answer = choices[int(answer) - 1]
-            if as_indices and isinstance(_answer, tuple):
-                return _answer[0]
-            return _answer
-        except Exception:
-            _warn(f"Could not cast answer '{answer}' to an integer.", stacklevel=3)
+            return choices_values_indices.get(answer, answer) if as_indices else answer
+        
+        key = get_key(answer)
+        return key if as_indices else choices_indices[key]
 
     if not numeric:
-        return answers
+        return [choices_values_indices.get(a, a) for a in answers] if as_indices else answers
 
-    _answers = []
+    final_answers = []
     for a in answers:
-        try:
-            _answer = choices[int(a) - 1]
-            _answer_to_return = altered_choices.get(_answer, _answer)
-            if isinstance(_answer_to_return, tuple) and as_indices:
-                _answer_to_return = _answer_to_return[0]
-            _answers.append(_answer_to_return)
-        except Exception:
-            _warn(f"Could not cast answer '{a}' to an integer.", stacklevel=3)
-    return _answers
+        key = get_key(a)
+        final_answers.append(key if as_indices else choices_indices[key])
+    return final_answers
+
 
 
 def get_password(
@@ -570,6 +596,7 @@ def check_noask(noask: bool = False) -> bool:
     NOASK = STATIC_CONFIG['environment']['noask']
     if noask:
         return True
+
     return (
         os.environ.get(NOASK, 'false').lower()
         in ('1', 'true')

meerschaum/utils/sql.py

@@ -1586,6 +1586,8 @@ def get_update_queries(
     datetime_col: Optional[str] = None,
     schema: Optional[str] = None,
     patch_schema: Optional[str] = None,
+    target_cols_types: Optional[Dict[str, str]] = None,
+    patch_cols_types: Optional[Dict[str, str]] = None,
     identity_insert: bool = False,
     null_indices: bool = True,
     cast_columns: bool = True,
@@ -1626,6 +1628,14 @@ def get_update_queries(
         If provided, use this schema when quoting the patch table.
         Defaults to `schema`.
 
+    target_cols_types: Optional[Dict[str, Any]], default None
+        If provided, use these as the columns-types dictionary for the target table.
+        Default will infer from the database context.
+
+    patch_cols_types: Optional[Dict[str, Any]], default None
+        If provided, use these as the columns-types dictionary for the target table.
+        Default will infer from the database context.
+
     identity_insert: bool, default False
         If `True`, include `SET IDENTITY_INSERT` queries before and after the update queries.
         Only applies for MSSQL upserts.
@@ -1672,14 +1682,14 @@ def get_update_queries(
         flavor=flavor,
         schema=schema,
         debug=debug,
-    )
+    ) if not target_cols_types else target_cols_types
     patch_table_columns = get_table_cols_types(
         patch,
         connectable,
         flavor=flavor,
         schema=patch_schema,
         debug=debug,
-    )
+    ) if not patch_cols_types else patch_cols_types
 
     patch_cols_str = ', '.join(
         [