meerschaum 2.4.11__py3-none-any.whl → 2.4.13__py3-none-any.whl

meerschaum/connectors/sql/_fetch.py

@@ -7,27 +7,29 @@ Implement the Connector fetch() method
 """
 
 from __future__ import annotations
+
 from datetime import datetime, timedelta
 import meerschaum as mrsm
-from meerschaum.utils.typing import Optional, Union, Callable, Any
+from meerschaum.utils.typing import Optional, Union, Callable, Any, List, Dict
+
 
 def fetch(
-        self,
-        pipe: meerschaum.Pipe,
-        begin: Union[datetime, int, str, None] = '',
-        end: Union[datetime, int, str, None] = None,
-        check_existing: bool = True,
-        chunk_hook: Optional[Callable[[pd.DataFrame], Any]] = None,
-        chunksize: Optional[int] = -1,
-        workers: Optional[int] = None,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union['pd.DataFrame', List[Any], None]:
+    self,
+    pipe: mrsm.Pipe,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, str, None] = None,
+    check_existing: bool = True,
+    chunk_hook: Optional[Callable[['pd.DataFrame'], Any]] = None,
+    chunksize: Optional[int] = -1,
+    workers: Optional[int] = None,
+    debug: bool = False,
+    **kw: Any
+) -> Union['pd.DataFrame', List[Any], None]:
     """Execute the SQL definition and return a Pandas DataFrame.
 
     Parameters
     ----------
-    pipe: meerschaum.Pipe
+    pipe: mrsm.Pipe
         The pipe object which contains the `fetch` metadata.
         
         - pipe.columns['datetime']: str
@@ -63,7 +65,7 @@ def fetch(
 
     debug: bool, default False
         Verbosity toggle.
-       
+
     Returns
     -------
     A pandas DataFrame or `None`.
@@ -71,20 +73,20 @@ def fetch(
     """
     meta_def = self.get_pipe_metadef(
         pipe,
-        begin = begin,
-        end = end,
-        check_existing = check_existing,
-        debug = debug,
+        begin=begin,
+        end=end,
+        check_existing=check_existing,
+        debug=debug,
         **kw
     )
     as_hook_results = chunk_hook is not None
     chunks = self.read(
         meta_def,
-        chunk_hook = chunk_hook,
-        as_hook_results = as_hook_results,
-        chunksize = chunksize,
-        workers = workers,
-        debug = debug,
+        chunk_hook=chunk_hook,
+        as_hook_results=as_hook_results,
+        chunksize=chunksize,
+        workers=workers,
+        debug=debug,
     )
     ### if sqlite, parse for datetimes
     if not as_hook_results and self.flavor == 'sqlite':
@@ -97,8 +99,8 @@ def fetch(
         return (
             parse_df_datetimes(
                 chunk,
-                ignore_cols = ignore_cols,
-                debug = debug,
+                ignore_cols=ignore_cols,
+                debug=debug,
             )
             for chunk in chunks
         )
@@ -106,15 +108,15 @@ def fetch(
 
 
 def get_pipe_metadef(
-        self,
-        pipe: meerschaum.Pipe,
-        params: Optional[Dict[str, Any]] = None,
-        begin: Union[datetime, int, str, None] = '',
-        end: Union[datetime, int, str, None] = None,
-        check_existing: bool = True,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union[str, None]:
+    self,
+    pipe: mrsm.Pipe,
+    params: Optional[Dict[str, Any]] = None,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, str, None] = None,
+    check_existing: bool = True,
+    debug: bool = False,
+    **kw: Any
+) -> Union[str, None]:
     """
     Return a pipe's meta definition fetch query.
 
@@ -173,7 +175,6 @@ def get_pipe_metadef(
                     stack = False
                 )
 
-
     apply_backtrack = begin == '' and check_existing
     backtrack_interval = pipe.get_backtrack_interval(check_existing=check_existing, debug=debug)
     btm = (
@@ -189,35 +190,34 @@ def get_pipe_metadef(
 
     if begin and end and begin >= end:
         begin = None
-    
-    da = None
+
     if dt_name:
         begin_da = (
             dateadd_str(
-                flavor = self.flavor,
-                datepart = 'minute',
-                number = ((-1 * btm) if apply_backtrack else 0), 
-                begin = begin,
+                flavor=self.flavor,
+                datepart='minute',
+                number=((-1 * btm) if apply_backtrack else 0), 
+                begin=begin,
             )
             if begin
             else None
         )
         end_da = (
             dateadd_str(
-                flavor = self.flavor,
-                datepart = 'minute',
-                number = 0,
-                begin = end,
+                flavor=self.flavor,
+                datepart='minute',
+                number=0,
+                begin=end,
             )
             if end
             else None
         )
 
     meta_def = (
-        _simple_fetch_query(pipe) if (
+        _simple_fetch_query(pipe, self.flavor) if (
             (not (pipe.columns or {}).get('id', None))
             or (not get_config('system', 'experimental', 'join_fetch'))
-        ) else _join_fetch_query(pipe, debug=debug, **kw)
+        ) else _join_fetch_query(pipe, self.flavor, debug=debug, **kw)
     )
 
     has_where = 'where' in meta_def.lower()[meta_def.lower().rfind('definition'):]
@@ -300,25 +300,30 @@ def set_pipe_query(pipe: mrsm.Pipe, query: str) -> None:
     dict_to_set[key_to_set] = query
 
 
-def _simple_fetch_query(pipe, debug: bool=False, **kw) -> str:
+def _simple_fetch_query(
+    pipe: mrsm.Pipe,
+    flavor: str,
+    debug: bool = False,
+    **kw
+) -> str:
     """Build a fetch query from a pipe's definition."""
-    def_name = 'definition'
+    from meerschaum.utils.sql import format_cte_subquery
     definition = get_pipe_query(pipe)
-    return (
-        f"WITH {def_name} AS (\n{definition}\n) SELECT * FROM {def_name}"
-        if pipe.connector.flavor not in ('mysql', 'mariadb')
-        else f"SELECT * FROM (\n{definition}\n) AS {def_name}"
-    )
+    if definition is None:
+        raise ValueError(f"No SQL definition could be found for {pipe}.")
+    return format_cte_subquery(definition, flavor, 'definition')
+
 
 def _join_fetch_query(
-        pipe,
-        debug: bool = False,
-        new_ids: bool = True,
-        **kw
-    ) -> str:
+    pipe: mrsm.Pipe,
+    flavor: str,
+    debug: bool = False,
+    new_ids: bool = True,
+    **kw
+) -> str:
     """Build a fetch query based on the datetime and ID indices."""
     if not pipe.exists(debug=debug):
-        return _simple_fetch_query(pipe, debug=debug, **kw)
+        return _simple_fetch_query(pipe, flavor, debug=debug, **kw)
 
     from meerschaum.utils.sql import sql_item_name, dateadd_str
     pipe_instance_name = sql_item_name(
@@ -350,7 +355,8 @@ def _join_fetch_query(
     """
     sync_times = pipe.instance_connector.read(sync_times_query, debug=debug, silent=False)
     if sync_times is None:
-        return _simple_fetch_query(pipe, debug=debug, **kw)
+        return _simple_fetch_query(pipe, flavor, debug=debug, **kw)
+
     _sync_times_q = f",\n{sync_times_remote_name} AS ("
     for _id, _st in sync_times.itertuples(index=False):
         _sync_times_q += (

meerschaum/connectors/sql/_pipes.py

@@ -2096,7 +2096,7 @@ def get_pipe_rowcount(
     An `int` for the number of rows if the `pipe` exists, otherwise `None`.
 
     """
-    from meerschaum.utils.sql import dateadd_str, sql_item_name, NO_CTE_FLAVORS
+    from meerschaum.utils.sql import dateadd_str, sql_item_name, wrap_query_with_cte
     from meerschaum.connectors.sql._fetch import get_pipe_query
     if remote:
         msg = f"'fetch:definition' must be an attribute of {pipe} to get a remote rowcount."
@@ -2175,20 +2175,10 @@ def get_pipe_rowcount(
         if not remote
         else get_pipe_query(pipe)
     )
-    query = (
-        f"""
-        WITH src AS ({src})
-        SELECT COUNT(*)
-        FROM src
-        """
-    ) if self.flavor not in ('mysql', 'mariadb') else (
-        f"""
-        SELECT COUNT(*)
-        FROM ({src}) AS src
-        """
-    )
+    parent_query = f"SELECT COUNT(*)\nFROM {sql_item_name('src', self.flavor)}"
+    query = wrap_query_with_cte(src, parent_query, self.flavor)
     if begin is not None or end is not None:
-        query += "WHERE"
+        query += "\nWHERE"
     if begin is not None:
         query += f"""
         {dt} >= {dateadd_str(self.flavor, datepart='minute', number=0, begin=begin)}
@@ -2330,10 +2320,10 @@ def clear_pipe(
 
 
 def get_pipe_table(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> sqlalchemy.Table:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Union['sqlalchemy.Table', None]:
     """
     Return the `sqlalchemy.Table` object for a `mrsm.Pipe`.
 
@@ -2352,18 +2342,18 @@ def get_pipe_table(
         return None
     return get_sqlalchemy_table(
         pipe.target,
-        connector = self,
-        schema = self.get_pipe_schema(pipe),
-        debug = debug,
-        refresh = True,
+        connector=self,
+        schema=self.get_pipe_schema(pipe),
+        debug=debug,
+        refresh=True,
     )
 
 
 def get_pipe_columns_types(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> Dict[str, str]:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Dict[str, str]:
     """
     Get the pipe's columns and types.
 
@@ -2394,8 +2384,8 @@ def get_pipe_columns_types(
         return get_table_cols_types(
             pipe.target,
             self,
-            flavor = self.flavor,
-            schema = self.schema,
+            flavor=self.flavor,
+            schema=self.get_pipe_schema(pipe),
         )
 
     table_columns = {}
@@ -2448,6 +2438,7 @@ def get_add_columns_queries(
     from meerschaum.utils.sql import (
         sql_item_name,
         SINGLE_ALTER_TABLE_FLAVORS,
+        get_table_cols_types,
     )
     from meerschaum.utils.dtypes.sql import (
         get_pd_type_from_db_type,
@@ -2480,6 +2471,14 @@ def get_add_columns_queries(
     db_cols_types = {
         col: get_pd_type_from_db_type(str(typ.type))
         for col, typ in table_obj.columns.items()
+    } if table_obj is not None else {
+        col: get_pd_type_from_db_type(typ)
+        for col, typ in get_table_cols_types(
+            pipe.target,
+            self,
+            schema=self.get_pipe_schema(pipe),
+            debug=debug,
+        ).items()
     }
     new_cols = set(df_cols_types) - set(db_cols_types)
     if not new_cols:
@@ -2552,7 +2551,7 @@ def get_alter_columns_queries(
     """
     if not pipe.exists(debug=debug):
         return []
-    from meerschaum.utils.sql import sql_item_name, DROP_IF_EXISTS_FLAVORS
+    from meerschaum.utils.sql import sql_item_name, DROP_IF_EXISTS_FLAVORS, get_table_cols_types
     from meerschaum.utils.dataframe import get_numeric_cols
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
@@ -2583,6 +2582,14 @@ def get_alter_columns_queries(
     db_cols_types = {
         col: get_pd_type_from_db_type(str(typ.type))
         for col, typ in table_obj.columns.items()
+    } if table_obj is not None else {
+        col: get_pd_type_from_db_type(typ)
+        for col, typ in get_table_cols_types(
+            pipe.target,
+            self,
+            schema=self.get_pipe_schema(pipe),
+            debug=debug,
+        ).items()
     }
     pipe_bool_cols = [col for col, typ in pipe.dtypes.items() if are_dtypes_equal(str(typ), 'bool')]
     pd_db_df_aliases = {

meerschaum/plugins/__init__.py

@@ -316,7 +316,11 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
 
         ### NOTE: Allow plugins to be installed via `pip`.
         packaged_plugin_paths = []
-        discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        try:
+            discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        except TypeError:
+            discovered_packaged_plugins_eps = []
+
         for ep in discovered_packaged_plugins_eps:
             module_name = ep.name
             for package_file_path in ep.dist.files:
@@ -330,7 +334,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
         if is_symlink(PLUGINS_RESOURCES_PATH) or not PLUGINS_RESOURCES_PATH.exists():
             try:
                 PLUGINS_RESOURCES_PATH.unlink()
-            except Exception as e:
+            except Exception:
                 pass
 
         PLUGINS_RESOURCES_PATH.mkdir(exist_ok=True)

meerschaum/plugins/bootstrap.py

@@ -47,7 +47,6 @@ IMPORTS_LINES: Dict[str, str] = {
     ),
 }
 
-### TODO: Add feature for custom connectors.
 FEATURE_LINES: Dict[str, str] = {
     'header': (
         "#! /usr/bin/env python3\n"
@@ -94,7 +93,7 @@ FEATURE_LINES: Dict[str, str] = {
         "class {plugin_name_capitalized}Connector(Connector):\n"
         "    \"\"\"Implement '{plugin_name_lower}' connectors.\"\"\"\n\n"
         "    REQUIRED_ATTRIBUTES: list[str] = []\n"
-        "    \n"
+        "\n"
         "    def fetch(\n"
         "        self,\n"
         "        pipe: mrsm.Pipe,\n"
@@ -149,11 +148,12 @@ FEATURE_LINES: Dict[str, str] = {
     ),
 }
 
+
 def bootstrap_plugin(
-        plugin_name: str,
-        debug: bool = False,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    plugin_name: str,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Prompt the user for features and create a plugin file.
     """
@@ -177,9 +177,9 @@ def bootstrap_plugin(
     features: List[str] = choose(
         "Which of the following features would you like to add to your plugin?",
         list(FEATURE_CHOICES.items()),
-        default = 'fetch',
-        multiple = True,
-        as_indices = True,
+        default='fetch',
+        multiple=True,
+        as_indices=True,
         **kwargs
     )
 
@@ -256,7 +256,7 @@ def bootstrap_plugin(
         _ = prompt(
             f"Press [Enter] to edit plugin '{plugin_name}',"
             + " [CTRL+C] to skip.",
-            icon = False,
+            icon=False,
         )
     except (KeyboardInterrupt, Exception):
         return True, "Success"
@@ -267,7 +267,7 @@ def bootstrap_plugin(
 
 def _get_plugins_dir_path() -> pathlib.Path:
     from meerschaum.config.paths import PLUGINS_DIR_PATHS
-    
+
     if not PLUGINS_DIR_PATHS:
         raise EnvironmentError("No plugin dir path could be found.")
 
@@ -278,9 +278,9 @@ def _get_plugins_dir_path() -> pathlib.Path:
         choose(
             "In which directory do you want to write your plugin?",
             [path.as_posix() for path in PLUGINS_DIR_PATHS],
-            numeric = True,
-            multiple = False,
-            default = PLUGINS_DIR_PATHS[0].as_posix(),
+            numeric=True,
+            multiple=False,
+            default=PLUGINS_DIR_PATHS[0].as_posix(),
         )
     )
  
@@ -290,7 +290,7 @@ def _ask_to_uninstall(plugin: mrsm.Plugin, **kwargs: Any) -> SuccessTuple:
     warn(f"Plugin '{plugin}' is already installed!", stack=False)
     uninstall_plugin = yes_no(
         f"Do you want to first uninstall '{plugin}'?",
-        default = 'n',
+        default='n',
         **kwargs
     )
     if not uninstall_plugin:

meerschaum/utils/formatting/__init__.py

@@ -11,7 +11,7 @@ import platform
 import os
 import sys
 import meerschaum as mrsm
-from meerschaum.utils.typing import Optional, Union, Any, Dict
+from meerschaum.utils.typing import Optional, Union, Any, Dict, Iterable
 from meerschaum.utils.formatting._shell import make_header
 from meerschaum.utils.formatting._pprint import pprint
 from meerschaum.utils.formatting._pipes import (
@@ -322,14 +322,14 @@ def format_success_tuple(
     from meerschaum.config import get_config
     status_config = get_config('formatting', status, patch=True)
 
-    msg = (' ' * left_padding) + status_config[CHARSET]['icon'] + ' ' + str(tup[1])
+    msg = (' ' * left_padding) + status_config[CHARSET]['icon'] + ' ' + str(highlight_pipes(tup[1]))
     lines = msg.split('\n')
     lines = [lines[0]] + [
         (('    ' + line if not line.startswith(' ') else line))
         for line in lines[1:]
     ]
     if ANSI:
-        lines[0] = fill_ansi(highlight_pipes(lines[0]), **status_config['ansi']['rich'])
+        lines[0] = fill_ansi(lines[0], **status_config['ansi']['rich'])
 
     msg = '\n'.join(lines)
     msg = ('\n' * upper_padding) + msg + ('\n' * lower_padding)
@@ -337,7 +337,7 @@ def format_success_tuple(
 
 
 def print_options(
-    options: Optional[Dict[str, Any]] = None,
+    options: Optional[Iterable[Any]] = None,
     nopretty: bool = False,
     no_rich: bool = False,
     name: str = 'options',
@@ -345,6 +345,7 @@ def print_options(
     num_cols: Optional[int] = None,
     adjust_cols: bool = True,
     sort_options: bool = False,
+    number_options: bool = False,
     **kw
 ) -> None:
     """
@@ -373,6 +374,12 @@ def print_options(
     adjust_cols: bool, default True
         If `True`, adjust the number of columns depending on the terminal size.
 
+    sort_options: bool, default False
+        If `True`, print the options in sorted order.
+
+    number_options: bool, default False
+        If `True`, print the option's number in the list (1 index).
+
     """
     import os
     from meerschaum.utils.packages import import_rich
@@ -398,9 +405,10 @@ def print_options(
             print()
             print(make_header(_header))
         ### print actions
-        for option in _options:
+        for i, option in enumerate(_options):
+            marker = '-' if not number_options else (str(i + 1) + '.')
             if not nopretty:
-                print("  - ", end="")
+                print(f"  {marker} ", end="")
             print(option)
         if not nopretty:
             print()
@@ -434,11 +442,11 @@ def print_options(
 
     if _header is not None:
         table = Table(
-            title = _header,
-            box = box.SIMPLE,
-            show_header = False,
-            show_footer = False,
-            title_style = '',
+            title=_header,
+            box=box.SIMPLE,
+            show_header=False,
+            show_footer=False,
+            title_style='',
             expand = True,
         )
     else:
@@ -447,18 +455,26 @@ def print_options(
         table.add_column()
 
     if len(_options) < 12:
-        # If fewer than 12 items, use a single column
-        for option in _options:
-            table.add_row(Text.from_ansi(highlight_pipes(option)))
+        ### If fewer than 12 items, use a single column
+        for i, option in enumerate(_options):
+            item = highlight_pipes(option)
+            if number_options:
+                item = str(i + 1) + '. ' + item
+            table.add_row(Text.from_ansi(item))
     else:
-        # Otherwise, use multiple columns as before
+        ### Otherwise, use multiple columns as before
         num_rows = (len(_options) + num_cols - 1) // num_cols
+        item_ix = 0
         for i in range(num_rows):
             row = []
             for j in range(num_cols):
                 index = i + j * num_rows
                 if index < len(_options):
-                    row.append(Text.from_ansi(highlight_pipes(_options[index])))
+                    item = highlight_pipes(_options[index])
+                    if number_options:
+                        item = str(i + 1) + '. ' + item
+                    row.append(Text.from_ansi(item))
+                    item_ix += 1
                 else:
                     row.append('')
             table.add_row(*row)

meerschaum/utils/formatting/_pipes.py

@@ -323,7 +323,7 @@ def pipe_repr(
     )
     if as_rich_text:
         return text_obj
-    return rich_text_to_str(text_obj)
+    return rich_text_to_str(text_obj).replace('\n', '')
 
 
 def highlight_pipes(message: str) -> str:

meerschaum/utils/formatting/_shell.py

@@ -10,10 +10,11 @@ from re import sub
 from meerschaum.utils.threading import Lock
 _locks = {'_tried_clear_command': Lock()}
 
+
 def make_header(
-        message : str,
-        ruler : str = '─',
-    ) -> str:
+    message: str,
+    ruler: str = '─',
+) -> str:
     """Format a message string with a ruler.
     Length of the ruler is the length of the longest word.
     

meerschaum/utils/process.py

@@ -178,22 +178,32 @@ def run_process(
 
     return ret
 
+
 def poll_process(
-        proc: subprocess.Popen,
-        line_callback: Callable[[bytes], Any],
-        timeout_seconds: Union[int, float, None] = None,
-        timeout_callback: Optional[Callable[[Any], Any]] = None,
-        timeout_callback_args: Optional[Tuple[Any]] = None,
-        timeout_callback_kwargs: Optional[Dict[str, Any]] = None,
-    ) -> int:
+    proc: subprocess.Popen,
+    line_callback: Callable[[bytes], Any],
+    timeout_seconds: Union[int, float, None] = None,
+    timeout_callback: Optional[Callable[[Any], Any]] = None,
+    timeout_callback_args: Optional[Tuple[Any]] = None,
+    timeout_callback_kwargs: Optional[Dict[str, Any]] = None,
+) -> int:
     """
     Poll a process and execute a callback function for each line printed to the process's `stdout`.
     """
     from meerschaum.utils.threading import Timer
+    from meerschaum.utils.warnings import warn
 
     def timeout_handler():
         nonlocal timeout_callback_args, timeout_callback_kwargs
-        proc.terminate()
+        try:
+            if platform.system() != 'Windows':
+                ### The process being killed may have children.
+                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+            else:
+                proc.send_signal(signal.CTRL_BREAK_EVENT)
+            proc.terminate()
+        except Exception as e:
+            warn(f"Failed to kill process:\n{e}")
         if timeout_callback_args is None:
             timeout_callback_args = []
         if timeout_callback_kwargs is None: