meerschaum 2.4.10__py3-none-any.whl → 2.4.12__py3-none-any.whl

meerschaum/connectors/sql/_pipes.py

@@ -2096,7 +2096,7 @@ def get_pipe_rowcount(
     An `int` for the number of rows if the `pipe` exists, otherwise `None`.
 
     """
-    from meerschaum.utils.sql import dateadd_str, sql_item_name, NO_CTE_FLAVORS
+    from meerschaum.utils.sql import dateadd_str, sql_item_name, wrap_query_with_cte
     from meerschaum.connectors.sql._fetch import get_pipe_query
     if remote:
         msg = f"'fetch:definition' must be an attribute of {pipe} to get a remote rowcount."
@@ -2175,20 +2175,10 @@ def get_pipe_rowcount(
         if not remote
         else get_pipe_query(pipe)
     )
-    query = (
-        f"""
-        WITH src AS ({src})
-        SELECT COUNT(*)
-        FROM src
-        """
-    ) if self.flavor not in ('mysql', 'mariadb') else (
-        f"""
-        SELECT COUNT(*)
-        FROM ({src}) AS src
-        """
-    )
+    parent_query = f"SELECT COUNT(*)\nFROM {sql_item_name('src', self.flavor)}"
+    query = wrap_query_with_cte(src, parent_query, self.flavor)
     if begin is not None or end is not None:
-        query += "WHERE"
+        query += "\nWHERE"
     if begin is not None:
         query += f"""
         {dt} >= {dateadd_str(self.flavor, datepart='minute', number=0, begin=begin)}
@@ -2330,10 +2320,10 @@ def clear_pipe(
 
 
 def get_pipe_table(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> sqlalchemy.Table:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Union['sqlalchemy.Table', None]:
     """
     Return the `sqlalchemy.Table` object for a `mrsm.Pipe`.
 
@@ -2352,18 +2342,18 @@ def get_pipe_table(
         return None
     return get_sqlalchemy_table(
         pipe.target,
-        connector = self,
-        schema = self.get_pipe_schema(pipe),
-        debug = debug,
-        refresh = True,
+        connector=self,
+        schema=self.get_pipe_schema(pipe),
+        debug=debug,
+        refresh=True,
     )
 
 
 def get_pipe_columns_types(
-        self,
-        pipe: mrsm.Pipe,
-        debug: bool = False,
-    ) -> Dict[str, str]:
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+) -> Dict[str, str]:
     """
     Get the pipe's columns and types.
 
@@ -2394,8 +2384,8 @@ def get_pipe_columns_types(
         return get_table_cols_types(
             pipe.target,
             self,
-            flavor = self.flavor,
-            schema = self.schema,
+            flavor=self.flavor,
+            schema=self.get_pipe_schema(pipe),
         )
 
     table_columns = {}
@@ -2448,6 +2438,7 @@ def get_add_columns_queries(
     from meerschaum.utils.sql import (
         sql_item_name,
         SINGLE_ALTER_TABLE_FLAVORS,
+        get_table_cols_types,
     )
     from meerschaum.utils.dtypes.sql import (
         get_pd_type_from_db_type,
@@ -2480,6 +2471,14 @@ def get_add_columns_queries(
     db_cols_types = {
         col: get_pd_type_from_db_type(str(typ.type))
         for col, typ in table_obj.columns.items()
+    } if table_obj is not None else {
+        col: get_pd_type_from_db_type(typ)
+        for col, typ in get_table_cols_types(
+            pipe.target,
+            self,
+            schema=self.get_pipe_schema(pipe),
+            debug=debug,
+        ).items()
     }
     new_cols = set(df_cols_types) - set(db_cols_types)
     if not new_cols:
@@ -2552,7 +2551,7 @@ def get_alter_columns_queries(
     """
     if not pipe.exists(debug=debug):
         return []
-    from meerschaum.utils.sql import sql_item_name, DROP_IF_EXISTS_FLAVORS
+    from meerschaum.utils.sql import sql_item_name, DROP_IF_EXISTS_FLAVORS, get_table_cols_types
     from meerschaum.utils.dataframe import get_numeric_cols
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import (
@@ -2583,6 +2582,14 @@ def get_alter_columns_queries(
     db_cols_types = {
         col: get_pd_type_from_db_type(str(typ.type))
         for col, typ in table_obj.columns.items()
+    } if table_obj is not None else {
+        col: get_pd_type_from_db_type(typ)
+        for col, typ in get_table_cols_types(
+            pipe.target,
+            self,
+            schema=self.get_pipe_schema(pipe),
+            debug=debug,
+        ).items()
     }
     pipe_bool_cols = [col for col, typ in pipe.dtypes.items() if are_dtypes_equal(str(typ), 'bool')]
     pd_db_df_aliases = {

meerschaum/core/Pipe/_data.py

@@ -453,7 +453,7 @@ def get_backtrack_data(
     backtrack_interval = self.get_backtrack_interval(debug=debug)
     if backtrack_minutes is None:
         backtrack_minutes = (
-            (backtrack_interval.total_seconds() * 60)
+            (backtrack_interval.total_seconds() / 60)
             if isinstance(backtrack_interval, timedelta)
             else backtrack_interval
         )

meerschaum/utils/formatting/__init__.py

@@ -11,7 +11,7 @@ import platform
 import os
 import sys
 import meerschaum as mrsm
-from meerschaum.utils.typing import Optional, Union, Any, Dict
+from meerschaum.utils.typing import Optional, Union, Any, Dict, Iterable
 from meerschaum.utils.formatting._shell import make_header
 from meerschaum.utils.formatting._pprint import pprint
 from meerschaum.utils.formatting._pipes import (
@@ -322,14 +322,14 @@ def format_success_tuple(
     from meerschaum.config import get_config
     status_config = get_config('formatting', status, patch=True)
 
-    msg = (' ' * left_padding) + status_config[CHARSET]['icon'] + ' ' + str(tup[1])
+    msg = (' ' * left_padding) + status_config[CHARSET]['icon'] + ' ' + str(highlight_pipes(tup[1]))
     lines = msg.split('\n')
     lines = [lines[0]] + [
         (('    ' + line if not line.startswith(' ') else line))
         for line in lines[1:]
     ]
     if ANSI:
-        lines[0] = fill_ansi(highlight_pipes(lines[0]), **status_config['ansi']['rich'])
+        lines[0] = fill_ansi(lines[0], **status_config['ansi']['rich'])
 
     msg = '\n'.join(lines)
     msg = ('\n' * upper_padding) + msg + ('\n' * lower_padding)
@@ -337,7 +337,7 @@ def format_success_tuple(
 
 
 def print_options(
-    options: Optional[Dict[str, Any]] = None,
+    options: Optional[Iterable[Any]] = None,
     nopretty: bool = False,
     no_rich: bool = False,
     name: str = 'options',
@@ -345,6 +345,7 @@ def print_options(
     num_cols: Optional[int] = None,
     adjust_cols: bool = True,
     sort_options: bool = False,
+    number_options: bool = False,
     **kw
 ) -> None:
     """
@@ -373,6 +374,12 @@ def print_options(
     adjust_cols: bool, default True
         If `True`, adjust the number of columns depending on the terminal size.
 
+    sort_options: bool, default False
+        If `True`, print the options in sorted order.
+
+    number_options: bool, default False
+        If `True`, print the option's number in the list (1 index).
+
     """
     import os
     from meerschaum.utils.packages import import_rich
@@ -398,9 +405,10 @@ def print_options(
             print()
             print(make_header(_header))
         ### print actions
-        for option in _options:
+        for i, option in enumerate(_options):
+            marker = '-' if not number_options else (str(i + 1) + '.')
             if not nopretty:
-                print("  - ", end="")
+                print(f"  {marker} ", end="")
             print(option)
         if not nopretty:
             print()
@@ -434,11 +442,11 @@ def print_options(
 
     if _header is not None:
         table = Table(
-            title = _header,
-            box = box.SIMPLE,
-            show_header = False,
-            show_footer = False,
-            title_style = '',
+            title=_header,
+            box=box.SIMPLE,
+            show_header=False,
+            show_footer=False,
+            title_style='',
             expand = True,
         )
     else:
@@ -447,18 +455,26 @@ def print_options(
         table.add_column()
 
     if len(_options) < 12:
-        # If fewer than 12 items, use a single column
-        for option in _options:
-            table.add_row(Text.from_ansi(highlight_pipes(option)))
+        ### If fewer than 12 items, use a single column
+        for i, option in enumerate(_options):
+            item = highlight_pipes(option)
+            if number_options:
+                item = str(i + 1) + '. ' + item
+            table.add_row(Text.from_ansi(item))
     else:
-        # Otherwise, use multiple columns as before
+        ### Otherwise, use multiple columns as before
         num_rows = (len(_options) + num_cols - 1) // num_cols
+        item_ix = 0
         for i in range(num_rows):
             row = []
             for j in range(num_cols):
                 index = i + j * num_rows
                 if index < len(_options):
-                    row.append(Text.from_ansi(highlight_pipes(_options[index])))
+                    item = highlight_pipes(_options[index])
+                    if number_options:
+                        item = str(i + 1) + '. ' + item
+                    row.append(Text.from_ansi(item))
+                    item_ix += 1
                 else:
                     row.append('')
             table.add_row(*row)

meerschaum/utils/formatting/_pipes.py

@@ -323,7 +323,7 @@ def pipe_repr(
     )
     if as_rich_text:
         return text_obj
-    return rich_text_to_str(text_obj)
+    return rich_text_to_str(text_obj).replace('\n', '')
 
 
 def highlight_pipes(message: str) -> str:

meerschaum/utils/formatting/_shell.py

@@ -10,10 +10,11 @@ from re import sub
 from meerschaum.utils.threading import Lock
 _locks = {'_tried_clear_command': Lock()}
 
+
 def make_header(
-        message : str,
-        ruler : str = '─',
-    ) -> str:
+    message: str,
+    ruler: str = '─',
+) -> str:
     """Format a message string with a ruler.
     Length of the ruler is the length of the longest word.
     

meerschaum/utils/prompt.py

@@ -10,17 +10,18 @@ from __future__ import annotations
 import os
 from meerschaum.utils.typing import Any, Union, Optional, Tuple, List
 
+
 def prompt(
-        question: str,
-        icon: bool = True,
-        default: Union[str, Tuple[str, str], None] = None,
-        default_editable: Optional[str] = None,
-        detect_password: bool = True,
-        is_password: bool = False,
-        wrap_lines: bool = True,
-        noask: bool = False,
-        **kw: Any
-    ) -> str:
+    question: str,
+    icon: bool = True,
+    default: Union[str, Tuple[str, str], None] = None,
+    default_editable: Optional[str] = None,
+    detect_password: bool = True,
+    is_password: bool = False,
+    wrap_lines: bool = True,
+    noask: bool = False,
+    **kw: Any
+) -> str:
     """
     Ask the user a question and return the answer.
     Wrapper around `prompt_toolkit.prompt()` with modified behavior.
@@ -75,7 +76,7 @@ def prompt(
     ### if a default is provided, append it to the question.
     default_answer = default
     if default is not None:
-        question += f" (default: "
+        question += " (default: "
         if isinstance(default, tuple) and len(default) > 1:
             question += f"{default[0]} [{default[1]}]"
             default_answer = default[0]
@@ -86,7 +87,7 @@ def prompt(
     ### detect password
     if (detect_password and 'password' in question.lower()) or is_password:
         kw['is_password'] = True
-  
+
     ### Add the icon and only color the first line.
     lines = question.split('\n')
     first_line = lines[0]
@@ -107,15 +108,15 @@ def prompt(
         answer = (
             prompt_toolkit.prompt(
                 prompt_toolkit.formatted_text.ANSI(question),
-                wrap_lines = wrap_lines,
-                default = default_editable or '',
+                wrap_lines=wrap_lines,
+                default=default_editable or '',
                 **filter_keywords(prompt_toolkit.prompt, **kw)
             ) if not noask else ''
         )
     else:
         print(question, end='\n', flush=True)
         try:
-            answer = input()
+            answer = input() if not noask else ''
         except EOFError:
             answer = ''
     if noask:

meerschaum/utils/sql.py

@@ -758,11 +758,11 @@ def build_where(
 
 
 def table_exists(
-        table: str,
-        connector: mrsm.connectors.sql.SQLConnector,
-        schema: Optional[str] = None,
-        debug: bool = False,
-    ) -> bool:
+    table: str,
+    connector: mrsm.connectors.sql.SQLConnector,
+    schema: Optional[str] = None,
+    debug: bool = False,
+) -> bool:
     """Check if a table exists.
 
     Parameters
@@ -793,12 +793,12 @@ def table_exists(
 
 
 def get_sqlalchemy_table(
-        table: str,
-        connector: Optional[meerschaum.connectors.sql.SQLConnector] = None,
-        schema: Optional[str] = None,
-        refresh: bool = False,
-        debug: bool = False,
-    ) -> 'sqlalchemy.Table':
+    table: str,
+    connector: Optional[mrsm.connectors.sql.SQLConnector] = None,
+    schema: Optional[str] = None,
+    refresh: bool = False,
+    debug: bool = False,
+) -> Union['sqlalchemy.Table', None]:
     """
     Construct a SQLAlchemy table from its name.
 
@@ -829,6 +829,9 @@ def get_sqlalchemy_table(
         from meerschaum import get_connector
         connector = get_connector('sql')
 
+    if connector.flavor == 'duckdb':
+        return None
+
     from meerschaum.connectors.sql.tables import get_tables
     from meerschaum.utils.packages import attempt_import
     from meerschaum.utils.warnings import warn
@@ -1333,11 +1336,11 @@ def get_rename_table_queries(
 
 
 def get_create_table_query(
-        query: str,
-        new_table: str,
-        flavor: str,
-        schema: Optional[str] = None,
-    ) -> str:
+    query: str,
+    new_table: str,
+    flavor: str,
+    schema: Optional[str] = None,
+) -> str:
     """
     Return a query to create a new table from a `SELECT` query.
 
@@ -1365,10 +1368,8 @@ def get_create_table_query(
     new_table_name = sql_item_name(new_table, flavor, schema)
     if flavor in ('mssql',):
         query = query.lstrip()
-        original_query = query
         if 'with ' in query.lower():
             final_select_ix = query.lower().rfind('select')
-            def_name = query[len('WITH '):].split(' ', maxsplit=1)[0]
             return (
                 query[:final_select_ix].rstrip() + ',\n'
                 + f"{create_cte_name} AS (\n"
@@ -1405,12 +1406,86 @@ def get_create_table_query(
     return textwrap.dedent(create_table_query)
 
 
+def wrap_query_with_cte(
+    sub_query: str,
+    parent_query: str,
+    flavor: str,
+    cte_name: str = "src",
+) -> str:
+    """
+    Wrap a subquery in a CTE and append an encapsulating query.
+
+    Parameters
+    ----------
+    sub_query: str
+        The query to be referenced. This may itself contain CTEs.
+        Unless `cte_name` is provided, this will be aliased as `src`.
+
+    parent_query: str
+        The larger query to append which references the subquery.
+        This must not contain CTEs.
+
+    flavor: str
+        The database flavor, e.g. `'mssql'`.
+
+    cte_name: str, default 'src'
+        The CTE alias, defaults to `src`.
+
+    Returns
+    -------
+    An encapsulating query which allows you to treat `sub_query` as a temporary table.
+
+    Examples
+    --------
+
+    ```python
+    from meerschaum.utils.sql import wrap_query_with_cte
+    sub_query = "WITH foo AS (SELECT 1 AS val) SELECT (val * 2) AS newval FROM foo"
+    parent_query = "SELECT newval * 3 FROM src"
+    query = wrap_query_with_cte(sub_query, parent_query, 'mssql')
+    print(query)
+    # WITH foo AS (SELECT 1 AS val),
+    # [src] AS (
+    #     SELECT (val * 2) AS newval FROM foo
+    # )
+    # SELECT newval * 3 FROM src
+    ```
+
+    """
+    sub_query = sub_query.lstrip()
+    cte_name_quoted = sql_item_name(cte_name, flavor, None)
+
+    if flavor in NO_CTE_FLAVORS:
+        return (
+            parent_query
+            .replace(cte_name_quoted, '--MRSM_SUBQUERY--')
+            .replace(cte_name, '--MRSM_SUBQUERY--')
+            .replace('--MRSM_SUBQUERY--', f"(\n{sub_query}\n) AS {cte_name_quoted}")
+        )
+
+    if 'with ' in sub_query.lower():
+        final_select_ix = sub_query.lower().rfind('select')
+        return (
+            sub_query[:final_select_ix].rstrip() + ',\n'
+            + f"{cte_name_quoted} AS (\n"
+            + '    ' + sub_query[final_select_ix:]
+            + "\n)\n"
+            + parent_query
+        )
+
+    return (
+        f"WITH {cte_name_quoted} AS (\n"
+        f"    {sub_query}\n"
+        f")\n{parent_query}"
+    )
+
+
 def format_cte_subquery(
-        sub_query: str,
-        flavor: str,
-        sub_name: str = 'src',
-        cols_to_select: Union[List[str], str] = '*',
-    ) -> str:
+    sub_query: str,
+    flavor: str,
+    sub_name: str = 'src',
+    cols_to_select: Union[List[str], str] = '*',
+) -> str:
     """
     Given a subquery, build a wrapper query that selects from the CTE subquery.
 
@@ -1434,28 +1509,25 @@ def format_cte_subquery(
     -------
     A wrapper query that selects from the CTE.
     """
-    import textwrap
     quoted_sub_name = sql_item_name(sub_name, flavor, None)
     cols_str = (
         cols_to_select
         if isinstance(cols_to_select, str)
         else ', '.join([sql_item_name(col, flavor, None) for col in cols_to_select])
     )
-    return textwrap.dedent(
-        f"""
-        SELECT {cols_str}
-        FROM ({sub_query})"""
-        + (f' AS {quoted_sub_name}' if flavor != 'oracle' else '') + """
-        """
+    parent_query = (
+        f"SELECT {cols_str}\n"
+        f"FROM {quoted_sub_name}"
     )
+    return wrap_query_with_cte(sub_query, parent_query, flavor, cte_name=sub_name)
 
 
 def session_execute(
-        session: 'sqlalchemy.orm.session.Session',
-        queries: Union[List[str], str],
-        with_results: bool = False,
-        debug: bool = False,
-    ) -> Union[mrsm.SuccessTuple, Tuple[mrsm.SuccessTuple, List['sqlalchemy.sql.ResultProxy']]]:
+    session: 'sqlalchemy.orm.session.Session',
+    queries: Union[List[str], str],
+    with_results: bool = False,
+    debug: bool = False,
+) -> Union[mrsm.SuccessTuple, Tuple[mrsm.SuccessTuple, List['sqlalchemy.sql.ResultProxy']]]:
     """
     Similar to `SQLConnector.exec_queries()`, execute a list of queries
     and roll back when one fails.

{meerschaum-2.4.10.dist-info → meerschaum-2.4.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meerschaum
-Version: 2.4.10
+Version: 2.4.12
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares