meerschaum 2.4.13__py3-none-any.whl → 2.5.1__py3-none-any.whl

meerschaum/_internal/entry.py

@@ -13,6 +13,7 @@ import os
 import sys
 import pathlib
 
+import meerschaum as mrsm
 from meerschaum.utils.typing import SuccessTuple, List, Optional, Dict, Callable, Any
 from meerschaum.config.static import STATIC_CONFIG as _STATIC_CONFIG
 
@@ -229,7 +230,7 @@ def entry_with_args(
         _ = get_shell(**kw).cmdloop()
         return True, "Success"
 
-    skip_schedule = False
+    _skip_schedule = False
 
     executor_keys = kw.get('executor_keys', None)
     if executor_keys is None:
@@ -246,7 +247,7 @@ def entry_with_args(
             api_label = executor_keys.split(':')[-1]
             kw['action'].insert(0, 'api')
             kw['action'].insert(1, api_label)
-            skip_schedule = True
+            _skip_schedule = True
 
     ### If the `--daemon` flag is present, prepend 'start job'.
     if kw.get('daemon', False) and kw['action'][0] != 'stack':
@@ -271,7 +272,7 @@ def entry_with_args(
         and kw['action'][0] == 'start'
         and kw['action'][1] in ('job', 'jobs')
     ):
-        skip_schedule = True
+        _skip_schedule = True
 
     kw['action'] = remove_leading_action(kw['action'], _actions=_actions)
 
@@ -279,10 +280,11 @@ def entry_with_args(
         _do_action_wrapper,
         action_function,
         plugin_name,
+        _skip_schedule=_skip_schedule,
         **kw
     )
 
-    if kw.get('schedule', None) and not skip_schedule:
+    if kw.get('schedule', None) and not _skip_schedule:
         from meerschaum.utils.schedule import schedule_function
         from meerschaum.utils.misc import interval_str
         import time
@@ -304,7 +306,12 @@ def entry_with_args(
     return result
 
 
-def _do_action_wrapper(action_function, plugin_name, **kw):
+def _do_action_wrapper(
+    action_function,
+    plugin_name,
+    _skip_schedule: bool = False,
+    **kw
+):
     from meerschaum.plugins import Plugin
     from meerschaum.utils.venv import Venv
     from meerschaum.utils.misc import filter_keywords
@@ -328,6 +335,10 @@ def _do_action_wrapper(action_function, plugin_name, **kw):
             )
         except KeyboardInterrupt:
             result = False, f"Cancelled action `{action_name.lstrip()}`."
+
+    if kw.get('schedule', None) and not _skip_schedule:
+        mrsm.pprint(result)
+
     return result
 
 _shells = []

meerschaum/actions/upgrade.py

@@ -109,6 +109,7 @@ def _upgrade_meerschaum(
 class NoVenv:
     pass
 
+
 def _upgrade_packages(
     action: Optional[List[str]] = None,
     venv: Union[str, None, NoVenv] = NoVenv,
@@ -121,7 +122,7 @@ def _upgrade_packages(
     """
     Upgrade and install dependencies.
     If provided, upgrade only a dependency group, otherwise default to `full`.
-    
+
     Examples:
         upgrade packages
         upgrade packages full
@@ -160,7 +161,7 @@ def _upgrade_packages(
     to_install = [
         install_name
         for install_name in to_install
-        if install_name not in prereleases_to_install
+        if (install_name not in prereleases_to_install) or group == '_internal'
     ]
 
     success, msg = False, f"Nothing installed."

meerschaum/api/dash/pipes.py

@@ -346,6 +346,8 @@ def accordion_items_from_pipe(
     for item in skip_items:
         _ = items_titles.pop(item, None)
 
+    pipe_meta_str = json.dumps(pipe.meta, sort_keys=True)
+
     ### Only generate items if they're in the `active_items` list.
     items_bodies = {}
     if 'overview' in active_items:
@@ -358,22 +360,103 @@ def accordion_items_from_pipe(
             html.Tr([html.Td("Instance"), html.Td(html.Pre(f"{pipe.instance_keys}"))]),
             html.Tr([html.Td("Target Table"), html.Td(html.Pre(f"{pipe.target}"))]),
         ]
-        columns = pipe.columns.copy()
-        if columns:
-            datetime_index = columns.pop('datetime', None)
-            columns_items = []
-            if datetime_index:
-                columns_items.append(html.Li(f"{datetime_index} (datetime)"))
-            columns_items.extend([
-                html.Li(f"{col}")
-                for col_key, col in columns.items()
-            ])
+
+        indices_header = [
+            html.Thead(
+                html.Tr(
+                    [
+                        html.Th(
+                            html.Span(
+                                "Key",
+                                id={'type': 'key-table-header', 'id': pipe_meta_str},
+                                style={"textDecoration": "underline", "cursor": "pointer"},
+                            ),
+                        ),
+                        html.Th(
+                            html.Span(
+                                "Column",
+                                id={'type': 'column-table-header', 'id': pipe_meta_str},
+                                style={"textDecoration": "underline", "cursor": "pointer"},
+                            ),
+                        ),
+                        html.Th(
+                            html.Span(
+                                "Index",
+                                id={'type': 'index-table-header', 'id': pipe_meta_str},
+                                style={"textDecoration": "underline", "cursor": "pointer"},
+                            ),
+                        ),
+                        html.Th(
+                            html.Span(
+                                "Is Composite",
+                                id={'type': 'is-composite-table-header', 'id': pipe_meta_str},
+                                style={"textDecoration": "underline", "cursor": "pointer"},
+                            ),
+                        ),
+                        dbc.Tooltip(
+                            "Unique reference name for the index "
+                            "(e.g. `datetime` for the range axis)",
+                            target={'type': 'key-table-header', 'id': pipe_meta_str},
+                        ),
+                        dbc.Tooltip(
+                            "The actual column (field name) in the target dataset.",
+                            target={'type': 'column-table-header', 'id': pipe_meta_str},
+                        ),
+                        dbc.Tooltip(
+                            "The name of the index created on the given columns.",
+                            target={'type': 'index-table-header', 'id': pipe_meta_str},
+                        ),
+                        dbc.Tooltip(
+                            "Whether the column is used in the composite primary key "
+                            "to determine updates.",
+                            target={'type': 'is-composite-table-header', 'id': pipe_meta_str},
+                        ),
+                    ]
+                )
+            )
+        ]
+
+        indices = pipe.indices
+        columns = pipe.columns
+        index_column_names = pipe.get_indices()
+        indices_rows = []
+        for ix_key, ix_name in index_column_names.items():
+            col = columns.get(ix_key, None)
+            ix_cols = indices.get(ix_key, None)
+            if not col and not ix_cols:
+                continue
+            if not isinstance(ix_cols, (list, tuple)):
+                ix_cols = [ix_cols]
+            if col:
+                col_item = html.Pre(col)
+            elif len(ix_cols) == 1:
+                col_item = html.Pre(ix_cols[0])
+            else:
+                col_item = html.Pre(html.Ul([html.Li(_c) for _c in ix_cols]))
+            is_composite_item = "✅" if col else ""
+            ix_key_item = html.Pre(ix_key) if ix_key != 'datetime' else html.Pre(f"🕓 {ix_key}")
+            indices_rows.append(
+                html.Tr([
+                    html.Td(ix_key_item),
+                    html.Td(col_item),
+                    html.Td(html.Pre(ix_name)),
+                    html.Td(is_composite_item),
+                ])
+            )
+        indices_table = dbc.Table(
+            indices_header + [html.Tbody(indices_rows)],
+            bordered=True,
+            hover=False,
+            striped=True,
+        )
+        if indices_rows:
             overview_rows.append(
                 html.Tr([
-                    html.Td("Indices" if len(columns_items) != 1 else "Index"),
-                    html.Td(html.Pre(html.Ul(columns_items))),
+                    html.Td("Indices" if len(indices_rows) != 1 else "Index"),
+                    html.Td(indices_table),
                 ])
             )
+
         tags = pipe.tags
         if tags:
             tags_items = html.Ul([
@@ -420,7 +503,6 @@ def accordion_items_from_pipe(
         if newest_time is not None:
             stats_rows.append(html.Tr([html.Td("Newest time"), html.Td(str(newest_time))]))
 
-
         items_bodies['stats'] = dbc.Table(stats_header + [html.Tbody(stats_rows)], hover=True)
 
     if 'columns' in active_items:

meerschaum/api/routes/_webterm.py

@@ -13,6 +13,7 @@ from meerschaum.utils.packages import attempt_import
 from meerschaum.api.dash.sessions import is_session_authenticated
 fastapi, fastapi_responses = attempt_import('fastapi', 'fastapi.responses')
 import starlette
+httpcore = attempt_import('httpcore')
 httpx = attempt_import('httpx')
 websockets = attempt_import('websockets')
 Request = fastapi.Request

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.4.13"
+__version__ = "2.5.1"

meerschaum/connectors/sql/_create_engine.py

@@ -63,6 +63,8 @@ flavor_configs = {
             'fast_executemany': True,
             'isolation_level': 'AUTOCOMMIT',
             'use_setinputsizes': False,
+            'pool_pre_ping': True,
+            'ignore_no_transaction_on_rollback': True,
         },
         'omit_create_engine': {'method',},
         'to_sql': {
@@ -189,15 +191,18 @@ def create_engine(
     ### Install and patch required drivers.
     if self.flavor in install_flavor_drivers:
         attempt_import(*install_flavor_drivers[self.flavor], debug=debug, lazy=False, warn=False)
+        if self.flavor == 'mssql':
+            pyodbc = attempt_import('pyodbc', debug=debug, lazy=False, warn=False)
+            pyodbc.pooling = False
     if self.flavor in require_patching_flavors:
         from meerschaum.utils.packages import determine_version, _monkey_patch_get_distribution
         import pathlib
         for install_name, import_name in require_patching_flavors[self.flavor]:
             pkg = attempt_import(
                 import_name,
-                debug = debug,
-                lazy = False,
-                warn = False
+                debug=debug,
+                lazy=False,
+                warn=False
             )
             _monkey_patch_get_distribution(
                 install_name, determine_version(pathlib.Path(pkg.__file__), venv='mrsm')

meerschaum/connectors/sql/_fetch.py

@@ -31,7 +31,7 @@ def fetch(
     ----------
     pipe: mrsm.Pipe
         The pipe object which contains the `fetch` metadata.
-        
+
         - pipe.columns['datetime']: str
             - Name of the datetime column for the remote table.
         - pipe.parameters['fetch']: Dict[str, Any]
@@ -196,7 +196,7 @@ def get_pipe_metadef(
             dateadd_str(
                 flavor=self.flavor,
                 datepart='minute',
-                number=((-1 * btm) if apply_backtrack else 0), 
+                number=((-1 * btm) if apply_backtrack else 0),
                 begin=begin,
             )
             if begin

meerschaum/connectors/sql/_instance.py

@@ -88,9 +88,9 @@ def _drop_temporary_tables(self, debug: bool = False) -> SuccessTuple:
     from meerschaum.connectors.sql.tables import get_tables
     sqlalchemy = mrsm.attempt_import('sqlalchemy')
     temp_tables_table = get_tables(
-        mrsm_instance = self,
-        create = False,
-        debug = debug,
+        mrsm_instance=self,
+        create=False,
+        debug=debug,
     )['temp_tables']
     query = (
         sqlalchemy.select(temp_tables_table.c.table)

meerschaum/connectors/sql/_pipes.py

@@ -384,7 +384,7 @@ def get_create_index_queries(
 
     Returns
     -------
-    A dictionary of column names mapping to lists of queries.
+    A dictionary of index names mapping to lists of queries.
     """
     ### NOTE: Due to recent breaking changes in DuckDB, indices don't behave properly.
     if self.flavor == 'duckdb':
@@ -400,7 +400,8 @@ def get_create_index_queries(
     index_queries = {}
 
     upsert = pipe.parameters.get('upsert', False) and (self.flavor + '-upsert') in update_queries
-    indices = pipe.get_indices()
+    index_names = pipe.get_indices()
+    indices = pipe.indices
 
     _datetime = pipe.get_columns('datetime', error=False)
     _datetime_type = pipe.dtypes.get(_datetime, 'datetime64[ns]')
@@ -409,8 +410,8 @@ def get_create_index_queries(
         if _datetime is not None else None
     )
     _datetime_index_name = (
-        sql_item_name(indices['datetime'], self.flavor, None)
-        if indices.get('datetime', None)
+        sql_item_name(index_names['datetime'], self.flavor, None)
+        if index_names.get('datetime', None)
         else None
     )
     _id = pipe.get_columns('id', error=False)
@@ -421,8 +422,8 @@ def get_create_index_queries(
     )
 
     _id_index_name = (
-        sql_item_name(indices['id'], self.flavor, None)
-        if indices.get('id', None)
+        sql_item_name(index_names['id'], self.flavor, None)
+        if index_names.get('id', None)
         else None
     )
     _pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
@@ -491,18 +492,22 @@ def get_create_index_queries(
         if id_query is not None:
             index_queries[_id] = id_query if isinstance(id_query, list) else [id_query]
 
-
     ### Create indices for other labels in `pipe.columns`.
-    other_indices = {
+    other_index_names = {
         ix_key: ix_unquoted
-        for ix_key, ix_unquoted in pipe.get_indices().items()
+        for ix_key, ix_unquoted in index_names.items()
         if ix_key not in ('datetime', 'id')
     }
-    for ix_key, ix_unquoted in other_indices.items():
+    for ix_key, ix_unquoted in other_index_names.items():
         ix_name = sql_item_name(ix_unquoted, self.flavor, None)
-        col = pipe.columns[ix_key]
-        col_name = sql_item_name(col, self.flavor, None)
-        index_queries[col] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({col_name})"]
+        cols = indices[ix_key]
+        if not isinstance(cols, (list, tuple)):
+            cols = [cols]
+        cols_names = [sql_item_name(col, self.flavor, None) for col in cols if col]
+        if not cols_names:
+            continue
+        cols_names_str = ", ".join(cols_names)
+        index_queries[ix_key] = [f"CREATE INDEX {ix_name} ON {_pipe_name} ({cols_names_str})"]
 
     existing_cols_types = pipe.get_columns_types(debug=debug)
     indices_cols_str = ', '.join(

meerschaum/connectors/sql/_sql.py

@@ -17,8 +17,9 @@ from meerschaum.utils.warnings import warn
 ### database flavors that can use bulk insert
 _bulk_flavors = {'postgresql', 'timescaledb', 'citus'}
 ### flavors that do not support chunks
-_disallow_chunks_flavors = {'duckdb', 'mssql'}
+_disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000,}
+SKIP_READ_TRANSACTION_FLAVORS: list[str] = ['mssql']
 
 
 def read(
@@ -97,7 +98,7 @@ def read(
         Defaults to `SQLConnector.schema`.
 
     as_chunks: bool, default False
-        If `True`, return a list of DataFrames. 
+        If `True`, return a list of DataFrames.
         Otherwise return a single DataFrame.
 
     as_iterator: bool, default False
@@ -127,7 +128,6 @@ def read(
     from meerschaum.utils.pool import get_pool
     from meerschaum.utils.dataframe import chunksize_to_npartitions, get_numeric_cols
     import warnings
-    import inspect
     import traceback
     from decimal import Decimal
     pd = import_pandas()
@@ -140,6 +140,7 @@ def read(
         chunksize = None
     schema = schema or self.schema
 
+    pool = get_pool(workers=workers)
     sqlalchemy = attempt_import("sqlalchemy")
     default_chunksize = self._sys_config.get('chunksize', None)
     chunksize = chunksize if chunksize != -1 else default_chunksize
@@ -157,7 +158,7 @@ def read(
                     f"The specified chunksize of {chunksize} exceeds the maximum of "
                     + f"{_max_chunks_flavors[self.flavor]} for flavor '{self.flavor}'.\n"
                     + f"    Falling back to a chunksize of {_max_chunks_flavors[self.flavor]}.",
-                    stacklevel = 3,
+                    stacklevel=3,
                 )
             chunksize = _max_chunks_flavors[self.flavor]
 
@@ -184,8 +185,8 @@ def read(
         truncated_table_name = truncate_item_name(str(query_or_table), self.flavor)
         if truncated_table_name != str(query_or_table) and not silent:
             warn(
-                f"Table '{name}' is too long for '{self.flavor}',"
-                + f" will instead create the table '{truncated_name}'."
+                f"Table '{query_or_table}' is too long for '{self.flavor}',"
+                + f" will instead read the table '{truncated_table_name}'."
             )
 
         query_or_table = sql_item_name(str(query_or_table), self.flavor, schema)
@@ -204,6 +205,34 @@ def read(
 
     chunk_list = []
     chunk_hook_results = []
+    def _process_chunk(_chunk, _retry_on_failure: bool = True):
+        if not as_hook_results:
+            chunk_list.append(_chunk)
+        if chunk_hook is None:
+            return None
+
+        result = None
+        try:
+            result = chunk_hook(
+                _chunk,
+                workers=workers,
+                chunksize=chunksize,
+                debug=debug,
+                **kw
+            )
+        except Exception:
+            result = False, traceback.format_exc()
+            from meerschaum.utils.formatting import get_console
+            if not silent:
+                get_console().print_exception()
+
+        ### If the chunk fails to process, try it again one more time.
+        if isinstance(result, tuple) and result[0] is False:
+            if _retry_on_failure:
+                return _process_chunk(_chunk, _retry_on_failure=False)
+
+        return result
+
     try:
         stream_results = not as_iterator and chunk_hook is not None and chunksize is not None
         with warnings.catch_warnings():
@@ -235,52 +264,32 @@ def read(
                 )
             else:
 
-                with self.engine.begin() as transaction:
-                    with transaction.execution_options(stream_results=stream_results) as connection:
-                        chunk_generator = pd.read_sql_query(
-                            formatted_query,
-                            connection,
-                            **read_sql_query_kwargs
+                def get_chunk_generator(connectable):
+                    chunk_generator = pd.read_sql_query(
+                        formatted_query,
+                        self.engine,
+                        **read_sql_query_kwargs
+                    )
+                    to_return = (
+                        chunk_generator
+                        if as_iterator or chunksize is None
+                        else (
+                            list(pool.imap(_process_chunk, chunk_generator))
+                            if as_hook_results
+                            else None
                         )
+                    )
+                    return chunk_generator, to_return
+
+                if self.flavor in SKIP_READ_TRANSACTION_FLAVORS:
+                    chunk_generator, to_return = get_chunk_generator(self.engine)
+                else:
+                    with self.engine.begin() as transaction:
+                        with transaction.execution_options(stream_results=stream_results) as connection:
+                            chunk_generator, to_return = get_chunk_generator(connection)
 
-                        ### `stream_results` must be False (will load everything into memory).
-                        if as_iterator or chunksize is None:
-                            return chunk_generator
-
-                        ### We must consume the generator in this context if using server-side cursors.
-                        if stream_results:
-
-                            pool = get_pool(workers=workers)
-
-                            def _process_chunk(_chunk, _retry_on_failure: bool = True):
-                                if not as_hook_results:
-                                    chunk_list.append(_chunk)
-                                result = None
-                                if chunk_hook is not None:
-                                    try:
-                                        result = chunk_hook(
-                                            _chunk,
-                                            workers = workers,
-                                            chunksize = chunksize,
-                                            debug = debug,
-                                            **kw
-                                        )
-                                    except Exception as e:
-                                        result = False, traceback.format_exc()
-                                        from meerschaum.utils.formatting import get_console
-                                        if not silent:
-                                            get_console().print_exception()
-
-                                    ### If the chunk fails to process, try it again one more time.
-                                    if isinstance(result, tuple) and result[0] is False:
-                                        if _retry_on_failure:
-                                            return _process_chunk(_chunk, _retry_on_failure=False)
-
-                                return result
-
-                            chunk_hook_results = list(pool.imap(_process_chunk, chunk_generator))
-                            if as_hook_results:
-                                return chunk_hook_results
+                if to_return is not None:
+                    return to_return
 
     except Exception as e:
         if debug:

meerschaum/core/Pipe/__init__.py

@@ -98,6 +98,8 @@ class Pipe:
         attributes,
         parameters,
         columns,
+        indices,
+        indexes,
         dtypes,
         get_columns,
         get_columns_types,
@@ -145,6 +147,7 @@ class Pipe:
         location: Optional[str] = None,
         parameters: Optional[Dict[str, Any]] = None,
         columns: Union[Dict[str, str], List[str], None] = None,
+        indices: Optional[Dict[str, Union[str, List[str]]]] = None,
         tags: Optional[List[str]] = None,
         target: Optional[str] = None,
         dtypes: Optional[Dict[str, str]] = None,
@@ -156,6 +159,7 @@ class Pipe:
         connector_keys: Optional[str] = None,
         metric_key: Optional[str] = None,
         location_key: Optional[str] = None,
+        indexes: Union[Dict[str, str], List[str], None] = None,
     ):
         """
         Parameters
@@ -174,10 +178,14 @@ class Pipe:
             e.g. columns and other attributes.
             You can edit these parameters with `edit pipes`.
 
-        columns: Optional[Dict[str, str]], default None
+        columns: Union[Dict[str, str], List[str], None], default None
             Set the `columns` dictionary of `parameters`.
             If `parameters` is also provided, this dictionary is added under the `'columns'` key.
 
+        indices: Optional[Dict[str, Union[str, List[str]]]], default None
+            Set the `indices` dictionary of `parameters`.
+            If `parameters` is also provided, this dictionary is added under the `'indices'` key.
+
         tags: Optional[List[str]], default None
             A list of strings to be added under the `'tags'` key of `parameters`.
             You can select pipes with certain tags using `--tags`.
@@ -255,6 +263,20 @@ class Pipe:
         elif columns is not None:
             warn(f"The provided columns are of invalid type '{type(columns)}'.")
 
+        indices = (
+            indices
+            or indexes
+            or self._attributes.get('parameters', {}).get('indices', None)
+            or self._attributes.get('parameters', {}).get('indexes', None)
+        ) or columns
+        if isinstance(indices, dict):
+            indices_key = (
+                'indexes'
+                if 'indexes' in self._attributes['parameters']
+                else 'indices'
+            )
+            self._attributes['parameters'][indices_key] = indices
+
         if isinstance(tags, (list, tuple)):
             self._attributes['parameters']['tags'] = tags
         elif tags is not None: