meerschaum 2.4.12__py3-none-any.whl → 2.5.0__py3-none-any.whl

meerschaum/connectors/sql/_sql.py

@@ -17,8 +17,9 @@ from meerschaum.utils.warnings import warn
 ### database flavors that can use bulk insert
 _bulk_flavors = {'postgresql', 'timescaledb', 'citus'}
 ### flavors that do not support chunks
-_disallow_chunks_flavors = {'duckdb', 'mssql'}
+_disallow_chunks_flavors = ['duckdb']
 _max_chunks_flavors = {'sqlite': 1000,}
+SKIP_READ_TRANSACTION_FLAVORS: list[str] = ['mssql']
 
 
 def read(
@@ -97,7 +98,7 @@ def read(
         Defaults to `SQLConnector.schema`.
 
     as_chunks: bool, default False
-        If `True`, return a list of DataFrames. 
+        If `True`, return a list of DataFrames.
         Otherwise return a single DataFrame.
 
     as_iterator: bool, default False
@@ -127,7 +128,6 @@ def read(
     from meerschaum.utils.pool import get_pool
     from meerschaum.utils.dataframe import chunksize_to_npartitions, get_numeric_cols
     import warnings
-    import inspect
     import traceback
     from decimal import Decimal
     pd = import_pandas()
@@ -140,6 +140,7 @@ def read(
         chunksize = None
     schema = schema or self.schema
 
+    pool = get_pool(workers=workers)
     sqlalchemy = attempt_import("sqlalchemy")
     default_chunksize = self._sys_config.get('chunksize', None)
     chunksize = chunksize if chunksize != -1 else default_chunksize
@@ -157,7 +158,7 @@ def read(
                     f"The specified chunksize of {chunksize} exceeds the maximum of "
                     + f"{_max_chunks_flavors[self.flavor]} for flavor '{self.flavor}'.\n"
                     + f"    Falling back to a chunksize of {_max_chunks_flavors[self.flavor]}.",
-                    stacklevel = 3,
+                    stacklevel=3,
                 )
             chunksize = _max_chunks_flavors[self.flavor]
 
@@ -184,8 +185,8 @@ def read(
         truncated_table_name = truncate_item_name(str(query_or_table), self.flavor)
         if truncated_table_name != str(query_or_table) and not silent:
             warn(
-                f"Table '{name}' is too long for '{self.flavor}',"
-                + f" will instead create the table '{truncated_name}'."
+                f"Table '{query_or_table}' is too long for '{self.flavor}',"
+                + f" will instead read the table '{truncated_table_name}'."
             )
 
         query_or_table = sql_item_name(str(query_or_table), self.flavor, schema)
@@ -204,6 +205,34 @@ def read(
 
     chunk_list = []
     chunk_hook_results = []
+    def _process_chunk(_chunk, _retry_on_failure: bool = True):
+        if not as_hook_results:
+            chunk_list.append(_chunk)
+        if chunk_hook is None:
+            return None
+
+        result = None
+        try:
+            result = chunk_hook(
+                _chunk,
+                workers=workers,
+                chunksize=chunksize,
+                debug=debug,
+                **kw
+            )
+        except Exception:
+            result = False, traceback.format_exc()
+            from meerschaum.utils.formatting import get_console
+            if not silent:
+                get_console().print_exception()
+
+        ### If the chunk fails to process, try it again one more time.
+        if isinstance(result, tuple) and result[0] is False:
+            if _retry_on_failure:
+                return _process_chunk(_chunk, _retry_on_failure=False)
+
+        return result
+
     try:
         stream_results = not as_iterator and chunk_hook is not None and chunksize is not None
         with warnings.catch_warnings():
@@ -235,52 +264,32 @@ def read(
                 )
             else:
 
-                with self.engine.begin() as transaction:
-                    with transaction.execution_options(stream_results=stream_results) as connection:
-                        chunk_generator = pd.read_sql_query(
-                            formatted_query,
-                            connection,
-                            **read_sql_query_kwargs
+                def get_chunk_generator(connectable):
+                    chunk_generator = pd.read_sql_query(
+                        formatted_query,
+                        self.engine,
+                        **read_sql_query_kwargs
+                    )
+                    to_return = (
+                        chunk_generator
+                        if as_iterator or chunksize is None
+                        else (
+                            list(pool.imap(_process_chunk, chunk_generator))
+                            if as_hook_results
+                            else None
                         )
+                    )
+                    return chunk_generator, to_return
+
+                if self.flavor in SKIP_READ_TRANSACTION_FLAVORS:
+                    chunk_generator, to_return = get_chunk_generator(self.engine)
+                else:
+                    with self.engine.begin() as transaction:
+                        with transaction.execution_options(stream_results=stream_results) as connection:
+                            chunk_generator, to_return = get_chunk_generator(connection)
 
-                        ### `stream_results` must be False (will load everything into memory).
-                        if as_iterator or chunksize is None:
-                            return chunk_generator
-
-                        ### We must consume the generator in this context if using server-side cursors.
-                        if stream_results:
-
-                            pool = get_pool(workers=workers)
-
-                            def _process_chunk(_chunk, _retry_on_failure: bool = True):
-                                if not as_hook_results:
-                                    chunk_list.append(_chunk)
-                                result = None
-                                if chunk_hook is not None:
-                                    try:
-                                        result = chunk_hook(
-                                            _chunk,
-                                            workers = workers,
-                                            chunksize = chunksize,
-                                            debug = debug,
-                                            **kw
-                                        )
-                                    except Exception as e:
-                                        result = False, traceback.format_exc()
-                                        from meerschaum.utils.formatting import get_console
-                                        if not silent:
-                                            get_console().print_exception()
-
-                                    ### If the chunk fails to process, try it again one more time.
-                                    if isinstance(result, tuple) and result[0] is False:
-                                        if _retry_on_failure:
-                                            return _process_chunk(_chunk, _retry_on_failure=False)
-
-                                return result
-
-                            chunk_hook_results = list(pool.imap(_process_chunk, chunk_generator))
-                            if as_hook_results:
-                                return chunk_hook_results
+                if to_return is not None:
+                    return to_return
 
     except Exception as e:
         if debug:

meerschaum/core/Pipe/__init__.py

@@ -98,6 +98,8 @@ class Pipe:
         attributes,
         parameters,
         columns,
+        indices,
+        indexes,
         dtypes,
         get_columns,
         get_columns_types,
@@ -145,6 +147,7 @@ class Pipe:
         location: Optional[str] = None,
         parameters: Optional[Dict[str, Any]] = None,
         columns: Union[Dict[str, str], List[str], None] = None,
+        indices: Optional[Dict[str, Union[str, List[str]]]] = None,
         tags: Optional[List[str]] = None,
         target: Optional[str] = None,
         dtypes: Optional[Dict[str, str]] = None,
@@ -156,6 +159,7 @@ class Pipe:
         connector_keys: Optional[str] = None,
         metric_key: Optional[str] = None,
         location_key: Optional[str] = None,
+        indexes: Union[Dict[str, str], List[str], None] = None,
     ):
         """
         Parameters
@@ -174,10 +178,14 @@ class Pipe:
             e.g. columns and other attributes.
             You can edit these parameters with `edit pipes`.
 
-        columns: Optional[Dict[str, str]], default None
+        columns: Union[Dict[str, str], List[str], None], default None
             Set the `columns` dictionary of `parameters`.
             If `parameters` is also provided, this dictionary is added under the `'columns'` key.
 
+        indices: Optional[Dict[str, Union[str, List[str]]]], default None
+            Set the `indices` dictionary of `parameters`.
+            If `parameters` is also provided, this dictionary is added under the `'indices'` key.
+
         tags: Optional[List[str]], default None
             A list of strings to be added under the `'tags'` key of `parameters`.
             You can select pipes with certain tags using `--tags`.
@@ -255,6 +263,20 @@ class Pipe:
         elif columns is not None:
             warn(f"The provided columns are of invalid type '{type(columns)}'.")
 
+        indices = (
+            indices
+            or indexes
+            or self._attributes.get('parameters', {}).get('indices', None)
+            or self._attributes.get('parameters', {}).get('indexes', None)
+        ) or columns
+        if isinstance(indices, dict):
+            indices_key = (
+                'indexes'
+                if 'indexes' in self._attributes['parameters']
+                else 'indices'
+            )
+            self._attributes['parameters'][indices_key] = indices
+
         if isinstance(tags, (list, tuple)):
             self._attributes['parameters']['tags'] = tags
         elif tags is not None:

meerschaum/core/Pipe/_attributes.py

@@ -77,15 +77,69 @@ def columns(self) -> Union[Dict[str, str], None]:
 
 
 @columns.setter
-def columns(self, columns: Dict[str, str]) -> None:
+def columns(self, _columns: Union[Dict[str, str], List[str]]) -> None:
     """
     Override the columns dictionary of the in-memory pipe.
     Call `meerschaum.Pipe.edit()` to persist changes.
     """
+    if isinstance(_columns, (list, tuple)):
+        _columns = {col: col for col in _columns}
     if not isinstance(columns, dict):
-        warn(f"{self}.columns must be a dictionary, received {type(columns)}")
+        warn(f"{self}.columns must be a dictionary, received {type(_columns)}.")
         return
-    self.parameters['columns'] = columns
+    self.parameters['columns'] = _columns
+
+
+@property
+def indices(self) -> Union[Dict[str, Union[str, List[str]]], None]:
+    """
+    Return the `indices` dictionary defined in `meerschaum.Pipe.parameters`.
+    """
+    indices_key = (
+        'indexes'
+        if 'indexes' in self.parameters
+        else 'indices'
+    )
+    if indices_key not in self.parameters:
+        self.parameters[indices_key] = {}
+    _indices = self.parameters[indices_key]
+    if not isinstance(_indices, dict):
+        _indices = {}
+        self.parameters[indices_key] = _indices
+    return {**self.columns, **_indices}
+
+
+@property
+def indexes(self) -> Union[Dict[str, Union[str, List[str]]], None]:
+    """
+    Alias for `meerschaum.Pipe.indices`.
+    """
+    return self.indices
+
+
+@indices.setter
+def indices(self, _indices: Union[Dict[str, Union[str, List[str]]], List[str]]) -> None:
+    """
+    Override the indices dictionary of the in-memory pipe.
+    Call `meerschaum.Pipe.edit()` to persist changes.
+    """
+    if not isinstance(_indices, dict):
+        warn(f"{self}.indices must be a dictionary, received {type(_indices)}.")
+        return
+    indices_key = (
+        'indexes'
+        if 'indexes' in self.parameters
+        else 'indices'
+    )
+    self.parameters[indices_key] = _indices
+
+
+@indexes.setter
+def indexes(self, _indexes: Union[Dict[str, Union[str, List[str]]], List[str]]) -> None:
+    """
+    Alias for `meerschaum.Pipe.indices`.
+    """
+    self.indices = _indexes
 
 
 @property
@@ -415,27 +469,55 @@ def guess_datetime(self) -> Union[str, None]:
     """
     Try to determine a pipe's datetime column.
     """
-    dtypes = self.dtypes
+    _dtypes = self.dtypes
 
     ### Abort if the user explictly disallows a datetime index.
-    if 'datetime' in dtypes:
-        if dtypes['datetime'] is None:
+    if 'datetime' in _dtypes:
+        if _dtypes['datetime'] is None:
             return None
 
+    from meerschaum.utils.dtypes import are_dtypes_equal
     dt_cols = [
-        col for col, typ in self.dtypes.items()
-        if str(typ).startswith('datetime')
+        col
+        for col, typ in _dtypes.items()
+        if are_dtypes_equal(typ, 'datetime')
     ]
     if not dt_cols:
         return None
-    return dt_cols[0]    
+    return dt_cols[0]
 
 
 def get_indices(self) -> Dict[str, str]:
     """
-    Return a dictionary in the form of `pipe.columns` but map to index names.
-    """
-    return {
-        ix: (self.target + '_' + col + '_index')
-        for ix, col in self.columns.items() if col
+    Return a dictionary mapping index keys to their names on the database.
+
+    Returns
+    -------
+    A dictionary of index keys to column names.
+    """
+    _parameters = self.parameters
+    _index_template = _parameters.get('index_template', "IX_{target}_{column_names}")
+    _indices = self.indices
+    _target = self.target
+    _column_names = {
+        ix: (
+            '_'.join(cols)
+            if isinstance(cols, (list, tuple))
+            else str(cols)
+        )
+        for ix, cols in _indices.items()
+        if cols
+    }
+    _index_names = {
+        ix: (
+            _index_template.format(
+                target=_target,
+                column_names=column_names,
+                connector_keys=self.connector_keys,
+                metric_key=self.connector_key,
+                location_key=self.location_key,
+            )
+        )
+        for ix, column_names in _column_names.items()
     }
+    return _index_names

meerschaum/plugins/__init__.py

@@ -316,7 +316,11 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
 
         ### NOTE: Allow plugins to be installed via `pip`.
         packaged_plugin_paths = []
-        discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        try:
+            discovered_packaged_plugins_eps = entry_points(group='meerschaum.plugins')
+        except TypeError:
+            discovered_packaged_plugins_eps = []
+
         for ep in discovered_packaged_plugins_eps:
             module_name = ep.name
             for package_file_path in ep.dist.files:
@@ -330,7 +334,7 @@ def sync_plugins_symlinks(debug: bool = False, warn: bool = True) -> None:
         if is_symlink(PLUGINS_RESOURCES_PATH) or not PLUGINS_RESOURCES_PATH.exists():
             try:
                 PLUGINS_RESOURCES_PATH.unlink()
-            except Exception as e:
+            except Exception:
                 pass
 
         PLUGINS_RESOURCES_PATH.mkdir(exist_ok=True)

meerschaum/plugins/bootstrap.py

@@ -47,7 +47,6 @@ IMPORTS_LINES: Dict[str, str] = {
     ),
 }
 
-### TODO: Add feature for custom connectors.
 FEATURE_LINES: Dict[str, str] = {
     'header': (
         "#! /usr/bin/env python3\n"
@@ -94,7 +93,7 @@ FEATURE_LINES: Dict[str, str] = {
         "class {plugin_name_capitalized}Connector(Connector):\n"
         "    \"\"\"Implement '{plugin_name_lower}' connectors.\"\"\"\n\n"
         "    REQUIRED_ATTRIBUTES: list[str] = []\n"
-        "    \n"
+        "\n"
         "    def fetch(\n"
         "        self,\n"
         "        pipe: mrsm.Pipe,\n"
@@ -149,11 +148,12 @@ FEATURE_LINES: Dict[str, str] = {
     ),
 }
 
+
 def bootstrap_plugin(
-        plugin_name: str,
-        debug: bool = False,
-        **kwargs: Any
-    ) -> SuccessTuple:
+    plugin_name: str,
+    debug: bool = False,
+    **kwargs: Any
+) -> SuccessTuple:
     """
     Prompt the user for features and create a plugin file.
     """
@@ -177,9 +177,9 @@ def bootstrap_plugin(
     features: List[str] = choose(
         "Which of the following features would you like to add to your plugin?",
         list(FEATURE_CHOICES.items()),
-        default = 'fetch',
-        multiple = True,
-        as_indices = True,
+        default='fetch',
+        multiple=True,
+        as_indices=True,
         **kwargs
     )
 
@@ -256,7 +256,7 @@ def bootstrap_plugin(
         _ = prompt(
             f"Press [Enter] to edit plugin '{plugin_name}',"
             + " [CTRL+C] to skip.",
-            icon = False,
+            icon=False,
         )
     except (KeyboardInterrupt, Exception):
         return True, "Success"
@@ -267,7 +267,7 @@ def bootstrap_plugin(
 
 def _get_plugins_dir_path() -> pathlib.Path:
     from meerschaum.config.paths import PLUGINS_DIR_PATHS
-    
+
     if not PLUGINS_DIR_PATHS:
         raise EnvironmentError("No plugin dir path could be found.")
 
@@ -278,9 +278,9 @@ def _get_plugins_dir_path() -> pathlib.Path:
         choose(
             "In which directory do you want to write your plugin?",
             [path.as_posix() for path in PLUGINS_DIR_PATHS],
-            numeric = True,
-            multiple = False,
-            default = PLUGINS_DIR_PATHS[0].as_posix(),
+            numeric=True,
+            multiple=False,
+            default=PLUGINS_DIR_PATHS[0].as_posix(),
         )
     )
  
@@ -290,7 +290,7 @@ def _ask_to_uninstall(plugin: mrsm.Plugin, **kwargs: Any) -> SuccessTuple:
     warn(f"Plugin '{plugin}' is already installed!", stack=False)
     uninstall_plugin = yes_no(
         f"Do you want to first uninstall '{plugin}'?",
-        default = 'n',
+        default='n',
         **kwargs
     )
     if not uninstall_plugin:

meerschaum/utils/dataframe.py

@@ -413,6 +413,7 @@ def parse_df_datetimes(
     from meerschaum.utils.packages import import_pandas, attempt_import
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.warnings import warn
+    from meerschaum.utils.misc import items_str
     import traceback
     pd = import_pandas()
     pandas = attempt_import('pandas')
@@ -503,14 +504,14 @@ def parse_df_datetimes(
         else:
             df[datetime_cols] = df[datetime_cols].apply(
                 pd.to_datetime,
-                utc = True,
-                axis = 1,
-                meta = {
+                utc=True,
+                axis=1,
+                meta={
                     col: 'datetime64[ns]'
                     for col in datetime_cols
                 }
             )
-    except Exception as e:
+    except Exception:
         warn(
             f"Unable to apply `pd.to_datetime` to {items_str(datetime_cols)}:\n"
             + f"{traceback.format_exc()}"
@@ -519,7 +520,7 @@ def parse_df_datetimes(
     for dt in datetime_cols:
         try:
             df[dt] = df[dt].dt.tz_localize(None)
-        except Exception as e:
+        except Exception:
             warn(f"Unable to convert column '{dt}' to naive datetime:\n{traceback.format_exc()}")
 
     return df
@@ -567,6 +568,9 @@ def get_json_cols(df: 'pd.DataFrame') -> List[str]:
     -------
     A list of columns to be encoded as JSON.
     """
+    if df is None:
+        return []
+
     is_dask = 'dask' in df.__module__ if hasattr(df, '__module__') else False
     if is_dask:
         df = get_first_valid_dask_partition(df)
@@ -602,6 +606,8 @@ def get_numeric_cols(df: 'pd.DataFrame') -> List[str]:
     -------
     A list of columns to treat as numerics.
     """
+    if df is None:
+        return []
     from decimal import Decimal
     is_dask = 'dask' in df.__module__
     if is_dask:
@@ -638,6 +644,8 @@ def get_uuid_cols(df: 'pd.DataFrame') -> List[str]:
     -------
     A list of columns to treat as numerics.
     """
+    if df is None:
+        return []
     from uuid import UUID
     is_dask = 'dask' in df.__module__
     if is_dask:
@@ -883,6 +891,8 @@ def get_datetime_bound_from_df(
     -------
     The minimum or maximum datetime value in the dataframe, or `None`.
     """
+    if df is None:
+        return None
     if not datetime_column:
         return None
 
@@ -955,6 +965,8 @@ def get_unique_index_values(
     -------
     A dictionary mapping indices to unique values.
     """
+    if df is None:
+        return {}
     if 'dataframe' in str(type(df)).lower():
         pandas = mrsm.attempt_import('pandas')
         return {

meerschaum/utils/packages/__init__.py

@@ -11,7 +11,12 @@ from __future__ import annotations
 import importlib.util, os, pathlib, re
 from meerschaum.utils.typing import Any, List, SuccessTuple, Optional, Union, Tuple, Dict, Iterable
 from meerschaum.utils.threading import Lock, RLock
-from meerschaum.utils.packages._packages import packages, all_packages, get_install_names
+from meerschaum.utils.packages._packages import (
+    packages,
+    all_packages,
+    get_install_names,
+    _MRSM_PACKAGE_ARCHIVES_PREFIX,
+)
 from meerschaum.utils.venv import (
     activate_venv,
     deactivate_venv,
@@ -35,14 +40,14 @@ _locks = {
 }
 _checked_for_updates = set()
 _is_installed_first_check: Dict[str, bool] = {}
-_MRSM_PACKAGE_ARCHIVES_PREFIX: str = "https://meerschaum.io/files/archives/"
+
 
 def get_module_path(
-        import_name: str,
-        venv: Optional[str] = 'mrsm',
-        debug: bool = False,
-        _try_install_name_on_fail: bool = True,
-    ) -> Union[pathlib.Path, None]:
+    import_name: str,
+    venv: Optional[str] = 'mrsm',
+    debug: bool = False,
+    _try_install_name_on_fail: bool = True,
+) -> Union[pathlib.Path, None]:
     """
     Get a module's path without importing.
     """
@@ -232,10 +237,10 @@ def manually_import_module(
         if check_update:
             if need_update(
                 None,
-                import_name = root_name,
-                version = _version,
-                check_pypi = check_pypi,
-                debug = debug,
+                import_name=root_name,
+                version=_version,
+                check_pypi=check_pypi,
+                debug=debug,
             ):
                 if install:
                     if not pip_install(
@@ -491,6 +496,8 @@ def _get_package_metadata(import_name: str, venv: Optional[str]) -> Dict[str, st
     import re
     from meerschaum.config._paths import VIRTENV_RESOURCES_PATH
     install_name = _import_to_install_name(import_name)
+    if install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX):
+        return {}
     _args = ['pip', 'show', install_name]
     if venv is not None:
         cache_dir_path = VIRTENV_RESOURCES_PATH / venv / 'cache'
@@ -586,7 +593,10 @@ def need_update(
         _checked_for_updates.add(install_name)
 
     _install_no_version = get_install_no_version(install_name)
-    required_version = install_name.replace(_install_no_version, '')
+    required_version = (
+        install_name
+        .replace(_install_no_version, '')
+    )
     if ']' in required_version:
         required_version = required_version.split(']')[1]
 
@@ -681,8 +691,8 @@ def need_update(
     )
 
     if 'a' in required_version:
-        required_version = required_version.replace('a', '-dev').replace('+mrsm', '')
-        version = version.replace('a', '-dev').replace('+mrsm', '')
+        required_version = required_version.replace('a', '-pre.').replace('+mrsm', '')
+        version = version.replace('a', '-pre.').replace('+mrsm', '')
     try:
         return (
             (not semver.Version.parse(version).match(required_version))
@@ -780,7 +790,7 @@ def pip_install(
         This includes version restrictions.
         Use `_import_to_install_name()` to get the predefined `install_name` for a package
         from its import name.
-        
+
     args: Optional[List[str]], default None
         A list of command line arguments to pass to `pip`.
         If not provided, default to `['--upgrade']` if `_uninstall` is `False`, else `[]`.
@@ -975,7 +985,11 @@ def pip_install(
                 pass
 
         _packages = [
-            (install_name if not _uninstall else get_install_no_version(install_name))
+            (
+                get_install_no_version(install_name)
+                if _uninstall or install_name.startswith(_MRSM_PACKAGE_ARCHIVES_PREFIX)
+                else install_name
+            )
             for install_name in install_names
         ]
         msg = "Installing packages:" if not _uninstall else "Uninstalling packages:"
@@ -1774,13 +1788,17 @@ def is_installed(
 
         found = (
             not need_update(
-                None, import_name = root_name,
-                _run_determine_version = False,
-                check_pypi = False,
-                version = determine_version(
-                    spec_path, venv=venv, debug=debug, import_name=root_name
+                None,
+                import_name=root_name,
+                _run_determine_version=False,
+                check_pypi=False,
+                version=determine_version(
+                    spec_path,
+                    venv=venv,
+                    debug=debug,
+                    import_name=root_name,
                 ),
-                debug = debug,
+                debug=debug,
             )
         ) if spec_path is not None else False