meerschaum 2.2.3__py3-none-any.whl → 2.2.5__py3-none-any.whl

meerschaum/api/_oauth2.py

@@ -13,6 +13,23 @@ fastapi = attempt_import('fastapi', lazy=False, check_update=CHECK_UPDATE)
 fastapi_responses = attempt_import('fastapi.responses', lazy=False, check_update=CHECK_UPDATE)
 fastapi_login = attempt_import('fastapi_login', check_update=CHECK_UPDATE)
 
+class CustomOAuth2PasswordRequestForm:
+    def __init__(
+        self,
+        grant_type: str = fastapi.Form(None, regex="password|client_credentials"),
+        username: str = fastapi.Form(...),
+        password: str = fastapi.Form(...),
+        scope: str = fastapi.Form(""),
+        client_id: str = fastapi.Form(None),
+        client_secret: str = fastapi.Form(None),
+    ):
+        self.grant_type = grant_type
+        self.username = username
+        self.password = password
+        self.scope = scope
+        self.client_id = client_id
+        self.client_secret = client_secret
+
 LoginManager = fastapi_login.LoginManager
 def generate_secret_key() -> str:
     """

meerschaum/api/dash/__init__.py

@@ -50,7 +50,6 @@ stylesheets = [
     '/static/css/bootstrap.min.css',
     '/static/css/dbc_dark.css',
     '/static/css/dash.css',
-    #  '/static/js/node_modules/xterm/css/xterm.css',
 ]
 scripts = ['/static/js/node_modules/xterm/lib/xterm.js']
 dash_app = enrich.DashProxy(

meerschaum/api/dash/callbacks/custom.py

@@ -21,7 +21,7 @@ def init_dash_plugins():
         for _function in _functions:
             try:
                 _function(dash_app)
-            except Exception as e:
+            except Exception:
                 warn(
                     f"Failed to load function '{_function.__name__}' "
                     + f"from plugin '{_module_name}':\n"

meerschaum/api/dash/plugins.py

@@ -7,7 +7,7 @@ Functions for interacting with plugins via the web interface.
 """
 
 from __future__ import annotations
-from meerschaum.utils.typing import List, Tuple, SuccessTuple
+from meerschaum.utils.typing import List, Tuple, SuccessTuple, Optional, WebState, Dict, Any
 from meerschaum.utils.packages import import_dcc, import_html
 from meerschaum.api import get_api_connector, endpoints, CHECK_UPDATE
 html, dcc = import_html(check_update=CHECK_UPDATE), import_dcc(check_update=CHECK_UPDATE)
@@ -17,10 +17,10 @@ from meerschaum.api.dash import dash_app, debug, active_sessions
 
 
 def get_plugins_cards(
-        state: Optional[WebState] = None,
-        search_term: Optional[str] = None,
-        session_data: Optional[Dict[str, Any]] = None,
-    ) -> Tuple[List[dbc.Card], List[SuccessTuple]]:
+    state: Optional[WebState] = None,
+    search_term: Optional[str] = None,
+    session_data: Optional[Dict[str, Any]] = None,
+) -> Tuple[List[dbc.Card], List[SuccessTuple]]:
     """
     Return the cards and alerts for plugins.
     """
@@ -96,4 +96,3 @@ def is_plugin_owner(plugin_name: str, session_data: Dict['str', Any]) -> bool:
         _username is not None
         and _username == _plugin_username
     )
-

meerschaum/api/routes/_login.py

@@ -8,13 +8,17 @@ Manage access and refresh tokens.
 
 from datetime import datetime, timedelta, timezone
 import fastapi
+from fastapi import Request, status
 from fastapi_login.exceptions import InvalidCredentialsException
-from fastapi.security import OAuth2PasswordRequestForm
+from fastapi.exceptions import RequestValidationError
 from starlette.responses import Response, JSONResponse
 from meerschaum.api import endpoints, get_api_connector, app, debug, manager, no_auth
 from meerschaum.core import User
 from meerschaum.config.static import STATIC_CONFIG
-from meerschaum.utils.typing import Dict, Any
+from meerschaum.utils.typing import Dict, Any, Optional
+from meerschaum.core.User._User import verify_password
+from meerschaum.utils.warnings import warn
+from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
 
 
 @manager.user_loader()
@@ -28,18 +32,18 @@ def load_user(
 
 
 @app.post(endpoints['login'], tags=['Users'])
-def login(
-        data: OAuth2PasswordRequestForm = fastapi.Depends()
+async def login(
+        data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
+        #  data: dict[str, str],
+        #  request: Request
     ) -> Dict[str, Any]:
     """
     Login and set the session token.
     """
     username, password = (
-        (data['username'], data['password']) if isinstance(data, dict)
-        else (data.username, data.password)
+        (data.username, data.password)
     ) if not no_auth else ('no-auth', 'no-auth')
 
-    from meerschaum.core.User._User import verify_password
     user = User(username, password)
     correct_password = no_auth or verify_password(
         password,
@@ -60,3 +64,15 @@ def login(
         'token_type': 'bearer',
         'expires' : expires_dt,
     }
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    """
+    Log validation errors as warnings.
+    """
+    warn(f"Validation error: {exc.errors()}", stack=False)
+    return JSONResponse(
+        status_code = status.HTTP_422_UNPROCESSABLE_ENTITY,
+        content = {"detail": exc.errors()},
+    )

meerschaum/config/__init__.py

@@ -12,7 +12,7 @@ from __future__ import annotations
 import os, shutil, sys, pathlib, copy
 from meerschaum.utils.typing import Any, Dict, Optional, Union
 from meerschaum.utils.threading import RLock
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.warnings import warn, error
 
 from meerschaum.config._version import __version__
 from meerschaum.config._edit import edit_config, write_config
@@ -239,17 +239,28 @@ def get_config(
     return c
 
 
-def get_plugin_config(*keys : str, **kw : Any) -> Optional[Any]:
+def get_plugin_config(
+        *keys: str,
+        warn: bool = False,
+        **kw: Any
+    ) -> Optional[Any]:
     """
     This may only be called from within a Meerschaum plugin.
     See `meerschaum.config.get_config` for arguments.
     """
-    from meerschaum.utils.warnings import warn, error
     from meerschaum.plugins import _get_parent_plugin
     parent_plugin_name = _get_parent_plugin(2)
     if parent_plugin_name is None:
-        error(f"You may only call `get_plugin_config()` from within a Meerschaum plugin.")
-    return get_config(*(['plugins', parent_plugin_name] + list(keys)), **kw)
+        error(
+            "You may only call `get_plugin_config()` "
+            "from within a Meerschaum plugin."
+        )
+
+    return get_config(
+        *(['plugins', parent_plugin_name] + list(keys)),
+        warn=warn,
+        **kw
+    )
 
 
 def write_plugin_config(
@@ -259,7 +270,6 @@ def write_plugin_config(
     """
     Write a plugin's configuration dictionary.
     """
-    from meerschaum.utils.warnings import warn, error
     from meerschaum.plugins import _get_parent_plugin
     parent_plugin_name = _get_parent_plugin(2)
     if parent_plugin_name is None:

meerschaum/config/_edit.py

@@ -171,7 +171,7 @@ def general_write_yaml_config(
         path = pathlib.Path(fp)
         path.parent.mkdir(parents=True, exist_ok=True)
         path.touch(exist_ok=True)
-        with open(path, 'w+') as f:
+        with open(path, 'w+', encoding='utf-8') as f:
             if header is not None:
                 if debug:
                     dprint(f"Header detected, writing to {path}...")

meerschaum/config/_paths.py

@@ -138,6 +138,9 @@ paths = {
     'SQLITE_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'sqlite'),
     'SQLITE_DB_PATH'                 : ('{SQLITE_RESOURCES_PATH}', 'mrsm_local.db'),
 
+    'BACKUP_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'backup'),
+    'BACKUP_DATABASE_PATH'           : ('{BACKUP_RESOURCES_PATH}', 'backup.sql'),
+
     'DUCKDB_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'duckdb'),
     'DUCKDB_PATH'                    : ('{DUCKDB_RESOURCES_PATH}', 'duck.db'),
 

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.2.3"
+__version__ = "2.2.5"

meerschaum/config/stack/__init__.py

@@ -56,6 +56,7 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
 
 volumes = {
     'api_root': '/meerschaum',
+    'api_user_local': '/home/meerschaum/.local',
     'meerschaum_db_data': '/var/lib/postgresql/data',
     'grafana_storage': '/var/lib/grafana',
 }
@@ -122,7 +123,7 @@ default_docker_compose_config = {
             ],
             'hostname' : f'{db_hostname}',
             'volumes' : [
-                'meerschaum_db_data' + ':' + volumes['meerschaum_db_data'],
+                'meerschaum_db_data:' + volumes['meerschaum_db_data'],
             ],
             'shm_size': '1024m',
             'networks' : [
@@ -159,6 +160,7 @@ default_docker_compose_config = {
             },
             'volumes' : [
                 'api_root:' + volumes['api_root'],
+                'api_user_local:' + volumes['api_user_local'],
             ],
         },
         'grafana': {

meerschaum/connectors/Connector.py

@@ -18,7 +18,7 @@ class InvalidAttributesError(Exception):
 
 class Connector(metaclass=abc.ABCMeta):
     """
-    The base connector class to hold connection attributes,
+    The base connector class to hold connection attributes.
     """
     def __init__(
             self,
@@ -27,6 +27,8 @@ class Connector(metaclass=abc.ABCMeta):
             **kw: Any
         ):
         """
+        Set the given keyword arguments as attributes.
+
         Parameters
         ----------
         type: str
@@ -35,9 +37,12 @@ class Connector(metaclass=abc.ABCMeta):
         label: str
             The `label` for the connector.
 
+
+        Examples
+        --------
         Run `mrsm edit config` and to edit connectors in the YAML file:
 
-        ```
+        ```yaml
         meerschaum:
             connections:
                 {type}:

meerschaum/connectors/__init__.py

@@ -24,6 +24,7 @@ from meerschaum.connectors.api.APIConnector import APIConnector
 from meerschaum.connectors.sql._create_engine import flavor_configs as sql_flavor_configs
 
 __all__ = (
+    "make_connector",
     "Connector",
     "SQLConnector",
     "APIConnector",
@@ -290,13 +291,14 @@ def make_connector(
     --------
     >>> import meerschaum as mrsm
     >>> from meerschaum.connectors import make_connector, Connector
+    >>> 
+    >>> @make_connector
     >>> class FooConnector(Connector):
-    ...     def __init__(self, label: str, **kw):
-    ...         super().__init__('foo', label, **kw)
+    ...     REQUIRED_ATTRIBUTES: list[str] = ['username', 'password']
     ... 
-    >>> make_connector(FooConnector)
-    >>> mrsm.get_connector('foo', 'bar')
-    foo:bar
+    >>> conn = mrsm.get_connector('foo:bar', username='dog', password='cat')
+    >>> print(conn.username, conn.password)
+    dog cat
     >>> 
     """
     import re

meerschaum/core/Pipe/_data.py

@@ -8,24 +8,32 @@ Retrieve Pipes' data from instances.
 
 from __future__ import annotations
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import Optional, Dict, Any, Union, Generator, List, Tuple, Iterator
+
+import meerschaum as mrsm
+from meerschaum.utils.typing import (
+    Optional, Dict, Any, Union, List, Tuple, Iterator, TYPE_CHECKING,
+)
 from meerschaum.config import get_config
 
+if TYPE_CHECKING:
+    pd = mrsm.attempt_import('pandas')
+
+
 def get_data(
-        self,
-        select_columns: Optional[List[str]] = None,
-        omit_columns: Optional[List[str]] = None,
-        begin: Union[datetime, int, None] = None,
-        end: Union[datetime, int, None] = None,
-        params: Optional[Dict[str, Any]] = None,
-        as_iterator: bool = False,
-        as_chunks: bool = False,
-        as_dask: bool = False,
-        chunk_interval: Union[timedelta, int, None] = None,
-        fresh: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> Union['pd.DataFrame', Generator['pd.DataFrame'], None]:
+    self,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    as_iterator: bool = False,
+    as_chunks: bool = False,
+    as_dask: bool = False,
+    chunk_interval: Union[timedelta, int, None] = None,
+    fresh: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> Union['pd.DataFrame', Iterator['pd.DataFrame'], None]:
     """
     Get a pipe's data from the instance connector.
 

meerschaum/core/Pipe/_deduplicate.py

@@ -8,7 +8,7 @@ Delete duplicate rows within a pipe's table.
 
 from __future__ import annotations
 from datetime import datetime, timedelta
-from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple
+from meerschaum.utils.typing import SuccessTuple, Any, Optional, Dict, Tuple, Union
 
 
 def deduplicate(

meerschaum/core/Pipe/_dtypes.py

@@ -8,7 +8,12 @@ Enforce data types for a pipe's underlying table.
 
 from __future__ import annotations
 from io import StringIO
+import meerschaum as mrsm
 from meerschaum.utils.typing import Dict, Any, Optional
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    pd = mrsm.attempt_import('pandas')
 
 def enforce_dtypes(
         self,

meerschaum/core/Pipe/_fetch.py

@@ -8,11 +8,15 @@ Functions for fetching new data into the Pipe
 
 from __future__ import annotations
 from datetime import timedelta, datetime
+
 import meerschaum as mrsm
-from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
+from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator, TYPE_CHECKING
 from meerschaum.config import get_config
 from meerschaum.utils.warnings import warn
 
+if TYPE_CHECKING:
+    pd = mrsm.attempt_import('pandas')
+
 def fetch(
         self,
         begin: Union[datetime, str, None] = '',
@@ -54,6 +58,7 @@ def fetch(
 
     from meerschaum.connectors import custom_types, get_connector_plugin
     from meerschaum.utils.debug import dprint, _checkpoint
+    from meerschaum.utils.misc import filter_arguments
 
     _chunk_hook = kw.pop('chunk_hook', None)
     kw['workers'] = self.get_num_workers(kw.get('workers', None))
@@ -71,29 +76,30 @@ def fetch(
                 chunk_message = '\n' + chunk_label + '\n' + chunk_message
             return chunk_success, chunk_message
 
-
     with mrsm.Venv(get_connector_plugin(self.connector)):
-        df = self.connector.fetch(
+        _args, _kwargs = filter_arguments(
+            self.connector.fetch,
             self,
-            begin = _determine_begin(
+            begin=_determine_begin(
                 self,
                 begin,
-                check_existing = check_existing,
-                debug = debug,
+                check_existing=check_existing,
+                debug=debug,
             ),
-            end = end,
-            chunk_hook = _chunk_hook,
-            debug = debug,
+            end=end,
+            chunk_hook=_chunk_hook,
+            debug=debug,
             **kw
         )
+        df = self.connector.fetch(*_args, **_kwargs)
     return df
 
 
 def get_backtrack_interval(
-        self,
-        check_existing: bool = True,
-        debug: bool = False,
-    ) -> Union[timedelta, int]:
+    self,
+    check_existing: bool = True,
+    debug: bool = False,
+) -> Union[timedelta, int]:
     """
     Get the chunk interval to use for this pipe.
 
@@ -127,17 +133,17 @@ def get_backtrack_interval(
 
 
 def _determine_begin(
-        pipe: mrsm.Pipe,
-        begin: Union[datetime, int, str] = '',
-        check_existing: bool = True,
-        debug: bool = False,
-    ) -> Union[datetime, int, None]:
+    pipe: mrsm.Pipe,
+    begin: Union[datetime, int, str, None] = '',
+    check_existing: bool = True,
+    debug: bool = False,
+) -> Union[datetime, int, None]:
     """
     Apply the backtrack interval if `--begin` is not provided.
 
     Parameters
     ----------
-    begin: Union[datetime, int, str], default ''
+    begin: Union[datetime, int, str, None], default ''
         The provided begin timestamp.
 
     check_existing: bool, default True
@@ -160,4 +166,4 @@ def _determine_begin(
         return sync_time - backtrack_interval
     except Exception as e:
         warn(f"Unable to substract backtrack interval {backtrack_interval} from {sync_time}.")
-        return sync_time
+    return sync_time