meerschaum 2.2.4__py3-none-any.whl → 2.2.5.dev2__py3-none-any.whl

meerschaum/actions/uninstall.py

@@ -25,9 +25,9 @@ def uninstall(
 
 
 def _complete_uninstall(
-        action: Optional[List[str]] = None,
-        **kw: Any
-    ) -> List[str]:
+    action: Optional[List[str]] = None,
+    **kw: Any
+) -> List[str]:
     """
     Override the default Meerschaum `complete_` function.
     """
@@ -49,15 +49,16 @@ def _complete_uninstall(
     from meerschaum._internal.shell import default_action_completer
     return default_action_completer(action=(['uninstall'] + action), **kw)
 
+
 def _uninstall_plugins(
-        action: Optional[List[str]] = None,
-        repository: Optional[str] = None,
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    repository: Optional[str] = None,
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Remove installed plugins. Does not affect repository registrations.
     """
@@ -135,26 +136,20 @@ def _uninstall_plugins(
 
 
 def _complete_uninstall_plugins(action: Optional[List[str]] = None, **kw) -> List[str]:
-    from meerschaum.plugins import get_plugins_names
-    _plugin_names = get_plugins_names()
-    if not action:
-        return _plugin_names
-    possibilities = []
-    for name in _plugin_names:
-        if name.startswith(action[0]) and action[0] != name:
-            possibilities.append(name)
-    return possibilities
+    from meerschaum.actions.edit import _complete_edit_plugins
+    return _complete_edit_plugins(action=action, **kw)
+
 
 def _uninstall_packages(
-        action: Optional[List[str]] = None,
-        sub_args: Optional[List[str]] = None,
-        venv: Optional[str] = 'mrsm',
-        yes: bool = False,
-        force: bool = False,
-        noask: bool = False,
-        debug: bool = False,
-        **kw: Any
-    ) -> SuccessTuple:
+    action: Optional[List[str]] = None,
+    sub_args: Optional[List[str]] = None,
+    venv: Optional[str] = 'mrsm',
+    yes: bool = False,
+    force: bool = False,
+    noask: bool = False,
+    debug: bool = False,
+    **kw: Any
+) -> SuccessTuple:
     """
     Uninstall PyPI packages from the Meerschaum virtual environment.
     

meerschaum/api/_oauth2.py

@@ -13,6 +13,23 @@ fastapi = attempt_import('fastapi', lazy=False, check_update=CHECK_UPDATE)
 fastapi_responses = attempt_import('fastapi.responses', lazy=False, check_update=CHECK_UPDATE)
 fastapi_login = attempt_import('fastapi_login', check_update=CHECK_UPDATE)
 
+class CustomOAuth2PasswordRequestForm:
+    def __init__(
+        self,
+        grant_type: str = fastapi.Form(None, regex="password|client_credentials"),
+        username: str = fastapi.Form(...),
+        password: str = fastapi.Form(...),
+        scope: str = fastapi.Form(""),
+        client_id: str = fastapi.Form(None),
+        client_secret: str = fastapi.Form(None),
+    ):
+        self.grant_type = grant_type
+        self.username = username
+        self.password = password
+        self.scope = scope
+        self.client_id = client_id
+        self.client_secret = client_secret
+
 LoginManager = fastapi_login.LoginManager
 def generate_secret_key() -> str:
     """

meerschaum/api/routes/_login.py

@@ -8,13 +8,17 @@ Manage access and refresh tokens.
 
 from datetime import datetime, timedelta, timezone
 import fastapi
+from fastapi import Request, status
 from fastapi_login.exceptions import InvalidCredentialsException
-from fastapi.security import OAuth2PasswordRequestForm
+from fastapi.exceptions import RequestValidationError
 from starlette.responses import Response, JSONResponse
 from meerschaum.api import endpoints, get_api_connector, app, debug, manager, no_auth
 from meerschaum.core import User
 from meerschaum.config.static import STATIC_CONFIG
-from meerschaum.utils.typing import Dict, Any
+from meerschaum.utils.typing import Dict, Any, Optional
+from meerschaum.core.User._User import verify_password
+from meerschaum.utils.warnings import warn
+from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
 
 
 @manager.user_loader()
@@ -28,18 +32,18 @@ def load_user(
 
 
 @app.post(endpoints['login'], tags=['Users'])
-def login(
-        data: OAuth2PasswordRequestForm = fastapi.Depends()
+async def login(
+        data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
+        #  data: dict[str, str],
+        #  request: Request
     ) -> Dict[str, Any]:
     """
     Login and set the session token.
     """
     username, password = (
-        (data['username'], data['password']) if isinstance(data, dict)
-        else (data.username, data.password)
+        (data.username, data.password)
     ) if not no_auth else ('no-auth', 'no-auth')
 
-    from meerschaum.core.User._User import verify_password
     user = User(username, password)
     correct_password = no_auth or verify_password(
         password,
@@ -60,3 +64,15 @@ def login(
         'token_type': 'bearer',
         'expires' : expires_dt,
     }
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    """
+    Log validation errors as warnings.
+    """
+    warn(f"Validation error: {exc.errors()}", stack=False)
+    return JSONResponse(
+        status_code = status.HTTP_422_UNPROCESSABLE_ENTITY,
+        content = {"detail": exc.errors()},
+    )

meerschaum/config/__init__.py

@@ -12,7 +12,7 @@ from __future__ import annotations
 import os, shutil, sys, pathlib, copy
 from meerschaum.utils.typing import Any, Dict, Optional, Union
 from meerschaum.utils.threading import RLock
-from meerschaum.utils.warnings import warn
+from meerschaum.utils.warnings import warn, error
 
 from meerschaum.config._version import __version__
 from meerschaum.config._edit import edit_config, write_config
@@ -239,17 +239,28 @@ def get_config(
     return c
 
 
-def get_plugin_config(*keys : str, **kw : Any) -> Optional[Any]:
+def get_plugin_config(
+        *keys: str,
+        warn: bool = False,
+        **kw: Any
+    ) -> Optional[Any]:
     """
     This may only be called from within a Meerschaum plugin.
     See `meerschaum.config.get_config` for arguments.
     """
-    from meerschaum.utils.warnings import warn, error
     from meerschaum.plugins import _get_parent_plugin
     parent_plugin_name = _get_parent_plugin(2)
     if parent_plugin_name is None:
-        error(f"You may only call `get_plugin_config()` from within a Meerschaum plugin.")
-    return get_config(*(['plugins', parent_plugin_name] + list(keys)), **kw)
+        error(
+            "You may only call `get_plugin_config()` "
+            "from within a Meerschaum plugin."
+        )
+
+    return get_config(
+        *(['plugins', parent_plugin_name] + list(keys)),
+        warn=warn,
+        **kw
+    )
 
 
 def write_plugin_config(
@@ -259,7 +270,6 @@ def write_plugin_config(
     """
     Write a plugin's configuration dictionary.
     """
-    from meerschaum.utils.warnings import warn, error
     from meerschaum.plugins import _get_parent_plugin
     parent_plugin_name = _get_parent_plugin(2)
     if parent_plugin_name is None:

meerschaum/config/_edit.py

@@ -171,7 +171,7 @@ def general_write_yaml_config(
         path = pathlib.Path(fp)
         path.parent.mkdir(parents=True, exist_ok=True)
         path.touch(exist_ok=True)
-        with open(path, 'w+') as f:
+        with open(path, 'w+', encoding='utf-8') as f:
             if header is not None:
                 if debug:
                     dprint(f"Header detected, writing to {path}...")

meerschaum/config/_paths.py

@@ -138,6 +138,9 @@ paths = {
     'SQLITE_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'sqlite'),
     'SQLITE_DB_PATH'                 : ('{SQLITE_RESOURCES_PATH}', 'mrsm_local.db'),
 
+    'BACKUP_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'backup'),
+    'BACKUP_DATABASE_PATH'           : ('{BACKUP_RESOURCES_PATH}', 'backup.sql'),
+
     'DUCKDB_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'duckdb'),
     'DUCKDB_PATH'                    : ('{DUCKDB_RESOURCES_PATH}', 'duck.db'),
 

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.2.4"
+__version__ = "2.2.5.dev2"

meerschaum/config/stack/__init__.py

@@ -56,6 +56,7 @@ env_dict['MEERSCHAUM_API_CONFIG'] = json.dumps(
 
 volumes = {
     'api_root': '/meerschaum',
+    'api_user_local': '/home/meerschaum/.local',
     'meerschaum_db_data': '/var/lib/postgresql/data',
     'grafana_storage': '/var/lib/grafana',
 }
@@ -122,7 +123,7 @@ default_docker_compose_config = {
             ],
             'hostname' : f'{db_hostname}',
             'volumes' : [
-                'meerschaum_db_data' + ':' + volumes['meerschaum_db_data'],
+                'meerschaum_db_data:' + volumes['meerschaum_db_data'],
             ],
             'shm_size': '1024m',
             'networks' : [
@@ -159,6 +160,7 @@ default_docker_compose_config = {
             },
             'volumes' : [
                 'api_root:' + volumes['api_root'],
+                'api_user_local:' + volumes['api_user_local'],
             ],
         },
         'grafana': {

meerschaum/core/Pipe/_fetch.py

@@ -12,6 +12,7 @@ import meerschaum as mrsm
 from meerschaum.utils.typing import Optional, Any, Union, SuccessTuple, Iterator
 from meerschaum.config import get_config
 from meerschaum.utils.warnings import warn
+from meerschaum.utils.misc import filter_keywords
 
 def fetch(
         self,
@@ -75,25 +76,28 @@ def fetch(
     with mrsm.Venv(get_connector_plugin(self.connector)):
         df = self.connector.fetch(
             self,
-            begin = _determine_begin(
-                self,
-                begin,
-                check_existing = check_existing,
-                debug = debug,
-            ),
-            end = end,
-            chunk_hook = _chunk_hook,
-            debug = debug,
-            **kw
+            **filter_keywords(
+                self.connector.fetch,
+                begin=_determine_begin(
+                    self,
+                    begin,
+                    check_existing=check_existing,
+                    debug=debug,
+                ),
+                end=end,
+                chunk_hook=_chunk_hook,
+                debug=debug,
+                **kw
+            )
         )
     return df
 
 
 def get_backtrack_interval(
-        self,
-        check_existing: bool = True,
-        debug: bool = False,
-    ) -> Union[timedelta, int]:
+    self,
+    check_existing: bool = True,
+    debug: bool = False,
+) -> Union[timedelta, int]:
     """
     Get the chunk interval to use for this pipe.
 
@@ -127,17 +131,17 @@ def get_backtrack_interval(
 
 
 def _determine_begin(
-        pipe: mrsm.Pipe,
-        begin: Union[datetime, int, str] = '',
-        check_existing: bool = True,
-        debug: bool = False,
-    ) -> Union[datetime, int, None]:
+    pipe: mrsm.Pipe,
+    begin: Union[datetime, int, str, None] = '',
+    check_existing: bool = True,
+    debug: bool = False,
+) -> Union[datetime, int, None]:
     """
     Apply the backtrack interval if `--begin` is not provided.
 
     Parameters
     ----------
-    begin: Union[datetime, int, str], default ''
+    begin: Union[datetime, int, str, None], default ''
         The provided begin timestamp.
 
     check_existing: bool, default True
@@ -160,4 +164,4 @@ def _determine_begin(
         return sync_time - backtrack_interval
     except Exception as e:
         warn(f"Unable to substract backtrack interval {backtrack_interval} from {sync_time}.")
-        return sync_time
+    return sync_time

meerschaum/core/Pipe/_sync.py

@@ -34,29 +34,29 @@ class InferFetch:
     MRSM_INFER_FETCH: bool = True
 
 def sync(
-        self,
-        df: Union[
-            pd.DataFrame,
-            Dict[str, List[Any]],
-            List[Dict[str, Any]],
-            InferFetch
-        ] = InferFetch,
-        begin: Union[datetime, int, str, None] = '',
-        end: Union[datetime, int] = None,
-        force: bool = False,
-        retries: int = 10,
-        min_seconds: int = 1,
-        check_existing: bool = True,
-        blocking: bool = True,
-        workers: Optional[int] = None,
-        callback: Optional[Callable[[Tuple[bool, str]], Any]] = None,
-        error_callback: Optional[Callable[[Exception], Any]] = None,
-        chunksize: Optional[int] = -1,
-        sync_chunks: bool = True,
-        debug: bool = False,
-        _inplace: bool = True,
-        **kw: Any
-    ) -> SuccessTuple:
+    self,
+    df: Union[
+        pd.DataFrame,
+        Dict[str, List[Any]],
+        List[Dict[str, Any]],
+        InferFetch
+    ] = InferFetch,
+    begin: Union[datetime, int, str, None] = '',
+    end: Union[datetime, int, None] = None,
+    force: bool = False,
+    retries: int = 10,
+    min_seconds: int = 1,
+    check_existing: bool = True,
+    blocking: bool = True,
+    workers: Optional[int] = None,
+    callback: Optional[Callable[[Tuple[bool, str]], Any]] = None,
+    error_callback: Optional[Callable[[Exception], Any]] = None,
+    chunksize: Optional[int] = -1,
+    sync_chunks: bool = True,
+    debug: bool = False,
+    _inplace: bool = True,
+    **kw: Any
+) -> SuccessTuple:
     """
     Fetch new data from the source and update the pipe's table with new data.
     
@@ -125,7 +125,7 @@ def sync(
     from meerschaum.utils.formatting import get_console
     from meerschaum.utils.venv import Venv
     from meerschaum.connectors import get_connector_plugin
-    from meerschaum.utils.misc import df_is_chunk_generator
+    from meerschaum.utils.misc import df_is_chunk_generator, filter_keywords
     from meerschaum.utils.pool import get_pool
     from meerschaum.config import get_config
 
@@ -186,7 +186,7 @@ def sync(
         ### use that instead.
         ### NOTE: The DataFrame must be omitted for the plugin sync method to apply.
         ### If a DataFrame is provided, continue as expected.
-        if hasattr(df, 'MRSM_INFER_FETCH'):                   
+        if hasattr(df, 'MRSM_INFER_FETCH'):
             try:
                 if p.connector is None:
                     msg = f"{p} does not have a valid connector."
@@ -194,7 +194,7 @@ def sync(
                         msg += f"\n    Perhaps {p.connector_keys} has a syntax error?"
                     p._exists = None
                     return False, msg
-            except Exception as e:
+            except Exception:
                 p._exists = None
                 return False, f"Unable to create the connector for {p}."
 
@@ -210,14 +210,28 @@ def sync(
             ):
                 with Venv(get_connector_plugin(self.instance_connector)):
                     p._exists = None
-                    return self.instance_connector.sync_pipe_inplace(p, debug=debug, **kw)
+                    return self.instance_connector.sync_pipe_inplace(
+                        p,
+                        **filter_keywords(
+                            p.instance_connector.sync_pipe_inplace,
+                            debug=debug,
+                            **kw
+                        )
+                    )
 
 
             ### Activate and invoke `sync(pipe)` for plugin connectors with `sync` methods.
             try:
                 if getattr(p.connector, 'sync', None) is not None:
                     with Venv(get_connector_plugin(p.connector), debug=debug):
-                        return_tuple = p.connector.sync(p, debug=debug, **kw)
+                        return_tuple = p.connector.sync(
+                            p,
+                            **filter_keywords(
+                                p.connector.sync,
+                                debug=debug,
+                                **kw
+                            )
+                        )
                     p._exists = None
                     if not isinstance(return_tuple, tuple):
                         return_tuple = (
@@ -237,13 +251,19 @@ def sync(
             ### Fetch the dataframe from the connector's `fetch()` method.
             try:
                 with Venv(get_connector_plugin(p.connector), debug=debug):
-                    df = p.fetch(debug=debug, **kw)
+                    df = p.fetch(
+                        **filter_keywords(
+                            p.fetch,
+                            debug=debug,
+                            **kw
+                        )
+                    )
 
             except Exception as e:
                 get_console().print_exception(
-                    suppress = [
-                        'meerschaum/core/Pipe/_sync.py', 
-                        'meerschaum/core/Pipe/_fetch.py', 
+                    suppress=[
+                        'meerschaum/core/Pipe/_sync.py',
+                        'meerschaum/core/Pipe/_fetch.py',
                     ]
                 )
                 msg = f"Failed to fetch data from {p.connector}:\n    {e}"
@@ -289,7 +309,7 @@ def sync(
             if not chunk_success:
                 return chunk_success, f"Unable to sync initial chunk for {p}:\n{chunk_msg}"
             if debug:
-                dprint(f"Successfully synced the first chunk, attemping the rest...")
+                dprint("Successfully synced the first chunk, attemping the rest...")
 
             failed_chunks = []
             def _process_chunk(_chunk):
@@ -309,7 +329,6 @@ def sync(
                     )
                 )
 
-
             results = sorted(
                 [(chunk_success, chunk_msg)] + (
                     list(pool.imap(_process_chunk, df))
@@ -329,7 +348,7 @@ def sync(
             retry_success = True
             if not success and any(success_bools):
                 if debug:
-                    dprint(f"Retrying failed chunks...")
+                    dprint("Retrying failed chunks...")
                 chunks_to_retry = [c for c in failed_chunks]
                 failed_chunks = []
                 for chunk in chunks_to_retry:
@@ -361,9 +380,9 @@ def sync(
         while run:
             with Venv(get_connector_plugin(self.instance_connector)):
                 return_tuple = p.instance_connector.sync_pipe(
-                    pipe = p,
-                    df = df,
-                    debug = debug,
+                    pipe=p,
+                    df=df,
+                    debug=debug,
                     **kw
                 )
             _retries += 1
@@ -382,7 +401,7 @@ def sync(
         _checkpoint(**kw)
         if self.cache_pipe is not None:
             if debug:
-                dprint(f"Caching retrieved dataframe.", **kw)
+                dprint("Caching retrieved dataframe.", **kw)
                 _sync_cache_tuple = self.cache_pipe.sync(df, debug=debug, **kw)
                 if not _sync_cache_tuple[0]:
                     warn(f"Failed to sync local cache for {self}.")
@@ -395,10 +414,10 @@ def sync(
         return _sync(self, df = df)
 
     from meerschaum.utils.threading import Thread
-    def default_callback(result_tuple : SuccessTuple):
+    def default_callback(result_tuple: SuccessTuple):
         dprint(f"Asynchronous result from {self}: {result_tuple}", **kw)
 
-    def default_error_callback(x : Exception):
+    def default_error_callback(x: Exception):
         dprint(f"Error received for {self}: {x}", **kw)
 
     if callback is None and debug:
@@ -407,12 +426,12 @@ def sync(
         error_callback = default_error_callback
     try:
         thread = Thread(
-            target = _sync,
-            args = (self,),
-            kwargs = {'df' : df},
-            daemon = False,
-            callback = callback,
-            error_callback = error_callback
+            target=_sync,
+            args=(self,),
+            kwargs={'df': df},
+            daemon=False,
+            callback=callback,
+            error_callback=error_callback,
         )
         thread.start()
     except Exception as e:
@@ -424,12 +443,13 @@ def sync(
 
 
 def get_sync_time(
-        self,
-        params: Optional[Dict[str, Any]] = None,
-        newest: bool = True,
-        round_down: bool = False, 
-        debug: bool = False
-    ) -> Union['datetime', None]:
+    self,
+    params: Optional[Dict[str, Any]] = None,
+    newest: bool = True,
+    apply_backtrack_interval: bool = False,
+    round_down: bool = False,
+    debug: bool = False
+) -> Union['datetime', None]:
     """
     Get the most recent datetime value for a Pipe.
 
@@ -443,6 +463,9 @@ def get_sync_time(
         If `True`, get the most recent datetime (honoring `params`).
         If `False`, get the oldest datetime (`ASC` instead of `DESC`).
 
+    apply_backtrack_interval: bool, default False
+        If `True`, subtract the backtrack interval from the sync time.
+
     round_down: bool, default False
         If `True`, round down the datetime value to the nearest minute.
 
@@ -461,15 +484,22 @@ def get_sync_time(
     with Venv(get_connector_plugin(self.instance_connector)):
         sync_time = self.instance_connector.get_sync_time(
             self,
-            params = params,
-            newest = newest,
-            debug = debug,
+            params=params,
+            newest=newest,
+            debug=debug,
         )
 
-    if not round_down or not isinstance(sync_time, datetime):
-        return sync_time
+    if round_down and isinstance(sync_time, datetime):
+        sync_time = round_time(sync_time, timedelta(minutes=1))
+
+    if apply_backtrack_interval and sync_time is not None:
+        backtrack_interval = self.get_backtrack_interval(debug=debug)
+        try:
+            sync_time -= backtrack_interval
+        except Exception as e:
+            warn(f"Failed to apply backtrack interval:\n{e}")
 
-    return round_time(sync_time, timedelta(minutes=1))
+    return sync_time
 
 
 def exists(