meerschaum 3.0.0rc3__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

meerschaum/api/routes/_jobs.py

@@ -54,7 +54,7 @@ def _get_job(name: str, sysargs: Union[str, List[str], None] = None):
 
 @app.get(endpoints['jobs'], tags=['Jobs'])
 def get_jobs(
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:read'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:read'])),
 ) -> Dict[str, Dict[str, Any]]:
     """
     Return metadata about the current jobs.
@@ -83,7 +83,7 @@ def get_jobs(
 @app.get(endpoints['jobs'] + '/{name}', tags=['Jobs'])
 def get_job(
     name: str,
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:read'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:read'])),
 ) -> Dict[str, Any]:
     """
     Return metadata for a single job.
@@ -134,7 +134,7 @@ def clean_sysargs(sysargs: List[str]) -> List[str]:
 def create_job(
     name: str,
     metadata: Union[List[str], Dict[str, Any]],
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:execute', 'jobs:write'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:execute', 'jobs:write'])),
 ) -> SuccessTuple:
     """
     Create and start a new job.
@@ -167,7 +167,7 @@ def create_job(
 @app.delete(endpoints['jobs'] + '/{name}', tags=['Jobs'])
 def delete_job(
     name: str,
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:delete'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:delete'])),
 ) -> SuccessTuple:
     """
     Delete a job.
@@ -182,7 +182,7 @@ def delete_job(
 @app.get(endpoints['jobs'] + '/{name}/exists', tags=['Jobs'])
 def get_job_exists(
     name: str,
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:read'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:read'])),
 ) -> bool:
     """
     Return whether a job exists.
@@ -194,7 +194,7 @@ def get_job_exists(
 @app.get(endpoints['logs'] + '/{name}', tags=['Jobs'])
 def get_logs(
     name: str,
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:read', 'logs:read'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:read', 'logs:read'])),
 ) -> Union[str, None]:
     """
     Return a job's log text.
@@ -213,7 +213,7 @@ def get_logs(
 @app.post(endpoints['jobs'] + '/{name}/start', tags=['Jobs'])
 def start_job(
     name: str,
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:execute'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:execute'])),
 ) -> SuccessTuple:
     """
     Start a job if stopped.
@@ -234,7 +234,7 @@ def start_job(
 @app.post(endpoints['jobs'] + '/{name}/stop', tags=['Jobs'])
 def stop_job(
     name: str,
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:execute', 'josb:stop'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:execute', 'josb:stop'])),
 ) -> SuccessTuple:
     """
     Stop a job if running.
@@ -255,7 +255,7 @@ def stop_job(
 @app.post(endpoints['jobs'] + '/{name}/pause', tags=['Jobs'])
 def pause_job(
     name: str,
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:execute', 'jobs:pause'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:execute', 'jobs:pause'])),
 ) -> SuccessTuple:
     """
     Pause a job if running.
@@ -276,7 +276,7 @@ def pause_job(
 @app.get(endpoints['jobs'] + '/{name}/stop_time', tags=['Jobs'])
 def get_stop_time(
     name: str,
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:read'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:read'])),
 ) -> Union[datetime, None]:
     """
     Get the timestamp when the job was manually stopped.
@@ -288,7 +288,7 @@ def get_stop_time(
 @app.get(endpoints['jobs'] + '/{name}/is_blocking_on_stdin', tags=['Jobs'])
 def get_is_blocking_on_stdin(
     name: str,
-    curr_user=fastapi.Depends(ScopedAuth(['jobs:read'])),
+    curr_user=fastapi.Security(ScopedAuth(['jobs:read'])),
 ) -> bool:
     """
     Return whether a job is blocking on stdin.
@@ -297,6 +297,18 @@ def get_is_blocking_on_stdin(
     return job.is_blocking_on_stdin()
 
 
+@app.get(endpoints['jobs'] + '{name}/prompt_kwargs', tags=['Jobs'])
+def get_prompt_kwargs(
+    name: str,
+    curr_user=fastapi.Security(ScopedAuth(['jobs:read'])),
+) -> Dict[str, Any]:
+    """
+    Return the kwargs for the blocking `prompt`, if available.
+    """
+    job = _get_job(name)
+    return job.get_prompt_kwargs()
+
+
 _job_clients = defaultdict(lambda: [])
 _job_stop_events = defaultdict(lambda: asyncio.Event())
 _job_queues = defaultdict(lambda: asyncio.Queue())

meerschaum/api/routes/_login.py

@@ -16,22 +16,69 @@ from fastapi_login.exceptions import InvalidCredentialsException
 from fastapi.exceptions import RequestValidationError
 from starlette.responses import JSONResponse
 
-from meerschaum.api import endpoints, get_api_connector, app, debug, manager, no_auth
+import meerschaum as mrsm
+from meerschaum.api import (
+    endpoints,
+    get_api_connector,
+    get_cache_connector,
+    app,
+    debug,
+    manager,
+    no_auth,
+)
 from meerschaum.core import User
 from meerschaum._internal.static import STATIC_CONFIG
 from meerschaum.utils.typing import Dict, Any
-from meerschaum.utils.misc import is_uuid
+from meerschaum.utils.misc import is_uuid, is_int
 from meerschaum.core.User import verify_password
 from meerschaum.utils.warnings import warn
 from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
 
 
+USER_ID_CACHE_EXPIRES_SECONDS: int = mrsm.get_config('system', 'api', 'cache', 'session_expires_minutes') * 60
+_active_user_ids = {}
+
+
 @manager.user_loader()
-def load_user(username: str) -> User:
+def load_user_or_token_from_manager(username_or_token_id: str) -> User:
     """
     Create the `meerschaum.core.User` object from the username.
     """
-    return User(username, instance=get_api_connector())
+    cache_conn = get_cache_connector()
+    api_conn = get_api_connector()
+
+    is_token = is_uuid(username_or_token_id)
+
+    if is_token:
+        return api_conn.get_token(username_or_token_id)
+
+    username = username_or_token_id
+
+    cached_user_id = (
+        _active_user_ids.get(username)
+        if cache_conn is None
+        else cache_conn.get(f'mrsm:users:{username}:id')
+    )
+    if isinstance(cached_user_id, str):
+        if is_int(cached_user_id):
+            cached_user_id = int(cached_user_id)
+        elif is_uuid(cached_user_id):
+            cached_user_id = uuid.UUID(cached_user_id)
+
+    user = User(username, instance=api_conn, user_id=cached_user_id)
+
+    if cached_user_id is not None:
+        return user
+
+    user_id = api_conn.get_user_id(user)
+    if user_id is not None:
+        user._user_id = user_id
+        if cache_conn is not None:
+            cache_conn.set(f'mrsm:users:{username}:id', str(user_id), ex=USER_ID_CACHE_EXPIRES_SECONDS)
+        else:
+            _active_user_ids[username] = user_id
+    return user
+
 
 
 @app.post(endpoints['login'], tags=['Users'])
@@ -63,7 +110,10 @@ def login(
             else 'client_credentials'
         )
 
+    allowed_scopes = []
+    type_ = None
     expires_dt: Union[datetime, None] = None
+    sub_id = None
     if grant_type == 'password':
         user = User(str(username), str(password), instance=get_api_connector())
         correct_password = no_auth or verify_password(
@@ -73,6 +123,10 @@ def login(
         if not correct_password:
             raise InvalidCredentialsException
 
+        allowed_scopes = user.get_scopes(debug=debug)
+        type_ = get_api_connector().get_user_type(user, debug=debug)
+        sub_id = username
+
     elif grant_type == 'client_credentials':
         if not is_uuid(str(client_id)):
             raise InvalidCredentialsException
@@ -83,15 +137,29 @@ def login(
         )
         if not correct_password:
             raise InvalidCredentialsException
+
+        allowed_scopes = get_api_connector().get_token_scopes(token_id, debug=debug)
+        sub_id = client_id
+
     else:
         raise InvalidCredentialsException
 
+    requested_scopes = data.scope.split()
+    if '*' in allowed_scopes:
+        final_scopes = requested_scopes or ['*']
+    else:
+        final_scopes = [
+            s for s in requested_scopes if s in allowed_scopes
+        ] if requested_scopes else allowed_scopes
+
     expires_minutes = STATIC_CONFIG['api']['oauth']['token_expires_minutes']
     expires_delta = timedelta(minutes=expires_minutes)
     expires_dt = datetime.now(timezone.utc).replace(tzinfo=None) + expires_delta
     access_token = manager.create_access_token(
         data={
-            'sub': (username if grant_type == 'password' else client_id),
+            'sub': sub_id,
+            'scopes': final_scopes,
+            'type': type_,
         },
         expires=expires_delta
     )

meerschaum/api/routes/_pipes.py

@@ -25,6 +25,7 @@ from meerschaum.api import (
     _get_pipes,
     debug,
     ScopedAuth,
+    manager,
 )
 from meerschaum.models import (
     ConnectorKeysModel,
@@ -148,7 +149,7 @@ def delete_pipe(
     location_key: str,
     instance_keys: Optional[str] = None,
     curr_user = fastapi.Security(ScopedAuth(['pipes:delete'])),
-) -> SuccessTuple:
+) -> mrsm.SuccessTuple:
     """
     Delete a Pipe (without dropping its table).
     """
@@ -187,7 +188,10 @@ async def fetch_pipes_keys(
         tags=json.loads(tags),
         params=json.loads(params),
     )
-    return keys
+    return [
+        (keys_tuple[0], keys_tuple[1], keys_tuple[2])
+        for keys_tuple in keys
+    ]
 
 
 @app.get(pipes_endpoint, tags=['Pipes: Attributes'])
@@ -197,7 +201,6 @@ async def get_pipes(
     location_keys: str = "",
     instance_keys: str = "",
     curr_user=fastapi.Security(ScopedAuth(['pipes:read'])),
-    debug: bool = False,
 ) -> PipesWithParametersDictModel:
     """
     Get all registered Pipes with metadata, excluding parameters.
@@ -384,7 +387,6 @@ async def sync_pipe(
     workers: Optional[int] = None,
     columns: Optional[str] = None,
     curr_user = fastapi.Security(ScopedAuth(['pipes:write'])),
-    debug: bool = False,
 ) -> mrsm.SuccessTuple:
     """
     Add data to an existing Pipe.

meerschaum/api/routes/_webterm.py

@@ -8,7 +8,7 @@ Routes to the Webterm proxy.
 
 import asyncio
 from meerschaum.utils.typing import Optional
-from meerschaum.api import app, endpoints
+from meerschaum.api import app, endpoints, webterm_port
 from meerschaum.utils.packages import attempt_import
 from meerschaum.api.dash.sessions import is_session_authenticated, get_username_from_session
 fastapi, fastapi_responses = attempt_import('fastapi', 'fastapi.responses')
@@ -71,7 +71,7 @@ async def get_webterm(
 
     username = get_username_from_session(session_id)
     async with httpx.AsyncClient() as client:
-        webterm_url = f"http://localhost:8765/webterm/{username or session_id}"
+        webterm_url = f"http://localhost:{webterm_port}/webterm/{username or session_id}"
         response = await client.get(webterm_url)
         text = response.text
         if request.url.scheme == 'https':
@@ -100,7 +100,7 @@ async def webterm_websocket(websocket: WebSocket, session_id: str):
 
     username = get_username_from_session(session_id)
 
-    ws_url = f"ws://localhost:8765/websocket/{username or session_id}"
+    ws_url = f"ws://localhost:{webterm_port}/websocket/{username or session_id}"
     async with websockets.connect(ws_url) as ws:
         async def forward_messages():
             try:

meerschaum/config/__init__.py

@@ -13,8 +13,9 @@ import os
 import shutil
 import sys
 import copy
+import contextlib
 
-from meerschaum.utils.typing import Any, Dict, Optional
+from meerschaum.utils.typing import Any, Dict, Optional, Union
 from meerschaum.utils.threading import RLock
 
 from meerschaum.config._version import __version__
@@ -43,6 +44,7 @@ __all__ = (
     'write_config',
     'edit_config',
     'set_config',
+    'replace_config',
     'search_and_substitute_config',
     'revert_symlinks_config',
     'get_possible_keys',
@@ -57,17 +59,23 @@ _locks = {'config': RLock()}
 
 ### apply config preprocessing (e.g. main to meta)
 config = {}
+_backup_config = None
+_allow_write_missing: bool = True
+
+
 def _config(
     *keys: str,
     reload: bool = False,
     substitute: bool = True,
     sync_files: bool = True,
+    allow_replaced: bool = True,
     write_missing: bool = True,
 ) -> Dict[str, Any]:
     """
     Read and process the configuration file.
     """
-    global config
+    global config, _backup_config
+
     if config is None or reload:
         with _locks['config']:
             config = {}
@@ -77,12 +85,16 @@ def _config(
         key_config = read_config(
             keys = [keys[0]],
             substitute = substitute,
-            write_missing = write_missing,
+            write_missing = write_missing and _allow_write_missing,
         )
         if keys[0] in key_config:
             config[keys[0]] = key_config[keys[0]]
-            if sync_files:
+            if sync_files and _allow_write_missing:
                 _sync_files(keys=[keys[0] if keys else None])
+
+    if not allow_replaced:
+        return _backup_config if _backup_config is not None else config
+
     return config
 
 
@@ -158,7 +170,11 @@ def get_config(
         dprint(f"Indexing keys: {keys}", color=False)
 
     if len(keys) == 0:
-        _rc = _config(substitute=substitute, sync_files=sync_files, write_missing=write_missing)
+        _rc = _config(
+            substitute=substitute,
+            sync_files=sync_files,
+            write_missing=(write_missing and _allow_write_missing),
+        )
         if as_tuple:
             return True, _rc 
         return _rc
@@ -177,15 +193,15 @@ def get_config(
         except Exception:
             import traceback
             traceback.print_exc()
+            _subbed = {keys[0]: config[keys[0]]}
+
         config[keys[0]] = _subbed[keys[0]]
         if symlinks_key in _subbed:
             if symlinks_key not in config:
                 config[symlinks_key] = {}
-            if keys[0] not in config[symlinks_key]:
-                config[symlinks_key][keys[0]] = {}
-            config[symlinks_key][keys[0]] = apply_patch_to_config(
-                _subbed,
-                {symlinks_key: config[symlinks_key][keys[0]]}
+            config[symlinks_key] = apply_patch_to_config(
+                _subbed.get(symlinks_key, {}),
+                config.get(symlinks_key, {}),
             )
 
     from meerschaum.config._sync import sync_files as _sync_files
@@ -312,13 +328,44 @@ def write_plugin_config(
     return write_config(cf, **kw)
 
 
+@contextlib.contextmanager
+def replace_config(config_: Union[Dict[str, Any], None]):
+    """
+    Temporarily override the Meerschaum config dictionary.
+
+    Parameters
+    ----------
+    config_: Dict[str, Any]
+        The new config dictionary to temporarily replace the canonical `config`.
+    """
+    if config_ is None:
+        try:
+            yield
+        finally:
+            return
+
+    global _backup_config, _allow_write_missing
+
+    _backup_config = _config()
+    _allow_write_missing = False
+    set_config(config_)
+
+    try:
+        yield
+    finally:
+        set_config(_backup_config)
+        _allow_write_missing = True
+
 ### This need to be below get_config to avoid a circular import.
 from meerschaum.config._read_config import read_config
 
 ### If environment variable MRSM_CONFIG or MRSM_PATCH is set, patch config before anything else.
-from meerschaum.config._environment import apply_environment_patches, apply_environment_uris
-apply_environment_uris()
-apply_environment_patches()
+from meerschaum.config.environment import (
+    apply_environment_patches as _apply_environment_patches,
+    apply_environment_uris as _apply_environment_uris,
+)
+_apply_environment_uris()
+_apply_environment_patches()
 
 
 from meerschaum.config._paths import PATCH_DIR_PATH, PERMANENT_PATCH_DIR_PATH

meerschaum/config/_default.py

@@ -36,8 +36,7 @@ CONNECTOR_ATTRIBUTES: Dict[str, Dict[str, Any]] = {
 default_meerschaum_config = {
     'instance': 'sql:main',
     'api_instance': 'MRSM{meerschaum:instance}',
-    'web_instance': 'MRSM{meerschaum:instance}',
-    'default_repository': 'api:mrsm',
+    'repository': 'api:mrsm',
     'connectors': {
         'sql': {
             'default': {},
@@ -51,7 +50,7 @@ default_meerschaum_config = {
             },
             'local': {
                 'flavor': 'sqlite',
-                'database': SQLITE_DB_PATH.as_posix(),
+                'database': "{SQLITE_DB_PATH}",
             },
             'memory': {
                 'flavor': 'sqlite',
@@ -117,75 +116,107 @@ default_system_config = {
                 'connect_args': {},
             },
         },
-
         'api': {
         },
     },
-    ### not to be confused with system_config['connectors']['api'], this is the configuration
-    ### for the API server itself.
-    'api': {
-        'uvicorn': {
-            'app': 'meerschaum.api:app',
-            'port': 8000,
-            'host': '0.0.0.0',
-            'workers': max(int(multiprocessing.cpu_count() / 2), 1),
-            'proxy_headers': True,
-            'forwarded_allow_ips': '*',
-        },
-        'cache': {
-            'connector': 'valkey:main',
-            'session_expires_minutes': 43200,
+    'api': 'MRSM{api}',
+    'webterm': 'MRSM{api:webterm}',
+    'cli': {
+        'max_daemons': (multiprocessing.cpu_count() * 3),
+        'refresh_seconds': 0.1,
+        'allowed_prefixes': ['*'],
+        'disallowed_prefixes': [
+            'edit',
+            'start daemon',
+            'start job',
+            'stop daemon',
+            'show daemon',
+            'restart daemon',
+            'reload',
+            'start worker',
+            'start job',
+            'python',
+            'login',
+            'executor',
+            'os ',
+            'sh ',
+            'start api',
+            'start webterm',
+            'stack',
+            'instance',
+            'debug',
+            'bootstrap',
+            'daemon',
+        ],
+    },
+    'experimental': {
+        'fetch': False,
+        'cache': True,
+        'space': False,
+        'join_fetch': False,
+        'inplace_sync': True,
+        'uv_pip': True,
+        'systemd_healthcheck': False,
+        'valkey_session_cache': True,
+        'cli_daemon': True,
+    },
+}
+
+default_api_config = {
+    'uvicorn': {
+        'app': 'meerschaum.api:app',
+        'port': 8000,
+        'host': '0.0.0.0',
+        'workers': max(int(multiprocessing.cpu_count() / 2), 1),
+        'proxy_headers': True,
+        'forwarded_allow_ips': '*',
+    },
+    'cache': {
+        'connector': 'valkey:main',
+        'session_expires_minutes': 43200,
+    },
+    'data': {
+        'max_response_row_limit': 100_000,
+        'chunks': {
+            'ttl_seconds': 1800,
         },
-        'data': {
-            'max_response_row_limit': 100_000,
-            'chunks': {
-                'ttl_seconds': 1800,
-            },
+    },
+    'endpoints': {
+        'docs_in_production': True,
+    },
+    'tokens': {
+        'valid_refresh_minutes': 60,
+        'default_expiration_days': 366,
+    },
+    'permissions':       {
+        'registration': {
+            'users': True,
+            'pipes': True,
+            'plugins': True,
         },
-        'endpoints': {
-            'docs_in_production': True,
+        'actions': {
+            'non_admin': True,
         },
-        'tokens': {
-            'valid_refresh_minutes': 60,
-            'default_expiration_days': 366,
+        'chaining': {
+            'insecure_parent_instance': False,
+            'child_apis': False,
         },
-        'permissions':       {
-            'registration': {
-                'users': True,
-                'pipes': True,
-                'plugins': True,
-            },
-            'actions': {
-                'non_admin': True,
-            },
-            'chaining': {
-                'insecure_parent_instance': False,
-                'child_apis': False,
-            },
-            'instances': {
-                'allow_multiple_instances': True,
-                'allowed_instance_keys': ['*']
-            },
+        'instances': {
+            'allow_multiple_instances': True,
+            'allowed_instance_keys': ['*']
         },
-        'protocol': default_meerschaum_config['connectors']['api']['default']['protocol'],
     },
+    'protocol': default_meerschaum_config['connectors']['api']['default']['protocol'],
     'webterm': {
         'tmux': {
             'enabled': True,
             'session_suffix': '_mrsm',
         },
-    },
-    'experimental': {
-        'fetch': False,
-        'cache': True,
-        'space': False,
-        'join_fetch': False,
-        'inplace_sync': True,
-        'uv_pip': True,
-        'systemd_healthcheck': False,
-        'valkey_session_cache': True,
+        'host': '127.0.0.1',
+        'port': 8765,
     },
 }
+
 default_pipes_config = {
     'parameters': {
         'columns': {
@@ -221,15 +252,12 @@ default_pipes_config = {
     },
 }
 default_plugins_config = {}
-default_experimental_config = {
-    'venv': True,
-}
-
 
 ### build default config dictionary
 default_config = {}
 default_config['meerschaum'] = default_meerschaum_config
 default_config['system'] = default_system_config
+default_config['api'] = default_api_config
 from meerschaum.config._formatting import default_formatting_config
 default_config['formatting'] = default_formatting_config
 from meerschaum.config._shell import default_shell_config
@@ -241,7 +269,7 @@ default_config['jobs'] = default_jobs_config
 ### add configs from other packages
 try:
     import meerschaum.config.stack
-except ImportError as e:
+except ImportError:
     pass
 finally:
     from meerschaum.config.stack import default_stack_config