meerschaum 2.3.5.dev0__py3-none-any.whl → 2.4.0.dev0__py3-none-any.whl

meerschaum/api/dash/pipes.py

@@ -48,13 +48,14 @@ def pipe_from_ctx(ctx, trigger_property: str = 'n_clicks') -> Union[mrsm.Pipe, N
         return None
     return mrsm.Pipe(**meta)
 
+
 def keys_from_state(
-        state: Dict[str, Any],
-        with_params: bool = False
-    ) -> Union[
-        Tuple[List[str], List[str], List[str]],
-        Tuple[List[str], List[str], List[str], str],
-    ]:
+    state: Dict[str, Any],
+    with_params: bool = False
+) -> Union[
+    Tuple[List[str], List[str], List[str]],
+    Tuple[List[str], List[str], List[str], str],
+]:
     """
     Read the current state and return the selected keys lists.
     """
@@ -85,10 +86,11 @@ def keys_from_state(
         keys.append(params)
     return tuple(keys)
 
+
 def pipes_from_state(
-        state: Dict[str, Any],
-        **kw
-    ):
+    state: Dict[str, Any],
+    **kw
+):
     _ck, _mk, _lk, _params = keys_from_state(state, with_params=True)
     try:
         _pipes = _get_pipes(
@@ -103,10 +105,10 @@ def pipes_from_state(
 
 
 def build_pipe_card(
-        pipe: mrsm.Pipe,
-        authenticated: bool = False,
-        _build_children_num: int = 10,
-    ) -> 'dbc.Card':
+    pipe: mrsm.Pipe,
+    authenticated: bool = False,
+    _build_children_num: int = 10,
+) -> 'dbc.Card':
     """
     Return a card for the given pipe.
 
@@ -128,11 +130,11 @@ def build_pipe_card(
             dbc.Col(
                 (
                     dbc.DropdownMenu(
-                        label = "Manage",
-                        children = [
+                        label="Manage",
+                        children=[
                             dbc.DropdownMenuItem(
                                 'Open in Python',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'python',
@@ -140,7 +142,7 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Delete',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'delete',
@@ -148,7 +150,7 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Drop',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'drop',
@@ -156,7 +158,7 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Clear',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'clear',
@@ -164,7 +166,7 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Verify',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'verify',
@@ -172,56 +174,86 @@ def build_pipe_card(
                             ),
                             dbc.DropdownMenuItem(
                                 'Sync',
-                                id = {
+                                id={
                                     'type': 'manage-pipe-button',
                                     'index': meta_str,
                                     'action': 'sync',
                                 },
                             ),
                         ],
-                        direction = "up",
-                        menu_variant = "dark",
-                        size = 'sm',
-                        color = 'secondary',
+                        direction="up",
+                        menu_variant="dark",
+                        size='sm',
+                        color='secondary',
                     )
                 ) if authenticated else [],
-                width = 2,
+                width=2,
             ),
             dbc.Col(width=6),
             dbc.Col(
                 dbc.Button(
                     'Download CSV',
-                    size = 'sm',
-                    color = 'link',
-                    style = {'float': 'right'},
-                    id = {'type': 'pipe-download-csv-button', 'index': meta_str},
+                    size='sm',
+                    color='link',
+                    style={'float': 'right'},
+                    id={'type': 'pipe-download-csv-button', 'index': meta_str},
                 ),
-                width = 4,
+                width=4,
             ),
         ],
-        justify = 'start',
+        justify='start',
     )
     card_body_children = [
-        html.H5(
-            html.B(str(pipe)),
-            className = 'card-title',
-            style = {'font-family': ['monospace']}
-        ),
         html.Div(
             dbc.Accordion(
                 accordion_items_from_pipe(
                     pipe,
-                    authenticated = authenticated,
-                    _build_children_num = _build_children_num,
+                    authenticated=authenticated,
+                    _build_children_num=_build_children_num,
                 ),
-                flush = True,
-                start_collapsed = True,
-                id = {'type': 'pipe-accordion', 'index': meta_str},
+                flush=True,
+                start_collapsed=True,
+                id={'type': 'pipe-accordion', 'index': meta_str},
             )
         )
 
     ]
+
+    pipe_url = (
+        f"/dash/pipes/{pipe.connector_keys}/{pipe.metric_key}/{pipe.location_key}"
+    )
+
+    card_header_children = dbc.Row(
+        [
+            dbc.Col(
+                html.H5(
+                    html.B(str(pipe)),
+                    className='card-title',
+                    style={'font-family': ['monospace']}
+                ),
+                width=11,
+            ),
+            dbc.Col(
+                dbc.Button(
+                    [
+                        html.I(
+                            className="bi bi-share",
+                        ),
+                    ],
+                    #  href=pipe_url,
+                    style={'float': 'right'},
+                    outline=True,
+                    color='link',
+                    id={'type': 'share-pipe-button', 'index': meta_str},
+                ),
+                width=1,
+            ),
+        ],
+        justify='start',
+    )
+
     return dbc.Card([
+        dbc.CardHeader(children=card_header_children),
         dbc.CardBody(children=card_body_children),
         dbc.CardFooter(children=footer_children),
     ])
@@ -270,11 +302,11 @@ def get_pipes_cards(*keys, session_data: Optional[Dict[str, Any]] = None):
 
 
 def accordion_items_from_pipe(
-        pipe: mrsm.Pipe,
-        active_items: Optional[List[str]] = None,
-        authenticated: bool = False,
-        _build_children_num: int = 10,
-    ) -> 'List[dbc.AccordionItem]':
+    pipe: mrsm.Pipe,
+    active_items: Optional[List[str]] = None,
+    authenticated: bool = False,
+    _build_children_num: int = 10,
+) -> 'List[dbc.AccordionItem]':
     """
     Build the accordion items for a given pipe.
     """

meerschaum/api/dash/users.py

@@ -18,6 +18,7 @@ from meerschaum.core import User
 dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 
+
 def get_users_cards(state: WebState) -> Tuple[List[dbc.Card], List[SuccessTuple]]:
     """
     Return the cards and alerts for users.
@@ -52,21 +53,33 @@ def is_session_authenticated(session_id: str) -> bool:
     """
     if no_auth:
         return True
+    if session_id not in active_sessions:
+        return False
     if session_id in unauthenticated_sessions:
         return False
     if session_id in authenticated_sessions:
         return True
+
     permissions = get_config('system', 'api', 'permissions')
     allow_non_admin = permissions.get('actions', {}).get('non_admin', False)
-    if allow_non_admin:
-        return True
-    conn = get_api_connector()
+
+    is_auth = True if allow_non_admin else session_is_admin(session_id)
     username = active_sessions.get(session_id, {}).get('username', None)
-    user = User(username, instance=conn)
-    user_type = conn.get_user_type(user, debug=debug)
-    is_auth = user_type == 'admin'
+
     if is_auth:
         authenticated_sessions[session_id] = username
     else:
         unauthenticated_sessions[session_id] = username
+
     return is_auth
+
+
+def session_is_admin(session_id: str) -> bool:
+    """
+    Check whether a session ID corresponds to an admin user.
+    """
+    conn = get_api_connector()
+    username = active_sessions.get(session_id, {}).get('username', None)
+    user = User(username, instance=conn)
+    user_type = conn.get_user_type(user, debug=debug)
+    return user_type == 'admin'

meerschaum/api/routes/_actions.py

@@ -67,104 +67,6 @@ def get_actions(
     return list(actions)
 
 
-async def notify_client(client, content: str):
-    """
-    Send a line of text to a client.
-    """
-    try:
-        await client.send_text(content)
-    except (RuntimeError, WebSocketDisconnect, Exception):
-        raise StopMonitoringLogs
-
-_temp_jobs = {}
-@app.websocket(actions_endpoint + '/ws')
-async def do_action_websocket(websocket: WebSocket):
-    """
-    Execute an action and stream the output to the client.
-    """
-    await websocket.accept()
-
-    stop_event = asyncio.Event()
-    job_name = '.' + generate_password(12)
-    job = None
-
-    async def monitor_logs(job):
-        success, msg = job.start()
-        await job.monitor_logs_async(
-            partial(notify_client, websocket),
-            stop_event=stop_event,
-            stop_on_exit=True,
-        )
-
-    try:
-        token = await websocket.receive_text()
-        user = await manager.get_current_user(token) if not no_auth else None
-        if user is None and not no_auth:
-            stop_event.set()
-            if job is not None:
-                job.delete()
-                _ = _temp_jobs.pop(job_name, None)
-            raise fastapi.HTTPException(
-                status_code=401,
-                detail="Invalid credentials.",
-            )
-
-        auth_success, auth_msg = (
-            is_user_allowed_to_execute(user)
-            if not no_auth
-            else (True, "Success")
-        )
-        auth_payload = {
-            'is_authenticated': auth_success,
-            'timestamp': datetime.now(timezone.utc).isoformat(),
-        }
-        await websocket.send_json(auth_payload)
-        if not auth_success:
-            stop_event.set()
-            job.stop()
-            await websocket.close()
-
-        sysargs = clean_sysargs(await websocket.receive_json())
-        job = Job(
-            job_name,
-            sysargs,
-            executor_keys='local',
-            _properties={
-                'logs': {
-                    'write_timestamps': False,
-                },
-            },
-        )
-        _temp_jobs[job_name] = job
-        monitor_task = asyncio.create_task(monitor_logs(job))
-
-        ### NOTE: Await incoming text to trigger `WebSocketDisconnect`.
-        while True:
-            await websocket.receive_text()
-
-    except fastapi.HTTPException:
-        await websocket.send_text("Invalid credentials.")
-        await websocket.close()
-    except (WebSocketDisconnect, asyncio.CancelledError):
-        stop_event.set()
-        job.stop()
-    except Exception:
-        stop_event.set()
-        job.stop()
-        warn(f"Error in logs websocket:\n{traceback.format_exc()}")
-    finally:
-        stop_event.set()
-        job.stop()
-        monitor_task.cancel()
-        if job is not None:
-            job.delete()
-        _ = _temp_jobs.pop(job_name, None)
-        try:
-            await websocket.close()
-        except RuntimeError:
-            pass
-
-
 @app.post(actions_endpoint + "/{action}", tags=['Actions'])
 def do_action_legacy(
     action: str,

meerschaum/api/routes/_jobs.py

@@ -40,7 +40,15 @@ from meerschaum.config.static import STATIC_CONFIG
 
 JOBS_STDIN_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stdin_message']
 JOBS_STOP_MESSAGE: str = STATIC_CONFIG['api']['jobs']['stop_message']
-EXECUTOR_KEYS: str = get_executor_keys_from_context()
+EXECUTOR_KEYS: str = 'local'
+
+
+def _get_job(name: str):
+    systemd_job = Job(name, executor_keys='systemd')
+    if systemd_job.exists():
+        return systemd_job
+
+    return Job(name, executor_keys=EXECUTOR_KEYS)
 
 
 @app.get(endpoints['jobs'], tags=['Jobs'])
@@ -52,7 +60,7 @@ def get_jobs(
     """
     Return metadata about the current jobs.
     """
-    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=False)
+    jobs = _get_jobs(executor_keys=EXECUTOR_KEYS, combine_local_and_systemd=True)
     return {
         name: {
             'sysargs': job.sysargs,
@@ -83,7 +91,7 @@ def get_job(
     """
     Return metadata for a single job.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -163,7 +171,7 @@ def delete_job(
     """
     Delete a job.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.delete()
 
 
@@ -177,7 +185,7 @@ def get_job_exists(
     """
     Return whether a job exists.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.exists()
 
 
@@ -192,7 +200,7 @@ def get_logs(
     Return a job's log text.
     To stream log text, connect to the WebSocket endpoint `/logs/{name}/ws`.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -212,7 +220,7 @@ def start_job(
     """
     Start a job if stopped.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -231,7 +239,7 @@ def stop_job(
     """
     Stop a job if running.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -250,7 +258,7 @@ def pause_job(
     """
     Pause a job if running.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     if not job.exists():
         raise fastapi.HTTPException(
             status_code=404,
@@ -269,7 +277,7 @@ def get_stop_time(
     """
     Get the timestamp when the job was manually stopped.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.stop_time
 
 
@@ -283,12 +291,13 @@ def get_is_blocking_on_stdin(
     """
     Return whether a job is blocking on stdin.
     """
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     return job.is_blocking_on_stdin()
 
 
 _job_clients = defaultdict(lambda: [])
 _job_stop_events = defaultdict(lambda: asyncio.Event())
+_job_queues = defaultdict(lambda: asyncio.Queue())
 async def notify_clients(name: str, websocket: WebSocket, content: str):
     """
     Write the given content to all connected clients.
@@ -315,12 +324,16 @@ async def get_input_from_clients(name: str, websocket: WebSocket) -> str:
     async def _read_client(client):
         try:
             await client.send_text(JOBS_STDIN_MESSAGE)
-            data = await client.receive_text()
+            data = await _job_queues[name].get()
         except WebSocketDisconnect:
             if client in _job_clients[name]:
                 _job_clients[name].remove(client)
+            if not _job_clients[name]:
+                _job_stop_events[name].set()
         except Exception:
             pass
+        finally:
+            _job_queues[name].task_done()
         return data
 
     read_tasks = [
@@ -357,10 +370,12 @@ async def logs_websocket(name: str, websocket: WebSocket):
     Stream logs from a job over a websocket.
     """
     await websocket.accept()
-    job = Job(name, executor_keys=EXECUTOR_KEYS)
+    job = _get_job(name)
     _job_clients[name].append(websocket)
 
+    _task = None
     async def monitor_logs():
+        nonlocal _task
         try:
             callback_function = partial(
                 notify_clients,
@@ -377,16 +392,17 @@ async def logs_websocket(name: str, websocket: WebSocket):
                 name,
                 websocket,
             )
-            await job.monitor_logs_async(
+            _task = asyncio.create_task(job.monitor_logs_async(
                 callback_function=callback_function,
                 input_callback_function=input_callback_function,
                 stop_callback_function=stop_callback_function,
                 stop_event=_job_stop_events[name],
                 stop_on_exit=True,
                 accept_input=True,
-            )
+            ))
         except Exception:
             warn(traceback.format_exc())
+            _task.cancel()
 
     try:
         token = await websocket.receive_text()
@@ -397,13 +413,17 @@ async def logs_websocket(name: str, websocket: WebSocket):
                 detail="Invalid credentials.",
             )
         monitor_task = asyncio.create_task(monitor_logs())
-        await monitor_task
+        while True:
+            text = await websocket.receive_text()
+            await _job_queues[name].put(text)
+
     except fastapi.HTTPException:
         await websocket.send_text("Invalid credentials.")
         await websocket.close()
     except WebSocketDisconnect:
-        _job_stop_events[name].set()
-        monitor_task.cancel()
+        if not _job_clients[name]:
+            _job_stop_events[name].set()
+            monitor_task.cancel()
     except asyncio.CancelledError:
         pass
     except Exception:

meerschaum/api/routes/_login.py

@@ -23,8 +23,8 @@ from meerschaum.api._oauth2 import CustomOAuth2PasswordRequestForm
 
 @manager.user_loader()
 def load_user(
-        username: str
-    ) -> User:
+    username: str
+) -> User:
     """
     Create the `meerschaum.core.User` object from the username.
     """
@@ -33,8 +33,8 @@ def load_user(
 
 @app.post(endpoints['login'], tags=['Users'])
 def login(
-        data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
-    ) -> Dict[str, Any]:
+    data: CustomOAuth2PasswordRequestForm = fastapi.Depends()
+) -> Dict[str, Any]:
     """
     Login and set the session token.
     """

meerschaum/api/routes/_pipes.py

@@ -474,9 +474,9 @@ def get_pipe_data(
         df[col] = df[col].apply(lambda x: f'{x:f}' if isinstance(x, Decimal) else x)
 
     json_content = df.to_json(
-        date_format = 'iso',
-        orient = 'records',
-        date_unit = 'us',
+        date_format='iso',
+        orient='records',
+        date_unit='us',
     )
 
     return fastapi.Response(

meerschaum/config/_default.py

@@ -16,7 +16,6 @@ default_meerschaum_config = {
     'api_instance': 'MRSM{meerschaum:instance}',
     'web_instance': 'MRSM{meerschaum:instance}',
     'default_repository': 'api:mrsm',
-    #  'default_executor': 'local',
     'connectors': {
         'sql': {
             'default': {},
@@ -53,6 +52,14 @@ default_meerschaum_config = {
                 'protocol': 'https',
             },
         },
+        'valkey': {
+            'main': {
+                'host': 'localhost',
+                'username': 'default',
+                'password': 'mrsm',
+                'port': 6379,
+            },
+        },
     },
 }
 default_system_config = {
@@ -75,7 +82,7 @@ default_system_config = {
             },
         },
 
-        'api' : {
+        'api': {
         },
     },
     ### not to be confused with system_config['connectors']['api'], this is the configuration

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.3.5.dev0"
+__version__ = "2.4.0.dev0"