bfabric-web-apps 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

bfabric_web_apps/__init__.py

@@ -16,85 +16,42 @@ from .utils.get_power_user_wrapper import get_power_user_wrapper
 from .utils.create_app_in_bfabric import create_app_in_bfabric
 
 # Export callbacks
-from .utils.callbacks import process_url_and_token, submit_bug_report
+from .utils.callbacks import (
+    process_url_and_token, 
+    submit_bug_report,
+    populate_workunit_details,
+    get_redis_queue_layout
+)
 
-from .utils import defaults
+from .utils.config import settings as config
 
-from bfabric_web_apps.utils.resource_utilities import create_workunit, create_resource
-HOST = os.getenv("HOST", defaults.HOST)
-PORT = int(os.getenv("PORT", defaults.PORT))  # Convert to int since env variables are strings
-DEV = os.getenv("DEV", str(defaults.DEV)).lower() in ["true", "1", "yes"]  # Convert to bool
-CONFIG_FILE_PATH = os.getenv("CONFIG_FILE_PATH", defaults.CONFIG_FILE_PATH)
+from. utils.run_main_pipeline import run_main_job
 
-DEVELOPER_EMAIL_ADDRESS = os.getenv("DEVELOPER_EMAIL_ADDRESS", defaults.DEVELOPER_EMAIL_ADDRESS)
-BUG_REPORT_EMAIL_ADDRESS = os.getenv("BUG_REPORT_EMAIL_ADDRESS", defaults.BUG_REPORT_EMAIL_ADDRESS)
+from .utils.resource_utilities import (
+    create_workunit, 
+    create_resource, 
+    create_workunits, 
+    create_resources
+)
 
+from .utils.redis_worker_init import run_worker, test_job
+from .utils.redis_queue import q
 
-# Define __all__ for controlled imports
-__all__ = [
-    "BfabricInterface",
-    "Logger",
-    "components",
-    "get_static_layout",
-    "create_app",
-    "process_url_and_token",
-    "submit_bug_report",
-    'get_logger',
-    'get_power_user_wrapper',
-    'HOST',
-    'PORT', 
-    'DEV',
-    'CONFIG_FILE_PATH',
-    'DEVELOPER_EMAIL_ADDRESS',
-    'BUG_REPORT_EMAIL_ADDRESS',
-    'create_app_in_bfabric',
-    'create_workunit',
-    'create_resource'
-]
+REDIS_HOST = config.REDIS_HOST
+REDIS_PORT = config.REDIS_PORT
 
+HOST = config.HOST
+PORT = config.PORT
+DEV = config.DEV
+DEBUG = config.DEBUG
 
+CONFIG_FILE_PATH = config.CONFIG_FILE_PATH
 
-'''
-import os
-from .utils import defaults
-
-# Private variable for CONFIG_FILE_PATH
-_CONFIG_FILE_PATH = os.getenv("CONFIG_FILE_PATH", defaults.CONFIG_FILE_PATH)
-
-def set_config_file_path(path):
-    """
-    Setter for the CONFIG_FILE_PATH variable.
-    """
-    global _CONFIG_FILE_PATH
-    if not isinstance(path, str):
-        raise ValueError("CONFIG_FILE_PATH must be a string.")
-    _CONFIG_FILE_PATH = path
-
-def get_config_file_path():
-    """
-    Getter for the CONFIG_FILE_PATH variable.
-    """
-    return _CONFIG_FILE_PATH
-
-# Expose CONFIG_FILE_PATH as a read-only property
-class Config:
-    @property
-    def CONFIG_FILE_PATH(self):
-        return get_config_file_path()
-
-config = Config()
-
-'''
-
-
-
-'''
-from bfabric import config
-
-config.CONFIG_FILE_PATH
-'''
+DEVELOPER_EMAIL_ADDRESS = config.DEVELOPER_EMAIL_ADDRESS
+BUG_REPORT_EMAIL_ADDRESS = config.BUG_REPORT_EMAIL_ADDRESS
 
-'''
-from bfabric import set_config_file_path
-set_config_file_path("new/path/to/config.json")
-'''
+GSTORE_REMOTE_PATH = config.GSTORE_REMOTE_PATH
+SCRATCH_PATH = config.SCRATCH_PATH
+TRX_LOGIN = config.TRX_LOGIN
+TRX_SSH_KEY = config.TRX_SSH_KEY
+URL = config.URL

bfabric_web_apps/layouts/layouts.py

@@ -2,7 +2,7 @@ from dash import html, dcc
 import dash_bootstrap_components as dbc
 import bfabric_web_apps
 
-def get_static_layout(base_title=None, main_content=None, documentation_content=None):
+def get_static_layout(base_title=None, main_content=None, documentation_content=None, layout_config={}):
     """
     Returns a layout with static tabs for Main, Documentation, and Report a Bug.
     The main content is customizable, while the other tabs are generic.
@@ -11,10 +11,24 @@ def get_static_layout(base_title=None, main_content=None, documentation_content=
         base_title (str): The main title to be displayed in the banner.
         main_content (html.Div): Content to be displayed in the "Main" tab.
         documentation_content (html.Div): Content for the "Documentation" tab.
+        layout_config (dict): Configuration for the layout, determining which tabs are shown.
 
     Returns:
         html.Div: The complete static layout of the web app.
     """
+
+    tab_list = [
+        dbc.Tab(main_content, label="Main", tab_id="main"),
+        dbc.Tab(dcc.Loading(get_documentation_tab(documentation_content)), label="Documentation", tab_id="documentation"),
+    ]
+
+    if layout_config.get("workunits", False):
+        tab_list.append(dbc.Tab(dcc.Loading(get_workunits_tab()), label="Workunits", tab_id="workunits"))
+    if layout_config.get("queue", False):
+        tab_list.append(dbc.Tab(get_queue_tab(), label="Queue", tab_id="queue"))
+    if layout_config.get("bug", False):
+        tab_list.append(dbc.Tab(dcc.Loading(get_report_bug_tab()), label="Report a Bug", tab_id="report-bug"))
+
     return html.Div(
         children=[
             dcc.Location(id='url', refresh=False),
@@ -142,11 +156,7 @@ def get_static_layout(base_title=None, main_content=None, documentation_content=
 
                     # Tabs Section
                     dbc.Tabs(
-                        [
-                            dbc.Tab(main_content, label="Main", tab_id="main"),
-                            dbc.Tab(get_documentation_tab(documentation_content), label="Documentation", tab_id="documentation"),
-                            dbc.Tab(get_report_bug_tab(), label="Report a Bug", tab_id="report-bug"),
-                        ],
+                        tab_list,
                         id="tabs",
                         active_tab="main",
                     ),
@@ -250,10 +260,92 @@ def get_report_bug_tab():
                         "margin-left": "2vw",
                         "font-size": "20px",
                         "padding-right": "40px",
+                        "overflow-y": "scroll",
+                        "max-height": "65vh",
                     },
                 ),
                 width=9,
             ),
         ],
         style={"margin-top": "0px", "min-height": "40vh"},
+    )
+
+
+def get_workunits_tab():
+    """
+    Returns the content for the Workunits tab with the upgraded layout.
+    """
+    return dbc.Row(
+        id="page-content-workunits",
+        children=[
+            dbc.Col(
+                html.Div(
+                    id="sidebar_workunits",
+                    children=[],  # Optional: Add sidebar content here if needed
+                    style={
+                        "border-right": "2px solid #d4d7d9",
+                        "height": "100%",
+                        "padding": "20px",
+                        "font-size": "20px",
+                    },
+                ),
+                width=3,
+            ),
+            dbc.Col(
+                html.Div(
+                    id="page-content-workunits-children",
+                    children=[
+                        html.H2("Workunits"),
+                        html.Div(id="refresh-workunits", children=[]),
+                        html.Div(
+                            id="workunits-content"
+                        )
+                    ],
+                    style={
+                        "margin-top": "2vh",
+                        "margin-left": "2vw",
+                        "font-size": "20px",
+                        "padding-right": "40px",
+                        "overflow-y": "scroll",
+                        "max-height": "65vh",
+                    },
+                ),
+                width=9,
+            ),
+        ],
+        style={"margin-top": "0px", "min-height": "40vh"},
+    )
+
+
+def get_queue_tab(): 
+
+    return dbc.Row(
+        id="page-content-queue",
+        children=[
+            dbc.Col(
+                html.Div(
+                    id="page-content-queue-children",
+                    children=[],
+                    style={
+                        "margin-top": "2vh",
+                        "margin-left": "2vw",
+                        "font-size": "20px",
+                        "padding-right": "40px",
+                        "overflow-y": "scroll",
+                        "max-height": "65vh",
+                    },
+                ),
+            ),
+            dbc.Col(
+                children = [
+                    dcc.Interval(
+                        id="queue-interval",
+                        interval=5 * 1000,  # in milliseconds
+                        n_intervals=0,
+                    ),
+                ],
+                style={"display": "none"}
+            )
+        ],
+        style={"margin-top": "0px", "min-height": "40vh"},
     )

bfabric_web_apps/objects/BfabricInterface.py

@@ -9,12 +9,9 @@ from bfabric_web_apps.utils.get_logger import get_logger
 import os
 import bfabric_web_apps
 
-
-
 VALIDATION_URL = "https://fgcz-bfabric.uzh.ch/bfabric/rest/token/validate?token="
 HOST = "fgcz-bfabric.uzh.ch"
 
-
 class BfabricInterface( Bfabric ):
     _instance = None  # Singleton instance
     _wrapper = None   # Shared wrapper instance
@@ -50,8 +47,7 @@ class BfabricInterface( Bfabric ):
         if self._wrapper is None:
             raise RuntimeError("Bfabric wrapper is not initialized. Token validation must run first.")
         return self._wrapper
-
-
+    
 
     def token_to_data(self, token):
         """
@@ -96,8 +92,6 @@ class BfabricInterface( Bfabric ):
             
             environment_dict = {"Production":"https://fgcz-bfabric.uzh.ch/bfabric","Test":"https://fgcz-bfabric-test.uzh.ch/bfabric"}
             
-            print('userinfo', userinfo)
-
             token_data = dict(
                 environment = userinfo['environment'],
                 user_data = userinfo['user'],
@@ -267,7 +261,7 @@ class BfabricInterface( Bfabric ):
 
         # Extract App ID, Name, and Description
         app_info = app_data_dict[0]  # First (and only) result
-        print('app_info', app_info)
+
         json_data = json.dumps({
             "id": app_info.get("id", "Unknown"),
             "name": app_info.get("name", "Unknown"),
@@ -312,10 +306,9 @@ class BfabricInterface( Bfabric ):
         os.system(mail)
 
         return True
+    
 
-
-
-
+    
 # Create a globally accessible instance
 bfabric_interface = BfabricInterface()
 

bfabric_web_apps/utils/callbacks.py

@@ -1,9 +1,13 @@
 from dash import Input, Output, State, html, dcc
-from bfabric_web_apps.objects.BfabricInterface import BfabricInterface
+from bfabric_web_apps.objects.BfabricInterface import bfabric_interface
 import json
 import dash_bootstrap_components as dbc
 from datetime import datetime as dt
 from bfabric_web_apps.utils.get_logger import get_logger
+from rq import Queue
+from .redis_connection import redis_conn
+from rq.registry import StartedJobRegistry, FailedJobRegistry, FinishedJobRegistry
+
 
 def process_url_and_token(url_params):
     """
@@ -28,7 +32,6 @@ def process_url_and_token(url_params):
         return None, None, None, None, base_title, None, None
 
     token = "".join(url_params.split('token=')[1:])
-    bfabric_interface = BfabricInterface()
     tdata_raw = bfabric_interface.token_to_data(token)
 
     if tdata_raw:
@@ -88,7 +91,6 @@ def process_url_and_token(url_params):
         return None, None, None, None, base_title, None, None
 
 
-
 def submit_bug_report(n_clicks, bug_description, token, entity_data):
     """
     Submits a bug report based on user input, token, and entity data.
@@ -103,7 +105,7 @@ def submit_bug_report(n_clicks, bug_description, token, entity_data):
         tuple: A tuple containing two boolean values indicating success and failure status of the submission.
                (is_open_success, is_open_failure)
     """
-    bfabric_interface = BfabricInterface()
+
     print("submit bug report", token)
 
     # Parse token data if token is provided, otherwise set it to an empty dictionary
@@ -168,3 +170,169 @@ def submit_bug_report(n_clicks, bug_description, token, entity_data):
 
     return False, False
 
+
+def populate_workunit_details(token_data):
+
+    """
+    Function to populate workunit data for the current app instance.
+
+    Args: 
+        token_data (dict): Token metadata.
+
+    Returns:
+        html.Div: A div containing the populated workunit data.
+    """
+
+    environment_urls = {
+        "Test": "https://fgcz-bfabric-test.uzh.ch/bfabric/workunit/show.html?id=",
+        "Production": "https://fgcz-bfabric.uzh.ch/bfabric/workunit/show.html?id="
+    }
+
+    if token_data:
+
+        jobId = token_data.get('jobId', None)
+        print("jobId", jobId)
+        
+        job = bfabric_interface.get_wrapper().read("job", {"id": jobId})[0]
+        workunits = job.get("workunit", [])
+
+        if workunits:
+            wus = bfabric_interface.get_wrapper().read(
+                "workunit", 
+                {"id": [wu["id"] for wu in workunits]}
+            )
+        else:
+            return html.Div(
+                [
+                    html.P("No workunits found for the current job.")
+                ]
+            )
+
+        wu_cards = []
+
+        for wu in wus: 
+            print(wu)
+            wu_card = html.A(
+                dbc.Card([
+                    dbc.CardHeader(html.B(f"Workunit {wu['id']}")),
+                    dbc.CardBody([
+                        html.P(f"Name: {wu.get('name', 'n/a')}"),
+                        html.P(f"Description: {wu.get('description', 'n/a')}"),
+                        html.P(f"Num Resources: {len(wu.get('resource', []))}"),
+                        html.P(f"Created: {wu.get('created', 'n/a')}"),
+                        html.P(f"Status: {wu.get('status', 'n/a')}")
+                    ])
+                ], style={"width": "400px", "margin":"10px"}),
+                href=environment_urls[token_data.get("environment", "Test")] + str(wu["id"]),
+                target="_blank",
+                style={"text-decoration": "none"}
+            )
+
+            wu_cards.append(wu_card)
+
+        return dbc.Container(wu_cards, style={"display": "flex", "flex-wrap": "wrap"})
+    else:
+        return html.Div()
+
+def get_redis_queue_layout():
+    # Get all queues dynamically
+    queues = Queue.all(connection=redis_conn)
+
+    queue_cards = []
+
+    print("QUEUES", queues)
+    
+    for queue in queues:
+        queue_name = queue.name
+
+        # Get queue stats
+        started_registry = StartedJobRegistry(queue_name, connection=redis_conn)
+        failed_registry = FailedJobRegistry(queue_name, connection=redis_conn)
+        finished_registry = FinishedJobRegistry(queue_name, connection=redis_conn)
+
+        stats = {
+            "Jobs in queue": queue.count,
+            "Running": started_registry.count,
+            "Failed": failed_registry.count,
+            "Completed": finished_registry.count,
+        }
+
+        print("STAT", stats)
+
+        stats_row = dbc.Row([
+            dbc.Col([
+                html.P([html.B("Jobs in queue: "), f"{queue.count}"]),
+                html.P([html.B("Running: "), f"{started_registry.count}"]),
+            ],width=6),
+            dbc.Col([
+                html.P([html.B("Failed: "), f"{failed_registry.count}"]),
+                html.P([html.B("Completed: "), f"{finished_registry.count}"]),
+            ], width=6)
+        ])
+
+        # Fetch job details
+        job_cards = []
+        for job_id in started_registry.get_job_ids():
+            job = queue.fetch_job(job_id)
+            if job:
+                job_cards.append(
+                    dbc.Card(
+                        dbc.CardBody([
+                            html.H6(f"Job ID: {job.id}", className="card-title"),
+                            html.P(f"Function: {job.func_name}", className="card-text"),
+                            html.P(f"Status: Running", className="text-success"),
+                        ]),
+                        style={"maxWidth": "36vw", "backgroundColor": "#d4edda"}, className="mb-2"
+                    )
+                )
+
+        for job_id in failed_registry.get_job_ids():
+            job = queue.fetch_job(job_id)
+            if job:
+                job_cards.append(
+                    dbc.Card(
+                        dbc.CardBody([
+                            html.H6(f"Job ID: {job.id}", className="card-title"),
+                            html.P(f"Function: {job.func_name}", className="card-text"),
+                            html.P(f"Status: Failed", className="text-danger"),
+                        ]),
+                        style={"maxWidth": "36vw", "backgroundColor": "#f8d7da"}, className="mb-2"
+                    )
+                )
+
+        for job_id in finished_registry.get_job_ids():
+            job = queue.fetch_job(job_id)
+            if job:
+                finished_time = job.ended_at.strftime("%Y-%m-%d %H:%M:%S") if job.ended_at else "Unknown"
+                job_cards.append(
+                    dbc.Card(
+                        dbc.CardBody([
+                            html.H6(f"Job ID: {job.id}", className="card-title"),
+                            html.P(f"Function: {job.func_name}", className="card-text"),
+                            html.P(f"Status: Completed", className="text-primary"),
+                            html.P(f"Finished at: {finished_time}", className="text-muted"),
+                        ]),
+                        style={"maxWidth": "36vw", "backgroundColor": "#d1ecf1"}, className="mb-2"
+                    )
+                )
+
+        # Create queue card
+        queue_card = dbc.Col([
+            dbc.Card(
+                [
+                    dbc.CardHeader(html.H5(f"Queue: {queue_name}")),
+                    dbc.CardBody([
+                        stats_row,  # Fixed: Convert dictionary to list
+                        html.Hr(),
+                        *job_cards  # Add job sub-cards
+                    ], style={"maxHeight": "58vh", "overflow-y": "scroll"})
+                ],
+                style={"maxWidth": "36vw", "backgroundColor": "#f8f9fa", "max-height":"60vh"}, className="mb-4"
+            )
+        ])
+
+        queue_cards.append(queue_card)
+
+    container_children = dbc.Row(queue_cards)
+
+    return dbc.Container(container_children, className="mt-4")

bfabric_web_apps/utils/config.py

@@ -0,0 +1,38 @@
+from pydantic_settings import BaseSettings
+from pydantic import EmailStr
+
+class Settings(BaseSettings):
+
+    REDIS_HOST: str = "localhost"
+    REDIS_PORT: int = 6379
+
+    CONFIG_FILE_PATH: str = "~/.bfabricpy.yml"
+
+    HOST: str = "127.0.0.1"
+    PORT: int = 8050
+
+    DEV: bool = False
+    DEBUG: bool = False
+
+    DEVELOPER_EMAIL_ADDRESS: EmailStr = "griffin@gwcustom.com"
+    BUG_REPORT_EMAIL_ADDRESS: EmailStr = "gwtools@fgcz.system"
+
+    #Run main pipeline config (only FGCZ specific)
+    GSTORE_REMOTE_PATH: str = "/path/to/remote/gstore"
+    SCRATCH_PATH: str = "/scratch/folder"
+    TRX_LOGIN: str = "trxcopy@fgcz-server.uzh.ch" 
+    TRX_SSH_KEY: str = "/home/user/.ssh/your_ssh_key"
+    URL: str = "https:/fgcz/dummy/url"
+
+    class Config:
+
+        env_file = ".env"  
+
+        # Disable reading from environment variables
+        @classmethod
+        def customise_sources(cls, init_settings, env_settings, file_secret_settings):
+            return file_secret_settings, init_settings 
+
+# Instantiate settings
+settings = Settings()
+

bfabric_web_apps/utils/redis_connection.py

@@ -0,0 +1,6 @@
+
+from .config import settings as config
+
+from redis import Redis
+
+redis_conn = Redis(host=config.REDIS_HOST, port=config.REDIS_PORT)

bfabric_web_apps/utils/redis_queue.py

@@ -0,0 +1,6 @@
+from rq import Queue
+from .redis_connection import redis_conn as conn
+
+
+def q(queue_name):
+    return Queue(name=queue_name, connection=conn, default_timeout=10000000)

bfabric_web_apps/utils/redis_worker_init.py

@@ -0,0 +1,28 @@
+import redis 
+from rq import Worker, Queue, Connection
+import time
+
+def test_job(): 
+
+    """
+    A test job that prints a message to the console.
+    """
+    print("Hello, this is a test job!")
+    time.sleep(10) 
+    print("Test job finished!")
+    return
+
+
+def run_worker(host, port, queue_names): 
+    """
+    Provides internal interface for running workers on a specified host and port.
+
+    Args:
+        host (str): The host to run
+        port (int): The port to run
+        queue_names (list): A list of queue names to listen to
+    """
+    conn = redis.Redis(host=host, port=port) 
+    with Connection(conn): 
+        worker = Worker(map(Queue, queue_names)) 
+        worker.work()