bfabric-web-apps 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

bfabric_web_apps/__init__.py

@@ -19,42 +19,39 @@ from .utils.create_app_in_bfabric import create_app_in_bfabric
 from .utils.callbacks import (
     process_url_and_token, 
     submit_bug_report,
-    populate_workunit_details
+    populate_workunit_details,
+    get_redis_queue_layout
 )
 
-from .utils import defaults
-
-from bfabric_web_apps.utils.resource_utilities import create_workunit, create_resource, create_workunits, create_resources
-HOST = os.getenv("HOST", defaults.HOST)
-PORT = int(os.getenv("PORT", defaults.PORT))  # Convert to int since env variables are strings
-DEV = os.getenv("DEV", str(defaults.DEV)).lower() in ["true", "1", "yes"]  # Convert to bool
-CONFIG_FILE_PATH = os.getenv("CONFIG_FILE_PATH", defaults.CONFIG_FILE_PATH)
-
-DEVELOPER_EMAIL_ADDRESS = os.getenv("DEVELOPER_EMAIL_ADDRESS", defaults.DEVELOPER_EMAIL_ADDRESS)
-BUG_REPORT_EMAIL_ADDRESS = os.getenv("BUG_REPORT_EMAIL_ADDRESS", defaults.BUG_REPORT_EMAIL_ADDRESS)
-
-
-# Define __all__ for controlled imports
-__all__ = [
-    "BfabricInterface",
-    "Logger",
-    "components",
-    "get_static_layout",
-    "create_app",
-    "process_url_and_token",
-    "submit_bug_report",
-    'get_logger',
-    'get_power_user_wrapper',
-    'HOST',
-    'PORT', 
-    'DEV',
-    'CONFIG_FILE_PATH',
-    'DEVELOPER_EMAIL_ADDRESS',
-    'BUG_REPORT_EMAIL_ADDRESS',
-    'create_app_in_bfabric',
-    'create_workunit',
-    'create_resource',
-    'create_workunits',
-    'create_resources',
-    'populate_workunit_details',
-]
+from .utils.config import settings as config
+
+from. utils.run_main_pipeline import run_main_job
+
+from .utils.resource_utilities import (
+    create_workunit, 
+    create_resource, 
+    create_workunits, 
+    create_resources
+)
+
+from .utils.redis_worker_init import run_worker, test_job
+from .utils.redis_queue import q
+
+REDIS_HOST = config.REDIS_HOST
+REDIS_PORT = config.REDIS_PORT
+
+HOST = config.HOST
+PORT = config.PORT
+DEV = config.DEV
+DEBUG = config.DEBUG
+
+CONFIG_FILE_PATH = config.CONFIG_FILE_PATH
+
+DEVELOPER_EMAIL_ADDRESS = config.DEVELOPER_EMAIL_ADDRESS
+BUG_REPORT_EMAIL_ADDRESS = config.BUG_REPORT_EMAIL_ADDRESS
+
+GSTORE_REMOTE_PATH = config.GSTORE_REMOTE_PATH
+SCRATCH_PATH = config.SCRATCH_PATH
+TRX_LOGIN = config.TRX_LOGIN
+TRX_SSH_KEY = config.TRX_SSH_KEY
+URL = config.URL

bfabric_web_apps/layouts/layouts.py

@@ -2,7 +2,7 @@ from dash import html, dcc
 import dash_bootstrap_components as dbc
 import bfabric_web_apps
 
-def get_static_layout(base_title=None, main_content=None, documentation_content=None):
+def get_static_layout(base_title=None, main_content=None, documentation_content=None, layout_config={}):
     """
     Returns a layout with static tabs for Main, Documentation, and Report a Bug.
     The main content is customizable, while the other tabs are generic.
@@ -11,10 +11,24 @@ def get_static_layout(base_title=None, main_content=None, documentation_content=
         base_title (str): The main title to be displayed in the banner.
         main_content (html.Div): Content to be displayed in the "Main" tab.
         documentation_content (html.Div): Content for the "Documentation" tab.
+        layout_config (dict): Configuration for the layout, determining which tabs are shown.
 
     Returns:
         html.Div: The complete static layout of the web app.
     """
+
+    tab_list = [
+        dbc.Tab(main_content, label="Main", tab_id="main"),
+        dbc.Tab(dcc.Loading(get_documentation_tab(documentation_content)), label="Documentation", tab_id="documentation"),
+    ]
+
+    if layout_config.get("workunits", False):
+        tab_list.append(dbc.Tab(dcc.Loading(get_workunits_tab()), label="Workunits", tab_id="workunits"))
+    if layout_config.get("queue", False):
+        tab_list.append(dbc.Tab(get_queue_tab(), label="Queue", tab_id="queue"))
+    if layout_config.get("bug", False):
+        tab_list.append(dbc.Tab(dcc.Loading(get_report_bug_tab()), label="Report a Bug", tab_id="report-bug"))
+
     return html.Div(
         children=[
             dcc.Location(id='url', refresh=False),
@@ -142,12 +156,7 @@ def get_static_layout(base_title=None, main_content=None, documentation_content=
 
                     # Tabs Section
                     dbc.Tabs(
-                        [
-                            dbc.Tab(main_content, label="Main", tab_id="main"),
-                            dbc.Tab(dcc.Loading(get_documentation_tab(documentation_content)), label="Documentation", tab_id="documentation"),
-                            dbc.Tab(dcc.Loading(get_workunits_tab()), label="Workunits", tab_id="workunits"),
-                            dbc.Tab(dcc.Loading(get_report_bug_tab()), label="Report a Bug", tab_id="report-bug"),
-                        ],
+                        tab_list,
                         id="tabs",
                         active_tab="main",
                     ),
@@ -305,4 +314,38 @@ def get_workunits_tab():
             ),
         ],
         style={"margin-top": "0px", "min-height": "40vh"},
+    )
+
+
+def get_queue_tab(): 
+
+    return dbc.Row(
+        id="page-content-queue",
+        children=[
+            dbc.Col(
+                html.Div(
+                    id="page-content-queue-children",
+                    children=[],
+                    style={
+                        "margin-top": "2vh",
+                        "margin-left": "2vw",
+                        "font-size": "20px",
+                        "padding-right": "40px",
+                        "overflow-y": "scroll",
+                        "max-height": "65vh",
+                    },
+                ),
+            ),
+            dbc.Col(
+                children = [
+                    dcc.Interval(
+                        id="queue-interval",
+                        interval=5 * 1000,  # in milliseconds
+                        n_intervals=0,
+                    ),
+                ],
+                style={"display": "none"}
+            )
+        ],
+        style={"margin-top": "0px", "min-height": "40vh"},
     )

bfabric_web_apps/utils/callbacks.py

@@ -4,6 +4,10 @@ import json
 import dash_bootstrap_components as dbc
 from datetime import datetime as dt
 from bfabric_web_apps.utils.get_logger import get_logger
+from rq import Queue
+from .redis_connection import redis_conn
+from rq.registry import StartedJobRegistry, FailedJobRegistry, FinishedJobRegistry
+
 
 def process_url_and_token(url_params):
     """
@@ -181,7 +185,7 @@ def populate_workunit_details(token_data):
 
     environment_urls = {
         "Test": "https://fgcz-bfabric-test.uzh.ch/bfabric/workunit/show.html?id=",
-        "Prod": "https://fgcz-bfabric.uzh.ch/bfabric/workunit/show.html?id="
+        "Production": "https://fgcz-bfabric.uzh.ch/bfabric/workunit/show.html?id="
     }
 
     if token_data:
@@ -228,4 +232,107 @@ def populate_workunit_details(token_data):
 
         return dbc.Container(wu_cards, style={"display": "flex", "flex-wrap": "wrap"})
     else:
-        return html.Div()
+        return html.Div()
+
+def get_redis_queue_layout():
+    # Get all queues dynamically
+    queues = Queue.all(connection=redis_conn)
+
+    queue_cards = []
+
+    print("QUEUES", queues)
+    
+    for queue in queues:
+        queue_name = queue.name
+
+        # Get queue stats
+        started_registry = StartedJobRegistry(queue_name, connection=redis_conn)
+        failed_registry = FailedJobRegistry(queue_name, connection=redis_conn)
+        finished_registry = FinishedJobRegistry(queue_name, connection=redis_conn)
+
+        stats = {
+            "Jobs in queue": queue.count,
+            "Running": started_registry.count,
+            "Failed": failed_registry.count,
+            "Completed": finished_registry.count,
+        }
+
+        print("STAT", stats)
+
+        stats_row = dbc.Row([
+            dbc.Col([
+                html.P([html.B("Jobs in queue: "), f"{queue.count}"]),
+                html.P([html.B("Running: "), f"{started_registry.count}"]),
+            ],width=6),
+            dbc.Col([
+                html.P([html.B("Failed: "), f"{failed_registry.count}"]),
+                html.P([html.B("Completed: "), f"{finished_registry.count}"]),
+            ], width=6)
+        ])
+
+        # Fetch job details
+        job_cards = []
+        for job_id in started_registry.get_job_ids():
+            job = queue.fetch_job(job_id)
+            if job:
+                job_cards.append(
+                    dbc.Card(
+                        dbc.CardBody([
+                            html.H6(f"Job ID: {job.id}", className="card-title"),
+                            html.P(f"Function: {job.func_name}", className="card-text"),
+                            html.P(f"Status: Running", className="text-success"),
+                        ]),
+                        style={"maxWidth": "36vw", "backgroundColor": "#d4edda"}, className="mb-2"
+                    )
+                )
+
+        for job_id in failed_registry.get_job_ids():
+            job = queue.fetch_job(job_id)
+            if job:
+                job_cards.append(
+                    dbc.Card(
+                        dbc.CardBody([
+                            html.H6(f"Job ID: {job.id}", className="card-title"),
+                            html.P(f"Function: {job.func_name}", className="card-text"),
+                            html.P(f"Status: Failed", className="text-danger"),
+                        ]),
+                        style={"maxWidth": "36vw", "backgroundColor": "#f8d7da"}, className="mb-2"
+                    )
+                )
+
+        for job_id in finished_registry.get_job_ids():
+            job = queue.fetch_job(job_id)
+            if job:
+                finished_time = job.ended_at.strftime("%Y-%m-%d %H:%M:%S") if job.ended_at else "Unknown"
+                job_cards.append(
+                    dbc.Card(
+                        dbc.CardBody([
+                            html.H6(f"Job ID: {job.id}", className="card-title"),
+                            html.P(f"Function: {job.func_name}", className="card-text"),
+                            html.P(f"Status: Completed", className="text-primary"),
+                            html.P(f"Finished at: {finished_time}", className="text-muted"),
+                        ]),
+                        style={"maxWidth": "36vw", "backgroundColor": "#d1ecf1"}, className="mb-2"
+                    )
+                )
+
+        # Create queue card
+        queue_card = dbc.Col([
+            dbc.Card(
+                [
+                    dbc.CardHeader(html.H5(f"Queue: {queue_name}")),
+                    dbc.CardBody([
+                        stats_row,  # Fixed: Convert dictionary to list
+                        html.Hr(),
+                        *job_cards  # Add job sub-cards
+                    ], style={"maxHeight": "58vh", "overflow-y": "scroll"})
+                ],
+                style={"maxWidth": "36vw", "backgroundColor": "#f8f9fa", "max-height":"60vh"}, className="mb-4"
+            )
+        ])
+
+        queue_cards.append(queue_card)
+
+    container_children = dbc.Row(queue_cards)
+
+    return dbc.Container(container_children, className="mt-4")

bfabric_web_apps/utils/config.py

@@ -0,0 +1,38 @@
+from pydantic_settings import BaseSettings
+from pydantic import EmailStr
+
+class Settings(BaseSettings):
+
+    REDIS_HOST: str = "localhost"
+    REDIS_PORT: int = 6379
+
+    CONFIG_FILE_PATH: str = "~/.bfabricpy.yml"
+
+    HOST: str = "127.0.0.1"
+    PORT: int = 8050
+
+    DEV: bool = False
+    DEBUG: bool = False
+
+    DEVELOPER_EMAIL_ADDRESS: EmailStr = "griffin@gwcustom.com"
+    BUG_REPORT_EMAIL_ADDRESS: EmailStr = "gwtools@fgcz.system"
+
+    #Run main pipeline config (only FGCZ specific)
+    GSTORE_REMOTE_PATH: str = "/path/to/remote/gstore"
+    SCRATCH_PATH: str = "/scratch/folder"
+    TRX_LOGIN: str = "trxcopy@fgcz-server.uzh.ch" 
+    TRX_SSH_KEY: str = "/home/user/.ssh/your_ssh_key"
+    URL: str = "https:/fgcz/dummy/url"
+
+    class Config:
+
+        env_file = ".env"  
+
+        # Disable reading from environment variables
+        @classmethod
+        def customise_sources(cls, init_settings, env_settings, file_secret_settings):
+            return file_secret_settings, init_settings 
+
+# Instantiate settings
+settings = Settings()
+

bfabric_web_apps/utils/redis_connection.py

@@ -0,0 +1,6 @@
+
+from .config import settings as config
+
+from redis import Redis
+
+redis_conn = Redis(host=config.REDIS_HOST, port=config.REDIS_PORT)

bfabric_web_apps/utils/redis_queue.py

@@ -0,0 +1,6 @@
+from rq import Queue
+from .redis_connection import redis_conn as conn
+
+
+def q(queue_name):
+    return Queue(name=queue_name, connection=conn, default_timeout=10000000)

bfabric_web_apps/utils/redis_worker_init.py

@@ -0,0 +1,28 @@
+import redis 
+from rq import Worker, Queue, Connection
+import time
+
+def test_job(): 
+
+    """
+    A test job that prints a message to the console.
+    """
+    print("Hello, this is a test job!")
+    time.sleep(10) 
+    print("Test job finished!")
+    return
+
+
+def run_worker(host, port, queue_names): 
+    """
+    Provides internal interface for running workers on a specified host and port.
+
+    Args:
+        host (str): The host to run
+        port (int): The port to run
+        queue_names (list): A list of queue names to listen to
+    """
+    conn = redis.Redis(host=host, port=port) 
+    with Connection(conn): 
+        worker = Worker(map(Queue, queue_names)) 
+        worker.work()

bfabric_web_apps/utils/resource_utilities.py

@@ -16,7 +16,7 @@ def create_workunit(token_data, application_name, application_description, appli
         container_id (int): Container ID (Order ID).
     
     Returns:
-        int: Created workunit ID or None if creation fails.
+        obj: Created workunit object or None if creation fails.
     """
     L = get_logger(token_data)
     wrapper = bfabric_interface.get_wrapper()
@@ -50,7 +50,7 @@ def create_workunit(token_data, application_name, application_description, appli
             params=None,
             flush_logs=True
         )
-        return workunit_id
+        return workunit_response[0]
 
     except Exception as e:
         L.log_operation(
@@ -75,82 +75,95 @@ def create_workunits(token_data, application_name, application_description, appl
         container_ids (list): List of container IDs.
     
     Returns:
-        list: List of created workunit IDs.
+        list[obj]: List of created workunit objects.
     """
     if not isinstance(container_ids, list):
         container_ids = [container_ids]  # Ensure it's a list
 
-    workunit_ids = [
+    workunits = [
         create_workunit(token_data, application_name, application_description, application_id, container_id)
         for container_id in container_ids
     ]
 
-    return [wu_id for wu_id in workunit_ids if wu_id is not None]  # Filter out None values
+    return [wu_id for wu_id in workunits if wu_id is not None]  # Filter out None values
 
 
-def create_resource(token_data, workunit_id, gz_file_path):
+from pathlib import Path
+
+def create_resource(token_data, workunit_id, file_path, storage_id="20"): # GWC Server is storage id 20. 
     """
-    Upload a single .gz resource to an existing B-Fabric workunit.
+    Attach a single file as a resource to an existing B-Fabric workunit.
 
     Args:
         token_data (dict): Authentication token data.
         workunit_id (int): ID of the workunit to associate the resource with.
-        gz_file_path (str): Full path to the .gz file to upload.
+        file_path (str): Full path to the file to attach.
     
     Returns:
-        int: Resource ID if successful, None otherwise.
+        obj: Resource object if successful, None otherwise.
     """
     L = get_logger(token_data)
     wrapper = get_power_user_wrapper(token_data)
 
     try:
-        file_path = Path(gz_file_path)
-
-        # Upload the resource
-        print("Uploading:", file_path, "to workunit:", workunit_id)
-        result = bfabric_upload_resource(wrapper, file_path, workunit_id)
+        file_path = Path(file_path)
+        
+        # Attaching the resource
+        print(f"Attaching: {file_path.name} to workunit: {workunit_id}")
+        
+        result = wrapper.save(
+            endpoint="resource",
+            obj={
+                "workunitid": str(workunit_id),
+                "name": file_path.name,
+                "description": f"Resource attached to workunit {workunit_id}",
+                "relativepath": file_path.name,
+                "storageid": str(storage_id),
+            }
+        )
 
         if result:
-            print(f"Resource uploaded: {file_path.name}")
+            resource_id = result[0].get("id")
+            print(f"Resource attached: {file_path.name} (ID: {resource_id})")
             L.log_operation(
-                "upload_resource",
-                f"Resource uploaded successfully: {file_path.name}",
+                "Attach_resource",
+                f"Resource attached successfully: {file_path.name}",
                 params=None,
                 flush_logs=True,
             )
-            return result
+            return result[0]
         else:
-            raise ValueError(f"Failed to upload resource: {file_path.name}")
+            raise ValueError(f"Failed to attach resource: {file_path.name}")
 
     except Exception as e:
         L.log_operation(
             "error",
-            f"Failed to upload resource: {e}",
+            f"Failed to attach resource: {e}",
             params=None,
             flush_logs=True,
         )
-        print(f"Failed to upload resource: {e}")
+        print(f"Failed to attach resource: {e}")
         return None
 
 
-def create_resources(token_data, workunit_id, gz_file_paths):
+def create_resources(token_data, workunit_id, file_paths):
     """
-    Upload multiple .gz resources to an existing B-Fabric workunit.
+    Attach multiple files as resources to an existing B-Fabric workunit.
 
     Args:
         token_data (dict): Authentication token data.
         workunit_id (int): ID of the workunit to associate the resources with.
-        gz_file_paths (list): List of full paths to .gz files to upload.
+        file_paths (list): List of full paths to files to attach.
     
     Returns:
-        list: List of successfully uploaded resource IDs.
+        list[obj]: List of successfully attached resource objects.
     """
-    if not isinstance(gz_file_paths, list):
-        gz_file_paths = [gz_file_paths]  # Ensure it's a list
+    if not isinstance(file_paths, list):
+        file_paths = [file_paths]  # Ensure it's a list
 
-    resource_ids = [
-        create_resource(token_data, workunit_id, gz_file_path)
-        for gz_file_path in gz_file_paths
+    resources = [
+        create_resource(token_data, workunit_id, file_path)
+        for file_path in file_paths
     ]
 
-    return [res_id for res_id in resource_ids if res_id is not None]  # Filter out None values
+    return [res_id for res_id in resources if res_id is not None]  # Filter out None values

bfabric_web_apps/utils/run_main_pipeline.py

@@ -0,0 +1,414 @@
+import redis
+from rq import Queue
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from .get_logger import get_logger
+from .get_power_user_wrapper import get_power_user_wrapper
+from .callbacks import process_url_and_token
+from bfabric_web_apps.objects import BfabricInterface
+from .resource_utilities import (
+    create_workunit, 
+    create_resource, 
+    create_workunits, 
+    create_resources
+)
+
+from .config import settings as config
+from datetime import datetime as dt
+
+GSTORE_REMOTE_PATH = config.GSTORE_REMOTE_PATH
+SCRATCH_PATH = config.SCRATCH_PATH
+TRX_LOGIN = config.TRX_LOGIN
+TRX_SSH_KEY = config.TRX_SSH_KEY
+URL = config.URL
+
+def run_main_job(files_as_byte_strings: dict, 
+                 bash_commands: list[str], 
+                 resource_paths: dict,
+                 attachment_paths: list[dict], 
+                 token: str):
+    """
+    Main function to handle:
+      1) Save Files on Server
+      2) Execute local bash commands
+      3) Create workunits in B-Fabric
+      4) Register resources in B-Fabric
+      5) Attach additional gstore files (logs/reports/etc.) to entities in B-Fabric
+
+    :param files_as_byte_strings: {destination_path: file as byte strings}
+    :param bash_commands: List of bash commands to execute
+    :param resource_paths: dict, {resource_path: container_id}
+    :param attachment_paths: Dictionary mapping source file paths to their corresponding file names ({"path/test.txt": "name.txt"})
+                             for attachment to a B-Fabric entity (e.g., logs, final reports, etc.)
+    :param token: Authentication token
+
+    
+Dev Notes:
+    !!! All exceptions get logged (make sure to log the exception message i.e. "except Exception as e: log(e)") !!!
+    !!! If an exception doesn't occur, log that some step ran successfully to the job object !!!
+    """
+
+    # STEP 0: Parse token, logger, etc.
+    token, token_data, entity_data, app_data, page_title, session_details, job_link = process_url_and_token(token)
+
+    if token is None:
+        raise ValueError("Error: 'token' is None")
+    if token_data is None:
+        raise ValueError("Error: 'token_data' is None")
+    if entity_data is None:
+        raise ValueError("Error: 'entity_data' is None")
+    if app_data is None:
+        raise ValueError("Error: 'app_data' is None")
+    
+    
+    L = get_logger(token_data)
+    print("Token Data:", token_data)
+    print("Entity Data:", entity_data)
+    print("App Data:", app_data)
+     
+
+    # Step 1: Save files to the server
+    try:
+        summary = save_files_from_bytes(files_as_byte_strings, L)
+        L.log_operation("Success", f"File copy summary: {summary}", params=None, flush_logs=True)
+        print("Summary:", summary)
+    except Exception as e:
+        # If something unexpected blows up the entire process
+        L.log_operation("Error", f"Failed to copy files: {e}", params=None, flush_logs=True)
+        print("Error copying files:", e)
+
+    
+    # STEP 2: Execute bash commands
+    try:
+        bash_log = execute_and_log_bash_commands(bash_commands, L)
+        L.log_operation("Success", f"Bash commands executed successfully:\n{bash_log}", 
+                        params=None, flush_logs=True)
+    except Exception as e:
+        L.log_operation("Error", f"Failed to execute bash commands: {e}", 
+                        params=None, flush_logs=True)
+        print("Error executing bash commands:", e)
+
+
+    # STEP 3: Create Workunits
+    try:
+        workunit_map = create_workunits_step(token_data, app_data, resource_paths, L)
+    except Exception as e:
+        L.log_operation("Error", f"Failed to create workunits in B-Fabric: {e}", 
+                        params=None, flush_logs=True)
+        print("Error creating workunits:", e)
+        workunit_map = []
+
+    # STEP 4: Register Resources (Refactored)
+    try:
+        attach_resources_to_workunits(token_data, L, workunit_map)
+    except Exception as e:
+        L.log_operation("Error", f"Failed to register resources: {e}", params=None, flush_logs=True)
+        print("Error registering resources:", e)
+
+    # STEP 5: Attach gstore files (logs, reports, etc.) to B-Fabric entity as a Link
+    try:
+        attach_gstore_files_to_entities_as_link(token_data, L, attachment_paths)
+        print("Attachment Paths:", attachment_paths)
+    except Exception as e:
+        L.log_operation("Error", f"Failed to attach extra files: {e}", params=None, flush_logs=True)
+        print("Error attaching extra files:", e)
+
+
+
+#---------------------------------------------------------------------------------------------------------------------
+#---------------------------------------------------------------------------------------------------------------------
+
+
+# -----------------------------------------------------------------------------
+# Step 1: Save Files from bytes
+# -----------------------------------------------------------------------------
+
+import os
+
+def save_files_from_bytes(files_as_byte_strings: dict, logger):
+    """
+    Saves byte string files to their respective paths.
+
+    :param files_as_byte_strings: Dictionary where keys are destination paths and values are byte strings
+    :param logger: Logging instance
+    :return: Summary indicating how many files succeeded vs. failed
+    """
+    results = {}  # Store results: (destination) -> True (if success) or error message (if failure)
+
+    # First pass: attempt to write all files
+    for destination, file_bytes in files_as_byte_strings.items():
+        try:
+            # Ensure the directory exists
+            os.makedirs(os.path.dirname(destination), exist_ok=True)
+
+            # Write file from byte string
+            with open(destination, "wb") as f:
+                f.write(file_bytes)
+            logger.log_operation("Files saved", "All files saved successfully.", params=None, flush_logs=True)
+            return "All files saved successfully."
+        
+        except Exception as e:
+            error_msg = f"Error saving file: {destination}, Error: {str(e)}"
+            logger.log_operation("Error", error_msg, params=None, flush_logs=True)
+            print(error_msg)
+            raise RuntimeError(error_msg)
+
+
+# -----------------------------------------------------------------------------
+# Step 2: Execute Bash Commands
+# -----------------------------------------------------------------------------
+
+def execute_and_log_bash_commands(bash_commands: list[str], logger):
+    """
+    Executes a list of bash commands locally, logs and returns the output.
+
+    :param bash_commands: List of commands to execute
+    :param logger: Logging instance
+    :return: A single string containing logs for all commands
+    """
+    logstring = ""
+
+    for cmd in bash_commands:
+        logstring += "---------------------------------------------------------\n"
+        logstring += f"Executing Command: {cmd}\n"
+
+        try:
+            # Execute the command and capture both stdout and stderr
+            result = subprocess.run(cmd, shell=True, text=True, capture_output=True)
+            output = result.stdout.strip()
+            error_output = result.stderr.strip()
+
+            # Check if command executed successfully
+            if result.returncode == 0:
+                status = "SUCCESS"
+                log_entry = f"Command: {cmd}\nStatus: {status}\nOutput:\n{output}\n"
+                logger.log_operation("Info", log_entry, params=None, flush_logs=True)
+            else:
+                status = "FAILURE"
+                log_entry = f"Command: {cmd}\nStatus: {status}\nError Output:\n{error_output}\n"
+                logger.log_operation("Error", log_entry, params=None, flush_logs=True)
+
+            logstring += log_entry
+            print(log_entry)
+
+        except Exception as e:
+            logstring += f"Command: {cmd}\nStatus: ERROR\nException: {str(e)}\n"
+            logger.log_operation("Error", f"Command: {cmd} failed with Exception: {str(e)}", 
+                                 params=None, flush_logs=True)
+    
+    return logstring
+
+
+# -----------------------------------------------------------------------------
+# Step 3: Create Workunits in B-Fabric
+# -----------------------------------------------------------------------------
+
+def create_workunits_step(token_data, app_data, resource_paths, logger):
+    """
+    Creates multiple workunits in B-Fabric based on unique order IDs found in resource_paths.
+
+    :param token_data: dict with token/auth info
+    :param app_data: dict with fields like {"id": <app_id>} or other app info
+    :param resource_paths: Dictionary {file_path: container_id}
+    :param logger: a logger instance
+    :return: A dictionary mapping file_paths to workunit objects {file_path: workunit}
+    """
+    app_id = app_data["id"]  # Extract the application ID
+
+    # Extract unique order IDs from resource_paths
+    container_ids = list(set(resource_paths.values()))
+
+    if not container_ids:
+        raise ValueError("No order IDs found in resource_paths; cannot create workunits.")
+
+    # Create all workunits in one API call
+    created_workunits = create_workunits(
+        token_data=token_data,
+        application_name="Test Workunit",
+        application_description="Workunits for batch processing",
+        application_id=app_id,
+        container_ids=container_ids
+    )
+
+    if not created_workunits or len(created_workunits) != len(container_ids):
+        raise ValueError(f"Mismatch in workunit creation: Expected {len(container_ids)} workunits, got {len(created_workunits)}.")
+
+    workunit_map = {
+        file_path: wu["id"]
+        for wu in created_workunits
+        for file_path, container_id in resource_paths.items()
+        if container_id == wu["container"]["id"]
+    }
+
+    logger.log_operation("Success", f"Total created Workunits: {list(workunit_map.values())}", params=None, flush_logs=True)
+    print(f"Total created Workunits: {list(workunit_map.values())}")
+
+    print(workunit_map)
+    return workunit_map  # Returning {file_path: workunit}
+
+
+
+# -----------------------------------------------------------------------------
+# Step 4: Attach Resources in B-Fabric
+# -----------------------------------------------------------------------------
+
+def attach_resources_to_workunits(token_data, logger, workunit_map):
+    """
+    Attaches each file to its corresponding workunit.
+
+    Uses `create_resource` to upload files one by one.
+
+    :param token_data: B-Fabric token data
+    :param logger: Logger instance
+    :param workunit_map: Dictionary mapping file_path to workunit_id {file_path: workunit_id}
+    """
+    if not workunit_map:
+        logger.log_operation("Info", "No workunits found, skipping resource registration.",
+                             params=None, flush_logs=True)
+        print("No workunits found, skipping resource registration.")
+        return
+    
+    print("Workunit Map:", workunit_map)
+
+    for file_path, workunit_id in workunit_map.items():
+        print(f"Processing file: {file_path}, Workunit ID: {workunit_id}")  # Corrected print statement
+        # Upload the file as a resource
+        resource = create_resource(token_data, workunit_id, file_path)
+        resource_id = resource.get("id")
+        print("Resource ID:", resource_id)
+
+        if resource_id:
+            logger.log_operation("Success", f"Resource {resource_id} attached to Workunit {workunit_id}",
+                                 params=None, flush_logs=True)
+            print(f"Resource {resource_id} attached to Workunit {workunit_id}")
+        else:
+            logger.log_operation("Error", f"Failed to attach resource {file_path} for Workunit {workunit_id}",
+                                 params=None, flush_logs=True)
+            print(f"Failed to attach resource {file_path} for Workunit {workunit_id}")
+
+
+
+# -----------------------------------------------------------------------------
+# Step 5: Attachments of gstore in B-Fabric as a Link
+# -----------------------------------------------------------------------------
+
+def attach_gstore_files_to_entities_as_link(token_data, logger, attachment_paths: dict):
+    
+
+    """
+    Attaches files to a B-Fabric entity by copying them to the FGCZ storage and creating an API link.
+
+    Args:
+        token_data (dict): Authentication token data.
+        logger: Logger instance for logging operations.
+        attachment_paths (dict): Dictionary mapping source file paths to their corresponding file names.
+    
+    Returns:
+        None
+    """
+
+    # Extract entity details from token data
+    entity_class = token_data.get("entityClass_data", None)
+    entity_id = token_data.get("entity_id_data", None)
+
+    # Check if we have access to the FGCZ server
+    local = local_access(GSTORE_REMOTE_PATH)
+
+    # Process each attachment
+    for source_path, file_name in attachment_paths.items():
+        if not source_path or not file_name:
+            logger.log_operation("Error", f"Missing required attachment details: {source_path} -> {file_name}", params=None, flush_logs=True)
+            print(f"Error: Missing required attachment details: {source_path} -> {file_name}")
+            continue
+
+        try:
+            # Define entity folder
+            entity_folder = f"{entity_class}_{entity_id}" if entity_class and entity_id else "unknown_entity"
+            final_remote_path = f"{GSTORE_REMOTE_PATH}/{entity_folder}/"
+
+            print("local access:", local)
+            print("source path:", source_path)
+            print("file name:", file_name)
+            print("final remote path:", final_remote_path)
+
+            if local:  # We have direct access → Copy directly
+                g_req_copy(source_path, final_remote_path)
+                
+            else:  # We don't have direct access → Send to migration folder first
+                remote_tmp_path = f"{SCRATCH_PATH}/{file_name}"
+                scp_copy(source_path, TRX_LOGIN, TRX_SSH_KEY, remote_tmp_path)
+
+                # Move to final location
+                ssh_move(TRX_LOGIN, TRX_SSH_KEY, remote_tmp_path, final_remote_path)
+
+            # Log success
+            success_msg = f"Successfully attached '{file_name}' to {entity_class} (ID={entity_id})"
+            logger.log_operation("Success", success_msg, params=None, flush_logs=True)
+            print(success_msg)
+
+            # Step 3: Create API link
+            create_api_link(token_data, logger, entity_class, entity_id, file_name, entity_folder)
+
+        except Exception as e:
+            error_msg = f"Exception while processing '{file_name}': {e}"
+            logger.log_operation("Error", error_msg, params=None, flush_logs=True)
+            print(error_msg)
+
+def local_access(remote_path):
+    """Checks if the remote gstore path (i.e. /srv/gstore/projects/) exists locally""" 
+    result = os.path.exists(remote_path)
+    print("Remote Path Exists:", result)
+    return result
+
+
+def scp_copy(source_path, ssh_user, ssh_key, remote_path):
+    """Copies a file to a remote location using SCP with the correct FGCZ server address."""
+    cmd = ["scp", "-i", ssh_key, source_path, f"{ssh_user}:{remote_path}"]
+    subprocess.run(cmd, check=True)
+    print(f"Copied {source_path} to {remote_path}")
+
+
+def ssh_move(ssh_user, ssh_key, remote_tmp_path, final_remote_path):
+    """Moves a file on the remote server to its final location using SSH."""
+    cmd = ["ssh", "-i", ssh_key, ssh_user, f"/usr/local/ngseq/bin/g-req copynow -f {remote_tmp_path} {final_remote_path}"]
+
+    subprocess.run(cmd, check=True)
+    print(f"Moved {remote_tmp_path} to {final_remote_path}")
+
+
+def g_req_copy(source_path, destination_path):
+    """Copies a file using g-req command when direct access is available."""
+    cmd = ["/usr/local/ngseq/bin/g-req", "copynow", "-f", source_path, destination_path]
+    subprocess.run(cmd, check=True)
+    print(f"Copied {source_path} using g-req")
+
+
+def create_api_link(token_data, logger, entity_class, entity_id, file_name, folder_name):
+    """Creates an API link in B-Fabric for the attached file."""
+    wrapper = get_power_user_wrapper(token_data)
+    url = f"{URL}/{folder_name}/{file_name}"
+    timestamped_filename = f"{dt.now().strftime('%Y-%m-%d_%H:%M:%S')}_{file_name}"
+
+    data = {
+        "name": timestamped_filename,
+        "parentclassname": entity_class,
+        "parentid": entity_id,
+        "url": url
+    }
+
+    try:
+        link_result = wrapper.save("link", data)
+        if link_result:
+            success_msg = f"API link created for '{file_name}': {url}"
+            logger.log_operation("Success", success_msg, params=None, flush_logs=True)
+            print(success_msg)
+        else:
+            raise ValueError("API link creation failed")
+    except Exception as e:
+        error_msg = f"Failed to create API link for '{file_name}': {e}"
+        logger.log_operation("Error", error_msg, params=None, flush_logs=True)
+        print(error_msg)
+

{bfabric_web_apps-0.1.4.dist-info → bfabric_web_apps-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bfabric-web-apps
-Version: 0.1.4
+Version: 0.1.5
 Summary: A package containing handy boilerplate utilities for developing bfabric web-applications
 Author: Marc Zuber, Griffin White, GWC GmbH
 Requires-Python: >=3.8,<4.0
@@ -9,3 +9,6 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0)
+Requires-Dist: pydantic[email] (>=2.10.6,<3.0.0)
+Requires-Dist: rq (==1.15.1)

bfabric_web_apps-0.1.5.dist-info/RECORD

@@ -0,0 +1,20 @@
+bfabric_web_apps/__init__.py,sha256=D2jTCEYedVnp6yQxNGLRsjihupDTQxNkPVFQAUYXQys,1409
+bfabric_web_apps/layouts/layouts.py,sha256=z8gL4n4wwLdpLGomO9CftBLnGpc3r6OpmUc2-wBg8uo,14661
+bfabric_web_apps/objects/BfabricInterface.py,sha256=wmcL9JuSC0QEopgImvkZxmtCIS7izt6bwb6y_ch0zus,10178
+bfabric_web_apps/objects/Logger.py,sha256=62LC94xhm7YG5LUw3yH46NqvJQsAX7wnc9D4zbY16rA,5224
+bfabric_web_apps/utils/app_init.py,sha256=RCdpCXp19cF74bouYJLPe-KSETZ0Vwqtd02Ta2VXEF8,428
+bfabric_web_apps/utils/callbacks.py,sha256=m5d6IPiYX77-kJN8I2OptZN-GPxZgrI76o1DGFxjpPU,12686
+bfabric_web_apps/utils/components.py,sha256=V7ECGmF2XYy5O9ciDJVH1nofJYP2a_ELQF3z3X_ADbo,844
+bfabric_web_apps/utils/config.py,sha256=i93fe49Ak4Z7cm_G80m2cBCPp-5qCYLAJEtEr-mYSwQ,1044
+bfabric_web_apps/utils/create_app_in_bfabric.py,sha256=eVk3cQDXxW-yo9b9n_zzGO6kLg_SLxYbIDECyvEPJXU,2752
+bfabric_web_apps/utils/get_logger.py,sha256=0Y3SrXW93--eglS0_ZOc34NOriAt6buFPik5n0ltzRA,434
+bfabric_web_apps/utils/get_power_user_wrapper.py,sha256=T33z64XjmJ0KSlmfEmrEP8eYpbpINCVD6Xld_V7PR2g,1027
+bfabric_web_apps/utils/redis_connection.py,sha256=qXSPxW6m55Ogv44BhmPCl9ACuvzmpfZNU73UJhHRXL4,133
+bfabric_web_apps/utils/redis_queue.py,sha256=MCx7z_I2NusJ4P42mcLvV7STtXBFMIIvun83fM8zOGI,168
+bfabric_web_apps/utils/redis_worker_init.py,sha256=9SUc9bbgBeMbUdqJD9EkWPA4wcJjvyX6Tzanv5JfqEg,691
+bfabric_web_apps/utils/resource_utilities.py,sha256=cJTak0sXAiMSQ7VwJ4ImDUCmW8tAKGObBZCSr5uARBg,5931
+bfabric_web_apps/utils/run_main_pipeline.py,sha256=1YSbk3uP_T3tL6mZZXGv7a7FJc8exro_Eb49gnJjdrs,16864
+bfabric_web_apps-0.1.5.dist-info/LICENSE,sha256=k0O_i2k13i9e35aO-j7FerJafAqzzu8x0kkBs0OWF3c,1065
+bfabric_web_apps-0.1.5.dist-info/METADATA,sha256=Cg8oOpeNHTs2EpD_CLp5LoKmomHml-ermak1AzF14tA,608
+bfabric_web_apps-0.1.5.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+bfabric_web_apps-0.1.5.dist-info/RECORD,,

bfabric_web_apps/utils/defaults.py

@@ -1,11 +0,0 @@
-# defaults.py
-CONFIG_FILE_PATH = "~/.bfabricpy.yml"
-
-# Default values for application settings
-HOST = "0.0.0.0"
-PORT = 8050
-DEV = False
-
-# Developer and bug report email addresses
-DEVELOPER_EMAIL_ADDRESS = "griffin@gwcustom.com"
-BUG_REPORT_EMAIL_ADDRESS = "gwtools@fgcz.system"

bfabric_web_apps-0.1.4.dist-info/RECORD

@@ -1,16 +0,0 @@
-bfabric_web_apps/__init__.py,sha256=jU5o22wl7kXHNJVCH6aqW0llZLfxeQssCIeX1OerQfI,1790
-bfabric_web_apps/layouts/layouts.py,sha256=fmv_QTJeAmiOxreAwx14QojzyRV_8RHu1c4sCPN5r5U,13382
-bfabric_web_apps/objects/BfabricInterface.py,sha256=wmcL9JuSC0QEopgImvkZxmtCIS7izt6bwb6y_ch0zus,10178
-bfabric_web_apps/objects/Logger.py,sha256=62LC94xhm7YG5LUw3yH46NqvJQsAX7wnc9D4zbY16rA,5224
-bfabric_web_apps/utils/app_init.py,sha256=RCdpCXp19cF74bouYJLPe-KSETZ0Vwqtd02Ta2VXEF8,428
-bfabric_web_apps/utils/callbacks.py,sha256=XbRMK2sL55twtR6IWGAf5B1m2fnMTOpkhyR55-76nes,8444
-bfabric_web_apps/utils/components.py,sha256=V7ECGmF2XYy5O9ciDJVH1nofJYP2a_ELQF3z3X_ADbo,844
-bfabric_web_apps/utils/create_app_in_bfabric.py,sha256=eVk3cQDXxW-yo9b9n_zzGO6kLg_SLxYbIDECyvEPJXU,2752
-bfabric_web_apps/utils/defaults.py,sha256=B82j3JEbysLEU9JDZgoDBTX7WGvW3Hn5YMZaWAcjZew,278
-bfabric_web_apps/utils/get_logger.py,sha256=0Y3SrXW93--eglS0_ZOc34NOriAt6buFPik5n0ltzRA,434
-bfabric_web_apps/utils/get_power_user_wrapper.py,sha256=T33z64XjmJ0KSlmfEmrEP8eYpbpINCVD6Xld_V7PR2g,1027
-bfabric_web_apps/utils/resource_utilities.py,sha256=q0gC_Lr5GQlMBU0_gLm48zjq3XlXbT4QArqzJcmxrTo,5476
-bfabric_web_apps-0.1.4.dist-info/LICENSE,sha256=k0O_i2k13i9e35aO-j7FerJafAqzzu8x0kkBs0OWF3c,1065
-bfabric_web_apps-0.1.4.dist-info/METADATA,sha256=5TKRMRQB4an34gV7b4_hValbQEvuLvOmJpB9DsKBFR4,480
-bfabric_web_apps-0.1.4.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-bfabric_web_apps-0.1.4.dist-info/RECORD,,