bfabric-web-apps 0.1.6__py3-none-any.whl → 0.2.0__py3-none-any.whl

bfabric_web_apps/__init__.py

@@ -2,6 +2,7 @@ import os
 
 # Export objects and classes
 from bfabric_web_apps.objects import BfabricInterface, Logger
+from bfabric_web_apps.objects.BfabricInterface import bfabric_interface
 
 # Export components
 from .utils import components
@@ -14,6 +15,10 @@ from .utils.app_init import create_app
 from .utils.get_logger import get_logger
 from .utils.get_power_user_wrapper import get_power_user_wrapper
 from .utils.create_app_in_bfabric import create_app_in_bfabric
+from .utils.dataset_utils import (
+    dataset_to_dictionary, 
+    dictionary_to_dataset
+)
 
 # Export callbacks
 from .utils.callbacks import (
@@ -25,7 +30,7 @@ from .utils.callbacks import (
 
 from .utils.config import settings as config
 
-from. utils.run_main_pipeline import run_main_job
+from. utils.run_main_pipeline import run_main_job, read_file_as_bytes
 
 from .utils.resource_utilities import (
     create_workunit, 
@@ -34,6 +39,7 @@ from .utils.resource_utilities import (
     create_resources
 )
 
+from .utils.charging import create_charge
 from .utils.redis_worker_init import run_worker, test_job
 from .utils.redis_queue import q
 
@@ -54,4 +60,7 @@ GSTORE_REMOTE_PATH = config.GSTORE_REMOTE_PATH
 SCRATCH_PATH = config.SCRATCH_PATH
 TRX_LOGIN = config.TRX_LOGIN
 TRX_SSH_KEY = config.TRX_SSH_KEY
-URL = config.URL
+URL = config.URL
+
+SERVICE_ID = config.SERVICE_ID
+DATASET_TEMPLATE_ID = config.DATASET_TEMPLATE_ID

bfabric_web_apps/objects/BfabricInterface.py

@@ -67,7 +67,7 @@ class BfabricInterface( Bfabric ):
 
         validation_url = VALIDATION_URL + token
         res = requests.get(validation_url, headers={"Host": HOST})
-        
+
         if res.status_code != 200:
             res = requests.get(validation_url)
         
@@ -83,11 +83,9 @@ class BfabricInterface( Bfabric ):
             userinfo = json.loads(res.text)
             expiry_time = userinfo['expiryDateTime']
             current_time = datetime.datetime.now()
-            five_minutes_later = current_time + datetime.timedelta(minutes=5)
-
             # Comparing the parsed expiry time with the five minutes later time
 
-            if not five_minutes_later <= datetime.datetime.strptime(expiry_time, "%Y-%m-%d %H:%M:%S"):
+            if current_time > datetime.datetime.strptime(expiry_time, "%Y-%m-%d %H:%M:%S") + datetime.timedelta(days=7):
                 return "EXPIRED"
             
             environment_dict = {"Production":"https://fgcz-bfabric.uzh.ch/bfabric","Test":"https://fgcz-bfabric-test.uzh.ch/bfabric"}
@@ -104,10 +102,18 @@ class BfabricInterface( Bfabric ):
                 userWsPassword = userinfo['userWsPassword'],
                 jobId = userinfo['jobId']
             )
-
             # Initialize the wrapper right after validating the token
             self._initialize_wrapper(token_data)
 
+            # Log the token validation process
+            L = get_logger(token_data)
+            L.log_operation(
+                    operation="Authentication Process",
+                    message=f"Token validated successfully. User {token_data.get('user_data')} authenticated.",
+                    params=None,
+                    flush_logs=True
+                )
+
             return json.dumps(token_data)
         
 
@@ -152,7 +158,10 @@ class BfabricInterface( Bfabric ):
             "Project": "container",
             "Order": "container",
             "Container": "container",
-            "Plate": "plate"
+            "Plate": "plate",
+            "Workunit": "workunit",
+            "Resource": "resource",
+            "Dataset": "dataset"
         }
 
         if not token_data:
@@ -176,7 +185,7 @@ class BfabricInterface( Bfabric ):
                 obj={"id": entity_id},
                 max_results=None,
                 params=None,
-                flush_logs=True
+                flush_logs=False
             )[0]
 
             
@@ -186,6 +195,7 @@ class BfabricInterface( Bfabric ):
                     "createdby": entity_data_dict.get("createdby"),
                     "created": entity_data_dict.get("created"),
                     "modified": entity_data_dict.get("modified"),
+                    "full_api_response": entity_data_dict,
                 })
                 return json_data
             else:
@@ -246,7 +256,7 @@ class BfabricInterface( Bfabric ):
             obj={"id": app_id},  # Query using the App ID
             max_results=None,
             params=None,
-            flush_logs=True
+            flush_logs=False
         )
 
         # If API call fails, return empty JSON

bfabric_web_apps/utils/callbacks.py

@@ -335,4 +335,4 @@ def get_redis_queue_layout():
 
     container_children = dbc.Row(queue_cards)
 
-    return dbc.Container(container_children, className="mt-4")
+    return dbc.Container(container_children, className="mt-4")

bfabric_web_apps/utils/charging.py

@@ -0,0 +1,40 @@
+
+from bfabric_web_apps.utils.get_logger import get_logger
+from bfabric_web_apps.utils.get_power_user_wrapper import get_power_user_wrapper
+
+def create_charge(token_data, container_id, service_id):
+    """
+    Create a charge in B-Fabric.
+    
+    Args:
+        token_data (dict): Authentication token data.
+        container_id (int): Container ID (Order ID).
+        service_id (int): Service ID.
+    
+    Returns:
+        list[dict]: List of charge data.
+    """
+    
+    # Get a logger and an api wrapper
+    L = get_logger(token_data)
+    wrapper = get_power_user_wrapper(token_data)
+
+    # Get the user ID from the token data to assign a charger
+    usr_id = wrapper.read("user", {"login": token_data.get("user_data")})[0]['id']
+
+    charge_data = {
+        "serviceid": service_id,
+        "containerid": container_id,
+        "chargerid": usr_id
+    }
+
+    # Create and log the charge
+    charge = L.logthis(
+        api_call=wrapper.save,
+        endpoint="charge",
+        obj=charge_data,
+        params=None,
+        flush_logs=True
+    )
+
+    return charge

bfabric_web_apps/utils/components.py

@@ -1,4 +1,5 @@
 from dash import html
+import dash_daq as daq
 
 DEVELOPER_EMAIL = "gwhite@fgcz.ethz.ch"
 
@@ -19,4 +20,9 @@ auth = [html.Div(id="auth-div")]
 no_auth = [
     html.P("You are not currently logged into an active session. Please log into bfabric to continue:"),
     html.A('Login to Bfabric', href='https://fgcz-bfabric.uzh.ch/bfabric/')
-]
+]
+
+charge_switch = [
+    daq.BooleanSwitch(id='charge_run', on=True, label="Charge project for run"),
+    html.Br()
+]

bfabric_web_apps/utils/config.py

@@ -24,6 +24,12 @@ class Settings(BaseSettings):
     TRX_SSH_KEY: str = "/home/user/.ssh/your_ssh_key"
     URL: str = "https:/fgcz/dummy/url"
 
+    # Which service id to use for the charge 
+    SERVICE_ID: int = 0
+
+    # Which dataset template id to use for dataset creation
+    DATASET_TEMPLATE_ID: int = 0
+
     class Config:
 
         env_file = ".env"  

bfabric_web_apps/utils/create_app_in_bfabric.py

@@ -11,7 +11,7 @@ def get_user_input():
             "2": "Proteomics",
             "4": "Metabolomics / Biophysics",
             "6": "General",
-            "10": "New Tech"
+            "10": "Bioinformatics"
         },
         "PRODUCTION": {
             "1": "Genomics / Transcriptomics",

bfabric_web_apps/utils/dataset_utils.py

@@ -0,0 +1,93 @@
+
+def is_numeric(value):
+    try:
+        float(value)  # or int(value) if you only want integers
+        return True
+    except ValueError:
+        return False
+
+
+def dataset_to_dictionary(dataset): 
+
+    """
+    Convert B-Fabric API Dataset Response 
+    to a dictionary. The dictionary will have the attribute names as keys and the field values as lists, 
+    so that it can be easily converted to a pandas dataframe.
+
+    Args: 
+        dataset (dict): B-Fabric API Dataset Response
+
+    Returns:
+        dict: A dictionary where the keys are the attribute names and the values are lists of field values, ready to become a pandas dataframe.
+    """
+
+    # Check if the dataset is empty
+    if not dataset:
+        return {}
+
+    attributes = dataset.get("attribute", []) 
+    items = [elt.get("field") for elt in dataset.get("item", [])]
+
+    position_map = {str(elt.get("position")): elt.get("name") for elt in attributes} # Create a mapping of attribute positions to names
+    df_dict = {elt : [] for elt in position_map.values()} # Create a dictionary to hold the dataframe data
+
+    for item in items: 
+        for field in item: 
+            attribute_position = field.get("attributeposition")
+            df_dict[position_map.get(attribute_position)].append(field.get("value")) # Append the field value to the corresponding attribute name in the dictionary
+                
+    # Create a dataframe from the dictionary
+    return df_dict
+
+
+def dictionary_to_dataset(dictionary, dataset_name, containerid, dataset_template_id=0, linked_workunit_id=0): 
+    
+    """
+    Convert a dictionary to a B-Fabric API Dataset
+
+    Args: 
+        dictionary (dict): A dictionary where the keys are the attribute names and the values are lists of field values.
+
+    Returns:
+        dict: A B-Fabric API Dataset ready to be sent to the API.
+    """
+
+    if not isinstance(dictionary, dict):
+        raise ValueError("Input must be a dictionary.")
+    
+    if not isinstance(dataset_name, str):
+        raise ValueError("Dataset name must be a string.")
+    
+    if not is_numeric(containerid):
+        raise ValueError("Container ID must be a numeric string or integer.")
+    
+    if not isinstance(dataset_template_id, int):
+        raise ValueError("Dataset template ID must be an integer.")
+    
+    if not isinstance(linked_workunit_id, int):
+        raise ValueError("Linked workunit ID must be an integer.")
+
+    # Check if the dictionary is empty
+    if not dictionary:
+        return {}
+
+    # Create a list of attributes
+    attributes = [{"name": name, "position": str(i+1)} for i, name in enumerate(dictionary.keys())]
+
+    # Create a list of items
+    items = []
+    for i in range(len(next(iter(dictionary.values())))):  # Get the length of the first value list
+        item = [{"attributeposition": str(j+1), "value": dictionary[name][i]} for j, name in enumerate(dictionary.keys())]
+        items.append({"field": item, "position": str(i+1)})
+
+    to_return = {"attribute": attributes, "item": items, "name": dataset_name, "containerid": containerid}
+
+    if dataset_template_id:
+        # Add the dataset template ID to the dataset
+        to_return["datasettemplateid"] = dataset_template_id
+
+    if linked_workunit_id:
+        # Add the linked workunit ID to the dataset
+        to_return["workunitid"] = linked_workunit_id
+
+    return to_return

bfabric_web_apps/utils/redis_worker_init.py

@@ -1,28 +1,38 @@
-import redis 
+import redis
 from rq import Worker, Queue, Connection
 import time
+import threading
 
-def test_job(): 
-
-    """
-    A test job that prints a message to the console.
-    """
+def test_job():
     print("Hello, this is a test job!")
-    time.sleep(10) 
+    time.sleep(10)
     print("Test job finished!")
     return
 
-
-def run_worker(host, port, queue_names): 
+def keepalive_ping(conn, interval=60):
     """
-    Provides internal interface for running workers on a specified host and port.
+    Periodically ping Redis to keep the TCP connection alive on platforms like Azure.
+    """
+    while True:
+        try:
+            conn.ping()
+        except Exception as e:
+            print("Redis keepalive ping failed:", e)
+        time.sleep(interval)
 
-    Args:
-        host (str): The host to run
-        port (int): The port to run
-        queue_names (list): A list of queue names to listen to
+def run_worker(host, port, queue_names):
+    """
+    Starts an RQ worker with a background Redis keepalive thread to prevent Azure from dropping idle connections.
     """
-    conn = redis.Redis(host=host, port=port) 
-    with Connection(conn): 
-        worker = Worker(map(Queue, queue_names)) 
-        worker.work()
+    conn = redis.Redis(
+        host=host,
+        port=port,
+        socket_keepalive=True
+    )
+
+    # Start Redis keepalive thread
+    threading.Thread(target=keepalive_ping, args=(conn,), daemon=True).start()
+
+    with Connection(conn):
+        worker = Worker(map(Queue, queue_names))
+        worker.work(logging_level="INFO")

bfabric_web_apps/utils/resource_utilities.py

@@ -22,8 +22,8 @@ def create_workunit(token_data, application_name, application_description, appli
     wrapper = bfabric_interface.get_wrapper()
 
     workunit_data = {
-        "name": f"{application_name} - Order {container_id}",
-        "description": f"{application_description} for Order {container_id}",
+        "name": f"Workunit - {application_name} - Container {container_id}",
+        "description": f"{application_description} for Container {container_id}",
         "applicationid": int(application_id),
         "containerid": container_id, 
     }
@@ -54,7 +54,7 @@ def create_workunit(token_data, application_name, application_description, appli
 
     except Exception as e:
         L.log_operation(
-            "Error",
+            "Error | ORIGIN: run_main_job function",
             f"Failed to create workunit for Order {container_id}: {e}",
             params=None,
             flush_logs=True,
@@ -85,7 +85,7 @@ def create_workunits(token_data, application_name, application_description, appl
         for container_id in container_ids
     ]
 
-    return [wu_id for wu_id in workunits if wu_id is not None]  # Filter out None values
+    return [wu for wu in workunits if wu is not None]  # Filter out None values
 
 
 from pathlib import Path
@@ -117,7 +117,7 @@ def create_resource(token_data, workunit_id, file_path, storage_id="20"): # GWC
                 "workunitid": str(workunit_id),
                 "name": file_path.name,
                 "description": f"Resource attached to workunit {workunit_id}",
-                "relativepath": file_path.name,
+                "relativepath": file_path,
                 "storageid": str(storage_id),
             }
         )
@@ -125,19 +125,13 @@ def create_resource(token_data, workunit_id, file_path, storage_id="20"): # GWC
         if result:
             resource_id = result[0].get("id")
             print(f"Resource attached: {file_path.name} (ID: {resource_id})")
-            L.log_operation(
-                "Attach_resource",
-                f"Resource attached successfully: {file_path.name}",
-                params=None,
-                flush_logs=True,
-            )
             return result[0]
         else:
             raise ValueError(f"Failed to attach resource: {file_path.name}")
 
     except Exception as e:
         L.log_operation(
-            "error",
+            "error | ORIGIN: run_main_job function",
             f"Failed to attach resource: {e}",
             params=None,
             flush_logs=True,

bfabric_web_apps/utils/run_main_pipeline.py

@@ -4,6 +4,8 @@ import os
 import shutil
 import subprocess
 from pathlib import Path
+import time
+from collections import defaultdict
 
 from .get_logger import get_logger
 from .get_power_user_wrapper import get_power_user_wrapper
@@ -15,6 +17,12 @@ from .resource_utilities import (
     create_workunits, 
     create_resources
 )
+from .dataset_utils import (
+    dataset_to_dictionary,
+    dictionary_to_dataset
+)
+
+from .charging import create_charge
 
 from .config import settings as config
 from datetime import datetime as dt
@@ -24,19 +32,29 @@ SCRATCH_PATH = config.SCRATCH_PATH
 TRX_LOGIN = config.TRX_LOGIN
 TRX_SSH_KEY = config.TRX_SSH_KEY
 URL = config.URL
+DATASET_TEMPLATE_ID = config.DATASET_TEMPLATE_ID
+
+def run_main_job(
+    files_as_byte_strings: dict, 
+    bash_commands: list[str], 
+    resource_paths: dict,
+    attachment_paths: list[dict], 
+    token: str,
+    service_id: int = 0,
+    charge: list[int] = [],
+    dataset_dict: dict = {}
+):
+
 
-def run_main_job(files_as_byte_strings: dict, 
-                 bash_commands: list[str], 
-                 resource_paths: dict,
-                 attachment_paths: list[dict], 
-                 token: str):
     """
     Main function to handle:
       1) Save Files on Server
       2) Execute local bash commands
       3) Create workunits in B-Fabric
-      4) Register resources in B-Fabric
-      5) Attach additional gstore files (logs/reports/etc.) to entities in B-Fabric
+      4) Create Dataset in B-Fabric
+      5) Register resources in B-Fabric
+      6) Attach additional gstore files (logs/reports/etc.) to entities in B-Fabric
+      7) Automatically charge the relevant container for the service
 
     :param files_as_byte_strings: {destination_path: file as byte strings}
     :param bash_commands: List of bash commands to execute
@@ -44,9 +62,13 @@ def run_main_job(files_as_byte_strings: dict,
     :param attachment_paths: Dictionary mapping source file paths to their corresponding file names ({"path/test.txt": "name.txt"})
                              for attachment to a B-Fabric entity (e.g., logs, final reports, etc.)
     :param token: Authentication token
+    :param service_id: ID of the service to charge
+    :param charge: A list of container IDs to be charged.
+    :param dataset_dict: A dictionary to create a dataset in B-Fabric. keys are container IDs and values are dictionaries whose keys are field names and values are lists of values.
+
 
     
-Dev Notes:
+    Dev Notes:
     !!! All exceptions get logged (make sure to log the exception message i.e. "except Exception as e: log(e)") !!!
     !!! If an exception doesn't occur, log that some step ran successfully to the job object !!!
     """
@@ -73,50 +95,98 @@ Dev Notes:
     # Step 1: Save files to the server
     try:
         summary = save_files_from_bytes(files_as_byte_strings, L)
-        L.log_operation("Success", f"File copy summary: {summary}", params=None, flush_logs=True)
+        L.log_operation("Success | ORIGIN: run_main_job function", f"File copy summary: {summary}", params=None, flush_logs=True)
         print("Summary:", summary)
+        
     except Exception as e:
         # If something unexpected blows up the entire process
-        L.log_operation("Error", f"Failed to copy files: {e}", params=None, flush_logs=True)
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to copy files: {e}", params=None, flush_logs=True)
         print("Error copying files:", e)
 
     
     # STEP 2: Execute bash commands
     try:
-        bash_log = execute_and_log_bash_commands(bash_commands, L)
-        L.log_operation("Success", f"Bash commands executed successfully:\n{bash_log}", 
+        bash_log = execute_and_log_bash_commands(bash_commands)
+        L.log_operation("Success | ORIGIN: run_main_job function", f"Bash commands executed success | origin: run_main_job functionfully:\n{bash_log}", 
                         params=None, flush_logs=True)
     except Exception as e:
-        L.log_operation("Error", f"Failed to execute bash commands: {e}", 
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to execute bash commands: {e}", 
                         params=None, flush_logs=True)
         print("Error executing bash commands:", e)
 
 
     # STEP 3: Create Workunits
     try:
-        workunit_map = create_workunits_step(token_data, app_data, resource_paths, L)
+        workunit_map, workunit_container_map = create_workunits_step(token_data, app_data, resource_paths, L)
     except Exception as e:
-        L.log_operation("Error", f"Failed to create workunits in B-Fabric: {e}", 
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to create workunits in B-Fabric: {e}", 
                         params=None, flush_logs=True)
         print("Error creating workunits:", e)
         workunit_map = []
 
-    # STEP 4: Register Resources (Refactored)
+
+    # STEP 4: Create Dataset
+    if dataset_dict:
+        for container_id, dataset_data in dataset_dict.items():
+
+            dataset_name = f'Dataset - {str(app_data.get("name", "Unknown App"))} - Container {container_id}'
+            linked_workunit_id = workunit_container_map.get(str(container_id), None)
+
+            try:
+                dataset = dictionary_to_dataset(dataset_data, dataset_name, container_id, DATASET_TEMPLATE_ID, linked_workunit_id)
+                dataset = create_dataset(token_data, dataset)
+                L.log_operation("Success | ORIGIN: run_main_job function", f'Dataset {dataset.get("id", "Null")} created successfully for container {container_id}', params=None, flush_logs=True)
+                print(f"Dataset created successfully for container {container_id}")
+            except Exception as e:
+                L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to create dataset for container {container_id}: {e}", params=None, flush_logs=True)
+                print(f"Error creating dataset for container {container_id}:", e)
+    else:
+        L.log_operation("Info | ORIGIN: run_main_job function", "No dataset creation requested.", params=None, flush_logs=True)
+        print("No dataset creation requested.")
+
+
+    # STEP 5: Register Resources (Refactored)
     try:
         attach_resources_to_workunits(token_data, L, workunit_map)
     except Exception as e:
-        L.log_operation("Error", f"Failed to register resources: {e}", params=None, flush_logs=True)
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to register resources: {e}", params=None, flush_logs=True)
         print("Error registering resources:", e)
 
-    # STEP 5: Attach gstore files (logs, reports, etc.) to B-Fabric entity as a Link
+    # STEP 6: Attach gstore files (logs, reports, etc.) to B-Fabric entity as a Link
     try:
         attach_gstore_files_to_entities_as_link(token_data, L, attachment_paths)
         print("Attachment Paths:", attachment_paths)
     except Exception as e:
-        L.log_operation("Error", f"Failed to attach extra files: {e}", params=None, flush_logs=True)
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to attach extra files: {e}", params=None, flush_logs=True)
         print("Error attaching extra files:", e)
 
 
+    # STEP 7: Charge the container for the service
+    if charge: 
+        
+        if service_id == 0:
+            print("Service ID not provided. Skipping charge creation.")
+            L.log_operation("Info | ORIGIN: run_main_job function", "Service ID not provided. Skipping charge creation.", params=None, flush_logs=True)
+        else:
+            container_ids = charge
+            print("Container IDs to charge:", container_ids)
+            if not container_ids:
+                L.log_operation("Error | ORIGIN: run_main_job function", "No container IDs found for charging.", params=None, flush_logs=True)
+                print("Error: No container IDs found for charging.")
+                return
+            for container_id in container_ids:
+                charges = create_charge(token_data, container_id, service_id)
+                charge_id = charges[0].get("id")
+                L.log_operation("Success | ORIGIN: run_main_job function", f"Charge created for container {container_id} with service ID {service_id} and charge id {charge_id}", params=None, flush_logs=False)
+                print(f"Charge created with id {charge_id} for container {container_id} with service ID {service_id}")
+            L.flush_logs()
+    else:
+        L.log_operation("Info | ORIGIN: run_main_job function", "Charge creation skipped.", params=None, flush_logs=True)
+        print("Charge creation skipped.")
+    
+    # Final log message
+    L.log_operation("Success | ORIGIN: run_main_job function", "All steps completed successfully.", params=None, flush_logs=True)
+    print("All steps completed successfully.")
 
 #---------------------------------------------------------------------------------------------------------------------
 #---------------------------------------------------------------------------------------------------------------------
@@ -126,8 +196,6 @@ Dev Notes:
 # Step 1: Save Files from bytes
 # -----------------------------------------------------------------------------
 
-import os
-
 def save_files_from_bytes(files_as_byte_strings: dict, logger):
     """
     Saves byte string files to their respective paths.
@@ -136,44 +204,42 @@ def save_files_from_bytes(files_as_byte_strings: dict, logger):
     :param logger: Logging instance
     :return: Summary indicating how many files succeeded vs. failed
     """
-    results = {}  # Store results: (destination) -> True (if success) or error message (if failure)
+
+    message = "All files saved successfully."
 
     # First pass: attempt to write all files
     for destination, file_bytes in files_as_byte_strings.items():
         try:
-            # Ensure the directory exists
-            os.makedirs(os.path.dirname(destination), exist_ok=True)
-
             # Write file from byte string
-            with open(destination, "wb") as f:
+            with open(destination, "+wb") as f:
                 f.write(file_bytes)
-            logger.log_operation("Files saved", "All files saved successfully.", params=None, flush_logs=True)
-            return "All files saved successfully."
+            logger.log_operation("File saved | ORIGIN: run_main_job function", f"File {destination} saved successfully.", params=None, flush_logs=True)
         
         except Exception as e:
             error_msg = f"Error saving file: {destination}, Error: {str(e)}"
-            logger.log_operation("Error", error_msg, params=None, flush_logs=True)
+            logger.log_operation("Error | ORIGIN: run_main_job function", error_msg, params=None, flush_logs=True)
             print(error_msg)
-            raise RuntimeError(error_msg)
+            message = f"Error saving some files." 
+
+    return message
 
 
 # -----------------------------------------------------------------------------
 # Step 2: Execute Bash Commands
 # -----------------------------------------------------------------------------
 
-def execute_and_log_bash_commands(bash_commands: list[str], logger):
+def execute_and_log_bash_commands(bash_commands: list[str]):
     """
     Executes a list of bash commands locally, logs and returns the output.
 
     :param bash_commands: List of commands to execute
-    :param logger: Logging instance
     :return: A single string containing logs for all commands
     """
     logstring = ""
 
     for cmd in bash_commands:
         logstring += "---------------------------------------------------------\n"
-        logstring += f"Executing Command: {cmd}\n"
+        logstring += f"Executing Command\n"
 
         try:
             # Execute the command and capture both stdout and stderr
@@ -185,19 +251,15 @@ def execute_and_log_bash_commands(bash_commands: list[str], logger):
             if result.returncode == 0:
                 status = "SUCCESS"
                 log_entry = f"Command: {cmd}\nStatus: {status}\nOutput:\n{output}\n"
-                logger.log_operation("Info", log_entry, params=None, flush_logs=True)
             else:
                 status = "FAILURE"
                 log_entry = f"Command: {cmd}\nStatus: {status}\nError Output:\n{error_output}\n"
-                logger.log_operation("Error", log_entry, params=None, flush_logs=True)
 
             logstring += log_entry
             print(log_entry)
 
         except Exception as e:
             logstring += f"Command: {cmd}\nStatus: ERROR\nException: {str(e)}\n"
-            logger.log_operation("Error", f"Command: {cmd} failed with Exception: {str(e)}", 
-                                 params=None, flush_logs=True)
     
     return logstring
 
@@ -212,17 +274,33 @@ def create_workunits_step(token_data, app_data, resource_paths, logger):
 
     :param token_data: dict with token/auth info
     :param app_data: dict with fields like {"id": <app_id>} or other app info
-    :param resource_paths: Dictionary {file_path: container_id}
+    :param resource_paths: Dictionary {file_path or dir_path: container_id}
     :param logger: a logger instance
-    :return: A dictionary mapping file_paths to workunit objects {file_path: workunit}
+    :return: A tuple containing: 
+        A dictionary mapping file_paths to workunit objects {file_path: workunit_id}
+        A dictionary mapping container_ids to workunit objects {container_id: workunit_id}
     """
     app_id = app_data["id"]  # Extract the application ID
 
-    # Extract unique order IDs from resource_paths
-    container_ids = list(set(resource_paths.values()))
+    # Expand any directories into individual files
+    expanded_paths = {}
+
+    for path_str, container_id in resource_paths.items():
+        path = Path(path_str)
+        if path.is_file():
+            expanded_paths[str(path)] = int(container_id)
+        elif path.is_dir():
+            for file in path.rglob("*"): #is a method that returns all files and folders in the directory and its subdirectories
+                if file.is_file():
+                    expanded_paths[str(file)] = int(container_id)
+        else:
+            logger.log_operation("Warning | ORIGIN: run_main_job function", f"Path {path_str} does not exist.", flush_logs=True)
+            print(f"Warning: Path {path_str} does not exist or is not accessible.")
+
+    if not expanded_paths:
+        raise ValueError("No valid file paths found in resource_paths.")
 
-    if not container_ids:
-        raise ValueError("No order IDs found in resource_paths; cannot create workunits.")
+    container_ids = list(set(expanded_paths.values()))
 
     # Create all workunits in one API call
     created_workunits = create_workunits(
@@ -239,20 +317,45 @@ def create_workunits_step(token_data, app_data, resource_paths, logger):
     workunit_map = {
         file_path: wu["id"]
         for wu in created_workunits
-        for file_path, container_id in resource_paths.items()
+        for file_path, container_id in expanded_paths.items()
         if container_id == wu["container"]["id"]
     }
 
-    logger.log_operation("Success", f"Total created Workunits: {list(workunit_map.values())}", params=None, flush_logs=True)
-    print(f"Total created Workunits: {list(workunit_map.values())}")
+    workunit_container_map = {
+        str(wu["container"]["id"]): wu["id"]
+        for wu in created_workunits
+    }
 
+    workunit_ids = [wu.get("id") for wu in created_workunits]
+    logger.log_operation("Success | ORIGIN: run_main_job function", f"Total created Workunits: {workunit_ids}", params=None, flush_logs=True)
+    print(f"Total created Workunits: {workunit_ids}")
     print(workunit_map)
-    return workunit_map  # Returning {file_path: workunit}
+
+    return workunit_map, workunit_container_map  # Returning {file_path: workunit}
 
 
 
 # -----------------------------------------------------------------------------
-# Step 4: Attach Resources in B-Fabric
+# Step 4: Create Dataset in B-Fabric
+# -----------------------------------------------------------------------------
+def create_dataset(token_data, dataset_data): 
+
+    """
+    Creates a dataset in B-Fabric using the provided dataset data.
+    :param dataset_data: Dictionary containing dataset information
+    :param token_data: B-Fabric token data
+    :return: The created dataset object
+    """
+
+    wrapper = get_power_user_wrapper(token_data) 
+    dataset = wrapper.save("dataset", dataset_data)  # Save the dataset
+
+    return dataset[0]
+
+
+
+# -----------------------------------------------------------------------------
+# Step 5: Attach Resources in B-Fabric
 # -----------------------------------------------------------------------------
 
 def attach_resources_to_workunits(token_data, logger, workunit_map):
@@ -266,33 +369,45 @@ def attach_resources_to_workunits(token_data, logger, workunit_map):
     :param workunit_map: Dictionary mapping file_path to workunit_id {file_path: workunit_id}
     """
     if not workunit_map:
-        logger.log_operation("Info", "No workunits found, skipping resource registration.",
+        logger.log_operation("Info | ORIGIN: run_main_job function", "No workunits found, skipping resource registration.",
                              params=None, flush_logs=True)
         print("No workunits found, skipping resource registration.")
         return
-    
+
     print("Workunit Map:", workunit_map)
 
+    # Dictionary to count successfully created resources per workunit
+    # defaultdict(int) automatically starts each new key with a value of 0
+    workunit_resource_count = defaultdict(int)
+
     for file_path, workunit_id in workunit_map.items():
-        print(f"Processing file: {file_path}, Workunit ID: {workunit_id}")  # Corrected print statement
+        print(f"Processing file: {file_path}, Workunit ID: {workunit_id}")
         # Upload the file as a resource
         resource = create_resource(token_data, workunit_id, file_path)
         resource_id = resource.get("id")
         print("Resource ID:", resource_id)
-
         if resource_id:
-            logger.log_operation("Success", f"Resource {resource_id} attached to Workunit {workunit_id}",
-                                 params=None, flush_logs=True)
+            workunit_resource_count[workunit_id] += 1
             print(f"Resource {resource_id} attached to Workunit {workunit_id}")
         else:
-            logger.log_operation("Error", f"Failed to attach resource {file_path} for Workunit {workunit_id}",
+            logger.log_operation("Error | ORIGIN: run_main_job function", f"Failed to attach resource {file_path} for Workunit {workunit_id}",
                                  params=None, flush_logs=True)
             print(f"Failed to attach resource {file_path} for Workunit {workunit_id}")
 
+    # Log a summary per workunit
+    for workunit_id, count in workunit_resource_count.items():
+        logger.log_operation(
+            "Success | ORIGIN: run_main_job function",
+            f"Created {count} resource(s) for Workunit ID {workunit_id}",
+            params=None,
+            flush_logs=True
+        )
+        print(f"Created {count} resource(s) for Workunit ID {workunit_id}")
+
 
 
 # -----------------------------------------------------------------------------
-# Step 5: Attachments of gstore in B-Fabric as a Link
+# Step 6: Attachments of gstore in B-Fabric as a Link
 # -----------------------------------------------------------------------------
 
 def attach_gstore_files_to_entities_as_link(token_data, logger, attachment_paths: dict):
@@ -320,7 +435,7 @@ def attach_gstore_files_to_entities_as_link(token_data, logger, attachment_paths
     # Process each attachment
     for source_path, file_name in attachment_paths.items():
         if not source_path or not file_name:
-            logger.log_operation("Error", f"Missing required attachment details: {source_path} -> {file_name}", params=None, flush_logs=True)
+            logger.log_operation("Error | ORIGIN: run_main_job function", f"Missing required attachment details: {source_path} -> {file_name}", params=None, flush_logs=True)
             print(f"Error: Missing required attachment details: {source_path} -> {file_name}")
             continue
 
@@ -340,21 +455,22 @@ def attach_gstore_files_to_entities_as_link(token_data, logger, attachment_paths
             else:  # We don't have direct access → Send to migration folder first
                 remote_tmp_path = f"{SCRATCH_PATH}/{file_name}"
                 scp_copy(source_path, TRX_LOGIN, TRX_SSH_KEY, remote_tmp_path)
+                print("scp copy done:")
 
                 # Move to final location
                 ssh_move(TRX_LOGIN, TRX_SSH_KEY, remote_tmp_path, final_remote_path)
+                print("ssh move done:")
 
             # Log success
             success_msg = f"Successfully attached '{file_name}' to {entity_class} (ID={entity_id})"
-            logger.log_operation("Success", success_msg, params=None, flush_logs=True)
             print(success_msg)
 
             # Step 3: Create API link
-            create_api_link(token_data, logger, entity_class, entity_id, file_name, entity_folder)
+            create_attachment_link(token_data, logger, entity_class, entity_id, file_name, entity_folder)
 
         except Exception as e:
             error_msg = f"Exception while processing '{file_name}': {e}"
-            logger.log_operation("Error", error_msg, params=None, flush_logs=True)
+            logger.log_operation("Error | ORIGIN: run_main_job function", error_msg, params=None, flush_logs=True)
             print(error_msg)
 
 def local_access(remote_path):
@@ -366,8 +482,11 @@ def local_access(remote_path):
 
 def scp_copy(source_path, ssh_user, ssh_key, remote_path):
     """Copies a file to a remote location using SCP with the correct FGCZ server address."""
+    print("SCP Copying...")
     cmd = ["scp", "-i", ssh_key, source_path, f"{ssh_user}:{remote_path}"]
+    print("SCP Command:")
     subprocess.run(cmd, check=True)
+    print("SCP Command Executed:", cmd)
     print(f"Copied {source_path} to {remote_path}")
 
 
@@ -377,6 +496,9 @@ def ssh_move(ssh_user, ssh_key, remote_tmp_path, final_remote_path):
 
     subprocess.run(cmd, check=True)
     print(f"Moved {remote_tmp_path} to {final_remote_path}")
+    
+    # Wait 10 second before next move
+    time.sleep(10)
 
 
 def g_req_copy(source_path, destination_path):
@@ -386,8 +508,8 @@ def g_req_copy(source_path, destination_path):
     print(f"Copied {source_path} using g-req")
 
 
-def create_api_link(token_data, logger, entity_class, entity_id, file_name, folder_name):
-    """Creates an API link in B-Fabric for the attached file."""
+def create_attachment_link(token_data, logger, entity_class, entity_id, file_name, folder_name):
+    """Creates an attachment link in B-Fabric for the attached file."""
     wrapper = get_power_user_wrapper(token_data)
     url = f"{URL}/{folder_name}/{file_name}"
     timestamped_filename = f"{dt.now().strftime('%Y-%m-%d_%H:%M:%S')}_{file_name}"
@@ -402,13 +524,24 @@ def create_api_link(token_data, logger, entity_class, entity_id, file_name, fold
     try:
         link_result = wrapper.save("link", data)
         if link_result:
-            success_msg = f"API link created for '{file_name}': {url}"
-            logger.log_operation("Success", success_msg, params=None, flush_logs=True)
+            success_msg = f"Attachment link created for '{file_name}': {url}"
+            logger.log_operation("Success | ORIGIN: run_main_job function", success_msg, params=None, flush_logs=True)
             print(success_msg)
         else:
-            raise ValueError("API link creation failed")
+            raise ValueError("Attachment link creation failed")
     except Exception as e:
-        error_msg = f"Failed to create API link for '{file_name}': {e}"
-        logger.log_operation("Error", error_msg, params=None, flush_logs=True)
+        error_msg = f"Failed to create attachment link for '{file_name}': {e}"
+        logger.log_operation("Error | ORIGIN: run_main_job function", error_msg, params=None, flush_logs=True)
         print(error_msg)
 
+
+def read_file_as_bytes(file_path, max_size_mb=400):
+    """Reads any file type and stores it as a byte string in a dictionary."""
+    file_size_mb = os.path.getsize(file_path) / (1024 * 1024)  # Convert bytes to MB
+    if file_size_mb > max_size_mb:
+        raise ValueError(f"File {file_path} exceeds {max_size_mb}MB limit ({file_size_mb:.2f}MB).")
+
+    with open(file_path, "rb") as f:  # Read as bytes
+        file_as_bytes = f.read()
+
+    return file_as_bytes

{bfabric_web_apps-0.1.6.dist-info → bfabric_web_apps-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bfabric-web-apps
-Version: 0.1.6
+Version: 0.2.0
 Summary: A package containing handy boilerplate utilities for developing bfabric web-applications
 Author: Marc Zuber, Griffin White, GWC GmbH
 Requires-Python: >=3.10,<4.0
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: bfabric (>=1.13.23,<2.0.0)
 Requires-Dist: dash (>=3.0.2,<4.0.0)
 Requires-Dist: dash-bootstrap-components (>=2.0.0,<3.0.0)
+Requires-Dist: dash-daq (>=0.6.0,<0.7.0)
 Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0)
 Requires-Dist: pydantic[email] (>=2.10.6,<3.0.0)
 Requires-Dist: rq (==1.15.1)

bfabric_web_apps-0.2.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+bfabric_web_apps/__init__.py,sha256=eRYBvXrDM8Bhdm7cbI80L1hjtbRFqUasJiBW52Hx3TI,1715
+bfabric_web_apps/layouts/layouts.py,sha256=z8gL4n4wwLdpLGomO9CftBLnGpc3r6OpmUc2-wBg8uo,14661
+bfabric_web_apps/objects/BfabricInterface.py,sha256=cEUcAdr4iUH8aS2VGe3CFnAbNQiTj13hYuSVcDg725A,10646
+bfabric_web_apps/objects/Logger.py,sha256=62LC94xhm7YG5LUw3yH46NqvJQsAX7wnc9D4zbY16rA,5224
+bfabric_web_apps/utils/app_init.py,sha256=RCdpCXp19cF74bouYJLPe-KSETZ0Vwqtd02Ta2VXEF8,428
+bfabric_web_apps/utils/callbacks.py,sha256=tB1xtHl_ePY6KJWNz3erkrZw3HFhRneewGqZm9xIYtI,12687
+bfabric_web_apps/utils/charging.py,sha256=oNNazH59SFkbxJKPvCel0IxdsRHC8xpJ0AXCLvI88FI,1069
+bfabric_web_apps/utils/components.py,sha256=X3NRnv--LsHWMtWL83Pzr2whOZLSEJIwXTklQdAQpZE,984
+bfabric_web_apps/utils/config.py,sha256=F4EExu7EkY7upOnxk6BU6zTLt9eU6_iy2y8esIlxTSc,1209
+bfabric_web_apps/utils/create_app_in_bfabric.py,sha256=Z7puke8QB4SBuDJ9x3_OjgApzovKu0Nt1g8EqkOHJpc,2758
+bfabric_web_apps/utils/dataset_utils.py,sha256=p_UtoOl1kJpSm2BGdg31Ji0C7ctst40wp4LX1tUe4tI,3360
+bfabric_web_apps/utils/get_logger.py,sha256=0Y3SrXW93--eglS0_ZOc34NOriAt6buFPik5n0ltzRA,434
+bfabric_web_apps/utils/get_power_user_wrapper.py,sha256=T33z64XjmJ0KSlmfEmrEP8eYpbpINCVD6Xld_V7PR2g,1027
+bfabric_web_apps/utils/redis_connection.py,sha256=qXSPxW6m55Ogv44BhmPCl9ACuvzmpfZNU73UJhHRXL4,133
+bfabric_web_apps/utils/redis_queue.py,sha256=MCx7z_I2NusJ4P42mcLvV7STtXBFMIIvun83fM8zOGI,168
+bfabric_web_apps/utils/redis_worker_init.py,sha256=wtjQL48PLNXD1s-5s3Oq5EC8BmcfKcd7IhUbTH_EYz8,1014
+bfabric_web_apps/utils/resource_utilities.py,sha256=N4EiUkxXHZ18jnU2OuRqaGSroCZ73Ogb9lkeA21Kvq4,5716
+bfabric_web_apps/utils/run_main_pipeline.py,sha256=whmUbO9mMom9voOCv7iS51wh6St4DfawCD337BuPvtY,23112
+bfabric_web_apps-0.2.0.dist-info/LICENSE,sha256=k0O_i2k13i9e35aO-j7FerJafAqzzu8x0kkBs0OWF3c,1065
+bfabric_web_apps-0.2.0.dist-info/METADATA,sha256=vhqne76HPRsHA9uFhTPgP7wudTKYJq74V7GEQYR9S1I,687
+bfabric_web_apps-0.2.0.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+bfabric_web_apps-0.2.0.dist-info/RECORD,,

bfabric_web_apps-0.1.6.dist-info/RECORD

@@ -1,20 +0,0 @@
-bfabric_web_apps/__init__.py,sha256=D2jTCEYedVnp6yQxNGLRsjihupDTQxNkPVFQAUYXQys,1409
-bfabric_web_apps/layouts/layouts.py,sha256=z8gL4n4wwLdpLGomO9CftBLnGpc3r6OpmUc2-wBg8uo,14661
-bfabric_web_apps/objects/BfabricInterface.py,sha256=wmcL9JuSC0QEopgImvkZxmtCIS7izt6bwb6y_ch0zus,10178
-bfabric_web_apps/objects/Logger.py,sha256=62LC94xhm7YG5LUw3yH46NqvJQsAX7wnc9D4zbY16rA,5224
-bfabric_web_apps/utils/app_init.py,sha256=RCdpCXp19cF74bouYJLPe-KSETZ0Vwqtd02Ta2VXEF8,428
-bfabric_web_apps/utils/callbacks.py,sha256=m5d6IPiYX77-kJN8I2OptZN-GPxZgrI76o1DGFxjpPU,12686
-bfabric_web_apps/utils/components.py,sha256=V7ECGmF2XYy5O9ciDJVH1nofJYP2a_ELQF3z3X_ADbo,844
-bfabric_web_apps/utils/config.py,sha256=i93fe49Ak4Z7cm_G80m2cBCPp-5qCYLAJEtEr-mYSwQ,1044
-bfabric_web_apps/utils/create_app_in_bfabric.py,sha256=eVk3cQDXxW-yo9b9n_zzGO6kLg_SLxYbIDECyvEPJXU,2752
-bfabric_web_apps/utils/get_logger.py,sha256=0Y3SrXW93--eglS0_ZOc34NOriAt6buFPik5n0ltzRA,434
-bfabric_web_apps/utils/get_power_user_wrapper.py,sha256=T33z64XjmJ0KSlmfEmrEP8eYpbpINCVD6Xld_V7PR2g,1027
-bfabric_web_apps/utils/redis_connection.py,sha256=qXSPxW6m55Ogv44BhmPCl9ACuvzmpfZNU73UJhHRXL4,133
-bfabric_web_apps/utils/redis_queue.py,sha256=MCx7z_I2NusJ4P42mcLvV7STtXBFMIIvun83fM8zOGI,168
-bfabric_web_apps/utils/redis_worker_init.py,sha256=9SUc9bbgBeMbUdqJD9EkWPA4wcJjvyX6Tzanv5JfqEg,691
-bfabric_web_apps/utils/resource_utilities.py,sha256=4LnV_eQjKkcpZJBsWFx--dmASyE7jfJfktk2hdHn5Fk,5856
-bfabric_web_apps/utils/run_main_pipeline.py,sha256=1YSbk3uP_T3tL6mZZXGv7a7FJc8exro_Eb49gnJjdrs,16864
-bfabric_web_apps-0.1.6.dist-info/LICENSE,sha256=k0O_i2k13i9e35aO-j7FerJafAqzzu8x0kkBs0OWF3c,1065
-bfabric_web_apps-0.1.6.dist-info/METADATA,sha256=bFw2hFwCuonWxGFxyCSj6y_l2rIjKfxbzXCAGK3O0Ek,646
-bfabric_web_apps-0.1.6.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-bfabric_web_apps-0.1.6.dist-info/RECORD,,