bfabric-web-apps 0.1.7__py3-none-any.whl → 0.2.1__py3-none-any.whl

bfabric_web_apps/__init__.py

@@ -2,6 +2,7 @@ import os
 
 # Export objects and classes
 from bfabric_web_apps.objects import BfabricInterface, Logger
+from bfabric_web_apps.objects.BfabricInterface import bfabric_interface
 
 # Export components
 from .utils import components
@@ -14,6 +15,10 @@ from .utils.app_init import create_app
 from .utils.get_logger import get_logger
 from .utils.get_power_user_wrapper import get_power_user_wrapper
 from .utils.create_app_in_bfabric import create_app_in_bfabric
+from .utils.dataset_utils import (
+    dataset_to_dictionary, 
+    dictionary_to_dataset
+)
 
 # Export callbacks
 from .utils.callbacks import (
@@ -57,4 +62,5 @@ TRX_LOGIN = config.TRX_LOGIN
 TRX_SSH_KEY = config.TRX_SSH_KEY
 URL = config.URL
 
-SERVICE_ID = config.SERVICE_ID
+SERVICE_ID = config.SERVICE_ID
+DATASET_TEMPLATE_ID = config.DATASET_TEMPLATE_ID

bfabric_web_apps/objects/BfabricInterface.py

@@ -83,14 +83,13 @@ class BfabricInterface( Bfabric ):
             userinfo = json.loads(res.text)
             expiry_time = userinfo['expiryDateTime']
             current_time = datetime.datetime.now()
-            five_minutes_later = current_time + datetime.timedelta(minutes=5)
-
             # Comparing the parsed expiry time with the five minutes later time
 
-            if not five_minutes_later <= datetime.datetime.strptime(expiry_time, "%Y-%m-%d %H:%M:%S"):
+            if current_time > datetime.datetime.strptime(expiry_time, "%Y-%m-%d %H:%M:%S") + datetime.timedelta(days=7):
                 return "EXPIRED"
             
-            environment_dict = {"Production":"https://fgcz-bfabric.uzh.ch/bfabric","Test":"https://fgcz-bfabric-test.uzh.ch/bfabric"}
+            envioronment_name = str(userinfo['environment']).strip().lower()
+            environment_dict = {"production":"https://fgcz-bfabric.uzh.ch/bfabric","prod":"https://fgcz-bfabric.uzh.ch/bfabric","test":"https://fgcz-bfabric-test.uzh.ch/bfabric"}
             
             token_data = dict(
                 environment = userinfo['environment'],
@@ -98,16 +97,24 @@ class BfabricInterface( Bfabric ):
                 token_expires = expiry_time,
                 entity_id_data = userinfo['entityId'],
                 entityClass_data = userinfo['entityClassName'],
-                webbase_data = environment_dict.get(userinfo['environment'], None),
+                webbase_data = environment_dict[envioronment_name],
                 application_params_data = {},
                 application_data = str(userinfo['applicationId']),
                 userWsPassword = userinfo['userWsPassword'],
                 jobId = userinfo['jobId']
             )
-
             # Initialize the wrapper right after validating the token
             self._initialize_wrapper(token_data)
 
+            # Log the token validation process
+            L = get_logger(token_data)
+            L.log_operation(
+                    operation="Authentication Process",
+                    message=f"Token validated successfully. User {token_data.get('user_data')} authenticated.",
+                    params=None,
+                    flush_logs=True
+                )
+
             return json.dumps(token_data)
         
 
@@ -123,10 +130,9 @@ class BfabricInterface( Bfabric ):
         Returns:
             Bfabric: An authenticated Bfabric instance.
         """
-
+     
         bfabric_auth = BfabricAuth(login=token_response.get('user_data'), password=token_response.get('userWsPassword'))
-        bfabric_client_config = BfabricClientConfig(base_url=token_response.get('webbase_data')) 
-
+        bfabric_client_config = BfabricClientConfig(base_url=token_response['webbase_data'])
         bfabric_wrapper = bfabric.Bfabric(config=bfabric_client_config, auth=bfabric_auth)
 
         return bfabric_wrapper
@@ -152,7 +158,10 @@ class BfabricInterface( Bfabric ):
             "Project": "container",
             "Order": "container",
             "Container": "container",
-            "Plate": "plate"
+            "Plate": "plate",
+            "Workunit": "workunit",
+            "Resource": "resource",
+            "Dataset": "dataset"
         }
 
         if not token_data:
@@ -176,7 +185,7 @@ class BfabricInterface( Bfabric ):
                 obj={"id": entity_id},
                 max_results=None,
                 params=None,
-                flush_logs=True
+                flush_logs=False
             )[0]
 
             
@@ -186,6 +195,7 @@ class BfabricInterface( Bfabric ):
                     "createdby": entity_data_dict.get("createdby"),
                     "created": entity_data_dict.get("created"),
                     "modified": entity_data_dict.get("modified"),
+                    "full_api_response": entity_data_dict,
                 })
                 return json_data
             else:
@@ -246,7 +256,7 @@ class BfabricInterface( Bfabric ):
             obj={"id": app_id},  # Query using the App ID
             max_results=None,
             params=None,
-            flush_logs=True
+            flush_logs=False
         )
 
         # If API call fails, return empty JSON

bfabric_web_apps/utils/config.py

@@ -27,6 +27,9 @@ class Settings(BaseSettings):
     # Which service id to use for the charge 
     SERVICE_ID: int = 0
 
+    # Which dataset template id to use for dataset creation
+    DATASET_TEMPLATE_ID: int = 0
+
     class Config:
 
         env_file = ".env"  

bfabric_web_apps/utils/create_app_in_bfabric.py

@@ -11,7 +11,7 @@ def get_user_input():
             "2": "Proteomics",
             "4": "Metabolomics / Biophysics",
             "6": "General",
-            "10": "New Tech"
+            "10": "Bioinformatics"
         },
         "PRODUCTION": {
             "1": "Genomics / Transcriptomics",

bfabric_web_apps/utils/dataset_utils.py

@@ -0,0 +1,93 @@
+
+def is_numeric(value):
+    try:
+        float(value)  # or int(value) if you only want integers
+        return True
+    except ValueError:
+        return False
+
+
+def dataset_to_dictionary(dataset): 
+
+    """
+    Convert B-Fabric API Dataset Response 
+    to a dictionary. The dictionary will have the attribute names as keys and the field values as lists, 
+    so that it can be easily converted to a pandas dataframe.
+
+    Args: 
+        dataset (dict): B-Fabric API Dataset Response
+
+    Returns:
+        dict: A dictionary where the keys are the attribute names and the values are lists of field values, ready to become a pandas dataframe.
+    """
+
+    # Check if the dataset is empty
+    if not dataset:
+        return {}
+
+    attributes = dataset.get("attribute", []) 
+    items = [elt.get("field") for elt in dataset.get("item", [])]
+
+    position_map = {str(elt.get("position")): elt.get("name") for elt in attributes} # Create a mapping of attribute positions to names
+    df_dict = {elt : [] for elt in position_map.values()} # Create a dictionary to hold the dataframe data
+
+    for item in items: 
+        for field in item: 
+            attribute_position = field.get("attributeposition")
+            df_dict[position_map.get(attribute_position)].append(field.get("value")) # Append the field value to the corresponding attribute name in the dictionary
+                
+    # Create a dataframe from the dictionary
+    return df_dict
+
+
+def dictionary_to_dataset(dictionary, dataset_name, containerid, dataset_template_id=0, linked_workunit_id=0): 
+    
+    """
+    Convert a dictionary to a B-Fabric API Dataset
+
+    Args: 
+        dictionary (dict): A dictionary where the keys are the attribute names and the values are lists of field values.
+
+    Returns:
+        dict: A B-Fabric API Dataset ready to be sent to the API.
+    """
+
+    if not isinstance(dictionary, dict):
+        raise ValueError("Input must be a dictionary.")
+    
+    if not isinstance(dataset_name, str):
+        raise ValueError("Dataset name must be a string.")
+    
+    if not is_numeric(containerid):
+        raise ValueError("Container ID must be a numeric string or integer.")
+    
+    if not isinstance(dataset_template_id, int):
+        raise ValueError("Dataset template ID must be an integer.")
+    
+    if not isinstance(linked_workunit_id, int):
+        raise ValueError("Linked workunit ID must be an integer.")
+
+    # Check if the dictionary is empty
+    if not dictionary:
+        return {}
+
+    # Create a list of attributes
+    attributes = [{"name": name, "position": str(i+1)} for i, name in enumerate(dictionary.keys())]
+
+    # Create a list of items
+    items = []
+    for i in range(len(next(iter(dictionary.values())))):  # Get the length of the first value list
+        item = [{"attributeposition": str(j+1), "value": dictionary[name][i]} for j, name in enumerate(dictionary.keys())]
+        items.append({"field": item, "position": str(i+1)})
+
+    to_return = {"attribute": attributes, "item": items, "name": dataset_name, "containerid": containerid}
+
+    if dataset_template_id:
+        # Add the dataset template ID to the dataset
+        to_return["datasettemplateid"] = dataset_template_id
+
+    if linked_workunit_id:
+        # Add the linked workunit ID to the dataset
+        to_return["workunitid"] = linked_workunit_id
+
+    return to_return

bfabric_web_apps/utils/redis_worker_init.py

@@ -1,28 +1,38 @@
-import redis 
+import redis
 from rq import Worker, Queue, Connection
 import time
+import threading
 
-def test_job(): 
-
-    """
-    A test job that prints a message to the console.
-    """
+def test_job():
     print("Hello, this is a test job!")
-    time.sleep(10) 
+    time.sleep(10)
     print("Test job finished!")
     return
 
-
-def run_worker(host, port, queue_names): 
+def keepalive_ping(conn, interval=60):
     """
-    Provides internal interface for running workers on a specified host and port.
+    Periodically ping Redis to keep the TCP connection alive on platforms like Azure.
+    """
+    while True:
+        try:
+            conn.ping()
+        except Exception as e:
+            print("Redis keepalive ping failed:", e)
+        time.sleep(interval)
 
-    Args:
-        host (str): The host to run
-        port (int): The port to run
-        queue_names (list): A list of queue names to listen to
+def run_worker(host, port, queue_names):
+    """
+    Starts an RQ worker with a background Redis keepalive thread to prevent Azure from dropping idle connections.
     """
-    conn = redis.Redis(host=host, port=port) 
-    with Connection(conn): 
-        worker = Worker(map(Queue, queue_names)) 
-        worker.work()
+    conn = redis.Redis(
+        host=host,
+        port=port,
+        socket_keepalive=True
+    )
+
+    # Start Redis keepalive thread
+    threading.Thread(target=keepalive_ping, args=(conn,), daemon=True).start()
+
+    with Connection(conn):
+        worker = Worker(map(Queue, queue_names))
+        worker.work(logging_level="INFO")

bfabric_web_apps/utils/resource_utilities.py

@@ -22,8 +22,8 @@ def create_workunit(token_data, application_name, application_description, appli
     wrapper = bfabric_interface.get_wrapper()
 
     workunit_data = {
-        "name": f"{application_name} - Order {container_id}",
-        "description": f"{application_description} for Order {container_id}",
+        "name": f"Workunit - {application_name} - Container {container_id}",
+        "description": f"{application_description} for Container {container_id}",
         "applicationid": int(application_id),
         "containerid": container_id, 
     }
@@ -54,7 +54,7 @@ def create_workunit(token_data, application_name, application_description, appli
 
     except Exception as e:
         L.log_operation(
-            "Error",
+            "Error | ORIGIN: run_main_job function",
             f"Failed to create workunit for Order {container_id}: {e}",
             params=None,
             flush_logs=True,
@@ -85,7 +85,7 @@ def create_workunits(token_data, application_name, application_description, appl
         for container_id in container_ids
     ]
 
-    return [wu_id for wu_id in workunits if wu_id is not None]  # Filter out None values
+    return [wu for wu in workunits if wu is not None]  # Filter out None values
 
 
 from pathlib import Path
@@ -117,7 +117,7 @@ def create_resource(token_data, workunit_id, file_path, storage_id="20"): # GWC
                 "workunitid": str(workunit_id),
                 "name": file_path.name,
                 "description": f"Resource attached to workunit {workunit_id}",
-                "relativepath": file_path.name,
+                "relativepath": file_path,
                 "storageid": str(storage_id),
             }
         )
@@ -125,19 +125,13 @@ def create_resource(token_data, workunit_id, file_path, storage_id="20"): # GWC
         if result:
             resource_id = result[0].get("id")
             print(f"Resource attached: {file_path.name} (ID: {resource_id})")
-            L.log_operation(
-                "Attach_resource",
-                f"Resource attached successfully: {file_path.name}",
-                params=None,
-                flush_logs=True,
-            )
             return result[0]
         else:
             raise ValueError(f"Failed to attach resource: {file_path.name}")
 
     except Exception as e:
         L.log_operation(
-            "error",
+            "error | ORIGIN: run_main_job function",
             f"Failed to attach resource: {e}",
             params=None,
             flush_logs=True,

bfabric_web_apps/utils/run_main_pipeline.py

@@ -4,6 +4,8 @@ import os
 import shutil
 import subprocess
 from pathlib import Path
+import time
+from collections import defaultdict
 
 from .get_logger import get_logger
 from .get_power_user_wrapper import get_power_user_wrapper
@@ -15,6 +17,10 @@ from .resource_utilities import (
     create_workunits, 
     create_resources
 )
+from .dataset_utils import (
+    dataset_to_dictionary,
+    dictionary_to_dataset
+)
 
 from .charging import create_charge
 
@@ -26,6 +32,7 @@ SCRATCH_PATH = config.SCRATCH_PATH
 TRX_LOGIN = config.TRX_LOGIN
 TRX_SSH_KEY = config.TRX_SSH_KEY
 URL = config.URL
+DATASET_TEMPLATE_ID = config.DATASET_TEMPLATE_ID
 
 def run_main_job(
     files_as_byte_strings: dict, 
@@ -34,16 +41,20 @@ def run_main_job(
     attachment_paths: list[dict], 
     token: str,
     service_id: int = 0,
-    charge: bool = False,
+    charge: list[int] = [],
+    dataset_dict: dict = {}
 ):
+
+
     """
     Main function to handle:
       1) Save Files on Server
       2) Execute local bash commands
       3) Create workunits in B-Fabric
-      4) Register resources in B-Fabric
-      5) Attach additional gstore files (logs/reports/etc.) to entities in B-Fabric
-      6) Automatically charge the relevant container for the service
+      4) Create Dataset in B-Fabric
+      5) Register resources in B-Fabric
+      6) Attach additional gstore files (logs/reports/etc.) to entities in B-Fabric
+      7) Automatically charge the relevant container for the service
 
     :param files_as_byte_strings: {destination_path: file as byte strings}
     :param bash_commands: List of bash commands to execute
@@ -52,10 +63,12 @@ def run_main_job(
                              for attachment to a B-Fabric entity (e.g., logs, final reports, etc.)
     :param token: Authentication token
     :param service_id: ID of the service to charge
-    :param charge: Boolean indicating whether to charge the container for the service
+    :param charge: A list of container IDs to be charged.
+    :param dataset_dict: A dictionary to create a dataset in B-Fabric. keys are container IDs and values are dictionaries whose keys are field names and values are lists of values.
+
 
     
-Dev Notes:
+    Dev Notes:
     !!! All exceptions get logged (make sure to log the exception message i.e. "except Exception as e: log(e)") !!!
     !!! If an exception doesn't occur, log that some step ran successfully to the job object !!!
     """
@@ -82,72 +95,98 @@ Dev Notes:
     # Step 1: Save files to the server
     try:
         summary = save_files_from_bytes(files_as_byte_strings, L)
-        L.log_operation("Success", f"File copy summary: {summary}", params=None, flush_logs=True)
+        L.log_operation("Success | ORIGIN: run_main_job function", f"File copy summary: {summary}", params=None, flush_logs=True)
         print("Summary:", summary)
         
     except Exception as e:
         # If something unexpected blows up the entire process
-        L.log_operation("Error", f"Failed to copy files: {e}", params=None, flush_logs=True)
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to copy files: {e}", params=None, flush_logs=True)
         print("Error copying files:", e)
 
     
     # STEP 2: Execute bash commands
     try:
-        bash_log = execute_and_log_bash_commands(bash_commands, L)
-        L.log_operation("Success", f"Bash commands executed successfully:\n{bash_log}", 
+        bash_log = execute_and_log_bash_commands(bash_commands)
+        L.log_operation("Success | ORIGIN: run_main_job function", f"Bash commands executed success | origin: run_main_job functionfully:\n{bash_log}", 
                         params=None, flush_logs=True)
     except Exception as e:
-        L.log_operation("Error", f"Failed to execute bash commands: {e}", 
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to execute bash commands: {e}", 
                         params=None, flush_logs=True)
         print("Error executing bash commands:", e)
 
 
     # STEP 3: Create Workunits
     try:
-        workunit_map = create_workunits_step(token_data, app_data, resource_paths, L)
+        workunit_map, workunit_container_map = create_workunits_step(token_data, app_data, resource_paths, L)
     except Exception as e:
-        L.log_operation("Error", f"Failed to create workunits in B-Fabric: {e}", 
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to create workunits in B-Fabric: {e}", 
                         params=None, flush_logs=True)
         print("Error creating workunits:", e)
         workunit_map = []
 
-    # STEP 4: Register Resources (Refactored)
+
+    # STEP 4: Create Dataset
+    if dataset_dict:
+        for container_id, dataset_data in dataset_dict.items():
+
+            dataset_name = f'Dataset - {str(app_data.get("name", "Unknown App"))} - Container {container_id}'
+            linked_workunit_id = workunit_container_map.get(str(container_id), None)
+
+            try:
+                dataset = dictionary_to_dataset(dataset_data, dataset_name, container_id, DATASET_TEMPLATE_ID, linked_workunit_id)
+                dataset = create_dataset(token_data, dataset)
+                L.log_operation("Success | ORIGIN: run_main_job function", f'Dataset {dataset.get("id", "Null")} created successfully for container {container_id}', params=None, flush_logs=True)
+                print(f"Dataset created successfully for container {container_id}")
+            except Exception as e:
+                L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to create dataset for container {container_id}: {e}", params=None, flush_logs=True)
+                print(f"Error creating dataset for container {container_id}:", e)
+    else:
+        L.log_operation("Info | ORIGIN: run_main_job function", "No dataset creation requested.", params=None, flush_logs=True)
+        print("No dataset creation requested.")
+
+
+    # STEP 5: Register Resources (Refactored)
     try:
         attach_resources_to_workunits(token_data, L, workunit_map)
     except Exception as e:
-        L.log_operation("Error", f"Failed to register resources: {e}", params=None, flush_logs=True)
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to register resources: {e}", params=None, flush_logs=True)
         print("Error registering resources:", e)
 
-    # STEP 5: Attach gstore files (logs, reports, etc.) to B-Fabric entity as a Link
+    # STEP 6: Attach gstore files (logs, reports, etc.) to B-Fabric entity as a Link
     try:
         attach_gstore_files_to_entities_as_link(token_data, L, attachment_paths)
         print("Attachment Paths:", attachment_paths)
     except Exception as e:
-        L.log_operation("Error", f"Failed to attach extra files: {e}", params=None, flush_logs=True)
+        L.log_operation("Error | ORIGIN: run_main_job function", f"Failed to attach extra files: {e}", params=None, flush_logs=True)
         print("Error attaching extra files:", e)
 
 
-    # STEP 6: Charge the container for the service
+    # STEP 7: Charge the container for the service
     if charge: 
         
         if service_id == 0:
             print("Service ID not provided. Skipping charge creation.")
-            L.log_operation("Info", "Service ID not provided. Skipping charge creation.", params=None, flush_logs=True)
+            L.log_operation("Info | ORIGIN: run_main_job function", "Service ID not provided. Skipping charge creation.", params=None, flush_logs=True)
         else:
-            container_ids = list(set(list(resource_paths.values())))
+            container_ids = charge
+            print("Container IDs to charge:", container_ids)
             if not container_ids:
-                L.log_operation("Error", "No container IDs found for charging.", params=None, flush_logs=True)
+                L.log_operation("Error | ORIGIN: run_main_job function", "No container IDs found for charging.", params=None, flush_logs=True)
                 print("Error: No container IDs found for charging.")
                 return
             for container_id in container_ids:
                 charges = create_charge(token_data, container_id, service_id)
                 charge_id = charges[0].get("id")
-                L.log_operation("Success", f"Charge created for container {container_id} with service ID {service_id} and charge id {charge_id}", params=None, flush_logs=False)
+                L.log_operation("Success | ORIGIN: run_main_job function", f"Charge created for container {container_id} with service ID {service_id} and charge id {charge_id}", params=None, flush_logs=False)
                 print(f"Charge created with id {charge_id} for container {container_id} with service ID {service_id}")
             L.flush_logs()
     else:
-        L.log_operation("Info", "Charge creation skipped.", params=None, flush_logs=True)
+        L.log_operation("Info | ORIGIN: run_main_job function", "Charge creation skipped.", params=None, flush_logs=True)
         print("Charge creation skipped.")
+    
+    # Final log message
+    L.log_operation("Success | ORIGIN: run_main_job function", "All steps completed successfully.", params=None, flush_logs=True)
+    print("All steps completed successfully.")
 
 #---------------------------------------------------------------------------------------------------------------------
 #---------------------------------------------------------------------------------------------------------------------
@@ -157,8 +196,6 @@ Dev Notes:
 # Step 1: Save Files from bytes
 # -----------------------------------------------------------------------------
 
-import os
-
 def save_files_from_bytes(files_as_byte_strings: dict, logger):
     """
     Saves byte string files to their respective paths.
@@ -167,7 +204,6 @@ def save_files_from_bytes(files_as_byte_strings: dict, logger):
     :param logger: Logging instance
     :return: Summary indicating how many files succeeded vs. failed
     """
-    results = {}  # Store results: (destination) -> True (if success) or error message (if failure)
 
     message = "All files saved successfully."
 
@@ -175,13 +211,15 @@ def save_files_from_bytes(files_as_byte_strings: dict, logger):
     for destination, file_bytes in files_as_byte_strings.items():
         try:
             # Write file from byte string
+            if destination.startswith("~"): 
+                destination = os.path.expanduser(destination)
             with open(destination, "+wb") as f:
                 f.write(file_bytes)
-            logger.log_operation(f"File saved", f"File {destination} saved successfully.", params=None, flush_logs=True)
+            logger.log_operation("File saved | ORIGIN: run_main_job function", f"File {destination} saved successfully.", params=None, flush_logs=True)
         
         except Exception as e:
             error_msg = f"Error saving file: {destination}, Error: {str(e)}"
-            logger.log_operation("Error", error_msg, params=None, flush_logs=True)
+            logger.log_operation("Error | ORIGIN: run_main_job function", error_msg, params=None, flush_logs=True)
             print(error_msg)
             message = f"Error saving some files." 
 
@@ -192,19 +230,18 @@ def save_files_from_bytes(files_as_byte_strings: dict, logger):
 # Step 2: Execute Bash Commands
 # -----------------------------------------------------------------------------
 
-def execute_and_log_bash_commands(bash_commands: list[str], logger):
+def execute_and_log_bash_commands(bash_commands: list[str]):
     """
     Executes a list of bash commands locally, logs and returns the output.
 
     :param bash_commands: List of commands to execute
-    :param logger: Logging instance
     :return: A single string containing logs for all commands
     """
     logstring = ""
 
     for cmd in bash_commands:
         logstring += "---------------------------------------------------------\n"
-        logstring += f"Executing Command: {cmd}\n"
+        logstring += f"Executing Command\n"
 
         try:
             # Execute the command and capture both stdout and stderr
@@ -216,19 +253,15 @@ def execute_and_log_bash_commands(bash_commands: list[str], logger):
             if result.returncode == 0:
                 status = "SUCCESS"
                 log_entry = f"Command: {cmd}\nStatus: {status}\nOutput:\n{output}\n"
-                logger.log_operation("Info", log_entry, params=None, flush_logs=True)
             else:
                 status = "FAILURE"
                 log_entry = f"Command: {cmd}\nStatus: {status}\nError Output:\n{error_output}\n"
-                logger.log_operation("Error", log_entry, params=None, flush_logs=True)
 
             logstring += log_entry
             print(log_entry)
 
         except Exception as e:
             logstring += f"Command: {cmd}\nStatus: ERROR\nException: {str(e)}\n"
-            logger.log_operation("Error", f"Command: {cmd} failed with Exception: {str(e)}", 
-                                 params=None, flush_logs=True)
     
     return logstring
 
@@ -243,17 +276,33 @@ def create_workunits_step(token_data, app_data, resource_paths, logger):
 
     :param token_data: dict with token/auth info
     :param app_data: dict with fields like {"id": <app_id>} or other app info
-    :param resource_paths: Dictionary {file_path: container_id}
+    :param resource_paths: Dictionary {file_path or dir_path: container_id}
     :param logger: a logger instance
-    :return: A dictionary mapping file_paths to workunit objects {file_path: workunit}
+    :return: A tuple containing: 
+        A dictionary mapping file_paths to workunit objects {file_path: workunit_id}
+        A dictionary mapping container_ids to workunit objects {container_id: workunit_id}
     """
     app_id = app_data["id"]  # Extract the application ID
 
-    # Extract unique order IDs from resource_paths
-    container_ids = list(set(resource_paths.values()))
+    # Expand any directories into individual files
+    expanded_paths = {}
+
+    for path_str, container_id in resource_paths.items():
+        path = Path(path_str)
+        if path.is_file():
+            expanded_paths[str(path)] = int(container_id)
+        elif path.is_dir():
+            for file in path.rglob("*"): #is a method that returns all files and folders in the directory and its subdirectories
+                if file.is_file():
+                    expanded_paths[str(file)] = int(container_id)
+        else:
+            logger.log_operation("Warning | ORIGIN: run_main_job function", f"Path {path_str} does not exist.", flush_logs=True)
+            print(f"Warning: Path {path_str} does not exist or is not accessible.")
+
+    if not expanded_paths:
+        raise ValueError("No valid file paths found in resource_paths.")
 
-    if not container_ids:
-        raise ValueError("No order IDs found in resource_paths; cannot create workunits.")
+    container_ids = list(set(expanded_paths.values()))
 
     # Create all workunits in one API call
     created_workunits = create_workunits(
@@ -270,20 +319,45 @@ def create_workunits_step(token_data, app_data, resource_paths, logger):
     workunit_map = {
         file_path: wu["id"]
         for wu in created_workunits
-        for file_path, container_id in resource_paths.items()
+        for file_path, container_id in expanded_paths.items()
         if container_id == wu["container"]["id"]
     }
 
-    logger.log_operation("Success", f"Total created Workunits: {list(workunit_map.values())}", params=None, flush_logs=True)
-    print(f"Total created Workunits: {list(workunit_map.values())}")
+    workunit_container_map = {
+        str(wu["container"]["id"]): wu["id"]
+        for wu in created_workunits
+    }
 
+    workunit_ids = [wu.get("id") for wu in created_workunits]
+    logger.log_operation("Success | ORIGIN: run_main_job function", f"Total created Workunits: {workunit_ids}", params=None, flush_logs=True)
+    print(f"Total created Workunits: {workunit_ids}")
     print(workunit_map)
-    return workunit_map  # Returning {file_path: workunit}
+
+    return workunit_map, workunit_container_map  # Returning {file_path: workunit}
 
 
 
 # -----------------------------------------------------------------------------
-# Step 4: Attach Resources in B-Fabric
+# Step 4: Create Dataset in B-Fabric
+# -----------------------------------------------------------------------------
+def create_dataset(token_data, dataset_data): 
+
+    """
+    Creates a dataset in B-Fabric using the provided dataset data.
+    :param dataset_data: Dictionary containing dataset information
+    :param token_data: B-Fabric token data
+    :return: The created dataset object
+    """
+
+    wrapper = get_power_user_wrapper(token_data) 
+    dataset = wrapper.save("dataset", dataset_data)  # Save the dataset
+
+    return dataset[0]
+
+
+
+# -----------------------------------------------------------------------------
+# Step 5: Attach Resources in B-Fabric
 # -----------------------------------------------------------------------------
 
 def attach_resources_to_workunits(token_data, logger, workunit_map):
@@ -297,33 +371,45 @@ def attach_resources_to_workunits(token_data, logger, workunit_map):
     :param workunit_map: Dictionary mapping file_path to workunit_id {file_path: workunit_id}
     """
     if not workunit_map:
-        logger.log_operation("Info", "No workunits found, skipping resource registration.",
+        logger.log_operation("Info | ORIGIN: run_main_job function", "No workunits found, skipping resource registration.",
                              params=None, flush_logs=True)
         print("No workunits found, skipping resource registration.")
         return
-    
+
     print("Workunit Map:", workunit_map)
 
+    # Dictionary to count successfully created resources per workunit
+    # defaultdict(int) automatically starts each new key with a value of 0
+    workunit_resource_count = defaultdict(int)
+
     for file_path, workunit_id in workunit_map.items():
-        print(f"Processing file: {file_path}, Workunit ID: {workunit_id}")  # Corrected print statement
+        print(f"Processing file: {file_path}, Workunit ID: {workunit_id}")
         # Upload the file as a resource
         resource = create_resource(token_data, workunit_id, file_path)
         resource_id = resource.get("id")
         print("Resource ID:", resource_id)
-
         if resource_id:
-            logger.log_operation("Success", f"Resource {resource_id} attached to Workunit {workunit_id}",
-                                 params=None, flush_logs=True)
+            workunit_resource_count[workunit_id] += 1
             print(f"Resource {resource_id} attached to Workunit {workunit_id}")
         else:
-            logger.log_operation("Error", f"Failed to attach resource {file_path} for Workunit {workunit_id}",
+            logger.log_operation("Error | ORIGIN: run_main_job function", f"Failed to attach resource {file_path} for Workunit {workunit_id}",
                                  params=None, flush_logs=True)
             print(f"Failed to attach resource {file_path} for Workunit {workunit_id}")
 
+    # Log a summary per workunit
+    for workunit_id, count in workunit_resource_count.items():
+        logger.log_operation(
+            "Success | ORIGIN: run_main_job function",
+            f"Created {count} resource(s) for Workunit ID {workunit_id}",
+            params=None,
+            flush_logs=True
+        )
+        print(f"Created {count} resource(s) for Workunit ID {workunit_id}")
+
 
 
 # -----------------------------------------------------------------------------
-# Step 5: Attachments of gstore in B-Fabric as a Link
+# Step 6: Attachments of gstore in B-Fabric as a Link
 # -----------------------------------------------------------------------------
 
 def attach_gstore_files_to_entities_as_link(token_data, logger, attachment_paths: dict):
@@ -351,7 +437,7 @@ def attach_gstore_files_to_entities_as_link(token_data, logger, attachment_paths
     # Process each attachment
     for source_path, file_name in attachment_paths.items():
         if not source_path or not file_name:
-            logger.log_operation("Error", f"Missing required attachment details: {source_path} -> {file_name}", params=None, flush_logs=True)
+            logger.log_operation("Error | ORIGIN: run_main_job function", f"Missing required attachment details: {source_path} -> {file_name}", params=None, flush_logs=True)
             print(f"Error: Missing required attachment details: {source_path} -> {file_name}")
             continue
 
@@ -371,21 +457,22 @@ def attach_gstore_files_to_entities_as_link(token_data, logger, attachment_paths
             else:  # We don't have direct access → Send to migration folder first
                 remote_tmp_path = f"{SCRATCH_PATH}/{file_name}"
                 scp_copy(source_path, TRX_LOGIN, TRX_SSH_KEY, remote_tmp_path)
+                print("scp copy done:")
 
                 # Move to final location
                 ssh_move(TRX_LOGIN, TRX_SSH_KEY, remote_tmp_path, final_remote_path)
+                print("ssh move done:")
 
             # Log success
             success_msg = f"Successfully attached '{file_name}' to {entity_class} (ID={entity_id})"
-            logger.log_operation("Success", success_msg, params=None, flush_logs=True)
             print(success_msg)
 
             # Step 3: Create API link
-            create_api_link(token_data, logger, entity_class, entity_id, file_name, entity_folder)
+            create_attachment_link(token_data, logger, entity_class, entity_id, file_name, entity_folder)
 
         except Exception as e:
             error_msg = f"Exception while processing '{file_name}': {e}"
-            logger.log_operation("Error", error_msg, params=None, flush_logs=True)
+            logger.log_operation("Error | ORIGIN: run_main_job function", error_msg, params=None, flush_logs=True)
             print(error_msg)
 
 def local_access(remote_path):
@@ -397,8 +484,11 @@ def local_access(remote_path):
 
 def scp_copy(source_path, ssh_user, ssh_key, remote_path):
     """Copies a file to a remote location using SCP with the correct FGCZ server address."""
+    print("SCP Copying...")
     cmd = ["scp", "-i", ssh_key, source_path, f"{ssh_user}:{remote_path}"]
+    print("SCP Command:")
     subprocess.run(cmd, check=True)
+    print("SCP Command Executed:", cmd)
     print(f"Copied {source_path} to {remote_path}")
 
 
@@ -408,6 +498,9 @@ def ssh_move(ssh_user, ssh_key, remote_tmp_path, final_remote_path):
 
     subprocess.run(cmd, check=True)
     print(f"Moved {remote_tmp_path} to {final_remote_path}")
+    
+    # Wait 10 second before next move
+    time.sleep(10)
 
 
 def g_req_copy(source_path, destination_path):
@@ -417,8 +510,8 @@ def g_req_copy(source_path, destination_path):
     print(f"Copied {source_path} using g-req")
 
 
-def create_api_link(token_data, logger, entity_class, entity_id, file_name, folder_name):
-    """Creates an API link in B-Fabric for the attached file."""
+def create_attachment_link(token_data, logger, entity_class, entity_id, file_name, folder_name):
+    """Creates an attachment link in B-Fabric for the attached file."""
     wrapper = get_power_user_wrapper(token_data)
     url = f"{URL}/{folder_name}/{file_name}"
     timestamped_filename = f"{dt.now().strftime('%Y-%m-%d_%H:%M:%S')}_{file_name}"
@@ -433,14 +526,14 @@ def create_api_link(token_data, logger, entity_class, entity_id, file_name, fold
     try:
         link_result = wrapper.save("link", data)
         if link_result:
-            success_msg = f"API link created for '{file_name}': {url}"
-            logger.log_operation("Success", success_msg, params=None, flush_logs=True)
+            success_msg = f"Attachment link created for '{file_name}': {url}"
+            logger.log_operation("Success | ORIGIN: run_main_job function", success_msg, params=None, flush_logs=True)
             print(success_msg)
         else:
-            raise ValueError("API link creation failed")
+            raise ValueError("Attachment link creation failed")
     except Exception as e:
-        error_msg = f"Failed to create API link for '{file_name}': {e}"
-        logger.log_operation("Error", error_msg, params=None, flush_logs=True)
+        error_msg = f"Failed to create attachment link for '{file_name}': {e}"
+        logger.log_operation("Error | ORIGIN: run_main_job function", error_msg, params=None, flush_logs=True)
         print(error_msg)
 
 

{bfabric_web_apps-0.1.7.dist-info → bfabric_web_apps-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bfabric-web-apps
-Version: 0.1.7
+Version: 0.2.1
 Summary: A package containing handy boilerplate utilities for developing bfabric web-applications
 Author: Marc Zuber, Griffin White, GWC GmbH
 Requires-Python: >=3.10,<4.0

bfabric_web_apps-0.2.1.dist-info/RECORD

@@ -0,0 +1,22 @@
+bfabric_web_apps/__init__.py,sha256=eRYBvXrDM8Bhdm7cbI80L1hjtbRFqUasJiBW52Hx3TI,1715
+bfabric_web_apps/layouts/layouts.py,sha256=z8gL4n4wwLdpLGomO9CftBLnGpc3r6OpmUc2-wBg8uo,14661
+bfabric_web_apps/objects/BfabricInterface.py,sha256=iz3m3SXtUVaEMm0ZRm-2IFePd2UkVu1ob87mq1ogLXA,10751
+bfabric_web_apps/objects/Logger.py,sha256=62LC94xhm7YG5LUw3yH46NqvJQsAX7wnc9D4zbY16rA,5224
+bfabric_web_apps/utils/app_init.py,sha256=RCdpCXp19cF74bouYJLPe-KSETZ0Vwqtd02Ta2VXEF8,428
+bfabric_web_apps/utils/callbacks.py,sha256=tB1xtHl_ePY6KJWNz3erkrZw3HFhRneewGqZm9xIYtI,12687
+bfabric_web_apps/utils/charging.py,sha256=oNNazH59SFkbxJKPvCel0IxdsRHC8xpJ0AXCLvI88FI,1069
+bfabric_web_apps/utils/components.py,sha256=X3NRnv--LsHWMtWL83Pzr2whOZLSEJIwXTklQdAQpZE,984
+bfabric_web_apps/utils/config.py,sha256=F4EExu7EkY7upOnxk6BU6zTLt9eU6_iy2y8esIlxTSc,1209
+bfabric_web_apps/utils/create_app_in_bfabric.py,sha256=Z7puke8QB4SBuDJ9x3_OjgApzovKu0Nt1g8EqkOHJpc,2758
+bfabric_web_apps/utils/dataset_utils.py,sha256=p_UtoOl1kJpSm2BGdg31Ji0C7ctst40wp4LX1tUe4tI,3360
+bfabric_web_apps/utils/get_logger.py,sha256=0Y3SrXW93--eglS0_ZOc34NOriAt6buFPik5n0ltzRA,434
+bfabric_web_apps/utils/get_power_user_wrapper.py,sha256=T33z64XjmJ0KSlmfEmrEP8eYpbpINCVD6Xld_V7PR2g,1027
+bfabric_web_apps/utils/redis_connection.py,sha256=qXSPxW6m55Ogv44BhmPCl9ACuvzmpfZNU73UJhHRXL4,133
+bfabric_web_apps/utils/redis_queue.py,sha256=MCx7z_I2NusJ4P42mcLvV7STtXBFMIIvun83fM8zOGI,168
+bfabric_web_apps/utils/redis_worker_init.py,sha256=wtjQL48PLNXD1s-5s3Oq5EC8BmcfKcd7IhUbTH_EYz8,1014
+bfabric_web_apps/utils/resource_utilities.py,sha256=N4EiUkxXHZ18jnU2OuRqaGSroCZ73Ogb9lkeA21Kvq4,5716
+bfabric_web_apps/utils/run_main_pipeline.py,sha256=-5o2QZW27OIvPJ_BtJ3s5UXoONOUcgUeXH4RSTN8oZg,23219
+bfabric_web_apps-0.2.1.dist-info/LICENSE,sha256=k0O_i2k13i9e35aO-j7FerJafAqzzu8x0kkBs0OWF3c,1065
+bfabric_web_apps-0.2.1.dist-info/METADATA,sha256=abK_cUI5x_js8GFj0gAHqo7pw7G98YW-IygicddE78o,687
+bfabric_web_apps-0.2.1.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+bfabric_web_apps-0.2.1.dist-info/RECORD,,

bfabric_web_apps-0.1.7.dist-info/RECORD

@@ -1,21 +0,0 @@
-bfabric_web_apps/__init__.py,sha256=9OSketawJ_bkCbCN0Q3KaM6_u1_zyE74oC7jtrznFP8,1503
-bfabric_web_apps/layouts/layouts.py,sha256=z8gL4n4wwLdpLGomO9CftBLnGpc3r6OpmUc2-wBg8uo,14661
-bfabric_web_apps/objects/BfabricInterface.py,sha256=2BNskMzV5K1a-tXFHQubcBk7Rt-8g9du7mNAJrGohMY,10170
-bfabric_web_apps/objects/Logger.py,sha256=62LC94xhm7YG5LUw3yH46NqvJQsAX7wnc9D4zbY16rA,5224
-bfabric_web_apps/utils/app_init.py,sha256=RCdpCXp19cF74bouYJLPe-KSETZ0Vwqtd02Ta2VXEF8,428
-bfabric_web_apps/utils/callbacks.py,sha256=tB1xtHl_ePY6KJWNz3erkrZw3HFhRneewGqZm9xIYtI,12687
-bfabric_web_apps/utils/charging.py,sha256=oNNazH59SFkbxJKPvCel0IxdsRHC8xpJ0AXCLvI88FI,1069
-bfabric_web_apps/utils/components.py,sha256=X3NRnv--LsHWMtWL83Pzr2whOZLSEJIwXTklQdAQpZE,984
-bfabric_web_apps/utils/config.py,sha256=vJzhmc6ooFb46MM1Eg3m8gNrM4fJa-l5Tao2Py-SF_I,1115
-bfabric_web_apps/utils/create_app_in_bfabric.py,sha256=eVk3cQDXxW-yo9b9n_zzGO6kLg_SLxYbIDECyvEPJXU,2752
-bfabric_web_apps/utils/get_logger.py,sha256=0Y3SrXW93--eglS0_ZOc34NOriAt6buFPik5n0ltzRA,434
-bfabric_web_apps/utils/get_power_user_wrapper.py,sha256=T33z64XjmJ0KSlmfEmrEP8eYpbpINCVD6Xld_V7PR2g,1027
-bfabric_web_apps/utils/redis_connection.py,sha256=qXSPxW6m55Ogv44BhmPCl9ACuvzmpfZNU73UJhHRXL4,133
-bfabric_web_apps/utils/redis_queue.py,sha256=MCx7z_I2NusJ4P42mcLvV7STtXBFMIIvun83fM8zOGI,168
-bfabric_web_apps/utils/redis_worker_init.py,sha256=9SUc9bbgBeMbUdqJD9EkWPA4wcJjvyX6Tzanv5JfqEg,691
-bfabric_web_apps/utils/resource_utilities.py,sha256=4LnV_eQjKkcpZJBsWFx--dmASyE7jfJfktk2hdHn5Fk,5856
-bfabric_web_apps/utils/run_main_pipeline.py,sha256=RG-Jb3-O1Ok8L0i1gddWVeAEqJT6REITtcD9XDoC_Dc,18766
-bfabric_web_apps-0.1.7.dist-info/LICENSE,sha256=k0O_i2k13i9e35aO-j7FerJafAqzzu8x0kkBs0OWF3c,1065
-bfabric_web_apps-0.1.7.dist-info/METADATA,sha256=Jf3A0eVEOXvoMzFJCmxpWnoGvKNnHd2ckHg4o9w6nm0,687
-bfabric_web_apps-0.1.7.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-bfabric_web_apps-0.1.7.dist-info/RECORD,,