sl-shared-assets 3.0.0rc14__py3-none-any.whl → 3.1.0__py3-none-any.whl

sl_shared_assets/tools/project_management_tools.py

@@ -11,15 +11,15 @@ from ataraxis_base_utilities import console
 
 from ..data_classes import (
     SessionData,
+    SessionTypes,
     ProcessingTracker,
     RunTrainingDescriptor,
     LickTrainingDescriptor,
+    WindowCheckingDescriptor,
     MesoscopeExperimentDescriptor,
 )
 from .packaging_tools import calculate_directory_checksum
 
-_valid_session_types = {"lick training", "run training", "mesoscope experiment", "window checking"}
-
 
 class ProjectManifest:
     """Wraps the contents of a Sun lab project manifest .feather file and exposes methods for visualizing and
@@ -220,8 +220,7 @@ class ProjectManifest:
 
         Returns:
             A Polars DataFrame with the following columns: 'animal', 'date', 'notes', 'session', 'type', 'complete',
-            'intensity_verification', 'suite2p', 'behavior', 'video',
-            'dataset'.
+            'intensity_verification', 'suite2p', 'behavior', 'video', 'dataset'.
         """
 
         df = self._data
@@ -330,23 +329,31 @@ def generate_project_manifest(
 
         # Depending on the session type, instantiates the appropriate descriptor instance and uses it to read the
         # experimenter notes
-        if session_data.session_type == "lick training":
+        if session_data.session_type == SessionTypes.LICK_TRAINING:
             descriptor: LickTrainingDescriptor = LickTrainingDescriptor.from_yaml(  # type: ignore
                 file_path=session_data.raw_data.session_descriptor_path
             )
             manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == "run training":
+        elif session_data.session_type == SessionTypes.RUN_TRAINING:
             descriptor: RunTrainingDescriptor = RunTrainingDescriptor.from_yaml(  # type: ignore
                 file_path=session_data.raw_data.session_descriptor_path
             )
             manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == "mesoscope experiment":
+        elif session_data.session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
             descriptor: MesoscopeExperimentDescriptor = MesoscopeExperimentDescriptor.from_yaml(  # type: ignore
                 file_path=session_data.raw_data.session_descriptor_path
             )
             manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == "window checking":
-            manifest["notes"].append("N/A")
+        elif session_data.session_type == SessionTypes.WINDOW_CHECKING:
+            # sl-experiment version 3.0.0 added session descriptors to Window Checking runtimes. Since the file does not
+            # exist in prior versions, this section is written to statically handle the discrepancy.
+            try:
+                descriptor: WindowCheckingDescriptor = WindowCheckingDescriptor.from_yaml(  # type: ignore
+                    file_path=session_data.raw_data.session_descriptor_path
+                )
+                manifest["notes"].append(descriptor.experimenter_notes)
+            except Exception:
+                manifest["notes"].append("N/A")
 
         # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
         manifest["complete"].append(session_data.raw_data.telomere_path.exists())
@@ -377,9 +384,7 @@ def generate_project_manifest(
         tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
         manifest["video"].append(tracker.is_complete)
 
-        # Tracks whether the session's data is ready for dataset integration. To be considered ready, the data must be
-        # successfully processed with all relevant pipelines. Any session currently being processed with any processing
-        # pipeline is considered NOT ready.
+        # Tracks whether the session's data is currently in the processing or dataset integration mode.
         manifest["dataset"].append(session_data.processed_data.p53_path.exists())
 
     # If all animal IDs are integer-convertible, stores them as numbers to promote proper sorting. Otherwise, stores
@@ -419,7 +424,10 @@ def generate_project_manifest(
 
 
 def verify_session_checksum(
-    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+    session_path: Path,
+    create_processed_data_directory: bool = True,
+    processed_data_root: None | Path = None,
+    update_manifest: bool = False,
 ) -> None:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
@@ -435,6 +443,9 @@ def verify_session_checksum(
         This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
         part of the data preprocessing runtime performed by a data acquisition system.
 
+        Since version 3.1.0, this functon also supports (re) generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
+
     Args:
         session_path: The path to the session directory to be verified. Note, the input session directory must contain
             the 'raw_data' subdirectory.
@@ -442,6 +453,9 @@ def verify_session_checksum(
         processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """
 
     # Loads session data layout. If configured to do so, also creates the processed data hierarchy
@@ -487,12 +501,33 @@ def verify_session_checksum(
         if tracker.is_running:
             tracker.error()
 
+        # If the runtime is configured to generate the project manifest file, attempts to generate and overwrite the
+        # existing manifest file for the target project.
+        if update_manifest:
+            # All sessions are stored under root/project/animal/session. Therefore, the grandparent of the session is
+            # the raw project directory.
+            raw_directory = session_path.parents[1]
+
+            # Depending on the processed_data_root configuration, determines the path for the project's processed
+            # data directory.
+            processed_directory: Path | None = None
+            if processed_data_root is not None:
+                processed_directory = processed_data_root.joinpath(session_data.project_name)
+
+            # Generates the manifest file inside the root raw data project directory
+            generate_project_manifest(
+                raw_project_directory=session_path.parents[1],
+                processed_project_directory=processed_directory,
+                output_directory=raw_directory,
+            )
+
 
 def resolve_p53_marker(
     session_path: Path,
     create_processed_data_directory: bool = True,
     processed_data_root: None | Path = None,
     remove: bool = False,
+    update_manifest: bool = False,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker file for the target session.
 
@@ -504,11 +539,12 @@ def resolve_p53_marker(
         from altering the data while it is integrated into a dataset. The p53.bin marker solves this issue by ensuring
         that only one type of runtimes (processing or dataset integration) is allowed to work with the session.
 
-        For the p53.bin marker to be created, the session must currently not undergo any processing and must be
-        successfully processed with the minimal set of pipelines for its session type. Removing the p53.bin marker does
-        not have any dependencies and will be executed even if the session is currently undergoing dataset integration.
-        Due to this limitation, it is only possible to call this function with the 'remove' flag manually (via the
-        dedicated CLI).
+        For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
+        p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab compute server.
+
+        Since version 3.1.0, this functon also supports (re)generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
 
     Args:
         session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
@@ -518,6 +554,9 @@ def resolve_p53_marker(
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
         remove: Determines whether this function is called to create or remove the p53.bin marker.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """
 
     # Loads session data layout. If configured to do so, also creates the processed data hierarchy
@@ -528,7 +567,7 @@ def resolve_p53_marker(
     )
 
     # If the p53.bin marker exists and the runtime is configured to remove it, removes the marker file. If the runtime
-    # is configured to create the marker, aborts the runtime (as the marker already exists).
+    # is configured to create the marker, the method aborts the runtime (as the marker already exists).
     if session_data.processed_data.p53_path.exists():
         if remove:
             session_data.processed_data.p53_path.unlink()
@@ -544,43 +583,50 @@ def resolve_p53_marker(
     # Queries the type of the processed session
     session_type = session_data.session_type
 
-    # If the session type is not supported, aborts with an error
-    if session_type not in _valid_session_types:
-        message = (
-            f"Unable to determine the mandatory processing pipelines for session {session_data.session_name} of animal "
-            f"{session_data.animal_id} and project {session_data.processed_data}. The type of the session "
-            f"{session_type} is not one of the supported session types: {', '.join(_valid_session_types)}."
-        )
-        console.error(message=message, error=ValueError)
-
-    # Window checking sessions are not designed to be integrated into datasets, so they cannot be marked with p53.bin
-    # file. Similarly, any incomplete session is automatically excluded from dataset formation.
-    if session_type == "window checking" or not session_data.raw_data.telomere_path.exists():
+    # Window checking sessions are not designed to be integrated into datasets, so they cannot be marked with the
+    # p53.bin file. Similarly, any incomplete session is automatically excluded from dataset formation.
+    if session_type == SessionTypes.WINDOW_CHECKING or not session_data.raw_data.telomere_path.exists():
         return
 
     # Training sessions collect similar data and share processing pipeline requirements
-    if session_type == "lick training" or session_type == "run training":
-        # If the session has not been successfully processed with the behavior processing pipeline, aborts without
-        # creating the marker file. Also ensures that the video tracking pipeline is not actively running, although it
-        # is not required
+    if session_type == SessionTypes.LICK_TRAINING or session_type == SessionTypes.RUN_TRAINING:
+        # Ensures that the session is not being processed with one of the supported pipelines.
         behavior_tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
         video_tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
-        if not behavior_tracker.is_complete or video_tracker.is_running:
+        if behavior_tracker.is_running or video_tracker.is_running:
             # Note, training runtimes do not require suite2p processing.
             return
 
     # Mesoscope experiment sessions require additional processing with suite2p
-    if session_type == "mesoscope experiment":
+    if session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
         behavior_tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
         suite2p_tracker = ProcessingTracker(file_path=session_data.processed_data.suite2p_processing_tracker_path)
         video_tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
 
-        # Similar to above, if the session is not processed with the behavior pipeline or the suite2p pipeline, aborts
-        # without creating the marker file. Video tracker is not required for p53 marker creation, but the video
-        # tracking pipeline must not be actively running.
-        if not behavior_tracker.is_complete or not suite2p_tracker.is_complete or video_tracker.is_running:
+        # Similar to the above, ensures that the session is not being processed with one of the supported pipelines.
+        if behavior_tracker.is_running or suite2p_tracker.is_running or video_tracker.is_running:
             return
 
     # If the runtime reached this point, the session is eligible for dataset integration. Creates the p53.bin marker
     # file, preventing the session from being processed again as long as the marker exists.
     session_data.processed_data.p53_path.touch()
+
+    # If the runtime is configured to generate the project manifest file, attempts to generate and overwrite the
+    # existing manifest file for the target project.
+    if update_manifest:
+        # All sessions are stored under root/project/animal/session. Therefore, the grandparent of the session is
+        # the raw project directory.
+        raw_directory = session_path.parents[1]
+
+        # Depending on the processed_data_root configuration, determines the path for the project's processed
+        # data directory.
+        processed_directory: Path | None = None
+        if processed_data_root is not None:
+            processed_directory = processed_data_root.joinpath(session_data.project_name)
+
+        # Generates the manifest file inside the root raw data project directory
+        generate_project_manifest(
+            raw_project_directory=session_path.parents[1],
+            processed_project_directory=processed_directory,
+            output_directory=raw_directory,
+        )

sl_shared_assets/tools/project_management_tools.pyi

@@ -1,19 +1,18 @@
 from pathlib import Path
 
 import polars as pl
-from _typeshed import Incomplete
 
 from ..data_classes import (
     SessionData as SessionData,
+    SessionTypes as SessionTypes,
     ProcessingTracker as ProcessingTracker,
     RunTrainingDescriptor as RunTrainingDescriptor,
     LickTrainingDescriptor as LickTrainingDescriptor,
+    WindowCheckingDescriptor as WindowCheckingDescriptor,
     MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
 )
 from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
 
-_valid_session_types: Incomplete
-
 class ProjectManifest:
     """Wraps the contents of a Sun lab project manifest .feather file and exposes methods for visualizing and
     working with the data stored inside the file.
@@ -104,8 +103,7 @@ class ProjectManifest:
 
         Returns:
             A Polars DataFrame with the following columns: 'animal', 'date', 'notes', 'session', 'type', 'complete',
-            'intensity_verification', 'suite2p', 'behavior', 'video',
-            'dataset'.
+            'intensity_verification', 'suite2p', 'behavior', 'video', 'dataset'.
         """
 
 def generate_project_manifest(
@@ -133,7 +131,10 @@ def generate_project_manifest(
     """
 
 def verify_session_checksum(
-    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+    session_path: Path,
+    create_processed_data_directory: bool = True,
+    processed_data_root: None | Path = None,
+    update_manifest: bool = False,
 ) -> None:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
@@ -149,6 +150,9 @@ def verify_session_checksum(
         This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
         part of the data preprocessing runtime performed by a data acquisition system.
 
+        Since version 3.1.0, this functon also supports (re) generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
+
     Args:
         session_path: The path to the session directory to be verified. Note, the input session directory must contain
             the 'raw_data' subdirectory.
@@ -156,6 +160,9 @@ def verify_session_checksum(
         processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """
 
 def resolve_p53_marker(
@@ -163,6 +170,7 @@ def resolve_p53_marker(
     create_processed_data_directory: bool = True,
     processed_data_root: None | Path = None,
     remove: bool = False,
+    update_manifest: bool = False,
 ) -> None:
     """Depending on configuration, either creates or removes the p53.bin marker file for the target session.
 
@@ -174,11 +182,12 @@ def resolve_p53_marker(
         from altering the data while it is integrated into a dataset. The p53.bin marker solves this issue by ensuring
         that only one type of runtimes (processing or dataset integration) is allowed to work with the session.
 
-        For the p53.bin marker to be created, the session must currently not undergo any processing and must be
-        successfully processed with the minimal set of pipelines for its session type. Removing the p53.bin marker does
-        not have any dependencies and will be executed even if the session is currently undergoing dataset integration.
-        Due to this limitation, it is only possible to call this function with the 'remove' flag manually (via the
-        dedicated CLI).
+        For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
+        p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab compute server.
+
+        Since version 3.1.0, this functon also supports (re)generating the processed session's project manifest file,
+        which is used to support further Sun lab data processing pipelines.
 
     Args:
         session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
@@ -188,4 +197,7 @@ def resolve_p53_marker(
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
         remove: Determines whether this function is called to create or remove the p53.bin marker.
+        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
+            project. This should always be enabled when working with remote compute server(s) to ensure that the
+            project manifest file contains the most actual snapshot of the project's state.
     """

sl_shared_assets/tools/transfer_tools.py

@@ -45,7 +45,7 @@ def transfer_directory(source: Path, destination: Path, num_threads: int = 1, ve
         done before copying the files.
 
         The method executes a multithreading copy operation. It does not clean up the source files. That job is handed
-        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that calls this function.
+        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that call this function.
 
         If the method is configured to verify transferred file integrity, it reruns the xxHash3-128 checksum calculation
         and compares the returned checksum to the one stored in the source directory. The method assumes that all input

sl_shared_assets/tools/transfer_tools.pyi

@@ -30,7 +30,7 @@ def transfer_directory(source: Path, destination: Path, num_threads: int = 1, ve
         done before copying the files.
 
         The method executes a multithreading copy operation. It does not clean up the source files. That job is handed
-        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that calls this function.
+        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that call this function.
 
         If the method is configured to verify transferred file integrity, it reruns the xxHash3-128 checksum calculation
         and compares the returned checksum to the one stored in the source directory. The method assumes that all input

{sl_shared_assets-3.0.0rc14.dist-info → sl_shared_assets-3.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 3.0.0rc14
+Version: 3.1.0
 Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
@@ -772,6 +772,7 @@ A Python library that stores assets shared between multiple Sun (NeuroAI) lab da
 ![PyPI - License](https://img.shields.io/pypi/l/sl-shared-assets)
 ![PyPI - Status](https://img.shields.io/pypi/status/sl-shared-assets)
 ![PyPI - Wheel](https://img.shields.io/pypi/wheel/sl-shared-assets)
+
 ___
 
 ## Detailed Description
@@ -813,7 +814,7 @@ All software library dependencies are installed automatically as part of library
 
 Note, installation from source is ***highly discouraged*** for everyone who is not an active project developer.
 
-1. Download this repository to your local machine using your preferred method, such as Git-cloning. Use one
+1. Download this repository to your local machine using any method, such as Git-cloning. Use one
    of the stable releases from [GitHub](https://github.com/Sun-Lab-NBB/sl-shared-assets/releases).
 2. Unpack the downloaded zip and note the path to the binary wheel (`.whl`) file contained in the archive.
 3. Run ```python -m pip install WHEEL_PATH```, replacing 'WHEEL_PATH' with the path to the wheel file, to install the 
@@ -826,12 +827,128 @@ Use the following command to install the library using pip: ```pip install sl-sh
 
 ## Usage
 
-All library components are intended to be used via other Sun lab libraries. Developers should study the API and CLI 
+Most library components are intended to be used via other Sun lab libraries. Developers should study the API and CLI 
 documentation below to learn how to use library components in other Sun lab libraries. For notes on using shared 
 assets for data acquisition, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment) library ReadMe.
 For notes on using shared assets for data processing, see the [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) 
 library ReadMe.
 
+The only exception to the note above is the **server.py** package exposed by this library. This package exposes an API 
+for running headless and a CLI for running interactive Simple Linux Utility for Resource Management (SLURM)-managed 
+jobs on remote compute clusters.
+
+### Generating Access Credentials
+
+To access any remote server, the user is required to first generate the access credentials. The credentials are stored 
+inside the 'server_credentials.yaml' file, which is generated by using the `sl-create-server-credentials` command.
+**Note!** Users are advised to generate this file in a secure (non-shared) location on their local machine.
+
+### Running Headless Jobs
+
+A headless job is a job that does not require any user interaction during runtime. Currently, all headless jobs in the 
+lab rely on pip-installable packages that expose a callable Command-Line Interface to carry out some type of
+data processing. In this regard, **running a headless job is equivalent to calling a CLI command on your local 
+machine**, except that the command is executed on a remote compute server. Therefore, the primary purpose of the API 
+exposed by this library is to transfer the target command request to the remote server, execute it, and monitor the 
+runtime status until it is complete.
+
+For example, the [sl-suite2p package](https://github.com/Sun-Lab-NBB/suite2p) maintained in the lab exposes a CLI to 
+process 2-Photon data from experiment sessions. During data processing by the 
+[sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library, a remote job is sent to the server that uses the CLI 
+exposed by the sl-suite2p package to process target session(s).
+
+### Creating Jobs
+All remote jobs are sent to the server in the form of an executable *shell* (.sh) script. The script is composed on the 
+local machine that uses this library and transferred to a temporary server directory using Secure Shell File 
+Transfer Protocol (SFTP). The server is then instructed to evaluate (run) the script using SLURM job manager, via a 
+Secure Shell (SSH) session.
+
+Broadly, each job consists of three major steps, which correspond to three major sections of the job shell script:
+1. **Setting up the job environment**. Each job script starts with a SLURM job parameter block, which tells SLURM 
+   what resources (CPUs, GPUs, RAM, etc.) the job requires. When resources become available, SLURM generates a virtual
+   environment and runs the rest of the job script in that environment. This forms the basis for using the shared
+   compute resources fairly, as SLURM balances resource allocation and the order of job execution for all users.
+2. **Activating the target conda environment**. Currently, all jobs are assumed to use Python libraries to execute the 
+   intended data processing. Similar to processing data locally, each job expects the remote server to provide a 
+   Conda environment preconfigured with necessary assets (packages) to run the job. Therefore, each job contains a 
+   section that activates the user-defined conda environment before running the rest of the job.
+3. **Executing processing**. The final section is typically unique to each job and calls specific CLI commands or runs 
+   specific Python modules. Since each job is submitted as a shell script, it can do anything a server shell can
+   do. Therefore, despite python-centric approach to data processing in the lab, a remote job composed via this library 
+   can execute ***any*** arbitrary command available to the user on the remove server.
+
+Use the *Job* class exposed by this library to compose remote jobs. **Steps 1 and 2** of each job are configured when 
+initializing the Job instance, while **step 3** is added via the `add_command()` method of the Job class:
+```
+# First, import the job class
+from pathlib import Path
+from sl_shared_assets import Job
+
+# Next, instantiate a new Job object. For example, this job is used to verify the integrity of raw experiment data as
+# it is transferred to the long-term storage destination (server) by the sl-experiment library.
+job = Job(
+    job_name="data_integrity_verification",
+    output_log=Path("/temp/output.txt"),
+    error_log=Path("/temp/errors.txt"),
+    working_directory=Path("/temp/test_job"),
+    conda_environment="test_environment",
+    cpus_to_use=20,
+    ram_gb=50,
+    time_limit=20,
+)
+
+# Finally, add a CLI command call (the actual work to be done by the job). Here, the job calls the
+# 'sl-verify-session' command exposed by the sl-shared-assets library installed in the target environment on the server.
+# Use this method to add commands as you would type them in your local terminal / shell / command line.
+job.add_command(f"sl-verify-session -sp /temp/test_session")
+```
+
+### Submitting and Monitoring Jobs:
+To submit the job to the remote server, use a **Server** class instance. This class encapsulates access to the target 
+remote compute server and uses the server_credentials.yaml file to determine server access credentials (see above):
+```
+# Initialize the Server class using precreated server credentials file
+server = Server(credentials_path=Path("/temp/server_credentials.yaml"))
+
+# Submit the job (generated in the previous code snippet) to the server
+job = server.submit_job(job)
+
+# Wait for the server to complete the job
+delay_timer = PrecisionTimer("s")
+while not server.job_complete(job=job):
+    delay_timer.delay_noblock(delay=5, allow_sleep=True)
+```
+
+**Note!** The Server class only checks whether the job is running on the server, but not the outcome of the job. For 
+that, you can either manually check the output and error logs for the job or come up with a programmatic way of 
+checking the outcome. All developers are highly advised to study the API documentation for the Job and Server classes 
+to use them most effectively.
+
+**Critical!** Since running remote jobs is largely equivalent to executing them locally, all users are highly encouraged
+to test their job scripts locally before deploying them server-side. If a script works on a local machine, it is likely 
+that the script would behave similarly and work on the server.
+
+### Interactive Jobs
+
+Interactive jobs are a special extension of the headless job type discussed above. Specifically, an interactive job is 
+a headless job, whose only purpose is to **create and maintain a Jupyter lab server** under the SLURM control. 
+Specifically, it requests SLURM to set up an isolated environment, starts a Jupyter server in that environment, and 
+sends the credentials for the started server back to the user.
+
+In essence, this allocates a set of resources the user can use interactively by running various Jupyter notebooks. 
+While convenient for certain data analysis cases, this type of jobs has the potential to inefficiently hog server 
+resources for prolonged periods of time. Therefore, users are encouraged to only resort to this type of jobs when 
+strictly necessary and to minimize the resources and time allocated to running these jobs.
+
+To run an interactive job, call the `sl-start-jupyter` CLI command exposed by this library and follow the instructions 
+printed to the terminal by the command during runtime.
+
+**Critical!** While this command tries to minimize collisions with other users, it is possible that an access port 
+collision occurs when multiple users try to instantiate a jupyter server at the same time. If you cannot authenticate
+with the Jupyter server, this likely indicates that the target port was in use and Jupyter automatically incremented the
+port number by 1. In this case, add 1 to your port number and try connecting to that port using the Jupyter credentials 
+provided by the command. For example, if your target port was '8888,' try port '8889.'
+
 ---
 
 ## API Documentation
@@ -847,7 +964,7 @@ ___
 
 ## Versioning
 
-We use [semantic versioning](https://semver.org/) for this project. For the versions available, see the 
+This project uses [semantic versioning](https://semver.org/). For the versions available, see the 
 [tags on this repository](https://github.com/Sun-Lab-NBB/sl-shared-assets/tags).
 
 ---
@@ -870,7 +987,7 @@ ___
 
 - All Sun lab [members](https://neuroai.github.io/sunlab/people) for providing the inspiration and comments during the
   development of this library.
-- The creators of all other projects used in our development automation pipelines and source code 
+- The creators of all other projects used in the development automation pipelines and source code of this project
   [see pyproject.toml](pyproject.toml).
 
 ---

sl_shared_assets-3.1.0.dist-info/RECORD

@@ -0,0 +1,36 @@
+sl_shared_assets/__init__.py,sha256=ybThh0XDtijjwahKkSEnnQ44rxrN2SVyjB5dHaXts0E,2391
+sl_shared_assets/__init__.pyi,sha256=Cb-umRqvnynk2udbgqAJ6h5_tiJyvVtWmx0kLKrL2Yg,2678
+sl_shared_assets/cli.py,sha256=OIwXf6pNPnzqzUPL7mSmEw17KIa3yAOpP0Mpo1Zpf88,19087
+sl_shared_assets/cli.pyi,sha256=5hEbOnYaH4q5qdqJ-zhM9-ElzgcaBeMAX34tuHaUDos,5328
+sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sl_shared_assets/data_classes/__init__.py,sha256=mP__bBIIjMf0EETM4PgQzKy1ZKsjp6paRPNDWWbPRV4,1962
+sl_shared_assets/data_classes/__init__.pyi,sha256=J7ZCH9qQ4qz-3Wq9ILdihlmK9zFR3iU1cpLcSaN45Y8,2238
+sl_shared_assets/data_classes/configuration_data.py,sha256=npMYu9WQMJnoanzC2_5rheDbWS-r4SWAyQZT5Tw1AYQ,36359
+sl_shared_assets/data_classes/configuration_data.pyi,sha256=1_kmBDPGkHmVwXEGYR_3uERBSsenQOTuquMBtjKVTA8,11068
+sl_shared_assets/data_classes/runtime_data.py,sha256=kmXTUk5rDAJBN3XrYLYgusJRfVJ5WiBBk0RPNiSk2pE,16725
+sl_shared_assets/data_classes/runtime_data.pyi,sha256=Hyc-dBePM0xIGgkSIoKmwwUUmdOokm1LUwy1OHHalyU,6771
+sl_shared_assets/data_classes/session_data.py,sha256=YKXako1sNB87LDkGEXx9WZFs6lG3aD619lga5g4L4Ks,49172
+sl_shared_assets/data_classes/session_data.pyi,sha256=CgB4nIDBl4bY1JvcIILfFTlos3ukl3WK4AOaku4CL3Y,15959
+sl_shared_assets/data_classes/surgery_data.py,sha256=5B1OPKFq4bnzbAoe-_c5dFV3kbSD5YFzXbX2zXmfGs8,7485
+sl_shared_assets/data_classes/surgery_data.pyi,sha256=rf59lJ3tGSYKHQlEGXg75MnjajBwl0DYhL4TClAO4SM,2605
+sl_shared_assets/server/__init__.py,sha256=GOQ7wWjiS5Xg_WgTqeEqCTRF9ms9GXx0nffCr-BmKsA,453
+sl_shared_assets/server/__init__.pyi,sha256=Zc12G90fZdgEMwaVZbFzrRVV1wH_LEj3sxaV3lhk1Cw,316
+sl_shared_assets/server/job.py,sha256=wZbppMrv6fqch79bKLjOGQ9AYfjiDKDnTyUe7xgAT44,19461
+sl_shared_assets/server/job.pyi,sha256=wop4ulVY2u6eb3twajeA9MS0EAtNb89aA56pPoGF1Xc,11673
+sl_shared_assets/server/server.py,sha256=oEwdXisyel72Hdk7ZpEwTPq3Lu64UbQWfGHArV8Y6nI,32978
+sl_shared_assets/server/server.pyi,sha256=84XFtqU9fYbxu6Ldf-OMB2nFe6wdGneZM1MFtR9rz4s,15133
+sl_shared_assets/tools/__init__.py,sha256=i-oUVw_un3lzyyII4Sc75s4BnUfZh_aUbQe6dP2Vrbc,743
+sl_shared_assets/tools/__init__.pyi,sha256=pi-5AJyQYeuqIFGWpJ_HhUpXLq6P_nItIqDhsdaIJFU,686
+sl_shared_assets/tools/ascension_tools.py,sha256=xI-hrkR9NIgb7lyhj-ntc8tCYQvDEv6YgYJXl1yvxCs,14639
+sl_shared_assets/tools/ascension_tools.pyi,sha256=fs5j7nbnZ4WpgK8D75A7WJcvFMwK_MUO9ULIYo1YkGo,3739
+sl_shared_assets/tools/packaging_tools.py,sha256=VxQoluGPDUWjPj1ftEt2dvUcdmj0g7T1frGZhZPM8NE,7541
+sl_shared_assets/tools/packaging_tools.pyi,sha256=vgGbAQCExwg-0A5F72MzEhzHxu97Nqg1yuz-5P89ycU,3118
+sl_shared_assets/tools/project_management_tools.py,sha256=vutKi0pdQn5dxBk1OcxPB4XspzQyJwzerNhGi4Vg4iw,31935
+sl_shared_assets/tools/project_management_tools.pyi,sha256=hdn0U9e3_j9McJH75Dzoas-FxcB9nVCTHEFHPofdLtg,11361
+sl_shared_assets/tools/transfer_tools.py,sha256=vqYO4sERZV0W1DFNFnTpJA6QBZ4QJA94a2TyUhZW2Qk,6605
+sl_shared_assets/tools/transfer_tools.pyi,sha256=WtUGfaKV9FP_CnhBg_UvclpuDvOlEESOSMlEDtWpOLg,3293
+sl_shared_assets-3.1.0.dist-info/METADATA,sha256=SbnWSGHffTfwIaQGGP04zsSZ-T2yFga1jL79eLLoib8,56944
+sl_shared_assets-3.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-3.1.0.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
+sl_shared_assets-3.1.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-3.1.0.dist-info/RECORD,,

sl_shared_assets-3.0.0rc14.dist-info/RECORD

@@ -1,36 +0,0 @@
-sl_shared_assets/__init__.py,sha256=rCu1VYs2Lc1l0jqHO3UtfuymU0uY2ccxEn4UyscIut8,2347
-sl_shared_assets/__init__.pyi,sha256=WCWIS-I3ToP4XybNZAi3fA7j2CZ48dl9D-fmd7oZKCo,2615
-sl_shared_assets/cli.py,sha256=R_h_Dlla48mG1LpFDDE9flZ_NyDC9UguRUAYZL6gA9s,18383
-sl_shared_assets/cli.pyi,sha256=8ZJK56_jh2QlF3XCN6c7fI6Z022XtehB0eCrQDJbAsU,5515
-sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sl_shared_assets/data_classes/__init__.py,sha256=bdm0hyQpNF0RL2SPhUgaOz33FsRzpM2L_z5-91HyZBE,1998
-sl_shared_assets/data_classes/__init__.pyi,sha256=J7ZCH9qQ4qz-3Wq9ILdihlmK9zFR3iU1cpLcSaN45Y8,2238
-sl_shared_assets/data_classes/configuration_data.py,sha256=HAdYUVy_8gg_ECN56RAEih9UB7DoZ4J3S5a8ky8o4dQ,36076
-sl_shared_assets/data_classes/configuration_data.pyi,sha256=11Q6OJjN6PdrlZwVPYCL4TEGxkbKL7mGb2S1sSMKsqs,11027
-sl_shared_assets/data_classes/runtime_data.py,sha256=MLIef6s9n2gG6sbp197gpFfzb05e_8vwVzyS_oSmXYQ,16722
-sl_shared_assets/data_classes/runtime_data.pyi,sha256=LzNuEWu-GlPGdyyi8Hn2OFUjGCWOaOplKsRQBbjn2vQ,6768
-sl_shared_assets/data_classes/session_data.py,sha256=R1xYEEKL6DzqyURWRZXNN37uyu8C7L77ECoMfMdh8bM,48237
-sl_shared_assets/data_classes/session_data.pyi,sha256=g53jIe-v8VkQJHc7ITS0KBGRhzn6LOIb6f96SEbEGig,15898
-sl_shared_assets/data_classes/surgery_data.py,sha256=5B1OPKFq4bnzbAoe-_c5dFV3kbSD5YFzXbX2zXmfGs8,7485
-sl_shared_assets/data_classes/surgery_data.pyi,sha256=rf59lJ3tGSYKHQlEGXg75MnjajBwl0DYhL4TClAO4SM,2605
-sl_shared_assets/server/__init__.py,sha256=w7y73RXXjBrWQsjU5g1QNCv_gsXDYnHos3NpOoR2AHA,452
-sl_shared_assets/server/__init__.pyi,sha256=Zc12G90fZdgEMwaVZbFzrRVV1wH_LEj3sxaV3lhk1Cw,316
-sl_shared_assets/server/job.py,sha256=DnEVIswZXm9queBgy6MlpIrCosXvQ_tweOeko7LN9yc,19431
-sl_shared_assets/server/job.pyi,sha256=uYfOuKgPL1hSHQvy5nmXzFkVjS316F3IZTdT-PmluZU,11663
-sl_shared_assets/server/server.py,sha256=MGk1v49aEFeIChMDsiR7CXjVkWwDpD9kA1TK0fwuTXw,32926
-sl_shared_assets/server/server.pyi,sha256=5Yxq4txhjtd9w-6U9fPehzMeIZL5GcprVCHd9mPP6FI,15113
-sl_shared_assets/tools/__init__.py,sha256=NktXk62E_HHOrO_93z_MVmSd6-Oir3mE4xE9Yr8Qa7U,682
-sl_shared_assets/tools/__init__.pyi,sha256=0UXorfCXXmHQOP5z7hODpsqEX0DAkOta5VZqN6FSS-w,623
-sl_shared_assets/tools/ascension_tools.py,sha256=tRV_tpoQURDD03slrRdh12Qbf9_ZQo4RU0PgYbUWOc0,14620
-sl_shared_assets/tools/ascension_tools.pyi,sha256=fs5j7nbnZ4WpgK8D75A7WJcvFMwK_MUO9ULIYo1YkGo,3739
-sl_shared_assets/tools/packaging_tools.py,sha256=QikjeaE_A8FyVJi3cnWLeW-hUXy1-FF1N23muA5VfT4,7526
-sl_shared_assets/tools/packaging_tools.pyi,sha256=vgGbAQCExwg-0A5F72MzEhzHxu97Nqg1yuz-5P89ycU,3118
-sl_shared_assets/tools/project_management_tools.py,sha256=Z_U0R26w9Le1O-u66gyF5CG8M_YaLFNpH9diQeH1AZQ,29381
-sl_shared_assets/tools/project_management_tools.pyi,sha256=4kok98nOZ4KnT-Sg-ZCZYg-WIM5qZqiyK8g1XiiDjHM,10375
-sl_shared_assets/tools/transfer_tools.py,sha256=J26kwOp_NpPSY0-xu5FTw9udte-rm_mW1FJyaTNoqQI,6606
-sl_shared_assets/tools/transfer_tools.pyi,sha256=FoH7eYZe7guGHfPr0MK5ggO62uXKwD2aJ7h1Bu7PaEE,3294
-sl_shared_assets-3.0.0rc14.dist-info/METADATA,sha256=lofwFKkF_-Qf2uQ7HeRBTB2CtEV4ktJPcgNKfT9JzH4,49214
-sl_shared_assets-3.0.0rc14.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sl_shared_assets-3.0.0rc14.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
-sl_shared_assets-3.0.0rc14.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-sl_shared_assets-3.0.0rc14.dist-info/RECORD,,