sl-shared-assets 2.0.1__py3-none-any.whl → 3.0.0__py3-none-any.whl

sl_shared_assets/tools/project_management_tools.py

@@ -11,15 +11,15 @@ from ataraxis_base_utilities import console
 
 from ..data_classes import (
     SessionData,
+    SessionTypes,
     ProcessingTracker,
     RunTrainingDescriptor,
     LickTrainingDescriptor,
+    WindowCheckingDescriptor,
     MesoscopeExperimentDescriptor,
 )
 from .packaging_tools import calculate_directory_checksum
 
-_valid_session_types = {"lick training", "run training", "mesoscope experiment", "window checking"}
-
 
 class ProjectManifest:
     """Wraps the contents of a Sun lab project manifest .feather file and exposes methods for visualizing and
@@ -76,11 +76,11 @@ class ProjectManifest:
             "session",
             "type",
             "complete",
-            "integrity_verification",
-            "suite2p_processing",
-            "behavior_processing",
-            "video_processing",
-            "dataset_formation",
+            "integrity",
+            "suite2p",
+            "behavior",
+            "video",
+            "dataset",
         ]
 
         # Retrieves the data
@@ -93,7 +93,7 @@ class ProjectManifest:
                 animal = str(animal)
             else:
                 animal = int(animal)
-        df = df.filter(pl.col("animal") == animal)
+            df = df.filter(pl.col("animal") == animal)
 
         # Ensures the data displays properly
         with pl.Config(
@@ -157,7 +157,13 @@ class ProjectManifest:
         """
         return tuple(self._data.select("session").sort("session").to_series().to_list())
 
-    def get_sessions_for_animal(self, animal: str | int, exclude_incomplete: bool = True) -> tuple[str, ...]:
+    def get_sessions_for_animal(
+        self,
+        animal: str | int,
+        exclude_incomplete: bool = True,
+        dataset_ready_only: bool = False,
+        not_dataset_ready_only: bool = False,
+    ) -> tuple[str, ...]:
         """Returns all session IDs for the target animal.
 
         This provides a tuple of all sessions performed by the target animal as part of the target project.
@@ -166,6 +172,11 @@ class ProjectManifest:
             animal: The ID of the animal for which to get the session data.
             exclude_incomplete: Determines whether to exclude sessions not marked as 'complete' from the output
                 list.
+            dataset_ready_only: Determines whether to exclude sessions not marked as 'dataset' integration ready from
+                the output list. Enabling this option only shows sessions that can be integrated into a dataset.
+            not_dataset_ready_only: The opposite of 'dataset_ready_only'. Determines whether to exclude sessions marked
+                as 'dataset' integration ready from the output list. Note, when both this and 'dataset_ready_only' are
+                enabled, the 'dataset_ready_only' option takes precedence.
 
         Raises:
             ValueError: If the specified animal is not found in the manifest file.
@@ -188,6 +199,12 @@ class ProjectManifest:
         if exclude_incomplete:
             data = data.filter(pl.col("complete") == 1)
 
+        # Optionally filters sessions based on their readiness for dataset integration.
+        if dataset_ready_only:  # Dataset-ready option always takes precedence
+            data = data.filter(pl.col("dataset") == 1)
+        elif not_dataset_ready_only:
+            data = data.filter(pl.col("dataset") == 0)
+
         # Formats and returns session IDs to the caller
         sessions = data.select("session").sort("session").to_series().to_list()
         return tuple(sessions)
@@ -203,8 +220,7 @@ class ProjectManifest:
 
         Returns:
             A Polars DataFrame with the following columns: 'animal', 'date', 'notes', 'session', 'type', 'complete',
-            'intensity_verification', 'suite2p_processing', 'behavior_processing', 'video_processing',
-            'dataset_formation'.
+            'intensity_verification', 'suite2p', 'behavior', 'video', 'dataset'.
         """
 
         df = self._data
@@ -264,12 +280,12 @@ def generate_project_manifest(
         # Determines whether the session data is complete (ran for the intended duration and has all expected data).
         "complete": [],
         # Determines whether the session data integrity has been verified upon transfer to a storage machine.
-        "integrity_verification": [],
-        "suite2p_processing": [],  # Determines whether the session has been processed with the single-day s2p pipeline.
+        "integrity": [],
+        "suite2p": [],  # Determines whether the session has been processed with the single-day s2p pipeline.
         # Determines whether the session has been processed with the behavior extraction pipeline.
-        "behavior_processing": [],
-        "video_processing": [],  # Determines whether the session has been processed with the DeepLabCut pipeline.
-        "dataset_formation": [],  # Determines whether the session's data has been integrated into a dataset.
+        "behavior": [],
+        "video": [],  # Determines whether the session has been processed with the DeepLabCut pipeline.
+        "dataset": [],  # Determines whether the session's data is ready to be integrated into a dataset.
     }
 
     # Loops over each session of every animal in the project and extracts session ID information and information
@@ -313,56 +329,63 @@ def generate_project_manifest(
 
         # Depending on the session type, instantiates the appropriate descriptor instance and uses it to read the
         # experimenter notes
-        if session_data.session_type == "lick training":
+        if session_data.session_type == SessionTypes.LICK_TRAINING:
             descriptor: LickTrainingDescriptor = LickTrainingDescriptor.from_yaml(  # type: ignore
                 file_path=session_data.raw_data.session_descriptor_path
             )
             manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == "run training":
+        elif session_data.session_type == SessionTypes.RUN_TRAINING:
             descriptor: RunTrainingDescriptor = RunTrainingDescriptor.from_yaml(  # type: ignore
                 file_path=session_data.raw_data.session_descriptor_path
             )
             manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == "mesoscope experiment":
+        elif session_data.session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
             descriptor: MesoscopeExperimentDescriptor = MesoscopeExperimentDescriptor.from_yaml(  # type: ignore
                 file_path=session_data.raw_data.session_descriptor_path
             )
             manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == "window checking":
-            manifest["notes"].append("N/A")
+        elif session_data.session_type == SessionTypes.WINDOW_CHECKING:
+            # sl-experiment version 3.0.0 added session descriptors to Window Checking runtimes. Since the file does not
+            # exist in prior versions, this section is written to statically handle the discrepancy.
+            try:
+                descriptor: WindowCheckingDescriptor = WindowCheckingDescriptor.from_yaml(  # type: ignore
+                    file_path=session_data.raw_data.session_descriptor_path
+                )
+                manifest["notes"].append(descriptor.experimenter_notes)
+            except Exception:
+                manifest["notes"].append("N/A")
 
         # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
         manifest["complete"].append(session_data.raw_data.telomere_path.exists())
 
         # Data verification status
         tracker = ProcessingTracker(file_path=session_data.raw_data.integrity_verification_tracker_path)
-        manifest["integrity_verification"].append(tracker.is_complete)
+        manifest["integrity"].append(tracker.is_complete)
 
         # If the session is incomplete or unverified, marks all processing steps as FALSE, as automatic processing is
         # disabled for incomplete sessions. If the session is unverified, the case is even more severe, as its data may
         # be corrupted.
-        if not manifest["complete"][-1] or not manifest["integrity_verification"][-1]:
-            manifest["suite2p_processing"].append(False)
-            manifest["dataset_formation"].append(False)
-            manifest["behavior_processing"].append(False)
-            manifest["video_processing"].append(False)
+        if not manifest["complete"][-1] or not manifest["integrity"][-1]:
+            manifest["suite2p"].append(False)
+            manifest["dataset"].append(False)
+            manifest["behavior"].append(False)
+            manifest["video"].append(False)
             continue  # Cycles to the next session
 
-        # Suite2p (single-day) status
+        # Suite2p (single-day) processing status.
         tracker = ProcessingTracker(file_path=session_data.processed_data.suite2p_processing_tracker_path)
-        manifest["suite2p_processing"].append(tracker.is_complete)
-
-        # Dataset formation (integration) status. Tracks whether the session has been added to any dataset(s).
-        tracker = ProcessingTracker(file_path=session_data.processed_data.dataset_formation_tracker_path)
-        manifest["dataset_formation"].append(tracker.is_complete)
+        manifest["suite2p"].append(tracker.is_complete)
 
-        # Dataset formation (integration) status. Tracks whether the session has been added to any dataset(s).
+        # Behavior data processing status.
         tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
-        manifest["behavior_processing"].append(tracker.is_complete)
+        manifest["behavior"].append(tracker.is_complete)
 
         # DeepLabCut (video) processing status.
         tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
-        manifest["video_processing"].append(tracker.is_complete)
+        manifest["video"].append(tracker.is_complete)
+
+        # Tracks whether the session's data is currently in the processing or dataset integration mode.
+        manifest["dataset"].append(session_data.processed_data.p53_path.exists())
 
     # If all animal IDs are integer-convertible, stores them as numbers to promote proper sorting. Otherwise, stores
     # them as strings. The latter options are primarily kept for compatibility with Tyche data
@@ -382,11 +405,11 @@ def generate_project_manifest(
         "type": pl.String,
         "notes": pl.String,
         "complete": pl.UInt8,
-        "integrity_verification": pl.UInt8,
-        "suite2p_processing": pl.UInt8,
-        "dataset_formation": pl.UInt8,
-        "behavior_processing": pl.UInt8,
-        "video_processing": pl.UInt8,
+        "integrity": pl.UInt8,
+        "suite2p": pl.UInt8,
+        "dataset": pl.UInt8,
+        "behavior": pl.UInt8,
+        "video": pl.UInt8,
     }
     df = pl.DataFrame(manifest, schema=schema, strict=False)
 
@@ -468,3 +491,86 @@ def verify_session_checksum(
         # runtime finished with an error to prevent deadlocking the runtime.
         if tracker.is_running:
             tracker.error()
+
+
+def resolve_p53_marker(
+    session_path: Path,
+    create_processed_data_directory: bool = True,
+    processed_data_root: None | Path = None,
+    remove: bool = False,
+) -> None:
+    """Depending on configuration, either creates or removes the p53.bin marker file for the target session.
+
+    The marker file statically determines whether the session can be targeted by data processing or dataset formation
+    pipelines.
+
+    Notes:
+        Since dataset integration relies on data processing outputs, it is essential to prevent processing pipelines
+        from altering the data while it is integrated into a dataset. The p53.bin marker solves this issue by ensuring
+        that only one type of runtimes (processing or dataset integration) is allowed to work with the session.
+
+        For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
+        p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab BioHPC server.
+
+    Args:
+        session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
+            input session directory must contain the 'raw_data' subdirectory.
+        create_processed_data_directory: Determines whether to create the processed data hierarchy during runtime.
+        processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
+            the root directory where to store the processed data from all projects, and it will be automatically
+            modified to include the project name, the animal name, and the session ID.
+        remove: Determines whether this function is called to create or remove the p53.bin marker.
+    """
+
+    # Loads session data layout. If configured to do so, also creates the processed data hierarchy
+    session_data = SessionData.load(
+        session_path=session_path,
+        processed_data_root=processed_data_root,
+        make_processed_data_directory=create_processed_data_directory,
+    )
+
+    # If the p53.bin marker exists and the runtime is configured to remove it, removes the marker file. If the runtime
+    # is configured to create the marker, aborts the runtime (as the marker already exists).
+    if session_data.processed_data.p53_path.exists():
+        if remove:
+            session_data.processed_data.p53_path.unlink()
+            return  # Ends remove runtime
+
+        return  # Ends create runtime
+
+    # If the marker does not exist and the function is called in 'remove' mode, aborts the runtime
+    elif remove:
+        return  # Ends remove runtime
+
+    # The rest of the runtime deals with determining whether it is safe to create the marker file.
+    # Queries the type of the processed session
+    session_type = session_data.session_type
+
+    # Window checking sessions are not designed to be integrated into datasets, so they cannot be marked with p53.bin
+    # file. Similarly, any incomplete session is automatically excluded from dataset formation.
+    if session_type == SessionTypes.WINDOW_CHECKING or not session_data.raw_data.telomere_path.exists():
+        return
+
+    # Training sessions collect similar data and share processing pipeline requirements
+    if session_type == SessionTypes.LICK_TRAINING or session_type == SessionTypes.RUN_TRAINING:
+        # Ensures that the session is not being processed with one of the supported pipelines.
+        behavior_tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
+        video_tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
+        if behavior_tracker.is_running or video_tracker.is_running:
+            # Note, training runtimes do not require suite2p processing.
+            return
+
+    # Mesoscope experiment sessions require additional processing with suite2p
+    if session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
+        behavior_tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
+        suite2p_tracker = ProcessingTracker(file_path=session_data.processed_data.suite2p_processing_tracker_path)
+        video_tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
+
+        # Similar to above, ensures that the session is not being processed with one of the supported pipelines.
+        if behavior_tracker.is_running or suite2p_tracker.is_running or video_tracker.is_running:
+            return
+
+    # If the runtime reached this point, the session is eligible for dataset integration. Creates the p53.bin marker
+    # file, preventing the session from being processed again as long as the marker exists.
+    session_data.processed_data.p53_path.touch()

sl_shared_assets/tools/project_management_tools.pyi

@@ -1,19 +1,18 @@
 from pathlib import Path
 
 import polars as pl
-from _typeshed import Incomplete
 
 from ..data_classes import (
     SessionData as SessionData,
+    SessionTypes as SessionTypes,
     ProcessingTracker as ProcessingTracker,
     RunTrainingDescriptor as RunTrainingDescriptor,
     LickTrainingDescriptor as LickTrainingDescriptor,
+    WindowCheckingDescriptor as WindowCheckingDescriptor,
     MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
 )
 from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
 
-_valid_session_types: Incomplete
-
 class ProjectManifest:
     """Wraps the contents of a Sun lab project manifest .feather file and exposes methods for visualizing and
     working with the data stored inside the file.
@@ -69,7 +68,13 @@ class ProjectManifest:
         This provides a tuple of all sessions, independent of the participating animal, that were recorded as part
         of the target project.
         """
-    def get_sessions_for_animal(self, animal: str | int, exclude_incomplete: bool = True) -> tuple[str, ...]:
+    def get_sessions_for_animal(
+        self,
+        animal: str | int,
+        exclude_incomplete: bool = True,
+        dataset_ready_only: bool = False,
+        not_dataset_ready_only: bool = False,
+    ) -> tuple[str, ...]:
         """Returns all session IDs for the target animal.
 
         This provides a tuple of all sessions performed by the target animal as part of the target project.
@@ -78,6 +83,11 @@ class ProjectManifest:
             animal: The ID of the animal for which to get the session data.
             exclude_incomplete: Determines whether to exclude sessions not marked as 'complete' from the output
                 list.
+            dataset_ready_only: Determines whether to exclude sessions not marked as 'dataset' integration ready from
+                the output list. Enabling this option only shows sessions that can be integrated into a dataset.
+            not_dataset_ready_only: The opposite of 'dataset_ready_only'. Determines whether to exclude sessions marked
+                as 'dataset' integration ready from the output list. Note, when both this and 'dataset_ready_only' are
+                enabled, the 'dataset_ready_only' option takes precedence.
 
         Raises:
             ValueError: If the specified animal is not found in the manifest file.
@@ -93,8 +103,7 @@ class ProjectManifest:
 
         Returns:
             A Polars DataFrame with the following columns: 'animal', 'date', 'notes', 'session', 'type', 'complete',
-            'intensity_verification', 'suite2p_processing', 'behavior_processing', 'video_processing',
-            'dataset_formation'.
+            'intensity_verification', 'suite2p', 'behavior', 'video', 'dataset'.
         """
 
 def generate_project_manifest(
@@ -146,3 +155,33 @@ def verify_session_checksum(
             the root directory where to store the processed data from all projects, and it will be automatically
             modified to include the project name, the animal name, and the session ID.
     """
+
+def resolve_p53_marker(
+    session_path: Path,
+    create_processed_data_directory: bool = True,
+    processed_data_root: None | Path = None,
+    remove: bool = False,
+) -> None:
+    """Depending on configuration, either creates or removes the p53.bin marker file for the target session.
+
+    The marker file statically determines whether the session can be targeted by data processing or dataset formation
+    pipelines.
+
+    Notes:
+        Since dataset integration relies on data processing outputs, it is essential to prevent processing pipelines
+        from altering the data while it is integrated into a dataset. The p53.bin marker solves this issue by ensuring
+        that only one type of runtimes (processing or dataset integration) is allowed to work with the session.
+
+        For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
+        p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab BioHPC server.
+
+    Args:
+        session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
+            input session directory must contain the 'raw_data' subdirectory.
+        create_processed_data_directory: Determines whether to create the processed data hierarchy during runtime.
+        processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
+            the root directory where to store the processed data from all projects, and it will be automatically
+            modified to include the project name, the animal name, and the session ID.
+        remove: Determines whether this function is called to create or remove the p53.bin marker.
+    """

{sl_shared_assets-2.0.1.dist-info → sl_shared_assets-3.0.0.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 2.0.1
-Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
+Version: 3.0.0
+Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
 Author: Ivan Kondratyev, Kushaan Gupta, Natalie Yeung
@@ -681,7 +681,7 @@ License:                     GNU GENERAL PUBLIC LICENSE
         Public License instead of this License.  But first, please read
         <https://www.gnu.org/licenses/why-not-lgpl.html>.
 License-File: LICENSE
-Keywords: acquisition,assets,data,processing,sunlab
+Keywords: acquisition,assets,data,processing,server,sunlab
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
@@ -691,13 +691,13 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.11
 Requires-Dist: appdirs==1.4.4
-Requires-Dist: ataraxis-base-utilities==3.0.1
+Requires-Dist: ataraxis-base-utilities==3.1.0
 Requires-Dist: ataraxis-data-structures==3.1.1
 Requires-Dist: ataraxis-time==3.0.0
 Requires-Dist: click==8.2.1
 Requires-Dist: filelock==3.18.0
 Requires-Dist: natsort==8.4.0
-Requires-Dist: numpy<2.3.0,>=2.0.2
+Requires-Dist: numpy==2.2.6
 Requires-Dist: paramiko==3.5.1
 Requires-Dist: polars==1.31.0
 Requires-Dist: pyarrow==20.0.0
@@ -706,7 +706,6 @@ Requires-Dist: simple-slurm==0.3.6
 Requires-Dist: tqdm==4.67.1
 Requires-Dist: xxhash==3.5.0
 Provides-Extra: conda
-Requires-Dist: grayskull<3,>=2; extra == 'conda'
 Requires-Dist: hatchling<2,>=1; extra == 'conda'
 Requires-Dist: importlib-metadata<9,>=8; extra == 'conda'
 Requires-Dist: mypy<2,>=1; extra == 'conda'
@@ -725,7 +724,7 @@ Requires-Dist: appdirs==1.4.4; extra == 'condarun'
 Requires-Dist: click==8.2.1; extra == 'condarun'
 Requires-Dist: filelock==3.18.0; extra == 'condarun'
 Requires-Dist: natsort==8.4.0; extra == 'condarun'
-Requires-Dist: numpy<2.3.0,>=2.0.2; extra == 'condarun'
+Requires-Dist: numpy==2.2.6; extra == 'condarun'
 Requires-Dist: paramiko==3.5.1; extra == 'condarun'
 Requires-Dist: polars==1.31.0; extra == 'condarun'
 Requires-Dist: pyarrow==20.0.0; extra == 'condarun'
@@ -734,7 +733,6 @@ Requires-Dist: tqdm==4.67.1; extra == 'condarun'
 Provides-Extra: dev
 Requires-Dist: ataraxis-automation<6,>=5; extra == 'dev'
 Requires-Dist: build<2,>=1; extra == 'dev'
-Requires-Dist: grayskull<3,>=2; extra == 'dev'
 Requires-Dist: hatchling<2,>=1; extra == 'dev'
 Requires-Dist: importlib-metadata<9,>=8; extra == 'dev'
 Requires-Dist: mypy<2,>=1; extra == 'dev'
@@ -774,6 +772,7 @@ A Python library that stores assets shared between multiple Sun (NeuroAI) lab da
 ![PyPI - License](https://img.shields.io/pypi/l/sl-shared-assets)
 ![PyPI - Status](https://img.shields.io/pypi/status/sl-shared-assets)
 ![PyPI - Wheel](https://img.shields.io/pypi/wheel/sl-shared-assets)
+
 ___
 
 ## Detailed Description
@@ -815,7 +814,7 @@ All software library dependencies are installed automatically as part of library
 
 Note, installation from source is ***highly discouraged*** for everyone who is not an active project developer.
 
-1. Download this repository to your local machine using your preferred method, such as Git-cloning. Use one
+1. Download this repository to your local machine using any method, such as Git-cloning. Use one
    of the stable releases from [GitHub](https://github.com/Sun-Lab-NBB/sl-shared-assets/releases).
 2. Unpack the downloaded zip and note the path to the binary wheel (`.whl`) file contained in the archive.
 3. Run ```python -m pip install WHEEL_PATH```, replacing 'WHEEL_PATH' with the path to the wheel file, to install the 
@@ -828,12 +827,128 @@ Use the following command to install the library using pip: ```pip install sl-sh
 
 ## Usage
 
-All library components are intended to be used via other Sun lab libraries. Developers should study the API and CLI 
+Most library components are intended to be used via other Sun lab libraries. Developers should study the API and CLI 
 documentation below to learn how to use library components in other Sun lab libraries. For notes on using shared 
 assets for data acquisition, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment) library ReadMe.
 For notes on using shared assets for data processing, see the [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) 
 library ReadMe.
 
+The only exception to the note above is the **server.py** package exposed by this library. This package exposes an API 
+for running headless and a CLI for running interactive Simple Linux Utility for Resource Management (SLURM)-managed 
+jobs on remote compute clusters.
+
+### Generating Access Credentials
+
+To access any remote server, the user is required to first generate the access credentials. The credentials are stored 
+inside the 'server_credentials.yaml' file, which is generated by using the `sl-create-server-credentials` command.
+**Note!** Users are advised to generate this file in a secure (non-shared) location on their local machine.
+
+### Running Headless Jobs
+
+A headless job is a job that does not require any user interaction during runtime. Currently, all headless jobs in the 
+lab rely on pip-installable packages that expose a callable Command-Line Interface to carry out some type of
+data processing. In this regard, **running a headless job is equivalent to calling a CLI command on your local 
+machine**, except that the command is executed on a remote compute server. Therefore, the primary purpose of the API 
+exposed by this library is to transfer the target command request to the remote server, execute it, and monitor the 
+runtime status until it is complete.
+
+For example, the [sl-suite2p package](https://github.com/Sun-Lab-NBB/suite2p) maintained in the lab exposes a CLI to 
+process 2-Photon data from experiment sessions. During data processing by the 
+[sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library, a remote job is sent to the server that uses the CLI 
+exposed by the sl-suite2p package to process target session(s).
+
+### Creating Jobs
+All remote jobs are sent to the server in the form of an executable *shell* (.sh) script. The script is composed on the 
+local machine that uses this library and transferred to a temporary server directory using Secure Shell File 
+Transfer Protocol (SFTP). The server is then instructed to evaluate (run) the script using SLURM job manager, via a 
+Secure Shell (SSH) session.
+
+Broadly, each job consists of three major steps, which correspond to three major sections of the job shell script:
+1. **Setting up the job environment**. Each job script starts with a SLURM job parameter block, which tells SLURM 
+   what resources (CPUs, GPUs, RAM, etc.) the job requires. When resources become available, SLURM generates a virtual
+   environment and runs the rest of the job script in that environment. This forms the basis for using the shared
+   compute resources fairly, as SLURM balances resource allocation and the order of job execution for all users.
+2. **Activating the target conda environment**. Currently, all jobs are assumed to use Python libraries to execute the 
+   intended data processing. Similar to processing data locally, each job expects the remote server to provide a 
+   Conda environment preconfigured with necessary assets (packages) to run the job. Therefore, each job contains a 
+   section that activates the user-defined conda environment before running the rest of the job.
+3. **Executing processing**. The final section is typically unique to each job and calls specific CLI commands or runs 
+   specific Python modules. Since each job is submitted as a shell script, it can do anything a server shell can
+   do. Therefore, despite python-centric approach to data processing in the lab, a remote job composed via this library 
+   can execute ***any*** arbitrary command available to the user on the remove server.
+
+Use the *Job* class exposed by this library to compose remote jobs. **Steps 1 and 2** of each job are configured when 
+initializing the Job instance, while **step 3** is added via the `add_command()` method of the Job class:
+```
+# First, import the job class
+from pathlib import Path
+from sl_shared_assets import Job
+
+# Next, instantiate a new Job object. For example, this job is used to verify the integrity of raw experiment data as
+# it is transferred to the long-term storage destination (server) by the sl-experiment library.
+job = Job(
+    job_name="data_integrity_verification",
+    output_log=Path("/temp/output.txt"),
+    error_log=Path("/temp/errors.txt"),
+    working_directory=Path("/temp/test_job"),
+    conda_environment="test_environment",
+    cpus_to_use=20,
+    ram_gb=50,
+    time_limit=20,
+)
+
+# Finally, add a CLI command call (the actual work to be done by the job). Here, the job calls the
+# 'sl-verify-session' command exposed by the sl-shared-assets library installed in the target environment on the server.
+# Use this method to add commands as you would type them in your local terminal / shell / command line.
+job.add_command(f"sl-verify-session -sp /temp/test_session")
+```
+
+### Submitting and Monitoring Jobs:
+To submit the job to the remote server, use a **Server** class instance. This class encapsulates access to the target 
+remote compute server and uses the server_credentials.yaml file to determine server access credentials (see above):
+```
+# Initialize the Server class using precreated server credentials file
+server = Server(credentials_path=Path("/temp/server_credentials.yaml"))
+
+# Submit the job (generated in the previous code snippet) to the server
+job = server.submit_job(job)
+
+# Wait for the server to complete the job
+delay_timer = PrecisionTimer("s")
+while not server.job_complete(job=job):
+    delay_timer.delay_noblock(delay=5, allow_sleep=True)
+```
+
+**Note!** The Server class only checks whether the job is running on the server, but not the outcome of the job. For 
+that, you can either manually check the output and error logs for the job or come up with a programmatic way of 
+checking the outcome. All developers are highly advised to study the API documentation for the Job and Server classes 
+to use them most effectively.
+
+**Critical!** Since running remote jobs is largely equivalent to executing them locally, all users are highly encouraged
+to test their job scripts locally before deploying them server-side. If a script works on a local machine, it is likely 
+that the script would behave similarly and work on the server.
+
+### Interactive Jobs
+
+Interactive jobs are a special extension of the headless job type discussed above. Specifically, an interactive job is 
+a headless job, whose only purpose is to **create and maintain a Jupyter lab server** under the SLURM control. 
+Specifically, it requests SLURM to set up an isolated environment, starts a Jupyter server in that environment, and 
+sends the credentials for the started server back to the user.
+
+In essence, this allocates a set of resources the user can use interactively by running various Jupyter notebooks. 
+While convenient for certain data analysis cases, this type of jobs has the potential to inefficiently hog server 
+resources for prolonged periods of time. Therefore, users are encouraged to only resort to this type of jobs when 
+strictly necessary and to minimize the resources and time allocated to running these jobs.
+
+To run an interactive job, call the `sl-start-jupyter` CLI command exposed by this library and follow the instructions 
+printed to the terminal by the command during runtime.
+
+**Critical!** While this command tries to minimize collisions with other users, it is possible that an access port 
+collision occurs when multiple users try to instantiate a jupyter server at the same time. If you cannot authenticate
+with the Jupyter server, this likely indicates that the target port was in use and Jupyter automatically incremented the
+port number by 1. In this case, add 1 to your port number and try connecting to that port using the Jupyter credentials 
+provided by the command. For example, if your target port was '8888,' try port '8889.'
+
 ---
 
 ## API Documentation
@@ -849,7 +964,7 @@ ___
 
 ## Versioning
 
-We use [semantic versioning](https://semver.org/) for this project. For the versions available, see the 
+This project uses [semantic versioning](https://semver.org/). For the versions available, see the 
 [tags on this repository](https://github.com/Sun-Lab-NBB/sl-shared-assets/tags).
 
 ---
@@ -858,7 +973,6 @@ We use [semantic versioning](https://semver.org/) for this project. For the vers
 
 - Ivan Kondratyev ([Inkaros](https://github.com/Inkaros))
 - Kushaan Gupta ([kushaangupta](https://github.com/kushaangupta))
-- Yuantao Deng ([YuantaoDeng](https://github.com/YuantaoDeng))
 - Natalie Yeung
 
 ___
@@ -873,7 +987,7 @@ ___
 
 - All Sun lab [members](https://neuroai.github.io/sunlab/people) for providing the inspiration and comments during the
   development of this library.
-- The creators of all other projects used in our development automation pipelines and source code 
+- The creators of all other projects used in the development automation pipelines and source code of this project
   [see pyproject.toml](pyproject.toml).
 
 ---

sl_shared_assets-3.0.0.dist-info/RECORD

@@ -0,0 +1,36 @@
+sl_shared_assets/__init__.py,sha256=rCu1VYs2Lc1l0jqHO3UtfuymU0uY2ccxEn4UyscIut8,2347
+sl_shared_assets/__init__.pyi,sha256=WCWIS-I3ToP4XybNZAi3fA7j2CZ48dl9D-fmd7oZKCo,2615
+sl_shared_assets/cli.py,sha256=1TRpRhkwi0A1WlN125iLxWt4e_ST4s6gpHfORK3FEQk,18126
+sl_shared_assets/cli.pyi,sha256=kQjGw5bxMLBqCdXcYPDz4aSrgrotrR02tX4hny0O9RA,5258
+sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sl_shared_assets/data_classes/__init__.py,sha256=bdm0hyQpNF0RL2SPhUgaOz33FsRzpM2L_z5-91HyZBE,1998
+sl_shared_assets/data_classes/__init__.pyi,sha256=J7ZCH9qQ4qz-3Wq9ILdihlmK9zFR3iU1cpLcSaN45Y8,2238
+sl_shared_assets/data_classes/configuration_data.py,sha256=SN2I_HrJkWirBQamMxRpwIyIiv5oW15Bgtl_-We9Ia0,36348
+sl_shared_assets/data_classes/configuration_data.pyi,sha256=MJPBQ2_vkZSYJOOYzwalBxOwLtpYKAnCaO6szqZ6adI,11059
+sl_shared_assets/data_classes/runtime_data.py,sha256=MLIef6s9n2gG6sbp197gpFfzb05e_8vwVzyS_oSmXYQ,16722
+sl_shared_assets/data_classes/runtime_data.pyi,sha256=LzNuEWu-GlPGdyyi8Hn2OFUjGCWOaOplKsRQBbjn2vQ,6768
+sl_shared_assets/data_classes/session_data.py,sha256=PZ7QVUyPXdLIuEJH4wvHRpirQk2GDiGNHIm0VlCU6QU,48237
+sl_shared_assets/data_classes/session_data.pyi,sha256=g53jIe-v8VkQJHc7ITS0KBGRhzn6LOIb6f96SEbEGig,15898
+sl_shared_assets/data_classes/surgery_data.py,sha256=5B1OPKFq4bnzbAoe-_c5dFV3kbSD5YFzXbX2zXmfGs8,7485
+sl_shared_assets/data_classes/surgery_data.pyi,sha256=rf59lJ3tGSYKHQlEGXg75MnjajBwl0DYhL4TClAO4SM,2605
+sl_shared_assets/server/__init__.py,sha256=w7y73RXXjBrWQsjU5g1QNCv_gsXDYnHos3NpOoR2AHA,452
+sl_shared_assets/server/__init__.pyi,sha256=Zc12G90fZdgEMwaVZbFzrRVV1wH_LEj3sxaV3lhk1Cw,316
+sl_shared_assets/server/job.py,sha256=DnEVIswZXm9queBgy6MlpIrCosXvQ_tweOeko7LN9yc,19431
+sl_shared_assets/server/job.pyi,sha256=uYfOuKgPL1hSHQvy5nmXzFkVjS316F3IZTdT-PmluZU,11663
+sl_shared_assets/server/server.py,sha256=MGk1v49aEFeIChMDsiR7CXjVkWwDpD9kA1TK0fwuTXw,32926
+sl_shared_assets/server/server.pyi,sha256=5Yxq4txhjtd9w-6U9fPehzMeIZL5GcprVCHd9mPP6FI,15113
+sl_shared_assets/tools/__init__.py,sha256=NktXk62E_HHOrO_93z_MVmSd6-Oir3mE4xE9Yr8Qa7U,682
+sl_shared_assets/tools/__init__.pyi,sha256=0UXorfCXXmHQOP5z7hODpsqEX0DAkOta5VZqN6FSS-w,623
+sl_shared_assets/tools/ascension_tools.py,sha256=tRV_tpoQURDD03slrRdh12Qbf9_ZQo4RU0PgYbUWOc0,14620
+sl_shared_assets/tools/ascension_tools.pyi,sha256=fs5j7nbnZ4WpgK8D75A7WJcvFMwK_MUO9ULIYo1YkGo,3739
+sl_shared_assets/tools/packaging_tools.py,sha256=cLZu4GBwrmQcBtvcLUahY7UPsucK3_-MZzJdZk5aPMc,7540
+sl_shared_assets/tools/packaging_tools.pyi,sha256=vgGbAQCExwg-0A5F72MzEhzHxu97Nqg1yuz-5P89ycU,3118
+sl_shared_assets/tools/project_management_tools.py,sha256=VpGI4Vt0hBIZ1_6F6Hq9zESw3pR8cNffSJp9oCHQk1Y,28725
+sl_shared_assets/tools/project_management_tools.pyi,sha256=AeBG-8XUygiJndfsBCKACKIZdnvk0avQRibWO24ahtM,10238
+sl_shared_assets/tools/transfer_tools.py,sha256=J26kwOp_NpPSY0-xu5FTw9udte-rm_mW1FJyaTNoqQI,6606
+sl_shared_assets/tools/transfer_tools.pyi,sha256=FoH7eYZe7guGHfPr0MK5ggO62uXKwD2aJ7h1Bu7PaEE,3294
+sl_shared_assets-3.0.0.dist-info/METADATA,sha256=meOnoDUinqxwqgzCNLpflBVDm8ZQxYnGAAsVvnoSKYY,56944
+sl_shared_assets-3.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-3.0.0.dist-info/entry_points.txt,sha256=UmO1rl7ly9N7HWPwWyP9E0b5KBUStpBo4TRoqNtizDY,430
+sl_shared_assets-3.0.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-3.0.0.dist-info/RECORD,,