sl-shared-assets 3.0.0rc13__py3-none-any.whl → 3.0.1__py3-none-any.whl

sl_shared_assets/server/job.py

@@ -1,6 +1,6 @@
 """This module provides the core Job class, used as the starting point for all SLURM-managed job executed on lab compute
 server(s). Specifically, the Job class acts as a wrapper around the SLURM configuration and specific logic of each
-job. During runtime, Server class interacts with input job objects to manage their transfer and execution on the
+job. During runtime, the Server class interacts with input job objects to manage their transfer and execution on the
 remote servers.
 
 Since version 3.0.0, this module also provides the specialized JupyterJob class used to launch remote Jupyter
@@ -97,8 +97,8 @@ class Job:
     Attributes:
         remote_script_path: Stores the path to the script file relative to the root of the remote server that runs the
             command.
-        job_id: Stores the unique job identifier assigned by the SLURM manager to this job, when it is accepted for
-            execution. This field initialized to None and is overwritten by the Server class that submits the job.
+        job_id: Stores the unique job identifier assigned by the SLURM manager to this job when it is accepted for
+            execution. This field is initialized to None and is overwritten by the Server class that submits the job.
         job_name: Stores the descriptive name of the SLURM job.
         _command: Stores the managed SLURM command object.
     """
@@ -174,7 +174,7 @@ class Job:
         # initialization would not work as expected.
         fixed_script_content = script_content.replace("\\$", "$")
 
-        # Returns the script content to caller as a string
+        # Returns the script content to the caller as a string
         return fixed_script_content
 
 
@@ -202,8 +202,8 @@ class JupyterJob(Job):
         conda_environment: The name of the conda environment to activate on the server before running the job logic. The
             environment should contain the necessary Python packages and CLIs to support running the job's logic. For
             Jupyter jobs, this necessarily includes the Jupyter notebook and jupyterlab packages.
-        port: The connection port number for Jupyter server. Do not change the default value unless you know what you
-            are doing, as the server has most common communication ports closed for security reasons.
+        port: The connection port number for the Jupyter server. Do not change the default value unless you know what
+            you are doing, as the server has most common communication ports closed for security reasons.
         notebook_directory: The directory to use as Jupyter's root. During runtime, Jupyter will only have access to
             items stored in or under this directory. For most runtimes, this should be set to the user's root data or
             working directory.
@@ -270,7 +270,7 @@ class JupyterJob(Job):
         self._build_jupyter_command(jupyter_args)
 
     def _build_jupyter_command(self, jupyter_args: str) -> None:
-        """Builds the command to launch Jupyter notebook server on the remote Sun lab server."""
+        """Builds the command to launch the Jupyter notebook server on the remote Sun lab server."""
 
         # Gets the hostname of the compute node and caches it in the connection data file. Also caches the port name.
         self.add_command('echo "COMPUTE_NODE: $(hostname)" > {}'.format(self.connection_info_file))
@@ -297,7 +297,7 @@ class JupyterJob(Job):
         if jupyter_args:
             jupyter_cmd.append(jupyter_args)
 
-        # Adds resolved jupyter command to the list of job commands.
+        # Adds the resolved jupyter command to the list of job commands.
         jupyter_cmd_str = " ".join(jupyter_cmd)
         self.add_command(jupyter_cmd_str)
 
@@ -324,7 +324,7 @@ class JupyterJob(Job):
             message = f"Could not parse connection information file for the Jupyter server job with id {self.job_id}."
             console.error(message, ValueError)
 
-        # Stores extracted data inside connection_info attribute as a JupyterConnectionInfo instance.
+        # Stores extracted data inside the connection_info attribute as a JupyterConnectionInfo instance.
         self.connection_info = _JupyterConnectionInfo(
             compute_node=compute_node_match.group(1).strip(),  # type: ignore
             port=int(port_match.group(1)),  # type: ignore
@@ -352,7 +352,7 @@ class JupyterJob(Job):
             )
             return  # No connection information available, so does not proceed with printing.
 
-        # Prints generic connection details to terminal
+        # Prints generic connection details to the terminal
         console.echo(f"Jupyter is running on: {self.connection_info.compute_node}")
         console.echo(f"Port: {self.connection_info.port}")
         console.echo(f"Token: {self.connection_info.token}")

sl_shared_assets/server/job.pyi

@@ -73,8 +73,8 @@ class Job:
     Attributes:
         remote_script_path: Stores the path to the script file relative to the root of the remote server that runs the
             command.
-        job_id: Stores the unique job identifier assigned by the SLURM manager to this job, when it is accepted for
-            execution. This field initialized to None and is overwritten by the Server class that submits the job.
+        job_id: Stores the unique job identifier assigned by the SLURM manager to this job when it is accepted for
+            execution. This field is initialized to None and is overwritten by the Server class that submits the job.
         job_name: Stores the descriptive name of the SLURM job.
         _command: Stores the managed SLURM command object.
     """
@@ -138,8 +138,8 @@ class JupyterJob(Job):
         conda_environment: The name of the conda environment to activate on the server before running the job logic. The
             environment should contain the necessary Python packages and CLIs to support running the job's logic. For
             Jupyter jobs, this necessarily includes the Jupyter notebook and jupyterlab packages.
-        port: The connection port number for Jupyter server. Do not change the default value unless you know what you
-            are doing, as the server has most common communication ports closed for security reasons.
+        port: The connection port number for the Jupyter server. Do not change the default value unless you know what
+            you are doing, as the server has most common communication ports closed for security reasons.
         notebook_directory: The directory to use as Jupyter's root. During runtime, Jupyter will only have access to
             items stored in or under this directory. For most runtimes, this should be set to the user's root data or
             working directory.
@@ -184,7 +184,7 @@ class JupyterJob(Job):
         jupyter_args: str = "",
     ) -> None: ...
     def _build_jupyter_command(self, jupyter_args: str) -> None:
-        """Builds the command to launch Jupyter notebook server on the remote Sun lab server."""
+        """Builds the command to launch the Jupyter notebook server on the remote Sun lab server."""
     def parse_connection_info(self, info_file: Path) -> None:
         """Parses the connection information file created by the Jupyter job on the server.
 

sl_shared_assets/server/server.py

@@ -27,7 +27,7 @@ def generate_server_credentials(
     output_directory: Path,
     username: str,
     password: str,
-    host: str = "cbsuwsun.biohpc.cornell.edu",
+    host: str = "cbsuwsun.biopic.cornell.edu",
     storage_root: str = "/local/workdir",
     working_root: str = "/local/storage",
     shared_directory_name: str = "sun_data",
@@ -255,7 +255,7 @@ class Server:
             conda_environment: The name of the conda environment to activate on the server before running the job logic.
                 The environment should contain the necessary Python packages and CLIs to support running the job's
                 logic. For Jupyter jobs, this necessarily includes the Jupyter notebook and jupyterlab packages.
-            port: The connection port number for Jupyter server. If set to 0 (default), a random port number between
+            port: The connection port number for the Jupyter server. If set to 0 (default), a random port number between
                 8888 and 9999 will be assigned to this connection to reduce the possibility of colliding with other
                 user sessions.
             notebook_directory: The directory to use as Jupyter's root. During runtime, Jupyter will only have GUI
@@ -274,8 +274,8 @@ class Server:
             Do NOT re-submit the job to the server, as this is done as part of this method's runtime.
 
         Raises:
-            TimeoutError: If the target Jupyter server doesn't start within 120 minutes from this method being called.
-            RuntimeError: If job submission fails for any reason.
+            TimeoutError: If the target Jupyter server doesn't start within 120 minutes of this method being called.
+            RuntimeError: If the job submission fails for any reason.
         """
 
         # Statically configures the working directory to be stored under:
@@ -309,7 +309,7 @@ class Server:
     def submit_job(self, job: Job | JupyterJob) -> Job | JupyterJob:
         """Submits the input job to the managed BioHPC server via SLURM job manager.
 
-        This method submits various jobs for execution via SLURM-managed BioHPC cluster. As part of its runtime, the
+        This method submits various jobs for execution via the SLURM-managed BioHPC cluster. As part of its runtime, the
         method translates the Job object into the shell script, moves the script to the target working directory on
         the server, and instructs the server to execute the shell script (via SLURM).
 
@@ -400,7 +400,7 @@ class Server:
 
                 timer.delay_noblock(delay=5, allow_sleep=True)  # Waits for 5 seconds before checking again
             else:
-                # Only raises timeout error if the while loop is not broken in 120 seconds
+                # Only raises the timeout error if the while loop is not broken in 120 seconds
                 message = (
                     f"Remote jupyter server job {job.job_name} with id {job.job_id} did not start within 120 seconds "
                     f"from being submitted. Since all jupyter jobs are intended to be interactive and the server is "
@@ -418,7 +418,7 @@ class Server:
         """Returns True if the job managed by the input Job instance has been completed or terminated its runtime due
         to an error.
 
-        If the job is still running or is waiting inside the execution queue, returns False.
+        If the job is still running or is waiting inside the execution queue, the method returns False.
 
         Args:
             job: The Job object whose status needs to be checked.
@@ -446,7 +446,7 @@ class Server:
     def abort_job(self, job: Job | JupyterJob) -> None:
         """Aborts the target job if it is currently running on the server.
 
-        Use this method to immediately abort running or queued jobs, without waiting for the timeout guard. If the job
+        Use this method to immediately abort running or queued jobs without waiting for the timeout guard. If the job
         is queued, this method will remove it from the SLURM queue. If the job is already terminated, this method will
         do nothing.
 
@@ -507,12 +507,12 @@ class Server:
                 remote_item_path = remote_directory_path.joinpath(item.filename)
                 local_item_path = local_directory_path.joinpath(item.filename)
 
-                # Checks if item is a directory
+                # Checks if the item is a directory
                 if stat.S_ISDIR(item.st_mode):  # type: ignore
                     # Recursively pulls the subdirectory
                     self.pull_directory(local_item_path, remote_item_path)
                 else:
-                    # Pulls the individual file using existing method
+                    # Pulls the individual file using the existing method
                     sftp.get(localpath=str(local_item_path), remotepath=str(remote_item_path))
 
         finally:
@@ -535,7 +535,7 @@ class Server:
         sftp = self._client.open_sftp()
 
         try:
-            # Creates the remote directory using existing method
+            # Creates the remote directory using the existing method
             self.create_directory(remote_directory_path, parents=True)
 
             # Iterates through all items in the local directory
@@ -546,7 +546,7 @@ class Server:
                     # Recursively pushes subdirectory
                     self.push_directory(local_item_path, remote_item_path)
                 else:
-                    # Pushes the individual file using existing method
+                    # Pushes the individual file using the existing method
                     sftp.put(localpath=str(local_item_path), remotepath=str(remote_item_path))
 
         finally:
@@ -609,7 +609,7 @@ class Server:
                         current_path = part
 
                     try:
-                        # Checks if directory exists by trying to stat it
+                        # Checks if the directory exists by trying to 'stat' it
                         sftp.stat(current_path)
                     except FileNotFoundError:
                         # If the directory does not exist, creates it
@@ -617,7 +617,7 @@ class Server:
             else:
                 # Otherwise, only creates the final directory
                 try:
-                    # Checks if directory already exists
+                    # Checks if the directory already exists
                     sftp.stat(remote_path_str)
                 except FileNotFoundError:
                     # Creates the directory if it does not exist
@@ -632,7 +632,7 @@ class Server:
 
         sftp = self._client.open_sftp()
         try:
-            # Checks if the target file or directory exists by trying to stat it
+            # Checks if the target file or directory exists by trying to 'stat' it
             sftp.stat(str(remote_path))
 
             # If the request does not err, returns True (file or directory exists)

sl_shared_assets/server/server.pyi

@@ -15,7 +15,7 @@ def generate_server_credentials(
     output_directory: Path,
     username: str,
     password: str,
-    host: str = "cbsuwsun.biohpc.cornell.edu",
+    host: str = "cbsuwsun.biopic.cornell.edu",
     storage_root: str = "/local/workdir",
     working_root: str = "/local/storage",
     shared_directory_name: str = "sun_data",
@@ -140,7 +140,7 @@ class Server:
             conda_environment: The name of the conda environment to activate on the server before running the job logic.
                 The environment should contain the necessary Python packages and CLIs to support running the job's
                 logic. For Jupyter jobs, this necessarily includes the Jupyter notebook and jupyterlab packages.
-            port: The connection port number for Jupyter server. If set to 0 (default), a random port number between
+            port: The connection port number for the Jupyter server. If set to 0 (default), a random port number between
                 8888 and 9999 will be assigned to this connection to reduce the possibility of colliding with other
                 user sessions.
             notebook_directory: The directory to use as Jupyter's root. During runtime, Jupyter will only have GUI
@@ -159,13 +159,13 @@ class Server:
             Do NOT re-submit the job to the server, as this is done as part of this method's runtime.
 
         Raises:
-            TimeoutError: If the target Jupyter server doesn't start within 120 minutes from this method being called.
-            RuntimeError: If job submission fails for any reason.
+            TimeoutError: If the target Jupyter server doesn't start within 120 minutes of this method being called.
+            RuntimeError: If the job submission fails for any reason.
         """
     def submit_job(self, job: Job | JupyterJob) -> Job | JupyterJob:
         """Submits the input job to the managed BioHPC server via SLURM job manager.
 
-        This method submits various jobs for execution via SLURM-managed BioHPC cluster. As part of its runtime, the
+        This method submits various jobs for execution via the SLURM-managed BioHPC cluster. As part of its runtime, the
         method translates the Job object into the shell script, moves the script to the target working directory on
         the server, and instructs the server to execute the shell script (via SLURM).
 
@@ -183,7 +183,7 @@ class Server:
         """Returns True if the job managed by the input Job instance has been completed or terminated its runtime due
         to an error.
 
-        If the job is still running or is waiting inside the execution queue, returns False.
+        If the job is still running or is waiting inside the execution queue, the method returns False.
 
         Args:
             job: The Job object whose status needs to be checked.
@@ -195,7 +195,7 @@ class Server:
     def abort_job(self, job: Job | JupyterJob) -> None:
         """Aborts the target job if it is currently running on the server.
 
-        Use this method to immediately abort running or queued jobs, without waiting for the timeout guard. If the job
+        Use this method to immediately abort running or queued jobs without waiting for the timeout guard. If the job
         is queued, this method will remove it from the SLURM queue. If the job is already terminated, this method will
         do nothing.
 

sl_shared_assets/tools/ascension_tools.py

@@ -47,7 +47,7 @@ def _generate_session_name(acquisition_path: Path) -> str:
         console.error(message=message, error=FileNotFoundError)
         raise FileNotFoundError(message)  # Fall-back to appease mypy
 
-    # Gets last modified time (available on all platforms) and converts it to a UTC timestamp object.
+    # Gets the last modified time (available on all platforms) and converts it to a UTC timestamp object.
     mod_time = source.stat().st_mtime
     mod_datetime = datetime.datetime.fromtimestamp(mod_time)
 
@@ -57,7 +57,7 @@ def _generate_session_name(acquisition_path: Path) -> str:
     timestamp_bytes = np.array([(timestamp_microseconds >> (8 * i)) & 0xFF for i in range(8)], dtype=np.uint8)
     stamp = extract_timestamp_from_bytes(timestamp_bytes=timestamp_bytes)
 
-    # Returns the generated session name to caller.
+    # Returns the generated session name to the caller.
     return stamp
 
 
@@ -89,8 +89,8 @@ def _reorganize_data(session_data: SessionData, source_root: Path) -> bool:
     mesoscope_frames_path = source_root.joinpath("mesoscope_frames")
     ax_checksum_path = source_root.joinpath("ax_checksum.txt")
 
-    # These two file types are present for some, but not all folders. They are not as important as the group of files
-    # above though, as, currently, the data stored in these files is not used during processing.
+    # These two file types are present for some, but not all folders. They are not as important as the files mentioned
+    # above, though, as, currently, the data stored in these files is not used during processing.
     frame_metadata_path = source_root.joinpath("frame_metadata.npz")
     metadata_path = source_root.joinpath("metadata.json")
 
@@ -201,10 +201,10 @@ def ascend_tyche_data(root_directory: Path) -> None:
     # Statically defines project name and local root paths
     project_name = "Tyche"
 
-    # Assumes that root directory stores all animal folders to be processed
+    # Assumes that the root directory stores all animal folders to be processed
     for animal_folder in root_directory.iterdir():
-        # Each animal folder is named to include project name and a static animal ID, e.g.: Tyche-A7. This extracts each
-        # animal ID.
+        # Each animal folder is named to include a project name and a static animal ID, e.g.: Tyche-A7. This extracts
+        # each animal ID.
         animal_name = animal_folder.stem.split(sep="-")[1]
 
         # Under each animal root folder, there are day folders that use YYYY-MM-DD timestamps
@@ -230,7 +230,7 @@ def ascend_tyche_data(root_directory: Path) -> None:
                 session_data.runtime_initialized()
 
                 # Moves the data from the old hierarchy to the new hierarchy. If the process runs as expected, and
-                # fully empties the source acquisition folder, destroys the folder. Otherwise, notifies the user that
+                # fully empties the source acquisition folder, it destroys the folder. Otherwise, notifies the user that
                 # the runtime did not fully process the session data and requests intervention.
                 success = _reorganize_data(session_data, acquisition_folder)
                 if not success:

sl_shared_assets/tools/packaging_tools.py

@@ -10,7 +10,7 @@ from concurrent.futures import ProcessPoolExecutor, as_completed
 from tqdm import tqdm
 import xxhash
 
-# Defines a 'blacklist' set of files. Primarily, this lit contains the service files that may change after the session
+# Defines a 'blacklist' set of files. Primarily, this list contains the service files that may change after the session
 # data has been acquired. Therefore, it does not make sense to include them in the checksum, as they do not reflect the
 # data that should remain permanently unchanged. Note, make sure all service files are added to this set!
 _excluded_files = {
@@ -18,6 +18,7 @@ _excluded_files = {
     "ubiquitin.bin",
     "telomere.bin",
     "p53.bin",
+    "nk.bin",
     "suite2p_processing_tracker.yaml",
     "dataset_formation_tracker.yaml",
     "video_processing_tracker.yaml",

sl_shared_assets/tools/project_management_tools.py

@@ -11,15 +11,15 @@ from ataraxis_base_utilities import console
 
 from ..data_classes import (
     SessionData,
+    SessionTypes,
     ProcessingTracker,
     RunTrainingDescriptor,
     LickTrainingDescriptor,
+    WindowCheckingDescriptor,
     MesoscopeExperimentDescriptor,
 )
 from .packaging_tools import calculate_directory_checksum
 
-_valid_session_types = {"lick training", "run training", "mesoscope experiment", "window checking"}
-
 
 class ProjectManifest:
     """Wraps the contents of a Sun lab project manifest .feather file and exposes methods for visualizing and
@@ -220,8 +220,7 @@ class ProjectManifest:
 
         Returns:
             A Polars DataFrame with the following columns: 'animal', 'date', 'notes', 'session', 'type', 'complete',
-            'intensity_verification', 'suite2p', 'behavior', 'video',
-            'dataset'.
+            'intensity_verification', 'suite2p', 'behavior', 'video', 'dataset'.
         """
 
         df = self._data
@@ -330,23 +329,31 @@ def generate_project_manifest(
 
         # Depending on the session type, instantiates the appropriate descriptor instance and uses it to read the
         # experimenter notes
-        if session_data.session_type == "lick training":
+        if session_data.session_type == SessionTypes.LICK_TRAINING:
             descriptor: LickTrainingDescriptor = LickTrainingDescriptor.from_yaml(  # type: ignore
                 file_path=session_data.raw_data.session_descriptor_path
             )
             manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == "run training":
+        elif session_data.session_type == SessionTypes.RUN_TRAINING:
             descriptor: RunTrainingDescriptor = RunTrainingDescriptor.from_yaml(  # type: ignore
                 file_path=session_data.raw_data.session_descriptor_path
             )
             manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == "mesoscope experiment":
+        elif session_data.session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
             descriptor: MesoscopeExperimentDescriptor = MesoscopeExperimentDescriptor.from_yaml(  # type: ignore
                 file_path=session_data.raw_data.session_descriptor_path
             )
             manifest["notes"].append(descriptor.experimenter_notes)
-        elif session_data.session_type == "window checking":
-            manifest["notes"].append("N/A")
+        elif session_data.session_type == SessionTypes.WINDOW_CHECKING:
+            # sl-experiment version 3.0.0 added session descriptors to Window Checking runtimes. Since the file does not
+            # exist in prior versions, this section is written to statically handle the discrepancy.
+            try:
+                descriptor: WindowCheckingDescriptor = WindowCheckingDescriptor.from_yaml(  # type: ignore
+                    file_path=session_data.raw_data.session_descriptor_path
+                )
+                manifest["notes"].append(descriptor.experimenter_notes)
+            except Exception:
+                manifest["notes"].append("N/A")
 
         # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
         manifest["complete"].append(session_data.raw_data.telomere_path.exists())
@@ -377,9 +384,7 @@ def generate_project_manifest(
         tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
         manifest["video"].append(tracker.is_complete)
 
-        # Tracks whether the session's data is ready for dataset integration. To be considered ready, the data must be
-        # successfully processed with all relevant pipelines. Any session currently being processed with any processing
-        # pipeline is considered NOT ready.
+        # Tracks whether the session's data is currently in the processing or dataset integration mode.
         manifest["dataset"].append(session_data.processed_data.p53_path.exists())
 
     # If all animal IDs are integer-convertible, stores them as numbers to promote proper sorting. Otherwise, stores
@@ -504,11 +509,9 @@ def resolve_p53_marker(
         from altering the data while it is integrated into a dataset. The p53.bin marker solves this issue by ensuring
         that only one type of runtimes (processing or dataset integration) is allowed to work with the session.
 
-        For the p53.bin marker to be created, the session must currently not undergo any processing and must be
-        successfully processed with the minimal set of pipelines for its session type. Removing the p53.bin marker does
-        not have any dependencies and will be executed even if the session is currently undergoing dataset integration.
-        Due to this limitation, it is only possible to call this function with the 'remove' flag manually (via the
-        dedicated CLI).
+        For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
+        p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab BioHPC server.
 
     Args:
         session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
@@ -528,7 +531,7 @@ def resolve_p53_marker(
     )
 
     # If the p53.bin marker exists and the runtime is configured to remove it, removes the marker file. If the runtime
-    # is configured to create the marker, aborts the runtime (as the marker already exists).
+    # is configured to create the marker, the method aborts the runtime (as the marker already exists).
     if session_data.processed_data.p53_path.exists():
         if remove:
             session_data.processed_data.p53_path.unlink()
@@ -544,41 +547,28 @@ def resolve_p53_marker(
     # Queries the type of the processed session
     session_type = session_data.session_type
 
-    # If the session type is not supported, aborts with an error
-    if session_type not in _valid_session_types:
-        message = (
-            f"Unable to determine the mandatory processing pipelines for session {session_data.session_name} of animal "
-            f"{session_data.animal_id} and project {session_data.processed_data}. The type of the session "
-            f"{session_type} is not one of the supported session types: {', '.join(_valid_session_types)}."
-        )
-        console.error(message=message, error=ValueError)
-
-    # Window checking sessions are not designed to be integrated into datasets, so they cannot be marked with p53.bin
-    # file. Similarly, any incomplete session is automatically excluded from dataset formation.
-    if session_type == "window checking" or not session_data.raw_data.telomere_path.exists():
+    # Window checking sessions are not designed to be integrated into datasets, so they cannot be marked with the
+    # p53.bin file. Similarly, any incomplete session is automatically excluded from dataset formation.
+    if session_type == SessionTypes.WINDOW_CHECKING or not session_data.raw_data.telomere_path.exists():
         return
 
     # Training sessions collect similar data and share processing pipeline requirements
-    if session_type == "lick training" or session_type == "run training":
-        # If the session has not been successfully processed with the behavior processing pipeline, aborts without
-        # creating the marker file. Also ensures that the video tracking pipeline is not actively running, although it
-        # is not required
+    if session_type == SessionTypes.LICK_TRAINING or session_type == SessionTypes.RUN_TRAINING:
+        # Ensures that the session is not being processed with one of the supported pipelines.
         behavior_tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
         video_tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
-        if not behavior_tracker.is_complete or video_tracker.is_running:
+        if behavior_tracker.is_running or video_tracker.is_running:
             # Note, training runtimes do not require suite2p processing.
             return
 
     # Mesoscope experiment sessions require additional processing with suite2p
-    if session_type == "mesoscope experiment":
+    if session_type == SessionTypes.MESOSCOPE_EXPERIMENT:
         behavior_tracker = ProcessingTracker(file_path=session_data.processed_data.behavior_processing_tracker_path)
         suite2p_tracker = ProcessingTracker(file_path=session_data.processed_data.suite2p_processing_tracker_path)
         video_tracker = ProcessingTracker(file_path=session_data.processed_data.video_processing_tracker_path)
 
-        # Similar to above, if the session is not processed with the behavior pipeline or the suite2p pipeline, aborts
-        # without creating the marker file. Video tracker is not required for p53 marker creation, but the video
-        # tracking pipeline must not be actively running.
-        if not behavior_tracker.is_complete or not suite2p_tracker.is_complete or video_tracker.is_running:
+        # Similar to the above, ensures that the session is not being processed with one of the supported pipelines.
+        if behavior_tracker.is_running or suite2p_tracker.is_running or video_tracker.is_running:
             return
 
     # If the runtime reached this point, the session is eligible for dataset integration. Creates the p53.bin marker

sl_shared_assets/tools/project_management_tools.pyi

@@ -1,19 +1,18 @@
 from pathlib import Path
 
 import polars as pl
-from _typeshed import Incomplete
 
 from ..data_classes import (
     SessionData as SessionData,
+    SessionTypes as SessionTypes,
     ProcessingTracker as ProcessingTracker,
     RunTrainingDescriptor as RunTrainingDescriptor,
     LickTrainingDescriptor as LickTrainingDescriptor,
+    WindowCheckingDescriptor as WindowCheckingDescriptor,
     MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
 )
 from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
 
-_valid_session_types: Incomplete
-
 class ProjectManifest:
     """Wraps the contents of a Sun lab project manifest .feather file and exposes methods for visualizing and
     working with the data stored inside the file.
@@ -104,8 +103,7 @@ class ProjectManifest:
 
         Returns:
             A Polars DataFrame with the following columns: 'animal', 'date', 'notes', 'session', 'type', 'complete',
-            'intensity_verification', 'suite2p', 'behavior', 'video',
-            'dataset'.
+            'intensity_verification', 'suite2p', 'behavior', 'video', 'dataset'.
         """
 
 def generate_project_manifest(
@@ -174,11 +172,9 @@ def resolve_p53_marker(
         from altering the data while it is integrated into a dataset. The p53.bin marker solves this issue by ensuring
         that only one type of runtimes (processing or dataset integration) is allowed to work with the session.
 
-        For the p53.bin marker to be created, the session must currently not undergo any processing and must be
-        successfully processed with the minimal set of pipelines for its session type. Removing the p53.bin marker does
-        not have any dependencies and will be executed even if the session is currently undergoing dataset integration.
-        Due to this limitation, it is only possible to call this function with the 'remove' flag manually (via the
-        dedicated CLI).
+        For the p53.bin marker to be created, the session must currently not undergo any processing. Removing the
+        p53.bin marker does not have any dependencies and will be executed even if the session is currently undergoing
+        dataset integration. This is due to data access hierarchy limitations of the Sun lab BioHPC server.
 
     Args:
         session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the

sl_shared_assets/tools/transfer_tools.py

@@ -45,7 +45,7 @@ def transfer_directory(source: Path, destination: Path, num_threads: int = 1, ve
         done before copying the files.
 
         The method executes a multithreading copy operation. It does not clean up the source files. That job is handed
-        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that calls this function.
+        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that call this function.
 
         If the method is configured to verify transferred file integrity, it reruns the xxHash3-128 checksum calculation
         and compares the returned checksum to the one stored in the source directory. The method assumes that all input

sl_shared_assets/tools/transfer_tools.pyi

@@ -30,7 +30,7 @@ def transfer_directory(source: Path, destination: Path, num_threads: int = 1, ve
         done before copying the files.
 
         The method executes a multithreading copy operation. It does not clean up the source files. That job is handed
-        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that calls this function.
+        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that call this function.
 
         If the method is configured to verify transferred file integrity, it reruns the xxHash3-128 checksum calculation
         and compares the returned checksum to the one stored in the source directory. The method assumes that all input