sl-shared-assets 3.0.0rc14__py3-none-any.whl → 3.1.0__py3-none-any.whl

sl_shared_assets/data_classes/session_data.pyi

@@ -20,7 +20,7 @@ class SessionTypes(StrEnum):
 
     Notes:
         This enumeration does not differentiate between different acquisition systems. Different acquisition systems
-        support different session types, and may not be suited for acquiring some of the session types listed in this
+        support different session types and may not be suited for acquiring some of the session types listed in this
         enumeration.
     """
 
@@ -206,8 +206,8 @@ class SessionData(YamlConfig):
                 provide the path to the root project directory (directory that stores all Sun lab projects) on that
                 drive. The method will automatically resolve the project/animal/session/processed_data hierarchy using
                 this root path. If raw and processed data are kept on the same drive, keep this set to None.
-            make_processed_data_directory: Determines whether this method should create processed_data directory if it
-                does not exist.
+            make_processed_data_directory: Determines whether this method should create the processed_data directory if
+                it does not exist.
 
         Returns:
             An initialized SessionData instance for the session whose data is stored at the provided path.
@@ -226,7 +226,7 @@ class SessionData(YamlConfig):
     def _save(self) -> None:
         """Saves the instance data to the 'raw_data' directory of the managed session as a 'session_data.yaml' file.
 
-        This is used to save the data stored in the instance to disk, so that it can be reused during further stages of
+        This is used to save the data stored in the instance to disk so that it can be reused during further stages of
         data processing. The method is intended to only be used by the SessionData instance itself during its
         create() method runtime.
         """
@@ -245,13 +245,13 @@ class ProcessingTracker(YamlConfig):
     _encountered_error: bool = ...
     _is_running: bool = ...
     _lock_path: str = field(init=False)
+    _started_runtime: bool = ...
     def __post_init__(self) -> None: ...
     def __del__(self) -> None:
-        """If the instance is garbage-collected without calling the stop() method, assumes this is due to a runtime
-        error.
+        """If the instance as used to start a runtime, ensures that the instance properly marks the runtime as completed
+        or erred before being garbage-collected.
 
-        It is essential to always resolve the runtime as either 'stopped' or 'erred' to avoid deadlocking the session
-        data.
+        This is a security mechanism to prevent deadlocking the processed session and pipeline for future runtimes.
         """
     def _load_state(self) -> None:
         """Reads the current processing state from the wrapped .YAML file."""
@@ -264,7 +264,7 @@ class ProcessingTracker(YamlConfig):
         with an error.
 
         Raises:
-            TimeoutError: If the file lock for the target .YAML file cannot be acquired within the timeout period.
+            TimeoutError: If the .lock file for the target .YAML file cannot be acquired within the timeout period.
         """
     def error(self) -> None:
         """Configures the tracker file to indicate that the tracked processing runtime encountered an error and failed
@@ -276,7 +276,7 @@ class ProcessingTracker(YamlConfig):
         from the process that calls this method.
 
         Raises:
-            TimeoutError: If the file lock for the target .YAML file cannot be acquired within the timeout period.
+            TimeoutError: If the .lock file for the target .YAML file cannot be acquired within the timeout period.
         """
     def stop(self) -> None:
         """Configures the tracker file to indicate that the tracked processing runtime has been completed successfully.
@@ -286,7 +286,7 @@ class ProcessingTracker(YamlConfig):
         at the end of the runtime.
 
         Raises:
-            TimeoutError: If the file lock for the target .YAML file cannot be acquired within the timeout period.
+            TimeoutError: If the .lock file for the target .YAML file cannot be acquired within the timeout period.
         """
     @property
     def is_complete(self) -> bool:

sl_shared_assets/server/__init__.py

@@ -1,5 +1,5 @@
 """This package provides the classes and methods used by all Sun lab libraries to submit remote jobs to the BioHPC
-and other compute servers. This package is also used across all Sun lab members private code to interface with the
+and other compute servers. This package is also used across all Sun lab members' private code to interface with the
 shared server."""
 
 from .job import Job, JupyterJob

sl_shared_assets/server/job.py

@@ -1,6 +1,6 @@
 """This module provides the core Job class, used as the starting point for all SLURM-managed job executed on lab compute
 server(s). Specifically, the Job class acts as a wrapper around the SLURM configuration and specific logic of each
-job. During runtime, Server class interacts with input job objects to manage their transfer and execution on the
+job. During runtime, the Server class interacts with input job objects to manage their transfer and execution on the
 remote servers.
 
 Since version 3.0.0, this module also provides the specialized JupyterJob class used to launch remote Jupyter
@@ -97,8 +97,8 @@ class Job:
     Attributes:
         remote_script_path: Stores the path to the script file relative to the root of the remote server that runs the
             command.
-        job_id: Stores the unique job identifier assigned by the SLURM manager to this job, when it is accepted for
-            execution. This field initialized to None and is overwritten by the Server class that submits the job.
+        job_id: Stores the unique job identifier assigned by the SLURM manager to this job when it is accepted for
+            execution. This field is initialized to None and is overwritten by the Server class that submits the job.
         job_name: Stores the descriptive name of the SLURM job.
         _command: Stores the managed SLURM command object.
     """
@@ -174,7 +174,7 @@ class Job:
         # initialization would not work as expected.
         fixed_script_content = script_content.replace("\\$", "$")
 
-        # Returns the script content to caller as a string
+        # Returns the script content to the caller as a string
         return fixed_script_content
 
 
@@ -202,8 +202,8 @@ class JupyterJob(Job):
         conda_environment: The name of the conda environment to activate on the server before running the job logic. The
             environment should contain the necessary Python packages and CLIs to support running the job's logic. For
             Jupyter jobs, this necessarily includes the Jupyter notebook and jupyterlab packages.
-        port: The connection port number for Jupyter server. Do not change the default value unless you know what you
-            are doing, as the server has most common communication ports closed for security reasons.
+        port: The connection port number for the Jupyter server. Do not change the default value unless you know what
+            you are doing, as the server has most common communication ports closed for security reasons.
         notebook_directory: The directory to use as Jupyter's root. During runtime, Jupyter will only have access to
             items stored in or under this directory. For most runtimes, this should be set to the user's root data or
             working directory.
@@ -270,7 +270,7 @@ class JupyterJob(Job):
         self._build_jupyter_command(jupyter_args)
 
     def _build_jupyter_command(self, jupyter_args: str) -> None:
-        """Builds the command to launch Jupyter notebook server on the remote Sun lab server."""
+        """Builds the command to launch the Jupyter notebook server on the remote Sun lab server."""
 
         # Gets the hostname of the compute node and caches it in the connection data file. Also caches the port name.
         self.add_command('echo "COMPUTE_NODE: $(hostname)" > {}'.format(self.connection_info_file))
@@ -297,7 +297,7 @@ class JupyterJob(Job):
         if jupyter_args:
             jupyter_cmd.append(jupyter_args)
 
-        # Adds resolved jupyter command to the list of job commands.
+        # Adds the resolved jupyter command to the list of job commands.
         jupyter_cmd_str = " ".join(jupyter_cmd)
         self.add_command(jupyter_cmd_str)
 
@@ -324,7 +324,7 @@ class JupyterJob(Job):
             message = f"Could not parse connection information file for the Jupyter server job with id {self.job_id}."
             console.error(message, ValueError)
 
-        # Stores extracted data inside connection_info attribute as a JupyterConnectionInfo instance.
+        # Stores extracted data inside the connection_info attribute as a JupyterConnectionInfo instance.
         self.connection_info = _JupyterConnectionInfo(
             compute_node=compute_node_match.group(1).strip(),  # type: ignore
             port=int(port_match.group(1)),  # type: ignore
@@ -352,7 +352,7 @@ class JupyterJob(Job):
             )
             return  # No connection information available, so does not proceed with printing.
 
-        # Prints generic connection details to terminal
+        # Prints generic connection details to the terminal
         console.echo(f"Jupyter is running on: {self.connection_info.compute_node}")
         console.echo(f"Port: {self.connection_info.port}")
         console.echo(f"Token: {self.connection_info.token}")

sl_shared_assets/server/job.pyi

@@ -73,8 +73,8 @@ class Job:
     Attributes:
         remote_script_path: Stores the path to the script file relative to the root of the remote server that runs the
             command.
-        job_id: Stores the unique job identifier assigned by the SLURM manager to this job, when it is accepted for
-            execution. This field initialized to None and is overwritten by the Server class that submits the job.
+        job_id: Stores the unique job identifier assigned by the SLURM manager to this job when it is accepted for
+            execution. This field is initialized to None and is overwritten by the Server class that submits the job.
         job_name: Stores the descriptive name of the SLURM job.
         _command: Stores the managed SLURM command object.
     """
@@ -138,8 +138,8 @@ class JupyterJob(Job):
         conda_environment: The name of the conda environment to activate on the server before running the job logic. The
             environment should contain the necessary Python packages and CLIs to support running the job's logic. For
             Jupyter jobs, this necessarily includes the Jupyter notebook and jupyterlab packages.
-        port: The connection port number for Jupyter server. Do not change the default value unless you know what you
-            are doing, as the server has most common communication ports closed for security reasons.
+        port: The connection port number for the Jupyter server. Do not change the default value unless you know what
+            you are doing, as the server has most common communication ports closed for security reasons.
         notebook_directory: The directory to use as Jupyter's root. During runtime, Jupyter will only have access to
             items stored in or under this directory. For most runtimes, this should be set to the user's root data or
             working directory.
@@ -184,7 +184,7 @@ class JupyterJob(Job):
         jupyter_args: str = "",
     ) -> None: ...
     def _build_jupyter_command(self, jupyter_args: str) -> None:
-        """Builds the command to launch Jupyter notebook server on the remote Sun lab server."""
+        """Builds the command to launch the Jupyter notebook server on the remote Sun lab server."""
     def parse_connection_info(self, info_file: Path) -> None:
         """Parses the connection information file created by the Jupyter job on the server.
 

sl_shared_assets/server/server.py

@@ -27,7 +27,7 @@ def generate_server_credentials(
     output_directory: Path,
     username: str,
     password: str,
-    host: str = "cbsuwsun.biohpc.cornell.edu",
+    host: str = "cbsuwsun.biopic.cornell.edu",
     storage_root: str = "/local/workdir",
     working_root: str = "/local/storage",
     shared_directory_name: str = "sun_data",
@@ -255,7 +255,7 @@ class Server:
             conda_environment: The name of the conda environment to activate on the server before running the job logic.
                 The environment should contain the necessary Python packages and CLIs to support running the job's
                 logic. For Jupyter jobs, this necessarily includes the Jupyter notebook and jupyterlab packages.
-            port: The connection port number for Jupyter server. If set to 0 (default), a random port number between
+            port: The connection port number for the Jupyter server. If set to 0 (default), a random port number between
                 8888 and 9999 will be assigned to this connection to reduce the possibility of colliding with other
                 user sessions.
             notebook_directory: The directory to use as Jupyter's root. During runtime, Jupyter will only have GUI
@@ -274,8 +274,8 @@ class Server:
             Do NOT re-submit the job to the server, as this is done as part of this method's runtime.
 
         Raises:
-            TimeoutError: If the target Jupyter server doesn't start within 120 minutes from this method being called.
-            RuntimeError: If job submission fails for any reason.
+            TimeoutError: If the target Jupyter server doesn't start within 120 minutes of this method being called.
+            RuntimeError: If the job submission fails for any reason.
         """
 
         # Statically configures the working directory to be stored under:
@@ -309,7 +309,7 @@ class Server:
     def submit_job(self, job: Job | JupyterJob) -> Job | JupyterJob:
         """Submits the input job to the managed BioHPC server via SLURM job manager.
 
-        This method submits various jobs for execution via SLURM-managed BioHPC cluster. As part of its runtime, the
+        This method submits various jobs for execution via the SLURM-managed BioHPC cluster. As part of its runtime, the
         method translates the Job object into the shell script, moves the script to the target working directory on
         the server, and instructs the server to execute the shell script (via SLURM).
 
@@ -400,7 +400,7 @@ class Server:
 
                 timer.delay_noblock(delay=5, allow_sleep=True)  # Waits for 5 seconds before checking again
             else:
-                # Only raises timeout error if the while loop is not broken in 120 seconds
+                # Only raises the timeout error if the while loop is not broken in 120 seconds
                 message = (
                     f"Remote jupyter server job {job.job_name} with id {job.job_id} did not start within 120 seconds "
                     f"from being submitted. Since all jupyter jobs are intended to be interactive and the server is "
@@ -418,7 +418,7 @@ class Server:
         """Returns True if the job managed by the input Job instance has been completed or terminated its runtime due
         to an error.
 
-        If the job is still running or is waiting inside the execution queue, returns False.
+        If the job is still running or is waiting inside the execution queue, the method returns False.
 
         Args:
             job: The Job object whose status needs to be checked.
@@ -446,7 +446,7 @@ class Server:
     def abort_job(self, job: Job | JupyterJob) -> None:
         """Aborts the target job if it is currently running on the server.
 
-        Use this method to immediately abort running or queued jobs, without waiting for the timeout guard. If the job
+        Use this method to immediately abort running or queued jobs without waiting for the timeout guard. If the job
         is queued, this method will remove it from the SLURM queue. If the job is already terminated, this method will
         do nothing.
 
@@ -507,12 +507,12 @@ class Server:
                 remote_item_path = remote_directory_path.joinpath(item.filename)
                 local_item_path = local_directory_path.joinpath(item.filename)
 
-                # Checks if item is a directory
+                # Checks if the item is a directory
                 if stat.S_ISDIR(item.st_mode):  # type: ignore
                     # Recursively pulls the subdirectory
                     self.pull_directory(local_item_path, remote_item_path)
                 else:
-                    # Pulls the individual file using existing method
+                    # Pulls the individual file using the existing method
                     sftp.get(localpath=str(local_item_path), remotepath=str(remote_item_path))
 
         finally:
@@ -535,7 +535,7 @@ class Server:
         sftp = self._client.open_sftp()
 
         try:
-            # Creates the remote directory using existing method
+            # Creates the remote directory using the existing method
             self.create_directory(remote_directory_path, parents=True)
 
             # Iterates through all items in the local directory
@@ -546,7 +546,7 @@ class Server:
                     # Recursively pushes subdirectory
                     self.push_directory(local_item_path, remote_item_path)
                 else:
-                    # Pushes the individual file using existing method
+                    # Pushes the individual file using the existing method
                     sftp.put(localpath=str(local_item_path), remotepath=str(remote_item_path))
 
         finally:
@@ -609,7 +609,7 @@ class Server:
                         current_path = part
 
                     try:
-                        # Checks if directory exists by trying to stat it
+                        # Checks if the directory exists by trying to 'stat' it
                         sftp.stat(current_path)
                     except FileNotFoundError:
                         # If the directory does not exist, creates it
@@ -617,7 +617,7 @@ class Server:
             else:
                 # Otherwise, only creates the final directory
                 try:
-                    # Checks if directory already exists
+                    # Checks if the directory already exists
                     sftp.stat(remote_path_str)
                 except FileNotFoundError:
                     # Creates the directory if it does not exist
@@ -632,7 +632,7 @@ class Server:
 
         sftp = self._client.open_sftp()
         try:
-            # Checks if the target file or directory exists by trying to stat it
+            # Checks if the target file or directory exists by trying to 'stat' it
             sftp.stat(str(remote_path))
 
             # If the request does not err, returns True (file or directory exists)

sl_shared_assets/server/server.pyi

@@ -15,7 +15,7 @@ def generate_server_credentials(
     output_directory: Path,
     username: str,
     password: str,
-    host: str = "cbsuwsun.biohpc.cornell.edu",
+    host: str = "cbsuwsun.biopic.cornell.edu",
     storage_root: str = "/local/workdir",
     working_root: str = "/local/storage",
     shared_directory_name: str = "sun_data",
@@ -140,7 +140,7 @@ class Server:
             conda_environment: The name of the conda environment to activate on the server before running the job logic.
                 The environment should contain the necessary Python packages and CLIs to support running the job's
                 logic. For Jupyter jobs, this necessarily includes the Jupyter notebook and jupyterlab packages.
-            port: The connection port number for Jupyter server. If set to 0 (default), a random port number between
+            port: The connection port number for the Jupyter server. If set to 0 (default), a random port number between
                 8888 and 9999 will be assigned to this connection to reduce the possibility of colliding with other
                 user sessions.
             notebook_directory: The directory to use as Jupyter's root. During runtime, Jupyter will only have GUI
@@ -159,13 +159,13 @@ class Server:
             Do NOT re-submit the job to the server, as this is done as part of this method's runtime.
 
         Raises:
-            TimeoutError: If the target Jupyter server doesn't start within 120 minutes from this method being called.
-            RuntimeError: If job submission fails for any reason.
+            TimeoutError: If the target Jupyter server doesn't start within 120 minutes of this method being called.
+            RuntimeError: If the job submission fails for any reason.
         """
     def submit_job(self, job: Job | JupyterJob) -> Job | JupyterJob:
         """Submits the input job to the managed BioHPC server via SLURM job manager.
 
-        This method submits various jobs for execution via SLURM-managed BioHPC cluster. As part of its runtime, the
+        This method submits various jobs for execution via the SLURM-managed BioHPC cluster. As part of its runtime, the
         method translates the Job object into the shell script, moves the script to the target working directory on
         the server, and instructs the server to execute the shell script (via SLURM).
 
@@ -183,7 +183,7 @@ class Server:
         """Returns True if the job managed by the input Job instance has been completed or terminated its runtime due
         to an error.
 
-        If the job is still running or is waiting inside the execution queue, returns False.
+        If the job is still running or is waiting inside the execution queue, the method returns False.
 
         Args:
             job: The Job object whose status needs to be checked.
@@ -195,7 +195,7 @@ class Server:
     def abort_job(self, job: Job | JupyterJob) -> None:
         """Aborts the target job if it is currently running on the server.
 
-        Use this method to immediately abort running or queued jobs, without waiting for the timeout guard. If the job
+        Use this method to immediately abort running or queued jobs without waiting for the timeout guard. If the job
         is queued, this method will remove it from the SLURM queue. If the job is already terminated, this method will
         do nothing.
 

sl_shared_assets/tools/__init__.py

@@ -4,9 +4,15 @@ integrity of the data. The tools from this package are used by most other data p
 from .transfer_tools import transfer_directory
 from .ascension_tools import ascend_tyche_data
 from .packaging_tools import calculate_directory_checksum
-from .project_management_tools import resolve_p53_marker, verify_session_checksum, generate_project_manifest
+from .project_management_tools import (
+    ProjectManifest,
+    resolve_p53_marker,
+    verify_session_checksum,
+    generate_project_manifest,
+)
 
 __all__ = [
+    "ProjectManifest",
     "transfer_directory",
     "calculate_directory_checksum",
     "ascend_tyche_data",

sl_shared_assets/tools/__init__.pyi

@@ -2,12 +2,14 @@ from .transfer_tools import transfer_directory as transfer_directory
 from .ascension_tools import ascend_tyche_data as ascend_tyche_data
 from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
 from .project_management_tools import (
+    ProjectManifest as ProjectManifest,
     resolve_p53_marker as resolve_p53_marker,
     verify_session_checksum as verify_session_checksum,
     generate_project_manifest as generate_project_manifest,
 )
 
 __all__ = [
+    "ProjectManifest",
     "transfer_directory",
     "calculate_directory_checksum",
     "ascend_tyche_data",

sl_shared_assets/tools/ascension_tools.py

@@ -47,7 +47,7 @@ def _generate_session_name(acquisition_path: Path) -> str:
         console.error(message=message, error=FileNotFoundError)
         raise FileNotFoundError(message)  # Fall-back to appease mypy
 
-    # Gets last modified time (available on all platforms) and converts it to a UTC timestamp object.
+    # Gets the last modified time (available on all platforms) and converts it to a UTC timestamp object.
     mod_time = source.stat().st_mtime
     mod_datetime = datetime.datetime.fromtimestamp(mod_time)
 
@@ -57,7 +57,7 @@ def _generate_session_name(acquisition_path: Path) -> str:
     timestamp_bytes = np.array([(timestamp_microseconds >> (8 * i)) & 0xFF for i in range(8)], dtype=np.uint8)
     stamp = extract_timestamp_from_bytes(timestamp_bytes=timestamp_bytes)
 
-    # Returns the generated session name to caller.
+    # Returns the generated session name to the caller.
     return stamp
 
 
@@ -89,8 +89,8 @@ def _reorganize_data(session_data: SessionData, source_root: Path) -> bool:
     mesoscope_frames_path = source_root.joinpath("mesoscope_frames")
     ax_checksum_path = source_root.joinpath("ax_checksum.txt")
 
-    # These two file types are present for some, but not all folders. They are not as important as the group of files
-    # above though, as, currently, the data stored in these files is not used during processing.
+    # These two file types are present for some, but not all folders. They are not as important as the files mentioned
+    # above, though, as, currently, the data stored in these files is not used during processing.
     frame_metadata_path = source_root.joinpath("frame_metadata.npz")
     metadata_path = source_root.joinpath("metadata.json")
 
@@ -201,10 +201,10 @@ def ascend_tyche_data(root_directory: Path) -> None:
     # Statically defines project name and local root paths
     project_name = "Tyche"
 
-    # Assumes that root directory stores all animal folders to be processed
+    # Assumes that the root directory stores all animal folders to be processed
     for animal_folder in root_directory.iterdir():
-        # Each animal folder is named to include project name and a static animal ID, e.g.: Tyche-A7. This extracts each
-        # animal ID.
+        # Each animal folder is named to include a project name and a static animal ID, e.g.: Tyche-A7. This extracts
+        # each animal ID.
         animal_name = animal_folder.stem.split(sep="-")[1]
 
         # Under each animal root folder, there are day folders that use YYYY-MM-DD timestamps
@@ -230,7 +230,7 @@ def ascend_tyche_data(root_directory: Path) -> None:
                 session_data.runtime_initialized()
 
                 # Moves the data from the old hierarchy to the new hierarchy. If the process runs as expected, and
-                # fully empties the source acquisition folder, destroys the folder. Otherwise, notifies the user that
+                # fully empties the source acquisition folder, it destroys the folder. Otherwise, notifies the user that
                 # the runtime did not fully process the session data and requests intervention.
                 success = _reorganize_data(session_data, acquisition_folder)
                 if not success:

sl_shared_assets/tools/packaging_tools.py

@@ -10,7 +10,7 @@ from concurrent.futures import ProcessPoolExecutor, as_completed
 from tqdm import tqdm
 import xxhash
 
-# Defines a 'blacklist' set of files. Primarily, this lit contains the service files that may change after the session
+# Defines a 'blacklist' set of files. Primarily, this list contains the service files that may change after the session
 # data has been acquired. Therefore, it does not make sense to include them in the checksum, as they do not reflect the
 # data that should remain permanently unchanged. Note, make sure all service files are added to this set!
 _excluded_files = {
@@ -18,6 +18,7 @@ _excluded_files = {
     "ubiquitin.bin",
     "telomere.bin",
     "p53.bin",
+    "nk.bin",
     "suite2p_processing_tracker.yaml",
     "dataset_formation_tracker.yaml",
     "video_processing_tracker.yaml",