sl-shared-assets 1.0.0rc12__tar.gz → 1.0.0rc13__tar.gz

{sl_shared_assets-1.0.0rc12 → sl_shared_assets-1.0.0rc13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 1.0.0rc12
+Version: 1.0.0rc13
 Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

{sl_shared_assets-1.0.0rc12 → sl_shared_assets-1.0.0rc13}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 # Project metdata section. Provides the genral ID information about the project.
 [project]
 name = "sl-shared-assets"
-version = "1.0.0rc12"
+version = "1.0.0rc13"
 description = "Stores assets shared between multiple Sun (NeuroAI) lab data pipelines."
 readme = "README.md"
 license = { file = "LICENSE" }

{sl_shared_assets-1.0.0rc12 → sl_shared_assets-1.0.0rc13}/src/sl_shared_assets/__init__.py

@@ -19,6 +19,7 @@ from .data_classes import (
     ProcedureData,
     ZaberPositions,
     ExperimentState,
+    ProcessingTracker,
     MesoscopePositions,
     ProjectConfiguration,
     HardwareConfiguration,
@@ -53,6 +54,7 @@ __all__ = [
     "LickTrainingDescriptor",
     "ExperimentConfiguration",
     "MesoscopeExperimentDescriptor",
+    "ProcessingTracker"
     # Transfer tools module
     "transfer_directory",
     # Packaging tools module

{sl_shared_assets-1.0.0rc12 → sl_shared_assets-1.0.0rc13}/src/sl_shared_assets/data_classes.py

@@ -104,8 +104,8 @@ class ProjectConfiguration(YamlConfig):
     this field is configured automatically each time the class is instantiated through any method, so overwriting it 
     manually will not be respected."""
     local_server_directory: str | Path = Path("/home/cybermouse/server/storage/sun_data")
-    """The absolute path to the directory where all projects are stored on the BioHPC server. This directory should be 
-    locally accessible (mounted) using a network sharing protocol, such as SMB."""
+    """The absolute path to the directory where the raw data portion of all projects is stored on the BioHPC server. 
+    This directory should be locally accessible (mounted) using a network sharing protocol, such as SMB."""
     local_nas_directory: str | Path = Path("/home/cybermouse/nas/rawdata")
     """The absolute path to the directory where all projects are stored on the Synology NAS. This directory should be 
     locally accessible (mounted) using a network sharing protocol, such as SMB."""
@@ -113,6 +113,9 @@ class ProjectConfiguration(YamlConfig):
     """The absolute path to the root mesoscope (ScanImagePC) directory where all mesoscope-acquired data is aggregated 
     during acquisition runtime. This directory should be locally accessible (mounted) using a network sharing 
     protocol, such as SMB."""
+    local_server_working_directory: str | Path = Path("/home/cybermouse/server/workdir/sun_data")
+    """The absolute path to the directory where the processed data portion of all projects is stored on the BioHPC 
+    server. This directory should be locally accessible (mounted) using a network sharing protocol, such as SMB."""
     remote_storage_directory: str | Path = Path("/storage/sun_data")
     """The absolute path, relative to the BioHPC server root, to the directory where all projects are stored on the 
     slow (SSD) volume of the server. This path is used when running remote (server-side) jobs and, therefore, has to
@@ -257,6 +260,7 @@ class ProjectConfiguration(YamlConfig):
         instance.local_mesoscope_directory = Path(instance.local_mesoscope_directory)
         instance.local_nas_directory = Path(instance.local_nas_directory)
         instance.local_server_directory = Path(instance.local_server_directory)
+        instance.local_server_working_directory = Path(instance.local_server_working_directory)
         instance.remote_storage_directory = Path(instance.remote_storage_directory)
         instance.remote_working_directory = Path(instance.remote_working_directory)
         instance.google_credentials_path = Path(instance.google_credentials_path)
@@ -301,6 +305,7 @@ class ProjectConfiguration(YamlConfig):
         original.local_mesoscope_directory = str(original.local_mesoscope_directory)
         original.local_nas_directory = str(original.local_nas_directory)
         original.local_server_directory = str(original.local_server_directory)
+        original.local_server_working_directory = str(original.local_server_working_directory)
         original.remote_storage_directory = str(original.remote_storage_directory)
         original.remote_working_directory = str(original.remote_working_directory)
         original.google_credentials_path = str(original.google_credentials_path)
@@ -795,10 +800,29 @@ class Destinations:
     server_raw_data_path: str | Path
     """Stores the path to the session's raw_data directory on the BioHPC server, which is mounted to the VRPC via the 
     SMB or equivalent protocol."""
+    server_processed_data_path: str | Path
+    """Stores the path to the session's processed_data directory on the BioHPC server, which is mounted to the VRPC via 
+    the SMB or equivalent protocol."""
+    server_configuration_path: str | Path
+    """Stores the path to the project-specific 'configuration' directory on the BioHPC server, which is mounted to the 
+    VRPC via the SMB or equivalent protocol."""
     server_telomere_path: str | Path = "null"
     """Stores the path to the session's telomere.bin marker. This marker is generated as part of BioHPC-side data 
     processing pipeline to notify he VRPC that the server received the data intact. it is used to determine which 
     data can be safely removed from the VRPC filesystem."""
+    server_suite2p_configuration_path: str | Path = "null"
+    """Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
+    the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p 
+    registration pipeline, which is applied to each session that generates brain activity data."""
+    server_processing_tracker_path: str | Path = "null"
+    """Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on 
+    the fast BioHPC server volume. This file is used to track which processing pipelines to apply to the target 
+    session and the status (success / failure) of each processing step.
+    """
+    server_multiday_configuration_path: str | Path = "null"
+    """Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory 
+    on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday' 
+    sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
 
     def __post_init__(self) -> None:
         """This method is automatically called after class instantiation and ensures that all path fields of the class
@@ -806,9 +830,21 @@ class Destinations:
         """
         self.nas_raw_data_path = Path(self.nas_raw_data_path)
         self.server_raw_data_path = Path(self.server_raw_data_path)
+        self.server_configuration_path = Path(self.server_configuration_path)
+        self.server_processed_data_path = Path(self.server_processed_data_path)
 
         if self.server_telomere_path == "null":
             self.server_telomere_path = self.server_raw_data_path.joinpath("telomere.bin")
+        if self.server_suite2p_configuration_path == "null":
+            self.server_suite2p_configuration_path = self.server_configuration_path.joinpath(
+                "suite2p_configuration.yaml"
+            )
+        if self.server_processing_tracker_path == "null":
+            self.server_processing_tracker_path = self.server_processed_data_path.joinpath("processing_tracker.yaml")
+        if self.server_multiday_configuration_path == "null":
+            self.server_multiday_configuration_path = self.server_configuration_path.joinpath(
+                "multiday_configuration.yaml"
+            )
 
     def make_string(self) -> None:
         """Converts all Path objects stored inside the class to strings.
@@ -819,11 +855,18 @@ class Destinations:
         self.nas_raw_data_path = str(self.nas_raw_data_path)
         self.server_raw_data_path = str(self.server_raw_data_path)
         self.server_telomere_path = str(self.server_telomere_path)
+        self.server_configuration_path = str(self.server_configuration_path)
+        self.server_processed_data_path = str(self.server_processed_data_path)
+        self.server_suite2p_configuration_path = str(self.server_suite2p_configuration_path)
+        self.server_processing_tracker_path = str(self.server_processing_tracker_path)
+        self.server_multiday_configuration_path = str(self.server_multiday_configuration_path)
 
     def make_dirs(self) -> None:
         """Ensures that all destination directories exist."""
         ensure_directory_exists(Path(self.nas_raw_data_path))
         ensure_directory_exists(Path(self.server_raw_data_path))
+        ensure_directory_exists(Path(self.server_configuration_path))
+        ensure_directory_exists(Path(self.server_processed_data_path))
 
 
 @dataclass
@@ -948,6 +991,7 @@ class SessionData(YamlConfig):
         vrpc_root = Path(project_configuration.local_root_directory)
         mesoscope_root = Path(project_configuration.local_mesoscope_directory)
         biohpc_root = Path(project_configuration.local_server_directory)
+        biohpc_workdir = Path(project_configuration.local_server_working_directory)
         nas_root = Path(project_configuration.local_nas_directory)
 
         # Extracts the name of the project stored inside the project configuration file.
@@ -1001,6 +1045,8 @@ class SessionData(YamlConfig):
         destinations = Destinations(
             nas_raw_data_path=nas_root.joinpath(project_name, animal_id, session_name, "raw_data"),
             server_raw_data_path=biohpc_root.joinpath(project_name, animal_id, session_name, "raw_data"),
+            server_configuration_path=biohpc_workdir.joinpath(project_name, "configuration"),
+            server_processed_data_path=biohpc_workdir.joinpath(project_name, "processed_data"),
         )
         destinations.make_dirs()  # Generates all destination directory trees
 
@@ -1582,7 +1628,7 @@ class SurgeryData(YamlConfig):
 
 @dataclass()
 class ProcessingTracker(YamlConfig):
-    """ Tracks the data processing status for a single session.
+    """Tracks the data processing status for a single session.
 
     This class is used during BioHPC-server data processing runtimes to track which processing steps are enabled and
     have been successfully applied to a given session. This is used to optimize data processing and avoid unnecessary