sl-shared-assets 1.0.0rc5__tar.gz → 1.0.0rc7__tar.gz

{sl_shared_assets-1.0.0rc5 → sl_shared_assets-1.0.0rc7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 1.0.0rc5
+Version: 1.0.0rc7
 Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

{sl_shared_assets-1.0.0rc5 → sl_shared_assets-1.0.0rc7}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 # Project metdata section. Provides the genral ID information about the project.
 [project]
 name = "sl-shared-assets"
-version = "1.0.0rc5"
+version = "1.0.0rc7"
 description = "Stores assets shared between multiple Sun (NeuroAI) lab data pipelines."
 readme = "README.md"
 license = { file = "LICENSE" }

{sl_shared_assets-1.0.0rc5 → sl_shared_assets-1.0.0rc7}/src/sl_shared_assets/data_classes.py

@@ -87,7 +87,7 @@ class ProjectConfiguration(YamlConfig):
     local_root_directory: str | Path = Path("/media/Data/Experiments")
     """The absolute path to the root directory where all projects are stored on the local host-machine (VRPC). Note, 
     overwriting the value of this field is pointless, as it is automatically set each time the class is instantiated."""
-    local_server_directory: str | Path = Path("/media/cbsuwsun/storage/sun_data")
+    local_server_directory: str | Path = Path("/home/cybermouse/server/storage/sun_data")
     """The absolute path to the locally-mapped (via SMB protocol) root BioHPC server machine directory where to store 
     all projects."""
     local_nas_directory: str | Path = Path("/home/cybermouse/nas/rawdata")
@@ -205,12 +205,9 @@ class ProjectConfiguration(YamlConfig):
                 with open(path_file, "w") as f:
                     f.write(str(root_path))
 
-            # Otherwise, uses the root path and the project name to resolve the path to the project configuration
-            # directory and load the project configuration data.
-            else:
-                # Reads the root path from the file
-                with open(path_file, "r") as f:
-                    root_path = Path(f.read().strip())
+            # Once the location of the path storage file is resolved, reads the root path from the file
+            with open(path_file, "r") as f:
+                root_path = Path(f.read().strip())
 
             # Uses the root experiment directory path to generate the path to the target project's configuration file.
             configuration_path = root_path.joinpath(project_name, "configuration", "project_configuration.yaml")
@@ -459,8 +456,8 @@ class RawData:
 
         This service method is used by the SessionData class to convert all paths in this class to be relative to the
         new root. This is used to adjust the SessionData instance to work for the VRPC (one root) or the BioHPC server
-        (another root). Since this is the only subclass used by both the VRPC and the BioHPC server, this method is
-        only implemented for this class.
+        (another root). This method is only implemented for subclasses intended to be used both locally and on the
+        BioHPC server.
 
         Args:
             new_root: The new root directory to use for all paths inside the instance. This has to be the path to the
@@ -494,15 +491,20 @@ class ProcessedData:
     """Stores the paths to the directories and files that make up the 'processed_data' session directory.
 
     The processed_data directory stores the processed session data, which is generated by running various processing
-    pipelines on the BioHPC server. These pipelines use raw data to generate processed data, and the processed data is
-    usually only stored on the BioHPC server. Processed data represents an intermediate step between raw data and the
-    dataset used in the data analysis.
+    pipelines. These pipelines use raw data to generate processed data, which represents an intermediate step between
+    raw data and the dataset used in the data analysis.
+
+    Notes:
+        The paths from this section are typically used only on the BioHPC server. This is because most data processing
+        in the lab is performed using the processing server's resources. On the server, processed data is stored on
+        the fast volume, in contrast to raw data, which is stored on the slow volume. However, to support local testing,
+        the class resolves the paths in this section both locally and globally (on the server).
+
     """
 
     processed_data_path: str | Path
     """Stores the path to the root processed_data directory of the session. This directory stores the processed data 
-    as it is generated by various pipelines running on the BioHPC server. This directory is only stored on the BioHPC 
-    server and is not intended to be used directly for data analysis."""
+    as it is generated by various data processing pipelines."""
     camera_data_path: str | Path
     """Stores the output of the DeepLabCut pose estimation pipeline."""
     mesoscope_data_path: str | Path
@@ -556,6 +558,29 @@ class ProcessedData:
         ensure_directory_exists(Path(self.deeplabcut_root_path))
         ensure_directory_exists(Path(self.suite2p_configuration_path))
 
+    def switch_root(self, new_root: Path) -> None:
+        """Changes the root of the managed processed_data directory to the provided root path.
+
+        This service method is used by the SessionData class to convert all paths in this class to be relative to the
+        new root. This is used to adjust the SessionData instance to work for the VRPC (one root) or the BioHPC server
+        (another root). This method is only implemented for subclasses intended to be used both locally and on the
+        BioHPC server.
+
+        Args:
+            new_root: The new root directory to use for all paths inside the instance. This has to be the path to the
+                root session directory: pc_root/project/animal/session.
+        """
+        # Gets current root from the processed_data_path.
+        old_root = Path(self.processed_data_path).parents[2]
+
+        # Updates all paths by replacing old_root with new_root
+        self.processed_data_path = new_root.joinpath(Path(self.processed_data_path).relative_to(old_root))
+        self.camera_data_path = new_root.joinpath(Path(self.camera_data_path).relative_to(old_root))
+        self.mesoscope_data_path = new_root.joinpath(Path(self.mesoscope_data_path).relative_to(old_root))
+        self.behavior_data_path = new_root.joinpath(Path(self.behavior_data_path).relative_to(old_root))
+        self.deeplabcut_root_path = new_root.joinpath(Path(self.deeplabcut_root_path).relative_to(old_root))
+        self.suite2p_configuration_path = new_root.joinpath(Path(self.suite2p_configuration_path).relative_to(old_root))
+
 
 @dataclass()
 class PersistentData:
@@ -735,7 +760,7 @@ class SessionData(YamlConfig):
     """Stores the name (timestamp-based ID) of the managed session."""
     session_type: str
     """Stores the type of the session. Primarily, this determines how to read the session_descriptor.yaml file. Has 
-    to be set to one of the three supported types: 'Lick training', 'Run training' or 'Experiment'.
+    to be set to one of the four supported types: 'Lick training', 'Run training', 'Window checking' or 'Experiment'.
     """
     experiment_name: str | None
     """Stores the name of the experiment configuration file. If the session_type field is set to 'Experiment', this 
@@ -747,8 +772,9 @@ class SessionData(YamlConfig):
     class initialization location (VRPC or BioHPC server), the class automatically resolves the root directory path to 
     either the VRPC project directory or the BioHPC cluster storage volume."""
     processed_data: ProcessedData
-    """Stores the paths to various directories used to store processed session data. This is automatically 
-    resolved to the fast BioHPC volume (workdir) in all cases, as processed data should only exist on the server."""
+    """Stores the paths to various directories used to store processed session data. Note, when this section is 
+    resolved for VRPC, it uses the same local session directory as the raw_data folder. When this is resolved for the 
+    BioHPC server, it uses the 'fast' volume path."""
     persistent_data: PersistentData
     """Stores the paths to various files and directories kept on VRPC and ScanImagePC after the session data is 
     transferred to long-term storage destinations."""
@@ -806,16 +832,11 @@ class SessionData(YamlConfig):
         biohpc_root = Path(project_configuration.local_server_directory)
         nas_root = Path(project_configuration.local_nas_directory)
 
-        # Also extracts the path to fast (working) directory on the BioHPC server. This is used to configure the
-        # paths for data processing, which happens on the server.
-        biohpc_workdir = Path(project_configuration.remote_working_directory)
-
         # Extracts the name of the project stored inside the project configuration file.
         project_name = project_configuration.project_name
 
         # Constructs the session directory path and generates the directory
         session_path = vrpc_root.joinpath(project_name, animal_id, session_name)
-        remote_session_path = biohpc_workdir.joinpath(project_name, animal_id, session_name)
 
         # Handles potential session name conflicts
         counter = 0
@@ -823,7 +844,6 @@ class SessionData(YamlConfig):
             counter += 1
             new_session_name = f"{session_name}_{counter}"
             session_path = vrpc_root.joinpath(project_name, animal_id, new_session_name)
-            remote_session_path = biohpc_workdir.joinpath(project_name, animal_id, new_session_name)
 
         # If a conflict is detected and resolved, warns the user about the resolved conflict.
         if counter > 0:
@@ -854,14 +874,12 @@ class SessionData(YamlConfig):
         raw_data.make_dirs()  # Generates the local directory tree
 
         processed_data = ProcessedData(
-            processed_data_path=remote_session_path.joinpath("processed_data"),
-            camera_data_path=remote_session_path.joinpath("processed_data", "camera_data"),
-            mesoscope_data_path=remote_session_path.joinpath("processed_data", "mesoscope_data"),
-            behavior_data_path=remote_session_path.joinpath("processed_data", "behavior_data"),
-            deeplabcut_root_path=biohpc_workdir.joinpath(project_name, "deeplabcut"),
-            suite2p_configuration_path=biohpc_workdir.joinpath(
-                project_name, "configuration", "suite2p_configuration.yaml"
-            ),
+            processed_data_path=session_path.joinpath("processed_data"),
+            camera_data_path=session_path.joinpath("processed_data", "camera_data"),
+            mesoscope_data_path=session_path.joinpath("processed_data", "mesoscope_data"),
+            behavior_data_path=session_path.joinpath("processed_data", "behavior_data"),
+            deeplabcut_root_path=vrpc_root.joinpath(project_name, "deeplabcut"),
+            suite2p_configuration_path=vrpc_root.joinpath(project_name, "configuration", "suite2p_configuration.yaml"),
         )
 
         vrpc_persistent_path = vrpc_root.joinpath(project_name, animal_id, "persistent_data")
@@ -968,17 +986,32 @@ class SessionData(YamlConfig):
         # Loads class data from .yaml
         instance: SessionData = cls.from_yaml(file_path=session_path)  # type: ignore
 
-        # Depending on whether the class is initialized on the VRPC or BioHPC server, resolves the local (raw_data)
-        # directory path. With the way this class is used, if on_server is False, the class is already
-        # well-configured. This is because the class is always created on the VRPC and, when it leaves VRPC, it is
-        # never used on VRPC again. Therefore, additional processing is ONLY done when on_server is True.
-        if on_server:
-            # Reconfigures the raw_data section to use the root provided as part of the session_path.
-            instance.raw_data.switch_root(new_root=session_path)
+        # The method assumes that the 'donor' .yaml file is always stored inside the raw_data directory of the session
+        # to be processed. Since the directory itself might have moved (between or even within the same PC) relative to
+        # where it was when the SessionData snapshot was generated, reconfigures the paths to all raw_data files using
+        # the root from above.
+        instance.raw_data.switch_root(new_root=session_path)
+
+        # Resolves the paths to the processed_data directories. The resolution strategy depends on whether the method is
+        # called on the VRPC (locally) or the BioHPC server (remotely).
+        if not on_server:
+            # Local runtimes use the same root session directory for both raw_data and processed_data. This stems from
+            # the assumption that most local machines in the lab only use NVME (fast) volumes and, therefore, do not
+            # need to separate 'storage' and 'working' data.
+            instance.processed_data.switch_root(new_root=session_path)
+
+        else:
+            # The BioHPC server stores raw_data on slow volume and processed_data on fast (NVME) volume. Therefore, to
+            # configure processed_data paths, the method first needs to load the fast volume root path from the
+            # project_configuration.yaml file stored in the raw_data folder.
+            project_configuration: ProjectConfiguration = ProjectConfiguration.load(
+                project_name=instance.project_name,
+                configuration_path=Path(instance.raw_data.project_configuration_path),
+            )
+            instance.processed_data.switch_root(new_root=Path(project_configuration.remote_working_directory))
 
-            # Processed Data section is always configured to use the BioHPC server root. Calls its' make_dirs() method
-            # to setup directories
-            instance.processed_data.make_dirs()
+        # Generates processed_data directories
+        instance.processed_data.make_dirs()
 
         # Returns the initialized SessionData instance to caller
         return instance
@@ -1331,6 +1364,10 @@ class ProcedureData:
     """Stores surgeon's notes taken during the surgery."""
     post_op_notes: str
     """Stores surgeon's notes taken during the post-surgery recovery period."""
+    surgery_quality: int = 0
+    """Stores the quality of the surgical intervention as a numeric level. 0 indicates unusable (bad) result, 1 
+    indicates usable result that is not good enough to be included in a publication, 2 indicates publication-grade 
+    result."""
 
 
 @dataclass

{sl_shared_assets-1.0.0rc5 → sl_shared_assets-1.0.0rc7}/src/sl_shared_assets/data_classes.pyi

@@ -163,8 +163,8 @@ class RawData:
 
         This service method is used by the SessionData class to convert all paths in this class to be relative to the
         new root. This is used to adjust the SessionData instance to work for the VRPC (one root) or the BioHPC server
-        (another root). Since this is the only subclass used by both the VRPC and the BioHPC server, this method is
-        only implemented for this class.
+        (another root). This method is only implemented for subclasses intended to be used both locally and on the
+        BioHPC server.
 
         Args:
             new_root: The new root directory to use for all paths inside the instance. This has to be the path to the
@@ -176,9 +176,15 @@ class ProcessedData:
     """Stores the paths to the directories and files that make up the 'processed_data' session directory.
 
     The processed_data directory stores the processed session data, which is generated by running various processing
-    pipelines on the BioHPC server. These pipelines use raw data to generate processed data, and the processed data is
-    usually only stored on the BioHPC server. Processed data represents an intermediate step between raw data and the
-    dataset used in the data analysis.
+    pipelines. These pipelines use raw data to generate processed data, which represents an intermediate step between
+    raw data and the dataset used in the data analysis.
+
+    Notes:
+        The paths from this section are typically used only on the BioHPC server. This is because most data processing
+        in the lab is performed using the processing server's resources. On the server, processed data is stored on
+        the fast volume, in contrast to raw data, which is stored on the slow volume. However, to support local testing,
+        the class resolves the paths in this section both locally and globally (on the server).
+
     """
 
     processed_data_path: str | Path
@@ -203,6 +209,18 @@ class ProcessedData:
         This method is used by the BioHPC server to generate the processed_data directory as part of the sl-forgery
         library runtime.
         """
+    def switch_root(self, new_root: Path) -> None:
+        """Changes the root of the managed processed_data directory to the provided root path.
+
+        This service method is used by the SessionData class to convert all paths in this class to be relative to the
+        new root. This is used to adjust the SessionData instance to work for the VRPC (one root) or the BioHPC server
+        (another root). This method is only implemented for subclasses intended to be used both locally and on the
+        BioHPC server.
+
+        Args:
+            new_root: The new root directory to use for all paths inside the instance. This has to be the path to the
+                root session directory: pc_root/project/animal/session.
+        """
 
 @dataclass()
 class PersistentData:
@@ -586,6 +604,7 @@ class ProcedureData:
     protocol: str
     surgery_notes: str
     post_op_notes: str
+    surgery_quality: int = ...
 
 @dataclass
 class ImplantData: