sl-shared-assets 1.0.0rc22__tar.gz → 1.0.0rc23__tar.gz

{sl_shared_assets-1.0.0rc22 → sl_shared_assets-1.0.0rc23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 1.0.0rc22
+Version: 1.0.0rc23
 Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

{sl_shared_assets-1.0.0rc22 → sl_shared_assets-1.0.0rc23}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 # Project metdata section. Provides the genral ID information about the project.
 [project]
 name = "sl-shared-assets"
-version = "1.0.0rc22"
+version = "1.0.0rc23"
 description = "Stores assets shared between multiple Sun (NeuroAI) lab data pipelines."
 readme = "README.md"
 license = { file = "LICENSE" }

{sl_shared_assets-1.0.0rc22 → sl_shared_assets-1.0.0rc23}/src/sl_shared_assets/cli.py

@@ -25,7 +25,32 @@ from .data_classes import (
     required=True,
     help="The absolute path to the session whose raw data needs to be verified for potential corruption.",
 )
-def verify_session_integrity(session_path: str) -> None:
+@click.option(
+    "-c",
+    "--create_processed_directories",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help=(
+        "Determines whether to created the processed data hierarchy. This flag should be disabled for most runtimes. "
+        "Primarily, it is used by lab acquisition system code to generate processed data directories on the remote "
+        "compute servers as part of the data preprocessing pipeline."
+    ),
+)
+@click.option(
+    "-pdr",
+    "--processed_data_root",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=False,
+    help=(
+        "The absolute path to the directory where processed data from all projects is stored on the machine that runs "
+        "this command. This argument is used when calling the CLI on the BioHPC server, which uses different data "
+        "volumes for raw and processed data. Note, the input path must point to the root directory, as it will be "
+        "automatically modified to include the project name, the animal id, and the session ID. This argument is only "
+        "used if 'create_processed_directories' flag is True."
+    ),
+)
+def verify_session_integrity(session_path: str, create_processed_directories: bool, processed_data_root: Path) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
     This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
@@ -33,9 +58,14 @@ def verify_session_integrity(session_path: str) -> None:
     always verified the integrity of the 'raw_data' directory. It does not work with 'processed_data' or any other
     directories. If the session data was corrupted, the command removes the 'telomere.bin' file, marking the session as
     'incomplete' and automatically excluding it from all further automated processing runtimes.
+
+    The command is also used by Sun lab data acquisition systems to generate the processed data hierarchy for each
+    processed session. This use case is fully automated and should not be triggered manually by the user.
     """
     session = Path(session_path)
-    if verify_session_checksum(session):
+    if verify_session_checksum(
+        session, create_processed_data_directory=create_processed_directories, processed_data_root=processed_data_root
+    ):
         console.echo(message=f"Session {session.stem} raw data integrity: verified.", level=LogLevel.SUCCESS)
     else:
         console.echo(message=f"Session {session.stem} raw data integrity: compromised!", level=LogLevel.ERROR)
@@ -288,6 +318,15 @@ def generate_experiment_configuration_file(project: str, experiment: str, state_
     acquisition_system = get_system_configuration_data()
     file_path = acquisition_system.paths.root_directory.joinpath(project, "configuration", f"{experiment}.yaml")
 
+    if not acquisition_system.paths.root_directory.joinpath(project).exists():
+        message = (
+            f"Unable to generate the experiment {experiment} configuration file for the project {project}. "
+            f"The target project does not exist on the local machine (PC). Use the "
+            f"'sl-create-project' CLI command to create the project before creating new experiment configuration(s). "
+        )
+        console.error(message=message, error=ValueError)
+        raise ValueError(message)  # Fall-back to appease mypy, should not be reachable
+
     # Loops over the number of requested states and, for each, generates a precursor experiment state field inside the
     # 'states' dictionary.
     states = {}

{sl_shared_assets-1.0.0rc22 → sl_shared_assets-1.0.0rc23}/src/sl_shared_assets/cli.pyi

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 from .tools import (
     ascend_tyche_data as ascend_tyche_data,
     verify_session_checksum as verify_session_checksum,
@@ -13,7 +15,7 @@ from .data_classes import (
     set_system_configuration_file as set_system_configuration_file,
 )
 
-def verify_session_integrity(session_path: str) -> None:
+def verify_session_integrity(session_path: str, create_processed_directories: bool, processed_data_root: Path) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
     This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
@@ -21,6 +23,9 @@ def verify_session_integrity(session_path: str) -> None:
     always verified the integrity of the 'raw_data' directory. It does not work with 'processed_data' or any other
     directories. If the session data was corrupted, the command removes the 'telomere.bin' file, marking the session as
     'incomplete' and automatically excluding it from all further automated processing runtimes.
+
+    The command is also used by Sun lab data acquisition systems to generate the processed data hierarchy for each
+    processed session. This use case is fully automated and should not be triggered manually by the user.
     """
 
 def generate_project_manifest_file(

{sl_shared_assets-1.0.0rc22 → sl_shared_assets-1.0.0rc23}/src/sl_shared_assets/data_classes/configuration_data.py

@@ -111,6 +111,9 @@ class MesoscopePaths:
     sharing protocol, such as SMB."""
     harvesters_cti_path: Path = Path("/opt/mvIMPACT_Acquire/lib/x86_64/mvGenTLProducer.cti")
     """The path to the GeniCam CTI file used to connect to Harvesters-managed cameras."""
+    server_processed_data_root: Path = Path("/workdir/sun_data")
+    """The absolute path to the BioHPC server directory used to store the processed data from all Sun lab projects. 
+    This path is relative to the server root and is only used when submitting remote jobs to the server."""
 
 
 @dataclass()
@@ -125,10 +128,17 @@ class MesoscopeCameras:
     right_camera_index: int = 2
     """The index of the right body camera (from animal's perspective) in the list of all available OpenCV-managed
      cameras."""
-    quantization_parameter: int = 15
-    """The quantization parameter used by all cameras to encode acquired frames as video files. This controls how much
-    data is discarded when encoding each video frame, directly contributing to the encoding speed, resultant video file
-    size and video quality."""
+    face_camera_quantization_parameter: int = 15
+    """The quantization parameter used by the face camera to encode acquired frames as video files. This controls how
+    much data is discarded when encoding each video frame, directly contributing to the encoding speed, resultant video 
+    file size and video quality."""
+    body_camera_quantization_parameter: int = 15
+    """SThe quantization parameter used by the left and right body cameras to encode acquired frames as video files.
+    See 'face_camera_quantization_parameter' field for more information on what this parameter does."""
+    display_face_camera_frames: bool = True
+    """Determines whether to display the frames grabbed from the face camera during runtime."""
+    display_body_camera_frames: bool = True
+    """Determines whether to display the frames grabbed from the left and right body cameras during runtime."""
 
 
 @dataclass()
@@ -141,14 +151,13 @@ class MesoscopeMicroControllers:
     """The USB port used by the Sensor Microcontroller."""
     encoder_port: str = "/dev/ttyACM2"
     """The USB port used by the Encoder Microcontroller."""
-    mesoscope_start_ttl_module_id: int = 1
-    """The unique byte-code ID of the TTL module instance used to send mesoscope frame acquisition start trigger 
-    signals to the ScanImagePC."""
-    mesoscope_stop_ttl_module_id: int = 2
-    """The unique byte-code ID of the TTL module instance used to send mesoscope frame acquisition stop trigger 
-    signals to the ScanImagePC."""
+    debug: bool = False
+    """Determines whether to run the managed acquisition system in the 'debug mode'. This mode should be disabled 
+    during most runtimes. It is used during initial system calibration and testing and prints a lot of generally 
+    redundant information into the terminal."""
     mesoscope_ttl_pulse_duration_ms: int = 10
-    """The duration of the HIGH phase of all outgoing mesoscope TTL pulses, in milliseconds."""
+    """The duration of the HIGH phase of all outgoing TTL pulses that target the Mesoscope (enable or disable mesoscope
+    frame acquisition), in milliseconds."""
     minimum_break_strength_g_cm: float = 43.2047
     """The minimum torque applied by the running wheel break in gram centimeter. This is the torque the break delivers 
     at minimum voltage (break is disabled)."""
@@ -292,6 +301,7 @@ class MesoscopeSystemConfiguration(YamlConfig):
         self.paths.nas_directory = Path(self.paths.nas_directory)
         self.paths.mesoscope_directory = Path(self.paths.mesoscope_directory)
         self.paths.harvesters_cti_path = Path(self.paths.harvesters_cti_path)
+        self.paths.server_processed_data_root = Path(self.paths.server_processed_data_root)
 
         # Converts valve_calibration data from dictionary to a tuple of tuples format
         if not isinstance(self.microcontrollers.valve_calibration_data, tuple):
@@ -299,6 +309,23 @@ class MesoscopeSystemConfiguration(YamlConfig):
                 (k, v) for k, v in self.microcontrollers.valve_calibration_data.items()
             )
 
+        # Verifies the contents of the valve calibration data loaded from the config file.
+        valve_calibration_data = self.microcontrollers.valve_calibration_data
+        if not all(
+            isinstance(item, tuple)
+            and len(item) == 2
+            and isinstance(item[0], (int, float))
+            and isinstance(item[1], (int, float))
+            for item in valve_calibration_data
+        ):
+            message = (
+                f"Unable to initialize the MesoscopeSystemConfiguration class. Expected each item under the "
+                f"'valve_calibration_data' field of the Mesoscope-VR acquisition system configuration .yaml file to be "
+                f"a tuple of two integer or float values, but instead encountered {valve_calibration_data} with at "
+                f"least one incompatible element."
+            )
+            console.error(message=message, error=TypeError)
+
     def save(self, path: Path) -> None:
         """Saves class instance data to disk as a 'mesoscope_system_configuration.yaml' file.
 
@@ -323,6 +350,7 @@ class MesoscopeSystemConfiguration(YamlConfig):
         original.paths.nas_directory = str(original.paths.nas_directory)  # type: ignore
         original.paths.mesoscope_directory = str(original.paths.mesoscope_directory)  # type: ignore
         original.paths.harvesters_cti_path = str(original.paths.harvesters_cti_path)  # type: ignore
+        original.paths.server_processed_data_root = str(original.paths.server_processed_data_root)  # type: ignore
 
         # Converts valve calibration data into dictionary format
         if isinstance(original.microcontrollers.valve_calibration_data, tuple):

{sl_shared_assets-1.0.0rc22 → sl_shared_assets-1.0.0rc23}/src/sl_shared_assets/data_classes/configuration_data.pyi

@@ -58,6 +58,7 @@ class MesoscopePaths:
     nas_directory: Path = ...
     mesoscope_directory: Path = ...
     harvesters_cti_path: Path = ...
+    server_processed_data_root: Path = ...
 
 @dataclass()
 class MesoscopeCameras:
@@ -66,7 +67,10 @@ class MesoscopeCameras:
     face_camera_index: int = ...
     left_camera_index: int = ...
     right_camera_index: int = ...
-    quantization_parameter: int = ...
+    face_camera_quantization_parameter: int = ...
+    body_camera_quantization_parameter: int = ...
+    display_face_camera_frames: bool = ...
+    display_body_camera_frames: bool = ...
 
 @dataclass()
 class MesoscopeMicroControllers:
@@ -75,8 +79,7 @@ class MesoscopeMicroControllers:
     actor_port: str = ...
     sensor_port: str = ...
     encoder_port: str = ...
-    mesoscope_start_ttl_module_id: int = ...
-    mesoscope_stop_ttl_module_id: int = ...
+    debug: bool = ...
     mesoscope_ttl_pulse_duration_ms: int = ...
     minimum_break_strength_g_cm: float = ...
     maximum_break_strength_g_cm: float = ...

{sl_shared_assets-1.0.0rc22 → sl_shared_assets-1.0.0rc23}/src/sl_shared_assets/data_classes/session_data.py

@@ -438,6 +438,16 @@ class SessionData(YamlConfig):
         # Constructs the root session directory path
         session_path = acquisition_system.paths.root_directory.joinpath(project_name, animal_id, session_name)
 
+        # Prevents creating new sessions for non-existent projects.
+        if not acquisition_system.paths.root_directory.joinpath(project_name).exists():
+            message = (
+                f"Unable to create the session directory hierarchy for the session {session_name} of the animal "
+                f"'{animal_id}' and project '{project_name}'. The project does not exist on the local machine (PC). "
+                f"Use the 'sl-create-project' CLI command to create the project on the local machine before creating "
+                f"new sessions."
+            )
+            console.error(message=message, error=FileNotFoundError)
+
         # Handles potential session name conflicts
         counter = 0
         while session_path.exists():

{sl_shared_assets-1.0.0rc22 → sl_shared_assets-1.0.0rc23}/src/sl_shared_assets/tools/project_management_tools.py

@@ -5,6 +5,7 @@ up a given project."""
 from pathlib import Path
 
 import polars as pl
+from ataraxis_base_utilities import console
 
 from ..data_classes import SessionData
 from .packaging_tools import calculate_directory_checksum
@@ -33,9 +34,25 @@ def generate_project_manifest(
             is different from the 'raw_project_directory'. Typically, this would be the case on remote compute server(s)
             and not on local machines.
     """
+
+    if not raw_project_directory.exists():
+        message = (
+            f"Unable to generate the project manifest file for the requested project {raw_project_directory.stem}. The "
+            f"specified project directory does not exist."
+        )
+        console.error(message=message, error=FileNotFoundError)
+
     # Finds all raw data directories
     session_directories = [directory.parent for directory in raw_project_directory.rglob("raw_data")]
 
+    if len(session_directories) == 0:
+        message = (
+            f"Unable to generate the project manifest file for the requested project {raw_project_directory.stem}. The "
+            f"project does not contain any raw session data. To generate the manifest file, the project must contain "
+            f"at least one valid experiment or training session."
+        )
+        console.error(message=message, error=FileNotFoundError)
+
     # Precreates the 'manifest' dictionary structure
     manifest: dict[str, list[str | bool]] = {
         "animal": [],  # Animal IDs.
@@ -120,7 +137,9 @@ def generate_project_manifest(
     )
 
 
-def verify_session_checksum(session_path: Path) -> bool:
+def verify_session_checksum(
+    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+) -> bool:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
 
@@ -132,16 +151,27 @@ def verify_session_checksum(session_path: Path) -> bool:
         Removing the telomere.bin marker file from session's raw_data folder marks the session as incomplete, excluding
         it from all further automatic processing.
 
+        This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
+        part of the data preprocessing runtime performed by a data acquisition system.
+
     Args:
         session_path: The path to the session directory to be verified. Note, the input session directory must contain
             the 'raw_data' subdirectory.
+        create_processed_data_directory: Determines whether to create the processed data hierarchy during runtime.
+        processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
+            the root directory where to store the processed data from all projects, and it will be automatically
+            modified to include the project name, the animal name, and the session ID.
 
     Returns:
         True if the checksum matches, False otherwise.
     """
 
-    # Loads session data layout
-    session_data = SessionData.load(session_path=session_path)
+    # Loads session data layout. If configured to do so, also creates the processed data hierarchy
+    session_data = SessionData.load(
+        session_path=session_path,
+        processed_data_root=processed_data_root,
+        make_processed_data_directory=create_processed_data_directory,
+    )
 
     # Re-calculates the checksum for the raw_data directory
     calculated_checksum = calculate_directory_checksum(

{sl_shared_assets-1.0.0rc22 → sl_shared_assets-1.0.0rc23}/src/sl_shared_assets/tools/project_management_tools.pyi

@@ -27,7 +27,9 @@ def generate_project_manifest(
             and not on local machines.
     """
 
-def verify_session_checksum(session_path: Path) -> bool:
+def verify_session_checksum(
+    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+) -> bool:
     """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
     comparing it against the checksum stored in the ax_checksum.txt file.
 
@@ -39,9 +41,16 @@ def verify_session_checksum(session_path: Path) -> bool:
         Removing the telomere.bin marker file from session's raw_data folder marks the session as incomplete, excluding
         it from all further automatic processing.
 
+        This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
+        part of the data preprocessing runtime performed by a data acquisition system.
+
     Args:
         session_path: The path to the session directory to be verified. Note, the input session directory must contain
             the 'raw_data' subdirectory.
+        create_processed_data_directory: Determines whether to create the processed data hierarchy during runtime.
+        processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
+            the root directory where to store the processed data from all projects, and it will be automatically
+            modified to include the project name, the animal name, and the session ID.
 
     Returns:
         True if the checksum matches, False otherwise.