sl-shared-assets 3.1.2__py3-none-any.whl → 4.0.0__py3-none-any.whl

sl_shared_assets/__init__.py

@@ -11,6 +11,7 @@ from .tools import (
     ProjectManifest,
     resolve_p53_marker,
     transfer_directory,
+    generate_project_manifest,
     calculate_directory_checksum,
 )
 from .server import Job, Server, JupyterJob, ServerCredentials
@@ -30,6 +31,7 @@ from .data_classes import (
     ExperimentState,
     ExperimentTrial,
     MesoscopeCameras,
+    TrackerFileNames,
     ProcessingTracker,
     AcquisitionSystems,
     MesoscopePositions,
@@ -42,6 +44,8 @@ from .data_classes import (
     MesoscopeSystemConfiguration,
     MesoscopeExperimentDescriptor,
     MesoscopeExperimentConfiguration,
+    generate_manager_id,
+    get_processing_tracker,
     get_system_configuration_data,
     set_system_configuration_file,
 )
@@ -86,9 +90,13 @@ __all__ = [
     "SessionTypes",
     "AcquisitionSystems",
     "WindowCheckingDescriptor",
+    "get_processing_tracker",
+    "generate_manager_id",
+    "TrackerFileNames",
     # Tools package
     "ProjectManifest",
     "resolve_p53_marker",
     "transfer_directory",
     "calculate_directory_checksum",
+    "generate_project_manifest",
 ]

sl_shared_assets/__init__.pyi

@@ -2,6 +2,7 @@ from .tools import (
     ProjectManifest as ProjectManifest,
     resolve_p53_marker as resolve_p53_marker,
     transfer_directory as transfer_directory,
+    generate_project_manifest as generate_project_manifest,
     calculate_directory_checksum as calculate_directory_checksum,
 )
 from .server import (
@@ -26,6 +27,7 @@ from .data_classes import (
     ExperimentState as ExperimentState,
     ExperimentTrial as ExperimentTrial,
     MesoscopeCameras as MesoscopeCameras,
+    TrackerFileNames as TrackerFileNames,
     ProcessingTracker as ProcessingTracker,
     AcquisitionSystems as AcquisitionSystems,
     MesoscopePositions as MesoscopePositions,
@@ -38,6 +40,8 @@ from .data_classes import (
     MesoscopeSystemConfiguration as MesoscopeSystemConfiguration,
     MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
     MesoscopeExperimentConfiguration as MesoscopeExperimentConfiguration,
+    generate_manager_id as generate_manager_id,
+    get_processing_tracker as get_processing_tracker,
     get_system_configuration_data as get_system_configuration_data,
     set_system_configuration_file as set_system_configuration_file,
 )
@@ -76,8 +80,12 @@ __all__ = [
     "SessionTypes",
     "AcquisitionSystems",
     "WindowCheckingDescriptor",
+    "get_processing_tracker",
+    "generate_manager_id",
+    "TrackerFileNames",
     "ProjectManifest",
     "resolve_p53_marker",
     "transfer_directory",
     "calculate_directory_checksum",
+    "generate_project_manifest",
 ]

sl_shared_assets/cli.py

@@ -7,7 +7,7 @@ from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
 
 from .tools import ascend_tyche_data, resolve_p53_marker, verify_session_checksum, generate_project_manifest
 from .server import Server, JupyterJob, generate_server_credentials
-from .data_classes import SessionData, ProcessingTracker
+from .data_classes import SessionData, TrackerFileNames, get_processing_tracker
 
 
 @click.command()
@@ -18,6 +18,18 @@ from .data_classes import SessionData, ProcessingTracker
     required=True,
     help="The absolute path to the session directory whose raw data needs to be verified for potential corruption.",
 )
+@click.option(
+    "-id",
+    "--manager_id",
+    required=True,
+    default=0,
+    show_default=True,
+    help=(
+        "The xxHash-64 hash value that represents the unique identifier for the process that manages this runtime. "
+        "This is primarily used when calling this CLI on remote compute servers to ensure that only a single process "
+        "can execute the CLI at a time."
+    ),
+)
 @click.option(
     "-c",
     "--create_processed_directories",
@@ -26,7 +38,7 @@ from .data_classes import SessionData, ProcessingTracker
     default=False,
     help=(
         "Determines whether to create the processed data hierarchy. This flag should be disabled for most runtimes. "
-        "Primarily, it is used by lab acquisition system code to generate processed data directories on the remote "
+        "Primarily, it is used by acquisition systems to generate processed data directories on the remote "
         "compute servers as part of the data preprocessing pipeline."
     ),
 )
@@ -54,7 +66,11 @@ from .data_classes import SessionData, ProcessingTracker
     ),
 )
 def verify_session_integrity(
-    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None, update_manifest: bool
+    session_path: Path,
+    manager_id: int,
+    create_processed_directories: bool,
+    processed_data_root: Path | None,
+    update_manifest: bool,
 ) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
@@ -73,14 +89,15 @@ def verify_session_integrity(
 
     # Runs the verification process
     verify_session_checksum(
-        session,
+        session_path=session,
+        manager_id=manager_id,
         create_processed_data_directory=create_processed_directories,
         processed_data_root=processed_data_root,
         update_manifest=update_manifest,
     )
 
     # Checks the outcome of the verification process
-    tracker = ProcessingTracker(file_path=session_data.raw_data.integrity_verification_tracker_path)
+    tracker = get_processing_tracker(root=session_data.raw_data.raw_data_path, file_name=TrackerFileNames.INTEGRITY)
     if tracker.is_complete:
         # noinspection PyTypeChecker
         console.echo(message=f"Session {session.stem} raw data integrity: Verified.", level=LogLevel.SUCCESS)
@@ -105,18 +122,21 @@ def verify_session_integrity(
     help="The absolute path to the directory where to store the generated project manifest file.",
 )
 @click.option(
-    "-ppp",
-    "--project_processed_path",
+    "-pdr",
+    "--processed_data_root",
     type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
     required=False,
     help=(
-        "The absolute path to the project directory where processed session data is stored, if different from the "
-        "directory used to store raw session data. Typically, this extra argument is only used when processing data "
-        "stored on remote compute server(s)."
+        "The absolute path to the directory where processed data from all projects is stored on the machine that runs "
+        "this command. This argument is used when calling the CLI on the BioHPC server, which uses different data "
+        "volumes for raw and processed data. Note, the input path must point to the root directory, as it will be "
+        "automatically modified to include the project name. Note, if the system cannot properly resolve the path to "
+        "the processed data, the generated manifest will indicate that no data processing has been performed for the "
+        "project."
     ),
 )
 def generate_project_manifest_file(
-    project_path: Path, output_directory: Path, project_processed_path: Path | None
+    project_path: Path, output_directory: Path, processed_data_root: Path | None
 ) -> None:
     """Generates the manifest .feather file that provides information about the data-processing state of all available
     project sessions.
@@ -128,7 +148,7 @@ def generate_project_manifest_file(
     generate_project_manifest(
         raw_project_directory=Path(project_path),
         output_directory=Path(output_directory),
-        processed_data_root=Path(project_processed_path) if project_processed_path else None,
+        processed_data_root=Path(processed_data_root) if processed_data_root else None,
     )
     # noinspection PyTypeChecker
     console.echo(message=f"Project {Path(project_path).stem} data manifest file: generated.", level=LogLevel.SUCCESS)
@@ -428,14 +448,15 @@ def start_jupyter_server(
     help="Determines whether to create the processed data hierarchy. This flag should be disabled for most runtimes.",
 )
 @click.option(
-    "-ppp",
-    "--project_processed_path",
+    "-pdr",
+    "--processed_data_root",
     type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
     required=False,
     help=(
-        "The absolute path to the project directory where processed session data is stored, if different from the "
-        "directory used to store raw session data. Typically, this extra argument is only used when processing data "
-        "stored on remote compute server(s)."
+        "The absolute path to the directory where processed data from all projects is stored on the machine that runs "
+        "this command. This argument is used when calling the CLI on the BioHPC server, which uses different data "
+        "volumes for raw and processed data. Note, the input path must point to the root directory, as it will be "
+        "automatically modified to include the project name."
     ),
 )
 @click.option(
@@ -444,11 +465,7 @@ def start_jupyter_server(
     is_flag=True,
     show_default=True,
     default=False,
-    help=(
-        "Determines whether the command should create or remove the dataset integration marker. Do not enable this "
-        "flag unless you know what you are doing. It is only safe to enable this flag if the session is not currently "
-        "being integrated into any datasets."
-    ),
+    help="Determines whether the command should create or remove the dataset integration marker.",
 )
 @click.option(
     "-um",
@@ -463,7 +480,7 @@ def start_jupyter_server(
 def resolve_dataset_marker(
     session_path: Path,
     create_processed_directories: bool,
-    project_processed_path: Path | None,
+    processed_data_root: Path | None,
     remove: bool,
     update_manifest: bool,
 ) -> None:
@@ -477,7 +494,7 @@ def resolve_dataset_marker(
     resolve_p53_marker(
         session_path=session_path,
         create_processed_data_directory=create_processed_directories,
-        processed_data_root=project_processed_path,
+        processed_data_root=processed_data_root,
         remove=remove,
         update_manifest=update_manifest,
     )

sl_shared_assets/cli.pyi

@@ -13,11 +13,16 @@ from .server import (
 )
 from .data_classes import (
     SessionData as SessionData,
-    ProcessingTracker as ProcessingTracker,
+    TrackerFileNames as TrackerFileNames,
+    get_processing_tracker as get_processing_tracker,
 )
 
 def verify_session_integrity(
-    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None, update_manifest: bool
+    session_path: Path,
+    manager_id: int,
+    create_processed_directories: bool,
+    processed_data_root: Path | None,
+    update_manifest: bool,
 ) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
@@ -33,7 +38,7 @@ def verify_session_integrity(
     """
 
 def generate_project_manifest_file(
-    project_path: Path, output_directory: Path, project_processed_path: Path | None
+    project_path: Path, output_directory: Path, processed_data_root: Path | None
 ) -> None:
     """Generates the manifest .feather file that provides information about the data-processing state of all available
     project sessions.
@@ -88,7 +93,7 @@ def start_jupyter_server(
 def resolve_dataset_marker(
     session_path: Path,
     create_processed_directories: bool,
-    project_processed_path: Path | None,
+    processed_data_root: Path | None,
     remove: bool,
     update_manifest: bool,
 ) -> None:

sl_shared_assets/data_classes/__init__.py

@@ -12,7 +12,16 @@ from .runtime_data import (
     WindowCheckingDescriptor,
     MesoscopeExperimentDescriptor,
 )
-from .session_data import RawData, SessionData, SessionTypes, ProcessedData, ProcessingTracker
+from .session_data import (
+    RawData,
+    SessionData,
+    SessionTypes,
+    ProcessedData,
+    TrackerFileNames,
+    ProcessingTracker,
+    generate_manager_id,
+    get_processing_tracker,
+)
 from .surgery_data import (
     DrugData,
     ImplantData,
@@ -65,4 +74,7 @@ __all__ = [
     "AcquisitionSystems",
     "SessionTypes",
     "WindowCheckingDescriptor",
+    "get_processing_tracker",
+    "generate_manager_id",
+    "TrackerFileNames",
 ]

sl_shared_assets/data_classes/__init__.pyi

@@ -12,7 +12,10 @@ from .session_data import (
     SessionData as SessionData,
     SessionTypes as SessionTypes,
     ProcessedData as ProcessedData,
+    TrackerFileNames as TrackerFileNames,
     ProcessingTracker as ProcessingTracker,
+    generate_manager_id as generate_manager_id,
+    get_processing_tracker as get_processing_tracker,
 )
 from .surgery_data import (
     DrugData as DrugData,
@@ -66,4 +69,7 @@ __all__ = [
     "AcquisitionSystems",
     "SessionTypes",
     "WindowCheckingDescriptor",
+    "get_processing_tracker",
+    "generate_manager_id",
+    "TrackerFileNames",
 ]

sl_shared_assets/data_classes/configuration_data.py

@@ -560,7 +560,7 @@ def get_system_configuration_data() -> MesoscopeSystemConfiguration:
     app_dir = Path(appdirs.user_data_dir(appname="sun_lab_data", appauthor="sun_lab"))
     path_file = app_dir.joinpath("configuration_path.txt")
 
-    # If the cache file or the Sun lab data directory do not exist, aborts with an error
+    # If the cache file or the Sun lab data directory does not exist, aborts with an error
     if not path_file.exists():
         message = (
             "Unable to resolve the path to the local system configuration file, as local machine does not have the "

sl_shared_assets/data_classes/runtime_data.py

@@ -87,9 +87,6 @@ class LickTrainingDescriptor(YamlConfig):
     """The ID of the experimenter running the session."""
     mouse_weight_g: float
     """The weight of the animal, in grams, at the beginning of the session."""
-    dispensed_water_volume_ml: float
-    """Stores the total water volume, in milliliters, dispensed during runtime. This excludes the water volume 
-    dispensed during the paused (idle) state."""
     minimum_reward_delay_s: int
     """Stores the minimum delay, in seconds, that can separate the delivery of two consecutive water rewards."""
     maximum_reward_delay_s: int
@@ -102,6 +99,9 @@ class LickTrainingDescriptor(YamlConfig):
     """Stores the maximum number of consecutive rewards that can be delivered without the animal consuming them. If 
     the animal receives this many rewards without licking (consuming) them, reward delivery is paused until the animal 
     consumes the rewards."""
+    dispensed_water_volume_ml: float = 0.0
+    """Stores the total water volume, in milliliters, dispensed during runtime. This excludes the water volume 
+    dispensed during the paused (idle) state."""
     pause_dispensed_water_volume_ml: float = 0.0
     """Stores the total water volume, in milliliters, dispensed during the paused (idle) state."""
     experimenter_notes: str = "Replace this with your notes."
@@ -110,6 +110,9 @@ class LickTrainingDescriptor(YamlConfig):
     experimenter_given_water_volume_ml: float = 0.0
     """The additional volume of water, in milliliters, administered by the experimenter to the animal after the session.
     """
+    preferred_session_water_volume_ml: float = 0.0
+    """The volume of water, in milliliters, the animal should be receiving during the session runtime if its 
+    performance matches experimenter-specified threshold."""
     incomplete: bool = False
     """If this field is set to True, the session is marked as 'incomplete' and automatically excluded from all further 
     Sun lab automated processing and analysis."""
@@ -123,8 +126,6 @@ class RunTrainingDescriptor(YamlConfig):
     """The ID of the experimenter running the session."""
     mouse_weight_g: float
     """The weight of the animal, in grams, at the beginning of the session."""
-    dispensed_water_volume_ml: float
-    """Stores the total water volume, in milliliters, dispensed during runtime."""
     final_run_speed_threshold_cm_s: float
     """Stores the final running speed threshold, in centimeters per second, that was active at the end of training."""
     final_run_duration_threshold_s: float
@@ -154,6 +155,9 @@ class RunTrainingDescriptor(YamlConfig):
     """Stores the maximum time, in seconds, the animal can dip below the running speed threshold to still receive the 
     reward. This allows animals that 'run' by taking a series of large steps, briefly dipping below speed threshold at 
     the end of each step, to still get water rewards."""
+    dispensed_water_volume_ml: float = 0.0
+    """Stores the total water volume, in milliliters, dispensed during runtime. This excludes the water volume 
+    dispensed during the paused (idle) state."""
     pause_dispensed_water_volume_ml: float = 0.0
     """Stores the total water volume, in milliliters, dispensed during the paused (idle) state."""
     experimenter_notes: str = "Replace this with your notes."
@@ -162,6 +166,9 @@ class RunTrainingDescriptor(YamlConfig):
     experimenter_given_water_volume_ml: float = 0.0
     """The additional volume of water, in milliliters, administered by the experimenter to the animal after the session.
     """
+    preferred_session_water_volume_ml: float = 0.0
+    """The volume of water, in milliliters, the animal should be receiving during the session runtime if its 
+    performance matches experimenter-specified threshold."""
     incomplete: bool = False
     """If this field is set to True, the session is marked as 'incomplete' and automatically excluded from all further 
     Sun lab automated processing and analysis."""
@@ -175,20 +182,24 @@ class MesoscopeExperimentDescriptor(YamlConfig):
     """The ID of the experimenter running the session."""
     mouse_weight_g: float
     """The weight of the animal, in grams, at the beginning of the session."""
-    dispensed_water_volume_ml: float
-    """Stores the total water volume, in milliliters, dispensed during runtime."""
     maximum_unconsumed_rewards: int = 1
     """Stores the maximum number of consecutive rewards that can be delivered without the animal consuming them. If 
     the animal receives this many rewards without licking (consuming) them, reward delivery is paused until the animal 
     consumes the rewards."""
+    dispensed_water_volume_ml: float = 0.0
+    """Stores the total water volume, in milliliters, dispensed during runtime. This excludes the water volume 
+    dispensed during the paused (idle) state."""
+    pause_dispensed_water_volume_ml: float = 0.0
+    """Stores the total water volume, in milliliters, dispensed during the paused (idle) state."""
     experimenter_notes: str = "Replace this with your notes."
     """This field is not set during runtime. It is expected that each experimenter will replace this field with their 
     notes made during runtime."""
-    pause_dispensed_water_volume_ml: float = 0.0
-    """Stores the total water volume, in milliliters, dispensed during the paused (idle) state."""
     experimenter_given_water_volume_ml: float = 0.0
     """The additional volume of water, in milliliters, administered by the experimenter to the animal after the session.
     """
+    preferred_session_water_volume_ml: float = 0.0
+    """The volume of water, in milliliters, the animal should be receiving during the session runtime if its 
+    performance matches experimenter-specified threshold."""
     incomplete: bool = False
     """If this field is set to True, the session is marked as 'incomplete' and automatically excluded from all further 
     Sun lab automated processing and analysis."""

sl_shared_assets/data_classes/runtime_data.pyi

@@ -49,15 +49,16 @@ class LickTrainingDescriptor(YamlConfig):
 
     experimenter: str
     mouse_weight_g: float
-    dispensed_water_volume_ml: float
     minimum_reward_delay_s: int
     maximum_reward_delay_s: int
     maximum_water_volume_ml: float
     maximum_training_time_m: int
     maximum_unconsumed_rewards: int = ...
+    dispensed_water_volume_ml: float = ...
     pause_dispensed_water_volume_ml: float = ...
     experimenter_notes: str = ...
     experimenter_given_water_volume_ml: float = ...
+    preferred_session_water_volume_ml: float = ...
     incomplete: bool = ...
 
 @dataclass()
@@ -66,7 +67,6 @@ class RunTrainingDescriptor(YamlConfig):
 
     experimenter: str
     mouse_weight_g: float
-    dispensed_water_volume_ml: float
     final_run_speed_threshold_cm_s: float
     final_run_duration_threshold_s: float
     initial_run_speed_threshold_cm_s: float
@@ -78,9 +78,11 @@ class RunTrainingDescriptor(YamlConfig):
     maximum_training_time_m: int
     maximum_unconsumed_rewards: int = ...
     maximum_idle_time_s: float = ...
+    dispensed_water_volume_ml: float = ...
     pause_dispensed_water_volume_ml: float = ...
     experimenter_notes: str = ...
     experimenter_given_water_volume_ml: float = ...
+    preferred_session_water_volume_ml: float = ...
     incomplete: bool = ...
 
 @dataclass()
@@ -89,11 +91,12 @@ class MesoscopeExperimentDescriptor(YamlConfig):
 
     experimenter: str
     mouse_weight_g: float
-    dispensed_water_volume_ml: float
     maximum_unconsumed_rewards: int = ...
-    experimenter_notes: str = ...
+    dispensed_water_volume_ml: float = ...
     pause_dispensed_water_volume_ml: float = ...
+    experimenter_notes: str = ...
     experimenter_given_water_volume_ml: float = ...
+    preferred_session_water_volume_ml: float = ...
     incomplete: bool = ...
 
 @dataclass()