sl-shared-assets 1.0.0rc14__tar.gz → 1.0.0rc16__tar.gz

{sl_shared_assets-1.0.0rc14 → sl_shared_assets-1.0.0rc16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 1.0.0rc14
+Version: 1.0.0rc16
 Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

{sl_shared_assets-1.0.0rc14 → sl_shared_assets-1.0.0rc16}/docs/source/api.rst

@@ -21,29 +21,22 @@ Command Line Interfaces
    :prog: sl-ascend
    :nested: full
 
-Packaging Tools
-===============
-.. automodule:: sl_shared_assets.packaging_tools
+Tools
+=====
+.. automodule:: sl_shared_assets.tools
    :members:
    :undoc-members:
    :show-inheritance:
 
-Transfer Tools
-==============
-.. automodule:: sl_shared_assets.transfer_tools
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Suite2P Configuration Classes
-=============================
+Suite2P Configuration
+=====================
 .. automodule:: sl_shared_assets.suite2p
    :members:
    :undoc-members:
    :show-inheritance:
 
-General Configuration and Data Storage Classes
-==============================================
+General Data and Configuration Classes
+======================================
 .. automodule:: sl_shared_assets.data_classes
    :members:
    :undoc-members:
@@ -55,10 +48,3 @@ Compute Server Tools
    :members:
    :undoc-members:
    :show-inheritance:
-
-Ascension Tools
-===============
-.. automodule:: sl_shared_assets.ascension_tools
-   :members:
-   :undoc-members:
-   :show-inheritance:

{sl_shared_assets-1.0.0rc14 → sl_shared_assets-1.0.0rc16}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 # Project metdata section. Provides the genral ID information about the project.
 [project]
 name = "sl-shared-assets"
-version = "1.0.0rc14"
+version = "1.0.0rc16"
 description = "Stores assets shared between multiple Sun (NeuroAI) lab data pipelines."
 readme = "README.md"
 license = { file = "LICENSE" }

{sl_shared_assets-1.0.0rc14 → sl_shared_assets-1.0.0rc16}/src/sl_shared_assets/__init__.py

@@ -7,31 +7,35 @@ Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Yuantao Deng
 
 from ataraxis_base_utilities import console
 
+from .tools import transfer_directory, calculate_directory_checksum
 from .server import Server, ServerCredentials
-from .suite2p import (
-    Suite2PConfiguration,
-)
+from .suite2p import MultiDayS2PConfiguration, SingleDayS2PConfiguration
 from .data_classes import (
+    RawData,
     DrugData,
     ImplantData,
     SessionData,
     SubjectData,
     SurgeryData,
     InjectionData,
+    MesoscopeData,
     ProcedureData,
+    ProcessedData,
+    DeepLabCutData,
     ZaberPositions,
     ExperimentState,
-    ProcessingTracker,
+    VRPCDestinations,
+    ConfigurationData,
     MesoscopePositions,
+    VRPCPersistentData,
     ProjectConfiguration,
     HardwareConfiguration,
     RunTrainingDescriptor,
     LickTrainingDescriptor,
     ExperimentConfiguration,
+    ScanImagePCPersistentData,
     MesoscopeExperimentDescriptor,
 )
-from .transfer_tools import transfer_directory
-from .packaging_tools import calculate_directory_checksum
 
 # Ensures console is enabled when this library is imported
 if not console.enabled:
@@ -41,12 +45,21 @@ __all__ = [
     # Server module
     "Server",
     "ServerCredentials",
-    # Suite2p module
-    "Suite2PConfiguration",
+    # Suite2p package
+    "SingleDayS2PConfiguration",
+    "MultiDayS2PConfiguration",
     # Data classes module
     "DrugData",
     "ImplantData",
     "SessionData",
+    "RawData",
+    "ProcessedData",
+    "ConfigurationData",
+    "DeepLabCutData",
+    "VRPCPersistentData",
+    "ScanImagePCPersistentData",
+    "MesoscopeData",
+    "VRPCDestinations",
     "SubjectData",
     "SurgeryData",
     "InjectionData",
@@ -60,7 +73,6 @@ __all__ = [
     "LickTrainingDescriptor",
     "ExperimentConfiguration",
     "MesoscopeExperimentDescriptor",
-    "ProcessingTracker",
     # Transfer tools module
     "transfer_directory",
     # Packaging tools module

{sl_shared_assets-1.0.0rc14 → sl_shared_assets-1.0.0rc16}/src/sl_shared_assets/cli.py

@@ -4,9 +4,9 @@ from pathlib import Path
 
 import click
 
+from .tools import ascend_tyche_data
 from .server import generate_server_credentials
 from .data_classes import replace_root_path
-from .ascension_tools import ascend_tyche_data
 
 
 @click.command()

sl_shared_assets-1.0.0rc16/src/sl_shared_assets/data_classes/__init__.py

@@ -0,0 +1,63 @@
+"""This package provides the classes used to store data acquired at various stages of the data workflow and to
+configure various pipelines used in the Sun lab. These classes are used across all stages of data acquisition,
+preprocessing, and processing in the lab, across multiple machines (PCs). Many classes in this package are designed to
+be saved to disk as .yaml files and restored from the .yaml files as needed."""
+
+from .runtime_data import (
+    ZaberPositions,
+    MesoscopePositions,
+    HardwareConfiguration,
+    RunTrainingDescriptor,
+    LickTrainingDescriptor,
+    MesoscopeExperimentDescriptor,
+)
+from .session_data import (
+    RawData,
+    SessionData,
+    MesoscopeData,
+    ProcessedData,
+    DeepLabCutData,
+    VRPCDestinations,
+    ConfigurationData,
+    VRPCPersistentData,
+    ProjectConfiguration,
+    ScanImagePCPersistentData,
+    replace_root_path,
+)
+from .surgery_data import (
+    DrugData,
+    ImplantData,
+    SubjectData,
+    SurgeryData,
+    InjectionData,
+    ProcedureData,
+)
+from .configuration_data import ExperimentState, ExperimentConfiguration
+
+__all__ = [
+    "DrugData",
+    "ImplantData",
+    "SessionData",
+    "RawData",
+    "ProcessedData",
+    "ConfigurationData",
+    "DeepLabCutData",
+    "VRPCPersistentData",
+    "ScanImagePCPersistentData",
+    "MesoscopeData",
+    "VRPCDestinations",
+    "SubjectData",
+    "SurgeryData",
+    "InjectionData",
+    "ProcedureData",
+    "ZaberPositions",
+    "ExperimentState",
+    "MesoscopePositions",
+    "ProjectConfiguration",
+    "HardwareConfiguration",
+    "RunTrainingDescriptor",
+    "LickTrainingDescriptor",
+    "ExperimentConfiguration",
+    "MesoscopeExperimentDescriptor",
+    "replace_root_path",
+]

sl_shared_assets-1.0.0rc16/src/sl_shared_assets/data_classes/configuration_data.py

@@ -0,0 +1,64 @@
+"""This module provides classes used to configure data acquisition and processing runtimes in the Sun lab.
+Classes from this library are saved as .yaml files to be edited by the user when a new project and / or session
+is created by the sl-experiment library. The runtime settings are then loaded from user-edited .yaml files by various
+lab pipelines."""
+
+import copy
+from pathlib import Path
+from dataclasses import field, dataclass
+
+from ataraxis_data_structures import YamlConfig
+
+
+@dataclass()
+class ExperimentState:
+    """Encapsulates the information used to set and maintain the desired experiment and Mesoscope-VR system state.
+
+    Primarily, experiment runtime logic (task logic) is resolved by the Unity game engine. However, the Mesoscope-VR
+    system configuration may also need to change throughout the experiment to optimize the runtime by disabling or
+    reconfiguring specific hardware modules. For example, some experiment stages may require the running wheel to be
+    locked to prevent the animal from running, and other may require the VR screens to be turned off.
+    """
+
+    experiment_state_code: int
+    """The integer code of the experiment state. Experiment states do not have a predefined meaning, Instead, each 
+    project is expected to define and follow its own experiment state code mapping. Typically, the experiment state 
+    code is used to denote major experiment stages, such as 'baseline', 'task', 'cooldown', etc. Note, the same 
+    experiment state code can be used by multiple sequential ExperimentState instances to change the VR system states 
+    while maintaining the same experiment state."""
+    vr_state_code: int
+    """One of the supported VR system state-codes. Currently, the Mesoscope-VR system supports two state codes. State 
+    code '1' denotes 'REST' state and code '2' denotes 'RUN' state. Note, multiple consecutive ExperimentState 
+    instances with different experiment state codes can reuse the same VR state code."""
+    state_duration_s: float
+    """The time, in seconds, to maintain the current combination of the experiment and VR states."""
+
+
+@dataclass()
+class ExperimentConfiguration(YamlConfig):
+    """Stores the configuration of a single experiment runtime.
+
+    Primarily, this includes the sequence of experiment and Virtual Reality (Mesoscope-VR) states that defines the flow
+    of the experiment runtime. During runtime, the main runtime control function traverses the sequence of states
+    stored in this class instance start-to-end in the exact order specified by the user. Together with custom Unity
+    projects that define the task logic (how the system responds to animal interactions with the VR system) this class
+    allows flexibly implementing a wide range of experiments.
+
+    Each project should define one or more experiment configurations and save them as .yaml files inside the project
+    'configuration' folder. The name for each configuration file is defined by the user and is used to identify and load
+    the experiment configuration when 'sl-run-experiment' CLI command exposed by the sl-experiment library is executed.
+    """
+
+    cue_map: dict[int, float] = field(default_factory=lambda: {0: 30.0, 1: 30.0, 2: 30.0, 3: 30.0, 4: 30.0})
+    """A dictionary that maps each integer-code associated with a wall cue used in the Virtual Reality experiment 
+    environment to its length in real-world centimeters. It is used to map each VR cue to the distance the animal needs
+    to travel to fully traverse the wall cue region from start to end."""
+    experiment_states: dict[str, ExperimentState] = field(
+        default_factory=lambda: {
+            "baseline": ExperimentState(experiment_state_code=1, vr_state_code=1, state_duration_s=30),
+            "experiment": ExperimentState(experiment_state_code=2, vr_state_code=2, state_duration_s=120),
+            "cooldown": ExperimentState(experiment_state_code=3, vr_state_code=1, state_duration_s=15),
+        }
+    )
+    """A dictionary that uses human-readable state-names as keys and ExperimentState instances as values. Each 
+    ExperimentState instance represents a phase of the experiment."""

sl_shared_assets-1.0.0rc16/src/sl_shared_assets/data_classes/runtime_data.py

@@ -0,0 +1,233 @@
+"""This module provides classes used to store the training and experiment data acquired by the sl-experiment library.
+Some classes from this library store raw data later processed by Sun lab data processing pipelines. Others are used to
+restore the Mesoscope-VR system to the same state across training or experiment sessions of the same animal.
+"""
+
+from dataclasses import dataclass
+
+from ataraxis_data_structures import YamlConfig
+
+
+@dataclass()
+class HardwareConfiguration(YamlConfig):
+    """This class is used to save the runtime hardware configuration parameters as a .yaml file.
+
+    This information is used to read and decode the data saved to the .npz log files during runtime as part of data
+    processing.
+
+    Notes:
+        All fields in this dataclass initialize to None. During log processing, any log associated with a hardware
+        module that provides the data stored in a field will be processed, unless that field is None. Therefore, setting
+        any field in this dataclass to None also functions as a flag for whether to parse the log associated with the
+        module that provides this field's information.
+
+        This class is automatically configured by MesoscopeExperiment and BehaviorTraining classes from sl-experiment
+        library to facilitate log parsing.
+    """
+
+    cue_map: dict[int, float] | None = None
+    """MesoscopeExperiment instance property. Stores the dictionary that maps the integer id-codes associated with each 
+    wall cue in the Virtual Reality task environment with distances in real-world centimeters animals should run on the 
+    wheel to fully traverse the cue region on a linearized track."""
+    cm_per_pulse: float | None = None
+    """EncoderInterface instance property. Stores the conversion factor used to translate encoder pulses into 
+    real-world centimeters."""
+    maximum_break_strength: float | None = None
+    """BreakInterface instance property. Stores the breaking torque, in Newton centimeters, applied by the break to 
+    the edge of the running wheel when it is engaged at 100% strength."""
+    minimum_break_strength: float | None = None
+    """BreakInterface instance property. Stores the breaking torque, in Newton centimeters, applied by the break to 
+    the edge of the running wheel when it is engaged at 0% strength (completely disengaged)."""
+    lick_threshold: int | None = None
+    """LickInterface instance property. Determines the threshold, in 12-bit Analog to Digital Converter (ADC) units, 
+    above which an interaction value reported by the lick sensor is considered a lick (compared to noise or non-lick 
+    touch)."""
+    valve_scale_coefficient: float | None = None
+    """ValveInterface instance property. To dispense precise water volumes during runtime, ValveInterface uses power 
+    law equation applied to valve calibration data to determine how long to keep the valve open. This stores the 
+    scale_coefficient of the power law equation that describes the relationship between valve open time and dispensed 
+    water volume, derived from calibration data."""
+    valve_nonlinearity_exponent: float | None = None
+    """ValveInterface instance property. To dispense precise water volumes during runtime, ValveInterface uses power 
+    law equation applied to valve calibration data to determine how long to keep the valve open. This stores the 
+    nonlinearity_exponent of the power law equation that describes the relationship between valve open time and 
+    dispensed water volume, derived from calibration data."""
+    torque_per_adc_unit: float | None = None
+    """TorqueInterface instance property. Stores the conversion factor used to translate torque values reported by the 
+    sensor as 12-bit Analog to Digital Converter (ADC) units, into real-world Newton centimeters (N·cm) of torque that 
+    had to be applied to the edge of the running wheel to produce the observed ADC value."""
+    screens_initially_on: bool | None = None
+    """ScreenInterface instance property. Stores the initial state of the Virtual Reality screens at the beginning of 
+    the session runtime."""
+    recorded_mesoscope_ttl: bool | None = None
+    """TTLInterface instance property. A boolean flag that determines whether the processed session recorded brain 
+    activity data with the mesoscope."""
+
+
+@dataclass()
+class LickTrainingDescriptor(YamlConfig):
+    """This class is used to save the description information specific to lick training sessions as a .yaml file.
+
+    The information stored in this class instance is filled in two steps. The main runtime function fills most fields
+    of the class, before it is saved as a .yaml file. After runtime, the experimenter manually fills leftover fields,
+    such as 'experimenter_notes,' before the class instance is transferred to the long-term storage destination.
+
+    The fully filled instance data is also used during preprocessing to write the water restriction log entry for the
+    trained animal.
+    """
+
+    experimenter: str
+    """The ID of the experimenter running the session."""
+    mouse_weight_g: float
+    """The weight of the animal, in grams, at the beginning of the session."""
+    dispensed_water_volume_ml: float
+    """Stores the total water volume, in milliliters, dispensed during runtime."""
+    minimum_reward_delay: int
+    """Stores the minimum delay, in seconds, that can separate the delivery of two consecutive water rewards."""
+    maximum_reward_delay_s: int
+    """Stores the maximum delay, in seconds, that can separate the delivery of two consecutive water rewards."""
+    maximum_water_volume_ml: float
+    """Stores the maximum volume of water the system is allowed to dispense during training."""
+    maximum_training_time_m: int
+    """Stores the maximum time, in minutes, the system is allowed to run the training for."""
+    experimenter_notes: str = "Replace this with your notes."
+    """This field is not set during runtime. It is expected that each experimenter replaces this field with their 
+    notes made during runtime."""
+    experimenter_given_water_volume_ml: float = 0.0
+    """The additional volume of water, in milliliters, administered by the experimenter to the animal after the session.
+    """
+
+
+@dataclass()
+class RunTrainingDescriptor(YamlConfig):
+    """This class is used to save the description information specific to run training sessions as a .yaml file.
+
+    The information stored in this class instance is filled in two steps. The main runtime function fills most fields
+    of the class, before it is saved as a .yaml file. After runtime, the experimenter manually fills leftover fields,
+    such as 'experimenter_notes,' before the class instance is transferred to the long-term storage destination.
+
+    The fully filled instance data is also used during preprocessing to write the water restriction log entry for the
+    trained animal.
+    """
+
+    experimenter: str
+    """The ID of the experimenter running the session."""
+    mouse_weight_g: float
+    """The weight of the animal, in grams, at the beginning of the session."""
+    dispensed_water_volume_ml: float
+    """Stores the total water volume, in milliliters, dispensed during runtime."""
+    final_run_speed_threshold_cm_s: float
+    """Stores the final running speed threshold, in centimeters per second, that was active at the end of training."""
+    final_run_duration_threshold_s: float
+    """Stores the final running duration threshold, in seconds, that was active at the end of training."""
+    initial_run_speed_threshold_cm_s: float
+    """Stores the initial running speed threshold, in centimeters per second, used during training."""
+    initial_run_duration_threshold_s: float
+    """Stores the initial running duration threshold, in seconds, used during training."""
+    increase_threshold_ml: float
+    """Stores the volume of water delivered to the animal, in milliliters, that triggers the increase in the running 
+    speed and duration thresholds."""
+    run_speed_increase_step_cm_s: float
+    """Stores the value, in centimeters per second, used by the system to increment the running speed threshold each 
+    time the animal receives 'increase_threshold' volume of water."""
+    run_duration_increase_step_s: float
+    """Stores the value, in seconds, used by the system to increment the duration threshold each time the animal 
+    receives 'increase_threshold' volume of water."""
+    maximum_water_volume_ml: float
+    """Stores the maximum volume of water the system is allowed to dispense during training."""
+    maximum_training_time_m: int
+    """Stores the maximum time, in minutes, the system is allowed to run the training for."""
+    experimenter_notes: str = "Replace this with your notes."
+    """This field is not set during runtime. It is expected that each experimenter will replace this field with their 
+    notes made during runtime."""
+    experimenter_given_water_volume_ml: float = 0.0
+    """The additional volume of water, in milliliters, administered by the experimenter to the animal after the session.
+    """
+
+
+@dataclass()
+class MesoscopeExperimentDescriptor(YamlConfig):
+    """This class is used to save the description information specific to experiment sessions as a .yaml file.
+
+    The information stored in this class instance is filled in two steps. The main runtime function fills most fields
+    of the class, before it is saved as a .yaml file. After runtime, the experimenter manually fills leftover fields,
+    such as 'experimenter_notes,' before the class instance is transferred to the long-term storage destination.
+
+    The fully filled instance data is also used during preprocessing to write the water restriction log entry for the
+    animal participating in the experiment runtime.
+    """
+
+    experimenter: str
+    """The ID of the experimenter running the session."""
+    mouse_weight_g: float
+    """The weight of the animal, in grams, at the beginning of the session."""
+    dispensed_water_volume_ml: float
+    """Stores the total water volume, in milliliters, dispensed during runtime."""
+    experimenter_notes: str = "Replace this with your notes."
+    """This field is not set during runtime. It is expected that each experimenter will replace this field with their 
+    notes made during runtime."""
+    experimenter_given_water_volume_ml: float = 0.0
+    """The additional volume of water, in milliliters, administered by the experimenter to the animal after the session.
+    """
+
+
+@dataclass()
+class ZaberPositions(YamlConfig):
+    """This class is used to save Zaber motor positions as a .yaml file to reuse them between sessions.
+
+    The class is specifically designed to store, save, and load the positions of the LickPort and HeadBar motors
+    (axes). It is used to both store Zaber motor positions for each session for future analysis and to restore the same
+    Zaber motor positions across consecutive runtimes for the same project and animal combination.
+
+    Notes:
+        All positions are saved using native motor units. All class fields initialize to default placeholders that are
+        likely NOT safe to apply to the VR system. Do not apply the positions loaded from the file unless you are
+        certain they are safe to use.
+
+        Exercise caution when working with Zaber motors. The motors are powerful enough to damage the surrounding
+        equipment and manipulated objects. Do not modify the data stored inside the .yaml file unless you know what you
+        are doing.
+    """
+
+    headbar_z: int = 0
+    """The absolute position, in native motor units, of the HeadBar z-axis motor."""
+    headbar_pitch: int = 0
+    """The absolute position, in native motor units, of the HeadBar pitch-axis motor."""
+    headbar_roll: int = 0
+    """The absolute position, in native motor units, of the HeadBar roll-axis motor."""
+    lickport_z: int = 0
+    """The absolute position, in native motor units, of the LickPort z-axis motor."""
+    lickport_x: int = 0
+    """The absolute position, in native motor units, of the LickPort x-axis motor."""
+    lickport_y: int = 0
+    """The absolute position, in native motor units, of the LickPort y-axis motor."""
+
+
+@dataclass()
+class MesoscopePositions(YamlConfig):
+    """This class is used to save the real and virtual Mesoscope objective positions as a .yaml file to reuse it
+    between experiment sessions.
+
+    Primarily, the class is used to help the experimenter to position the Mesoscope at the same position across
+    multiple imaging sessions. It stores both the physical (real) position of the objective along the motorized
+    X, Y, Z, and Roll axes and the virtual (ScanImage software) tip, tilt, and fastZ focus axes.
+
+    Notes:
+        Since the API to read and write these positions automatically is currently not available, this class relies on
+        the experimenter manually entering all positions and setting the mesoscope to these positions when necessary.
+    """
+
+    mesoscope_x_position: float = 0.0
+    """The X-axis position, in centimeters, of the Mesoscope objective used during session runtime."""
+    mesoscope_y_position: float = 0.0
+    """The Y-axis position, in centimeters, of the Mesoscope objective used during session runtime."""
+    mesoscope_roll_position: float = 0.0
+    """The Roll-axis position, in degrees, of the Mesoscope objective used during session runtime."""
+    mesoscope_z_position: float = 0.0
+    """The Z-axis position, in centimeters, of the Mesoscope objective used during session runtime."""
+    mesoscope_fast_z_position: float = 0.0
+    """The Fast-Z-axis position, in micrometers, of the Mesoscope objective used during session runtime."""
+    mesoscope_tip_position: float = 0.0
+    """The Tilt-axis position, in degrees, of the Mesoscope objective used during session runtime."""
+    mesoscope_tilt_position: float = 0.0
+    """The Tip-axis position, in degrees, of the Mesoscope objective used during session runtime."""