sl-shared-assets 1.0.0rc19__py3-none-any.whl → 1.0.0rc21__py3-none-any.whl

sl_shared_assets/data_classes/session_data.pyi

@@ -0,0 +1,249 @@
+from pathlib import Path
+from dataclasses import field, dataclass
+
+from _typeshed import Incomplete
+from ataraxis_data_structures import YamlConfig
+
+from .configuration_data import get_system_configuration_data as get_system_configuration_data
+
+_valid_session_types: Incomplete
+
+@dataclass()
+class ProjectConfiguration(YamlConfig):
+    """Stores the project-specific configuration parameters that do not change between different animals and runtime
+    sessions.
+
+    An instance of this class is generated and saved as a .yaml file in the 'configuration' directory of each project
+    when it is created. After that, the stored data is reused for every runtime (training or experiment session) carried
+    out for each animal of the project. Additionally, a copy of the most actual configuration file is saved inside each
+    runtime session's 'raw_data' folder, providing seamless integration between the managed data and various Sun lab
+    (sl-) libraries.
+
+    Notes:
+        Together with SessionData, this class forms the entry point for all interactions with the data acquired in the
+        Sun lab. The fields of this class are used to flexibly configure the runtime behavior of major data acquisition
+        (sl-experiment) and processing (sl-forgery) libraries, adapting them for any project in the lab.
+    """
+
+    project_name: str = ...
+    surgery_sheet_id: str = ...
+    water_log_sheet_id: str = ...
+    @classmethod
+    def load(cls, configuration_path: Path) -> ProjectConfiguration:
+        """Loads the project configuration parameters from the specified project_configuration.yaml file.
+
+        This method is called during each interaction with any runtime session's data, including the creation of a new
+        session.
+
+        Args:
+            configuration_path: The path to the project_configuration.yaml file from which to load the data.
+
+        Returns:
+            The initialized ProjectConfiguration instance that stores the configuration data for the target project.
+
+        Raise:
+            FileNotFoundError: If the specified configuration file does not exist or is not a valid YAML file.
+        """
+    def save(self, path: Path) -> None:
+        """Saves class instance data to disk as a project_configuration.yaml file.
+
+        This method is automatically called from the 'sl_experiment' library when a new project is created. After this
+        method's runtime, all future project initialization calls will use the load() method to reuse configuration data
+        saved to the .yaml file created by this method.
+
+        Args:
+            path: The path to the .yaml file to save the data to.
+        """
+    def _verify_data(self) -> None:
+        """Verifies the user-modified data loaded from the project_configuration.yaml file.
+
+        Since this class is explicitly designed to be modified by the user, this verification step is carried out to
+        ensure that the loaded data matches expectations. This reduces the potential for user errors to impact the
+        runtime behavior of the libraries using this class. This internal method is automatically called by the load()
+        method.
+
+        Raises:
+            ValueError: If the loaded data does not match expected formats or values.
+        """
+
+@dataclass()
+class RawData:
+    """Stores the paths to the directories and files that make up the 'raw_data' session-specific directory.
+
+    The raw_data directory stores the data acquired during the session runtime before and after preprocessing. Since
+    preprocessing does not alter the data, any data in that folder is considered 'raw'.
+
+    Notes:
+        Sun lab data management strategy primarily relies on keeping multiple redundant copies of the raw_data for
+        each acquired session. Typically, one copy is stored on the lab's processing server and the other is stored on
+        the NAS.
+    """
+
+    raw_data_path: Path = ...
+    camera_data_path: Path = ...
+    mesoscope_data_path: Path = ...
+    behavior_data_path: Path = ...
+    zaber_positions_path: Path = ...
+    session_descriptor_path: Path = ...
+    hardware_state_path: Path = ...
+    surgery_metadata_path: Path = ...
+    project_configuration_path: Path = ...
+    session_data_path: Path = ...
+    experiment_configuration_path: Path = ...
+    mesoscope_positions_path: Path = ...
+    window_screenshot_path: Path = ...
+    system_configuration_path: Path = ...
+    checksum_path: Path = ...
+    telomere_path: Path = ...
+    ubiquitin_path: Path = ...
+    def resolve_paths(self, root_directory_path: Path) -> None:
+        """Resolves all paths managed by the class instance based on the input root directory path.
+
+        This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
+        machine that instantiates the class.
+
+        Args:
+            root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
+                hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
+                the managed session.
+        """
+    def make_directories(self) -> None:
+        """Ensures that all major subdirectories and the root directory exist, creating any missing directories."""
+
+@dataclass()
+class ProcessedData:
+    """Stores the paths to the directories and files that make up the 'processed_data' session-specific directory.
+
+    The processed_data directory stores the data generated by various processing pipelines from the raw data (contents
+    of the raw_data directory). Processed data represents an intermediate step between raw data and the dataset used in
+    the data analysis, but is not itself designed to be analyzed.
+    """
+
+    processed_data_path: Path = ...
+    camera_data_path: Path = ...
+    mesoscope_data_path: Path = ...
+    behavior_data_path: Path = ...
+    job_logs_path: Path = ...
+    single_day_suite2p_bin_path: Path = ...
+    multi_day_suite2p_bin_path: Path = ...
+    behavior_bin_path: Path = ...
+    dlc_bin_path: Path = ...
+    def resolve_paths(self, root_directory_path: Path) -> None:
+        """Resolves all paths managed by the class instance based on the input root directory path.
+
+        This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
+        machine that instantiates the class.
+
+        Args:
+            root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
+                hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
+                the managed session.
+        """
+    def make_directories(self) -> None:
+        """Ensures that all major subdirectories and the root directory exist, creating any missing directories."""
+
+@dataclass
+class SessionData(YamlConfig):
+    """Stores and manages the data layout of a single training or experiment session acquired in the Sun lab.
+
+    The primary purpose of this class is to maintain the session data structure across all supported destinations and
+    during all processing stages. It generates the paths used by all other classes from all Sun lab libraries that
+    interact with the session's data from the point of its creation and until the data is integrated into an
+    analysis dataset.
+
+    When necessary, the class can be used to either generate a new session or load the layout of an already existing
+    session. When the class is used to create a new session, it generates the new session's name using the current
+    UTC timestamp, accurate to microseconds. This ensures that each session name is unique and preserves the overall
+    session order.
+
+    Notes:
+        This class is specifically designed for working with the data from a single session, performed by a single
+        animal under the specific experiment. The class is used to manage both raw and processed data. It follows the
+        data through acquisition, preprocessing and processing stages of the Sun lab data workflow. Together with
+        ProjectConfiguration class, this class serves as an entry point for all interactions with the managed session's
+        data.
+    """
+
+    project_name: str
+    animal_id: str
+    session_name: str
+    session_type: str
+    acquisition_system: str
+    experiment_name: str | None
+    raw_data: RawData = field(default_factory=Incomplete)
+    processed_data: ProcessedData = field(default_factory=Incomplete)
+    @classmethod
+    def create(
+        cls,
+        project_name: str,
+        animal_id: str,
+        session_type: str,
+        experiment_name: str | None = None,
+        session_name: str | None = None,
+    ) -> SessionData:
+        """Creates a new SessionData object and generates the new session's data structure on the local PC.
+
+        This method is intended to be called exclusively by the sl-experiment library to create new training or
+        experiment sessions and generate the session data directory tree.
+
+        Notes:
+            To load an already existing session data structure, use the load() method instead.
+
+            This method automatically dumps the data of the created SessionData instance into the session_data.yaml file
+            inside the root raw_data directory of the created hierarchy. It also finds and dumps other configuration
+            files, such as project_configuration.yaml, experiment_configuration.yaml, and system_configuration.yaml into
+            the same raw_data directory. This ensures that if the session's runtime is interrupted unexpectedly, the
+            acquired data can still be processed.
+
+        Args:
+            project_name: The name of the project for which the data is acquired.
+            animal_id: The ID code of the animal for which the data is acquired.
+            session_type: The type of the session. Primarily, this determines how to read the session_descriptor.yaml
+                file. Valid options are 'Lick training', 'Run training', 'Window checking', or 'Experiment'.
+            experiment_name: The name of the experiment executed during managed session. This optional argument is only
+                used for 'Experiment' session types. It is used to find the experiment configuration .YAML file.
+            session_name: An optional session_name override. Generally, this argument should not be provided for most
+                sessions. When provided, the method uses this name instead of generating a new timestamp-based name.
+                This is only used during the 'ascension' runtime to convert old data structures to the modern
+                lab standards.
+
+        Returns:
+            An initialized SessionData instance that stores the layout of the newly created session's data.
+        """
+    @classmethod
+    def load(
+        cls, session_path: Path, processed_data_root: Path | None = None, make_processed_data_directory: bool = False
+    ) -> SessionData:
+        """Loads the SessionData instance from the target session's session_data.yaml file.
+
+        This method is used to load the data layout information of an already existing session. Primarily, this is used
+        when preprocessing or processing session data. Due to how SessionData is stored and used in the lab, this
+        method always loads the data layout from the session_data.yaml file stored inside the raw_data session
+        subfolder. Currently, all interactions with Sun lab data require access to the 'raw_data' folder.
+
+        Notes:
+            To create a new session, use the create() method instead.
+
+        Args:
+            session_path: The path to the root directory of an existing session, e.g.: root/project/animal/session.
+            processed_data_root: If processed data is kept on a drive different from the one that stores raw data,
+                provide the path to the root project directory (directory that stores all Sun lab projects) on that
+                drive. The method will automatically resolve the project/animal/session/processed_data hierarchy using
+                this root path. If raw and processed data are kept on the same drive, keep this set to None.
+            make_processed_data_directory: Determines whether this method should create processed_data directory if it
+                does not exist.
+
+        Returns:
+            An initialized SessionData instance for the session whose data is stored at the provided path.
+
+        Raises:
+            FileNotFoundError: If the 'session_data.yaml' file is not found under the session_path/raw_data/ subfolder.
+
+        """
+    def _save(self) -> None:
+        """Saves the instance data to the 'raw_data' directory of the managed session as a 'session_data.yaml' file.
+
+        This is used to save the data stored in the instance to disk, so that it can be reused during preprocessing or
+        data processing. The method is intended to only be used by the SessionData instance itself during its
+        create() method runtime.
+        """

sl_shared_assets/data_classes/surgery_data.py

@@ -1,5 +1,5 @@
 """This module provides classes to store animal surgery data. This is used to store the data extracted from the Sun lab
-surgery log, so that subject surgery data is always kept together with training and experiment data."""
+surgery log, so that subject (animal) surgery data is always kept together with training and experiment data."""
 
 from dataclasses import dataclass
 
@@ -130,10 +130,10 @@ class DrugData:
 
 @dataclass
 class SurgeryData(YamlConfig):
-    """Stores the data about a single mouse surgical intervention.
+    """Stores the data about a single animal surgical intervention.
 
     This class aggregates other dataclass instances that store specific data about the surgical procedure. Primarily, it
-    is used to save the data as a .yaml file to every session's raw_data directory of each animal used in every lab
+    is used to save the data as a .yaml file to every session's 'raw_data' directory of each animal used in every lab
     project. This way, the surgery data is always stored alongside the behavior and brain activity data collected
     during the session.
     """

sl_shared_assets/data_classes/surgery_data.pyi

@@ -0,0 +1,89 @@
+from dataclasses import dataclass
+
+from ataraxis_data_structures import YamlConfig
+
+@dataclass()
+class SubjectData:
+    """Stores the ID information of the surgical intervention's subject (animal)."""
+
+    id: int
+    ear_punch: str
+    sex: str
+    genotype: str
+    date_of_birth_us: int
+    weight_g: float
+    cage: int
+    location_housed: str
+    status: str
+
+@dataclass()
+class ProcedureData:
+    """Stores the general information about the surgical intervention."""
+
+    surgery_start_us: int
+    surgery_end_us: int
+    surgeon: str
+    protocol: str
+    surgery_notes: str
+    post_op_notes: str
+    surgery_quality: int = ...
+
+@dataclass
+class ImplantData:
+    """Stores the information about a single implantation performed during the surgical intervention.
+
+    Multiple ImplantData instances are used at the same time if the surgery involved multiple implants.
+    """
+
+    implant: str
+    implant_target: str
+    implant_code: int
+    implant_ap_coordinate_mm: float
+    implant_ml_coordinate_mm: float
+    implant_dv_coordinate_mm: float
+
+@dataclass
+class InjectionData:
+    """Stores the information about a single injection performed during surgical intervention.
+
+    Multiple InjectionData instances are used at the same time if the surgery involved multiple injections.
+    """
+
+    injection: str
+    injection_target: str
+    injection_volume_nl: float
+    injection_code: int
+    injection_ap_coordinate_mm: float
+    injection_ml_coordinate_mm: float
+    injection_dv_coordinate_mm: float
+
+@dataclass
+class DrugData:
+    """Stores the information about all drugs administered to the subject before, during, and immediately after the
+    surgical intervention.
+    """
+
+    lactated_ringers_solution_volume_ml: float
+    lactated_ringers_solution_code: int
+    ketoprofen_volume_ml: float
+    ketoprofen_code: int
+    buprenorphine_volume_ml: float
+    buprenorphine_code: int
+    dexamethasone_volume_ml: float
+    dexamethasone_code: int
+
+@dataclass
+class SurgeryData(YamlConfig):
+    """Stores the data about a single animal surgical intervention.
+
+    This class aggregates other dataclass instances that store specific data about the surgical procedure. Primarily, it
+    is used to save the data as a .yaml file to every session's 'raw_data' directory of each animal used in every lab
+    project. This way, the surgery data is always stored alongside the behavior and brain activity data collected
+    during the session.
+    """
+
+    subject: SubjectData
+    procedure: ProcedureData
+    drugs: DrugData
+    implants: list[ImplantData]
+    injections: list[InjectionData]

sl_shared_assets/server/__init__.pyi

@@ -0,0 +1,8 @@
+from .job import Job as Job
+from .server import (
+    Server as Server,
+    ServerCredentials as ServerCredentials,
+    generate_server_credentials as generate_server_credentials,
+)
+
+__all__ = ["Server", "ServerCredentials", "generate_server_credentials", "Job"]

sl_shared_assets/server/job.pyi

@@ -0,0 +1,94 @@
+from pathlib import Path
+
+from _typeshed import Incomplete
+from simple_slurm import Slurm
+
+class Job:
+    """Aggregates the data of a single SLURM-managed job to be executed on the Sun lab BioHPC cluster.
+
+    This class provides the API for constructing any server-side job in the Sun lab. Internally, it wraps an instance
+    of a Slurm class to package the job data into the format expected by the SLURM job manager. All jobs managed by this
+    class instance should be submitted to an initialized Server class 'submit_job' method to be executed on the server.
+
+    Notes:
+        The initialization method of the class contains the arguments for configuring the SLURM and Conda environments
+        used by the job. Do not submit additional SLURM or Conda commands via the 'add_command' method, as this may
+        produce unexpected behavior.
+
+        Each job can be conceptualized as a sequence of shell instructions to execute on the remote compute server. For
+        the lab, that means that the bulk of the command consists of calling various CLIs exposed by data processing or
+        analysis pipelines, installed in the Conda environment on the server. Other than that, the job contains commands
+        for activating the target conda environment and, in some cases, doing other preparatory or cleanup work. The
+        source code of a 'remote' job is typically identical to what a human operator would type in a 'local' terminal
+        to run the same job on their PC.
+
+        A key feature of server-side jobs is that they are executed on virtual machines managed by SLURM. Since the
+        server has a lot more compute and memory resources than likely needed by individual jobs, each job typically
+        requests a subset of these resources. Upon being executed, SLURM creates an isolated environment with the
+        requested resources and runs the job in that environment.
+
+        Since all jobs are expected to use the CLIs from python packages (pre)installed on the BioHPC server, make sure
+        that the target environment is installed and configured before submitting jobs to the server. See notes in
+        ReadMe to learn more about configuring server-side conda environments.
+
+    Args:
+        job_name: The descriptive name of the SLURM job to be created. Primarily, this name is used in terminal
+            printouts to identify the job to human operators.
+        output_log: The absolute path to the .txt file on the processing server, where to store the standard output
+            data of the job.
+        error_log: The absolute path to the .txt file on the processing server, where to store the standard error
+            data of the job.
+        working_directory: The absolute path to the directory where temporary job files will be stored. During runtime,
+            classes from this library use that directory to store files such as the job's shell script. All such files
+            are automatically removed from the directory at the end of a non-errors runtime.
+        conda_environment: The name of the conda environment to activate on the server before running the job logic. The
+            environment should contain the necessary Python packages and CLIs to support running the job's logic.
+        cpus_to_use: The number of CPUs to use for the job.
+        ram_gb: The amount of RAM to allocate for the job, in Gigabytes.
+        time_limit: The maximum time limit for the job, in minutes. If the job is still running at the end of this time
+            period, it will be forcibly terminated. It is highly advised to always set adequate maximum runtime limits
+            to prevent jobs from hogging the server in case of runtime or algorithm errors.
+
+    Attributes:
+        remote_script_path: Stores the path to the script file relative to the root of the remote server that runs the
+            command.
+        job_id: Stores the unique job identifier assigned by the SLURM manager to this job, when it is accepted for
+            execution. This field initialized to None and is overwritten by the Server class that submits the job.
+        job_name: Stores the descriptive name of the SLURM job.
+        _command: Stores the managed SLURM command object.
+    """
+
+    remote_script_path: Incomplete
+    job_id: str | None
+    job_name: str
+    _command: Slurm
+    def __init__(
+        self,
+        job_name: str,
+        output_log: Path,
+        error_log: Path,
+        working_directory: Path,
+        conda_environment: str,
+        cpus_to_use: int = 10,
+        ram_gb: int = 10,
+        time_limit: int = 60,
+    ) -> None: ...
+    def __repr__(self) -> str:
+        """Returns the string representation of the Job instance."""
+    def add_command(self, command: str) -> None:
+        """Adds the input command string to the end of the managed SLURM job command list.
+
+        This method is a wrapper around simple_slurm's 'add_cmd' method. It is used to iteratively build the shell
+        command sequence of the job.
+
+        Args:
+            command: The command string to add to the command list, e.g.: 'python main.py --input 1'.
+        """
+    @property
+    def command_script(self) -> str:
+        """Translates the managed job data into a shell-script-writable string and returns it to caller.
+
+        This method is used by the Server class to translate the job into the format that can be submitted to and
+        executed on the remote compute server. Do not call this method manually unless you know what you are doing.
+        The returned string is safe to dump into a .sh (shell script) file and move to the BioHPC server for execution.
+        """

sl_shared_assets/server/server.pyi

@@ -0,0 +1,95 @@
+from pathlib import Path
+from dataclasses import dataclass
+
+from simple_slurm import Slurm as Slurm
+from paramiko.client import SSHClient as SSHClient
+from ataraxis_data_structures import YamlConfig
+
+from .job import Job as Job
+
+def generate_server_credentials(
+    output_directory: Path, username: str, password: str, host: str = "cbsuwsun.biohpc.cornell.edu"
+) -> None:
+    """Generates a new server_credentials.yaml file under the specified directory, using input information.
+
+    This function provides a convenience interface for generating new BioHPC server credential files. Generally, this is
+    only used when setting up new host-computers in the lab.
+    """
+@dataclass()
+class ServerCredentials(YamlConfig):
+    """This class stores the hostname and credentials used to log into the BioHPC cluster to run Sun lab processing
+    pipelines.
+
+    Primarily, this is used as part of the sl-experiment library runtime to start data processing once it is
+    transferred to the BioHPC server during preprocessing. However, the same file can be used together with the Server
+    class API to run any computation jobs on the lab's BioHPC server.
+    """
+
+    username: str = ...
+    password: str = ...
+    host: str = ...
+
+class Server:
+    """Encapsulates access to the Sun lab BioHPC processing server.
+
+    This class provides the API that allows accessing the BioHPC server to create and submit various SLURM-managed jobs
+    to the server. It functions as the central interface used by all processing pipelines in the lab to execute costly
+    data processing on the server.
+
+    Notes:
+        All lab processing pipelines expect the data to be stored on the server and all processing logic to be packaged
+        and installed into dedicated conda environments on the server.
+
+        This class assumes that the target server has SLURM job manager installed and accessible to the user whose
+        credentials are used to connect to the server as part of this class instantiation.
+
+    Args:
+        credentials_path: The path to the locally stored .yaml file that contains the server hostname and access
+            credentials.
+
+    Attributes:
+        _open: Tracks whether the connection to the server is open or not.
+        _client: Stores the initialized SSHClient instance used to interface with the server.
+    """
+
+    _open: bool
+    _credentials: ServerCredentials
+    _client: SSHClient
+    def __init__(self, credentials_path: Path) -> None: ...
+    def __del__(self) -> None:
+        """If the instance is connected to the server, terminates the connection before the instance is destroyed."""
+    def submit_job(self, job: Job) -> Job:
+        """Submits the input job to the managed BioHPC server via SLURM job manager.
+
+        This method submits various jobs for execution via SLURM-managed BioHPC cluster. As part of its runtime, the
+        method translates the Job object into the shell script, moves the script to the target working directory on
+        the server, and instructs the server to execute the shell script (via SLURM).
+
+        Args:
+            job: The Job object that contains all job data.
+
+        Returns:
+            The job object whose 'job_id' attribute had been modified with the job ID, if the job was successfully
+            submitted.
+
+        Raises:
+            RuntimeError: If job submission to the server fails.
+        """
+    def job_complete(self, job: Job) -> bool:
+        """Returns True if the job managed by the input Job instance has been completed or terminated its runtime due
+        to an error.
+
+        If the job is still running or is waiting inside the execution queue, returns False.
+
+        Args:
+            job: The Job object whose status needs to be checked.
+
+        Raises:
+            ValueError: If the input Job object does not contain a valid job_id, suggesting that it has not been
+                submitted to the server.
+        """
+    def close(self) -> None:
+        """Closes the SSH connection to the server.
+
+        This method has to be called before destroying the class instance to ensure proper resource cleanup.
+        """

sl_shared_assets/tools/__init__.py

@@ -4,5 +4,12 @@ integrity of the data. The tools from this package are used by most other data p
 from .transfer_tools import transfer_directory
 from .ascension_tools import ascend_tyche_data
 from .packaging_tools import calculate_directory_checksum
+from .project_management_tools import verify_session_checksum, generate_project_manifest
 
-__all__ = ["transfer_directory", "calculate_directory_checksum", "ascend_tyche_data"]
+__all__ = [
+    "transfer_directory",
+    "calculate_directory_checksum",
+    "ascend_tyche_data",
+    "verify_session_checksum",
+    "generate_project_manifest",
+]

sl_shared_assets/tools/__init__.pyi

@@ -0,0 +1,15 @@
+from .transfer_tools import transfer_directory as transfer_directory
+from .ascension_tools import ascend_tyche_data as ascend_tyche_data
+from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
+from .project_management_tools import (
+    verify_session_checksum as verify_session_checksum,
+    generate_project_manifest as generate_project_manifest,
+)
+
+__all__ = [
+    "transfer_directory",
+    "calculate_directory_checksum",
+    "ascend_tyche_data",
+    "verify_session_checksum",
+    "generate_project_manifest",
+]