sl-shared-assets 1.0.0rc13__py3-none-any.whl → 1.0.0rc15__py3-none-any.whl

sl_shared_assets/data_classes/surgery_data.py

@@ -0,0 +1,152 @@
+"""This module provides classes to store animal surgery data. This is used to store the data extracted from the Sun lab
+surgery log, so that subject surgery data is always kept together with training and experiment data."""
+
+from dataclasses import dataclass
+
+from ataraxis_data_structures import YamlConfig
+
+
+@dataclass()
+class SubjectData:
+    """Stores the ID information of the surgical intervention's subject (animal)."""
+
+    id: int
+    """Stores the unique ID (name) of the subject. Assumes all animals are given a numeric ID, rather than a string 
+    name."""
+    ear_punch: str
+    """Stores the ear tag location of the subject."""
+    sex: str
+    """Stores the gender of the subject."""
+    genotype: str
+    """Stores the genotype of the subject."""
+    date_of_birth_us: int
+    """Stores the date of birth of the subject as the number of microseconds elapsed since UTC epoch onset."""
+    weight_g: float
+    """Stores the weight of the subject pre-surgery, in grams."""
+    cage: int
+    """Stores the number of the cage used to house the subject after surgery."""
+    location_housed: str
+    """Stores the location used to house the subject after the surgery."""
+    status: str
+    """Stores the current status of the subject (alive / deceased)."""
+
+
+@dataclass()
+class ProcedureData:
+    """Stores the general information about the surgical intervention."""
+
+    surgery_start_us: int
+    """Stores the date and time when the surgery has started as microseconds elapsed since UTC epoch onset."""
+    surgery_end_us: int
+    """Stores the date and time when the surgery has ended as microseconds elapsed since UTC epoch onset."""
+    surgeon: str
+    """Stores the name or ID of the surgeon. If the intervention was carried out by multiple surgeons, all participating
+    surgeon names and IDs are stored as part of the same string."""
+    protocol: str
+    """Stores the experiment protocol number (ID) used during the surgery."""
+    surgery_notes: str
+    """Stores surgeon's notes taken during the surgery."""
+    post_op_notes: str
+    """Stores surgeon's notes taken during the post-surgery recovery period."""
+    surgery_quality: int = 0
+    """Stores the quality of the surgical intervention as a numeric level. 0 indicates unusable (bad) result, 1 
+    indicates usable result that is not good enough to be included in a publication, 2 indicates publication-grade 
+    result."""
+
+
+@dataclass
+class ImplantData:
+    """Stores the information about a single implantation performed during the surgical intervention.
+
+    Multiple ImplantData instances are used at the same time if the surgery involved multiple implants.
+    """
+
+    implant: str
+    """The descriptive name of the implant."""
+    implant_target: str
+    """The name of the brain region or cranium section targeted by the implant."""
+    implant_code: int
+    """The manufacturer code or internal reference code for the implant. This code is used to identify the implant in 
+    additional datasheets and lab ordering documents."""
+    implant_ap_coordinate_mm: float
+    """Stores implant's antero-posterior stereotactic coordinate, in millimeters, relative to bregma."""
+    implant_ml_coordinate_mm: float
+    """Stores implant's medial-lateral stereotactic coordinate, in millimeters, relative to bregma."""
+    implant_dv_coordinate_mm: float
+    """Stores implant's dorsal-ventral stereotactic coordinate, in millimeters, relative to bregma."""
+
+
+@dataclass
+class InjectionData:
+    """Stores the information about a single injection performed during surgical intervention.
+
+    Multiple InjectionData instances are used at the same time if the surgery involved multiple injections.
+    """
+
+    injection: str
+    """The descriptive name of the injection."""
+    injection_target: str
+    """The name of the brain region targeted by the injection."""
+    injection_volume_nl: float
+    """The volume of substance, in nanoliters, delivered during the injection."""
+    injection_code: int
+    """The manufacturer code or internal reference code for the injected substance. This code is used to identify the 
+    substance in additional datasheets and lab ordering documents."""
+    injection_ap_coordinate_mm: float
+    """Stores injection's antero-posterior stereotactic coordinate, in millimeters, relative to bregma."""
+    injection_ml_coordinate_mm: float
+    """Stores injection's medial-lateral stereotactic coordinate, in millimeters, relative to bregma."""
+    injection_dv_coordinate_mm: float
+    """Stores injection's dorsal-ventral stereotactic coordinate, in millimeters, relative to bregma."""
+
+
+@dataclass
+class DrugData:
+    """Stores the information about all drugs administered to the subject before, during, and immediately after the
+    surgical intervention.
+    """
+
+    lactated_ringers_solution_volume_ml: float
+    """Stores the volume of Lactated Ringer's Solution (LRS) administered during surgery, in ml."""
+    lactated_ringers_solution_code: int
+    """Stores the manufacturer code or internal reference code for Lactated Ringer's Solution (LRS). This code is used 
+    to identify the LRS batch in additional datasheets and lab ordering documents."""
+    ketoprofen_volume_ml: float
+    """Stores the volume of ketoprofen diluted with saline administered during surgery, in ml."""
+    ketoprofen_code: int
+    """Stores the manufacturer code or internal reference code for ketoprofen. This code is used to identify the 
+    ketoprofen batch in additional datasheets and lab ordering documents."""
+    buprenorphine_volume_ml: float
+    """Stores the volume of buprenorphine diluted with saline administered during surgery, in ml."""
+    buprenorphine_code: int
+    """Stores the manufacturer code or internal reference code for buprenorphine. This code is used to identify the 
+    buprenorphine batch in additional datasheets and lab ordering documents."""
+    dexamethasone_volume_ml: float
+    """Stores the volume of dexamethasone diluted with saline administered during surgery, in ml."""
+    dexamethasone_code: int
+    """Stores the manufacturer code or internal reference code for dexamethasone. This code is used to identify the 
+    dexamethasone batch in additional datasheets and lab ordering documents."""
+
+
+@dataclass
+class SurgeryData(YamlConfig):
+    """Stores the data about a single mouse surgical intervention.
+
+    This class aggregates other dataclass instances that store specific data about the surgical procedure. Primarily, it
+    is used to save the data as a .yaml file to every session's raw_data directory of each animal used in every lab
+    project. This way, the surgery data is always stored alongside the behavior and brain activity data collected
+    during the session.
+    """
+
+    subject: SubjectData
+    """Stores the ID information about the subject (mouse)."""
+    procedure: ProcedureData
+    """Stores general data about the surgical intervention."""
+    drugs: DrugData
+    """Stores the data about the substances subcutaneously injected into the subject before, during and immediately 
+    after the surgical intervention."""
+    implants: list[ImplantData]
+    """Stores the data for all cranial and transcranial implants introduced to the subject during the surgical 
+    intervention."""
+    injections: list[InjectionData]
+    """Stores the data about all substances infused into the brain of the subject during the surgical intervention."""

sl_shared_assets/data_classes/surgery_data.pyi

@@ -0,0 +1,89 @@
+from dataclasses import dataclass
+
+from ataraxis_data_structures import YamlConfig
+
+@dataclass()
+class SubjectData:
+    """Stores the ID information of the surgical intervention's subject (animal)."""
+
+    id: int
+    ear_punch: str
+    sex: str
+    genotype: str
+    date_of_birth_us: int
+    weight_g: float
+    cage: int
+    location_housed: str
+    status: str
+
+@dataclass()
+class ProcedureData:
+    """Stores the general information about the surgical intervention."""
+
+    surgery_start_us: int
+    surgery_end_us: int
+    surgeon: str
+    protocol: str
+    surgery_notes: str
+    post_op_notes: str
+    surgery_quality: int = ...
+
+@dataclass
+class ImplantData:
+    """Stores the information about a single implantation performed during the surgical intervention.
+
+    Multiple ImplantData instances are used at the same time if the surgery involved multiple implants.
+    """
+
+    implant: str
+    implant_target: str
+    implant_code: int
+    implant_ap_coordinate_mm: float
+    implant_ml_coordinate_mm: float
+    implant_dv_coordinate_mm: float
+
+@dataclass
+class InjectionData:
+    """Stores the information about a single injection performed during surgical intervention.
+
+    Multiple InjectionData instances are used at the same time if the surgery involved multiple injections.
+    """
+
+    injection: str
+    injection_target: str
+    injection_volume_nl: float
+    injection_code: int
+    injection_ap_coordinate_mm: float
+    injection_ml_coordinate_mm: float
+    injection_dv_coordinate_mm: float
+
+@dataclass
+class DrugData:
+    """Stores the information about all drugs administered to the subject before, during, and immediately after the
+    surgical intervention.
+    """
+
+    lactated_ringers_solution_volume_ml: float
+    lactated_ringers_solution_code: int
+    ketoprofen_volume_ml: float
+    ketoprofen_code: int
+    buprenorphine_volume_ml: float
+    buprenorphine_code: int
+    dexamethasone_volume_ml: float
+    dexamethasone_code: int
+
+@dataclass
+class SurgeryData(YamlConfig):
+    """Stores the data about a single mouse surgical intervention.
+
+    This class aggregates other dataclass instances that store specific data about the surgical procedure. Primarily, it
+    is used to save the data as a .yaml file to every session's raw_data directory of each animal used in every lab
+    project. This way, the surgery data is always stored alongside the behavior and brain activity data collected
+    during the session.
+    """
+
+    subject: SubjectData
+    procedure: ProcedureData
+    drugs: DrugData
+    implants: list[ImplantData]
+    injections: list[InjectionData]

sl_shared_assets/server/__init__.py

@@ -0,0 +1,8 @@
+"""This package provides the classes and methods used by all Sun lab libraries to submit remote jobs to the BioHPC
+and other compute servers. This package is also used across all Sun lab members private code to interface with the
+shared server."""
+
+from .job import Job
+from .server import Server, ServerCredentials, generate_server_credentials
+
+__all__ = ["Server", "ServerCredentials", "generate_server_credentials", "Job"]

sl_shared_assets/server/__init__.pyi

@@ -0,0 +1,8 @@
+from .job import Job as Job
+from .server import (
+    Server as Server,
+    ServerCredentials as ServerCredentials,
+    generate_server_credentials as generate_server_credentials,
+)
+
+__all__ = ["Server", "ServerCredentials", "generate_server_credentials", "Job"]

sl_shared_assets/server/job.py

@@ -0,0 +1,140 @@
+"""This module provides the core Job class, used as the starting point for all SLURM-managed job executed on lab compute
+server(s). Specifically, the Job class acts as a wrapper around the SLURM configuration and specific logic of each
+job. During runtime, Server class interacts with input job objects to manage their transfer and execution on the
+remote servers."""
+
+# noinspection PyProtectedMember
+from pathlib import Path
+import datetime
+
+from simple_slurm import Slurm  # type: ignore
+
+
+class Job:
+    """Aggregates the data of a single SLURM-managed job to be executed on the Sun lab BioHPC cluster.
+
+    This class provides the API for constructing any server-side job in the Sun lab. Internally, it wraps an instance
+    of a Slurm class to package the job data into the format expected by the SLURM job manager. All jobs managed by this
+    class instance should be submitted to an initialized Server class 'submit_job' method to be executed on the server.
+
+    Notes:
+        The initialization method of the class contains the arguments for configuring the SLURM and Conda environments
+        used by the job. Do not submit additional SLURM or Conda commands via the 'add_command' method, as this may
+        produce unexpected behavior.
+
+        Each job can be conceptualized as a sequence of shell instructions to execute on the remote compute server. For
+        the lab, that means that the bulk of the command consists of calling various CLIs exposed by data processing or
+        analysis pipelines, installed in the Conda environment on the server. Other than that, the job contains commands
+        for activating the target conda environment and, in some cases, doing other preparatory or cleanup work. The
+        source code of a 'remote' job is typically identical to what a human operator would type in a 'local' terminal
+        to run the same job on their PC.
+
+        A key feature of server-side jobs is that they are executed on virtual machines managed by SLURM. Since the
+        server has a lot more compute and memory resources than likely needed by individual jobs, each job typically
+        requests a subset of these resources. Upon being executed, SLURM creates an isolated environment with the
+        requested resources and runs the job in that environment.
+
+        Since all jobs are expected to use the CLIs from python packages (pre)installed on the BioHPC server, make sure
+        that the target environment is installed and configured before submitting jobs to the server. See notes in
+        ReadMe to learn more about configuring server-side conda environments.
+
+    Args:
+        job_name: The descriptive name of the SLURM job to be created. Primarily, this name is used in terminal
+            printouts to identify the job to human operators.
+        output_log: The absolute path to the .txt file on the processing server, where to store the standard output
+            data of the job.
+        error_log: The absolute path to the .txt file on the processing server, where to store the standard error
+            data of the job.
+        working_directory: The absolute path to the directory where temporary job files will be stored. During runtime,
+            classes from this library use that directory to store files such as the job's shell script. All such files
+            are automatically removed from the directory at the end of a non-errors runtime.
+        conda_environment: The name of the conda environment to activate on the server before running the job logic. The
+            environment should contain the necessary Python packages and CLIs to support running the job's logic.
+        cpus_to_use: The number of CPUs to use for the job.
+        ram_gb: The amount of RAM to allocate for the job, in Gigabytes.
+        time_limit: The maximum time limit for the job, in minutes. If the job is still running at the end of this time
+            period, it will be forcibly terminated. It is highly advised to always set adequate maximum runtime limits
+            to prevent jobs from hogging the server in case of runtime or algorithm errors.
+
+    Attributes:
+        remote_script_path: Stores the path to the script file relative to the root of the remote server that runs the
+            command.
+        job_id: Stores the unique job identifier assigned by the SLURM manager to this job, when it is accepted for
+            execution. This field initialized to None and is overwritten by the Server class that submits the job.
+        job_name: Stores the descriptive name of the SLURM job.
+        _command: Stores the managed SLURM command object.
+    """
+
+    def __init__(
+        self,
+        job_name: str,
+        output_log: Path,
+        error_log: Path,
+        working_directory: Path,
+        conda_environment: str,
+        cpus_to_use: int = 10,
+        ram_gb: int = 10,
+        time_limit: int = 60,
+    ) -> None:
+        # Resolves the paths to the remote (server-side) .sh script file. This is the path where the job script
+        # will be stored on the server, once it is transferred by the Server class instance.
+        self.remote_script_path = str(working_directory.joinpath(f"{job_name}.sh"))
+
+        # Defines additional arguments used by the Server class that executed the job.
+        self.job_id: str | None = None  # This is set by the Server that submits the job.
+        self.job_name: str = job_name  # Also stores the job name to support more informative terminal prints
+
+        # Builds the slurm command object filled with configuration information
+        self._command: Slurm = Slurm(
+            cpus_per_task=cpus_to_use,
+            job_name=job_name,
+            output=str(output_log),
+            error=str(error_log),
+            mem=f"{ram_gb}G",
+            time=datetime.timedelta(minutes=time_limit),
+        )
+
+        # Conda shell initialization commands
+        self._command.add_cmd("eval $(conda shell.bash hook)")
+        self._command.add_cmd("conda init bash")
+
+        # Activates the target conda environment for the command.
+        self._command.add_cmd(f"conda activate {conda_environment}")
+
+    def __repr__(self) -> str:
+        """Returns the string representation of the Job instance."""
+        return f"Job(name={self.job_name}, id={self.job_id})"
+
+    def add_command(self, command: str) -> None:
+        """Adds the input command string to the end of the managed SLURM job command list.
+
+        This method is a wrapper around simple_slurm's 'add_cmd' method. It is used to iteratively build the shell
+        command sequence of the job.
+
+        Args:
+            command: The command string to add to the command list, e.g.: 'python main.py --input 1'.
+        """
+
+        self._command.add_cmd(command)
+
+    @property
+    def command_script(self) -> str:
+        """Translates the managed job data into a shell-script-writable string and returns it to caller.
+
+        This method is used by the Server class to translate the job into the format that can be submitted to and
+        executed on the remote compute server. Do not call this method manually unless you know what you are doing.
+        The returned string is safe to dump into a .sh (shell script) file and move to the BioHPC server for execution.
+        """
+
+        # Appends the command to clean up (remove) the temporary script file after processing runtime is over
+        self._command.add_cmd(f"rm -f {self.remote_script_path}")
+
+        # Translates the command to string format
+        script_content = str(self._command)
+
+        # Replaces escaped $ (/$) with $. This is essential, as without this correction, things like conda
+        # initialization would not work as expected.
+        fixed_script_content = script_content.replace("\\$", "$")
+
+        # Returns the script content to caller as a string
+        return fixed_script_content

sl_shared_assets/server/job.pyi

@@ -0,0 +1,94 @@
+from pathlib import Path
+
+from _typeshed import Incomplete
+from simple_slurm import Slurm
+
+class Job:
+    """Aggregates the data of a single SLURM-managed job to be executed on the Sun lab BioHPC cluster.
+
+    This class provides the API for constructing any server-side job in the Sun lab. Internally, it wraps an instance
+    of a Slurm class to package the job data into the format expected by the SLURM job manager. All jobs managed by this
+    class instance should be submitted to an initialized Server class 'submit_job' method to be executed on the server.
+
+    Notes:
+        The initialization method of the class contains the arguments for configuring the SLURM and Conda environments
+        used by the job. Do not submit additional SLURM or Conda commands via the 'add_command' method, as this may
+        produce unexpected behavior.
+
+        Each job can be conceptualized as a sequence of shell instructions to execute on the remote compute server. For
+        the lab, that means that the bulk of the command consists of calling various CLIs exposed by data processing or
+        analysis pipelines, installed in the Conda environment on the server. Other than that, the job contains commands
+        for activating the target conda environment and, in some cases, doing other preparatory or cleanup work. The
+        source code of a 'remote' job is typically identical to what a human operator would type in a 'local' terminal
+        to run the same job on their PC.
+
+        A key feature of server-side jobs is that they are executed on virtual machines managed by SLURM. Since the
+        server has a lot more compute and memory resources than likely needed by individual jobs, each job typically
+        requests a subset of these resources. Upon being executed, SLURM creates an isolated environment with the
+        requested resources and runs the job in that environment.
+
+        Since all jobs are expected to use the CLIs from python packages (pre)installed on the BioHPC server, make sure
+        that the target environment is installed and configured before submitting jobs to the server. See notes in
+        ReadMe to learn more about configuring server-side conda environments.
+
+    Args:
+        job_name: The descriptive name of the SLURM job to be created. Primarily, this name is used in terminal
+            printouts to identify the job to human operators.
+        output_log: The absolute path to the .txt file on the processing server, where to store the standard output
+            data of the job.
+        error_log: The absolute path to the .txt file on the processing server, where to store the standard error
+            data of the job.
+        working_directory: The absolute path to the directory where temporary job files will be stored. During runtime,
+            classes from this library use that directory to store files such as the job's shell script. All such files
+            are automatically removed from the directory at the end of a non-errors runtime.
+        conda_environment: The name of the conda environment to activate on the server before running the job logic. The
+            environment should contain the necessary Python packages and CLIs to support running the job's logic.
+        cpus_to_use: The number of CPUs to use for the job.
+        ram_gb: The amount of RAM to allocate for the job, in Gigabytes.
+        time_limit: The maximum time limit for the job, in minutes. If the job is still running at the end of this time
+            period, it will be forcibly terminated. It is highly advised to always set adequate maximum runtime limits
+            to prevent jobs from hogging the server in case of runtime or algorithm errors.
+
+    Attributes:
+        remote_script_path: Stores the path to the script file relative to the root of the remote server that runs the
+            command.
+        job_id: Stores the unique job identifier assigned by the SLURM manager to this job, when it is accepted for
+            execution. This field initialized to None and is overwritten by the Server class that submits the job.
+        job_name: Stores the descriptive name of the SLURM job.
+        _command: Stores the managed SLURM command object.
+    """
+
+    remote_script_path: Incomplete
+    job_id: str | None
+    job_name: str
+    _command: Slurm
+    def __init__(
+        self,
+        job_name: str,
+        output_log: Path,
+        error_log: Path,
+        working_directory: Path,
+        conda_environment: str,
+        cpus_to_use: int = 10,
+        ram_gb: int = 10,
+        time_limit: int = 60,
+    ) -> None: ...
+    def __repr__(self) -> str:
+        """Returns the string representation of the Job instance."""
+    def add_command(self, command: str) -> None:
+        """Adds the input command string to the end of the managed SLURM job command list.
+
+        This method is a wrapper around simple_slurm's 'add_cmd' method. It is used to iteratively build the shell
+        command sequence of the job.
+
+        Args:
+            command: The command string to add to the command list, e.g.: 'python main.py --input 1'.
+        """
+    @property
+    def command_script(self) -> str:
+        """Translates the managed job data into a shell-script-writable string and returns it to caller.
+
+        This method is used by the Server class to translate the job into the format that can be submitted to and
+        executed on the remote compute server. Do not call this method manually unless you know what you are doing.
+        The returned string is safe to dump into a .sh (shell script) file and move to the BioHPC server for execution.
+        """