sl-shared-assets 4.0.1__py3-none-any.whl → 5.0.0__py3-none-any.whl

sl_shared_assets/__init__.py

@@ -1,4 +1,4 @@
-"""A Python library that stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
+"""A Python library that provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
 
 See https://github.com/Sun-Lab-NBB/sl-shared-assets for more details.
 API documentation: https://sl-shared-assets-api-docs.netlify.app/
@@ -9,12 +9,22 @@ from ataraxis_base_utilities import console
 
 from .tools import (
     ProjectManifest,
-    resolve_p53_marker,
+    delete_directory,
     transfer_directory,
-    generate_project_manifest,
     calculate_directory_checksum,
 )
-from .server import Job, Server, JupyterJob, ServerCredentials
+from .server import (
+    Job,
+    Server,
+    JupyterJob,
+    ProcessingStatus,
+    TrackerFileNames,
+    ProcessingTracker,
+    ServerCredentials,
+    ProcessingPipeline,
+    ProcessingPipelines,
+    generate_manager_id,
+)
 from .data_classes import (
     RawData,
     DrugData,
@@ -31,8 +41,6 @@ from .data_classes import (
     ExperimentState,
     ExperimentTrial,
     MesoscopeCameras,
-    TrackerFileNames,
-    ProcessingTracker,
     AcquisitionSystems,
     MesoscopePositions,
     RunTrainingDescriptor,
@@ -44,10 +52,9 @@ from .data_classes import (
     MesoscopeSystemConfiguration,
     MesoscopeExperimentDescriptor,
     MesoscopeExperimentConfiguration,
-    generate_manager_id,
-    get_processing_tracker,
+    get_working_directory,
+    get_credentials_file_path,
     get_system_configuration_data,
-    set_system_configuration_file,
 )
 
 # Ensures console is enabled when this library is imported
@@ -55,48 +62,44 @@ if not console.enabled:
     console.enable()
 
 __all__ = [
-    # Server package
-    "Server",
-    "ServerCredentials",
-    "Job",
-    "JupyterJob",
-    # Data classes package
+    "AcquisitionSystems",
     "DrugData",
+    "ExperimentState",
+    "ExperimentTrial",
     "ImplantData",
-    "SessionData",
-    "RawData",
-    "ProcessedData",
-    "SubjectData",
-    "SurgeryData",
     "InjectionData",
-    "ProcessingTracker",
-    "ProcedureData",
-    "ZaberPositions",
-    "ExperimentState",
-    "MesoscopePositions",
-    "MesoscopeHardwareState",
-    "RunTrainingDescriptor",
+    "Job",
+    "JupyterJob",
     "LickTrainingDescriptor",
+    "MesoscopeAdditionalFirmware",
+    "MesoscopeCameras",
     "MesoscopeExperimentConfiguration",
     "MesoscopeExperimentDescriptor",
-    "MesoscopeSystemConfiguration",
-    "MesoscopePaths",
-    "MesoscopeCameras",
+    "MesoscopeHardwareState",
     "MesoscopeMicroControllers",
-    "MesoscopeAdditionalFirmware",
-    "get_system_configuration_data",
-    "set_system_configuration_file",
-    "ExperimentTrial",
+    "MesoscopePaths",
+    "MesoscopePositions",
+    "MesoscopeSystemConfiguration",
+    "ProcedureData",
+    "ProcessedData",
+    "ProcessingTracker",
+    "ProjectManifest",
+    "RawData",
+    "RunTrainingDescriptor",
+    "Server",
+    "ServerCredentials",
+    "SessionData",
     "SessionTypes",
-    "AcquisitionSystems",
+    "SubjectData",
+    "SurgeryData",
+    "TrackerFileNames",
     "WindowCheckingDescriptor",
-    "get_processing_tracker",
+    "ZaberPositions",
+    "calculate_directory_checksum",
+    "delete_directory",
     "generate_manager_id",
-    "TrackerFileNames",
-    # Tools package
-    "ProjectManifest",
-    "resolve_p53_marker",
+    "get_credentials_file_path",
+    "get_system_configuration_data",
+    "get_working_directory",
     "transfer_directory",
-    "calculate_directory_checksum",
-    "generate_project_manifest",
 ]

sl_shared_assets/command_line_interfaces/__init__.py

@@ -0,0 +1,3 @@
+"""This package provides the Command-Line Interfaces (CLIs) exposed by installing this library into a Python
+environment.
+"""

sl_shared_assets/command_line_interfaces/configure.py

@@ -0,0 +1,173 @@
+"""This module provides the Command-Line Interface (CLI) for configuring major components of the Sun lab data
+workflow."""
+
+from pathlib import Path
+
+import click
+from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
+
+from ..server import generate_server_credentials
+from ..data_classes import (
+    AcquisitionSystems,
+    get_working_directory,
+    set_working_directory,
+    create_system_configuration_file,
+)
+
+# Ensures that displayed CLICK help messages are formatted according to the lab standard.
+CONTEXT_SETTINGS = dict(max_content_width=120)  # or any width you want
+
+
+@click.group("manage", context_settings=CONTEXT_SETTINGS)
+def configure() -> None:
+    """This Command-Line Interface (CLI) allows configuring major components of the Sun lab data acquisition,
+    processing, and analysis workflow, such as acquisition systems and compute server(s)."""
+
+
+@configure.command("directory")
+@click.option(
+    "-d",
+    "--directory",
+    type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="The absolute path to the directory where to cache Sun lab configuration and local runtime data.",
+)
+def configure_directory(directory: Path) -> None:
+    """Sets the input directory as the Sun lab working directory, creating any missing path components.
+
+    This command as the initial entry-point for setting up any machine (PC) to work with Sun lab libraries and data.
+    After the working directory is configured, all calls to this and all other Sun lab libraries automatically use this
+    directory to store the configuration and runtime data required to perform any requested task. This allows all Sun
+    lab libraries to behave consistently across different user machines and runtime contexts.
+    """
+    # Creates the directory if it does not exist
+    ensure_directory_exists(directory)
+
+    # Sets the directory as the local working directory
+    set_working_directory(path=directory)
+
+    console.echo(message=f"Sun lab working directory set to: {directory}.", level=LogLevel.SUCCESS)
+
+
+@configure.command("server")
+@click.option(
+    "-u",
+    "--username",
+    type=str,
+    required=True,
+    help="The username to use for server authentication.",
+)
+@click.option(
+    "-p",
+    "--password",
+    type=str,
+    required=True,
+    help="The password to use for server authentication.",
+)
+@click.option(
+    "-s",
+    "--service",
+    is_flag=True,
+    default=False,
+    help=(
+        "Determines whether the credentials' file is created for a service account. This determines the name of the "
+        "generated file. Do not provide this flag unless creating a service credentials file."
+    ),
+)
+@click.option(
+    "-h",
+    "--host",
+    type=str,
+    required=True,
+    show_default=True,
+    default="cbsuwsun.biohpc.cornell.edu",
+    help="The host name or IP address of the server.",
+)
+@click.option(
+    "-sr",
+    "--storage-root",
+    type=str,
+    required=True,
+    show_default=True,
+    default="/local/storage",
+    help=(
+        "The absolute path to to the root storage server directory. Typically, this is the path to the "
+        "top-level (root) directory of the HDD RAID volume."
+    ),
+)
+@click.option(
+    "-wr",
+    "--working-root",
+    type=str,
+    required=True,
+    show_default=True,
+    default="/local/workdir",
+    help=(
+        "The absolute path to the root working server directory. Typically, this is the path to the top-level "
+        "(root) directory of the NVME RAID volume. If the server uses the same volume for both storing and working "
+        "with data, set this to the same path as the 'storage-root' argument."
+    ),
+)
+@click.option(
+    "-sd",
+    "--shared-directory",
+    type=str,
+    required=True,
+    show_default=True,
+    default="sun_data",
+    help="The name of the shared directory used to store all Sun lab project data on all server volumes.",
+)
+def generate_server_credentials_file(
+    username: str,
+    password: str,
+    service: bool,
+    host: str,
+    storage_root: str,
+    working_root: str,
+    shared_directory: str,
+) -> None:
+    """Generates a service or user server access credentials' file.
+
+    This command is used to set up access to the lab's remote compute server(s). The Server class uses the data stored
+    inside the generated credentials .yaml file to connect to and execute remote jobs on the target compute server(s).
+    Depending on the configuration, this command generates either the 'user_credentials.yaml' or
+    'service_credentials.yaml' file.
+    """
+
+    # Resolves the path to the local Sun lab working directory.
+    output_directory = get_working_directory()
+
+    # Generates the requested credentials' file.
+    generate_server_credentials(
+        output_directory=output_directory,
+        username=username,
+        password=password,
+        service=service,
+        host=host,
+        storage_root=storage_root,
+        working_root=working_root,
+        shared_directory_name=shared_directory,
+    )
+
+
+@configure.command("system")
+@click.option(
+    "-s",
+    "--system",
+    type=click.Choice(AcquisitionSystems, case_sensitive=False),
+    show_default=True,
+    required=True,
+    default=AcquisitionSystems.MESOSCOPE_VR,
+    help="The type (name) of the data acquisition system for which to generate the configuration file.",
+)
+def generate_system_configuration_file(system: AcquisitionSystems) -> None:
+    """Generates the configuration file for the specified data acquisition system.
+
+    This command is typically used when setting up new data acquisition systems in the lab. The sl-experiment library
+    uses the created file to load the acquisition system configuration data during data acquisition runtimes. The
+    system configuration only needs to be created on the machine (PC) that runs the sl-experiment library and manages
+    the acquisition runtime if the system uses multiple machines (PCs). Once the system configuration .yaml file is
+    created via this command, edit the file to modify the acquisition system configuration at any time.
+    """
+
+    create_system_configuration_file(system=system)

sl_shared_assets/command_line_interfaces/manage.py

@@ -0,0 +1,226 @@
+"""This module provides the Command-Line Interfaces (CLIs) for managing Sun lab sessions and projects. Most of these
+CLIs are intended to run on the remote compute server and should not be used by end-users directly."""
+
+from typing import Any
+from pathlib import Path
+
+import click
+from ataraxis_base_utilities import LogLevel, console
+
+from ..tools import (
+    archive_session,
+    prepare_session,
+    resolve_checksum,
+    generate_project_manifest,
+)
+
+# Ensures that displayed CLICK help messages are formatted according to the lab standard.
+CONTEXT_SETTINGS = dict(max_content_width=120)  # or any width you want
+
+
+@click.group("manage", context_settings=CONTEXT_SETTINGS)
+def manage() -> None:
+    """This Command-Line Interface (CLI) allows managing session and project data acquired in the Sun lab.
+
+    This CLI is intended to run on the Sun lab remote compute server(s) and should not be called by the end-user
+    directly. Instead, commands from this CLI are designed to be accessed through the bindings in the sl-experiment and
+    sl-forgery libraries.
+    """
+
+
+# Session data management commands
+@manage.group("session")
+@click.option(
+    "-sp",
+    "--session-path",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="The absolute path to the root session directory to process. This directory must contain the 'raw_data' "
+    "subdirectory.",
+)
+@click.option(
+    "-pdr",
+    "--processed-data-root",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=False,
+    help=(
+        "The absolute path to the directory that stores the processed data from all Sun lab projects, if it is "
+        "different from the root directory included in the 'session-path' argument value."
+    ),
+)
+@click.option(
+    "-id",
+    "--manager-id",
+    type=int,
+    required=True,
+    default=0,
+    show_default=True,
+    help="The unique identifier of the process that manages this runtime.",
+)
+@click.option(
+    "-r",
+    "--reset-tracker",
+    is_flag=True,
+    required=False,
+    help=(
+        "Determines whether to forcibly reset the tracker file for the target session management pipeline before "
+        "processing runtime. This flag should only be used in exceptional cases to recover from improper runtime "
+        "terminations."
+    ),
+)
+@click.pass_context
+def manage_session(
+    ctx: Any, session_path: Path, processed_data_root: Path | None, manager_id: int, reset_tracker: bool
+) -> None:
+    """This group provides commands for managing the data of a Sun lab data acquisition session.
+
+    Commands from this group are used to support data processing and dataset-formation (forging) on remote compute
+    servers."""
+    ctx.ensure_object(dict)
+    ctx.obj["session_path"] = session_path
+    ctx.obj["processed_data_root"] = processed_data_root
+    ctx.obj["manager_id"] = manager_id
+    ctx.obj["reset_tracker"] = reset_tracker
+
+
+# noinspection PyUnresolvedReferences
+@manage_session.command("checksum")
+@click.pass_context
+@click.option(
+    "-rc",
+    "--recalculate-checksum",
+    is_flag=True,
+    help=(
+        "Determines whether to recalculate and overwrite the cached checksum value for the processed session. When "
+        "the command is called with this flag, it effectively re-checksums the data instead of verifying its integrity."
+    ),
+)
+def resolve_session_checksum(ctx: Any, recalculate_checksum: bool) -> None:
+    """Resolves the data integrity checksum for the target session's 'raw_data' directory.
+
+    This command can be used to verify the integrity of the session's 'raw_data' directory using an existing
+    checksum or to re-generate the checksum to reflect the current state of the directory. It only works with the
+    'raw_data' session directory and ignores all other directories. Primarily, this command is used to verify the
+    integrity of the session's data as it is transferred from data acquisition systems to long-term storage
+    destinations.
+    """
+
+    # Extracts shared parameters from context
+    session_path = ctx.obj["session_path"]
+    processed_data_root = ctx.obj["processed_data_root"]
+    manager_id = ctx.obj["manager_id"]
+    reset_tracker = ctx.obj["reset_tracker"]
+
+    resolve_checksum(
+        session_path=session_path,
+        manager_id=manager_id,
+        processed_data_root=processed_data_root,
+        regenerate_checksum=recalculate_checksum,
+        reset_tracker=reset_tracker,
+    )
+
+
+# noinspection PyUnresolvedReferences
+@manage_session.command("prepare")
+@click.pass_context
+def prepare_session_for_processing(
+    ctx: Any,
+) -> None:
+    """Prepares the target session for data processing by moving all session data to the working volume.
+
+    This command is intended to run on remote compute servers that use slow HDD volumes to maximize data integrity and
+    fast NVME volumes to maximize data processing speed. For such systems, moving the data to the fast volume before
+    processing results in a measurable processing time decrease.
+    """
+    # Extracts shared parameters from context
+    session_path = ctx.obj["session_path"]
+    processed_data_root = ctx.obj["processed_data_root"]
+    manager_id = ctx.obj["manager_id"]
+    reset_tracker = ctx.obj["reset_tracker"]
+
+    prepare_session(
+        session_path=session_path,
+        manager_id=manager_id,
+        processed_data_root=processed_data_root,
+        reset_tracker=reset_tracker,
+    )
+
+
+# noinspection PyUnresolvedReferences
+@manage_session.command("archive")
+@click.pass_context
+def archive_session_for_storage(
+    ctx: Any,
+) -> None:
+    """Prepares the target session for long-term storage by moving all session data to the storage volume.
+
+    This command is primarily intended to run on remote compute servers that use slow HDD volumes to maximize data
+    integrity and fast NVME volumes to maximize data processing speed. For such systems, moving all sessions that are no
+    longer actively processed or analyzed to the slow drive volume frees up the processing volume space and ensures
+    long-term data integrity.
+    """
+    # Extracts shared parameters from context
+    session_path = ctx.obj["session_path"]
+    processed_data_root = ctx.obj["processed_data_root"]
+    manager_id = ctx.obj["manager_id"]
+    reset_tracker = ctx.obj["reset_tracker"]
+
+    archive_session(
+        session_path=session_path,
+        manager_id=manager_id,
+        processed_data_root=processed_data_root,
+        reset_tracker=reset_tracker,
+    )
+
+
+@manage.group("project")
+@click.pass_context
+@click.option(
+    "-pp",
+    "--project-path",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="The absolute path to the project-specific directory where raw session data is stored.",
+)
+@click.option(
+    "-pdr",
+    "--processed-data-root",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=False,
+    help=(
+        "The absolute path to the directory that stores the processed data from all Sun lab projects, if it is "
+        "different from the root directory included in the 'session-path' argument value."
+    ),
+)
+def manage_project(ctx: Any, project_path: Path, processed_data_root: Path | None) -> None:
+    """This group provides commands for managing the data of a Sun lab project.
+
+    Commands from this group are used to support all interactions with the data stored on the Sun lab remote compute
+    server(s)."""
+    ctx.ensure_object(dict)
+    ctx.obj["project_path"] = project_path
+    ctx.obj["processed_data_root"] = processed_data_root
+
+
+# noinspection PyUnresolvedReferences
+@manage_project.command("manifest")
+@click.pass_context
+def generate_project_manifest_file(ctx: Any) -> None:
+    """Generates the manifest .feather file that captures the current state of the target project's data.
+
+    The manifest file contains the comprehensive snapshot of the available project's data. It includes the information
+    about the management and processing pipelines that have been applied to each session's data, as well as the
+    descriptive information about each session. The manifest file is used as an entry-point for all interactions with
+    the Sun lab data stored on the remote compute server(s).
+    """
+    # Extracts shared parameters from context
+    project_path = ctx.obj["project_path"]
+    processed_data_root = ctx.obj["processed_data_root"]
+
+    generate_project_manifest(
+        raw_project_directory=project_path,
+        processed_data_root=processed_data_root,
+        manager_id=1,
+    )
+
+    console.echo(message=f"Project {Path(project_path).stem} data manifest file: generated.", level=LogLevel.SUCCESS)

sl_shared_assets/data_classes/__init__.py

@@ -1,7 +1,6 @@
-"""This package provides the classes used to store data acquired at various stages of the data workflow and to
-configure various pipelines used in the Sun lab. These classes are used across all stages of data acquisition,
-preprocessing, and processing in the lab. Many classes in this package are designed to be saved to disk as .yaml files
-and restored from the .yaml files as needed."""
+"""This package provides the classes used to store data acquired at all stages of the Sun lab data workflow and to
+configure various elements and pipelines making up the overall workflow. Many classes in this package are designed to
+be saved to disk as .yaml files and restored from the .yaml files as needed."""
 
 from .runtime_data import (
     ZaberPositions,
@@ -15,12 +14,10 @@ from .runtime_data import (
 from .session_data import (
     RawData,
     SessionData,
+    SessionLock,
     SessionTypes,
+    TrackingData,
     ProcessedData,
-    TrackerFileNames,
-    ProcessingTracker,
-    generate_manager_id,
-    get_processing_tracker,
 )
 from .surgery_data import (
     DrugData,
@@ -40,41 +37,45 @@ from .configuration_data import (
     MesoscopeAdditionalFirmware,
     MesoscopeSystemConfiguration,
     MesoscopeExperimentConfiguration,
+    get_working_directory,
+    set_working_directory,
+    get_credentials_file_path,
     get_system_configuration_data,
-    set_system_configuration_file,
+    create_system_configuration_file,
 )
 
 __all__ = [
+    "AcquisitionSystems",
     "DrugData",
+    "ExperimentState",
+    "ExperimentTrial",
     "ImplantData",
-    "SessionData",
-    "RawData",
-    "ProcessedData",
-    "SubjectData",
-    "SurgeryData",
     "InjectionData",
-    "ProcedureData",
-    "ZaberPositions",
-    "ExperimentState",
-    "MesoscopePositions",
-    "MesoscopeHardwareState",
-    "RunTrainingDescriptor",
     "LickTrainingDescriptor",
+    "MesoscopeAdditionalFirmware",
+    "MesoscopeCameras",
     "MesoscopeExperimentConfiguration",
     "MesoscopeExperimentDescriptor",
-    "MesoscopeSystemConfiguration",
-    "set_system_configuration_file",
-    "get_system_configuration_data",
-    "MesoscopePaths",
-    "MesoscopeCameras",
+    "MesoscopeHardwareState",
     "MesoscopeMicroControllers",
-    "MesoscopeAdditionalFirmware",
-    "ProcessingTracker",
-    "ExperimentTrial",
-    "AcquisitionSystems",
+    "MesoscopePaths",
+    "MesoscopePositions",
+    "MesoscopeSystemConfiguration",
+    "ProcedureData",
+    "ProcessedData",
+    "RawData",
+    "RunTrainingDescriptor",
+    "SessionData",
+    "SessionLock",
     "SessionTypes",
+    "SubjectData",
+    "SurgeryData",
+    "TrackingData",
     "WindowCheckingDescriptor",
-    "get_processing_tracker",
-    "generate_manager_id",
-    "TrackerFileNames",
+    "ZaberPositions",
+    "create_system_configuration_file",
+    "get_credentials_file_path",
+    "get_system_configuration_data",
+    "get_working_directory",
+    "set_working_directory",
 ]