sl-shared-assets 2.0.1__py3-none-any.whl → 3.0.0__py3-none-any.whl

sl_shared_assets/__init__.py

@@ -2,13 +2,17 @@
 
 See https://github.com/Sun-Lab-NBB/sl-shared-assets for more details.
 API documentation: https://sl-shared-assets-api-docs.netlify.app/
-Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Yuantao Deng, Natalie Yeung
+Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Natalie Yeung
 """
 
 from ataraxis_base_utilities import console
 
-from .tools import transfer_directory, verify_session_checksum, generate_project_manifest, calculate_directory_checksum
-from .server import Job, Server, ServerCredentials
+from .tools import (
+    resolve_p53_marker,
+    transfer_directory,
+    calculate_directory_checksum,
+)
+from .server import Job, Server, JupyterJob, ServerCredentials
 from .data_classes import (
     RawData,
     DrugData,
@@ -16,20 +20,22 @@ from .data_classes import (
     SessionData,
     SubjectData,
     SurgeryData,
-    VersionData,
+    SessionTypes,
     InjectionData,
     ProcedureData,
     ProcessedData,
     MesoscopePaths,
     ZaberPositions,
     ExperimentState,
+    ExperimentTrial,
     MesoscopeCameras,
     ProcessingTracker,
+    AcquisitionSystems,
     MesoscopePositions,
-    ProjectConfiguration,
     RunTrainingDescriptor,
     LickTrainingDescriptor,
     MesoscopeHardwareState,
+    WindowCheckingDescriptor,
     MesoscopeMicroControllers,
     MesoscopeAdditionalFirmware,
     MesoscopeSystemConfiguration,
@@ -48,12 +54,12 @@ __all__ = [
     "Server",
     "ServerCredentials",
     "Job",
+    "JupyterJob",
     # Data classes package
     "DrugData",
     "ImplantData",
     "SessionData",
     "RawData",
-    "VersionData",
     "ProcessedData",
     "SubjectData",
     "SurgeryData",
@@ -63,7 +69,6 @@ __all__ = [
     "ZaberPositions",
     "ExperimentState",
     "MesoscopePositions",
-    "ProjectConfiguration",
     "MesoscopeHardwareState",
     "RunTrainingDescriptor",
     "LickTrainingDescriptor",
@@ -76,9 +81,12 @@ __all__ = [
     "MesoscopeAdditionalFirmware",
     "get_system_configuration_data",
     "set_system_configuration_file",
+    "ExperimentTrial",
+    "SessionTypes",
+    "AcquisitionSystems",
+    "WindowCheckingDescriptor",
     # Tools package
+    "resolve_p53_marker",
     "transfer_directory",
-    "generate_project_manifest",
-    "verify_session_checksum",
     "calculate_directory_checksum",
 ]

sl_shared_assets/__init__.pyi

@@ -1,12 +1,12 @@
 from .tools import (
+    resolve_p53_marker as resolve_p53_marker,
     transfer_directory as transfer_directory,
-    verify_session_checksum as verify_session_checksum,
-    generate_project_manifest as generate_project_manifest,
     calculate_directory_checksum as calculate_directory_checksum,
 )
 from .server import (
     Job as Job,
     Server as Server,
+    JupyterJob as JupyterJob,
     ServerCredentials as ServerCredentials,
 )
 from .data_classes import (
@@ -16,20 +16,22 @@ from .data_classes import (
     SessionData as SessionData,
     SubjectData as SubjectData,
     SurgeryData as SurgeryData,
-    VersionData as VersionData,
+    SessionTypes as SessionTypes,
     InjectionData as InjectionData,
     ProcedureData as ProcedureData,
     ProcessedData as ProcessedData,
     MesoscopePaths as MesoscopePaths,
     ZaberPositions as ZaberPositions,
     ExperimentState as ExperimentState,
+    ExperimentTrial as ExperimentTrial,
     MesoscopeCameras as MesoscopeCameras,
     ProcessingTracker as ProcessingTracker,
+    AcquisitionSystems as AcquisitionSystems,
     MesoscopePositions as MesoscopePositions,
-    ProjectConfiguration as ProjectConfiguration,
     RunTrainingDescriptor as RunTrainingDescriptor,
     LickTrainingDescriptor as LickTrainingDescriptor,
     MesoscopeHardwareState as MesoscopeHardwareState,
+    WindowCheckingDescriptor as WindowCheckingDescriptor,
     MesoscopeMicroControllers as MesoscopeMicroControllers,
     MesoscopeAdditionalFirmware as MesoscopeAdditionalFirmware,
     MesoscopeSystemConfiguration as MesoscopeSystemConfiguration,
@@ -43,11 +45,11 @@ __all__ = [
     "Server",
     "ServerCredentials",
     "Job",
+    "JupyterJob",
     "DrugData",
     "ImplantData",
     "SessionData",
     "RawData",
-    "VersionData",
     "ProcessedData",
     "SubjectData",
     "SurgeryData",
@@ -57,7 +59,6 @@ __all__ = [
     "ZaberPositions",
     "ExperimentState",
     "MesoscopePositions",
-    "ProjectConfiguration",
     "MesoscopeHardwareState",
     "RunTrainingDescriptor",
     "LickTrainingDescriptor",
@@ -70,8 +71,11 @@ __all__ = [
     "MesoscopeAdditionalFirmware",
     "get_system_configuration_data",
     "set_system_configuration_file",
+    "ExperimentTrial",
+    "SessionTypes",
+    "AcquisitionSystems",
+    "WindowCheckingDescriptor",
+    "resolve_p53_marker",
     "transfer_directory",
-    "generate_project_manifest",
-    "verify_session_checksum",
     "calculate_directory_checksum",
 ]

sl_shared_assets/cli.py

@@ -3,10 +3,10 @@
 from pathlib import Path
 
 import click
-from ataraxis_base_utilities import LogLevel, console
+from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
 
-from .tools import ascend_tyche_data, verify_session_checksum, generate_project_manifest
-from .server import generate_server_credentials
+from .tools import ascend_tyche_data, resolve_p53_marker, verify_session_checksum, generate_project_manifest
+from .server import Server, JupyterJob, generate_server_credentials
 from .data_classes import SessionData, ProcessingTracker
 
 
@@ -16,7 +16,7 @@ from .data_classes import SessionData, ProcessingTracker
     "--session_path",
     type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
     required=True,
-    help="The absolute path to the session whose raw data needs to be verified for potential corruption.",
+    help="The absolute path to the session directory whose raw data needs to be verified for potential corruption.",
 )
 @click.option(
     "-c",
@@ -43,7 +43,9 @@ from .data_classes import SessionData, ProcessingTracker
         "used if 'create_processed_directories' flag is True."
     ),
 )
-def verify_session_integrity(session_path: str, create_processed_directories: bool, processed_data_root: Path) -> None:
+def verify_session_integrity(
+    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None
+) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
     This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
@@ -101,7 +103,7 @@ def verify_session_integrity(session_path: str, create_processed_directories: bo
     ),
 )
 def generate_project_manifest_file(
-    project_path: str, output_directory: str, project_processed_path: str | None
+    project_path: Path, output_directory: Path, project_processed_path: Path | None
 ) -> None:
     """Generates the manifest .feather file that provides information about the data-processing state of all available
     project sessions.
@@ -123,7 +125,7 @@ def generate_project_manifest_file(
 @click.option(
     "-od",
     "--output_directory",
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path),
     required=True,
     help="The absolute path to the directory where to store the generated server credentials file.",
 )
@@ -151,29 +153,50 @@ def generate_project_manifest_file(
     help="The password to use for server authentication.",
 )
 @click.option(
-    "-rdp",
-    "--raw_data_path",
+    "-sr",
+    "--storage_root",
     type=str,
     required=True,
-    default="/storage/sun_data",
+    show_default=True,
+    default="/local/storage",
     help=(
-        "The absolute path to the directory used to store raw data from all Sun lab projects, relative to the server "
-        "root."
+        "The absolute path to to the root storage (slow) server directory. Typically, this is the path to the "
+        "top-level (root) directory of the HDD RAID volume."
     ),
 )
 @click.option(
-    "-pdp",
-    "--processed_data_path",
+    "-wr",
+    "--working_root",
     type=str,
     required=True,
-    default="/workdir/sun_data",
+    show_default=True,
+    default="/local/workdir",
     help=(
-        "The absolute path to the directory used to store processed data from all Sun lab projects, relative to the "
-        "server root."
+        "The absolute path to the root working (fast) server directory. Typically, this is the path to the top-level "
+        "(root) directory of the NVME RAID volume. If the server uses the same volume for both storage and working "
+        "directories, enter the same path under both 'storage_root' and 'working_root'."
+    ),
+)
+@click.option(
+    "-sdn",
+    "--shared_directory_name",
+    type=str,
+    required=True,
+    show_default=True,
+    default="sun_data",
+    help=(
+        "The name of the shared directory used to store all Sun lab project data on the storage and working server "
+        "volumes."
     ),
 )
 def generate_server_credentials_file(
-    output_directory: str, host: str, username: str, password: str, raw_data_path: str, processed_data_path: str
+    output_directory: Path,
+    host: str,
+    username: str,
+    password: str,
+    storage_root: str,
+    working_root: str,
+    shared_directory_name: str,
 ) -> None:
     """Generates a new server_credentials.yaml file under the specified directory, using input information.
 
@@ -181,13 +204,19 @@ def generate_server_credentials_file(
     the server_credentials.yaml file generated by this command is used by the Server and Job classes used in many Sun
     lab data processing libraries.
     """
+
+    # If necessary, generates the output directory hierarchy before creating the credentials' file.
+    ensure_directory_exists(output_directory)
+
+    # Generates the credentials' file
     generate_server_credentials(
         output_directory=Path(output_directory),
         username=username,
         password=password,
         host=host,
-        raw_data_root=raw_data_path,
-        processed_data_root=processed_data_path,
+        storage_root=storage_root,
+        working_root=working_root,
+        shared_directory_name=shared_directory_name,
     )
     message = (
         f"Server access credentials file: generated. If necessary, remember to edit the data acquisition system "
@@ -195,6 +224,9 @@ def generate_server_credentials_file(
     )
     # noinspection PyTypeChecker
     console.echo(message=message, level=LogLevel.SUCCESS)
+    message = f"File location: {output_directory}"
+    # noinspection PyTypeChecker
+    console.echo(message=message, level=LogLevel.SUCCESS)
 
 
 @click.command()
@@ -205,7 +237,7 @@ def generate_server_credentials_file(
     required=True,
     help="The absolute path to the directory that stores original Tyche animal folders.",
 )
-def ascend_tyche_directory(input_directory: str) -> None:
+def ascend_tyche_directory(input_directory: Path) -> None:
     """Restructures old Tyche project data to use the modern Sun lab data structure and uploads them to the processing
     server.
 
@@ -216,3 +248,208 @@ def ascend_tyche_directory(input_directory: str) -> None:
     valid Sun lab data acquisition system, such as VRPC of the Mesoscope-VR system.
     """
     ascend_tyche_data(root_directory=Path(input_directory))
+
+
+@click.command()
+@click.option(
+    "-cp",
+    "--credentials_path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path),
+    required=True,
+    help=(
+        "The absolute path to the server_credentials.yaml file that stores access credentials for the target Sun lab "
+        "server. If necessary, use the 'sl-create-server-credentials' command to generate the file."
+    ),
+)
+@click.option(
+    "-n",
+    "--name",
+    type=str,
+    required=True,
+    show_default=True,
+    default="jupyter_server",
+    help=(
+        "The descriptive name to be given to the remote Jupyter server job. Primarily, this is used to identify the "
+        "job inside the log files."
+    ),
+)
+@click.option(
+    "-e",
+    "--environment",
+    type=str,
+    required=True,
+    help=(
+        "The name of the conda environment to use for running the Jupyter server. At a minimum, the target environment "
+        "must contain the 'jupyterlab' and the 'notebook' Python packages. Note, the user whose credentials are used "
+        "to connect to the server must have a configured conda / mamba shell that exposes the target environment for "
+        "the job to run as expected."
+    ),
+)
+@click.option(
+    "-d",
+    "--directory",
+    type=click.Path(exists=False, file_okay=True, dir_okay=True, path_type=Path),
+    required=False,
+    help=(
+        "The absolute path to the server directory to use as the root directory for the jupyter session. If not "
+        "provided, this is automatically resolved to user's working directory. Note, during runtime, Jupyter will only "
+        "have access to files stored in or under that root directory."
+    ),
+)
+@click.option(
+    "-c",
+    "--cores",
+    type=int,
+    required=True,
+    show_default=True,
+    default=2,
+    help=(
+        "The number of CPU cores to allocate to the Jupyter server. Note, during the interactive Jupyter runtime, it "
+        "is be impossible to use more than this number of CPU cores."
+    ),
+)
+@click.option(
+    "-m",
+    "--memory",
+    type=int,
+    required=True,
+    show_default=True,
+    default=32,
+    help=(
+        "The RAM, in Gigabytes, to allocate to the Jupyter server. Note, during the interactive Jupyter runtime, it "
+        "is be impossible to use more than this amount of RAM."
+    ),
+)
+@click.option(
+    "-t",
+    "--time",
+    type=int,
+    required=True,
+    show_default=True,
+    default=240,
+    help=(
+        "The maximum runtime duration for this Jupyter server instance, in minutes. If the server job is still running "
+        "at the end of this time limit, the job will be forcibly terminated by SLURM. Note, to prevent hogging the "
+        "server, make sure this parameter is always set to the smallest feasible period of time you intend to interact "
+        "with the server."
+    ),
+)
+@click.option(
+    "-p",
+    "--port",
+    type=int,
+    required=True,
+    show_default=True,
+    default=0,
+    help=(
+        "The port to use for the Jupyter server communication on the remote server. Valid port values are from 8888 "
+        "to 9999. Most runtimes should leave this set to the default value (0), which will randomly select one of the "
+        "valid ports. Using random selection minimizes the chances of colliding with other interactive jupyter "
+        "sessions."
+    ),
+)
+def start_jupyter_server(
+    credentials_path: Path, name: str, environment: str, directory: Path, cores: int, memory: int, time: int, port: int
+) -> None:
+    """Starts an interactive Jupyter session on the remote Sun lab server.
+
+    This command should be used to run Jupyter lab and notebooks sessions on the remote Sun lab server. Since all lab
+    data is stored on the server, this allows running light interactive analysis sessions on the same node as the data,
+    while leveraging considerable compute resources of the server.
+
+    Calling this command initializes a SLURM session that runs the interactive Jupyter server. Since this server
+    directly competes for resources with all other headless jobs running on the server, it is imperative that each
+    jupyter runtime uses only the minimum amount of resources and run-time as necessary. Do not use this command to run
+    heavy data processing pipelines! Instead, consult with library documentation and use the headless Job class.
+    """
+    # Initializes server connection
+    server = Server(credentials_path)
+    job: JupyterJob | None = None
+    try:
+        # If the caller did not provide an explicit notebook directory, defaults to user's working directory
+        if directory is None:
+            directory = (server.user_working_root,)
+
+        # Launches the specified Jupyter server
+        job = server.launch_jupyter_server(
+            job_name=name,
+            conda_environment=environment,
+            notebook_directory=directory,
+            cpus_to_use=cores,
+            ram_gb=memory,
+            port=port,
+            time_limit=time,
+        )
+
+        # Displays the server connection details to the user via terminal
+        job.print_connection_info()
+
+        # Blocks in-place until the user shuts down the server. This allows terminating the jupyter job early if the
+        # user is done working with the server
+        input("Enter anything to shut down the server: ")
+
+    # Ensures that the server created as part of this CLI is always terminated when the CLI terminates
+    finally:
+        # Terminates the server job
+        if job is not None:
+            server.abort_job(job)
+
+        # Closes server connection if it is still open
+        server.close()
+
+
+@click.command()
+@click.option(
+    "-sp",
+    "--session_path",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="The absolute path to the session directory for which to resolve the dataset integration readiness marker.",
+)
+@click.option(
+    "-c",
+    "--create_processed_directories",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Determines whether to create the processed data hierarchy. This flag should be disabled for most runtimes.",
+)
+@click.option(
+    "-ppp",
+    "--project_processed_path",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=False,
+    help=(
+        "The absolute path to the project directory where processed session data is stored, if different from the "
+        "directory used to store raw session data. Typically, this extra argument is only used when processing data "
+        "stored on remote compute server(s)."
+    ),
+)
+@click.option(
+    "-r",
+    "--remove",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help=(
+        "Determines whether the command should create or remove the dataset integration marker. Do not enable this "
+        "flag unless you know what you are doing. It is only safe to enable this flag if the session is not currently "
+        "being integrated into any datasets."
+    ),
+)
+def resolve_dataset_marker(
+    session_path: Path, create_processed_directories: bool, project_processed_path: Path | None, remove: bool
+) -> None:
+    """Depending on configuration, either creates or removes the p53.bin marker from the target session.
+
+    The p53.bin marker determines whether the session is ready for dataset integration. When the marker exists,
+    processing pipelines are not allowed to work with the session data, ensuring that all processed data remains
+    unchanged. If the marker does not exist, dataset integration pipelines are not allowed to work with the session
+    data, enabling processing pipelines to safely modify the data at any time.
+    """
+    resolve_p53_marker(
+        session_path=session_path,
+        create_processed_data_directory=create_processed_directories,
+        processed_data_root=project_processed_path,
+        remove=remove,
+    )

sl_shared_assets/cli.pyi

@@ -2,16 +2,23 @@ from pathlib import Path
 
 from .tools import (
     ascend_tyche_data as ascend_tyche_data,
+    resolve_p53_marker as resolve_p53_marker,
     verify_session_checksum as verify_session_checksum,
     generate_project_manifest as generate_project_manifest,
 )
-from .server import generate_server_credentials as generate_server_credentials
+from .server import (
+    Server as Server,
+    JupyterJob as JupyterJob,
+    generate_server_credentials as generate_server_credentials,
+)
 from .data_classes import (
     SessionData as SessionData,
     ProcessingTracker as ProcessingTracker,
 )
 
-def verify_session_integrity(session_path: str, create_processed_directories: bool, processed_data_root: Path) -> None:
+def verify_session_integrity(
+    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None
+) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
     This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
@@ -26,7 +33,7 @@ def verify_session_integrity(session_path: str, create_processed_directories: bo
     """
 
 def generate_project_manifest_file(
-    project_path: str, output_directory: str, project_processed_path: str | None
+    project_path: Path, output_directory: Path, project_processed_path: Path | None
 ) -> None:
     """Generates the manifest .feather file that provides information about the data-processing state of all available
     project sessions.
@@ -37,7 +44,13 @@ def generate_project_manifest_file(
     """
 
 def generate_server_credentials_file(
-    output_directory: str, host: str, username: str, password: str, raw_data_path: str, processed_data_path: str
+    output_directory: Path,
+    host: str,
+    username: str,
+    password: str,
+    storage_root: str,
+    working_root: str,
+    shared_directory_name: str,
 ) -> None:
     """Generates a new server_credentials.yaml file under the specified directory, using input information.
 
@@ -46,7 +59,7 @@ def generate_server_credentials_file(
     lab data processing libraries.
     """
 
-def ascend_tyche_directory(input_directory: str) -> None:
+def ascend_tyche_directory(input_directory: Path) -> None:
     """Restructures old Tyche project data to use the modern Sun lab data structure and uploads them to the processing
     server.
 
@@ -56,3 +69,29 @@ def ascend_tyche_directory(input_directory: str) -> None:
     will not work for any other project or data. Also, this command will only work on a machine (PC) that belongs to a
     valid Sun lab data acquisition system, such as VRPC of the Mesoscope-VR system.
     """
+
+def start_jupyter_server(
+    credentials_path: Path, name: str, environment: str, directory: Path, cores: int, memory: int, time: int, port: int
+) -> None:
+    """Starts an interactive Jupyter session on the remote Sun lab server.
+
+    This command should be used to run Jupyter lab and notebooks sessions on the remote Sun lab server. Since all lab
+    data is stored on the server, this allows running light interactive analysis sessions on the same node as the data,
+    while leveraging considerable compute resources of the server.
+
+    Calling this command initializes a SLURM session that runs the interactive Jupyter server. Since this server
+    directly competes for resources with all other headless jobs running on the server, it is imperative that each
+    jupyter runtime uses only the minimum amount of resources and run-time as necessary. Do not use this command to run
+    heavy data processing pipelines! Instead, consult with library documentation and use the headless Job class.
+    """
+
+def resolve_dataset_marker(
+    session_path: Path, create_processed_directories: bool, project_processed_path: Path | None, remove: bool
+) -> None:
+    """Depending on configuration, either creates or removes the p53.bin marker from the target session.
+
+    The p53.bin marker determines whether the session is ready for dataset integration. When the marker exists,
+    processing pipelines are not allowed to work with the session data, ensuring that all processed data remains
+    unchanged. If the marker does not exist, dataset integration pipelines are not allowed to work with the session
+    data, enabling processing pipelines to safely modify the data at any time.
+    """

sl_shared_assets/data_classes/__init__.py

@@ -9,9 +9,10 @@ from .runtime_data import (
     RunTrainingDescriptor,
     LickTrainingDescriptor,
     MesoscopeHardwareState,
+    WindowCheckingDescriptor,
     MesoscopeExperimentDescriptor,
 )
-from .session_data import RawData, SessionData, VersionData, ProcessedData, ProcessingTracker, ProjectConfiguration
+from .session_data import RawData, SessionData, SessionTypes, ProcessedData, ProcessingTracker
 from .surgery_data import (
     DrugData,
     ImplantData,
@@ -23,7 +24,9 @@ from .surgery_data import (
 from .configuration_data import (
     MesoscopePaths,
     ExperimentState,
+    ExperimentTrial,
     MesoscopeCameras,
+    AcquisitionSystems,
     MesoscopeMicroControllers,
     MesoscopeAdditionalFirmware,
     MesoscopeSystemConfiguration,
@@ -37,7 +40,6 @@ __all__ = [
     "ImplantData",
     "SessionData",
     "RawData",
-    "VersionData",
     "ProcessedData",
     "SubjectData",
     "SurgeryData",
@@ -46,7 +48,6 @@ __all__ = [
     "ZaberPositions",
     "ExperimentState",
     "MesoscopePositions",
-    "ProjectConfiguration",
     "MesoscopeHardwareState",
     "RunTrainingDescriptor",
     "LickTrainingDescriptor",
@@ -60,4 +61,8 @@ __all__ = [
     "MesoscopeMicroControllers",
     "MesoscopeAdditionalFirmware",
     "ProcessingTracker",
+    "ExperimentTrial",
+    "AcquisitionSystems",
+    "SessionTypes",
+    "WindowCheckingDescriptor",
 ]