sl-shared-assets 2.0.0__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

sl_shared_assets/__init__.py

@@ -2,13 +2,17 @@
 
 See https://github.com/Sun-Lab-NBB/sl-shared-assets for more details.
 API documentation: https://sl-shared-assets-api-docs.netlify.app/
-Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Yuantao Deng, Natalie Yeung
+Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Natalie Yeung
 """
 
 from ataraxis_base_utilities import console
 
-from .tools import transfer_directory, verify_session_checksum, generate_project_manifest, calculate_directory_checksum
-from .server import Job, Server, ServerCredentials
+from .tools import (
+    resolve_p53_marker,
+    transfer_directory,
+    calculate_directory_checksum,
+)
+from .server import Job, Server, JupyterJob, ServerCredentials
 from .data_classes import (
     RawData,
     DrugData,
@@ -48,6 +52,7 @@ __all__ = [
     "Server",
     "ServerCredentials",
     "Job",
+    "JupyterJob",
     # Data classes package
     "DrugData",
     "ImplantData",
@@ -77,8 +82,7 @@ __all__ = [
     "get_system_configuration_data",
     "set_system_configuration_file",
     # Tools package
+    "resolve_p53_marker",
     "transfer_directory",
-    "generate_project_manifest",
-    "verify_session_checksum",
     "calculate_directory_checksum",
 ]

sl_shared_assets/__init__.pyi

@@ -1,12 +1,12 @@
 from .tools import (
+    resolve_p53_marker as resolve_p53_marker,
     transfer_directory as transfer_directory,
-    verify_session_checksum as verify_session_checksum,
-    generate_project_manifest as generate_project_manifest,
     calculate_directory_checksum as calculate_directory_checksum,
 )
 from .server import (
     Job as Job,
     Server as Server,
+    JupyterJob as JupyterJob,
     ServerCredentials as ServerCredentials,
 )
 from .data_classes import (
@@ -43,6 +43,7 @@ __all__ = [
     "Server",
     "ServerCredentials",
     "Job",
+    "JupyterJob",
     "DrugData",
     "ImplantData",
     "SessionData",
@@ -70,8 +71,7 @@ __all__ = [
     "MesoscopeAdditionalFirmware",
     "get_system_configuration_data",
     "set_system_configuration_file",
+    "resolve_p53_marker",
     "transfer_directory",
-    "generate_project_manifest",
-    "verify_session_checksum",
     "calculate_directory_checksum",
 ]

sl_shared_assets/cli.py

@@ -3,10 +3,10 @@
 from pathlib import Path
 
 import click
-from ataraxis_base_utilities import LogLevel, console
+from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
 
-from .tools import ascend_tyche_data, verify_session_checksum, generate_project_manifest
-from .server import generate_server_credentials
+from .tools import ascend_tyche_data, resolve_p53_marker, verify_session_checksum, generate_project_manifest
+from .server import Server, JupyterJob, generate_server_credentials
 from .data_classes import SessionData, ProcessingTracker
 
 
@@ -16,7 +16,7 @@ from .data_classes import SessionData, ProcessingTracker
     "--session_path",
     type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
     required=True,
-    help="The absolute path to the session whose raw data needs to be verified for potential corruption.",
+    help="The absolute path to the session directory whose raw data needs to be verified for potential corruption.",
 )
 @click.option(
     "-c",
@@ -43,7 +43,9 @@ from .data_classes import SessionData, ProcessingTracker
         "used if 'create_processed_directories' flag is True."
     ),
 )
-def verify_session_integrity(session_path: str, create_processed_directories: bool, processed_data_root: Path) -> None:
+def verify_session_integrity(
+    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None
+) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
     This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
@@ -101,7 +103,7 @@ def verify_session_integrity(session_path: str, create_processed_directories: bo
     ),
 )
 def generate_project_manifest_file(
-    project_path: str, output_directory: str, project_processed_path: str | None
+    project_path: Path, output_directory: Path, project_processed_path: Path | None
 ) -> None:
     """Generates the manifest .feather file that provides information about the data-processing state of all available
     project sessions.
@@ -123,7 +125,7 @@ def generate_project_manifest_file(
 @click.option(
     "-od",
     "--output_directory",
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path),
     required=True,
     help="The absolute path to the directory where to store the generated server credentials file.",
 )
@@ -151,34 +153,70 @@ def generate_project_manifest_file(
     help="The password to use for server authentication.",
 )
 @click.option(
-    "-rdp",
-    "--raw_data_path",
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    "-sr",
+    "--storage_root",
+    type=str,
     required=True,
+    show_default=True,
+    default="/local/storage",
     help=(
-        "The absolute path to the directory used to store raw data from all Sun lab projects, relative to the server "
-        "root."
+        "The absolute path to to the root storage (slow) server directory. Typically, this is the path to the "
+        "top-level (root) directory of the HDD RAID volume."
     ),
 )
 @click.option(
-    "-pdp",
-    "--processed_data_path",
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    "-wr",
+    "--working_root",
+    type=str,
+    required=True,
+    show_default=True,
+    default="/local/workdir",
+    help=(
+        "The absolute path to the root working (fast) server directory. Typically, this is the path to the top-level "
+        "(root) directory of the NVME RAID volume. If the server uses the same volume for both storage and working "
+        "directories, enter the same path under both 'storage_root' and 'working_root'."
+    ),
+)
+@click.option(
+    "-sdn",
+    "--shared_directory_name",
+    type=str,
     required=True,
+    show_default=True,
+    default="sun_data",
     help=(
-        "The absolute path to the directory used to store processed data from all Sun lab projects, relative to the "
-        "server root."
+        "The name of the shared directory used to store all Sun lab project data on the storage and working server "
+        "volumes."
     ),
 )
-def generate_server_credentials_file(output_directory: str, host: str, username: str, password: str) -> None:
+def generate_server_credentials_file(
+    output_directory: Path,
+    host: str,
+    username: str,
+    password: str,
+    storage_root: str,
+    working_root: str,
+    shared_directory_name: str,
+) -> None:
     """Generates a new server_credentials.yaml file under the specified directory, using input information.
 
     This command is used to set up access to compute servers and clusters on new machines (PCs). The data stored inside
     the server_credentials.yaml file generated by this command is used by the Server and Job classes used in many Sun
     lab data processing libraries.
     """
+
+    # If necessary, generates the output directory hierarchy before creating the credentials' file.
+    ensure_directory_exists(output_directory)
+
+    # Generates the credentials' file
     generate_server_credentials(
-        output_directory=Path(output_directory), username=username, password=password, host=host
+        output_directory=Path(output_directory),
+        username=username,
+        password=password,
+        host=host,
+        storage_root=storage_root,
+        working_root=working_root,
+        shared_directory_name=shared_directory_name,
     )
     message = (
         f"Server access credentials file: generated. If necessary, remember to edit the data acquisition system "
@@ -186,6 +224,9 @@ def generate_server_credentials_file(output_directory: str, host: str, username:
     )
     # noinspection PyTypeChecker
     console.echo(message=message, level=LogLevel.SUCCESS)
+    message = f"File location: {output_directory}"
+    # noinspection PyTypeChecker
+    console.echo(message=message, level=LogLevel.SUCCESS)
 
 
 @click.command()
@@ -196,7 +237,7 @@ def generate_server_credentials_file(output_directory: str, host: str, username:
     required=True,
     help="The absolute path to the directory that stores original Tyche animal folders.",
 )
-def ascend_tyche_directory(input_directory: str) -> None:
+def ascend_tyche_directory(input_directory: Path) -> None:
     """Restructures old Tyche project data to use the modern Sun lab data structure and uploads them to the processing
     server.
 
@@ -207,3 +248,212 @@ def ascend_tyche_directory(input_directory: str) -> None:
     valid Sun lab data acquisition system, such as VRPC of the Mesoscope-VR system.
     """
     ascend_tyche_data(root_directory=Path(input_directory))
+
+
+@click.command()
+@click.option(
+    "-cp",
+    "--credentials_path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path),
+    required=True,
+    help=(
+        "The absolute path to the server_credentials.yaml file that stores access credentials for the target Sun lab "
+        "server. If necessary, use the 'sl-create-server-credentials' command to generate the file."
+    ),
+)
+@click.option(
+    "-n",
+    "--name",
+    type=str,
+    required=True,
+    show_default=True,
+    default="jupyter_server",
+    help=(
+        "The descriptive name to be given to the remote Jupyter server job. Primarily, this is used to identify the "
+        "job inside the log files."
+    ),
+)
+@click.option(
+    "-e",
+    "--environment",
+    type=str,
+    required=True,
+    help=(
+        "The name of the conda environment to use for running the Jupyter server. At a minimum, the target environment "
+        "must contain the 'jupyterlab' and the 'notebook' Python packages. Note, the user whose credentials are used "
+        "to connect to the server must have a configured conda / mamba shell that exposes the target environment for "
+        "the job to run as expected."
+    ),
+)
+@click.option(
+    "-d",
+    "--directory",
+    type=click.Path(exists=False, file_okay=True, dir_okay=True, path_type=Path),
+    required=False,
+    help=(
+        "The absolute path to the server directory to use as the root directory for the jupyter session. If not "
+        "provided, this is automatically resolved to user's working directory. Note, during runtime, Jupyter will only "
+        "have access to files stored in or under that root directory."
+    ),
+)
+@click.option(
+    "-c",
+    "--cores",
+    type=int,
+    required=True,
+    show_default=True,
+    default=2,
+    help=(
+        "The number of CPU cores to allocate to the Jupyter server. Note, during the interactive Jupyter runtime, it "
+        "is be impossible to use more than this number of CPU cores."
+    ),
+)
+@click.option(
+    "-m",
+    "--memory",
+    type=int,
+    required=True,
+    show_default=True,
+    default=32,
+    help=(
+        "The RAM, in Gigabytes, to allocate to the Jupyter server. Note, during the interactive Jupyter runtime, it "
+        "is be impossible to use more than this amount of RAM."
+    ),
+)
+@click.option(
+    "-t",
+    "--time",
+    type=int,
+    required=True,
+    show_default=True,
+    default=240,
+    help=(
+        "The maximum runtime duration for this Jupyter server instance, in minutes. If the server job is still running "
+        "at the end of this time limit, the job will be forcibly terminated by SLURM. Note, to prevent hogging the "
+        "server, make sure this parameter is always set to the smallest feasible period of time you intend to interact "
+        "with the server."
+    ),
+)
+@click.option(
+    "-p",
+    "--port",
+    type=int,
+    required=True,
+    show_default=True,
+    default=0,
+    help=(
+        "The port to use for the Jupyter server communication on the remote server. Valid port values are from 8888 "
+        "to 9999. Most runtimes should leave this set to the default value (0), which will randomly select one of the "
+        "valid ports. Using random selection minimizes the chances of colliding with other interactive jupyter "
+        "sessions."
+    ),
+)
+def start_jupyter_server(
+    credentials_path: Path, name: str, environment: str, directory: Path, cores: int, memory: int, time: int, port: int
+) -> None:
+    """Starts an interactive Jupyter session on the remote Sun lab server.
+
+    This command should be used to run Jupyter lab and notebooks sessions on the remote Sun lab server. Since all lab
+    data is stored on the server, this allows running light interactive analysis sessions on the same node as the data,
+    while leveraging considerable compute resources of the server.
+
+    Calling this command initializes a SLURM session that runs the interactive Jupyter server. Since this server
+    directly competes for resources with all other headless jobs running on the server, it is imperative that each
+    jupyter runtime uses only the minimum amount of resources and run-time as necessary. Do not use this command to run
+    heavy data processing pipelines! Instead, consult with library documentation and use the headless Job class.
+    """
+    # Initializes server connection
+    server = Server(credentials_path)
+    job: JupyterJob | None = None
+    try:
+        # If the caller did not provide an explicit notebook directory, defaults to user's working directory
+        if directory is None:
+            directory = (server.user_working_root,)
+
+        # Launches the specified Jupyter server
+        job = server.launch_jupyter_server(
+            job_name=name,
+            conda_environment=environment,
+            notebook_directory=directory,
+            cpus_to_use=cores,
+            ram_gb=memory,
+            port=port,
+            time_limit=time,
+        )
+
+        # Displays the server connection details to the user via terminal
+        job.print_connection_info()
+
+        # Blocks in-place until the user shuts down the server. This allows terminating the jupyter job early if the
+        # user is done working with the server
+        input("Enter anything to shut down the server: ")
+
+    # Ensures that the server created as part of this CLI is always terminated when the CLI terminates
+    finally:
+        # Terminates the server job
+        if job is not None:
+            server.abort_job(job)
+
+        # Closes server connection if it is still open
+        server.close()
+
+
+@click.command()
+@click.option(
+    "-sp",
+    "--session_path",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="The absolute path to the session directory for which to resolve the dataset integration readiness marker.",
+)
+@click.option(
+    "-c",
+    "--create_processed_directories",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Determines whether to create the processed data hierarchy. This flag should be disabled for most runtimes.",
+)
+@click.option(
+    "-ppp",
+    "--project_processed_path",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=False,
+    help=(
+        "The absolute path to the project directory where processed session data is stored, if different from the "
+        "directory used to store raw session data. Typically, this extra argument is only used when processing data "
+        "stored on remote compute server(s)."
+    ),
+)
+@click.option(
+    "-r",
+    "--remove",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help=(
+        "Determines whether the command should create or remove the dataset integration marker. Do not enable this "
+        "flag unless you know what you are doing. It is only safe to enable this flag if the session is not currently "
+        "being integrated into any datasets."
+    ),
+)
+def resolve_dataset_marker(
+    session_path: Path, create_processed_directories: bool, project_processed_path: Path | None, remove: bool
+) -> None:
+    """Depending on configuration, either creates or removes the p53.bin marker from the target session.
+
+    The p53.bin marker determines whether the session is ready for dataset integration. When the marker exists,
+    processing pipelines are not allowed to work with the session data, ensuring that all processed data remains
+    unchanged. If the marker does not exist, dataset integration pipelines are not allowed to work with the session
+    data, enabling processing pipelines to safely modify the data at any time.
+
+    This command is automatically called at the end of each processing runtime to automatically transfer processed
+    sessions to the dataset integration step by creating the p53.bin marker. In contrast, removing the marker can only
+    be done manually.
+    """
+    resolve_p53_marker(
+        session_path=session_path,
+        create_processed_data_directory=create_processed_directories,
+        processed_data_root=project_processed_path,
+        remove=remove,
+    )

sl_shared_assets/cli.pyi

@@ -2,16 +2,23 @@ from pathlib import Path
 
 from .tools import (
     ascend_tyche_data as ascend_tyche_data,
+    resolve_p53_marker as resolve_p53_marker,
     verify_session_checksum as verify_session_checksum,
     generate_project_manifest as generate_project_manifest,
 )
-from .server import generate_server_credentials as generate_server_credentials
+from .server import (
+    Server as Server,
+    JupyterJob as JupyterJob,
+    generate_server_credentials as generate_server_credentials,
+)
 from .data_classes import (
     SessionData as SessionData,
     ProcessingTracker as ProcessingTracker,
 )
 
-def verify_session_integrity(session_path: str, create_processed_directories: bool, processed_data_root: Path) -> None:
+def verify_session_integrity(
+    session_path: Path, create_processed_directories: bool, processed_data_root: Path | None
+) -> None:
     """Checks the integrity of the target session's raw data (contents of the raw_data directory).
 
     This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
@@ -26,7 +33,7 @@ def verify_session_integrity(session_path: str, create_processed_directories: bo
     """
 
 def generate_project_manifest_file(
-    project_path: str, output_directory: str, project_processed_path: str | None
+    project_path: Path, output_directory: Path, project_processed_path: Path | None
 ) -> None:
     """Generates the manifest .feather file that provides information about the data-processing state of all available
     project sessions.
@@ -36,7 +43,15 @@ def generate_project_manifest_file(
     between machines (as it is cached in a file).
     """
 
-def generate_server_credentials_file(output_directory: str, host: str, username: str, password: str) -> None:
+def generate_server_credentials_file(
+    output_directory: Path,
+    host: str,
+    username: str,
+    password: str,
+    storage_root: str,
+    working_root: str,
+    shared_directory_name: str,
+) -> None:
     """Generates a new server_credentials.yaml file under the specified directory, using input information.
 
     This command is used to set up access to compute servers and clusters on new machines (PCs). The data stored inside
@@ -44,7 +59,7 @@ def generate_server_credentials_file(output_directory: str, host: str, username:
     lab data processing libraries.
     """
 
-def ascend_tyche_directory(input_directory: str) -> None:
+def ascend_tyche_directory(input_directory: Path) -> None:
     """Restructures old Tyche project data to use the modern Sun lab data structure and uploads them to the processing
     server.
 
@@ -54,3 +69,33 @@ def ascend_tyche_directory(input_directory: str) -> None:
     will not work for any other project or data. Also, this command will only work on a machine (PC) that belongs to a
     valid Sun lab data acquisition system, such as VRPC of the Mesoscope-VR system.
     """
+
+def start_jupyter_server(
+    credentials_path: Path, name: str, environment: str, directory: Path, cores: int, memory: int, time: int, port: int
+) -> None:
+    """Starts an interactive Jupyter session on the remote Sun lab server.
+
+    This command should be used to run Jupyter lab and notebooks sessions on the remote Sun lab server. Since all lab
+    data is stored on the server, this allows running light interactive analysis sessions on the same node as the data,
+    while leveraging considerable compute resources of the server.
+
+    Calling this command initializes a SLURM session that runs the interactive Jupyter server. Since this server
+    directly competes for resources with all other headless jobs running on the server, it is imperative that each
+    jupyter runtime uses only the minimum amount of resources and run-time as necessary. Do not use this command to run
+    heavy data processing pipelines! Instead, consult with library documentation and use the headless Job class.
+    """
+
+def resolve_dataset_marker(
+    session_path: Path, create_processed_directories: bool, project_processed_path: Path | None, remove: bool
+) -> None:
+    """Depending on configuration, either creates or removes the p53.bin marker from the target session.
+
+    The p53.bin marker determines whether the session is ready for dataset integration. When the marker exists,
+    processing pipelines are not allowed to work with the session data, ensuring that all processed data remains
+    unchanged. If the marker does not exist, dataset integration pipelines are not allowed to work with the session
+    data, enabling processing pipelines to safely modify the data at any time.
+
+    This command is automatically called at the end of each processing runtime to automatically transfer processed
+    sessions to the dataset integration step by creating the p53.bin marker. In contrast, removing the marker can only
+    be done manually.
+    """

sl_shared_assets/data_classes/configuration_data.py

@@ -41,6 +41,21 @@ class ExperimentState:
     """The time, in seconds, to maintain the current combination of the experiment and system states."""
 
 
+@dataclass()
+class TrialCueSequence:
+    """Encapsulates information about the Virtual Reality (VR) wall cue sequence experienced by the animal as part of
+    the given trial.
+
+    All Virtual Reality environments can be broadly conceptualized as repeating motifs (sequences) of wall cues. Since
+    some experimental tasks can use multiple cue sequences as part of the same experiment session, multiple instances of
+    this class can be used to specify supported trial structures. The information stored in this class instance is used
+    during behavior data parsing to assign trial information to data collected from various sources.
+    """
+
+    cue_sequence: tuple[int, ...]
+    """Specifies the sequence of wall cues experienced by the animal while running this trial."""
+
+
 # noinspection PyArgumentList
 @dataclass()
 class MesoscopeExperimentConfiguration(YamlConfig):
@@ -74,6 +89,11 @@ class MesoscopeExperimentConfiguration(YamlConfig):
     )
     """A dictionary that uses human-readable state-names as keys and ExperimentState instances as values. Each 
     ExperimentState instance represents a phase of the experiment."""
+    trial_structures: dict[str, TrialCueSequence] = field(
+        default_factory=lambda: {"circular_4cue": TrialCueSequence(cue_sequence=(0, 1, 0, 2, 0, 3, 0, 4))}
+    )
+    """A dictionary that maps human-readable trial structure names as keys and TrialCueSequence instances as values. 
+    Each TrialCueSequence instance represents a specific VR wall cue sequence used by a given trial structure."""
 
 
 @dataclass()

sl_shared_assets/data_classes/configuration_data.pyi

@@ -24,6 +24,19 @@ class ExperimentState:
     system_state_code: int
     state_duration_s: float
 
+@dataclass()
+class TrialCueSequence:
+    """Encapsulates information about the Virtual Reality (VR) wall cue sequence experienced by the animal as part of
+    the given trial.
+
+    All Virtual Reality environments can be broadly conceptualized as repeating motifs (sequences) of wall cues. Since
+    some experimental tasks can use multiple cue sequences as part of the same experiment session, multiple instances of
+    this class can be used to specify supported trial structures. The information stored in this class instance is used
+    during behavior data parsing to assign trial information to data collected from various sources.
+    """
+
+    cue_sequence: tuple[int, ...]
+
 @dataclass()
 class MesoscopeExperimentConfiguration(YamlConfig):
     """Stores the configuration of a single experiment runtime that uses the Mesoscope_VR data acquisition system.
@@ -45,6 +58,7 @@ class MesoscopeExperimentConfiguration(YamlConfig):
 
     cue_map: dict[int, float] = field(default_factory=Incomplete)
     experiment_states: dict[str, ExperimentState] = field(default_factory=Incomplete)
+    trial_structures: dict[str, TrialCueSequence] = field(default_factory=Incomplete)
 
 @dataclass()
 class MesoscopePaths:

sl_shared_assets/data_classes/session_data.py

@@ -299,23 +299,21 @@ class ProcessedData:
     behavior_data_path: Path = Path()
     """Stores the path to the directory that contains the non-video and non-brain-activity data extracted from 
     .npz log files by our in-house log parsing pipeline."""
-    job_logs_path: Path = Path()
-    """Stores the path to the directory that stores the standard output and standard error data collected during 
-    server-side data processing pipeline runtimes. This directory is primarily used when running data processing jobs 
-    on the remote server. However, it is possible to configure local runtimes to also redirect log data to files 
-    stored in this directory (by editing ataraxis-base-utilities 'console' variable)."""
     suite2p_processing_tracker_path: Path = Path()
     """Stores the path to the suite2p_processing_tracker.yaml tracker file. This file stores the current state of the 
     sl-suite2p single-day data processing pipeline."""
-    dataset_formation_tracker_path: Path = Path()
-    """Same as suite2p_processing_tracker_path, but stores the current state of the dataset formation process that 
-    includes this session (communicates whether the session has been successfully added to any dataset(s))."""
     behavior_processing_tracker_path: Path = Path()
     """Stores the path to the behavior_processing_tracker.yaml file. This file stores the current state of the 
     behavior (log) data processing pipeline."""
     video_processing_tracker_path: Path = Path()
     """Stores the path to the video_processing_tracker.yaml file. This file stores the current state of the video 
     tracking (DeepLabCut) processing pipeline."""
+    p53_path: Path = Path()
+    """Stores the path to the p53.bin file. This file serves as a lock-in marker that determines whether the session is 
+    in the processing or dataset mode. Specifically, if the file does not exist, the session data cannot be integrated 
+    into any dataset, as it may be actively worked on by processing pipelines. Conversely, if the marker exists, 
+    processing pipelines are not allowed to work with the session, as it may be actively integrated into one or more 
+    datasets."""
 
     def resolve_paths(self, root_directory_path: Path) -> None:
         """Resolves all paths managed by the class instance based on the input root directory path.
@@ -333,11 +331,10 @@ class ProcessedData:
         self.camera_data_path = self.processed_data_path.joinpath("camera_data")
         self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
         self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
-        self.job_logs_path = self.processed_data_path.joinpath("job_logs")
         self.suite2p_processing_tracker_path = self.processed_data_path.joinpath("suite2p_processing_tracker.yaml")
-        self.dataset_formation_tracker_path = self.processed_data_path.joinpath("dataset_formation_tracker.yaml")
         self.behavior_processing_tracker_path = self.processed_data_path.joinpath("behavior_processing_tracker.yaml")
         self.video_processing_tracker_path = self.processed_data_path.joinpath("video_processing_tracker.yaml")
+        self.p53_path = self.processed_data_path.joinpath("p53.bin")
 
     def make_directories(self) -> None:
         """Ensures that all major subdirectories and the root directory exist, creating any missing directories."""
@@ -345,7 +342,6 @@ class ProcessedData:
         ensure_directory_exists(self.processed_data_path)
         ensure_directory_exists(self.camera_data_path)
         ensure_directory_exists(self.behavior_data_path)
-        ensure_directory_exists(self.job_logs_path)
 
 
 @dataclass

sl_shared_assets/data_classes/session_data.pyi

@@ -132,11 +132,10 @@ class ProcessedData:
     camera_data_path: Path = ...
     mesoscope_data_path: Path = ...
     behavior_data_path: Path = ...
-    job_logs_path: Path = ...
     suite2p_processing_tracker_path: Path = ...
-    dataset_formation_tracker_path: Path = ...
     behavior_processing_tracker_path: Path = ...
     video_processing_tracker_path: Path = ...
+    p53_path: Path = ...
     def resolve_paths(self, root_directory_path: Path) -> None:
         """Resolves all paths managed by the class instance based on the input root directory path.
 

sl_shared_assets/server/__init__.py

@@ -2,7 +2,7 @@
 and other compute servers. This package is also used across all Sun lab members private code to interface with the
 shared server."""
 
-from .job import Job
+from .job import Job, JupyterJob
 from .server import Server, ServerCredentials, generate_server_credentials
 
-__all__ = ["Server", "ServerCredentials", "generate_server_credentials", "Job"]
+__all__ = ["Server", "ServerCredentials", "generate_server_credentials", "Job", "JupyterJob"]