sl-shared-assets 4.0.0__py3-none-any.whl → 5.0.0__py3-none-any.whl

sl_shared_assets/tools/ascension_tools.py

@@ -1,265 +0,0 @@
-"""This module provides tools for translating ('ascending') old Tyche data to use the modern data structure used in the
-Sun lab. The tools from this module will not work for any other data and also assume that the Tyche data has been
-preprocessed with an early version of the Sun lab mesoscope processing pipeline. However, this module can be used as
-an example for how to convert other data formats to match use the Sun lab data structure."""
-
-from pathlib import Path
-import datetime
-
-import numpy as np
-from ataraxis_base_utilities import LogLevel, console
-from ataraxis_time.time_helpers import extract_timestamp_from_bytes
-
-from ..data_classes import SessionData, SessionTypes, get_system_configuration_data
-from .transfer_tools import transfer_directory
-from .packaging_tools import calculate_directory_checksum
-
-
-def _generate_session_name(acquisition_path: Path) -> str:
-    """Generates a session name using the last modification time of a zstack.mat or MotionEstimator.me file.
-
-    This worker function uses one of the motion estimation files stored in each Tyche 'acquisition' subfolder to
-    generate a modern Sun lab timestamp-based session name. This is used to translate the original Tyche session naming
-    pattern into the pattern used by all modern Sun lab projects and pipelines.
-
-    Args:
-        acquisition_path: The absolute path to the target acquisition folder. These folders are found under the 'day'
-            folders for each animal, e.g.: Tyche-A7/2022_01_03/1.
-
-    Returns:
-        The modernized session name.
-    """
-
-    # All well-formed sessions are expected to contain both the zstack.mat and the MotionEstimator.me file.
-    # We use the last modification time from one of these files to infer when the session was carried out. This allows
-    # us to gather the time information, which is missing from the original session naming pattern.
-    source: Path
-    if acquisition_path.joinpath("zstack.mat").exists():
-        source = acquisition_path.joinpath("zstack.mat")
-    elif acquisition_path.joinpath("MotionEstimator.me").exists():
-        source = acquisition_path.joinpath("MotionEstimator.me")
-    else:
-        message = (
-            f"Unable to find zstack.mat or MotionEstimator.me file in the target acquisition subfolder "
-            f"{acquisition_path} of the session {acquisition_path.parent}. Manual intervention is required to ascend "
-            f"the target session folder to the latest Sun lab data format."
-        )
-        console.error(message=message, error=FileNotFoundError)
-        raise FileNotFoundError(message)  # Fall-back to appease mypy
-
-    # Gets the last modified time (available on all platforms) and converts it to a UTC timestamp object.
-    mod_time = source.stat().st_mtime
-    mod_datetime = datetime.datetime.fromtimestamp(mod_time)
-
-    # Converts the timestamp to microseconds as uint64, then to an array of 8 uint8 bytes. The array is then reformatted
-    # to match the session name pattern used in the modern Sun lab data pipelines.
-    timestamp_microseconds = np.uint64(int(mod_datetime.timestamp() * 1_000_000))
-    timestamp_bytes = np.array([(timestamp_microseconds >> (8 * i)) & 0xFF for i in range(8)], dtype=np.uint8)
-    stamp = extract_timestamp_from_bytes(timestamp_bytes=timestamp_bytes)
-
-    # Returns the generated session name to the caller.
-    return stamp
-
-
-def _reorganize_data(session_data: SessionData, source_root: Path) -> bool:
-    """Reorganizes and moves the session's data from the source folder in the old Tyche data hierarchy to the raw_data
-    folder in the newly created modern hierarchy.
-
-    This worker function is used to physically rearrange the data from the original Tyche data structure to the
-    new data structure. It both moves the existing files to their new destinations and renames certain files to match
-    the modern naming convention used in the Sun lab.
-
-    Args:
-        session_data: The initialized SessionData instance managing the 'ascended' (modernized) session data hierarchy.
-        source_root: The absolute path to the old Tyche data hierarchy folder that stores session's data.
-
-    Returns:
-        True if the ascension process was successfully completed. False if the process encountered missing data or
-        otherwise did not go as expected. When the method returns False, the runtime function requests user intervention
-        to finalize the process manually.
-    """
-
-    # Resolves expected data targets:
-
-    # These files should be present in all well-formed session data folders. While not all session folders are
-    # well-formed, we will likely exclude any non-well-formed folders from processing.
-    zstack_path = source_root.joinpath("zstack.mat")
-    motion_estimator_path = source_root.joinpath("MotionEstimator.me")
-    ops_path = source_root.joinpath("ops.json")
-    mesoscope_frames_path = source_root.joinpath("mesoscope_frames")
-    ax_checksum_path = source_root.joinpath("ax_checksum.txt")
-
-    # These two file types are present for some, but not all folders. They are not as important as the files mentioned
-    # above, though, as, currently, the data stored in these files is not used during processing.
-    frame_metadata_path = source_root.joinpath("frame_metadata.npz")
-    metadata_path = source_root.joinpath("metadata.json")
-
-    # This tracker is used to mark the session for manual intervention if any expected data is missing from the source
-    # session folder. At the end of this function's runtime, it determines whether the function returns True or False
-    data_missing = False
-
-    # First, moves the mesoscope TIFF stacks to the newly created session data hierarchy as mesoscope_data subfolder
-    if mesoscope_frames_path.exists():
-        mesoscope_frames_path.rename(session_data.raw_data.mesoscope_data_path)
-    else:
-        data_missing = True
-
-    # Then, moves 'loose' mesoscope-related data files to the mesoscope_data folder.
-    if zstack_path.exists():
-        zstack_path.rename(Path(session_data.raw_data.mesoscope_data_path).joinpath("zstack.mat"))
-    else:
-        data_missing = True
-
-    if motion_estimator_path.exists():
-        motion_estimator_path.rename(Path(session_data.raw_data.mesoscope_data_path).joinpath("MotionEstimator.me"))
-    else:
-        data_missing = True
-
-    if ops_path.exists():
-        ops_path.rename(Path(session_data.raw_data.mesoscope_data_path).joinpath("ops.json"))
-    else:
-        data_missing = True
-
-    # If variant and invariant metadata files exist, also moves them to the mesoscope data folder and renames the
-    # files to use the latest naming convention. Missing any of these files is not considered a user-intervention-worthy
-    # situation.
-    if frame_metadata_path.exists():
-        frame_metadata_path.rename(
-            Path(session_data.raw_data.mesoscope_data_path).joinpath("frame_variant_metadata.npz")
-        )
-    if metadata_path.exists():
-        metadata_path.rename(Path(session_data.raw_data.mesoscope_data_path).joinpath("frame_invariant_metadata.json"))
-
-    # Loops over all camera video files (using the .avi extension) and moves them to the camera_data folder.
-    videos_found = 0
-    for video in source_root.glob("*.avi"):
-        videos_found += 1
-        video.rename(Path(session_data.raw_data.camera_data_path).joinpath(video.name))
-    if videos_found == 0:
-        data_missing = True
-
-    # Loops over all behavior log files (old GIMBL format) and moves them to the behavior_data folder.
-    logs_found = 0
-    for log in source_root.glob("Log Tyche-* ????-??-?? session *.json"):
-        logs_found += 1
-        log.rename(Path(session_data.raw_data.behavior_data_path).joinpath(log.name))
-    if logs_found == 0:
-        data_missing = True
-
-    # Removes the checksum file if it exists. Due to file name and location changes, the session data folder has to
-    # be re-checksummed after the reorganization anyway, so there is no need to keep the original file.
-    ax_checksum_path.unlink(missing_ok=True)
-
-    # Loops over all remaining contents of the directory.
-    for path in source_root.glob("*"):
-        # At this point, there should be no more subfolders left inside the root directory. If there are more
-        # subfolders, this case requires user intervention
-        if path.is_dir():
-            data_missing = True
-
-        # All non-subfolder files are moved to the root raw_data directory of the newly created session.
-        else:
-            path.rename(Path(session_data.raw_data.raw_data_path).joinpath(path.name))
-
-    # Session data has been fully reorganized. Depending on whether there was any missing data during processing,
-    # returns the boolean flag for whether user intervention is required
-    if data_missing:
-        return False
-    else:
-        return True
-
-
-def ascend_tyche_data(root_directory: Path) -> None:
-    """Reformats the old Tyche data to use the modern Sun lab layout and metadata files.
-
-    This function is used to convert old Tyche data to the modern data management standard. This is used to make the
-    data compatible with the modern Sun lab data workflows.
-
-    Notes:
-        This function is statically written to work with the raw Tyche dataset featured in the OSM manuscript:
-        https://www.nature.com/articles/s41586-024-08548-w. Additionally, it assumes that the dataset has been
-        preprocessed with the early Sun lab mesoscope compression pipeline. The function will not work for any other
-        project or data hierarchy.
-
-        As part of its runtime, the function automatically transfers the ascended session data to the BioHPC server.
-        Since transferring the data over the network is the bottleneck of this pipeline, it runs in a single-threaded
-        mode and is constrained by the communication channel between the local machine and the BioHPC server. Calling
-        this function for a large number of sessions will result in a long processing time due to the network data
-        transfer.
-
-        Since SessionData can only be created on a PC that has a valid acquisition system config, this function will
-        only work on a machine that is part of an active Sun lab acquisition system.
-
-    Args:
-        root_directory: The directory that stores one or more Tyche animal folders. This can be conceptualized as the
-            root directory for the Tyche project.
-    """
-    # The acquisition system config resolves most paths and filesystem configuration arguments
-    acquisition_system = get_system_configuration_data()
-    server_root_directory = acquisition_system.paths.server_storage_directory
-
-    # Statically defines project name and local root paths
-    project_name = "Tyche"
-
-    # Assumes that the root directory stores all animal folders to be processed
-    for animal_folder in root_directory.iterdir():
-        # Each animal folder is named to include a project name and a static animal ID, e.g.: Tyche-A7. This extracts
-        # each animal ID.
-        animal_name = animal_folder.stem.split(sep="-")[1]
-
-        # Under each animal root folder, there are day folders that use YYYY-MM-DD timestamps
-        for session_folder in animal_folder.iterdir():
-            # Inside each day folder, there are one or more acquisitions (sessions)
-            for acquisition_folder in session_folder.iterdir():
-                # For each session, we extract the modification time from either (preferentially) zstack.mat or
-                # MotionEstimator.me file. Any session without these files is flagged for additional user intervention.
-                # This procedure generates timestamp-based session names, analogous to how our modern pipeline does it.
-                session_name = _generate_session_name(acquisition_path=acquisition_folder)
-
-                # Uses derived session name and the derived project name to create the session data hierarchy using the
-                # output root. This generates a 'standard' Sun lab directory structure for the Tyche data.
-                session_data = SessionData.create(
-                    project_name=project_name,
-                    session_name=session_name,
-                    animal_id=animal_name,
-                    session_type=SessionTypes.MESOSCOPE_EXPERIMENT,
-                    experiment_name=None,
-                )
-
-                # Since this runtime reprocesses already acquired data, marks the session as fully initialized.
-                session_data.runtime_initialized()
-
-                # Moves the data from the old hierarchy to the new hierarchy. If the process runs as expected, and
-                # fully empties the source acquisition folder, it destroys the folder. Otherwise, notifies the user that
-                # the runtime did not fully process the session data and requests intervention.
-                success = _reorganize_data(session_data, acquisition_folder)
-                if not success:
-                    message = (
-                        f"Encountered issues when reorganizing {animal_name} session {session_name}. "
-                        f"User intervention is required to finish data reorganization process for this session."
-                    )
-                    # noinspection PyTypeChecker
-                    console.echo(message=message, level=LogLevel.WARNING)
-                else:
-                    # Generates the telomere.bin file to mark the session as 'complete'
-                    session_data.raw_data.telomere_path.touch()
-
-                    # If the local transfer process was successful, generates a new checksum for the moved data
-                    calculate_directory_checksum(directory=Path(session_data.raw_data.raw_data_path))
-
-                    # Next, copies the data to the BioHPC server for further processing
-                    transfer_directory(
-                        source=Path(session_data.raw_data.raw_data_path),
-                        destination=Path(
-                            server_root_directory.joinpath(project_name, animal_name, session_name, "raw_data")
-                        ),
-                        verify_integrity=False,
-                    )
-
-                    # Removes the now-empty old session data directory.
-                    acquisition_folder.rmdir()
-
-            # If the loop above removed all acquisition folders, all data for that day has been successfully converted
-            # to use the new session format. Removes the now-empty 'day' folder from the target animal
-            if len([folder for folder in session_folder.iterdir()]) == 0:
-                session_folder.rmdir()

sl_shared_assets/tools/ascension_tools.pyi

@@ -1,68 +0,0 @@
-from pathlib import Path
-
-from ..data_classes import (
-    SessionData as SessionData,
-    SessionTypes as SessionTypes,
-    get_system_configuration_data as get_system_configuration_data,
-)
-from .transfer_tools import transfer_directory as transfer_directory
-from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
-
-def _generate_session_name(acquisition_path: Path) -> str:
-    """Generates a session name using the last modification time of a zstack.mat or MotionEstimator.me file.
-
-    This worker function uses one of the motion estimation files stored in each Tyche 'acquisition' subfolder to
-    generate a modern Sun lab timestamp-based session name. This is used to translate the original Tyche session naming
-    pattern into the pattern used by all modern Sun lab projects and pipelines.
-
-    Args:
-        acquisition_path: The absolute path to the target acquisition folder. These folders are found under the 'day'
-            folders for each animal, e.g.: Tyche-A7/2022_01_03/1.
-
-    Returns:
-        The modernized session name.
-    """
-
-def _reorganize_data(session_data: SessionData, source_root: Path) -> bool:
-    """Reorganizes and moves the session's data from the source folder in the old Tyche data hierarchy to the raw_data
-    folder in the newly created modern hierarchy.
-
-    This worker function is used to physically rearrange the data from the original Tyche data structure to the
-    new data structure. It both moves the existing files to their new destinations and renames certain files to match
-    the modern naming convention used in the Sun lab.
-
-    Args:
-        session_data: The initialized SessionData instance managing the 'ascended' (modernized) session data hierarchy.
-        source_root: The absolute path to the old Tyche data hierarchy folder that stores session's data.
-
-    Returns:
-        True if the ascension process was successfully completed. False if the process encountered missing data or
-        otherwise did not go as expected. When the method returns False, the runtime function requests user intervention
-        to finalize the process manually.
-    """
-
-def ascend_tyche_data(root_directory: Path) -> None:
-    """Reformats the old Tyche data to use the modern Sun lab layout and metadata files.
-
-    This function is used to convert old Tyche data to the modern data management standard. This is used to make the
-    data compatible with the modern Sun lab data workflows.
-
-    Notes:
-        This function is statically written to work with the raw Tyche dataset featured in the OSM manuscript:
-        https://www.nature.com/articles/s41586-024-08548-w. Additionally, it assumes that the dataset has been
-        preprocessed with the early Sun lab mesoscope compression pipeline. The function will not work for any other
-        project or data hierarchy.
-
-        As part of its runtime, the function automatically transfers the ascended session data to the BioHPC server.
-        Since transferring the data over the network is the bottleneck of this pipeline, it runs in a single-threaded
-        mode and is constrained by the communication channel between the local machine and the BioHPC server. Calling
-        this function for a large number of sessions will result in a long processing time due to the network data
-        transfer.
-
-        Since SessionData can only be created on a PC that has a valid acquisition system config, this function will
-        only work on a machine that is part of an active Sun lab acquisition system.
-
-    Args:
-        root_directory: The directory that stores one or more Tyche animal folders. This can be conceptualized as the
-            root directory for the Tyche project.
-    """

sl_shared_assets/tools/packaging_tools.pyi

@@ -1,58 +0,0 @@
-from pathlib import Path
-
-from _typeshed import Incomplete
-
-from ..data_classes import TrackerFileNames as TrackerFileNames
-
-_excluded_files: Incomplete
-
-def _calculate_file_checksum(base_directory: Path, file_path: Path) -> tuple[str, bytes]:
-    """Calculates xxHash3-128 checksum for a single file and its path relative to the base directory.
-
-    This function is passed to parallel workers used by the calculate_directory_hash() method that iteratively
-    calculates the checksum for all files inside a directory. Each call to this function returns the checksum for the
-    target file, which includes both the contents of the file and its path relative to the base directory.
-
-    Args:
-        base_directory: The path to the base (root) directory which is being checksummed by the main
-            'calculate_directory_checksum' function.
-        file_path: The absolute path to the target file.
-
-    Returns:
-        A tuple with two elements. The first element is the path to the file relative to the base directory. The second
-        element is the xxHash3-128 checksum that covers the relative path and the contents of the file.
-    """
-
-def calculate_directory_checksum(
-    directory: Path, num_processes: int | None = None, batch: bool = False, save_checksum: bool = True
-) -> str:
-    """Calculates xxHash3-128 checksum for the input directory, which includes the data of all contained files and
-    the directory structure information.
-
-    This function is used to generate a checksum for the raw_data directory of each experiment or training session.
-    Checksums are used to verify the session data integrity during transmission between the PC that acquired the data
-    and long-term storage locations, such as the Synology NAS or the BioHPC server. The function can be configured to
-    write the generated checksum as a hexadecimal string to the ax_checksum.txt file stored at the highest level of the
-    input directory.
-
-    Note:
-        This method uses multiprocessing to efficiently parallelize checksum calculation for multiple files. In
-        combination with xxHash3, this achieves a significant speedup over more common checksums, such as MD5 and
-        SHA256. Note that xxHash3 is not suitable for security purposes and is only used to ensure data integrity.
-
-        The method notifies the user about the checksum calculation process via the terminal.
-
-        The returned checksum accounts for both the contents of each file and the layout of the input directory
-        structure.
-
-    Args:
-        directory: The Path to the directory to be checksummed.
-        num_processes: The number of CPU processes to use for parallelizing checksum calculation. If set to None, the
-            function defaults to using (logical CPU count - 4).
-        batch: Determines whether the function is called as part of batch-processing multiple directories. This is used
-            to optimize progress reporting to avoid cluttering the terminal.
-        save_checksum: Determines whether the checksum should be saved (written to) a .txt file.
-
-    Returns:
-        The xxHash3-128 checksum for the input directory as a hexadecimal string.
-    """

sl_shared_assets/tools/project_management_tools.pyi

@@ -1,239 +0,0 @@
-from pathlib import Path
-
-import polars as pl
-
-from ..data_classes import (
-    SessionData as SessionData,
-    SessionTypes as SessionTypes,
-    TrackerFileNames as TrackerFileNames,
-    RunTrainingDescriptor as RunTrainingDescriptor,
-    LickTrainingDescriptor as LickTrainingDescriptor,
-    WindowCheckingDescriptor as WindowCheckingDescriptor,
-    MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
-    get_processing_tracker as get_processing_tracker,
-)
-from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
-
-class ProjectManifest:
-    """Wraps the contents of a Sun lab project manifest .feather file and exposes methods for visualizing and
-    working with the data stored inside the file.
-
-    This class functions as a high-level API for working with Sun lab projects. It is used both to visualize the
-    current state of various projects and during automated data processing to determine which processing steps to
-    apply to different sessions.
-
-    Args:
-        manifest_file: The path to the .feather manifest file that stores the target project's state data.
-
-    Attributes:
-        _data: Stores the manifest data as a Polars DataFrame.
-        _animal_string: Determines whether animal IDs are stored as strings or unsigned integers.
-    """
-
-    _data: pl.DataFrame
-    _animal_string: bool
-    def __init__(self, manifest_file: Path) -> None: ...
-    def print_data(self) -> None:
-        """Prints the entire contents of the manifest file to the terminal."""
-    def print_summary(self, animal: str | int | None = None) -> None:
-        """Prints a summary view of the manifest file to the terminal, excluding the 'experimenter notes' data for
-        each session.
-
-        This data view is optimized for tracking which processing steps have been applied to each session inside the
-        project.
-
-        Args:
-            animal: The ID of the animal for which to display the data. If an ID is provided, this method will only
-                display the data for that animal. Otherwise, it will display the data for all animals.
-        """
-    def print_notes(self, animal: str | int | None = None) -> None:
-        """Prints only animal, session, and notes data from the manifest file.
-
-        This data view is optimized for experimenters to check what sessions have been recorded for each animal in the
-        project and refresh their memory on the outcomes of each session using experimenter notes.
-
-        Args:
-            animal: The ID of the animal for which to display the data. If an ID is provided, this method will only
-                display the data for that animal. Otherwise, it will display the data for all animals.
-        """
-    @property
-    def animals(self) -> tuple[str, ...]:
-        """Returns all unique animal IDs stored inside the manifest file.
-
-        This provides a tuple of all animal IDs participating in the target project.
-        """
-    def _get_filtered_sessions(
-        self,
-        animal: str | int | None = None,
-        exclude_incomplete: bool = True,
-        dataset_ready_only: bool = False,
-        not_dataset_ready_only: bool = False,
-    ) -> tuple[str, ...]:
-        """This worker method is used to get a list of sessions with optional filtering.
-
-        User-facing methods call this worker under-the-hood to fetch the filtered tuple of sessions.
-
-        Args:
-            animal: An optional animal ID to filter the sessions. If set to None, the method returns sessions for all
-                animals.
-            exclude_incomplete: Determines whether to exclude sessions not marked as 'complete' from the output
-                list.
-            dataset_ready_only: Determines whether to exclude sessions not marked as 'dataset' integration ready from
-                the output list. Enabling this option only shows sessions that can be integrated into a dataset.
-            not_dataset_ready_only: The opposite of 'dataset_ready_only'. Determines whether to exclude sessions marked
-                as 'dataset' integration ready from the output list. Note, when both this and 'dataset_ready_only' are
-                enabled, the 'dataset_ready_only' option takes precedence.
-
-        Returns:
-            The tuple of session IDs matching the filter criteria.
-
-        Raises:
-            ValueError: If the specified animal is not found in the manifest file.
-        """
-    @property
-    def sessions(self) -> tuple[str, ...]:
-        """Returns all session IDs stored inside the manifest file.
-
-        This property provides a tuple of all sessions, independent of the participating animal, that were recorded as
-        part of the target project. Use the get_sessions() method to get the list of session tuples with filtering.
-        """
-    def get_sessions(
-        self,
-        animal: str | int | None = None,
-        exclude_incomplete: bool = True,
-        dataset_ready_only: bool = False,
-        not_dataset_ready_only: bool = False,
-    ) -> tuple[str, ...]:
-        """Returns requested session IDs based on selected filtering criteria.
-
-        This method provides a tuple of sessions based on the specified filters. If no animal is specified, returns
-        sessions for all animals in the project.
-
-        Args:
-            animal: An optional animal ID to filter the sessions. If set to None, the method returns sessions for all
-                animals.
-            exclude_incomplete: Determines whether to exclude sessions not marked as 'complete' from the output
-                list.
-            dataset_ready_only: Determines whether to exclude sessions not marked as 'dataset' integration ready from
-                the output list. Enabling this option only shows sessions that can be integrated into a dataset.
-            not_dataset_ready_only: The opposite of 'dataset_ready_only'. Determines whether to exclude sessions marked
-                as 'dataset' integration ready from the output list. Note, when both this and 'dataset_ready_only' are
-                enabled, the 'dataset_ready_only' option takes precedence.
-
-        Returns:
-            The tuple of session IDs matching the filter criteria.
-
-        Raises:
-            ValueError: If the specified animal is not found in the manifest file.
-        """
-    def get_session_info(self, session: str) -> pl.DataFrame:
-        """Returns a Polars DataFrame that stores detailed information for the specified session.
-
-        Since session IDs are unique, it is expected that filtering by session ID is enough to get the requested
-        information.
-
-        Args:
-            session: The ID of the session for which to retrieve the data.
-
-        Returns:
-            A Polars DataFrame with the following columns: 'animal', 'date', 'notes', 'session', 'type', 'complete',
-            'intensity_verification', 'suite2p', 'behavior', 'video', 'dataset'.
-        """
-
-def generate_project_manifest(
-    raw_project_directory: Path, output_directory: Path, processed_data_root: Path | None = None
-) -> None:
-    """Builds and saves the project manifest .feather file under the specified output directory.
-
-    This function evaluates the input project directory and builds the 'manifest' file for the project. The file
-    includes the descriptive information about every session stored inside the input project folder and the state of
-    the session's data processing (which processing pipelines have been applied to each session). The file will be
-    created under the 'output_path' directory and use the following name pattern: ProjectName_manifest.feather.
-
-    Notes:
-        The manifest file is primarily used to capture and move project state information between machines, typically
-        in the context of working with data stored on a remote compute server or cluster. However, it can also be used
-        on a local machine, since an up-to-date manifest file is required to run most data processing pipelines in the
-        lab regardless of the runtime context.
-
-    Args:
-        raw_project_directory: The path to the root project directory used to store raw session data.
-        output_directory: The path to the directory where to save the generated manifest file.
-        processed_data_root: The path to the root directory (volume) used to store processed data for all Sun lab
-            projects if it is different from the parent of the 'raw_project_directory'. Typically, this would be the
-            case on remote compute server(s) and not on local machines.
-    """
-
-def verify_session_checksum(
-    session_path: Path,
-    manager_id: int,
-    create_processed_data_directory: bool = True,
-    processed_data_root: None | Path = None,
-    update_manifest: bool = False,
-) -> None:
-    """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
-    comparing it against the checksum stored in the ax_checksum.txt file.
-
-    Primarily, this function is used to verify data integrity after transferring it from a local PC to the remote
-    server for long-term storage. This function is designed to create the 'verified.bin' marker file if the checksum
-    matches and to remove the 'telomere.bin' and 'verified.bin' marker files if it does not.
-
-    Notes:
-        Removing the telomere.bin marker file from the session's raw_data folder marks the session as incomplete,
-        excluding it from all further automatic processing.
-
-        This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
-        part of the data preprocessing runtime performed by a data acquisition system.
-
-        Since version 3.1.0, this functon also supports (re) generating the processed session's project manifest file,
-        which is used to support further Sun lab data processing pipelines.
-
-    Args:
-        session_path: The path to the session directory to be verified. Note, the input session directory must contain
-            the 'raw_data' subdirectory.
-        manager_id: The xxHash-64 hash-value that specifies the unique identifier of the manager process that
-            manages the integrity verification runtime.
-        create_processed_data_directory: Determines whether to create the processed data hierarchy during runtime.
-        processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
-            the root directory where to store the processed data from all projects, and it will be automatically
-            modified to include the project name, the animal name, and the session ID.
-        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
-            project. This should always be enabled when working with remote compute server(s) to ensure that the
-            project manifest file contains the most actual snapshot of the project's state.
-    """
-
-def resolve_p53_marker(
-    session_path: Path,
-    create_processed_data_directory: bool = True,
-    processed_data_root: None | Path = None,
-    remove: bool = False,
-    update_manifest: bool = False,
-) -> None:
-    """Depending on configuration, either creates or removes the p53.bin marker file for the target session.
-
-    The marker file statically determines whether the session can be targeted by data processing or dataset formation
-    pipelines.
-
-    Notes:
-        Since dataset integration relies on data processing outputs, it is essential to prevent processing pipelines
-        from altering the data while it is integrated into a dataset. The p53.bin marker solves this issue by ensuring
-        that only one type of runtimes (processing or dataset integration) is allowed to work with the session.
-
-        For the p53.bin marker to be created, the session must not be undergoing processing. For the p53 marker
-        to be removed, the session must not be undergoing dataset integration.
-
-        Since version 3.1.0, this functon also supports (re)generating the processed session's project manifest file,
-        which is used to support further Sun lab data processing pipelines.
-
-    Args:
-        session_path: The path to the session directory for which the p53.bin marker needs to be resolved. Note, the
-            input session directory must contain the 'raw_data' subdirectory.
-        create_processed_data_directory: Determines whether to create the processed data hierarchy during runtime.
-        processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
-            the root directory where to store the processed data from all projects, and it will be automatically
-            modified to include the project name, the animal name, and the session ID.
-        remove: Determines whether this function is called to create or remove the p53.bin marker.
-        update_manifest: Determines whether to update (regenerate) the project manifest file for the processed session's
-            project. This should always be enabled when working with remote compute server(s) to ensure that the
-            project manifest file contains the most actual snapshot of the project's state.
-    """