sl-shared-assets 1.0.0__py3-none-any.whl

sl_shared_assets/tools/project_management_tools.py

@@ -0,0 +1,201 @@
+"""This module provides tools for managing the data of any Sun lab project. Tools from this module extend the
+functionality of SessionData class via a convenient API that allows working with the data of multiple sessions making
+up a given project."""
+
+from pathlib import Path
+
+import polars as pl
+from ataraxis_base_utilities import console
+
+from ..data_classes import SessionData
+from .packaging_tools import calculate_directory_checksum
+
+
+def generate_project_manifest(
+    raw_project_directory: Path, output_directory: Path, processed_project_directory: Path | None = None
+) -> None:
+    """Builds and saves the project manifest .feather file under the specified output directory.
+
+    This function evaluates the input project directory and builds the 'manifest' file for the project. The file
+    includes the descriptive information about every session stored inside the input project folder and the state of
+    session's data processing (which processing pipelines have been applied to each session). The file will be created
+    under the 'output_path' directory and use the following name pattern: {ProjectName}}_manifest.feather.
+
+    Notes:
+        The manifest file is primarily used to capture and move project state information between machines, typically
+        in the context of working with data stored on a remote compute server or cluster. However, it can also be used
+        on a local machine, since an up-to-date manifest file is required to run most data processing pipelines in the
+        lab regardless of the runtime context.
+
+    Args:
+        raw_project_directory: The path to the root project directory used to store raw session data.
+        output_directory: The path to the directory where to save the generated manifest file.
+        processed_project_directory: The path to the root project directory used to store processed session data if it
+            is different from the 'raw_project_directory'. Typically, this would be the case on remote compute server(s)
+            and not on local machines.
+    """
+
+    if not raw_project_directory.exists():
+        message = (
+            f"Unable to generate the project manifest file for the requested project {raw_project_directory.stem}. The "
+            f"specified project directory does not exist."
+        )
+        console.error(message=message, error=FileNotFoundError)
+
+    # Finds all raw data directories
+    session_directories = [directory.parent for directory in raw_project_directory.rglob("raw_data")]
+
+    if len(session_directories) == 0:
+        message = (
+            f"Unable to generate the project manifest file for the requested project {raw_project_directory.stem}. The "
+            f"project does not contain any raw session data. To generate the manifest file, the project must contain "
+            f"at least one valid experiment or training session."
+        )
+        console.error(message=message, error=FileNotFoundError)
+
+    # Precreates the 'manifest' dictionary structure
+    manifest: dict[str, list[str | bool]] = {
+        "animal": [],  # Animal IDs.
+        "session": [],  # Session names.
+        "type": [],  # Type of the session (e.g., Experiment, Training, etc.).
+        "raw_data": [],  # Server-side raw_data folder path.
+        "processed_data": [],  # Server-side processed_data folder path.
+        "complete": [],  # Determines if the session data is complete. Incomplete sessions are excluded from processing.
+        "verified": [],  # Determines if the session data integrity has been verified upon transfer to storage machine.
+        "single_day_suite2p": [],  # Determines whether the session has been processed with the single-day s2p pipeline.
+        "multi_day_suite2p": [],  # Determines whether the session has been processed with the multi-day s2p pipeline.
+        "behavior": [],  # Determines whether the session has been processed with the behavior extraction pipeline.
+        "dlc": [],  # Determines whether the session has been processed with the DeepLabCut pipeline.
+    }
+
+    # Loops over each session of every animal in the project and extracts session ID information and information
+    # about which processing steps have been successfully applied to the session.
+    for directory in session_directories:
+        # Instantiates the SessionData instance to resolve the paths to all session's data files and locations.
+        session_data = SessionData.load(
+            session_path=directory, processed_data_root=processed_project_directory, make_processed_data_directory=False
+        )
+
+        # Fills the manifest dictionary with data for the processed session:
+
+        # Extracts ID and data path information from the SessionData instance
+        manifest["animal"].append(session_data.animal_id)
+        manifest["session"].append(session_data.session_name)
+        manifest["type"].append(session_data.session_type)
+        manifest["raw_data"].append(str(session_data.raw_data.raw_data_path))
+        manifest["processed_data"].append(str(session_data.processed_data.processed_data_path))
+
+        # If the session raw_data folder contains the telomere.bin file, marks the session as complete.
+        manifest["complete"].append(session_data.raw_data.telomere_path.exists())
+
+        # If the session raw_data folder contains the verified.bin file, marks the session as verified.
+        manifest["verified"].append(session_data.raw_data.verified_bin_path.exists())
+
+        # If the session is incomplete or unverified, marks all processing steps as FALSE, as automatic processing is
+        # disabled for incomplete sessions. If the session unverified, the case is even more severe, as its data may be
+        # corrupted.
+        if not manifest["complete"][-1] or not not manifest["verified"][-1]:
+            manifest["single_day_suite2p"].append(False)
+            manifest["multi_day_suite2p"].append(False)
+            manifest["behavior"].append(False)
+            manifest["dlc"].append(False)
+            continue  # Cycles to the next session
+
+        # If the session processed_data folder contains the single-day suite2p.bin file, marks the single-day suite2p
+        # processing step as complete.
+        manifest["single_day_suite2p"].append(session_data.processed_data.single_day_suite2p_bin_path.exists())
+
+        # If the session processed_data folder contains the multi-day suite2p.bin file, marks the multi-day suite2p
+        # processing step as complete.
+        manifest["multi_day_suite2p"].append(session_data.processed_data.multi_day_suite2p_bin_path.exists())
+
+        # If the session processed_data folder contains the behavior.bin file, marks the behavior processing step as
+        # complete.
+        manifest["behavior"].append(session_data.processed_data.behavior_data_path.exists())
+
+        # If the session processed_data folder contains the dlc.bin file, marks the dlc processing step as
+        # complete.
+        manifest["dlc"].append(session_data.processed_data.dlc_bin_path.exists())
+
+    # Converts the manifest dictionary to a Polars Dataframe
+    schema = {
+        "animal": pl.String,
+        "session": pl.String,
+        "raw_data": pl.String,
+        "processed_data": pl.String,
+        "type": pl.String,
+        "complete": pl.Boolean,
+        "verified": pl.Boolean,
+        "single_day_suite2p": pl.Boolean,
+        "multi_day_suite2p": pl.Boolean,
+        "behavior": pl.Boolean,
+        "dlc": pl.Boolean,
+    }
+    df = pl.DataFrame(manifest, schema=schema)
+
+    # Sorts the DataFrame by animal and then session. Since we assign animal IDs sequentially and 'name' sessions based
+    # on acquisition timestamps, the sort order is chronological.
+    sorted_df = df.sort(["animal", "session"])
+
+    # Saves the generated manifest to the project-specific manifest .feather file for further processing.
+    sorted_df.write_ipc(
+        file=output_directory.joinpath(f"{raw_project_directory.stem}_manifest.feather"), compression="lz4"
+    )
+
+
+def verify_session_checksum(
+    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+) -> None:
+    """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
+    comparing it against the checksum stored in the ax_checksum.txt file.
+
+    Primarily, this function is used to verify data integrity after transferring it from a local PC to the remote
+    server for long-term storage. This function is designed to create the 'verified.bin' marker file if the checksum
+    matches and to remove the 'telomere.bin' and 'verified.bin' marker files if it does not.
+
+    Notes:
+        Removing the telomere.bin marker file from session's raw_data folder marks the session as incomplete, excluding
+        it from all further automatic processing.
+
+        This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
+        part of the data preprocessing runtime performed by a data acquisition system.
+
+    Args:
+        session_path: The path to the session directory to be verified. Note, the input session directory must contain
+            the 'raw_data' subdirectory.
+        create_processed_data_directory: Determines whether to create the processed data hierarchy during runtime.
+        processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
+            the root directory where to store the processed data from all projects, and it will be automatically
+            modified to include the project name, the animal name, and the session ID.
+    """
+
+    # Loads session data layout. If configured to do so, also creates the processed data hierarchy
+    session_data = SessionData.load(
+        session_path=session_path,
+        processed_data_root=processed_data_root,
+        make_processed_data_directory=create_processed_data_directory,
+    )
+
+    # Unlinks the verified.bin marker if it exists. The presence or absence of the marker is used as the
+    # primary heuristic for determining if the session data passed verification. Unlinking it early helps in the case
+    # the verification procedure aborts unexpectedly for any reason.
+    session_data.raw_data.verified_bin_path.unlink(missing_ok=True)
+
+    # Re-calculates the checksum for the raw_data directory
+    calculated_checksum = calculate_directory_checksum(
+        directory=session_data.raw_data.raw_data_path, batch=False, save_checksum=False
+    )
+
+    # Loads the checksum stored inside the ax_checksum.txt file
+    with open(session_data.raw_data.checksum_path, "r") as f:
+        stored_checksum = f.read().strip()
+
+    # If the two checksums do not match, this likely indicates data corruption.
+    if stored_checksum != calculated_checksum:
+        # If the telomere.bin file exists, removes this file. This automatically marks the session as incomplete for
+        # all other Sun lab runtimes.
+        session_data.raw_data.telomere_path.unlink(missing_ok=True)
+
+    # Otherwise, ensures that the session is marked with the verified.bin marker file.
+    else:
+        session_data.raw_data.verified_bin_path.touch(exist_ok=True)

sl_shared_assets/tools/project_management_tools.pyi

@@ -0,0 +1,54 @@
+from pathlib import Path
+
+from ..data_classes import SessionData as SessionData
+from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
+
+def generate_project_manifest(
+    raw_project_directory: Path, output_directory: Path, processed_project_directory: Path | None = None
+) -> None:
+    """Builds and saves the project manifest .feather file under the specified output directory.
+
+    This function evaluates the input project directory and builds the 'manifest' file for the project. The file
+    includes the descriptive information about every session stored inside the input project folder and the state of
+    session's data processing (which processing pipelines have been applied to each session). The file will be created
+    under the 'output_path' directory and use the following name pattern: {ProjectName}}_manifest.feather.
+
+    Notes:
+        The manifest file is primarily used to capture and move project state information between machines, typically
+        in the context of working with data stored on a remote compute server or cluster. However, it can also be used
+        on a local machine, since an up-to-date manifest file is required to run most data processing pipelines in the
+        lab regardless of the runtime context.
+
+    Args:
+        raw_project_directory: The path to the root project directory used to store raw session data.
+        output_directory: The path to the directory where to save the generated manifest file.
+        processed_project_directory: The path to the root project directory used to store processed session data if it
+            is different from the 'raw_project_directory'. Typically, this would be the case on remote compute server(s)
+            and not on local machines.
+    """
+
+def verify_session_checksum(
+    session_path: Path, create_processed_data_directory: bool = True, processed_data_root: None | Path = None
+) -> None:
+    """Verifies the integrity of the session's raw data by generating the checksum of the raw_data directory and
+    comparing it against the checksum stored in the ax_checksum.txt file.
+
+    Primarily, this function is used to verify data integrity after transferring it from a local PC to the remote
+    server for long-term storage. This function is designed to create the 'verified.bin' marker file if the checksum
+    matches and to remove the 'telomere.bin' and 'verified.bin' marker files if it does not.
+
+    Notes:
+        Removing the telomere.bin marker file from session's raw_data folder marks the session as incomplete, excluding
+        it from all further automatic processing.
+
+        This function is also used to create the processed data hierarchy on the BioHPC server, when it is called as
+        part of the data preprocessing runtime performed by a data acquisition system.
+
+    Args:
+        session_path: The path to the session directory to be verified. Note, the input session directory must contain
+            the 'raw_data' subdirectory.
+        create_processed_data_directory: Determines whether to create the processed data hierarchy during runtime.
+        processed_data_root: The root directory where to store the processed data hierarchy. This path has to point to
+            the root directory where to store the processed data from all projects, and it will be automatically
+            modified to include the project name, the animal name, and the session ID.
+    """

sl_shared_assets/tools/transfer_tools.py

@@ -0,0 +1,119 @@
+"""This module provides methods for moving session runtime data between the local machine, the ScanImage (Mesoscope) PC,
+the Synology NAS drive, and the lab BioHPC server. All methods in this module expect that the destinations and sources
+are mounted on the host file-system via the SMB or an equivalent protocol.
+"""
+
+import shutil
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from tqdm import tqdm
+from ataraxis_base_utilities import console, ensure_directory_exists
+
+from .packaging_tools import calculate_directory_checksum
+
+
+def _transfer_file(source_file: Path, source_directory: Path, destination_directory: Path) -> None:
+    """Copies the input file from the source directory to the destination directory while preserving the file metadata.
+
+    This is a worker method used by the transfer_directory() method to move multiple files in parallel.
+
+    Notes:
+        If the file is found under a hierarchy of subdirectories inside the input source_directory, that hierarchy will
+        be preserved in the destination directory.
+
+    Args:
+        source_file: The file to be copied.
+        source_directory: The root directory where the file is located.
+        destination_directory: The destination directory where to move the file.
+    """
+    relative = source_file.relative_to(source_directory)
+    dest_file = destination_directory / relative
+    shutil.copy2(source_file, dest_file)
+
+
+def transfer_directory(source: Path, destination: Path, num_threads: int = 1, verify_integrity: bool = True) -> None:
+    """Copies the contents of the input directory tree from source to destination while preserving the folder
+    structure.
+
+    This function is used to assemble the experimental data from all remote machines used in the acquisition process on
+    the VRPC before the data is preprocessed. It is also used to transfer the preprocessed data from the VRPC to the
+    SynologyNAS and the Sun lab BioHPC server.
+
+    Notes:
+        This method recreates the moved directory hierarchy on the destination if the hierarchy does not exist. This is
+        done before copying the files.
+
+        The method executes a multithreading copy operation. It does not clean up the source files. That job is handed
+        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that calls this function.
+
+        If the method is configured to verify transferred file integrity, it reruns the xxHash3-128 checksum calculation
+        and compares the returned checksum to the one stored in the source directory. The method assumes that all input
+        directories contain the 'ax_checksum.txt' file that stores the 'source' directory checksum at the highest level
+        of the input directory tree.
+
+    Args:
+        source: The path to the directory that needs to be moved.
+        destination: The path to the destination directory where to move the contents of the source directory.
+        num_threads: The number of threads to use for parallel file transfer. This number should be set depending on the
+            type of transfer (local or remote) and is not guaranteed to provide improved transfer performance. For local
+            transfers, setting this number above 1 will likely provide a performance boost. For remote transfers using
+            a single TCP / IP socket (such as non-multichannel SMB protocol), the number should be set to 1.
+        verify_integrity: Determines whether to perform integrity verification for the transferred files. Note,
+            integrity verification is a time-consuming process and generally would not be a concern for most runtimes.
+            Therefore, it is often fine to disable this option to optimize method runtime speed.
+
+    Raises:
+        RuntimeError: If the transferred files do not pass the xxHas3-128 checksum integrity verification.
+    """
+    if not source.exists():
+        message = f"Unable to move the directory {source}, as it does not exist."
+        console.error(message=message, error=FileNotFoundError)
+
+    # Ensures the destination root directory exists.
+    ensure_directory_exists(destination)
+
+    # Collects all items (files and directories) in the source directory.
+    all_items = tuple(source.rglob("*"))
+
+    # Loops over all items (files and directories). Adds files to the file_list variable. Uses directories to reinstate
+    # the source subdirectory hierarchy in the destination directory.
+    file_list = []
+    for item in sorted(all_items, key=lambda x: len(x.relative_to(source).parts)):
+        # Recreates directory structure on destination
+        if item.is_dir():
+            dest_dir = destination / item.relative_to(source)
+            dest_dir.mkdir(parents=True, exist_ok=True)
+        # Also builds the list of files to be moved
+        else:  # is_file()
+            file_list.append(item)
+
+    # Copies the data to the destination. For parallel workflows, the method uses the ThreadPoolExecutor to move
+    # multiple files at the same time. Since I/O operations do not hold GIL, we do not need to parallelize with
+    # Processes here.
+    if num_threads > 1:
+        with ThreadPoolExecutor(max_workers=num_threads) as executor:
+            futures = {executor.submit(_transfer_file, file, source, destination): file for file in file_list}
+            for future in tqdm(
+                as_completed(futures),
+                total=len(file_list),
+                desc=f"Transferring files to {Path(*destination.parts[-6:])}",
+                unit="file",
+            ):
+                # Propagates any exceptions from the file transfer.
+                future.result()
+    else:
+        for file in tqdm(file_list, desc=f"Transferring files to {Path(*destination.parts[-6:])}", unit="file"):
+            _transfer_file(file, source, destination)
+
+    # Verifies the integrity of the transferred directory by rerunning xxHash3-128 calculation.
+    if verify_integrity:
+        destination_checksum = calculate_directory_checksum(directory=destination, batch=False, save_checksum=False)
+        with open(file=source.joinpath("ax_checksum.txt"), mode="r") as local_checksum:
+            message = (
+                f"Checksum mismatch detected when transferring {Path(*source.parts[-6:])} to "
+                f"{Path(*destination.parts[-6:])}! The data was likely corrupted in transmission. User intervention "
+                f"required."
+            )
+            if not destination_checksum == local_checksum.readline().strip():
+                console.error(message=message, error=RuntimeError)

sl_shared_assets/tools/transfer_tools.pyi

@@ -0,0 +1,53 @@
+from pathlib import Path
+
+from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
+
+def _transfer_file(source_file: Path, source_directory: Path, destination_directory: Path) -> None:
+    """Copies the input file from the source directory to the destination directory while preserving the file metadata.
+
+    This is a worker method used by the transfer_directory() method to move multiple files in parallel.
+
+    Notes:
+        If the file is found under a hierarchy of subdirectories inside the input source_directory, that hierarchy will
+        be preserved in the destination directory.
+
+    Args:
+        source_file: The file to be copied.
+        source_directory: The root directory where the file is located.
+        destination_directory: The destination directory where to move the file.
+    """
+
+def transfer_directory(source: Path, destination: Path, num_threads: int = 1, verify_integrity: bool = True) -> None:
+    """Copies the contents of the input directory tree from source to destination while preserving the folder
+    structure.
+
+    This function is used to assemble the experimental data from all remote machines used in the acquisition process on
+    the VRPC before the data is preprocessed. It is also used to transfer the preprocessed data from the VRPC to the
+    SynologyNAS and the Sun lab BioHPC server.
+
+    Notes:
+        This method recreates the moved directory hierarchy on the destination if the hierarchy does not exist. This is
+        done before copying the files.
+
+        The method executes a multithreading copy operation. It does not clean up the source files. That job is handed
+        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that calls this function.
+
+        If the method is configured to verify transferred file integrity, it reruns the xxHash3-128 checksum calculation
+        and compares the returned checksum to the one stored in the source directory. The method assumes that all input
+        directories contain the 'ax_checksum.txt' file that stores the 'source' directory checksum at the highest level
+        of the input directory tree.
+
+    Args:
+        source: The path to the directory that needs to be moved.
+        destination: The path to the destination directory where to move the contents of the source directory.
+        num_threads: The number of threads to use for parallel file transfer. This number should be set depending on the
+            type of transfer (local or remote) and is not guaranteed to provide improved transfer performance. For local
+            transfers, setting this number above 1 will likely provide a performance boost. For remote transfers using
+            a single TCP / IP socket (such as non-multichannel SMB protocol), the number should be set to 1.
+        verify_integrity: Determines whether to perform integrity verification for the transferred files. Note,
+            integrity verification is a time-consuming process and generally would not be a concern for most runtimes.
+            Therefore, it is often fine to disable this option to optimize method runtime speed.
+
+    Raises:
+        RuntimeError: If the transferred files do not pass the xxHas3-128 checksum integrity verification.
+    """