sl-shared-assets 1.0.0rc13__py3-none-any.whl → 1.0.0rc15__py3-none-any.whl

sl_shared_assets/{suite2p.py → suite2p/single_day.py}

@@ -1,6 +1,7 @@
-"""This module provides the classes used to store suite2p runtime configuration parameters in .YAML files. This is used
-by the sl-forgery library to configure mesoscope data processing via the suite2p library, both during single-day and
-multiday registration processing."""
+"""This module stores the classes used to configure the single-day (within-session) sl-suite2p pipeline. This is the
+'original' suite2p pipeline used to process brain activity data collected as part of a single continuous recording. It
+is used as the first step of the multi-day brain activity processing pipeline used in the lab. Both single-day
+(original) and multi-day (extended) pipelines are available as part of the Sun lab maintained sl-suite2p package."""
 
 from typing import Any
 from dataclasses import field, asdict, dataclass
@@ -10,7 +11,7 @@ from ataraxis_data_structures import YamlConfig
 
 @dataclass
 class Main:
-    """Stores global settings used to broadly define the suite2p processing configuration."""
+    """Stores global parameters that broadly define the suite2p single-day processing configuration."""
 
     nplanes: int = 3
     """The number of imaging planes in each TIFF file sequence. For Mesoscope frames, this is the number of individual 
@@ -24,16 +25,17 @@ class Main:
 
     tau: float = 0.4
     """The timescale of the sensor, in seconds, used for computing the deconvolution kernel. The kernel is fixed to 
-    have this decay and is not fit to the data. Note, the default value was optimized for GCamp6f animals used in 
-    Weinan's OSM Manuscript."""
+    have this decay and is not fit to the data. Note, the default value is optimized for GCamp6f animals recorded with 
+    the Mesoscope."""
 
     force_sktiff: bool = True
-    """Determines whether to force the use of scikit-image for reading TIFF files. This HAS to be true for our tiff 
-    files, as they are compressed in a way that cannot be read with ScanImage tiff reader."""
+    """Determines whether to force the use of scikit-image for reading TIFF files. Generally, it is recommended to have 
+    this enabled as it forces suite2p to use tifffile library, which has better safety and compatibility than 
+    ScanImage tiff reader for certain types of tiff files."""
 
     fs: float = 10.0014
-    """The sampling rate per plane in Hertz. This is automatically overwritten using ops.json file generated by our 
-    preprocessing pipeline to match the specific acquisition frequency of the processed dataset."""
+    """The sampling rate per plane in Hertz. For instance, if you have a 10 plane recording acquired at 30Hz, then the 
+    sampling rate per plane is 3Hz, so set this to 3."""
 
     do_bidiphase: bool = False
     """Determines whether to perform computation of bidirectional phase offset for misaligned line scanning 
@@ -53,8 +55,8 @@ class Main:
      to process all available frames."""
 
     multiplane_parallel: bool = True
-    """Determines whether to parallelize plane processing for multiplane data. Assuming that this configuration class is
-    used together with Sun lab optimized suite2p, it is always recommended to have this set to True for most runtimes.
+    """Determines whether to parallelize plane processing for multiplane data. Note, while enabling this option improves
+    processing speeds, it also increases the memory (RAM) overhead resulting from processing all planes in-parallel.
     """
 
     ignore_flyback: list[int] = field(default_factory=list)
@@ -63,7 +65,7 @@ class Main:
 
 @dataclass
 class FileIO:
-    """Stores I/O settings used to specify input data file locations, formats, and output storage options."""
+    """Stores general I/O parameters that specify input data location, format, and working and output directories."""
 
     fast_disk: list[str] = field(default_factory=list)
     """Specifies the locations where to store the temporary binary files created during processing. If no directories 
@@ -107,7 +109,9 @@ class FileIO:
 
     save_folder: list[str] = field(default_factory=list)
     """Lists folder names under which the results should be stored. If this is not provided, the pipeline defaults to 
-    using 'suite2p' as the root folder, created under the path specified by save_path0."""
+    using 'suite2p' as the root folder, created under the path specified by save_path0. Note, if the data produced by 
+    the 'single-day' pipeline is also processed using sl-suite2p 'multi-day' pipeline, do not modify this field. The 
+    multi-day pipeline expects the save_folder to be 'suite2p' (default)."""
 
     look_one_level_down: bool = False
     """Determines whether to search for TIFF files in the subfolders when searching for Tiff files. If this is True, 
@@ -123,7 +127,7 @@ class FileIO:
 
 @dataclass
 class Output:
-    """Stores I/O settings used to define the output format and organization of the processing results."""
+    """Stores I/O settings that specify the output format and organization of the data processing results."""
 
     preclassify: float = 0.5
     """The probability threshold for pre-classification of cells to use before signal extraction. If this is set to 
@@ -149,7 +153,7 @@ class Output:
 
 @dataclass
 class Registration:
-    """Stores rigid registration settings used for correcting motion artifacts between frames."""
+    """Stores parameters for rigid registration, which is used to correct motion artifacts between frames."""
 
     do_registration: bool = True
     """Determines whether to run the motion registration."""
@@ -214,7 +218,8 @@ class Registration:
 
 @dataclass
 class OnePRegistration:
-    """Stores additional pre-registration processing settings used to improve the registration of 1-photon datasets."""
+    """Stores parameters for additional pre-registration processing used to improve the registration of 1-photon
+    datasets."""
 
     one_p_reg: bool = False
     """Determines whether to perform high-pass spatial filtering and tapering to improve one-photon image 
@@ -233,7 +238,8 @@ class OnePRegistration:
 
 @dataclass
 class NonRigid:
-    """Stores non-rigid registration settings used to improve motion registration in complex datasets."""
+    """Stores parameters for non-rigid registration, which is used to improve motion registration in complex
+    datasets."""
 
     nonrigid: bool = True
     """Determines whether to perform non-rigid registration to correct for local motion and deformation. This is used 
@@ -253,7 +259,7 @@ class NonRigid:
 
 @dataclass
 class ROIDetection:
-    """Stores ROI detection and extraction settings used to identify cells and their activity signals."""
+    """Stores parameters for cell ROI detection and extraction."""
 
     roidetect: bool = True
     """Determines whether to perform ROI detection and subsequent signal extraction."""
@@ -292,7 +298,8 @@ class ROIDetection:
     """The maximum number of iterations allowed for cell extraction."""
 
     nbinned: int = 5000
-    """The maximum number of binned frames to use for ROI detection to speed up processing."""
+    """The maximum number of binned frames to use for ROI detection. Settings this value to a higher number leads to 
+    more ROIs being detected, but reduces processing speed and increases RAM overhead."""
 
     denoise: bool = False
     """Determines whether to denoise the binned movie before cell detection in sparse mode to enhance performance. 
@@ -301,7 +308,7 @@ class ROIDetection:
 
 @dataclass
 class CellposeDetection:
-    """Stores Cellpose algorithm settings used for cell detection."""
+    """Stores parameters for the Cellpose algorithm, which can optionally be used to improve cell ROI extraction."""
 
     anatomical_only: int = 0
     """Specifies the Cellpose mode for cell detection:
@@ -332,7 +339,7 @@ class CellposeDetection:
 
 @dataclass
 class SignalExtraction:
-    """Stores settings used to extract fluorescence signals from ROIs and surrounding neuropil regions."""
+    """Stores parameters for extracting fluorescence signals from ROIs and surrounding neuropil regions."""
 
     neuropil_extract: bool = True
     """Determines whether to extract neuropil signals."""
@@ -353,10 +360,10 @@ class SignalExtraction:
 
 @dataclass
 class SpikeDeconvolution:
-    """Stores settings used to deconvolve calcium signals to infer spike trains."""
+    """Stores parameters for deconvolve fluorescence signals to infer spike trains."""
 
     spikedetect: bool = True
-    """Determines whether to perform spike deconvolution to convert calcium signals into estimated spike trains."""
+    """Determines whether to perform spike deconvolution."""
 
     neucoeff: float = 0.7
     """The neuropil coefficient applied for signal correction before deconvolution."""
@@ -382,7 +389,7 @@ class SpikeDeconvolution:
 
 @dataclass
 class Classification:
-    """Stores settings used to classify detected ROIs as real cells or artifacts."""
+    """Stores parameters for classifying detected ROIs as real cells or artifacts."""
 
     soma_crop: bool = True
     """Determines whether to crop dendritic regions from detected ROIs to focus on the cell body for classification 
@@ -397,20 +404,22 @@ class Classification:
 
 @dataclass
 class Channel2:
-    """Stores settings for processing the second channel in multichannel datasets."""
+    """Stores parameters for processing the second channel in multichannel datasets."""
 
     chan2_thres: float = 0.65
     """The threshold for considering an ROI registered in one channel as detected in the second channel."""
 
 
 @dataclass
-class Suite2PConfiguration(YamlConfig):
-    """Stores the user-addressable suite2p configuration parameters, organized into subsections.
+class SingleDayS2PConfiguration(YamlConfig):
+    """Stores the user-addressable suite2p configuration parameters for the single-day (original) pipeline, organized
+    into subsections.
 
-    This class is used during processing to instruct suite2p on how to process the data. Specifically, it provides a
-    user-friendly way of specifying all user-addressable parameters through a .YAML file. The sl-forgery library then
-    loads the data from .yaml file and uses it to configure the single-day suite2p pipeline and the multiday suite2p
-    pipeline.
+    This class is used during single-day processing to instruct suite2p on how to process the data. This class is based
+    on the 'default_ops' from the original suite2p package. As part of the suite2p refactoring performed in sl-suite2p
+    package, the 'default_ops' has been replaced with this class instance. Compared to 'original' ops, it allows saving
+    configuration parameters as a .YAML file, which offers a better way of viewing and editing the parameters and
+    running suite2p pipeline on remote compute servers.
 
     Notes:
         The .YAML file uses section names that match the suite2p documentation sections. This way, users can always
@@ -419,17 +428,31 @@ class Suite2PConfiguration(YamlConfig):
 
     # Define the instances of each nested settings class as fields
     main: Main = field(default_factory=Main)
+    """Stores global parameters that broadly define the suite2p single-day processing configuration."""
     file_io: FileIO = field(default_factory=FileIO)
+    """Stores general I/O parameters that specify input data location, format, and working and output directories."""
     output: Output = field(default_factory=Output)
+    """Stores I/O settings that specify the output format and organization of the data processing results."""
     registration: Registration = field(default_factory=Registration)
+    """Stores parameters for rigid registration, which is used to correct motion artifacts between frames."""
     one_p_registration: OnePRegistration = field(default_factory=OnePRegistration)
+    """Stores parameters for additional pre-registration processing used to improve the registration of 1-photon
+    datasets."""
     non_rigid: NonRigid = field(default_factory=NonRigid)
+    """Stores parameters for non-rigid registration, which is used to improve motion registration in complex 
+    datasets."""
     roi_detection: ROIDetection = field(default_factory=ROIDetection)
+    """Stores parameters for cell ROI detection and extraction."""
     cellpose_detection: CellposeDetection = field(default_factory=CellposeDetection)
+    """Stores parameters for the Cellpose algorithm, which can optionally be used to improve cell ROI extraction."""
     signal_extraction: SignalExtraction = field(default_factory=SignalExtraction)
+    """Stores parameters for extracting fluorescence signals from ROIs and surrounding neuropil regions."""
     spike_deconvolution: SpikeDeconvolution = field(default_factory=SpikeDeconvolution)
+    """Stores parameters for deconvolve fluorescence signals to infer spike trains."""
     classification: Classification = field(default_factory=Classification)
+    """Stores parameters for classifying detected ROIs as real cells or artifacts."""
     channel2: Channel2 = field(default_factory=Channel2)
+    """Stores parameters for processing the second channel in multichannel datasets."""
 
     def to_ops(self) -> dict[str, Any]:
         """Converts the class instance to a dictionary and returns it to caller.

sl_shared_assets/suite2p/single_day.pyi

@@ -0,0 +1,192 @@
+from typing import Any
+from dataclasses import field, dataclass
+
+from _typeshed import Incomplete
+from ataraxis_data_structures import YamlConfig
+
+@dataclass
+class Main:
+    """Stores global parameters that broadly define the suite2p single-day processing configuration."""
+
+    nplanes: int = ...
+    nchannels: int = ...
+    functional_chan: int = ...
+    tau: float = ...
+    force_sktiff: bool = ...
+    fs: float = ...
+    do_bidiphase: bool = ...
+    bidiphase: int = ...
+    bidi_corrected: bool = ...
+    frames_include: int = ...
+    multiplane_parallel: bool = ...
+    ignore_flyback: list[int] = field(default_factory=list)
+
+@dataclass
+class FileIO:
+    """Stores general I/O parameters that specify input data location, format, and working and output directories."""
+
+    fast_disk: list[str] = field(default_factory=list)
+    delete_bin: bool = ...
+    mesoscan: bool = ...
+    bruker: bool = ...
+    bruker_bidirectional: bool = ...
+    h5py: list[str] = field(default_factory=list)
+    h5py_key: str = ...
+    nwb_file: str = ...
+    nwb_driver: str = ...
+    nwb_series: str = ...
+    save_path0: list[str] = field(default_factory=list)
+    save_folder: list[str] = field(default_factory=list)
+    look_one_level_down: bool = ...
+    subfolders: list[str] = field(default_factory=list)
+    move_bin: bool = ...
+
+@dataclass
+class Output:
+    """Stores I/O settings that specify the output format and organization of the data processing results."""
+
+    preclassify: float = ...
+    save_nwb: bool = ...
+    save_mat: bool = ...
+    combined: bool = ...
+    aspect: float = ...
+    report_time: bool = ...
+
+@dataclass
+class Registration:
+    """Stores parameters for rigid registration, which is used to correct motion artifacts between frames."""
+
+    do_registration: bool = ...
+    align_by_chan: int = ...
+    nimg_init: int = ...
+    batch_size: int = ...
+    maxregshift: float = ...
+    smooth_sigma: float = ...
+    smooth_sigma_time: float = ...
+    keep_movie_raw: bool = ...
+    two_step_registration: bool = ...
+    reg_tif: bool = ...
+    reg_tif_chan2: bool = ...
+    subpixel: int = ...
+    th_badframes: float = ...
+    norm_frames: bool = ...
+    force_refImg: bool = ...
+    pad_fft: bool = ...
+
+@dataclass
+class OnePRegistration:
+    """Stores parameters for additional pre-registration processing used to improve the registration of 1-photon
+    datasets."""
+
+    one_p_reg: bool = ...
+    spatial_hp_reg: int = ...
+    pre_smooth: float = ...
+    spatial_taper: float = ...
+
+@dataclass
+class NonRigid:
+    """Stores parameters for non-rigid registration, which is used to improve motion registration in complex
+    datasets."""
+
+    nonrigid: bool = ...
+    block_size: list[int] = field(default_factory=Incomplete)
+    snr_thresh: float = ...
+    maxregshiftNR: float = ...
+
+@dataclass
+class ROIDetection:
+    """Stores parameters for cell ROI detection and extraction."""
+
+    roidetect: bool = ...
+    sparse_mode: bool = ...
+    spatial_scale: int = ...
+    connected: bool = ...
+    threshold_scaling: float = ...
+    spatial_hp_detect: int = ...
+    max_overlap: float = ...
+    high_pass: int = ...
+    smooth_masks: bool = ...
+    max_iterations: int = ...
+    nbinned: int = ...
+    denoise: bool = ...
+
+@dataclass
+class CellposeDetection:
+    """Stores parameters for the Cellpose algorithm, which can optionally be used to improve cell ROI extraction."""
+
+    anatomical_only: int = ...
+    diameter: int = ...
+    cellprob_threshold: float = ...
+    flow_threshold: float = ...
+    spatial_hp_cp: int = ...
+    pretrained_model: str = ...
+
+@dataclass
+class SignalExtraction:
+    """Stores parameters for extracting fluorescence signals from ROIs and surrounding neuropil regions."""
+
+    neuropil_extract: bool = ...
+    allow_overlap: bool = ...
+    min_neuropil_pixels: int = ...
+    inner_neuropil_radius: int = ...
+    lam_percentile: int = ...
+
+@dataclass
+class SpikeDeconvolution:
+    """Stores parameters for deconvolve fluorescence signals to infer spike trains."""
+
+    spikedetect: bool = ...
+    neucoeff: float = ...
+    baseline: str = ...
+    win_baseline: float = ...
+    sig_baseline: float = ...
+    prctile_baseline: float = ...
+
+@dataclass
+class Classification:
+    """Stores parameters for classifying detected ROIs as real cells or artifacts."""
+
+    soma_crop: bool = ...
+    use_builtin_classifier: bool = ...
+    classifier_path: str = ...
+
+@dataclass
+class Channel2:
+    """Stores parameters for processing the second channel in multichannel datasets."""
+
+    chan2_thres: float = ...
+
+@dataclass
+class SingleDayS2PConfiguration(YamlConfig):
+    """Stores the user-addressable suite2p configuration parameters for the single-day (original) pipeline, organized
+    into subsections.
+
+    This class is used during single-day processing to instruct suite2p on how to process the data. This class is based
+    on the 'default_ops' from the original suite2p package. As part of the suite2p refactoring performed in sl-suite2p
+    package, the 'default_ops' has been replaced with this class instance. Compared to 'original' ops, it allows saving
+    configuration parameters as a .YAML file, which offers a better way of viewing and editing the parameters and
+    running suite2p pipeline on remote compute servers.
+
+    Notes:
+        The .YAML file uses section names that match the suite2p documentation sections. This way, users can always
+        consult the suite2p documentation for information on the purpose of each field inside every subsection.
+    """
+
+    main: Main = field(default_factory=Main)
+    file_io: FileIO = field(default_factory=FileIO)
+    output: Output = field(default_factory=Output)
+    registration: Registration = field(default_factory=Registration)
+    one_p_registration: OnePRegistration = field(default_factory=OnePRegistration)
+    non_rigid: NonRigid = field(default_factory=NonRigid)
+    roi_detection: ROIDetection = field(default_factory=ROIDetection)
+    cellpose_detection: CellposeDetection = field(default_factory=CellposeDetection)
+    signal_extraction: SignalExtraction = field(default_factory=SignalExtraction)
+    spike_deconvolution: SpikeDeconvolution = field(default_factory=SpikeDeconvolution)
+    classification: Classification = field(default_factory=Classification)
+    channel2: Channel2 = field(default_factory=Channel2)
+    def to_ops(self) -> dict[str, Any]:
+        """Converts the class instance to a dictionary and returns it to caller.
+
+        This dictionary can be passed to suite2p functions either as an 'ops' or 'db' argument to control the
+        processing runtime.
+        """

sl_shared_assets/tools/__init__.py

@@ -0,0 +1,8 @@
+"""This package provides helper tools used to automate routine operations, such as transferring or verifying the
+integrity of the data. The tools from this package are used by most other data processing libraries in the lab."""
+
+from .transfer_tools import transfer_directory
+from .ascension_tools import ascend_tyche_data
+from .packaging_tools import calculate_directory_checksum
+
+__all__ = ["transfer_directory", "calculate_directory_checksum", "ascend_tyche_data"]

sl_shared_assets/tools/__init__.pyi

@@ -0,0 +1,5 @@
+from .transfer_tools import transfer_directory as transfer_directory
+from .ascension_tools import ascend_tyche_data as ascend_tyche_data
+from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
+
+__all__ = ["transfer_directory", "calculate_directory_checksum", "ascend_tyche_data"]

sl_shared_assets/{ascension_tools.py → tools/ascension_tools.py}

@@ -1,6 +1,7 @@
 """This module provides tools for translating ('ascending') old Tyche data to use the modern data structure used in the
 Sun lab. The tools from this module will not work for any other data and also assume that the Tyche data has been
-preprocessed with an early version of the Sun lab mesoscope processing pipeline."""
+preprocessed with an early version of the Sun lab mesoscope processing pipeline. However, this module can be used as
+an example for how to convert other data formats to match use the Sun lab data structure."""
 
 from pathlib import Path
 import datetime
@@ -10,14 +11,10 @@ import numpy as np
 from ataraxis_base_utilities import LogLevel, console
 from ataraxis_time.time_helpers import extract_timestamp_from_bytes
 
-from .data_classes import SessionData, ProjectConfiguration
+from ..data_classes import SessionData, ProjectConfiguration
 from .transfer_tools import transfer_directory
 from .packaging_tools import calculate_directory_checksum
 
-# Ensures the console is enabled when this file is imported
-if not console.enabled:
-    console.enable()
-
 
 def _generate_session_name(acquisition_path: Path) -> str:
     """Generates a session name using the last modification time of a zstack.mat or MotionEstimator.me file.

sl_shared_assets/tools/ascension_tools.pyi

@@ -0,0 +1,68 @@
+from pathlib import Path
+
+from ..data_classes import (
+    SessionData as SessionData,
+    ProjectConfiguration as ProjectConfiguration,
+)
+from .transfer_tools import transfer_directory as transfer_directory
+from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
+
+def _generate_session_name(acquisition_path: Path) -> str:
+    """Generates a session name using the last modification time of a zstack.mat or MotionEstimator.me file.
+
+    This worker function uses one of the motion estimation files stored in each Tyche 'acquisition' subfolder to
+    generate a modern Sun lab timestamp-based session name. This is used to translate the original Tyche session naming
+    pattern into the pattern used by all modern Sun lab projects and pipelines.
+
+    Args:
+        acquisition_path: The absolute path to the target acquisition folder. These folders are found under the 'day'
+            folders for each animal, e.g.: Tyche-A7/2022_01_03/1.
+
+    Returns:
+        The modernized session name.
+    """
+
+def _reorganize_data(session_data: SessionData, source_root: Path) -> bool:
+    """Reorganizes and moves the session's data from the source folder in the old Tyche data hierarchy to the raw_data
+    folder in the newly created modern hierarchy.
+
+    This worker function is used to physically rearrange the data from the original Tyche data structure to the
+    new data structure. It both moves the existing files to their new destinations and renames certain files to match
+    the modern naming convention used in the Sun lab.
+
+    Args:
+        session_data: The initialized SessionData instance managing the 'ascended' (modernized) session data hierarchy.
+        source_root: The absolute path to the old Tyche data hierarchy folder that stores session's data.
+
+    Returns:
+        True if the ascension process was successfully completed. False if the process encountered missing data or
+        otherwise did not go as expected. When the method returns False, the runtime function requests user intervention
+        to finalize the process manually.
+    """
+
+def ascend_tyche_data(root_directory: Path, output_root_directory: Path, server_root_directory: Path) -> None:
+    """Reformats the old Tyche data to use the modern Sun lab layout and metadata files.
+
+    This function is used to convert old Tyche data to the modern data management standard. This is used to make the
+    data compatible with the modern Sun lab data workflows.
+
+    Notes:
+        This function is statically written to work with the raw Tyche dataset featured in the OSM manuscript:
+        https://www.nature.com/articles/s41586-024-08548-w. Additionally, it assumes that the dataset has been
+        preprocessed with the early Sun lab mesoscope compression pipeline. The function will not work for any other
+        project or data hierarchy.
+
+        As part of its runtime, the function automatically transfers the ascended session data to the BioHPC server.
+        Since transferring the data over the network is the bottleneck of this pipeline, it runs in a single-threaded
+        mode and is constrained by the communication channel between the local machine and the BioHPC server. Calling
+        this function for a large number of sessions will result in a long processing time due to the network data
+        transfer.
+
+    Args:
+        root_directory: The directory that stores one or more Tyche animal folders. This can be conceptualized as the
+            root directory for the Tyche project.
+        output_root_directory: The path to the local directory where to generate the converted Tyche project hierarchy.
+            Typically, this is the 'root' directory where all other Sun lab projects are stored.
+        server_root_directory: The path to the local filesystem-mounted BioHPC server storage directory. Note, this
+            directory hs to be mapped to the local filesystem via the SMB or equivalent protocol.
+    """

sl_shared_assets/tools/packaging_tools.pyi

@@ -0,0 +1,52 @@
+from pathlib import Path
+
+def _calculate_file_checksum(base_directory: Path, file_path: Path) -> tuple[str, bytes]:
+    """Calculates xxHash3-128 checksum for a single file and its path relative to the base directory.
+
+    This function is passed to parallel workers used by the calculate_directory_hash() method that iteratively
+    calculates the checksum for all files inside a directory. Each call to this function returns the checksum for the
+    target file, which includes both the contents of the file and its path relative to the base directory.
+
+    Args:
+        base_directory: The path to the base (root) directory which is being checksummed by the main
+            'calculate_directory_checksum' function.
+        file_path: The absolute path to the target file.
+
+    Returns:
+        A tuple with two elements. The first element is the path to the file relative to the base directory. The second
+        element is the xxHash3-128 checksum that covers the relative path and the contents of the file.
+    """
+
+def calculate_directory_checksum(
+    directory: Path, num_processes: int | None = None, batch: bool = False, save_checksum: bool = True
+) -> str:
+    """Calculates xxHash3-128 checksum for the input directory, which includes the data of all contained files and
+    the directory structure information.
+
+    This function is used to generate a checksum for the raw_data directory of each experiment or training session.
+    Checksums are used to verify the session data integrity during transmission between the PC that acquired the data
+    and long-term storage locations, such as the Synology NAS or the BioHPC server. The function can be configured to
+    write the generated checksum as a hexadecimal string to the ax_checksum.txt file stored at the highest level of the
+    input directory.
+
+    Note:
+        This method uses multiprocessing to efficiently parallelize checksum calculation for multiple files. In
+        combination with xxHash3, this achieves a significant speedup over more common checksums, such as MD5 and
+        SHA256. Note that xxHash3 is not suitable for security purposes and is only used to ensure data integrity.
+
+        The method notifies the user about the checksum calculation process via the terminal.
+
+        The returned checksum accounts for both the contents of each file and the layout of the input directory
+        structure.
+
+    Args:
+        directory: The Path to the directory to be checksummed.
+        num_processes: The number of CPU processes to use for parallelizing checksum calculation. If set to None, the
+            function defaults to using (logical CPU count - 4).
+        batch: Determines whether the function is called as part of batch-processing multiple directories. This is used
+            to optimize progress reporting to avoid cluttering the terminal.
+        save_checksum: Determines whether the checksum should be saved (written to) a .txt file.
+
+    Returns:
+        The xxHash3-128 checksum for the input directory as a hexadecimal string.
+    """

sl_shared_assets/tools/transfer_tools.pyi

@@ -0,0 +1,53 @@
+from pathlib import Path
+
+from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
+
+def _transfer_file(source_file: Path, source_directory: Path, destination_directory: Path) -> None:
+    """Copies the input file from the source directory to the destination directory while preserving the file metadata.
+
+    This is a worker method used by the transfer_directory() method to move multiple files in parallel.
+
+    Notes:
+        If the file is found under a hierarchy of subdirectories inside the input source_directory, that hierarchy will
+        be preserved in the destination directory.
+
+    Args:
+        source_file: The file to be copied.
+        source_directory: The root directory where the file is located.
+        destination_directory: The destination directory where to move the file.
+    """
+
+def transfer_directory(source: Path, destination: Path, num_threads: int = 1, verify_integrity: bool = True) -> None:
+    """Copies the contents of the input directory tree from source to destination while preserving the folder
+    structure.
+
+    This function is used to assemble the experimental data from all remote machines used in the acquisition process on
+    the VRPC before the data is preprocessed. It is also used to transfer the preprocessed data from the VRPC to the
+    SynologyNAS and the Sun lab BioHPC server.
+
+    Notes:
+        This method recreates the moved directory hierarchy on the destination if the hierarchy does not exist. This is
+        done before copying the files.
+
+        The method executes a multithreading copy operation. It does not clean up the source files. That job is handed
+        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that calls this function.
+
+        If the method is configured to verify transferred file integrity, it reruns the xxHash3-128 checksum calculation
+        and compares the returned checksum to the one stored in the source directory. The method assumes that all input
+        directories contain the 'ax_checksum.txt' file that stores the 'source' directory checksum at the highest level
+        of the input directory tree.
+
+    Args:
+        source: The path to the directory that needs to be moved.
+        destination: The path to the destination directory where to move the contents of the source directory.
+        num_threads: The number of threads to use for parallel file transfer. This number should be set depending on the
+            type of transfer (local or remote) and is not guaranteed to provide improved transfer performance. For local
+            transfers, setting this number above 1 will likely provide a performance boost. For remote transfers using
+            a single TCP / IP socket (such as non-multichannel SMB protocol), the number should be set to 1.
+        verify_integrity: Determines whether to perform integrity verification for the transferred files. Note,
+            integrity verification is a time-consuming process and generally would not be a concern for most runtimes.
+            Therefore, it is often fine to disable this option to optimize method runtime speed.
+
+    Raises:
+        RuntimeError: If the transferred files do not pass the xxHas3-128 checksum integrity verification.
+    """

{sl_shared_assets-1.0.0rc13.dist-info → sl_shared_assets-1.0.0rc15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 1.0.0rc13
+Version: 1.0.0rc15
 Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/