sl-shared-assets 1.0.0rc18__py3-none-any.whl → 1.0.0rc20__py3-none-any.whl

sl_shared_assets/tools/ascension_tools.pyi

@@ -0,0 +1,68 @@
+from pathlib import Path
+
+from ..data_classes import (
+    SessionData as SessionData,
+    ProjectConfiguration as ProjectConfiguration,
+)
+from .transfer_tools import transfer_directory as transfer_directory
+from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
+
+def _generate_session_name(acquisition_path: Path) -> str:
+    """Generates a session name using the last modification time of a zstack.mat or MotionEstimator.me file.
+
+    This worker function uses one of the motion estimation files stored in each Tyche 'acquisition' subfolder to
+    generate a modern Sun lab timestamp-based session name. This is used to translate the original Tyche session naming
+    pattern into the pattern used by all modern Sun lab projects and pipelines.
+
+    Args:
+        acquisition_path: The absolute path to the target acquisition folder. These folders are found under the 'day'
+            folders for each animal, e.g.: Tyche-A7/2022_01_03/1.
+
+    Returns:
+        The modernized session name.
+    """
+
+def _reorganize_data(session_data: SessionData, source_root: Path) -> bool:
+    """Reorganizes and moves the session's data from the source folder in the old Tyche data hierarchy to the raw_data
+    folder in the newly created modern hierarchy.
+
+    This worker function is used to physically rearrange the data from the original Tyche data structure to the
+    new data structure. It both moves the existing files to their new destinations and renames certain files to match
+    the modern naming convention used in the Sun lab.
+
+    Args:
+        session_data: The initialized SessionData instance managing the 'ascended' (modernized) session data hierarchy.
+        source_root: The absolute path to the old Tyche data hierarchy folder that stores session's data.
+
+    Returns:
+        True if the ascension process was successfully completed. False if the process encountered missing data or
+        otherwise did not go as expected. When the method returns False, the runtime function requests user intervention
+        to finalize the process manually.
+    """
+
+def ascend_tyche_data(root_directory: Path, output_root_directory: Path, server_root_directory: Path) -> None:
+    """Reformats the old Tyche data to use the modern Sun lab layout and metadata files.
+
+    This function is used to convert old Tyche data to the modern data management standard. This is used to make the
+    data compatible with the modern Sun lab data workflows.
+
+    Notes:
+        This function is statically written to work with the raw Tyche dataset featured in the OSM manuscript:
+        https://www.nature.com/articles/s41586-024-08548-w. Additionally, it assumes that the dataset has been
+        preprocessed with the early Sun lab mesoscope compression pipeline. The function will not work for any other
+        project or data hierarchy.
+
+        As part of its runtime, the function automatically transfers the ascended session data to the BioHPC server.
+        Since transferring the data over the network is the bottleneck of this pipeline, it runs in a single-threaded
+        mode and is constrained by the communication channel between the local machine and the BioHPC server. Calling
+        this function for a large number of sessions will result in a long processing time due to the network data
+        transfer.
+
+    Args:
+        root_directory: The directory that stores one or more Tyche animal folders. This can be conceptualized as the
+            root directory for the Tyche project.
+        output_root_directory: The path to the local directory where to generate the converted Tyche project hierarchy.
+            Typically, this is the 'root' directory where all other Sun lab projects are stored.
+        server_root_directory: The path to the local filesystem-mounted BioHPC server storage directory. Note, this
+            directory hs to be mapped to the local filesystem via the SMB or equivalent protocol.
+    """

sl_shared_assets/tools/packaging_tools.pyi

@@ -0,0 +1,52 @@
+from pathlib import Path
+
+def _calculate_file_checksum(base_directory: Path, file_path: Path) -> tuple[str, bytes]:
+    """Calculates xxHash3-128 checksum for a single file and its path relative to the base directory.
+
+    This function is passed to parallel workers used by the calculate_directory_hash() method that iteratively
+    calculates the checksum for all files inside a directory. Each call to this function returns the checksum for the
+    target file, which includes both the contents of the file and its path relative to the base directory.
+
+    Args:
+        base_directory: The path to the base (root) directory which is being checksummed by the main
+            'calculate_directory_checksum' function.
+        file_path: The absolute path to the target file.
+
+    Returns:
+        A tuple with two elements. The first element is the path to the file relative to the base directory. The second
+        element is the xxHash3-128 checksum that covers the relative path and the contents of the file.
+    """
+
+def calculate_directory_checksum(
+    directory: Path, num_processes: int | None = None, batch: bool = False, save_checksum: bool = True
+) -> str:
+    """Calculates xxHash3-128 checksum for the input directory, which includes the data of all contained files and
+    the directory structure information.
+
+    This function is used to generate a checksum for the raw_data directory of each experiment or training session.
+    Checksums are used to verify the session data integrity during transmission between the PC that acquired the data
+    and long-term storage locations, such as the Synology NAS or the BioHPC server. The function can be configured to
+    write the generated checksum as a hexadecimal string to the ax_checksum.txt file stored at the highest level of the
+    input directory.
+
+    Note:
+        This method uses multiprocessing to efficiently parallelize checksum calculation for multiple files. In
+        combination with xxHash3, this achieves a significant speedup over more common checksums, such as MD5 and
+        SHA256. Note that xxHash3 is not suitable for security purposes and is only used to ensure data integrity.
+
+        The method notifies the user about the checksum calculation process via the terminal.
+
+        The returned checksum accounts for both the contents of each file and the layout of the input directory
+        structure.
+
+    Args:
+        directory: The Path to the directory to be checksummed.
+        num_processes: The number of CPU processes to use for parallelizing checksum calculation. If set to None, the
+            function defaults to using (logical CPU count - 4).
+        batch: Determines whether the function is called as part of batch-processing multiple directories. This is used
+            to optimize progress reporting to avoid cluttering the terminal.
+        save_checksum: Determines whether the checksum should be saved (written to) a .txt file.
+
+    Returns:
+        The xxHash3-128 checksum for the input directory as a hexadecimal string.
+    """

sl_shared_assets/tools/transfer_tools.pyi

@@ -0,0 +1,53 @@
+from pathlib import Path
+
+from .packaging_tools import calculate_directory_checksum as calculate_directory_checksum
+
+def _transfer_file(source_file: Path, source_directory: Path, destination_directory: Path) -> None:
+    """Copies the input file from the source directory to the destination directory while preserving the file metadata.
+
+    This is a worker method used by the transfer_directory() method to move multiple files in parallel.
+
+    Notes:
+        If the file is found under a hierarchy of subdirectories inside the input source_directory, that hierarchy will
+        be preserved in the destination directory.
+
+    Args:
+        source_file: The file to be copied.
+        source_directory: The root directory where the file is located.
+        destination_directory: The destination directory where to move the file.
+    """
+
+def transfer_directory(source: Path, destination: Path, num_threads: int = 1, verify_integrity: bool = True) -> None:
+    """Copies the contents of the input directory tree from source to destination while preserving the folder
+    structure.
+
+    This function is used to assemble the experimental data from all remote machines used in the acquisition process on
+    the VRPC before the data is preprocessed. It is also used to transfer the preprocessed data from the VRPC to the
+    SynologyNAS and the Sun lab BioHPC server.
+
+    Notes:
+        This method recreates the moved directory hierarchy on the destination if the hierarchy does not exist. This is
+        done before copying the files.
+
+        The method executes a multithreading copy operation. It does not clean up the source files. That job is handed
+        to the specific preprocessing function from the sl_experiment or sl-forgery libraries that calls this function.
+
+        If the method is configured to verify transferred file integrity, it reruns the xxHash3-128 checksum calculation
+        and compares the returned checksum to the one stored in the source directory. The method assumes that all input
+        directories contain the 'ax_checksum.txt' file that stores the 'source' directory checksum at the highest level
+        of the input directory tree.
+
+    Args:
+        source: The path to the directory that needs to be moved.
+        destination: The path to the destination directory where to move the contents of the source directory.
+        num_threads: The number of threads to use for parallel file transfer. This number should be set depending on the
+            type of transfer (local or remote) and is not guaranteed to provide improved transfer performance. For local
+            transfers, setting this number above 1 will likely provide a performance boost. For remote transfers using
+            a single TCP / IP socket (such as non-multichannel SMB protocol), the number should be set to 1.
+        verify_integrity: Determines whether to perform integrity verification for the transferred files. Note,
+            integrity verification is a time-consuming process and generally would not be a concern for most runtimes.
+            Therefore, it is often fine to disable this option to optimize method runtime speed.
+
+    Raises:
+        RuntimeError: If the transferred files do not pass the xxHas3-128 checksum integrity verification.
+    """

{sl_shared_assets-1.0.0rc18.dist-info → sl_shared_assets-1.0.0rc20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sl-shared-assets
-Version: 1.0.0rc18
+Version: 1.0.0rc20
 Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
 Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
 Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/

sl_shared_assets-1.0.0rc20.dist-info/RECORD

@@ -0,0 +1,40 @@
+sl_shared_assets/__init__.py,sha256=VASYVzsHnaFykp1xdEjOgQdnfuqwokaKIp387zXyFqA,2080
+sl_shared_assets/__init__.pyi,sha256=jCNSKprv3odCQv30EvLrW0mzxwdljZGWyX03mMajBkQ,2163
+sl_shared_assets/cli.py,sha256=a4Ja2gm4ykOac7QmHen14Jhgu6955IUQDZrysp1RpFM,4207
+sl_shared_assets/cli.pyi,sha256=AhD4v9NgVge9zpGA52L42WWLYHXCbyeFXKB9e2v8Xzk,1849
+sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sl_shared_assets/data_classes/__init__.py,sha256=pq-iyd4eyv1CPVcLcQQJBVAm7XCDWuZqxurrLOMW3e0,1702
+sl_shared_assets/data_classes/__init__.pyi,sha256=gnwx-0wTKxOCuq0yNoF25m63byryyzUPeWJWEcK2jMA,1794
+sl_shared_assets/data_classes/configuration_data.py,sha256=zKZ3cE2pZFDRiQ7off8hu2nV9JsBly9Phc5goY4qCH4,4163
+sl_shared_assets/data_classes/configuration_data.pyi,sha256=HeKJWhkVEyAcwDr2dje_Fs2Mfa41CVl38YSh7js9WRE,2024
+sl_shared_assets/data_classes/runtime_data.py,sha256=GupRuTRnbp-ggLiTeEyX9r12VaWqVHpwU7FxOm3MN3M,14909
+sl_shared_assets/data_classes/runtime_data.pyi,sha256=yetdFz29QQyvMmpGTdWGxKSgWAdBZ9zsPM3fluDeDIY,6808
+sl_shared_assets/data_classes/session_data.py,sha256=NyrDGvvD14VnYvGyW0VS4OUHhUWyJHMQ3KfoZo3JaBY,81887
+sl_shared_assets/data_classes/session_data.pyi,sha256=m5B_W4Nh9Ua-qsUOPtM8IvaPG-alnJnHe8KJiAYllr0,30830
+sl_shared_assets/data_classes/surgery_data.py,sha256=Z4QZAZCz0_w7qrGMMzYn-fwwwycw3ki9SPJU0kD5Ap4,7425
+sl_shared_assets/data_classes/surgery_data.pyi,sha256=jLDK1fMli2yF0RkmQcED_iMZq8XIlbIqLysZZGlEJKY,2602
+sl_shared_assets/server/__init__.py,sha256=nyX6-9ACcrQeRQOCNvBVrWSTHGjRPANIG_u0aq7HPTg,426
+sl_shared_assets/server/__init__.pyi,sha256=7o99f8uf6NuBjMZjNAM1FX69Qbu5uBluRSAyaUWbXOU,263
+sl_shared_assets/server/job.py,sha256=M7ZytqhluEV-YQPM9VQV3SqK-F9egQ3UcLc1SLBH4rA,7885
+sl_shared_assets/server/job.pyi,sha256=cxgHMpuwHsJGf_ZcTSSa2tZNzeR_GxqlICOsYGV_oy0,5655
+sl_shared_assets/server/server.py,sha256=NpqWVjvxdtxHZIooBVaKY2jEBDN3dGohBzrFA0TSKIU,9380
+sl_shared_assets/server/server.pyi,sha256=uu-bs5LBuOF-AL7qroYFbGc1aWgBalm-uHUXzZNxheU,4144
+sl_shared_assets/suite2p/__init__.py,sha256=8N2BQNINBkH7JNODmYACHhvUH_uurTuw7w0PW3rVRYM,411
+sl_shared_assets/suite2p/__init__.pyi,sha256=o2R9hcKuCbzzFwjcHdnkLyG03cMBp-Zpn93P-_A4ZFY,224
+sl_shared_assets/suite2p/multi_day.py,sha256=sjteUROBxx2mAziPv42_qMeP5cqofECvd2HE5WqGjvY,13114
+sl_shared_assets/suite2p/multi_day.pyi,sha256=_kxgHqtsxIG__09YVv-YKEdgClOpTdN0o6fTE1CW4Mo,4315
+sl_shared_assets/suite2p/single_day.py,sha256=v0_WAvXTYVB0RjTmNDaX2dWUKjFrtNM1suwIx8IIKBQ,29983
+sl_shared_assets/suite2p/single_day.pyi,sha256=UxieE15msb-MPsqnPQr_3CdbYvYLRYfx70M_yYHl-98,7924
+sl_shared_assets/tools/__init__.py,sha256=UWrNfseGupOrY4OK2PgDEJ-Q3LbSrPLC2pfaLm8wYBA,468
+sl_shared_assets/tools/__init__.pyi,sha256=HwriMRN-9oZRiyvl5VvP8fm5Um3_YWUB4nqnCLMDayo,314
+sl_shared_assets/tools/ascension_tools.py,sha256=y2w72Czpbw9LwgsbdEx_ckzMKTZ78cgKPTIk6vRYmAk,15877
+sl_shared_assets/tools/ascension_tools.pyi,sha256=AE1buiE-rSLGpSPf4TVOtxUu12u-fXL6tL10RQCsrnQ,3973
+sl_shared_assets/tools/packaging_tools.py,sha256=LOKCKvT6UD_cidCONaI4ctWyej5zEwwdhhgwYrj60Kg,6746
+sl_shared_assets/tools/packaging_tools.pyi,sha256=hlAP9AxF7NHtFIPKjj5ehm8Vr9qIn6xDk4VvL0JuAmk,3055
+sl_shared_assets/tools/transfer_tools.py,sha256=J26kwOp_NpPSY0-xu5FTw9udte-rm_mW1FJyaTNoqQI,6606
+sl_shared_assets/tools/transfer_tools.pyi,sha256=FoH7eYZe7guGHfPr0MK5ggO62uXKwD2aJ7h1Bu7PaEE,3294
+sl_shared_assets-1.0.0rc20.dist-info/METADATA,sha256=fgvVyVByniPRILIqCJO_SF5uG8ICEWGlw9oEi3sIke8,47884
+sl_shared_assets-1.0.0rc20.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+sl_shared_assets-1.0.0rc20.dist-info/entry_points.txt,sha256=bdnmVAcK3nrKi9QEYeNMrCLFH5LQ4BMBfwbLIgLPtq4,222
+sl_shared_assets-1.0.0rc20.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+sl_shared_assets-1.0.0rc20.dist-info/RECORD,,

sl_shared_assets-1.0.0rc18.dist-info/RECORD

@@ -1,23 +0,0 @@
-sl_shared_assets/__init__.py,sha256=VASYVzsHnaFykp1xdEjOgQdnfuqwokaKIp387zXyFqA,2080
-sl_shared_assets/cli.py,sha256=a4Ja2gm4ykOac7QmHen14Jhgu6955IUQDZrysp1RpFM,4207
-sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sl_shared_assets/data_classes/__init__.py,sha256=pq-iyd4eyv1CPVcLcQQJBVAm7XCDWuZqxurrLOMW3e0,1702
-sl_shared_assets/data_classes/configuration_data.py,sha256=zKZ3cE2pZFDRiQ7off8hu2nV9JsBly9Phc5goY4qCH4,4163
-sl_shared_assets/data_classes/runtime_data.py,sha256=jSCvZG01gkcSUexFiKITjTJ8ZoF18qRfxiwhyrWW7-I,13964
-sl_shared_assets/data_classes/session_data.py,sha256=z5rt099ZOMydefxE65i5-l3brRqfoSfDdRywLsuEYaI,81597
-sl_shared_assets/data_classes/surgery_data.py,sha256=Z4QZAZCz0_w7qrGMMzYn-fwwwycw3ki9SPJU0kD5Ap4,7425
-sl_shared_assets/server/__init__.py,sha256=nyX6-9ACcrQeRQOCNvBVrWSTHGjRPANIG_u0aq7HPTg,426
-sl_shared_assets/server/job.py,sha256=M7ZytqhluEV-YQPM9VQV3SqK-F9egQ3UcLc1SLBH4rA,7885
-sl_shared_assets/server/server.py,sha256=NpqWVjvxdtxHZIooBVaKY2jEBDN3dGohBzrFA0TSKIU,9380
-sl_shared_assets/suite2p/__init__.py,sha256=8N2BQNINBkH7JNODmYACHhvUH_uurTuw7w0PW3rVRYM,411
-sl_shared_assets/suite2p/multi_day.py,sha256=2_AHI91wuxwS07GFJn8S1454uLrQGmt0bS_qBoZsHVc,13102
-sl_shared_assets/suite2p/single_day.py,sha256=7dgYN9FnIvxNL0uHo0CtoLng2nRxZJYtV_AfskMt6tE,29961
-sl_shared_assets/tools/__init__.py,sha256=UWrNfseGupOrY4OK2PgDEJ-Q3LbSrPLC2pfaLm8wYBA,468
-sl_shared_assets/tools/ascension_tools.py,sha256=y2w72Czpbw9LwgsbdEx_ckzMKTZ78cgKPTIk6vRYmAk,15877
-sl_shared_assets/tools/packaging_tools.py,sha256=LOKCKvT6UD_cidCONaI4ctWyej5zEwwdhhgwYrj60Kg,6746
-sl_shared_assets/tools/transfer_tools.py,sha256=J26kwOp_NpPSY0-xu5FTw9udte-rm_mW1FJyaTNoqQI,6606
-sl_shared_assets-1.0.0rc18.dist-info/METADATA,sha256=oLVm883kOA9Mms93kAa0qK8iBybCV5oKW29EWrDLiFg,47884
-sl_shared_assets-1.0.0rc18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-sl_shared_assets-1.0.0rc18.dist-info/entry_points.txt,sha256=bdnmVAcK3nrKi9QEYeNMrCLFH5LQ4BMBfwbLIgLPtq4,222
-sl_shared_assets-1.0.0rc18.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-sl_shared_assets-1.0.0rc18.dist-info/RECORD,,