sl-shared-assets 4.0.1__py3-none-any.whl → 5.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sl-shared-assets might be problematic. Click here for more details.
- sl_shared_assets/__init__.py +48 -41
- sl_shared_assets/command_line_interfaces/__init__.py +3 -0
- sl_shared_assets/command_line_interfaces/configure.py +173 -0
- sl_shared_assets/command_line_interfaces/manage.py +226 -0
- sl_shared_assets/data_classes/__init__.py +33 -32
- sl_shared_assets/data_classes/configuration_data.py +267 -79
- sl_shared_assets/data_classes/session_data.py +226 -289
- sl_shared_assets/server/__init__.py +24 -4
- sl_shared_assets/server/job.py +6 -7
- sl_shared_assets/server/pipeline.py +585 -0
- sl_shared_assets/server/server.py +57 -25
- sl_shared_assets/tools/__init__.py +9 -8
- sl_shared_assets/tools/packaging_tools.py +14 -25
- sl_shared_assets/tools/project_management_tools.py +602 -523
- sl_shared_assets/tools/transfer_tools.py +88 -23
- {sl_shared_assets-4.0.1.dist-info → sl_shared_assets-5.0.1.dist-info}/METADATA +46 -203
- sl_shared_assets-5.0.1.dist-info/RECORD +23 -0
- sl_shared_assets-5.0.1.dist-info/entry_points.txt +3 -0
- sl_shared_assets/__init__.pyi +0 -91
- sl_shared_assets/cli.py +0 -501
- sl_shared_assets/cli.pyi +0 -106
- sl_shared_assets/data_classes/__init__.pyi +0 -75
- sl_shared_assets/data_classes/configuration_data.pyi +0 -235
- sl_shared_assets/data_classes/runtime_data.pyi +0 -157
- sl_shared_assets/data_classes/session_data.pyi +0 -379
- sl_shared_assets/data_classes/surgery_data.pyi +0 -89
- sl_shared_assets/server/__init__.pyi +0 -11
- sl_shared_assets/server/job.pyi +0 -205
- sl_shared_assets/server/server.pyi +0 -298
- sl_shared_assets/tools/__init__.pyi +0 -19
- sl_shared_assets/tools/ascension_tools.py +0 -265
- sl_shared_assets/tools/ascension_tools.pyi +0 -68
- sl_shared_assets/tools/packaging_tools.pyi +0 -58
- sl_shared_assets/tools/project_management_tools.pyi +0 -239
- sl_shared_assets/tools/transfer_tools.pyi +0 -53
- sl_shared_assets-4.0.1.dist-info/RECORD +0 -36
- sl_shared_assets-4.0.1.dist-info/entry_points.txt +0 -7
- {sl_shared_assets-4.0.1.dist-info → sl_shared_assets-5.0.1.dist-info}/WHEEL +0 -0
- {sl_shared_assets-4.0.1.dist-info → sl_shared_assets-5.0.1.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,22 +1,69 @@
|
|
|
1
|
-
"""This module provides
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
"""This module provides tools for moving the data between destinations within or across machines (PCs). All methods in
|
|
2
|
+
this module expect that the destinations and sources are mounted on the host-machine file-system via the SMB or an
|
|
3
|
+
equivalent protocol.
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
|
+
import os
|
|
6
7
|
import shutil
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
9
10
|
|
|
10
11
|
from tqdm import tqdm
|
|
12
|
+
from ataraxis_time import PrecisionTimer
|
|
11
13
|
from ataraxis_base_utilities import console, ensure_directory_exists
|
|
12
14
|
|
|
13
15
|
from .packaging_tools import calculate_directory_checksum
|
|
14
16
|
|
|
15
17
|
|
|
18
|
+
def delete_directory(directory_path: Path) -> None:
|
|
19
|
+
"""Removes the input directory and all its subdirectories using parallel processing.
|
|
20
|
+
|
|
21
|
+
This function outperforms default approaches like subprocess call with rm -rf and shutil rmtree for directories with
|
|
22
|
+
a comparably small number of large files. For example, this is the case for the mesoscope frame directories, which
|
|
23
|
+
are deleted ~6 times faster with this method over sh.rmtree. Potentially, it may also outperform these approaches
|
|
24
|
+
for all comparatively shallow directories.
|
|
25
|
+
|
|
26
|
+
Notes:
|
|
27
|
+
This function is often combined with the transfer_directory function to remove the source directory after
|
|
28
|
+
it has been transferred.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
directory_path: The path to the directory to delete.
|
|
32
|
+
"""
|
|
33
|
+
# Checks if the directory exists and, if not, aborts early
|
|
34
|
+
if not directory_path.exists():
|
|
35
|
+
return
|
|
36
|
+
|
|
37
|
+
# Builds the list of files and directories inside the input directory using Path
|
|
38
|
+
files = [p for p in directory_path.iterdir() if p.is_file()]
|
|
39
|
+
subdirectories = [p for p in directory_path.iterdir() if p.is_dir()]
|
|
40
|
+
|
|
41
|
+
# Deletes files in parallel
|
|
42
|
+
with ThreadPoolExecutor() as executor:
|
|
43
|
+
list(executor.map(os.unlink, files)) # Forces completion of all tasks
|
|
44
|
+
|
|
45
|
+
# Recursively deletes subdirectories
|
|
46
|
+
for subdir in subdirectories:
|
|
47
|
+
delete_directory(subdir)
|
|
48
|
+
|
|
49
|
+
# Removes the now-empty root directory. Since Windows is sometimes slow to release file handles, adds
|
|
50
|
+
# an optional delay step to give Windows time to release file handles.
|
|
51
|
+
max_attempts = 5
|
|
52
|
+
delay_timer = PrecisionTimer("ms")
|
|
53
|
+
for attempt in range(max_attempts):
|
|
54
|
+
# noinspection PyBroadException
|
|
55
|
+
try:
|
|
56
|
+
directory_path.rmdir()
|
|
57
|
+
break # Breaks early if the call succeeds
|
|
58
|
+
except Exception:
|
|
59
|
+
delay_timer.delay_noblock(delay=500, allow_sleep=True) # For each failed attempt, sleeps for 500 ms
|
|
60
|
+
continue
|
|
61
|
+
|
|
62
|
+
|
|
16
63
|
def _transfer_file(source_file: Path, source_directory: Path, destination_directory: Path) -> None:
|
|
17
64
|
"""Copies the input file from the source directory to the destination directory while preserving the file metadata.
|
|
18
65
|
|
|
19
|
-
This
|
|
66
|
+
This worker method is used by the transfer_directory() method to move multiple files in parallel.
|
|
20
67
|
|
|
21
68
|
Notes:
|
|
22
69
|
If the file is found under a hierarchy of subdirectories inside the input source_directory, that hierarchy will
|
|
@@ -32,25 +79,21 @@ def _transfer_file(source_file: Path, source_directory: Path, destination_direct
|
|
|
32
79
|
shutil.copy2(source_file, dest_file)
|
|
33
80
|
|
|
34
81
|
|
|
35
|
-
def transfer_directory(
|
|
82
|
+
def transfer_directory(
|
|
83
|
+
source: Path, destination: Path, num_threads: int = 1, verify_integrity: bool = False, remove_source: bool = False
|
|
84
|
+
) -> None:
|
|
36
85
|
"""Copies the contents of the input directory tree from source to destination while preserving the folder
|
|
37
86
|
structure.
|
|
38
87
|
|
|
39
|
-
This function is used to assemble the experimental data from all remote machines used in the acquisition process on
|
|
40
|
-
the VRPC before the data is preprocessed. It is also used to transfer the preprocessed data from the VRPC to the
|
|
41
|
-
SynologyNAS and the Sun lab BioHPC server.
|
|
42
|
-
|
|
43
88
|
Notes:
|
|
44
89
|
This method recreates the moved directory hierarchy on the destination if the hierarchy does not exist. This is
|
|
45
90
|
done before copying the files.
|
|
46
91
|
|
|
47
|
-
The method executes a multithreading copy operation
|
|
48
|
-
|
|
92
|
+
The method executes a multithreading copy operation and does not by default remove the source data after the
|
|
93
|
+
copy is complete.
|
|
49
94
|
|
|
50
|
-
If the method is configured to verify transferred
|
|
51
|
-
and
|
|
52
|
-
directories contain the 'ax_checksum.txt' file that stores the 'source' directory checksum at the highest level
|
|
53
|
-
of the input directory tree.
|
|
95
|
+
If the method is configured to verify transferred data integrity, it generates xxHash-128 checksum of the data
|
|
96
|
+
before and after the transfer and compares the two checksums to detect data corruption.
|
|
54
97
|
|
|
55
98
|
Args:
|
|
56
99
|
source: The path to the directory that needs to be moved.
|
|
@@ -58,18 +101,32 @@ def transfer_directory(source: Path, destination: Path, num_threads: int = 1, ve
|
|
|
58
101
|
num_threads: The number of threads to use for parallel file transfer. This number should be set depending on the
|
|
59
102
|
type of transfer (local or remote) and is not guaranteed to provide improved transfer performance. For local
|
|
60
103
|
transfers, setting this number above 1 will likely provide a performance boost. For remote transfers using
|
|
61
|
-
a single TCP / IP socket (such as non-multichannel SMB protocol), the number should be set to 1.
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
104
|
+
a single TCP / IP socket (such as non-multichannel SMB protocol), the number should be set to 1. Setting
|
|
105
|
+
this value to a number below 1 instructs the function to use all available CPU cores.
|
|
106
|
+
verify_integrity: Determines whether to perform integrity verification for the transferred files.
|
|
107
|
+
remove_source: Determines whether to remove the source directory and all of its contents after the transfer is
|
|
108
|
+
complete and optionally verified.
|
|
65
109
|
|
|
66
110
|
Raises:
|
|
67
111
|
RuntimeError: If the transferred files do not pass the xxHas3-128 checksum integrity verification.
|
|
68
112
|
"""
|
|
69
113
|
if not source.exists():
|
|
70
|
-
message = f"Unable to
|
|
114
|
+
message = f"Unable to transfer the source directory {source}, as it does not exist."
|
|
71
115
|
console.error(message=message, error=FileNotFoundError)
|
|
72
116
|
|
|
117
|
+
# If the number of threads is less than 1, interprets this as a directive to use all available CPU cores.
|
|
118
|
+
if num_threads < 1:
|
|
119
|
+
cpu_count = os.cpu_count()
|
|
120
|
+
if cpu_count is not None:
|
|
121
|
+
num_threads = cpu_count
|
|
122
|
+
else:
|
|
123
|
+
num_threads = 1
|
|
124
|
+
|
|
125
|
+
# If transfer integrity verification is enabled, but the source directory does not contain the 'ax_checksum.txt'
|
|
126
|
+
# file, checksums the directory before the transfer operation.
|
|
127
|
+
if verify_integrity and not source.joinpath("ax_checksum.txt").exists():
|
|
128
|
+
calculate_directory_checksum(directory=source, batch=False, save_checksum=True)
|
|
129
|
+
|
|
73
130
|
# Ensures the destination root directory exists.
|
|
74
131
|
ensure_directory_exists(destination)
|
|
75
132
|
|
|
@@ -109,11 +166,19 @@ def transfer_directory(source: Path, destination: Path, num_threads: int = 1, ve
|
|
|
109
166
|
# Verifies the integrity of the transferred directory by rerunning xxHash3-128 calculation.
|
|
110
167
|
if verify_integrity:
|
|
111
168
|
destination_checksum = calculate_directory_checksum(directory=destination, batch=False, save_checksum=False)
|
|
112
|
-
with
|
|
169
|
+
with source.joinpath("ax_checksum.txt").open("r") as local_checksum:
|
|
113
170
|
message = (
|
|
114
171
|
f"Checksum mismatch detected when transferring {Path(*source.parts[-6:])} to "
|
|
115
|
-
f"{Path(*destination.parts[-6:])}! The data was likely corrupted in transmission.
|
|
116
|
-
f"required."
|
|
172
|
+
f"{Path(*destination.parts[-6:])}! The data was likely corrupted in transmission."
|
|
117
173
|
)
|
|
118
174
|
if not destination_checksum == local_checksum.readline().strip():
|
|
119
175
|
console.error(message=message, error=RuntimeError)
|
|
176
|
+
|
|
177
|
+
# If necessary, removes the transferred directory from the original location.
|
|
178
|
+
if remove_source:
|
|
179
|
+
message = (
|
|
180
|
+
f"Removing the now-redundant source directory {source} and all of its contents following the successful "
|
|
181
|
+
f"transfer..."
|
|
182
|
+
)
|
|
183
|
+
console.echo(message=message)
|
|
184
|
+
delete_directory(source)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sl-shared-assets
|
|
3
|
-
Version:
|
|
3
|
+
Version: 5.0.1
|
|
4
4
|
Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
|
|
6
6
|
Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
|
|
@@ -689,78 +689,38 @@ Classifier: Operating System :: OS Independent
|
|
|
689
689
|
Classifier: Programming Language :: Python :: 3.11
|
|
690
690
|
Classifier: Programming Language :: Python :: 3.12
|
|
691
691
|
Classifier: Programming Language :: Python :: 3.13
|
|
692
|
+
Classifier: Topic :: Scientific/Engineering
|
|
693
|
+
Classifier: Typing :: Typed
|
|
692
694
|
Requires-Python: >=3.11
|
|
693
|
-
Requires-Dist: appdirs
|
|
694
|
-
Requires-Dist: ataraxis-base-utilities
|
|
695
|
-
Requires-Dist: ataraxis-data-structures
|
|
696
|
-
Requires-Dist: ataraxis-time
|
|
697
|
-
Requires-Dist: click
|
|
698
|
-
Requires-Dist: filelock
|
|
699
|
-
Requires-Dist: natsort
|
|
700
|
-
Requires-Dist: numpy
|
|
701
|
-
Requires-Dist: paramiko
|
|
702
|
-
Requires-Dist: polars
|
|
703
|
-
Requires-Dist: pyarrow
|
|
704
|
-
Requires-Dist: pytz
|
|
705
|
-
Requires-Dist: simple-slurm
|
|
706
|
-
Requires-Dist: tqdm
|
|
707
|
-
Requires-Dist: xxhash
|
|
708
|
-
Provides-Extra: conda
|
|
709
|
-
Requires-Dist: hatchling==1.27.0; extra == 'conda'
|
|
710
|
-
Requires-Dist: importlib-metadata==8.7.0; extra == 'conda'
|
|
711
|
-
Requires-Dist: mypy==1.17.1; extra == 'conda'
|
|
712
|
-
Requires-Dist: ruff==0.12.7; extra == 'conda'
|
|
713
|
-
Requires-Dist: sphinx-autodoc-typehints==3.2.0; extra == 'conda'
|
|
714
|
-
Requires-Dist: sphinx-click==6.0.0; extra == 'conda'
|
|
715
|
-
Requires-Dist: sphinx==8.2.3; extra == 'conda'
|
|
716
|
-
Requires-Dist: tox-uv==1.26.2; extra == 'conda'
|
|
717
|
-
Requires-Dist: tox==4.28.4; extra == 'conda'
|
|
718
|
-
Requires-Dist: twine==6.1.0; extra == 'conda'
|
|
719
|
-
Requires-Dist: types-appdirs==1.4.3.5; extra == 'conda'
|
|
720
|
-
Requires-Dist: types-filelock==3.2.7; extra == 'conda'
|
|
721
|
-
Requires-Dist: types-paramiko==3.5.0.20250801; extra == 'conda'
|
|
722
|
-
Requires-Dist: types-pytz==2025.2.0.20250516; extra == 'conda'
|
|
723
|
-
Requires-Dist: types-tqdm==4.67.0.20250516; extra == 'conda'
|
|
724
|
-
Requires-Dist: uv==0.8.4; extra == 'conda'
|
|
725
|
-
Provides-Extra: condarun
|
|
726
|
-
Requires-Dist: appdirs==1.4.4; extra == 'condarun'
|
|
727
|
-
Requires-Dist: filelock==3.18.0; extra == 'condarun'
|
|
728
|
-
Requires-Dist: natsort==8.4.0; extra == 'condarun'
|
|
729
|
-
Requires-Dist: numpy==2.2.6; extra == 'condarun'
|
|
730
|
-
Requires-Dist: polars==1.32.0; extra == 'condarun'
|
|
731
|
-
Requires-Dist: pyarrow==21.0.0; extra == 'condarun'
|
|
732
|
-
Requires-Dist: pytz==2025.2; extra == 'condarun'
|
|
733
|
-
Requires-Dist: tqdm==4.67.1; extra == 'condarun'
|
|
695
|
+
Requires-Dist: appdirs<2,>=1
|
|
696
|
+
Requires-Dist: ataraxis-base-utilities<4,>=3
|
|
697
|
+
Requires-Dist: ataraxis-data-structures<4,>=3
|
|
698
|
+
Requires-Dist: ataraxis-time<4,>=3
|
|
699
|
+
Requires-Dist: click<9,>=8
|
|
700
|
+
Requires-Dist: filelock<4,>=3
|
|
701
|
+
Requires-Dist: natsort<9,>=8
|
|
702
|
+
Requires-Dist: numpy<3,>=2
|
|
703
|
+
Requires-Dist: paramiko<5,>=4
|
|
704
|
+
Requires-Dist: polars<2,>=1
|
|
705
|
+
Requires-Dist: pyarrow<22,>=21
|
|
706
|
+
Requires-Dist: pytz<2026,>=2025
|
|
707
|
+
Requires-Dist: simple-slurm<1,>=0
|
|
708
|
+
Requires-Dist: tqdm<5,>=4
|
|
709
|
+
Requires-Dist: xxhash<4,>=3
|
|
734
710
|
Provides-Extra: dev
|
|
735
|
-
Requires-Dist: ataraxis-automation
|
|
736
|
-
Requires-Dist:
|
|
737
|
-
Requires-Dist:
|
|
738
|
-
Requires-Dist:
|
|
739
|
-
Requires-Dist:
|
|
740
|
-
Requires-Dist:
|
|
741
|
-
Requires-Dist:
|
|
742
|
-
Requires-Dist:
|
|
743
|
-
Requires-Dist:
|
|
744
|
-
Requires-Dist: sphinx-rtd-theme==3.0.2; extra == 'dev'
|
|
745
|
-
Requires-Dist: sphinx==8.2.3; extra == 'dev'
|
|
746
|
-
Requires-Dist: tox-uv==1.26.2; extra == 'dev'
|
|
747
|
-
Requires-Dist: tox==4.28.4; extra == 'dev'
|
|
748
|
-
Requires-Dist: twine==6.1.0; extra == 'dev'
|
|
749
|
-
Requires-Dist: types-appdirs==1.4.3.5; extra == 'dev'
|
|
750
|
-
Requires-Dist: types-filelock==3.2.7; extra == 'dev'
|
|
751
|
-
Requires-Dist: types-paramiko==3.5.0.20250801; extra == 'dev'
|
|
752
|
-
Requires-Dist: types-pytz==2025.2.0.20250516; extra == 'dev'
|
|
753
|
-
Requires-Dist: types-tqdm==4.67.0.20250516; extra == 'dev'
|
|
754
|
-
Requires-Dist: uv==0.8.4; extra == 'dev'
|
|
755
|
-
Provides-Extra: noconda
|
|
756
|
-
Requires-Dist: ataraxis-automation==5.0.0; extra == 'noconda'
|
|
757
|
-
Requires-Dist: build==1.3.0; extra == 'noconda'
|
|
758
|
-
Requires-Dist: sphinx-rtd-dark-mode==1.3.0; extra == 'noconda'
|
|
759
|
-
Requires-Dist: sphinx-rtd-theme==3.0.2; extra == 'noconda'
|
|
711
|
+
Requires-Dist: ataraxis-automation<7,>=6.0.1; extra == 'dev'
|
|
712
|
+
Requires-Dist: tox-uv<2,>=1; extra == 'dev'
|
|
713
|
+
Requires-Dist: tox<5,>=4; extra == 'dev'
|
|
714
|
+
Requires-Dist: types-appdirs<2,>=1; extra == 'dev'
|
|
715
|
+
Requires-Dist: types-filelock<4,>=3; extra == 'dev'
|
|
716
|
+
Requires-Dist: types-paramiko<5,>=4; extra == 'dev'
|
|
717
|
+
Requires-Dist: types-pytz<2026,>=2025; extra == 'dev'
|
|
718
|
+
Requires-Dist: types-tqdm<5,>=4; extra == 'dev'
|
|
719
|
+
Requires-Dist: uv<1,>=0; extra == 'dev'
|
|
760
720
|
Description-Content-Type: text/markdown
|
|
761
721
|
|
|
762
722
|
# sl-shared-assets
|
|
763
|
-
A
|
|
723
|
+
A python library that provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
|
|
764
724
|
|
|
765
725
|

|
|
766
726
|

|
|
@@ -777,14 +737,11 @@ ___
|
|
|
777
737
|
|
|
778
738
|
Primarily, this library is designed to make the two main Sun lab libraries used for data acquisition
|
|
779
739
|
([sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment)) and processing
|
|
780
|
-
([sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)) independent of each other.
|
|
781
|
-
libraries feature an extensive and largely incompatible set of runtime dependencies. Moreover, having a shared
|
|
782
|
-
repository of classes and tools reused across Sun lab pipelines streamlines the maintenance of these tools.
|
|
740
|
+
([sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)) independent of each other.
|
|
783
741
|
|
|
784
|
-
The library broadly stores two types of assets. First, it stores
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
processing, and provides the API for running various data processing jobs on remote compute servers.
|
|
742
|
+
The library broadly stores two types of assets. First, it stores dataclasses used to save the data acquired in the lab
|
|
743
|
+
and configure data acquisition and processing runtimes. Second, it provides the low-level tools and methods used to
|
|
744
|
+
manage the data at all stages of Sun lab data workflow: acquisition, processing, and analysis.
|
|
788
745
|
|
|
789
746
|
---
|
|
790
747
|
|
|
@@ -798,11 +755,12 @@ processing, and provides the API for running various data processing jobs on rem
|
|
|
798
755
|
- [Authors](#authors)
|
|
799
756
|
- [License](#license)
|
|
800
757
|
- [Acknowledgements](#Acknowledgments)
|
|
758
|
+
|
|
801
759
|
___
|
|
802
760
|
|
|
803
761
|
## Dependencies
|
|
804
762
|
|
|
805
|
-
All software library dependencies are installed automatically as part of library installation.
|
|
763
|
+
All software library dependencies are installed automatically as part of the library installation.
|
|
806
764
|
|
|
807
765
|
---
|
|
808
766
|
|
|
@@ -825,138 +783,23 @@ Use the following command to install the library using pip: ```pip install sl-sh
|
|
|
825
783
|
|
|
826
784
|
## Usage
|
|
827
785
|
|
|
828
|
-
Most library components are intended to be used via other Sun lab libraries.
|
|
829
|
-
|
|
830
|
-
assets for data
|
|
831
|
-
|
|
832
|
-
library ReadMe.
|
|
833
|
-
|
|
834
|
-
The only exception to the note above is the **server.py** package exposed by this library. This package exposes an API
|
|
835
|
-
for running headless and a CLI for running interactive Simple Linux Utility for Resource Management (SLURM)-managed
|
|
836
|
-
jobs on remote compute clusters.
|
|
837
|
-
|
|
838
|
-
### Generating Access Credentials
|
|
839
|
-
|
|
840
|
-
To access any remote server, the user is required to first generate the access credentials. The credentials are stored
|
|
841
|
-
inside the 'server_credentials.yaml' file, which is generated by using the `sl-create-server-credentials` command.
|
|
842
|
-
**Note!** Users are advised to generate this file in a secure (non-shared) location on their local machine.
|
|
843
|
-
|
|
844
|
-
### Running Headless Jobs
|
|
845
|
-
|
|
846
|
-
A headless job is a job that does not require any user interaction during runtime. Currently, all headless jobs in the
|
|
847
|
-
lab rely on pip-installable packages that expose a callable Command-Line Interface to carry out some type of
|
|
848
|
-
data processing. In this regard, **running a headless job is equivalent to calling a CLI command on your local
|
|
849
|
-
machine**, except that the command is executed on a remote compute server. Therefore, the primary purpose of the API
|
|
850
|
-
exposed by this library is to transfer the target command request to the remote server, execute it, and monitor the
|
|
851
|
-
runtime status until it is complete.
|
|
852
|
-
|
|
853
|
-
For example, the [sl-suite2p package](https://github.com/Sun-Lab-NBB/suite2p) maintained in the lab exposes a CLI to
|
|
854
|
-
process 2-Photon data from experiment sessions. During data processing by the
|
|
855
|
-
[sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library, a remote job is sent to the server that uses the CLI
|
|
856
|
-
exposed by the sl-suite2p package to process target session(s).
|
|
857
|
-
|
|
858
|
-
### Creating Jobs
|
|
859
|
-
All remote jobs are sent to the server in the form of an executable *shell* (.sh) script. The script is composed on the
|
|
860
|
-
local machine that uses this library and transferred to a temporary server directory using Secure Shell File
|
|
861
|
-
Transfer Protocol (SFTP). The server is then instructed to evaluate (run) the script using SLURM job manager, via a
|
|
862
|
-
Secure Shell (SSH) session.
|
|
863
|
-
|
|
864
|
-
Broadly, each job consists of three major steps, which correspond to three major sections of the job shell script:
|
|
865
|
-
1. **Setting up the job environment**. Each job script starts with a SLURM job parameter block, which tells SLURM
|
|
866
|
-
what resources (CPUs, GPUs, RAM, etc.) the job requires. When resources become available, SLURM generates a virtual
|
|
867
|
-
environment and runs the rest of the job script in that environment. This forms the basis for using the shared
|
|
868
|
-
compute resources fairly, as SLURM balances resource allocation and the order of job execution for all users.
|
|
869
|
-
2. **Activating the target conda environment**. Currently, all jobs are assumed to use Python libraries to execute the
|
|
870
|
-
intended data processing. Similar to processing data locally, each job expects the remote server to provide a
|
|
871
|
-
Conda environment preconfigured with necessary assets (packages) to run the job. Therefore, each job contains a
|
|
872
|
-
section that activates the user-defined conda environment before running the rest of the job.
|
|
873
|
-
3. **Executing processing**. The final section is typically unique to each job and calls specific CLI commands or runs
|
|
874
|
-
specific Python modules. Since each job is submitted as a shell script, it can do anything a server shell can
|
|
875
|
-
do. Therefore, despite python-centric approach to data processing in the lab, a remote job composed via this library
|
|
876
|
-
can execute ***any*** arbitrary command available to the user on the remove server.
|
|
877
|
-
|
|
878
|
-
Use the *Job* class exposed by this library to compose remote jobs. **Steps 1 and 2** of each job are configured when
|
|
879
|
-
initializing the Job instance, while **step 3** is added via the `add_command()` method of the Job class:
|
|
880
|
-
```
|
|
881
|
-
# First, import the job class
|
|
882
|
-
from pathlib import Path
|
|
883
|
-
from sl_shared_assets import Job
|
|
884
|
-
|
|
885
|
-
# Next, instantiate a new Job object. For example, this job is used to verify the integrity of raw experiment data as
|
|
886
|
-
# it is transferred to the long-term storage destination (server) by the sl-experiment library.
|
|
887
|
-
job = Job(
|
|
888
|
-
job_name="data_integrity_verification",
|
|
889
|
-
output_log=Path("/temp/output.txt"),
|
|
890
|
-
error_log=Path("/temp/errors.txt"),
|
|
891
|
-
working_directory=Path("/temp/test_job"),
|
|
892
|
-
conda_environment="test_environment",
|
|
893
|
-
cpus_to_use=20,
|
|
894
|
-
ram_gb=50,
|
|
895
|
-
time_limit=20,
|
|
896
|
-
)
|
|
897
|
-
|
|
898
|
-
# Finally, add a CLI command call (the actual work to be done by the job). Here, the job calls the
|
|
899
|
-
# 'sl-verify-session' command exposed by the sl-shared-assets library installed in the target environment on the server.
|
|
900
|
-
# Use this method to add commands as you would type them in your local terminal / shell / command line.
|
|
901
|
-
job.add_command(f"sl-verify-session -sp /temp/test_session")
|
|
902
|
-
```
|
|
903
|
-
|
|
904
|
-
### Submitting and Monitoring Jobs:
|
|
905
|
-
To submit the job to the remote server, use a **Server** class instance. This class encapsulates access to the target
|
|
906
|
-
remote compute server and uses the server_credentials.yaml file to determine server access credentials (see above):
|
|
907
|
-
```
|
|
908
|
-
# Initialize the Server class using precreated server credentials file
|
|
909
|
-
server = Server(credentials_path=Path("/temp/server_credentials.yaml"))
|
|
910
|
-
|
|
911
|
-
# Submit the job (generated in the previous code snippet) to the server
|
|
912
|
-
job = server.submit_job(job)
|
|
913
|
-
|
|
914
|
-
# Wait for the server to complete the job
|
|
915
|
-
delay_timer = PrecisionTimer("s")
|
|
916
|
-
while not server.job_complete(job=job):
|
|
917
|
-
delay_timer.delay_noblock(delay=5, allow_sleep=True)
|
|
918
|
-
```
|
|
919
|
-
|
|
920
|
-
**Note!** The Server class only checks whether the job is running on the server, but not the outcome of the job. For
|
|
921
|
-
that, you can either manually check the output and error logs for the job or come up with a programmatic way of
|
|
922
|
-
checking the outcome. All developers are highly advised to study the API documentation for the Job and Server classes
|
|
923
|
-
to use them most effectively.
|
|
924
|
-
|
|
925
|
-
**Critical!** Since running remote jobs is largely equivalent to executing them locally, all users are highly encouraged
|
|
926
|
-
to test their job scripts locally before deploying them server-side. If a script works on a local machine, it is likely
|
|
927
|
-
that the script would behave similarly and work on the server.
|
|
928
|
-
|
|
929
|
-
### Interactive Jobs
|
|
930
|
-
|
|
931
|
-
Interactive jobs are a special extension of the headless job type discussed above. Specifically, an interactive job is
|
|
932
|
-
a headless job, whose only purpose is to **create and maintain a Jupyter lab server** under the SLURM control.
|
|
933
|
-
Specifically, it requests SLURM to set up an isolated environment, starts a Jupyter server in that environment, and
|
|
934
|
-
sends the credentials for the started server back to the user.
|
|
935
|
-
|
|
936
|
-
In essence, this allocates a set of resources the user can use interactively by running various Jupyter notebooks.
|
|
937
|
-
While convenient for certain data analysis cases, this type of jobs has the potential to inefficiently hog server
|
|
938
|
-
resources for prolonged periods of time. Therefore, users are encouraged to only resort to this type of jobs when
|
|
939
|
-
strictly necessary and to minimize the resources and time allocated to running these jobs.
|
|
940
|
-
|
|
941
|
-
To run an interactive job, call the `sl-start-jupyter` CLI command exposed by this library and follow the instructions
|
|
942
|
-
printed to the terminal by the command during runtime.
|
|
943
|
-
|
|
944
|
-
**Critical!** While this command tries to minimize collisions with other users, it is possible that an access port
|
|
945
|
-
collision occurs when multiple users try to instantiate a jupyter server at the same time. If you cannot authenticate
|
|
946
|
-
with the Jupyter server, this likely indicates that the target port was in use and Jupyter automatically incremented the
|
|
947
|
-
port number by 1. In this case, add 1 to your port number and try connecting to that port using the Jupyter credentials
|
|
948
|
-
provided by the command. For example, if your target port was '8888,' try port '8889.'
|
|
786
|
+
Most library components are intended to be used via other Sun lab libraries. For details on using shared
|
|
787
|
+
assets for data acquisition and preprocessing, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment)
|
|
788
|
+
library. For details on using shared assets for data processing and dataset formation, see the
|
|
789
|
+
[sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library.
|
|
949
790
|
|
|
950
|
-
|
|
791
|
+
***Warning!*** End users should not use any component of this library directly or install this library into any Python
|
|
792
|
+
environment. All assets from this library are intended to be used exclusively by developers working on other Sun lab
|
|
793
|
+
libraries.
|
|
951
794
|
|
|
952
795
|
## API Documentation
|
|
953
796
|
|
|
954
|
-
|
|
955
|
-
detailed description of the methods and
|
|
797
|
+
Developers working on integrating sl-shared-assets into other libraries should see the
|
|
798
|
+
[API documentation](https://sl-shared-assets-api-docs.netlify.app/) for the detailed description of the methods and
|
|
799
|
+
classes exposed by components of this library.
|
|
956
800
|
|
|
957
|
-
**Note!** The API documentation includes important information about Command-Line
|
|
958
|
-
library as part of installation into a Python environment.
|
|
959
|
-
documentation to learn how to use library components via the terminal.
|
|
801
|
+
**Note!** The API documentation includes important information about Command-Line Interfaces (CLIs) exposed by this
|
|
802
|
+
library as part of installation into a Python environment.
|
|
960
803
|
|
|
961
804
|
___
|
|
962
805
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
sl_shared_assets/__init__.py,sha256=97oYSDN15AVCv-nk1MzIs_7FfsQP8ENR1wkJYQc6vEY,2677
|
|
2
|
+
sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
sl_shared_assets/command_line_interfaces/__init__.py,sha256=dbcCiuBCe1uqz6d1D-dFpSkRwYP603tEGkc1RbZcI6w,126
|
|
4
|
+
sl_shared_assets/command_line_interfaces/configure.py,sha256=lAWhLzJ7UmdvUZlges9Hg5pnblQuXSDlgzr12y41108,6140
|
|
5
|
+
sl_shared_assets/command_line_interfaces/manage.py,sha256=aGLn_iOrAGTXLFvhrjex4qXo7Crutmp8e8dWQ8YWuy4,8738
|
|
6
|
+
sl_shared_assets/data_classes/__init__.py,sha256=SiOY1JjK_W7lc-EkeZ4O8SCepMPsC5OILUkPe6geU1Q,2119
|
|
7
|
+
sl_shared_assets/data_classes/configuration_data.py,sha256=X6p5V15Verpu6ycO_fJhGOCyNp-IacVLktERc5eYF4k,45394
|
|
8
|
+
sl_shared_assets/data_classes/runtime_data.py,sha256=ruo5KZ1VSfGwSg_gtSyWTbLBFcfG7Az455cKxiUpDoc,17565
|
|
9
|
+
sl_shared_assets/data_classes/session_data.py,sha256=q34M6xDl_ntnvIG_tdDlGrkSXbHR1MzMGjhNBwM2rkk,47038
|
|
10
|
+
sl_shared_assets/data_classes/surgery_data.py,sha256=riwyULMdZ8pDXFLdyNByNSeMk1-mVTo1FcCl5FrThas,7495
|
|
11
|
+
sl_shared_assets/server/__init__.py,sha256=6REoeUvHW39V1Uwal_Af4uZ8J6UYDmR2N-saZoau30E,869
|
|
12
|
+
sl_shared_assets/server/job.py,sha256=92nCVaLYqEJH7-xaS0bZmIVoGp-nJ5gh6dnwyRyZ9gI,19406
|
|
13
|
+
sl_shared_assets/server/pipeline.py,sha256=0Peuq499BxKA3GaebQyhl_9VLepuaJQuwwt2vuM5ULg,33503
|
|
14
|
+
sl_shared_assets/server/server.py,sha256=BUELEQbO-sx-9E2epGURCYSGN-I9WAD2dRETMWEPcRU,36483
|
|
15
|
+
sl_shared_assets/tools/__init__.py,sha256=Pf8_MqZOLiityR112Txkermxf1bBLHYdEdw-raeY9Fc,737
|
|
16
|
+
sl_shared_assets/tools/packaging_tools.py,sha256=hvOrdRwQCoe199G_XHhIpbKtIx-I0ad9O4SLZwhgj7U,7035
|
|
17
|
+
sl_shared_assets/tools/project_management_tools.py,sha256=AcZpkzcXyAv4aBn-LIXawQgd7le7-vtZ9dl5MvMFmkg,40365
|
|
18
|
+
sl_shared_assets/tools/transfer_tools.py,sha256=6lwk4zyHWWB0QeWQageiQ1pPIGUV0IhBNNzsbgQ_qH4,9026
|
|
19
|
+
sl_shared_assets-5.0.1.dist-info/METADATA,sha256=kH_5OOIkoJbGKbiBoAG8_Ghg5iSisLLsxAV3NjTPY4E,46884
|
|
20
|
+
sl_shared_assets-5.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
21
|
+
sl_shared_assets-5.0.1.dist-info/entry_points.txt,sha256=19NzFPG5CcW-4RhEacTcX3J2TbrvmjOE4tlLCH2O5wI,161
|
|
22
|
+
sl_shared_assets-5.0.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
23
|
+
sl_shared_assets-5.0.1.dist-info/RECORD,,
|
sl_shared_assets/__init__.pyi
DELETED
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
from .tools import (
|
|
2
|
-
ProjectManifest as ProjectManifest,
|
|
3
|
-
resolve_p53_marker as resolve_p53_marker,
|
|
4
|
-
transfer_directory as transfer_directory,
|
|
5
|
-
generate_project_manifest as generate_project_manifest,
|
|
6
|
-
calculate_directory_checksum as calculate_directory_checksum,
|
|
7
|
-
)
|
|
8
|
-
from .server import (
|
|
9
|
-
Job as Job,
|
|
10
|
-
Server as Server,
|
|
11
|
-
JupyterJob as JupyterJob,
|
|
12
|
-
ServerCredentials as ServerCredentials,
|
|
13
|
-
)
|
|
14
|
-
from .data_classes import (
|
|
15
|
-
RawData as RawData,
|
|
16
|
-
DrugData as DrugData,
|
|
17
|
-
ImplantData as ImplantData,
|
|
18
|
-
SessionData as SessionData,
|
|
19
|
-
SubjectData as SubjectData,
|
|
20
|
-
SurgeryData as SurgeryData,
|
|
21
|
-
SessionTypes as SessionTypes,
|
|
22
|
-
InjectionData as InjectionData,
|
|
23
|
-
ProcedureData as ProcedureData,
|
|
24
|
-
ProcessedData as ProcessedData,
|
|
25
|
-
MesoscopePaths as MesoscopePaths,
|
|
26
|
-
ZaberPositions as ZaberPositions,
|
|
27
|
-
ExperimentState as ExperimentState,
|
|
28
|
-
ExperimentTrial as ExperimentTrial,
|
|
29
|
-
MesoscopeCameras as MesoscopeCameras,
|
|
30
|
-
TrackerFileNames as TrackerFileNames,
|
|
31
|
-
ProcessingTracker as ProcessingTracker,
|
|
32
|
-
AcquisitionSystems as AcquisitionSystems,
|
|
33
|
-
MesoscopePositions as MesoscopePositions,
|
|
34
|
-
RunTrainingDescriptor as RunTrainingDescriptor,
|
|
35
|
-
LickTrainingDescriptor as LickTrainingDescriptor,
|
|
36
|
-
MesoscopeHardwareState as MesoscopeHardwareState,
|
|
37
|
-
WindowCheckingDescriptor as WindowCheckingDescriptor,
|
|
38
|
-
MesoscopeMicroControllers as MesoscopeMicroControllers,
|
|
39
|
-
MesoscopeAdditionalFirmware as MesoscopeAdditionalFirmware,
|
|
40
|
-
MesoscopeSystemConfiguration as MesoscopeSystemConfiguration,
|
|
41
|
-
MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
|
|
42
|
-
MesoscopeExperimentConfiguration as MesoscopeExperimentConfiguration,
|
|
43
|
-
generate_manager_id as generate_manager_id,
|
|
44
|
-
get_processing_tracker as get_processing_tracker,
|
|
45
|
-
get_system_configuration_data as get_system_configuration_data,
|
|
46
|
-
set_system_configuration_file as set_system_configuration_file,
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
__all__ = [
|
|
50
|
-
"Server",
|
|
51
|
-
"ServerCredentials",
|
|
52
|
-
"Job",
|
|
53
|
-
"JupyterJob",
|
|
54
|
-
"DrugData",
|
|
55
|
-
"ImplantData",
|
|
56
|
-
"SessionData",
|
|
57
|
-
"RawData",
|
|
58
|
-
"ProcessedData",
|
|
59
|
-
"SubjectData",
|
|
60
|
-
"SurgeryData",
|
|
61
|
-
"InjectionData",
|
|
62
|
-
"ProcessingTracker",
|
|
63
|
-
"ProcedureData",
|
|
64
|
-
"ZaberPositions",
|
|
65
|
-
"ExperimentState",
|
|
66
|
-
"MesoscopePositions",
|
|
67
|
-
"MesoscopeHardwareState",
|
|
68
|
-
"RunTrainingDescriptor",
|
|
69
|
-
"LickTrainingDescriptor",
|
|
70
|
-
"MesoscopeExperimentConfiguration",
|
|
71
|
-
"MesoscopeExperimentDescriptor",
|
|
72
|
-
"MesoscopeSystemConfiguration",
|
|
73
|
-
"MesoscopePaths",
|
|
74
|
-
"MesoscopeCameras",
|
|
75
|
-
"MesoscopeMicroControllers",
|
|
76
|
-
"MesoscopeAdditionalFirmware",
|
|
77
|
-
"get_system_configuration_data",
|
|
78
|
-
"set_system_configuration_file",
|
|
79
|
-
"ExperimentTrial",
|
|
80
|
-
"SessionTypes",
|
|
81
|
-
"AcquisitionSystems",
|
|
82
|
-
"WindowCheckingDescriptor",
|
|
83
|
-
"get_processing_tracker",
|
|
84
|
-
"generate_manager_id",
|
|
85
|
-
"TrackerFileNames",
|
|
86
|
-
"ProjectManifest",
|
|
87
|
-
"resolve_p53_marker",
|
|
88
|
-
"transfer_directory",
|
|
89
|
-
"calculate_directory_checksum",
|
|
90
|
-
"generate_project_manifest",
|
|
91
|
-
]
|