sl-shared-assets 4.0.1__py3-none-any.whl → 5.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sl-shared-assets might be problematic. Click here for more details.

Files changed (39) hide show
  1. sl_shared_assets/__init__.py +48 -41
  2. sl_shared_assets/command_line_interfaces/__init__.py +3 -0
  3. sl_shared_assets/command_line_interfaces/configure.py +173 -0
  4. sl_shared_assets/command_line_interfaces/manage.py +226 -0
  5. sl_shared_assets/data_classes/__init__.py +33 -32
  6. sl_shared_assets/data_classes/configuration_data.py +267 -79
  7. sl_shared_assets/data_classes/session_data.py +226 -289
  8. sl_shared_assets/server/__init__.py +24 -4
  9. sl_shared_assets/server/job.py +6 -7
  10. sl_shared_assets/server/pipeline.py +585 -0
  11. sl_shared_assets/server/server.py +57 -25
  12. sl_shared_assets/tools/__init__.py +9 -8
  13. sl_shared_assets/tools/packaging_tools.py +14 -25
  14. sl_shared_assets/tools/project_management_tools.py +602 -523
  15. sl_shared_assets/tools/transfer_tools.py +88 -23
  16. {sl_shared_assets-4.0.1.dist-info → sl_shared_assets-5.0.1.dist-info}/METADATA +46 -203
  17. sl_shared_assets-5.0.1.dist-info/RECORD +23 -0
  18. sl_shared_assets-5.0.1.dist-info/entry_points.txt +3 -0
  19. sl_shared_assets/__init__.pyi +0 -91
  20. sl_shared_assets/cli.py +0 -501
  21. sl_shared_assets/cli.pyi +0 -106
  22. sl_shared_assets/data_classes/__init__.pyi +0 -75
  23. sl_shared_assets/data_classes/configuration_data.pyi +0 -235
  24. sl_shared_assets/data_classes/runtime_data.pyi +0 -157
  25. sl_shared_assets/data_classes/session_data.pyi +0 -379
  26. sl_shared_assets/data_classes/surgery_data.pyi +0 -89
  27. sl_shared_assets/server/__init__.pyi +0 -11
  28. sl_shared_assets/server/job.pyi +0 -205
  29. sl_shared_assets/server/server.pyi +0 -298
  30. sl_shared_assets/tools/__init__.pyi +0 -19
  31. sl_shared_assets/tools/ascension_tools.py +0 -265
  32. sl_shared_assets/tools/ascension_tools.pyi +0 -68
  33. sl_shared_assets/tools/packaging_tools.pyi +0 -58
  34. sl_shared_assets/tools/project_management_tools.pyi +0 -239
  35. sl_shared_assets/tools/transfer_tools.pyi +0 -53
  36. sl_shared_assets-4.0.1.dist-info/RECORD +0 -36
  37. sl_shared_assets-4.0.1.dist-info/entry_points.txt +0 -7
  38. {sl_shared_assets-4.0.1.dist-info → sl_shared_assets-5.0.1.dist-info}/WHEEL +0 -0
  39. {sl_shared_assets-4.0.1.dist-info → sl_shared_assets-5.0.1.dist-info}/licenses/LICENSE +0 -0
@@ -1,22 +1,69 @@
1
- """This module provides methods for moving session runtime data between the local machine, the ScanImage (Mesoscope) PC,
2
- the Synology NAS drive, and the lab BioHPC server. All methods in this module expect that the destinations and sources
3
- are mounted on the host file-system via the SMB or an equivalent protocol.
1
+ """This module provides tools for moving the data between destinations within or across machines (PCs). All methods in
2
+ this module expect that the destinations and sources are mounted on the host-machine file-system via the SMB or an
3
+ equivalent protocol.
4
4
  """
5
5
 
6
+ import os
6
7
  import shutil
7
8
  from pathlib import Path
8
9
  from concurrent.futures import ThreadPoolExecutor, as_completed
9
10
 
10
11
  from tqdm import tqdm
12
+ from ataraxis_time import PrecisionTimer
11
13
  from ataraxis_base_utilities import console, ensure_directory_exists
12
14
 
13
15
  from .packaging_tools import calculate_directory_checksum
14
16
 
15
17
 
18
+ def delete_directory(directory_path: Path) -> None:
19
+ """Removes the input directory and all its subdirectories using parallel processing.
20
+
21
+ This function outperforms default approaches like subprocess call with rm -rf and shutil rmtree for directories with
22
+ a comparably small number of large files. For example, this is the case for the mesoscope frame directories, which
23
+ are deleted ~6 times faster with this method over sh.rmtree. Potentially, it may also outperform these approaches
24
+ for all comparatively shallow directories.
25
+
26
+ Notes:
27
+ This function is often combined with the transfer_directory function to remove the source directory after
28
+ it has been transferred.
29
+
30
+ Args:
31
+ directory_path: The path to the directory to delete.
32
+ """
33
+ # Checks if the directory exists and, if not, aborts early
34
+ if not directory_path.exists():
35
+ return
36
+
37
+ # Builds the list of files and directories inside the input directory using Path
38
+ files = [p for p in directory_path.iterdir() if p.is_file()]
39
+ subdirectories = [p for p in directory_path.iterdir() if p.is_dir()]
40
+
41
+ # Deletes files in parallel
42
+ with ThreadPoolExecutor() as executor:
43
+ list(executor.map(os.unlink, files)) # Forces completion of all tasks
44
+
45
+ # Recursively deletes subdirectories
46
+ for subdir in subdirectories:
47
+ delete_directory(subdir)
48
+
49
+ # Removes the now-empty root directory. Since Windows is sometimes slow to release file handles, adds
50
+ # an optional delay step to give Windows time to release file handles.
51
+ max_attempts = 5
52
+ delay_timer = PrecisionTimer("ms")
53
+ for attempt in range(max_attempts):
54
+ # noinspection PyBroadException
55
+ try:
56
+ directory_path.rmdir()
57
+ break # Breaks early if the call succeeds
58
+ except Exception:
59
+ delay_timer.delay_noblock(delay=500, allow_sleep=True) # For each failed attempt, sleeps for 500 ms
60
+ continue
61
+
62
+
16
63
  def _transfer_file(source_file: Path, source_directory: Path, destination_directory: Path) -> None:
17
64
  """Copies the input file from the source directory to the destination directory while preserving the file metadata.
18
65
 
19
- This is a worker method used by the transfer_directory() method to move multiple files in parallel.
66
+ This worker method is used by the transfer_directory() method to move multiple files in parallel.
20
67
 
21
68
  Notes:
22
69
  If the file is found under a hierarchy of subdirectories inside the input source_directory, that hierarchy will
@@ -32,25 +79,21 @@ def _transfer_file(source_file: Path, source_directory: Path, destination_direct
32
79
  shutil.copy2(source_file, dest_file)
33
80
 
34
81
 
35
- def transfer_directory(source: Path, destination: Path, num_threads: int = 1, verify_integrity: bool = True) -> None:
82
+ def transfer_directory(
83
+ source: Path, destination: Path, num_threads: int = 1, verify_integrity: bool = False, remove_source: bool = False
84
+ ) -> None:
36
85
  """Copies the contents of the input directory tree from source to destination while preserving the folder
37
86
  structure.
38
87
 
39
- This function is used to assemble the experimental data from all remote machines used in the acquisition process on
40
- the VRPC before the data is preprocessed. It is also used to transfer the preprocessed data from the VRPC to the
41
- SynologyNAS and the Sun lab BioHPC server.
42
-
43
88
  Notes:
44
89
  This method recreates the moved directory hierarchy on the destination if the hierarchy does not exist. This is
45
90
  done before copying the files.
46
91
 
47
- The method executes a multithreading copy operation. It does not clean up the source files. That job is handed
48
- to the specific preprocessing function from the sl_experiment or sl-forgery libraries that call this function.
92
+ The method executes a multithreading copy operation and does not by default remove the source data after the
93
+ copy is complete.
49
94
 
50
- If the method is configured to verify transferred file integrity, it reruns the xxHash3-128 checksum calculation
51
- and compares the returned checksum to the one stored in the source directory. The method assumes that all input
52
- directories contain the 'ax_checksum.txt' file that stores the 'source' directory checksum at the highest level
53
- of the input directory tree.
95
+ If the method is configured to verify transferred data integrity, it generates xxHash-128 checksum of the data
96
+ before and after the transfer and compares the two checksums to detect data corruption.
54
97
 
55
98
  Args:
56
99
  source: The path to the directory that needs to be moved.
@@ -58,18 +101,32 @@ def transfer_directory(source: Path, destination: Path, num_threads: int = 1, ve
58
101
  num_threads: The number of threads to use for parallel file transfer. This number should be set depending on the
59
102
  type of transfer (local or remote) and is not guaranteed to provide improved transfer performance. For local
60
103
  transfers, setting this number above 1 will likely provide a performance boost. For remote transfers using
61
- a single TCP / IP socket (such as non-multichannel SMB protocol), the number should be set to 1.
62
- verify_integrity: Determines whether to perform integrity verification for the transferred files. Note,
63
- integrity verification is a time-consuming process and generally would not be a concern for most runtimes.
64
- Therefore, it is often fine to disable this option to optimize method runtime speed.
104
+ a single TCP / IP socket (such as non-multichannel SMB protocol), the number should be set to 1. Setting
105
+ this value to a number below 1 instructs the function to use all available CPU cores.
106
+ verify_integrity: Determines whether to perform integrity verification for the transferred files.
107
+ remove_source: Determines whether to remove the source directory and all of its contents after the transfer is
108
+ complete and optionally verified.
65
109
 
66
110
  Raises:
67
111
  RuntimeError: If the transferred files do not pass the xxHas3-128 checksum integrity verification.
68
112
  """
69
113
  if not source.exists():
70
- message = f"Unable to move the directory {source}, as it does not exist."
114
+ message = f"Unable to transfer the source directory {source}, as it does not exist."
71
115
  console.error(message=message, error=FileNotFoundError)
72
116
 
117
+ # If the number of threads is less than 1, interprets this as a directive to use all available CPU cores.
118
+ if num_threads < 1:
119
+ cpu_count = os.cpu_count()
120
+ if cpu_count is not None:
121
+ num_threads = cpu_count
122
+ else:
123
+ num_threads = 1
124
+
125
+ # If transfer integrity verification is enabled, but the source directory does not contain the 'ax_checksum.txt'
126
+ # file, checksums the directory before the transfer operation.
127
+ if verify_integrity and not source.joinpath("ax_checksum.txt").exists():
128
+ calculate_directory_checksum(directory=source, batch=False, save_checksum=True)
129
+
73
130
  # Ensures the destination root directory exists.
74
131
  ensure_directory_exists(destination)
75
132
 
@@ -109,11 +166,19 @@ def transfer_directory(source: Path, destination: Path, num_threads: int = 1, ve
109
166
  # Verifies the integrity of the transferred directory by rerunning xxHash3-128 calculation.
110
167
  if verify_integrity:
111
168
  destination_checksum = calculate_directory_checksum(directory=destination, batch=False, save_checksum=False)
112
- with open(file=source.joinpath("ax_checksum.txt"), mode="r") as local_checksum:
169
+ with source.joinpath("ax_checksum.txt").open("r") as local_checksum:
113
170
  message = (
114
171
  f"Checksum mismatch detected when transferring {Path(*source.parts[-6:])} to "
115
- f"{Path(*destination.parts[-6:])}! The data was likely corrupted in transmission. User intervention "
116
- f"required."
172
+ f"{Path(*destination.parts[-6:])}! The data was likely corrupted in transmission."
117
173
  )
118
174
  if not destination_checksum == local_checksum.readline().strip():
119
175
  console.error(message=message, error=RuntimeError)
176
+
177
+ # If necessary, removes the transferred directory from the original location.
178
+ if remove_source:
179
+ message = (
180
+ f"Removing the now-redundant source directory {source} and all of its contents following the successful "
181
+ f"transfer..."
182
+ )
183
+ console.echo(message=message)
184
+ delete_directory(source)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sl-shared-assets
3
- Version: 4.0.1
3
+ Version: 5.0.1
4
4
  Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
5
5
  Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
6
6
  Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
@@ -689,78 +689,38 @@ Classifier: Operating System :: OS Independent
689
689
  Classifier: Programming Language :: Python :: 3.11
690
690
  Classifier: Programming Language :: Python :: 3.12
691
691
  Classifier: Programming Language :: Python :: 3.13
692
+ Classifier: Topic :: Scientific/Engineering
693
+ Classifier: Typing :: Typed
692
694
  Requires-Python: >=3.11
693
- Requires-Dist: appdirs==1.4.4
694
- Requires-Dist: ataraxis-base-utilities==3.1.0
695
- Requires-Dist: ataraxis-data-structures==3.1.1
696
- Requires-Dist: ataraxis-time==3.0.0
697
- Requires-Dist: click==8.2.1
698
- Requires-Dist: filelock==3.18.0
699
- Requires-Dist: natsort==8.4.0
700
- Requires-Dist: numpy==2.2.6
701
- Requires-Dist: paramiko==4.0.0
702
- Requires-Dist: polars==1.32.0
703
- Requires-Dist: pyarrow==21.0.0
704
- Requires-Dist: pytz==2025.2
705
- Requires-Dist: simple-slurm==0.3.6
706
- Requires-Dist: tqdm==4.67.1
707
- Requires-Dist: xxhash==3.5.0
708
- Provides-Extra: conda
709
- Requires-Dist: hatchling==1.27.0; extra == 'conda'
710
- Requires-Dist: importlib-metadata==8.7.0; extra == 'conda'
711
- Requires-Dist: mypy==1.17.1; extra == 'conda'
712
- Requires-Dist: ruff==0.12.7; extra == 'conda'
713
- Requires-Dist: sphinx-autodoc-typehints==3.2.0; extra == 'conda'
714
- Requires-Dist: sphinx-click==6.0.0; extra == 'conda'
715
- Requires-Dist: sphinx==8.2.3; extra == 'conda'
716
- Requires-Dist: tox-uv==1.26.2; extra == 'conda'
717
- Requires-Dist: tox==4.28.4; extra == 'conda'
718
- Requires-Dist: twine==6.1.0; extra == 'conda'
719
- Requires-Dist: types-appdirs==1.4.3.5; extra == 'conda'
720
- Requires-Dist: types-filelock==3.2.7; extra == 'conda'
721
- Requires-Dist: types-paramiko==3.5.0.20250801; extra == 'conda'
722
- Requires-Dist: types-pytz==2025.2.0.20250516; extra == 'conda'
723
- Requires-Dist: types-tqdm==4.67.0.20250516; extra == 'conda'
724
- Requires-Dist: uv==0.8.4; extra == 'conda'
725
- Provides-Extra: condarun
726
- Requires-Dist: appdirs==1.4.4; extra == 'condarun'
727
- Requires-Dist: filelock==3.18.0; extra == 'condarun'
728
- Requires-Dist: natsort==8.4.0; extra == 'condarun'
729
- Requires-Dist: numpy==2.2.6; extra == 'condarun'
730
- Requires-Dist: polars==1.32.0; extra == 'condarun'
731
- Requires-Dist: pyarrow==21.0.0; extra == 'condarun'
732
- Requires-Dist: pytz==2025.2; extra == 'condarun'
733
- Requires-Dist: tqdm==4.67.1; extra == 'condarun'
695
+ Requires-Dist: appdirs<2,>=1
696
+ Requires-Dist: ataraxis-base-utilities<4,>=3
697
+ Requires-Dist: ataraxis-data-structures<4,>=3
698
+ Requires-Dist: ataraxis-time<4,>=3
699
+ Requires-Dist: click<9,>=8
700
+ Requires-Dist: filelock<4,>=3
701
+ Requires-Dist: natsort<9,>=8
702
+ Requires-Dist: numpy<3,>=2
703
+ Requires-Dist: paramiko<5,>=4
704
+ Requires-Dist: polars<2,>=1
705
+ Requires-Dist: pyarrow<22,>=21
706
+ Requires-Dist: pytz<2026,>=2025
707
+ Requires-Dist: simple-slurm<1,>=0
708
+ Requires-Dist: tqdm<5,>=4
709
+ Requires-Dist: xxhash<4,>=3
734
710
  Provides-Extra: dev
735
- Requires-Dist: ataraxis-automation==5.0.0; extra == 'dev'
736
- Requires-Dist: build==1.3.0; extra == 'dev'
737
- Requires-Dist: hatchling==1.27.0; extra == 'dev'
738
- Requires-Dist: importlib-metadata==8.7.0; extra == 'dev'
739
- Requires-Dist: mypy==1.17.1; extra == 'dev'
740
- Requires-Dist: ruff==0.12.7; extra == 'dev'
741
- Requires-Dist: sphinx-autodoc-typehints==3.2.0; extra == 'dev'
742
- Requires-Dist: sphinx-click==6.0.0; extra == 'dev'
743
- Requires-Dist: sphinx-rtd-dark-mode==1.3.0; extra == 'dev'
744
- Requires-Dist: sphinx-rtd-theme==3.0.2; extra == 'dev'
745
- Requires-Dist: sphinx==8.2.3; extra == 'dev'
746
- Requires-Dist: tox-uv==1.26.2; extra == 'dev'
747
- Requires-Dist: tox==4.28.4; extra == 'dev'
748
- Requires-Dist: twine==6.1.0; extra == 'dev'
749
- Requires-Dist: types-appdirs==1.4.3.5; extra == 'dev'
750
- Requires-Dist: types-filelock==3.2.7; extra == 'dev'
751
- Requires-Dist: types-paramiko==3.5.0.20250801; extra == 'dev'
752
- Requires-Dist: types-pytz==2025.2.0.20250516; extra == 'dev'
753
- Requires-Dist: types-tqdm==4.67.0.20250516; extra == 'dev'
754
- Requires-Dist: uv==0.8.4; extra == 'dev'
755
- Provides-Extra: noconda
756
- Requires-Dist: ataraxis-automation==5.0.0; extra == 'noconda'
757
- Requires-Dist: build==1.3.0; extra == 'noconda'
758
- Requires-Dist: sphinx-rtd-dark-mode==1.3.0; extra == 'noconda'
759
- Requires-Dist: sphinx-rtd-theme==3.0.2; extra == 'noconda'
711
+ Requires-Dist: ataraxis-automation<7,>=6.0.1; extra == 'dev'
712
+ Requires-Dist: tox-uv<2,>=1; extra == 'dev'
713
+ Requires-Dist: tox<5,>=4; extra == 'dev'
714
+ Requires-Dist: types-appdirs<2,>=1; extra == 'dev'
715
+ Requires-Dist: types-filelock<4,>=3; extra == 'dev'
716
+ Requires-Dist: types-paramiko<5,>=4; extra == 'dev'
717
+ Requires-Dist: types-pytz<2026,>=2025; extra == 'dev'
718
+ Requires-Dist: types-tqdm<5,>=4; extra == 'dev'
719
+ Requires-Dist: uv<1,>=0; extra == 'dev'
760
720
  Description-Content-Type: text/markdown
761
721
 
762
722
  # sl-shared-assets
763
- A Python library that stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
723
+ A python library that provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
764
724
 
765
725
  ![PyPI - Version](https://img.shields.io/pypi/v/sl-shared-assets)
766
726
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sl-shared-assets)
@@ -777,14 +737,11 @@ ___
777
737
 
778
738
  Primarily, this library is designed to make the two main Sun lab libraries used for data acquisition
779
739
  ([sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment)) and processing
780
- ([sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)) independent of each other. This is beneficial, as both
781
- libraries feature an extensive and largely incompatible set of runtime dependencies. Moreover, having a shared
782
- repository of classes and tools reused across Sun lab pipelines streamlines the maintenance of these tools.
740
+ ([sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)) independent of each other.
783
741
 
784
- The library broadly stores two types of assets. First, it stores various dataclasses used to save the data acquired
785
- during experiments in the lab and the dataclasses used to configure data acquisition and processing runtimes. Secondly,
786
- it stores the tools used to safely move the data between the machines (computers) used in the data acquisition and
787
- processing, and provides the API for running various data processing jobs on remote compute servers.
742
+ The library broadly stores two types of assets. First, it stores dataclasses used to save the data acquired in the lab
743
+ and configure data acquisition and processing runtimes. Second, it provides the low-level tools and methods used to
744
+ manage the data at all stages of Sun lab data workflow: acquisition, processing, and analysis.
788
745
 
789
746
  ---
790
747
 
@@ -798,11 +755,12 @@ processing, and provides the API for running various data processing jobs on rem
798
755
  - [Authors](#authors)
799
756
  - [License](#license)
800
757
  - [Acknowledgements](#Acknowledgments)
758
+
801
759
  ___
802
760
 
803
761
  ## Dependencies
804
762
 
805
- All software library dependencies are installed automatically as part of library installation.
763
+ All software library dependencies are installed automatically as part of the library installation.
806
764
 
807
765
  ---
808
766
 
@@ -825,138 +783,23 @@ Use the following command to install the library using pip: ```pip install sl-sh
825
783
 
826
784
  ## Usage
827
785
 
828
- Most library components are intended to be used via other Sun lab libraries. Developers should study the API and CLI
829
- documentation below to learn how to use library components in other Sun lab libraries. For notes on using shared
830
- assets for data acquisition, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment) library ReadMe.
831
- For notes on using shared assets for data processing, see the [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)
832
- library ReadMe.
833
-
834
- The only exception to the note above is the **server.py** package exposed by this library. This package exposes an API
835
- for running headless and a CLI for running interactive Simple Linux Utility for Resource Management (SLURM)-managed
836
- jobs on remote compute clusters.
837
-
838
- ### Generating Access Credentials
839
-
840
- To access any remote server, the user is required to first generate the access credentials. The credentials are stored
841
- inside the 'server_credentials.yaml' file, which is generated by using the `sl-create-server-credentials` command.
842
- **Note!** Users are advised to generate this file in a secure (non-shared) location on their local machine.
843
-
844
- ### Running Headless Jobs
845
-
846
- A headless job is a job that does not require any user interaction during runtime. Currently, all headless jobs in the
847
- lab rely on pip-installable packages that expose a callable Command-Line Interface to carry out some type of
848
- data processing. In this regard, **running a headless job is equivalent to calling a CLI command on your local
849
- machine**, except that the command is executed on a remote compute server. Therefore, the primary purpose of the API
850
- exposed by this library is to transfer the target command request to the remote server, execute it, and monitor the
851
- runtime status until it is complete.
852
-
853
- For example, the [sl-suite2p package](https://github.com/Sun-Lab-NBB/suite2p) maintained in the lab exposes a CLI to
854
- process 2-Photon data from experiment sessions. During data processing by the
855
- [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library, a remote job is sent to the server that uses the CLI
856
- exposed by the sl-suite2p package to process target session(s).
857
-
858
- ### Creating Jobs
859
- All remote jobs are sent to the server in the form of an executable *shell* (.sh) script. The script is composed on the
860
- local machine that uses this library and transferred to a temporary server directory using Secure Shell File
861
- Transfer Protocol (SFTP). The server is then instructed to evaluate (run) the script using SLURM job manager, via a
862
- Secure Shell (SSH) session.
863
-
864
- Broadly, each job consists of three major steps, which correspond to three major sections of the job shell script:
865
- 1. **Setting up the job environment**. Each job script starts with a SLURM job parameter block, which tells SLURM
866
- what resources (CPUs, GPUs, RAM, etc.) the job requires. When resources become available, SLURM generates a virtual
867
- environment and runs the rest of the job script in that environment. This forms the basis for using the shared
868
- compute resources fairly, as SLURM balances resource allocation and the order of job execution for all users.
869
- 2. **Activating the target conda environment**. Currently, all jobs are assumed to use Python libraries to execute the
870
- intended data processing. Similar to processing data locally, each job expects the remote server to provide a
871
- Conda environment preconfigured with necessary assets (packages) to run the job. Therefore, each job contains a
872
- section that activates the user-defined conda environment before running the rest of the job.
873
- 3. **Executing processing**. The final section is typically unique to each job and calls specific CLI commands or runs
874
- specific Python modules. Since each job is submitted as a shell script, it can do anything a server shell can
875
- do. Therefore, despite python-centric approach to data processing in the lab, a remote job composed via this library
876
- can execute ***any*** arbitrary command available to the user on the remove server.
877
-
878
- Use the *Job* class exposed by this library to compose remote jobs. **Steps 1 and 2** of each job are configured when
879
- initializing the Job instance, while **step 3** is added via the `add_command()` method of the Job class:
880
- ```
881
- # First, import the job class
882
- from pathlib import Path
883
- from sl_shared_assets import Job
884
-
885
- # Next, instantiate a new Job object. For example, this job is used to verify the integrity of raw experiment data as
886
- # it is transferred to the long-term storage destination (server) by the sl-experiment library.
887
- job = Job(
888
- job_name="data_integrity_verification",
889
- output_log=Path("/temp/output.txt"),
890
- error_log=Path("/temp/errors.txt"),
891
- working_directory=Path("/temp/test_job"),
892
- conda_environment="test_environment",
893
- cpus_to_use=20,
894
- ram_gb=50,
895
- time_limit=20,
896
- )
897
-
898
- # Finally, add a CLI command call (the actual work to be done by the job). Here, the job calls the
899
- # 'sl-verify-session' command exposed by the sl-shared-assets library installed in the target environment on the server.
900
- # Use this method to add commands as you would type them in your local terminal / shell / command line.
901
- job.add_command(f"sl-verify-session -sp /temp/test_session")
902
- ```
903
-
904
- ### Submitting and Monitoring Jobs:
905
- To submit the job to the remote server, use a **Server** class instance. This class encapsulates access to the target
906
- remote compute server and uses the server_credentials.yaml file to determine server access credentials (see above):
907
- ```
908
- # Initialize the Server class using precreated server credentials file
909
- server = Server(credentials_path=Path("/temp/server_credentials.yaml"))
910
-
911
- # Submit the job (generated in the previous code snippet) to the server
912
- job = server.submit_job(job)
913
-
914
- # Wait for the server to complete the job
915
- delay_timer = PrecisionTimer("s")
916
- while not server.job_complete(job=job):
917
- delay_timer.delay_noblock(delay=5, allow_sleep=True)
918
- ```
919
-
920
- **Note!** The Server class only checks whether the job is running on the server, but not the outcome of the job. For
921
- that, you can either manually check the output and error logs for the job or come up with a programmatic way of
922
- checking the outcome. All developers are highly advised to study the API documentation for the Job and Server classes
923
- to use them most effectively.
924
-
925
- **Critical!** Since running remote jobs is largely equivalent to executing them locally, all users are highly encouraged
926
- to test their job scripts locally before deploying them server-side. If a script works on a local machine, it is likely
927
- that the script would behave similarly and work on the server.
928
-
929
- ### Interactive Jobs
930
-
931
- Interactive jobs are a special extension of the headless job type discussed above. Specifically, an interactive job is
932
- a headless job, whose only purpose is to **create and maintain a Jupyter lab server** under the SLURM control.
933
- Specifically, it requests SLURM to set up an isolated environment, starts a Jupyter server in that environment, and
934
- sends the credentials for the started server back to the user.
935
-
936
- In essence, this allocates a set of resources the user can use interactively by running various Jupyter notebooks.
937
- While convenient for certain data analysis cases, this type of jobs has the potential to inefficiently hog server
938
- resources for prolonged periods of time. Therefore, users are encouraged to only resort to this type of jobs when
939
- strictly necessary and to minimize the resources and time allocated to running these jobs.
940
-
941
- To run an interactive job, call the `sl-start-jupyter` CLI command exposed by this library and follow the instructions
942
- printed to the terminal by the command during runtime.
943
-
944
- **Critical!** While this command tries to minimize collisions with other users, it is possible that an access port
945
- collision occurs when multiple users try to instantiate a jupyter server at the same time. If you cannot authenticate
946
- with the Jupyter server, this likely indicates that the target port was in use and Jupyter automatically incremented the
947
- port number by 1. In this case, add 1 to your port number and try connecting to that port using the Jupyter credentials
948
- provided by the command. For example, if your target port was '8888,' try port '8889.'
786
+ Most library components are intended to be used via other Sun lab libraries. For details on using shared
787
+ assets for data acquisition and preprocessing, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment)
788
+ library. For details on using shared assets for data processing and dataset formation, see the
789
+ [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library.
949
790
 
950
- ---
791
+ ***Warning!*** End users should not use any component of this library directly or install this library into any Python
792
+ environment. All assets from this library are intended to be used exclusively by developers working on other Sun lab
793
+ libraries.
951
794
 
952
795
  ## API Documentation
953
796
 
954
- See the [API documentation](https://sl-shared-assets-api-docs.netlify.app/) for the
955
- detailed description of the methods and classes exposed by components of this library.
797
+ Developers working on integrating sl-shared-assets into other libraries should see the
798
+ [API documentation](https://sl-shared-assets-api-docs.netlify.app/) for the detailed description of the methods and
799
+ classes exposed by components of this library.
956
800
 
957
- **Note!** The API documentation includes important information about Command-Line-Interfaces (CLIs) exposed by this
958
- library as part of installation into a Python environment. All users are highly encouraged to study the CLI
959
- documentation to learn how to use library components via the terminal.
801
+ **Note!** The API documentation includes important information about Command-Line Interfaces (CLIs) exposed by this
802
+ library as part of installation into a Python environment.
960
803
 
961
804
  ___
962
805
 
@@ -0,0 +1,23 @@
1
+ sl_shared_assets/__init__.py,sha256=97oYSDN15AVCv-nk1MzIs_7FfsQP8ENR1wkJYQc6vEY,2677
2
+ sl_shared_assets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ sl_shared_assets/command_line_interfaces/__init__.py,sha256=dbcCiuBCe1uqz6d1D-dFpSkRwYP603tEGkc1RbZcI6w,126
4
+ sl_shared_assets/command_line_interfaces/configure.py,sha256=lAWhLzJ7UmdvUZlges9Hg5pnblQuXSDlgzr12y41108,6140
5
+ sl_shared_assets/command_line_interfaces/manage.py,sha256=aGLn_iOrAGTXLFvhrjex4qXo7Crutmp8e8dWQ8YWuy4,8738
6
+ sl_shared_assets/data_classes/__init__.py,sha256=SiOY1JjK_W7lc-EkeZ4O8SCepMPsC5OILUkPe6geU1Q,2119
7
+ sl_shared_assets/data_classes/configuration_data.py,sha256=X6p5V15Verpu6ycO_fJhGOCyNp-IacVLktERc5eYF4k,45394
8
+ sl_shared_assets/data_classes/runtime_data.py,sha256=ruo5KZ1VSfGwSg_gtSyWTbLBFcfG7Az455cKxiUpDoc,17565
9
+ sl_shared_assets/data_classes/session_data.py,sha256=q34M6xDl_ntnvIG_tdDlGrkSXbHR1MzMGjhNBwM2rkk,47038
10
+ sl_shared_assets/data_classes/surgery_data.py,sha256=riwyULMdZ8pDXFLdyNByNSeMk1-mVTo1FcCl5FrThas,7495
11
+ sl_shared_assets/server/__init__.py,sha256=6REoeUvHW39V1Uwal_Af4uZ8J6UYDmR2N-saZoau30E,869
12
+ sl_shared_assets/server/job.py,sha256=92nCVaLYqEJH7-xaS0bZmIVoGp-nJ5gh6dnwyRyZ9gI,19406
13
+ sl_shared_assets/server/pipeline.py,sha256=0Peuq499BxKA3GaebQyhl_9VLepuaJQuwwt2vuM5ULg,33503
14
+ sl_shared_assets/server/server.py,sha256=BUELEQbO-sx-9E2epGURCYSGN-I9WAD2dRETMWEPcRU,36483
15
+ sl_shared_assets/tools/__init__.py,sha256=Pf8_MqZOLiityR112Txkermxf1bBLHYdEdw-raeY9Fc,737
16
+ sl_shared_assets/tools/packaging_tools.py,sha256=hvOrdRwQCoe199G_XHhIpbKtIx-I0ad9O4SLZwhgj7U,7035
17
+ sl_shared_assets/tools/project_management_tools.py,sha256=AcZpkzcXyAv4aBn-LIXawQgd7le7-vtZ9dl5MvMFmkg,40365
18
+ sl_shared_assets/tools/transfer_tools.py,sha256=6lwk4zyHWWB0QeWQageiQ1pPIGUV0IhBNNzsbgQ_qH4,9026
19
+ sl_shared_assets-5.0.1.dist-info/METADATA,sha256=kH_5OOIkoJbGKbiBoAG8_Ghg5iSisLLsxAV3NjTPY4E,46884
20
+ sl_shared_assets-5.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
21
+ sl_shared_assets-5.0.1.dist-info/entry_points.txt,sha256=19NzFPG5CcW-4RhEacTcX3J2TbrvmjOE4tlLCH2O5wI,161
22
+ sl_shared_assets-5.0.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
23
+ sl_shared_assets-5.0.1.dist-info/RECORD,,
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ sl-configure = sl_shared_assets.command_line_interfaces.configure:configure
3
+ sl-manage = sl_shared_assets.command_line_interfaces.manage:manage
@@ -1,91 +0,0 @@
1
- from .tools import (
2
- ProjectManifest as ProjectManifest,
3
- resolve_p53_marker as resolve_p53_marker,
4
- transfer_directory as transfer_directory,
5
- generate_project_manifest as generate_project_manifest,
6
- calculate_directory_checksum as calculate_directory_checksum,
7
- )
8
- from .server import (
9
- Job as Job,
10
- Server as Server,
11
- JupyterJob as JupyterJob,
12
- ServerCredentials as ServerCredentials,
13
- )
14
- from .data_classes import (
15
- RawData as RawData,
16
- DrugData as DrugData,
17
- ImplantData as ImplantData,
18
- SessionData as SessionData,
19
- SubjectData as SubjectData,
20
- SurgeryData as SurgeryData,
21
- SessionTypes as SessionTypes,
22
- InjectionData as InjectionData,
23
- ProcedureData as ProcedureData,
24
- ProcessedData as ProcessedData,
25
- MesoscopePaths as MesoscopePaths,
26
- ZaberPositions as ZaberPositions,
27
- ExperimentState as ExperimentState,
28
- ExperimentTrial as ExperimentTrial,
29
- MesoscopeCameras as MesoscopeCameras,
30
- TrackerFileNames as TrackerFileNames,
31
- ProcessingTracker as ProcessingTracker,
32
- AcquisitionSystems as AcquisitionSystems,
33
- MesoscopePositions as MesoscopePositions,
34
- RunTrainingDescriptor as RunTrainingDescriptor,
35
- LickTrainingDescriptor as LickTrainingDescriptor,
36
- MesoscopeHardwareState as MesoscopeHardwareState,
37
- WindowCheckingDescriptor as WindowCheckingDescriptor,
38
- MesoscopeMicroControllers as MesoscopeMicroControllers,
39
- MesoscopeAdditionalFirmware as MesoscopeAdditionalFirmware,
40
- MesoscopeSystemConfiguration as MesoscopeSystemConfiguration,
41
- MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
42
- MesoscopeExperimentConfiguration as MesoscopeExperimentConfiguration,
43
- generate_manager_id as generate_manager_id,
44
- get_processing_tracker as get_processing_tracker,
45
- get_system_configuration_data as get_system_configuration_data,
46
- set_system_configuration_file as set_system_configuration_file,
47
- )
48
-
49
- __all__ = [
50
- "Server",
51
- "ServerCredentials",
52
- "Job",
53
- "JupyterJob",
54
- "DrugData",
55
- "ImplantData",
56
- "SessionData",
57
- "RawData",
58
- "ProcessedData",
59
- "SubjectData",
60
- "SurgeryData",
61
- "InjectionData",
62
- "ProcessingTracker",
63
- "ProcedureData",
64
- "ZaberPositions",
65
- "ExperimentState",
66
- "MesoscopePositions",
67
- "MesoscopeHardwareState",
68
- "RunTrainingDescriptor",
69
- "LickTrainingDescriptor",
70
- "MesoscopeExperimentConfiguration",
71
- "MesoscopeExperimentDescriptor",
72
- "MesoscopeSystemConfiguration",
73
- "MesoscopePaths",
74
- "MesoscopeCameras",
75
- "MesoscopeMicroControllers",
76
- "MesoscopeAdditionalFirmware",
77
- "get_system_configuration_data",
78
- "set_system_configuration_file",
79
- "ExperimentTrial",
80
- "SessionTypes",
81
- "AcquisitionSystems",
82
- "WindowCheckingDescriptor",
83
- "get_processing_tracker",
84
- "generate_manager_id",
85
- "TrackerFileNames",
86
- "ProjectManifest",
87
- "resolve_p53_marker",
88
- "transfer_directory",
89
- "calculate_directory_checksum",
90
- "generate_project_manifest",
91
- ]