sl-shared-assets 2.0.1__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sl-shared-assets might be problematic. Click here for more details.

Files changed (51) hide show
  1. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/PKG-INFO +127 -13
  2. sl_shared_assets-3.0.0/README.md +230 -0
  3. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/docs/source/api.rst +8 -0
  4. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/docs/source/conf.py +1 -1
  5. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/docs/source/welcome.rst +3 -2
  6. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/envs/slsa_dev_lin.yml +51 -66
  7. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/envs/slsa_dev_lin_spec.txt +48 -63
  8. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/pyproject.toml +12 -11
  9. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/__init__.py +17 -9
  10. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/__init__.pyi +12 -8
  11. sl_shared_assets-3.0.0/src/sl_shared_assets/cli.py +455 -0
  12. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/cli.pyi +44 -5
  13. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/__init__.py +8 -3
  14. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/__init__.pyi +8 -4
  15. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/configuration_data.py +149 -30
  16. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/configuration_data.pyi +49 -11
  17. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/runtime_data.py +70 -49
  18. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/runtime_data.pyi +41 -33
  19. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/session_data.py +193 -253
  20. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/session_data.pyi +99 -116
  21. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/surgery_data.py +1 -1
  22. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/server/__init__.py +2 -2
  23. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/server/__init__.pyi +5 -2
  24. sl_shared_assets-3.0.0/src/sl_shared_assets/server/job.py +368 -0
  25. sl_shared_assets-3.0.0/src/sl_shared_assets/server/job.pyi +205 -0
  26. sl_shared_assets-3.0.0/src/sl_shared_assets/server/server.py +688 -0
  27. sl_shared_assets-3.0.0/src/sl_shared_assets/server/server.pyi +286 -0
  28. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/__init__.py +2 -1
  29. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/__init__.pyi +2 -0
  30. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/ascension_tools.py +9 -21
  31. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/ascension_tools.pyi +1 -1
  32. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/packaging_tools.py +2 -2
  33. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/project_management_tools.py +147 -41
  34. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/project_management_tools.pyi +45 -6
  35. sl_shared_assets-2.0.1/README.md +0 -114
  36. sl_shared_assets-2.0.1/src/sl_shared_assets/cli.py +0 -218
  37. sl_shared_assets-2.0.1/src/sl_shared_assets/server/job.py +0 -140
  38. sl_shared_assets-2.0.1/src/sl_shared_assets/server/job.pyi +0 -94
  39. sl_shared_assets-2.0.1/src/sl_shared_assets/server/server.py +0 -288
  40. sl_shared_assets-2.0.1/src/sl_shared_assets/server/server.pyi +0 -143
  41. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/.gitignore +0 -0
  42. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/LICENSE +0 -0
  43. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/docs/Makefile +0 -0
  44. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/docs/make.bat +0 -0
  45. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/docs/source/index.rst +0 -0
  46. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/data_classes/surgery_data.pyi +0 -0
  47. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/py.typed +0 -0
  48. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/packaging_tools.pyi +0 -0
  49. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/transfer_tools.py +0 -0
  50. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/src/sl_shared_assets/tools/transfer_tools.pyi +0 -0
  51. {sl_shared_assets-2.0.1 → sl_shared_assets-3.0.0}/tox.ini +0 -0
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sl-shared-assets
3
- Version: 2.0.1
4
- Summary: Stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
3
+ Version: 3.0.0
4
+ Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
5
5
  Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
6
6
  Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
7
7
  Author: Ivan Kondratyev, Kushaan Gupta, Natalie Yeung
@@ -681,7 +681,7 @@ License: GNU GENERAL PUBLIC LICENSE
681
681
  Public License instead of this License. But first, please read
682
682
  <https://www.gnu.org/licenses/why-not-lgpl.html>.
683
683
  License-File: LICENSE
684
- Keywords: acquisition,assets,data,processing,sunlab
684
+ Keywords: acquisition,assets,data,processing,server,sunlab
685
685
  Classifier: Development Status :: 5 - Production/Stable
686
686
  Classifier: Intended Audience :: Developers
687
687
  Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
@@ -691,13 +691,13 @@ Classifier: Programming Language :: Python :: 3.12
691
691
  Classifier: Programming Language :: Python :: 3.13
692
692
  Requires-Python: >=3.11
693
693
  Requires-Dist: appdirs==1.4.4
694
- Requires-Dist: ataraxis-base-utilities==3.0.1
694
+ Requires-Dist: ataraxis-base-utilities==3.1.0
695
695
  Requires-Dist: ataraxis-data-structures==3.1.1
696
696
  Requires-Dist: ataraxis-time==3.0.0
697
697
  Requires-Dist: click==8.2.1
698
698
  Requires-Dist: filelock==3.18.0
699
699
  Requires-Dist: natsort==8.4.0
700
- Requires-Dist: numpy<2.3.0,>=2.0.2
700
+ Requires-Dist: numpy==2.2.6
701
701
  Requires-Dist: paramiko==3.5.1
702
702
  Requires-Dist: polars==1.31.0
703
703
  Requires-Dist: pyarrow==20.0.0
@@ -706,7 +706,6 @@ Requires-Dist: simple-slurm==0.3.6
706
706
  Requires-Dist: tqdm==4.67.1
707
707
  Requires-Dist: xxhash==3.5.0
708
708
  Provides-Extra: conda
709
- Requires-Dist: grayskull<3,>=2; extra == 'conda'
710
709
  Requires-Dist: hatchling<2,>=1; extra == 'conda'
711
710
  Requires-Dist: importlib-metadata<9,>=8; extra == 'conda'
712
711
  Requires-Dist: mypy<2,>=1; extra == 'conda'
@@ -725,7 +724,7 @@ Requires-Dist: appdirs==1.4.4; extra == 'condarun'
725
724
  Requires-Dist: click==8.2.1; extra == 'condarun'
726
725
  Requires-Dist: filelock==3.18.0; extra == 'condarun'
727
726
  Requires-Dist: natsort==8.4.0; extra == 'condarun'
728
- Requires-Dist: numpy<2.3.0,>=2.0.2; extra == 'condarun'
727
+ Requires-Dist: numpy==2.2.6; extra == 'condarun'
729
728
  Requires-Dist: paramiko==3.5.1; extra == 'condarun'
730
729
  Requires-Dist: polars==1.31.0; extra == 'condarun'
731
730
  Requires-Dist: pyarrow==20.0.0; extra == 'condarun'
@@ -734,7 +733,6 @@ Requires-Dist: tqdm==4.67.1; extra == 'condarun'
734
733
  Provides-Extra: dev
735
734
  Requires-Dist: ataraxis-automation<6,>=5; extra == 'dev'
736
735
  Requires-Dist: build<2,>=1; extra == 'dev'
737
- Requires-Dist: grayskull<3,>=2; extra == 'dev'
738
736
  Requires-Dist: hatchling<2,>=1; extra == 'dev'
739
737
  Requires-Dist: importlib-metadata<9,>=8; extra == 'dev'
740
738
  Requires-Dist: mypy<2,>=1; extra == 'dev'
@@ -774,6 +772,7 @@ A Python library that stores assets shared between multiple Sun (NeuroAI) lab da
774
772
  ![PyPI - License](https://img.shields.io/pypi/l/sl-shared-assets)
775
773
  ![PyPI - Status](https://img.shields.io/pypi/status/sl-shared-assets)
776
774
  ![PyPI - Wheel](https://img.shields.io/pypi/wheel/sl-shared-assets)
775
+
777
776
  ___
778
777
 
779
778
  ## Detailed Description
@@ -815,7 +814,7 @@ All software library dependencies are installed automatically as part of library
815
814
 
816
815
  Note, installation from source is ***highly discouraged*** for everyone who is not an active project developer.
817
816
 
818
- 1. Download this repository to your local machine using your preferred method, such as Git-cloning. Use one
817
+ 1. Download this repository to your local machine using any method, such as Git-cloning. Use one
819
818
  of the stable releases from [GitHub](https://github.com/Sun-Lab-NBB/sl-shared-assets/releases).
820
819
  2. Unpack the downloaded zip and note the path to the binary wheel (`.whl`) file contained in the archive.
821
820
  3. Run ```python -m pip install WHEEL_PATH```, replacing 'WHEEL_PATH' with the path to the wheel file, to install the
@@ -828,12 +827,128 @@ Use the following command to install the library using pip: ```pip install sl-sh
828
827
 
829
828
  ## Usage
830
829
 
831
- All library components are intended to be used via other Sun lab libraries. Developers should study the API and CLI
830
+ Most library components are intended to be used via other Sun lab libraries. Developers should study the API and CLI
832
831
  documentation below to learn how to use library components in other Sun lab libraries. For notes on using shared
833
832
  assets for data acquisition, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment) library ReadMe.
834
833
  For notes on using shared assets for data processing, see the [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)
835
834
  library ReadMe.
836
835
 
836
+ The only exception to the note above is the **server.py** package exposed by this library. This package exposes an API
837
+ for running headless and a CLI for running interactive Simple Linux Utility for Resource Management (SLURM)-managed
838
+ jobs on remote compute clusters.
839
+
840
+ ### Generating Access Credentials
841
+
842
+ To access any remote server, the user is required to first generate the access credentials. The credentials are stored
843
+ inside the 'server_credentials.yaml' file, which is generated by using the `sl-create-server-credentials` command.
844
+ **Note!** Users are advised to generate this file in a secure (non-shared) location on their local machine.
845
+
846
+ ### Running Headless Jobs
847
+
848
+ A headless job is a job that does not require any user interaction during runtime. Currently, all headless jobs in the
849
+ lab rely on pip-installable packages that expose a callable Command-Line Interface to carry out some type of
850
+ data processing. In this regard, **running a headless job is equivalent to calling a CLI command on your local
851
+ machine**, except that the command is executed on a remote compute server. Therefore, the primary purpose of the API
852
+ exposed by this library is to transfer the target command request to the remote server, execute it, and monitor the
853
+ runtime status until it is complete.
854
+
855
+ For example, the [sl-suite2p package](https://github.com/Sun-Lab-NBB/suite2p) maintained in the lab exposes a CLI to
856
+ process 2-Photon data from experiment sessions. During data processing by the
857
+ [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library, a remote job is sent to the server that uses the CLI
858
+ exposed by the sl-suite2p package to process target session(s).
859
+
860
+ ### Creating Jobs
861
+ All remote jobs are sent to the server in the form of an executable *shell* (.sh) script. The script is composed on the
862
+ local machine that uses this library and transferred to a temporary server directory using Secure Shell File
863
+ Transfer Protocol (SFTP). The server is then instructed to evaluate (run) the script using SLURM job manager, via a
864
+ Secure Shell (SSH) session.
865
+
866
+ Broadly, each job consists of three major steps, which correspond to three major sections of the job shell script:
867
+ 1. **Setting up the job environment**. Each job script starts with a SLURM job parameter block, which tells SLURM
868
+ what resources (CPUs, GPUs, RAM, etc.) the job requires. When resources become available, SLURM generates a virtual
869
+ environment and runs the rest of the job script in that environment. This forms the basis for using the shared
870
+ compute resources fairly, as SLURM balances resource allocation and the order of job execution for all users.
871
+ 2. **Activating the target conda environment**. Currently, all jobs are assumed to use Python libraries to execute the
872
+ intended data processing. Similar to processing data locally, each job expects the remote server to provide a
873
+ Conda environment preconfigured with necessary assets (packages) to run the job. Therefore, each job contains a
874
+ section that activates the user-defined conda environment before running the rest of the job.
875
+ 3. **Executing processing**. The final section is typically unique to each job and calls specific CLI commands or runs
876
+ specific Python modules. Since each job is submitted as a shell script, it can do anything a server shell can
877
+ do. Therefore, despite python-centric approach to data processing in the lab, a remote job composed via this library
878
+ can execute ***any*** arbitrary command available to the user on the remove server.
879
+
880
+ Use the *Job* class exposed by this library to compose remote jobs. **Steps 1 and 2** of each job are configured when
881
+ initializing the Job instance, while **step 3** is added via the `add_command()` method of the Job class:
882
+ ```
883
+ # First, import the job class
884
+ from pathlib import Path
885
+ from sl_shared_assets import Job
886
+
887
+ # Next, instantiate a new Job object. For example, this job is used to verify the integrity of raw experiment data as
888
+ # it is transferred to the long-term storage destination (server) by the sl-experiment library.
889
+ job = Job(
890
+ job_name="data_integrity_verification",
891
+ output_log=Path("/temp/output.txt"),
892
+ error_log=Path("/temp/errors.txt"),
893
+ working_directory=Path("/temp/test_job"),
894
+ conda_environment="test_environment",
895
+ cpus_to_use=20,
896
+ ram_gb=50,
897
+ time_limit=20,
898
+ )
899
+
900
+ # Finally, add a CLI command call (the actual work to be done by the job). Here, the job calls the
901
+ # 'sl-verify-session' command exposed by the sl-shared-assets library installed in the target environment on the server.
902
+ # Use this method to add commands as you would type them in your local terminal / shell / command line.
903
+ job.add_command(f"sl-verify-session -sp /temp/test_session")
904
+ ```
905
+
906
+ ### Submitting and Monitoring Jobs:
907
+ To submit the job to the remote server, use a **Server** class instance. This class encapsulates access to the target
908
+ remote compute server and uses the server_credentials.yaml file to determine server access credentials (see above):
909
+ ```
910
+ # Initialize the Server class using precreated server credentials file
911
+ server = Server(credentials_path=Path("/temp/server_credentials.yaml"))
912
+
913
+ # Submit the job (generated in the previous code snippet) to the server
914
+ job = server.submit_job(job)
915
+
916
+ # Wait for the server to complete the job
917
+ delay_timer = PrecisionTimer("s")
918
+ while not server.job_complete(job=job):
919
+ delay_timer.delay_noblock(delay=5, allow_sleep=True)
920
+ ```
921
+
922
+ **Note!** The Server class only checks whether the job is running on the server, but not the outcome of the job. For
923
+ that, you can either manually check the output and error logs for the job or come up with a programmatic way of
924
+ checking the outcome. All developers are highly advised to study the API documentation for the Job and Server classes
925
+ to use them most effectively.
926
+
927
+ **Critical!** Since running remote jobs is largely equivalent to executing them locally, all users are highly encouraged
928
+ to test their job scripts locally before deploying them server-side. If a script works on a local machine, it is likely
929
+ that the script would behave similarly and work on the server.
930
+
931
+ ### Interactive Jobs
932
+
933
+ Interactive jobs are a special extension of the headless job type discussed above. Specifically, an interactive job is
934
+ a headless job, whose only purpose is to **create and maintain a Jupyter lab server** under the SLURM control.
935
+ Specifically, it requests SLURM to set up an isolated environment, starts a Jupyter server in that environment, and
936
+ sends the credentials for the started server back to the user.
937
+
938
+ In essence, this allocates a set of resources the user can use interactively by running various Jupyter notebooks.
939
+ While convenient for certain data analysis cases, this type of jobs has the potential to inefficiently hog server
940
+ resources for prolonged periods of time. Therefore, users are encouraged to only resort to this type of jobs when
941
+ strictly necessary and to minimize the resources and time allocated to running these jobs.
942
+
943
+ To run an interactive job, call the `sl-start-jupyter` CLI command exposed by this library and follow the instructions
944
+ printed to the terminal by the command during runtime.
945
+
946
+ **Critical!** While this command tries to minimize collisions with other users, it is possible that an access port
947
+ collision occurs when multiple users try to instantiate a jupyter server at the same time. If you cannot authenticate
948
+ with the Jupyter server, this likely indicates that the target port was in use and Jupyter automatically incremented the
949
+ port number by 1. In this case, add 1 to your port number and try connecting to that port using the Jupyter credentials
950
+ provided by the command. For example, if your target port was '8888,' try port '8889.'
951
+
837
952
  ---
838
953
 
839
954
  ## API Documentation
@@ -849,7 +964,7 @@ ___
849
964
 
850
965
  ## Versioning
851
966
 
852
- We use [semantic versioning](https://semver.org/) for this project. For the versions available, see the
967
+ This project uses [semantic versioning](https://semver.org/). For the versions available, see the
853
968
  [tags on this repository](https://github.com/Sun-Lab-NBB/sl-shared-assets/tags).
854
969
 
855
970
  ---
@@ -858,7 +973,6 @@ We use [semantic versioning](https://semver.org/) for this project. For the vers
858
973
 
859
974
  - Ivan Kondratyev ([Inkaros](https://github.com/Inkaros))
860
975
  - Kushaan Gupta ([kushaangupta](https://github.com/kushaangupta))
861
- - Yuantao Deng ([YuantaoDeng](https://github.com/YuantaoDeng))
862
976
  - Natalie Yeung
863
977
 
864
978
  ___
@@ -873,7 +987,7 @@ ___
873
987
 
874
988
  - All Sun lab [members](https://neuroai.github.io/sunlab/people) for providing the inspiration and comments during the
875
989
  development of this library.
876
- - The creators of all other projects used in our development automation pipelines and source code
990
+ - The creators of all other projects used in the development automation pipelines and source code of this project
877
991
  [see pyproject.toml](pyproject.toml).
878
992
 
879
993
  ---
@@ -0,0 +1,230 @@
1
+ # sl-shared-assets
2
+ A Python library that stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
3
+
4
+ ![PyPI - Version](https://img.shields.io/pypi/v/sl-shared-assets)
5
+ ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sl-shared-assets)
6
+ [![uv](https://tinyurl.com/uvbadge)](https://github.com/astral-sh/uv)
7
+ [![Ruff](https://tinyurl.com/ruffbadge)](https://github.com/astral-sh/ruff)
8
+ ![type-checked: mypy](https://img.shields.io/badge/type--checked-mypy-blue?style=flat-square&logo=python)
9
+ ![PyPI - License](https://img.shields.io/pypi/l/sl-shared-assets)
10
+ ![PyPI - Status](https://img.shields.io/pypi/status/sl-shared-assets)
11
+ ![PyPI - Wheel](https://img.shields.io/pypi/wheel/sl-shared-assets)
12
+
13
+ ___
14
+
15
+ ## Detailed Description
16
+
17
+ Primarily, this library is designed to make the two main Sun lab libraries used for data acquisition
18
+ ([sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment)) and processing
19
+ ([sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)) independent of each other. This is beneficial, as both
20
+ libraries feature an extensive and largely incompatible set of runtime dependencies. Moreover, having a shared
21
+ repository of classes and tools reused across Sun lab pipelines streamlines the maintenance of these tools.
22
+
23
+ The library broadly stores two types of assets. First, it stores various dataclasses used to save the data acquired
24
+ during experiments in the lab and the dataclasses used to configure data acquisition and processing runtimes. Secondly,
25
+ it stores the tools used to safely move the data between the machines (computers) used in the data acquisition and
26
+ processing, and provides the API for running various data processing jobs on remote compute servers.
27
+
28
+ ---
29
+
30
+ ## Table of Contents
31
+
32
+ - [Dependencies](#dependencies)
33
+ - [Installation](#installation)
34
+ - [Usage](#usage)
35
+ - [API Documentation](#api-documentation)
36
+ - [Versioning](#versioning)
37
+ - [Authors](#authors)
38
+ - [License](#license)
39
+ - [Acknowledgements](#Acknowledgments)
40
+ ___
41
+
42
+ ## Dependencies
43
+
44
+ All software library dependencies are installed automatically as part of library installation.
45
+
46
+ ---
47
+
48
+ ## Installation
49
+
50
+ ### Source
51
+
52
+ Note, installation from source is ***highly discouraged*** for everyone who is not an active project developer.
53
+
54
+ 1. Download this repository to your local machine using any method, such as Git-cloning. Use one
55
+ of the stable releases from [GitHub](https://github.com/Sun-Lab-NBB/sl-shared-assets/releases).
56
+ 2. Unpack the downloaded zip and note the path to the binary wheel (`.whl`) file contained in the archive.
57
+ 3. Run ```python -m pip install WHEEL_PATH```, replacing 'WHEEL_PATH' with the path to the wheel file, to install the
58
+ wheel into the active python environment.
59
+
60
+ ### pip
61
+ Use the following command to install the library using pip: ```pip install sl-shared-assets```.
62
+
63
+ ---
64
+
65
+ ## Usage
66
+
67
+ Most library components are intended to be used via other Sun lab libraries. Developers should study the API and CLI
68
+ documentation below to learn how to use library components in other Sun lab libraries. For notes on using shared
69
+ assets for data acquisition, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment) library ReadMe.
70
+ For notes on using shared assets for data processing, see the [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)
71
+ library ReadMe.
72
+
73
+ The only exception to the note above is the **server.py** package exposed by this library. This package exposes an API
74
+ for running headless and a CLI for running interactive Simple Linux Utility for Resource Management (SLURM)-managed
75
+ jobs on remote compute clusters.
76
+
77
+ ### Generating Access Credentials
78
+
79
+ To access any remote server, the user is required to first generate the access credentials. The credentials are stored
80
+ inside the 'server_credentials.yaml' file, which is generated by using the `sl-create-server-credentials` command.
81
+ **Note!** Users are advised to generate this file in a secure (non-shared) location on their local machine.
82
+
83
+ ### Running Headless Jobs
84
+
85
+ A headless job is a job that does not require any user interaction during runtime. Currently, all headless jobs in the
86
+ lab rely on pip-installable packages that expose a callable Command-Line Interface to carry out some type of
87
+ data processing. In this regard, **running a headless job is equivalent to calling a CLI command on your local
88
+ machine**, except that the command is executed on a remote compute server. Therefore, the primary purpose of the API
89
+ exposed by this library is to transfer the target command request to the remote server, execute it, and monitor the
90
+ runtime status until it is complete.
91
+
92
+ For example, the [sl-suite2p package](https://github.com/Sun-Lab-NBB/suite2p) maintained in the lab exposes a CLI to
93
+ process 2-Photon data from experiment sessions. During data processing by the
94
+ [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library, a remote job is sent to the server that uses the CLI
95
+ exposed by the sl-suite2p package to process target session(s).
96
+
97
+ ### Creating Jobs
98
+ All remote jobs are sent to the server in the form of an executable *shell* (.sh) script. The script is composed on the
99
+ local machine that uses this library and transferred to a temporary server directory using Secure Shell File
100
+ Transfer Protocol (SFTP). The server is then instructed to evaluate (run) the script using SLURM job manager, via a
101
+ Secure Shell (SSH) session.
102
+
103
+ Broadly, each job consists of three major steps, which correspond to three major sections of the job shell script:
104
+ 1. **Setting up the job environment**. Each job script starts with a SLURM job parameter block, which tells SLURM
105
+ what resources (CPUs, GPUs, RAM, etc.) the job requires. When resources become available, SLURM generates a virtual
106
+ environment and runs the rest of the job script in that environment. This forms the basis for using the shared
107
+ compute resources fairly, as SLURM balances resource allocation and the order of job execution for all users.
108
+ 2. **Activating the target conda environment**. Currently, all jobs are assumed to use Python libraries to execute the
109
+ intended data processing. Similar to processing data locally, each job expects the remote server to provide a
110
+ Conda environment preconfigured with necessary assets (packages) to run the job. Therefore, each job contains a
111
+ section that activates the user-defined conda environment before running the rest of the job.
112
+ 3. **Executing processing**. The final section is typically unique to each job and calls specific CLI commands or runs
113
+ specific Python modules. Since each job is submitted as a shell script, it can do anything a server shell can
114
+ do. Therefore, despite python-centric approach to data processing in the lab, a remote job composed via this library
115
+ can execute ***any*** arbitrary command available to the user on the remove server.
116
+
117
+ Use the *Job* class exposed by this library to compose remote jobs. **Steps 1 and 2** of each job are configured when
118
+ initializing the Job instance, while **step 3** is added via the `add_command()` method of the Job class:
119
+ ```
120
+ # First, import the job class
121
+ from pathlib import Path
122
+ from sl_shared_assets import Job
123
+
124
+ # Next, instantiate a new Job object. For example, this job is used to verify the integrity of raw experiment data as
125
+ # it is transferred to the long-term storage destination (server) by the sl-experiment library.
126
+ job = Job(
127
+ job_name="data_integrity_verification",
128
+ output_log=Path("/temp/output.txt"),
129
+ error_log=Path("/temp/errors.txt"),
130
+ working_directory=Path("/temp/test_job"),
131
+ conda_environment="test_environment",
132
+ cpus_to_use=20,
133
+ ram_gb=50,
134
+ time_limit=20,
135
+ )
136
+
137
+ # Finally, add a CLI command call (the actual work to be done by the job). Here, the job calls the
138
+ # 'sl-verify-session' command exposed by the sl-shared-assets library installed in the target environment on the server.
139
+ # Use this method to add commands as you would type them in your local terminal / shell / command line.
140
+ job.add_command(f"sl-verify-session -sp /temp/test_session")
141
+ ```
142
+
143
+ ### Submitting and Monitoring Jobs:
144
+ To submit the job to the remote server, use a **Server** class instance. This class encapsulates access to the target
145
+ remote compute server and uses the server_credentials.yaml file to determine server access credentials (see above):
146
+ ```
147
+ # Initialize the Server class using precreated server credentials file
148
+ server = Server(credentials_path=Path("/temp/server_credentials.yaml"))
149
+
150
+ # Submit the job (generated in the previous code snippet) to the server
151
+ job = server.submit_job(job)
152
+
153
+ # Wait for the server to complete the job
154
+ delay_timer = PrecisionTimer("s")
155
+ while not server.job_complete(job=job):
156
+ delay_timer.delay_noblock(delay=5, allow_sleep=True)
157
+ ```
158
+
159
+ **Note!** The Server class only checks whether the job is running on the server, but not the outcome of the job. For
160
+ that, you can either manually check the output and error logs for the job or come up with a programmatic way of
161
+ checking the outcome. All developers are highly advised to study the API documentation for the Job and Server classes
162
+ to use them most effectively.
163
+
164
+ **Critical!** Since running remote jobs is largely equivalent to executing them locally, all users are highly encouraged
165
+ to test their job scripts locally before deploying them server-side. If a script works on a local machine, it is likely
166
+ that the script would behave similarly and work on the server.
167
+
168
+ ### Interactive Jobs
169
+
170
+ Interactive jobs are a special extension of the headless job type discussed above. Specifically, an interactive job is
171
+ a headless job, whose only purpose is to **create and maintain a Jupyter lab server** under the SLURM control.
172
+ Specifically, it requests SLURM to set up an isolated environment, starts a Jupyter server in that environment, and
173
+ sends the credentials for the started server back to the user.
174
+
175
+ In essence, this allocates a set of resources the user can use interactively by running various Jupyter notebooks.
176
+ While convenient for certain data analysis cases, this type of jobs has the potential to inefficiently hog server
177
+ resources for prolonged periods of time. Therefore, users are encouraged to only resort to this type of jobs when
178
+ strictly necessary and to minimize the resources and time allocated to running these jobs.
179
+
180
+ To run an interactive job, call the `sl-start-jupyter` CLI command exposed by this library and follow the instructions
181
+ printed to the terminal by the command during runtime.
182
+
183
+ **Critical!** While this command tries to minimize collisions with other users, it is possible that an access port
184
+ collision occurs when multiple users try to instantiate a jupyter server at the same time. If you cannot authenticate
185
+ with the Jupyter server, this likely indicates that the target port was in use and Jupyter automatically incremented the
186
+ port number by 1. In this case, add 1 to your port number and try connecting to that port using the Jupyter credentials
187
+ provided by the command. For example, if your target port was '8888,' try port '8889.'
188
+
189
+ ---
190
+
191
+ ## API Documentation
192
+
193
+ See the [API documentation](https://sl-shared-assets-api-docs.netlify.app/) for the
194
+ detailed description of the methods and classes exposed by components of this library.
195
+
196
+ **Note!** The API documentation includes important information about Command-Line-Interfaces (CLIs) exposed by this
197
+ library as part of installation into a Python environment. All users are highly encouraged to study the CLI
198
+ documentation to learn how to use library components via the terminal.
199
+
200
+ ___
201
+
202
+ ## Versioning
203
+
204
+ This project uses [semantic versioning](https://semver.org/). For the versions available, see the
205
+ [tags on this repository](https://github.com/Sun-Lab-NBB/sl-shared-assets/tags).
206
+
207
+ ---
208
+
209
+ ## Authors
210
+
211
+ - Ivan Kondratyev ([Inkaros](https://github.com/Inkaros))
212
+ - Kushaan Gupta ([kushaangupta](https://github.com/kushaangupta))
213
+ - Natalie Yeung
214
+
215
+ ___
216
+
217
+ ## License
218
+
219
+ This project is licensed under the GPL3 License: see the [LICENSE](LICENSE) file for details.
220
+
221
+ ___
222
+
223
+ ## Acknowledgments
224
+
225
+ - All Sun lab [members](https://neuroai.github.io/sunlab/people) for providing the inspiration and comments during the
226
+ development of this library.
227
+ - The creators of all other projects used in the development automation pipelines and source code of this project
228
+ [see pyproject.toml](pyproject.toml).
229
+
230
+ ---
@@ -25,6 +25,14 @@ Command Line Interfaces
25
25
  :prog: sl-ascend-tyche
26
26
  :nested: full
27
27
 
28
+ .. click:: sl_shared_assets.cli:start_jupyter_server
29
+ :prog: sl-start-jupyter
30
+ :nested: full
31
+
32
+ .. click:: sl_shared_assets.cli:resolve_dataset_marker
33
+ :prog: sl-dataset-marker
34
+ :nested: full
35
+
28
36
  Tools
29
37
  =====
30
38
  .. automodule:: sl_shared_assets.tools
@@ -5,7 +5,7 @@ import importlib_metadata
5
5
  project = 'sl-shared-assets'
6
6
  # noinspection PyShadowingBuiltins
7
7
  copyright = '2025, Sun (NeuroAI) lab'
8
- authors = ['Ivan Kondratyev', 'Kushaan Gupta', 'Yuantao Deng', 'Natalie Yeung']
8
+ authors = ['Ivan Kondratyev', 'Kushaan Gupta', 'Natalie Yeung']
9
9
  # Extracts the project version from the .toml file.
10
10
  release = importlib_metadata.version("sl-shared-assets")
11
11
 
@@ -1,10 +1,11 @@
1
1
  Welcome to sl-shared-assets API documentation page
2
2
  ==================================================
3
3
 
4
- sl-shared-assets is a Python library that stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
4
+ sl-shared-assets is a Python library that provides data acquisition and processing assets shared between Sun (NeuroAI)
5
+ lab libraries.
5
6
 
6
7
  This library is developed in the `Sun lab <https://neuroai.github.io/sunlab/>`_ at Cornell University. The main purpose
7
- of this library is to provide the tools adn assets used by multiple other data acquisition and processing libraries in
8
+ of this library is to provide the tools and assets used by multiple other data acquisition and processing libraries in
8
9
  the lab to avoid cross-dependency issues.
9
10
 
10
11
  This website only contains the API documentation for the classes and methods offered by this library. See the project