sl-shared-assets 4.0.1__tar.gz → 5.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sl-shared-assets might be problematic. Click here for more details.

Files changed (59) hide show
  1. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/PKG-INFO +46 -203
  2. sl_shared_assets-5.0.1/README.md +113 -0
  3. sl_shared_assets-5.0.1/docs/source/api.rst +34 -0
  4. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/docs/source/welcome.rst +1 -1
  5. sl_shared_assets-5.0.1/envs/slsa_dev_lin.yml +45 -0
  6. sl_shared_assets-5.0.1/envs/slsa_dev_lin_spec.txt +42 -0
  7. sl_shared_assets-5.0.1/envs/slsa_dev_osx.yml +39 -0
  8. sl_shared_assets-5.0.1/envs/slsa_dev_osx_spec.txt +36 -0
  9. sl_shared_assets-5.0.1/envs/slsa_dev_win.yml +39 -0
  10. sl_shared_assets-5.0.1/envs/slsa_dev_win_spec.txt +36 -0
  11. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/pyproject.toml +40 -93
  12. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/__init__.py +48 -41
  13. sl_shared_assets-5.0.1/src/sl_shared_assets/command_line_interfaces/__init__.py +3 -0
  14. sl_shared_assets-5.0.1/src/sl_shared_assets/command_line_interfaces/configure.py +173 -0
  15. sl_shared_assets-5.0.1/src/sl_shared_assets/command_line_interfaces/manage.py +226 -0
  16. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/data_classes/__init__.py +33 -32
  17. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/data_classes/configuration_data.py +267 -79
  18. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/data_classes/session_data.py +226 -289
  19. sl_shared_assets-5.0.1/src/sl_shared_assets/server/__init__.py +28 -0
  20. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/server/job.py +6 -7
  21. sl_shared_assets-5.0.1/src/sl_shared_assets/server/pipeline.py +585 -0
  22. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/server/server.py +57 -25
  23. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/tools/__init__.py +9 -8
  24. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/tools/packaging_tools.py +14 -25
  25. sl_shared_assets-5.0.1/src/sl_shared_assets/tools/project_management_tools.py +841 -0
  26. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/tools/transfer_tools.py +88 -23
  27. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/tox.ini +49 -72
  28. sl_shared_assets-4.0.1/README.md +0 -230
  29. sl_shared_assets-4.0.1/docs/source/api.rst +0 -55
  30. sl_shared_assets-4.0.1/envs/slsa_dev_lin.yml +0 -218
  31. sl_shared_assets-4.0.1/envs/slsa_dev_lin_spec.txt +0 -213
  32. sl_shared_assets-4.0.1/src/sl_shared_assets/__init__.pyi +0 -91
  33. sl_shared_assets-4.0.1/src/sl_shared_assets/cli.py +0 -501
  34. sl_shared_assets-4.0.1/src/sl_shared_assets/cli.pyi +0 -106
  35. sl_shared_assets-4.0.1/src/sl_shared_assets/data_classes/__init__.pyi +0 -75
  36. sl_shared_assets-4.0.1/src/sl_shared_assets/data_classes/configuration_data.pyi +0 -235
  37. sl_shared_assets-4.0.1/src/sl_shared_assets/data_classes/runtime_data.pyi +0 -157
  38. sl_shared_assets-4.0.1/src/sl_shared_assets/data_classes/session_data.pyi +0 -379
  39. sl_shared_assets-4.0.1/src/sl_shared_assets/data_classes/surgery_data.pyi +0 -89
  40. sl_shared_assets-4.0.1/src/sl_shared_assets/server/__init__.py +0 -8
  41. sl_shared_assets-4.0.1/src/sl_shared_assets/server/__init__.pyi +0 -11
  42. sl_shared_assets-4.0.1/src/sl_shared_assets/server/job.pyi +0 -205
  43. sl_shared_assets-4.0.1/src/sl_shared_assets/server/server.pyi +0 -298
  44. sl_shared_assets-4.0.1/src/sl_shared_assets/tools/__init__.pyi +0 -19
  45. sl_shared_assets-4.0.1/src/sl_shared_assets/tools/ascension_tools.py +0 -265
  46. sl_shared_assets-4.0.1/src/sl_shared_assets/tools/ascension_tools.pyi +0 -68
  47. sl_shared_assets-4.0.1/src/sl_shared_assets/tools/packaging_tools.pyi +0 -58
  48. sl_shared_assets-4.0.1/src/sl_shared_assets/tools/project_management_tools.py +0 -762
  49. sl_shared_assets-4.0.1/src/sl_shared_assets/tools/project_management_tools.pyi +0 -239
  50. sl_shared_assets-4.0.1/src/sl_shared_assets/tools/transfer_tools.pyi +0 -53
  51. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/.gitignore +0 -0
  52. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/LICENSE +0 -0
  53. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/docs/Makefile +0 -0
  54. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/docs/make.bat +0 -0
  55. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/docs/source/conf.py +0 -0
  56. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/docs/source/index.rst +0 -0
  57. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/data_classes/runtime_data.py +0 -0
  58. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/data_classes/surgery_data.py +0 -0
  59. {sl_shared_assets-4.0.1 → sl_shared_assets-5.0.1}/src/sl_shared_assets/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sl-shared-assets
3
- Version: 4.0.1
3
+ Version: 5.0.1
4
4
  Summary: Provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
5
5
  Project-URL: Homepage, https://github.com/Sun-Lab-NBB/sl-shared-assets
6
6
  Project-URL: Documentation, https://sl-shared-assets-api-docs.netlify.app/
@@ -689,78 +689,38 @@ Classifier: Operating System :: OS Independent
689
689
  Classifier: Programming Language :: Python :: 3.11
690
690
  Classifier: Programming Language :: Python :: 3.12
691
691
  Classifier: Programming Language :: Python :: 3.13
692
+ Classifier: Topic :: Scientific/Engineering
693
+ Classifier: Typing :: Typed
692
694
  Requires-Python: >=3.11
693
- Requires-Dist: appdirs==1.4.4
694
- Requires-Dist: ataraxis-base-utilities==3.1.0
695
- Requires-Dist: ataraxis-data-structures==3.1.1
696
- Requires-Dist: ataraxis-time==3.0.0
697
- Requires-Dist: click==8.2.1
698
- Requires-Dist: filelock==3.18.0
699
- Requires-Dist: natsort==8.4.0
700
- Requires-Dist: numpy==2.2.6
701
- Requires-Dist: paramiko==4.0.0
702
- Requires-Dist: polars==1.32.0
703
- Requires-Dist: pyarrow==21.0.0
704
- Requires-Dist: pytz==2025.2
705
- Requires-Dist: simple-slurm==0.3.6
706
- Requires-Dist: tqdm==4.67.1
707
- Requires-Dist: xxhash==3.5.0
708
- Provides-Extra: conda
709
- Requires-Dist: hatchling==1.27.0; extra == 'conda'
710
- Requires-Dist: importlib-metadata==8.7.0; extra == 'conda'
711
- Requires-Dist: mypy==1.17.1; extra == 'conda'
712
- Requires-Dist: ruff==0.12.7; extra == 'conda'
713
- Requires-Dist: sphinx-autodoc-typehints==3.2.0; extra == 'conda'
714
- Requires-Dist: sphinx-click==6.0.0; extra == 'conda'
715
- Requires-Dist: sphinx==8.2.3; extra == 'conda'
716
- Requires-Dist: tox-uv==1.26.2; extra == 'conda'
717
- Requires-Dist: tox==4.28.4; extra == 'conda'
718
- Requires-Dist: twine==6.1.0; extra == 'conda'
719
- Requires-Dist: types-appdirs==1.4.3.5; extra == 'conda'
720
- Requires-Dist: types-filelock==3.2.7; extra == 'conda'
721
- Requires-Dist: types-paramiko==3.5.0.20250801; extra == 'conda'
722
- Requires-Dist: types-pytz==2025.2.0.20250516; extra == 'conda'
723
- Requires-Dist: types-tqdm==4.67.0.20250516; extra == 'conda'
724
- Requires-Dist: uv==0.8.4; extra == 'conda'
725
- Provides-Extra: condarun
726
- Requires-Dist: appdirs==1.4.4; extra == 'condarun'
727
- Requires-Dist: filelock==3.18.0; extra == 'condarun'
728
- Requires-Dist: natsort==8.4.0; extra == 'condarun'
729
- Requires-Dist: numpy==2.2.6; extra == 'condarun'
730
- Requires-Dist: polars==1.32.0; extra == 'condarun'
731
- Requires-Dist: pyarrow==21.0.0; extra == 'condarun'
732
- Requires-Dist: pytz==2025.2; extra == 'condarun'
733
- Requires-Dist: tqdm==4.67.1; extra == 'condarun'
695
+ Requires-Dist: appdirs<2,>=1
696
+ Requires-Dist: ataraxis-base-utilities<4,>=3
697
+ Requires-Dist: ataraxis-data-structures<4,>=3
698
+ Requires-Dist: ataraxis-time<4,>=3
699
+ Requires-Dist: click<9,>=8
700
+ Requires-Dist: filelock<4,>=3
701
+ Requires-Dist: natsort<9,>=8
702
+ Requires-Dist: numpy<3,>=2
703
+ Requires-Dist: paramiko<5,>=4
704
+ Requires-Dist: polars<2,>=1
705
+ Requires-Dist: pyarrow<22,>=21
706
+ Requires-Dist: pytz<2026,>=2025
707
+ Requires-Dist: simple-slurm<1,>=0
708
+ Requires-Dist: tqdm<5,>=4
709
+ Requires-Dist: xxhash<4,>=3
734
710
  Provides-Extra: dev
735
- Requires-Dist: ataraxis-automation==5.0.0; extra == 'dev'
736
- Requires-Dist: build==1.3.0; extra == 'dev'
737
- Requires-Dist: hatchling==1.27.0; extra == 'dev'
738
- Requires-Dist: importlib-metadata==8.7.0; extra == 'dev'
739
- Requires-Dist: mypy==1.17.1; extra == 'dev'
740
- Requires-Dist: ruff==0.12.7; extra == 'dev'
741
- Requires-Dist: sphinx-autodoc-typehints==3.2.0; extra == 'dev'
742
- Requires-Dist: sphinx-click==6.0.0; extra == 'dev'
743
- Requires-Dist: sphinx-rtd-dark-mode==1.3.0; extra == 'dev'
744
- Requires-Dist: sphinx-rtd-theme==3.0.2; extra == 'dev'
745
- Requires-Dist: sphinx==8.2.3; extra == 'dev'
746
- Requires-Dist: tox-uv==1.26.2; extra == 'dev'
747
- Requires-Dist: tox==4.28.4; extra == 'dev'
748
- Requires-Dist: twine==6.1.0; extra == 'dev'
749
- Requires-Dist: types-appdirs==1.4.3.5; extra == 'dev'
750
- Requires-Dist: types-filelock==3.2.7; extra == 'dev'
751
- Requires-Dist: types-paramiko==3.5.0.20250801; extra == 'dev'
752
- Requires-Dist: types-pytz==2025.2.0.20250516; extra == 'dev'
753
- Requires-Dist: types-tqdm==4.67.0.20250516; extra == 'dev'
754
- Requires-Dist: uv==0.8.4; extra == 'dev'
755
- Provides-Extra: noconda
756
- Requires-Dist: ataraxis-automation==5.0.0; extra == 'noconda'
757
- Requires-Dist: build==1.3.0; extra == 'noconda'
758
- Requires-Dist: sphinx-rtd-dark-mode==1.3.0; extra == 'noconda'
759
- Requires-Dist: sphinx-rtd-theme==3.0.2; extra == 'noconda'
711
+ Requires-Dist: ataraxis-automation<7,>=6.0.1; extra == 'dev'
712
+ Requires-Dist: tox-uv<2,>=1; extra == 'dev'
713
+ Requires-Dist: tox<5,>=4; extra == 'dev'
714
+ Requires-Dist: types-appdirs<2,>=1; extra == 'dev'
715
+ Requires-Dist: types-filelock<4,>=3; extra == 'dev'
716
+ Requires-Dist: types-paramiko<5,>=4; extra == 'dev'
717
+ Requires-Dist: types-pytz<2026,>=2025; extra == 'dev'
718
+ Requires-Dist: types-tqdm<5,>=4; extra == 'dev'
719
+ Requires-Dist: uv<1,>=0; extra == 'dev'
760
720
  Description-Content-Type: text/markdown
761
721
 
762
722
  # sl-shared-assets
763
- A Python library that stores assets shared between multiple Sun (NeuroAI) lab data pipelines.
723
+ A python library that provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
764
724
 
765
725
  ![PyPI - Version](https://img.shields.io/pypi/v/sl-shared-assets)
766
726
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sl-shared-assets)
@@ -777,14 +737,11 @@ ___
777
737
 
778
738
  Primarily, this library is designed to make the two main Sun lab libraries used for data acquisition
779
739
  ([sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment)) and processing
780
- ([sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)) independent of each other. This is beneficial, as both
781
- libraries feature an extensive and largely incompatible set of runtime dependencies. Moreover, having a shared
782
- repository of classes and tools reused across Sun lab pipelines streamlines the maintenance of these tools.
740
+ ([sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)) independent of each other.
783
741
 
784
- The library broadly stores two types of assets. First, it stores various dataclasses used to save the data acquired
785
- during experiments in the lab and the dataclasses used to configure data acquisition and processing runtimes. Secondly,
786
- it stores the tools used to safely move the data between the machines (computers) used in the data acquisition and
787
- processing, and provides the API for running various data processing jobs on remote compute servers.
742
+ The library broadly stores two types of assets. First, it stores dataclasses used to save the data acquired in the lab
743
+ and configure data acquisition and processing runtimes. Second, it provides the low-level tools and methods used to
744
+ manage the data at all stages of Sun lab data workflow: acquisition, processing, and analysis.
788
745
 
789
746
  ---
790
747
 
@@ -798,11 +755,12 @@ processing, and provides the API for running various data processing jobs on rem
798
755
  - [Authors](#authors)
799
756
  - [License](#license)
800
757
  - [Acknowledgements](#Acknowledgments)
758
+
801
759
  ___
802
760
 
803
761
  ## Dependencies
804
762
 
805
- All software library dependencies are installed automatically as part of library installation.
763
+ All software library dependencies are installed automatically as part of the library installation.
806
764
 
807
765
  ---
808
766
 
@@ -825,138 +783,23 @@ Use the following command to install the library using pip: ```pip install sl-sh
825
783
 
826
784
  ## Usage
827
785
 
828
- Most library components are intended to be used via other Sun lab libraries. Developers should study the API and CLI
829
- documentation below to learn how to use library components in other Sun lab libraries. For notes on using shared
830
- assets for data acquisition, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment) library ReadMe.
831
- For notes on using shared assets for data processing, see the [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)
832
- library ReadMe.
833
-
834
- The only exception to the note above is the **server.py** package exposed by this library. This package exposes an API
835
- for running headless and a CLI for running interactive Simple Linux Utility for Resource Management (SLURM)-managed
836
- jobs on remote compute clusters.
837
-
838
- ### Generating Access Credentials
839
-
840
- To access any remote server, the user is required to first generate the access credentials. The credentials are stored
841
- inside the 'server_credentials.yaml' file, which is generated by using the `sl-create-server-credentials` command.
842
- **Note!** Users are advised to generate this file in a secure (non-shared) location on their local machine.
843
-
844
- ### Running Headless Jobs
845
-
846
- A headless job is a job that does not require any user interaction during runtime. Currently, all headless jobs in the
847
- lab rely on pip-installable packages that expose a callable Command-Line Interface to carry out some type of
848
- data processing. In this regard, **running a headless job is equivalent to calling a CLI command on your local
849
- machine**, except that the command is executed on a remote compute server. Therefore, the primary purpose of the API
850
- exposed by this library is to transfer the target command request to the remote server, execute it, and monitor the
851
- runtime status until it is complete.
852
-
853
- For example, the [sl-suite2p package](https://github.com/Sun-Lab-NBB/suite2p) maintained in the lab exposes a CLI to
854
- process 2-Photon data from experiment sessions. During data processing by the
855
- [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library, a remote job is sent to the server that uses the CLI
856
- exposed by the sl-suite2p package to process target session(s).
857
-
858
- ### Creating Jobs
859
- All remote jobs are sent to the server in the form of an executable *shell* (.sh) script. The script is composed on the
860
- local machine that uses this library and transferred to a temporary server directory using Secure Shell File
861
- Transfer Protocol (SFTP). The server is then instructed to evaluate (run) the script using SLURM job manager, via a
862
- Secure Shell (SSH) session.
863
-
864
- Broadly, each job consists of three major steps, which correspond to three major sections of the job shell script:
865
- 1. **Setting up the job environment**. Each job script starts with a SLURM job parameter block, which tells SLURM
866
- what resources (CPUs, GPUs, RAM, etc.) the job requires. When resources become available, SLURM generates a virtual
867
- environment and runs the rest of the job script in that environment. This forms the basis for using the shared
868
- compute resources fairly, as SLURM balances resource allocation and the order of job execution for all users.
869
- 2. **Activating the target conda environment**. Currently, all jobs are assumed to use Python libraries to execute the
870
- intended data processing. Similar to processing data locally, each job expects the remote server to provide a
871
- Conda environment preconfigured with necessary assets (packages) to run the job. Therefore, each job contains a
872
- section that activates the user-defined conda environment before running the rest of the job.
873
- 3. **Executing processing**. The final section is typically unique to each job and calls specific CLI commands or runs
874
- specific Python modules. Since each job is submitted as a shell script, it can do anything a server shell can
875
- do. Therefore, despite python-centric approach to data processing in the lab, a remote job composed via this library
876
- can execute ***any*** arbitrary command available to the user on the remove server.
877
-
878
- Use the *Job* class exposed by this library to compose remote jobs. **Steps 1 and 2** of each job are configured when
879
- initializing the Job instance, while **step 3** is added via the `add_command()` method of the Job class:
880
- ```
881
- # First, import the job class
882
- from pathlib import Path
883
- from sl_shared_assets import Job
884
-
885
- # Next, instantiate a new Job object. For example, this job is used to verify the integrity of raw experiment data as
886
- # it is transferred to the long-term storage destination (server) by the sl-experiment library.
887
- job = Job(
888
- job_name="data_integrity_verification",
889
- output_log=Path("/temp/output.txt"),
890
- error_log=Path("/temp/errors.txt"),
891
- working_directory=Path("/temp/test_job"),
892
- conda_environment="test_environment",
893
- cpus_to_use=20,
894
- ram_gb=50,
895
- time_limit=20,
896
- )
897
-
898
- # Finally, add a CLI command call (the actual work to be done by the job). Here, the job calls the
899
- # 'sl-verify-session' command exposed by the sl-shared-assets library installed in the target environment on the server.
900
- # Use this method to add commands as you would type them in your local terminal / shell / command line.
901
- job.add_command(f"sl-verify-session -sp /temp/test_session")
902
- ```
903
-
904
- ### Submitting and Monitoring Jobs:
905
- To submit the job to the remote server, use a **Server** class instance. This class encapsulates access to the target
906
- remote compute server and uses the server_credentials.yaml file to determine server access credentials (see above):
907
- ```
908
- # Initialize the Server class using precreated server credentials file
909
- server = Server(credentials_path=Path("/temp/server_credentials.yaml"))
910
-
911
- # Submit the job (generated in the previous code snippet) to the server
912
- job = server.submit_job(job)
913
-
914
- # Wait for the server to complete the job
915
- delay_timer = PrecisionTimer("s")
916
- while not server.job_complete(job=job):
917
- delay_timer.delay_noblock(delay=5, allow_sleep=True)
918
- ```
919
-
920
- **Note!** The Server class only checks whether the job is running on the server, but not the outcome of the job. For
921
- that, you can either manually check the output and error logs for the job or come up with a programmatic way of
922
- checking the outcome. All developers are highly advised to study the API documentation for the Job and Server classes
923
- to use them most effectively.
924
-
925
- **Critical!** Since running remote jobs is largely equivalent to executing them locally, all users are highly encouraged
926
- to test their job scripts locally before deploying them server-side. If a script works on a local machine, it is likely
927
- that the script would behave similarly and work on the server.
928
-
929
- ### Interactive Jobs
930
-
931
- Interactive jobs are a special extension of the headless job type discussed above. Specifically, an interactive job is
932
- a headless job, whose only purpose is to **create and maintain a Jupyter lab server** under the SLURM control.
933
- Specifically, it requests SLURM to set up an isolated environment, starts a Jupyter server in that environment, and
934
- sends the credentials for the started server back to the user.
935
-
936
- In essence, this allocates a set of resources the user can use interactively by running various Jupyter notebooks.
937
- While convenient for certain data analysis cases, this type of jobs has the potential to inefficiently hog server
938
- resources for prolonged periods of time. Therefore, users are encouraged to only resort to this type of jobs when
939
- strictly necessary and to minimize the resources and time allocated to running these jobs.
940
-
941
- To run an interactive job, call the `sl-start-jupyter` CLI command exposed by this library and follow the instructions
942
- printed to the terminal by the command during runtime.
943
-
944
- **Critical!** While this command tries to minimize collisions with other users, it is possible that an access port
945
- collision occurs when multiple users try to instantiate a jupyter server at the same time. If you cannot authenticate
946
- with the Jupyter server, this likely indicates that the target port was in use and Jupyter automatically incremented the
947
- port number by 1. In this case, add 1 to your port number and try connecting to that port using the Jupyter credentials
948
- provided by the command. For example, if your target port was '8888,' try port '8889.'
786
+ Most library components are intended to be used via other Sun lab libraries. For details on using shared
787
+ assets for data acquisition and preprocessing, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment)
788
+ library. For details on using shared assets for data processing and dataset formation, see the
789
+ [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library.
949
790
 
950
- ---
791
+ ***Warning!*** End users should not use any component of this library directly or install this library into any Python
792
+ environment. All assets from this library are intended to be used exclusively by developers working on other Sun lab
793
+ libraries.
951
794
 
952
795
  ## API Documentation
953
796
 
954
- See the [API documentation](https://sl-shared-assets-api-docs.netlify.app/) for the
955
- detailed description of the methods and classes exposed by components of this library.
797
+ Developers working on integrating sl-shared-assets into other libraries should see the
798
+ [API documentation](https://sl-shared-assets-api-docs.netlify.app/) for the detailed description of the methods and
799
+ classes exposed by components of this library.
956
800
 
957
- **Note!** The API documentation includes important information about Command-Line-Interfaces (CLIs) exposed by this
958
- library as part of installation into a Python environment. All users are highly encouraged to study the CLI
959
- documentation to learn how to use library components via the terminal.
801
+ **Note!** The API documentation includes important information about Command-Line Interfaces (CLIs) exposed by this
802
+ library as part of installation into a Python environment.
960
803
 
961
804
  ___
962
805
 
@@ -0,0 +1,113 @@
1
+ # sl-shared-assets
2
+ A python library that provides data acquisition and processing assets shared between Sun (NeuroAI) lab libraries.
3
+
4
+ ![PyPI - Version](https://img.shields.io/pypi/v/sl-shared-assets)
5
+ ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sl-shared-assets)
6
+ [![uv](https://tinyurl.com/uvbadge)](https://github.com/astral-sh/uv)
7
+ [![Ruff](https://tinyurl.com/ruffbadge)](https://github.com/astral-sh/ruff)
8
+ ![type-checked: mypy](https://img.shields.io/badge/type--checked-mypy-blue?style=flat-square&logo=python)
9
+ ![PyPI - License](https://img.shields.io/pypi/l/sl-shared-assets)
10
+ ![PyPI - Status](https://img.shields.io/pypi/status/sl-shared-assets)
11
+ ![PyPI - Wheel](https://img.shields.io/pypi/wheel/sl-shared-assets)
12
+
13
+ ___
14
+
15
+ ## Detailed Description
16
+
17
+ Primarily, this library is designed to make the two main Sun lab libraries used for data acquisition
18
+ ([sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment)) and processing
19
+ ([sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery)) independent of each other.
20
+
21
+ The library broadly stores two types of assets. First, it stores dataclasses used to save the data acquired in the lab
22
+ and configure data acquisition and processing runtimes. Second, it provides the low-level tools and methods used to
23
+ manage the data at all stages of Sun lab data workflow: acquisition, processing, and analysis.
24
+
25
+ ---
26
+
27
+ ## Table of Contents
28
+
29
+ - [Dependencies](#dependencies)
30
+ - [Installation](#installation)
31
+ - [Usage](#usage)
32
+ - [API Documentation](#api-documentation)
33
+ - [Versioning](#versioning)
34
+ - [Authors](#authors)
35
+ - [License](#license)
36
+ - [Acknowledgements](#Acknowledgments)
37
+
38
+ ___
39
+
40
+ ## Dependencies
41
+
42
+ All software library dependencies are installed automatically as part of the library installation.
43
+
44
+ ---
45
+
46
+ ## Installation
47
+
48
+ ### Source
49
+
50
+ Note, installation from source is ***highly discouraged*** for everyone who is not an active project developer.
51
+
52
+ 1. Download this repository to your local machine using any method, such as Git-cloning. Use one
53
+ of the stable releases from [GitHub](https://github.com/Sun-Lab-NBB/sl-shared-assets/releases).
54
+ 2. Unpack the downloaded zip and note the path to the binary wheel (`.whl`) file contained in the archive.
55
+ 3. Run ```python -m pip install WHEEL_PATH```, replacing 'WHEEL_PATH' with the path to the wheel file, to install the
56
+ wheel into the active python environment.
57
+
58
+ ### pip
59
+ Use the following command to install the library using pip: ```pip install sl-shared-assets```.
60
+
61
+ ---
62
+
63
+ ## Usage
64
+
65
+ Most library components are intended to be used via other Sun lab libraries. For details on using shared
66
+ assets for data acquisition and preprocessing, see the [sl-experiment](https://github.com/Sun-Lab-NBB/sl-experiment)
67
+ library. For details on using shared assets for data processing and dataset formation, see the
68
+ [sl-forgery](https://github.com/Sun-Lab-NBB/sl-forgery) library.
69
+
70
+ ***Warning!*** End users should not use any component of this library directly or install this library into any Python
71
+ environment. All assets from this library are intended to be used exclusively by developers working on other Sun lab
72
+ libraries.
73
+
74
+ ## API Documentation
75
+
76
+ Developers working on integrating sl-shared-assets into other libraries should see the
77
+ [API documentation](https://sl-shared-assets-api-docs.netlify.app/) for the detailed description of the methods and
78
+ classes exposed by components of this library.
79
+
80
+ **Note!** The API documentation includes important information about Command-Line Interfaces (CLIs) exposed by this
81
+ library as part of installation into a Python environment.
82
+
83
+ ___
84
+
85
+ ## Versioning
86
+
87
+ This project uses [semantic versioning](https://semver.org/). For the versions available, see the
88
+ [tags on this repository](https://github.com/Sun-Lab-NBB/sl-shared-assets/tags).
89
+
90
+ ---
91
+
92
+ ## Authors
93
+
94
+ - Ivan Kondratyev ([Inkaros](https://github.com/Inkaros))
95
+ - Kushaan Gupta ([kushaangupta](https://github.com/kushaangupta))
96
+ - Natalie Yeung
97
+
98
+ ___
99
+
100
+ ## License
101
+
102
+ This project is licensed under the GPL3 License: see the [LICENSE](LICENSE) file for details.
103
+
104
+ ___
105
+
106
+ ## Acknowledgments
107
+
108
+ - All Sun lab [members](https://neuroai.github.io/sunlab/people) for providing the inspiration and comments during the
109
+ development of this library.
110
+ - The creators of all other projects used in the development automation pipelines and source code of this project
111
+ [see pyproject.toml](pyproject.toml).
112
+
113
+ ---
@@ -0,0 +1,34 @@
1
+ .. This file provides the instructions for how to display the API documentation generated using sphinx autodoc
2
+ extension. Use it to declare Python documentation sub-directories via appropriate modules (autodoc, etc.).
3
+
4
+ Command Line Interfaces
5
+ =======================
6
+
7
+ .. click:: sl_shared_assets.command_line_interfaces.manage:manage
8
+ :prog: sl-manage
9
+ :nested: full
10
+
11
+ .. click:: sl_shared_assets.command_line_interfaces.configure:configure
12
+ :prog: sl-configure
13
+ :nested: full
14
+
15
+ Tools
16
+ =====
17
+ .. automodule:: sl_shared_assets.tools
18
+ :members:
19
+ :undoc-members:
20
+ :show-inheritance:
21
+
22
+ Data and Configuration Assets
23
+ =============================
24
+ .. automodule:: sl_shared_assets.data_classes
25
+ :members:
26
+ :undoc-members:
27
+ :show-inheritance:
28
+
29
+ Server
30
+ ======
31
+ .. automodule:: sl_shared_assets.server
32
+ :members:
33
+ :undoc-members:
34
+ :show-inheritance:
@@ -6,7 +6,7 @@ lab libraries.
6
6
 
7
7
  This library is developed in the `Sun lab <https://neuroai.github.io/sunlab/>`_ at Cornell University. The main purpose
8
8
  of this library is to provide the tools and assets used by multiple other data acquisition and processing libraries in
9
- the lab to avoid cross-dependency issues.
9
+ the lab to avoid dependency collisions.
10
10
 
11
11
  This website only contains the API documentation for the classes and methods offered by this library. See the project
12
12
  GitHub repository for installation instructions and library usage examples:
@@ -0,0 +1,45 @@
1
+ name: slsa_dev_lin
2
+ channels:
3
+ - conda-forge
4
+ dependencies:
5
+ - _libgcc_mutex=0.1=conda_forge
6
+ - _openmp_mutex=4.5=2_gnu
7
+ - bzip2=1.0.8=h4bc722e_7
8
+ - ca-certificates=2025.8.3=hbd8a1cb_0
9
+ - cachetools=6.1.0=pyhd8ed1ab_0
10
+ - chardet=5.2.0=pyhd8ed1ab_3
11
+ - colorama=0.4.6=pyhd8ed1ab_1
12
+ - distlib=0.4.0=pyhd8ed1ab_0
13
+ - filelock=3.19.1=pyhd8ed1ab_0
14
+ - ld_impl_linux-64=2.44=h1423503_1
15
+ - libexpat=2.7.1=hecca717_0
16
+ - libffi=3.4.6=h2dba641_1
17
+ - libgcc=15.1.0=h767d61c_4
18
+ - libgcc-ng=15.1.0=h69a702a_4
19
+ - libgomp=15.1.0=h767d61c_4
20
+ - liblzma=5.8.1=hb9d3cd8_2
21
+ - libmpdec=4.0.0=hb9d3cd8_0
22
+ - libsqlite=3.50.4=h0c1763c_0
23
+ - libstdcxx=15.1.0=h8f9b012_4
24
+ - libuuid=2.38.1=h0b41bf4_0
25
+ - libzlib=1.3.1=hb9d3cd8_2
26
+ - ncurses=6.5=h2d0b736_3
27
+ - openssl=3.5.2=h26f9b46_0
28
+ - packaging=25.0=pyh29332c3_1
29
+ - pip=25.2=pyh145f28c_0
30
+ - platformdirs=4.3.8=pyhe01879c_0
31
+ - pluggy=1.6.0=pyhd8ed1ab_0
32
+ - pyproject-api=1.9.1=pyhe01879c_0
33
+ - python=3.13.5=hec9711d_102_cp313
34
+ - python-uv=0.8.11=pyhe01879c_0
35
+ - python_abi=3.13=8_cp313
36
+ - readline=8.2=h8c095d6_2
37
+ - tk=8.6.13=noxft_hd72426e_102
38
+ - tomli=2.2.1=pyhe01879c_2
39
+ - tox=4.28.4=pyhe01879c_0
40
+ - tox-uv=1.28.0=pyhd8ed1ab_0
41
+ - typing_extensions=4.14.1=pyhe01879c_0
42
+ - tzdata=2025b=h78e105d_0
43
+ - uv=0.8.11=heb9285d_0
44
+ - virtualenv=20.34.0=pyhd8ed1ab_0
45
+
@@ -0,0 +1,42 @@
1
+ List of packages in environment: "/home/cybermouse/miniforge3/envs/slsa_dev_lin"
2
+
3
+ https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
4
+ https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2
5
+ https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda
6
+ https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.8.3-hbd8a1cb_0.conda
7
+ https://conda.anaconda.org/conda-forge/noarch/cachetools-6.1.0-pyhd8ed1ab_0.conda
8
+ https://conda.anaconda.org/conda-forge/noarch/chardet-5.2.0-pyhd8ed1ab_3.conda
9
+ https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda
10
+ https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda
11
+ https://conda.anaconda.org/conda-forge/noarch/filelock-3.19.1-pyhd8ed1ab_0.conda
12
+ https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.44-h1423503_1.conda
13
+ https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.1-hecca717_0.conda
14
+ https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.6-h2dba641_1.conda
15
+ https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.1.0-h767d61c_4.conda
16
+ https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.1.0-h69a702a_4.conda
17
+ https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.1.0-h767d61c_4.conda
18
+ https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.1-hb9d3cd8_2.conda
19
+ https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb9d3cd8_0.conda
20
+ https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.50.4-h0c1763c_0.conda
21
+ https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.1.0-h8f9b012_4.conda
22
+ https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda
23
+ https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda
24
+ https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda
25
+ https://conda.anaconda.org/conda-forge/linux-64/openssl-3.5.2-h26f9b46_0.conda
26
+ https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda
27
+ https://conda.anaconda.org/conda-forge/noarch/pip-25.2-pyh145f28c_0.conda
28
+ https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.3.8-pyhe01879c_0.conda
29
+ https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda
30
+ https://conda.anaconda.org/conda-forge/noarch/pyproject-api-1.9.1-pyhe01879c_0.conda
31
+ https://conda.anaconda.org/conda-forge/linux-64/python-3.13.5-hec9711d_102_cp313.conda
32
+ https://conda.anaconda.org/conda-forge/noarch/python-uv-0.8.11-pyhe01879c_0.conda
33
+ https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda
34
+ https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda
35
+ https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd72426e_102.conda
36
+ https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhe01879c_2.conda
37
+ https://conda.anaconda.org/conda-forge/noarch/tox-4.28.4-pyhe01879c_0.conda
38
+ https://conda.anaconda.org/conda-forge/noarch/tox-uv-1.28.0-pyhd8ed1ab_0.conda
39
+ https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.14.1-pyhe01879c_0.conda
40
+ https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda
41
+ https://conda.anaconda.org/conda-forge/linux-64/uv-0.8.11-heb9285d_0.conda
42
+ https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.34.0-pyhd8ed1ab_0.conda
@@ -0,0 +1,39 @@
1
+ name: slsa_dev_osx
2
+ channels:
3
+ - conda-forge
4
+ dependencies:
5
+ - bzip2=1.0.8=h99b78c6_7
6
+ - ca-certificates=2025.8.3=hbd8a1cb_0
7
+ - cachetools=6.1.0=pyhd8ed1ab_0
8
+ - chardet=5.2.0=pyhd8ed1ab_3
9
+ - colorama=0.4.6=pyhd8ed1ab_1
10
+ - distlib=0.4.0=pyhd8ed1ab_0
11
+ - filelock=3.19.1=pyhd8ed1ab_0
12
+ - icu=75.1=hfee45f7_0
13
+ - libcxx=20.1.8=hf598326_1
14
+ - libexpat=2.7.1=hec049ff_0
15
+ - libffi=3.4.6=h1da3d7d_1
16
+ - liblzma=5.8.1=h39f12f2_2
17
+ - libmpdec=4.0.0=h5505292_0
18
+ - libsqlite=3.50.4=h4237e3c_0
19
+ - libzlib=1.3.1=h8359307_2
20
+ - ncurses=6.5=h5e97a16_3
21
+ - openssl=3.5.2=he92f556_0
22
+ - packaging=25.0=pyh29332c3_1
23
+ - pip=25.2=pyh145f28c_0
24
+ - platformdirs=4.3.8=pyhe01879c_0
25
+ - pluggy=1.6.0=pyhd8ed1ab_0
26
+ - pyproject-api=1.9.1=pyhe01879c_0
27
+ - python=3.13.5=hf3f3da0_102_cp313
28
+ - python-uv=0.8.11=pyhe01879c_0
29
+ - python_abi=3.13=8_cp313
30
+ - readline=8.2=h1d1bf99_2
31
+ - tk=8.6.13=h892fb3f_2
32
+ - tomli=2.2.1=pyhe01879c_2
33
+ - tox=4.28.4=pyhe01879c_0
34
+ - tox-uv=1.28.0=pyhd8ed1ab_0
35
+ - typing_extensions=4.14.1=pyhe01879c_0
36
+ - tzdata=2025b=h78e105d_0
37
+ - uv=0.8.11=hb521335_0
38
+ - virtualenv=20.34.0=pyhd8ed1ab_0
39
+
@@ -0,0 +1,36 @@
1
+ List of packages in environment: "/Users/InfamousOne/miniforge3/envs/slsa_dev_osx"
2
+
3
+ https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h99b78c6_7.conda
4
+ https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025.8.3-hbd8a1cb_0.conda
5
+ https://conda.anaconda.org/conda-forge/noarch/cachetools-6.1.0-pyhd8ed1ab_0.conda
6
+ https://conda.anaconda.org/conda-forge/noarch/chardet-5.2.0-pyhd8ed1ab_3.conda
7
+ https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda
8
+ https://conda.anaconda.org/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda
9
+ https://conda.anaconda.org/conda-forge/noarch/filelock-3.19.1-pyhd8ed1ab_0.conda
10
+ https://conda.anaconda.org/conda-forge/osx-arm64/icu-75.1-hfee45f7_0.conda
11
+ https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-20.1.8-hf598326_1.conda
12
+ https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.1-hec049ff_0.conda
13
+ https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.6-h1da3d7d_1.conda
14
+ https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.1-h39f12f2_2.conda
15
+ https://conda.anaconda.org/conda-forge/osx-arm64/libmpdec-4.0.0-h5505292_0.conda
16
+ https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.50.4-h4237e3c_0.conda
17
+ https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda
18
+ https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda
19
+ https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.5.2-he92f556_0.conda
20
+ https://conda.anaconda.org/conda-forge/noarch/packaging-25.0-pyh29332c3_1.conda
21
+ https://conda.anaconda.org/conda-forge/noarch/pip-25.2-pyh145f28c_0.conda
22
+ https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.3.8-pyhe01879c_0.conda
23
+ https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhd8ed1ab_0.conda
24
+ https://conda.anaconda.org/conda-forge/noarch/pyproject-api-1.9.1-pyhe01879c_0.conda
25
+ https://conda.anaconda.org/conda-forge/osx-arm64/python-3.13.5-hf3f3da0_102_cp313.conda
26
+ https://conda.anaconda.org/conda-forge/noarch/python-uv-0.8.11-pyhe01879c_0.conda
27
+ https://conda.anaconda.org/conda-forge/noarch/python_abi-3.13-8_cp313.conda
28
+ https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda
29
+ https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h892fb3f_2.conda
30
+ https://conda.anaconda.org/conda-forge/noarch/tomli-2.2.1-pyhe01879c_2.conda
31
+ https://conda.anaconda.org/conda-forge/noarch/tox-4.28.4-pyhe01879c_0.conda
32
+ https://conda.anaconda.org/conda-forge/noarch/tox-uv-1.28.0-pyhd8ed1ab_0.conda
33
+ https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.14.1-pyhe01879c_0.conda
34
+ https://conda.anaconda.org/conda-forge/noarch/tzdata-2025b-h78e105d_0.conda
35
+ https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.8.11-hb521335_0.conda
36
+ https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.34.0-pyhd8ed1ab_0.conda