np_codeocean 0.3.5__tar.gz → 0.3.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/PKG-INFO +38 -6
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/README.md +51 -28
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/pyproject.toml +83 -59
- np_codeocean-0.3.7/setup.cfg +4 -0
- np_codeocean-0.3.7/src/np_codeocean/__init__.py +2 -0
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/__init__.py +1 -1
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/common.py +1 -3
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/core.py +333 -331
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/dynamic_routing_task_etl.py +1 -1
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/model_templates/behavior_box.py +115 -115
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/model_templates/neuropixels_rig.py +544 -544
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/np.py +1 -1
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/rigs.py +1 -1
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/storage.py +78 -78
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/utils.py +1 -1
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/np_session_utils.py +462 -385
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/scripts/upload_dynamic_routing_behavior.py +483 -413
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/scripts/upload_dynamic_routing_ecephys.py +279 -217
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/scripts/upload_split_recordings_example.py +39 -33
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/utils.py +671 -563
- np_codeocean-0.3.7/src/np_codeocean.egg-info/PKG-INFO +91 -0
- np_codeocean-0.3.7/src/np_codeocean.egg-info/SOURCES.txt +27 -0
- np_codeocean-0.3.7/src/np_codeocean.egg-info/dependency_links.txt +1 -0
- np_codeocean-0.3.7/src/np_codeocean.egg-info/entry_points.txt +4 -0
- np_codeocean-0.3.7/src/np_codeocean.egg-info/requires.txt +25 -0
- np_codeocean-0.3.7/src/np_codeocean.egg-info/top_level.txt +1 -0
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/tests/test_metadata.py +99 -99
- np_codeocean-0.3.5/src/np_codeocean/__init__.py +0 -2
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/model_templates/__init__.py +0 -0
- {np_codeocean-0.3.5 → np_codeocean-0.3.7}/src/np_codeocean/metadata/update.py +1 -2
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: np_codeocean
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.7
|
|
4
4
|
Summary: Tools for uploading and interacting with Mindscope Neuropixels experiments on Code Ocean
|
|
5
|
-
Author-
|
|
5
|
+
Author-email: Ben Hardcastle <ben.hardcastle@alleninstitute.org>, Chris Mochizuki <chrism@alleninstitute.org>, Arjun Sridhar <arjun.sridhar@alleninstitute.org>
|
|
6
6
|
License: MIT
|
|
7
|
+
Project-URL: Source, https://github.com/AllenInstitute/np_codeocean
|
|
8
|
+
Project-URL: Issues, https://github.com/AllenInstitute/np_codeocean/issues
|
|
7
9
|
Classifier: Programming Language :: Python :: 3
|
|
8
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
9
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
@@ -11,9 +13,8 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
11
13
|
Classifier: License :: OSI Approved :: MIT License
|
|
12
14
|
Classifier: Operating System :: Microsoft :: Windows
|
|
13
15
|
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
-
Project-URL: Source, https://github.com/AllenInstitute/np_codeocean
|
|
15
|
-
Project-URL: Issues, https://github.com/AllenInstitute/np_codeocean/issues
|
|
16
16
|
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
17
18
|
Requires-Dist: np_session>=0.6.44
|
|
18
19
|
Requires-Dist: np-tools>=0.1.23
|
|
19
20
|
Requires-Dist: np-config>=0.4.33
|
|
@@ -27,7 +28,15 @@ Requires-Dist: cryptography<43.0
|
|
|
27
28
|
Requires-Dist: aind-data-transfer-service>=1.15.0
|
|
28
29
|
Requires-Dist: aind-slurm-rest-v2==0.0.3
|
|
29
30
|
Requires-Dist: aind-codeocean-pipeline-monitor>=0.5.2
|
|
30
|
-
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: bump>=1.3.2; extra == "dev"
|
|
33
|
+
Provides-Extra: dynamicrouting
|
|
34
|
+
Requires-Dist: npc-lims>=0.1.154; extra == "dynamicrouting"
|
|
35
|
+
Requires-Dist: npc-sessions>=0.0.273; extra == "dynamicrouting"
|
|
36
|
+
Requires-Dist: aind-codeocean-pipeline-monitor[full]>=0.5.0; extra == "dynamicrouting"
|
|
37
|
+
Requires-Dist: aind-metadata-mapper==0.18.2; extra == "dynamicrouting"
|
|
38
|
+
Provides-Extra: openscope
|
|
39
|
+
Requires-Dist: aind-metadata-mapper==0.18.1; extra == "openscope"
|
|
31
40
|
|
|
32
41
|
# np_codeocean
|
|
33
42
|
Tools for uploading Mindscope Neuropixels experiments to S3 (for Code Ocean).
|
|
@@ -35,6 +44,29 @@ Tools for uploading Mindscope Neuropixels experiments to S3 (for Code Ocean).
|
|
|
35
44
|
Requires running as admin on Windows in order to create remote-to-remote symlinks
|
|
36
45
|
on the Isilon.
|
|
37
46
|
|
|
47
|
+
## Install
|
|
48
|
+
For Dynamic Routing:
|
|
49
|
+
```shell
|
|
50
|
+
uv sync --extra dynamicrouting
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
For OpenScope:
|
|
54
|
+
```shell
|
|
55
|
+
uv sync --extra openscope
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Add dependencies
|
|
59
|
+
For shared utilities:
|
|
60
|
+
```shell
|
|
61
|
+
uv add <package-name>
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
For Dynamic Routing or OpenScope specific utilities:
|
|
65
|
+
```shell
|
|
66
|
+
uv add <package-name> --optional <dynamicrouting|openscope>
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Usage
|
|
38
70
|
- `upload` CLI tool is provided, which uses the
|
|
39
71
|
[`np_session`](https://github.com/AllenInstitute/np_session) interface to find
|
|
40
72
|
and upload
|
|
@@ -1,28 +1,51 @@
|
|
|
1
|
-
# np_codeocean
|
|
2
|
-
Tools for uploading Mindscope Neuropixels experiments to S3 (for Code Ocean).
|
|
3
|
-
|
|
4
|
-
Requires running as admin on Windows in order to create remote-to-remote symlinks
|
|
5
|
-
on the Isilon.
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
1
|
+
# np_codeocean
|
|
2
|
+
Tools for uploading Mindscope Neuropixels experiments to S3 (for Code Ocean).
|
|
3
|
+
|
|
4
|
+
Requires running as admin on Windows in order to create remote-to-remote symlinks
|
|
5
|
+
on the Isilon.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
For Dynamic Routing:
|
|
9
|
+
```shell
|
|
10
|
+
uv sync --extra dynamicrouting
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
For OpenScope:
|
|
14
|
+
```shell
|
|
15
|
+
uv sync --extra openscope
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Add dependencies
|
|
19
|
+
For shared utilities:
|
|
20
|
+
```shell
|
|
21
|
+
uv add <package-name>
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
For Dynamic Routing or OpenScope specific utilities:
|
|
25
|
+
```shell
|
|
26
|
+
uv add <package-name> --optional <dynamicrouting|openscope>
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Usage
|
|
30
|
+
- `upload` CLI tool is provided, which uses the
|
|
31
|
+
[`np_session`](https://github.com/AllenInstitute/np_session) interface to find
|
|
32
|
+
and upload
|
|
33
|
+
raw data for one ecephys session:
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
pip install np_codeocean
|
|
37
|
+
upload <session-id>
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
where session-id is any valid input to `np_session.Session()`, e.g.:
|
|
41
|
+
- a lims ID (`1333741475`)
|
|
42
|
+
- a workgroups foldername (`DRPilot_366122_20230101`)
|
|
43
|
+
- a path to a session folder ( `\\allen\programs\mindscope\workgroups\np-exp\1333741475_719667_20240227`)
|
|
44
|
+
|
|
45
|
+
- a folder of symlinks pointing to the raw data is created, with a new structure suitable for the KS2.5 sorting pipeline on Code Ocean
|
|
46
|
+
- the symlink folder, plus metadata, are entered into a csv file, which is
|
|
47
|
+
submitted to [`http://aind-data-transfer-service`](http://aind-data-transfer-service), which in turn runs the
|
|
48
|
+
[`aind-data-transfer`](https://github.com/AllenNeuralDynamics/aind-data-transfer)
|
|
49
|
+
tool on the HPC, which follows the symlinks to the original data,
|
|
50
|
+
median-subtracts/scales/compresses ephys data, then uploads with the AWS CLI tool
|
|
51
|
+
- all compression/zipping acts on copies in temporary folders: the original raw data is not altered in anyway
|
|
@@ -1,53 +1,7 @@
|
|
|
1
|
-
[tool.pdm.scripts]
|
|
2
|
-
ruff = "ruff --fix src"
|
|
3
|
-
blue = "blue src"
|
|
4
|
-
pytest = "pytest --cov"
|
|
5
|
-
bump = "bump -p pyproject.toml"
|
|
6
|
-
|
|
7
|
-
[tool.pdm.scripts.prebuild]
|
|
8
|
-
composite = [
|
|
9
|
-
"blue",
|
|
10
|
-
"ruff",
|
|
11
|
-
"pytest",
|
|
12
|
-
]
|
|
13
|
-
|
|
14
|
-
[tool.pdm.scripts.build]
|
|
15
|
-
composite = [
|
|
16
|
-
"prebuild",
|
|
17
|
-
"pdm build",
|
|
18
|
-
]
|
|
19
|
-
|
|
20
|
-
[tool.pdm.scripts.dry-run]
|
|
21
|
-
composite = [
|
|
22
|
-
"prebuild",
|
|
23
|
-
"pdm build",
|
|
24
|
-
"pdm publish --no-build --repository https://test.pypi.org/simple",
|
|
25
|
-
]
|
|
26
|
-
|
|
27
|
-
[tool.pdm.scripts.publish]
|
|
28
|
-
composite = [
|
|
29
|
-
"prebuild",
|
|
30
|
-
"bump",
|
|
31
|
-
"pdm build",
|
|
32
|
-
"pdm publish --no-build",
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
[tool.pdm.scripts.pub]
|
|
36
|
-
composite = [
|
|
37
|
-
"bump",
|
|
38
|
-
"pdm publish",
|
|
39
|
-
]
|
|
40
|
-
|
|
41
|
-
[tool.uv]
|
|
42
|
-
package = true
|
|
43
|
-
default-groups = [
|
|
44
|
-
"dev",
|
|
45
|
-
"dynamicrouting",
|
|
46
|
-
]
|
|
47
1
|
|
|
48
2
|
[project]
|
|
49
3
|
name = "np_codeocean"
|
|
50
|
-
version = "0.3.
|
|
4
|
+
version = "0.3.7"
|
|
51
5
|
description = "Tools for uploading and interacting with Mindscope Neuropixels experiments on Code Ocean"
|
|
52
6
|
authors = [
|
|
53
7
|
{ name = "Ben Hardcastle", email = "ben.hardcastle@alleninstitute.org" },
|
|
@@ -93,24 +47,94 @@ upload_dr_behavior = "np_codeocean.scripts.upload_dynamic_routing_behavior:main"
|
|
|
93
47
|
upload_dr_ecephys = "np_codeocean.scripts.upload_dynamic_routing_ecephys:main"
|
|
94
48
|
upload_sessions = "np_codeocean.scripts.upload_sessions:main"
|
|
95
49
|
|
|
96
|
-
[
|
|
97
|
-
dev = [
|
|
98
|
-
"bump>=1.3.2",
|
|
99
|
-
"pdm>=2.4.9",
|
|
100
|
-
]
|
|
50
|
+
[project.optional-dependencies]
|
|
51
|
+
dev = ["bump>=1.3.2"]
|
|
101
52
|
dynamicrouting = [
|
|
102
53
|
"npc-lims>=0.1.154",
|
|
103
54
|
"npc-sessions>=0.0.273",
|
|
104
55
|
"aind-codeocean-pipeline-monitor[full]>=0.5.0",
|
|
105
56
|
"aind-metadata-mapper==0.18.2",
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
"
|
|
109
|
-
"npc-sync>=0.1.25",
|
|
57
|
+
]
|
|
58
|
+
openscope = [
|
|
59
|
+
"aind-metadata-mapper==0.18.1",
|
|
110
60
|
]
|
|
111
61
|
|
|
62
|
+
[tool.uv]
|
|
63
|
+
conflicts = [
|
|
64
|
+
[
|
|
65
|
+
{ extra = "dynamicrouting" },
|
|
66
|
+
{ extra = "openscope" },
|
|
67
|
+
],
|
|
68
|
+
]
|
|
69
|
+
package = true
|
|
70
|
+
default-groups = "all"
|
|
71
|
+
|
|
72
|
+
|
|
112
73
|
[build-system]
|
|
113
|
-
requires = [
|
|
114
|
-
|
|
74
|
+
requires = ["setuptools>=61", "wheel"]
|
|
75
|
+
build-backend = "setuptools.build_meta"
|
|
76
|
+
|
|
77
|
+
[dependency-groups]
|
|
78
|
+
task_runner = ["poethepoet>=0.33.1"]
|
|
79
|
+
linting = ["black>=25.1.0", "isort>=6.0.1", "ruff>=0.11.2"]
|
|
80
|
+
testing = ["mypy>=1.4.1"]
|
|
81
|
+
bump = ["bump-my-version>=1.1.1", "git-changelog>=2.5.3"]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
[tool.poe.tasks]
|
|
85
|
+
mypy = "mypy src --install-types --non-interactive"
|
|
86
|
+
test = ["mypy"]
|
|
87
|
+
ruff = "ruff check src --fix-only"
|
|
88
|
+
black = "black src"
|
|
89
|
+
format = ["ruff", "black"]
|
|
90
|
+
bump_pub_tag = "bump-my-version bump patch"
|
|
91
|
+
release = ["format", "bump_pub_tag"]
|
|
92
|
+
|
|
93
|
+
[tool.bumpversion]
|
|
94
|
+
tag = true
|
|
95
|
+
allow_dirty = true
|
|
96
|
+
commit = true
|
|
97
|
+
commit_args = "-a"
|
|
98
|
+
message = "Bump version: {current_version} → {new_version} [skip actions]"
|
|
99
|
+
pre_commit_hooks = [
|
|
100
|
+
"uv lock --upgrade-package np_workflows",
|
|
101
|
+
"git-changelog -o CHANGELOG.md",
|
|
102
|
+
"uv build",
|
|
103
|
+
"uv publish",
|
|
104
|
+
]
|
|
105
|
+
post_commit_hooks = [
|
|
106
|
+
"git push origin main",
|
|
107
|
+
"git push --tags",
|
|
108
|
+
]
|
|
109
|
+
|
|
110
|
+
[tool.ruff]
|
|
111
|
+
line-length = 100
|
|
112
|
+
select = [
|
|
113
|
+
"B", # flake8-bugbear
|
|
114
|
+
"C4", # flake8-comprehensions
|
|
115
|
+
"E", # pycodestyle
|
|
116
|
+
"F", # pyflakes
|
|
117
|
+
"PGH", # pygrep-hooks
|
|
118
|
+
"RUF", # ruff
|
|
119
|
+
"W", # pycodestyle
|
|
120
|
+
"YTT", # flake8-2020
|
|
121
|
+
"ANN",
|
|
122
|
+
"C",
|
|
123
|
+
"I",
|
|
124
|
+
"RUF100",
|
|
125
|
+
"N",
|
|
126
|
+
"UP",
|
|
115
127
|
]
|
|
116
|
-
|
|
128
|
+
# extend-ignore = ["B018", "B019"]
|
|
129
|
+
src = ["src"]
|
|
130
|
+
extend-exclude = ["tests/fixtures"]
|
|
131
|
+
target-version = "py39"
|
|
132
|
+
|
|
133
|
+
[tool.ruff.flake8-tidy-imports]
|
|
134
|
+
ban-relative-imports = "all"
|
|
135
|
+
|
|
136
|
+
[tool.ruff.mccabe]
|
|
137
|
+
max-complexity = 10
|
|
138
|
+
|
|
139
|
+
[tool.ruff.isort]
|
|
140
|
+
known-first-party = ["np_workflows"]
|