npc_lims 0.1.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- npc_lims/__init__.py +0 -0
- npc_lims/jobs/__init__.py +1 -0
- npc_lims/paths/__init__.py +0 -0
- npc_lims/paths/codeocean.py +132 -0
- npc_lims/py.typed +1 -0
- npc_lims/state/__init__.py +1 -0
- npc_lims/state/redis_state.py +125 -0
- npc_lims/status/__init__.py +0 -0
- npc_lims-0.1.8.dist-info/METADATA +40 -0
- npc_lims-0.1.8.dist-info/RECORD +12 -0
- npc_lims-0.1.8.dist-info/WHEEL +5 -0
- npc_lims-0.1.8.dist-info/top_level.txt +1 -0
npc_lims/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
File without changes
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import functools
|
|
4
|
+
import operator
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
from typing import Any, Iterator, Literal
|
|
8
|
+
|
|
9
|
+
import npc_session
|
|
10
|
+
import npc_session.parsing as parsing
|
|
11
|
+
import upath
|
|
12
|
+
from aind_codeocean_api import codeocean # type: ignore
|
|
13
|
+
from typing_extensions import TypeAlias
|
|
14
|
+
|
|
15
|
+
CODE_OCEAN_API_TOKEN = os.getenv("CODE_OCEAN_API_TOKEN")
|
|
16
|
+
CODE_OCEAN_DOMAIN = os.getenv("CODE_OCEAN_DOMAIN")
|
|
17
|
+
|
|
18
|
+
DataAsset: TypeAlias = dict[
|
|
19
|
+
Literal[
|
|
20
|
+
"created",
|
|
21
|
+
"custom_metadata",
|
|
22
|
+
"description",
|
|
23
|
+
"files",
|
|
24
|
+
"id",
|
|
25
|
+
"last_used",
|
|
26
|
+
"name",
|
|
27
|
+
"size",
|
|
28
|
+
"sourceBucket",
|
|
29
|
+
"state",
|
|
30
|
+
"tags",
|
|
31
|
+
"type",
|
|
32
|
+
],
|
|
33
|
+
Any,
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@functools.cache
|
|
38
|
+
def get_subject_data_assets(subject: str | int) -> tuple[DataAsset, ...]:
|
|
39
|
+
"""
|
|
40
|
+
>>> assets = get_subject_data_assets(668759)
|
|
41
|
+
>>> assert len(assets) > 0
|
|
42
|
+
"""
|
|
43
|
+
codeocean_client = codeocean.CodeOceanClient(
|
|
44
|
+
domain=CODE_OCEAN_DOMAIN, token=CODE_OCEAN_API_TOKEN
|
|
45
|
+
)
|
|
46
|
+
response = codeocean_client.search_data_assets(
|
|
47
|
+
query=f"subject id: {npc_session.SubjectRecord(subject)}"
|
|
48
|
+
)
|
|
49
|
+
response.raise_for_status()
|
|
50
|
+
return response.json()["results"]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def get_sessions_with_data_assets(
|
|
54
|
+
subject: str | int,
|
|
55
|
+
) -> tuple[npc_session.SessionRecord, ...]:
|
|
56
|
+
"""
|
|
57
|
+
>>> sessions = get_sessions_with_data_assets(668759)
|
|
58
|
+
>>> assert len(sessions) > 0
|
|
59
|
+
"""
|
|
60
|
+
assets = get_subject_data_assets(subject)
|
|
61
|
+
return tuple({npc_session.SessionRecord(asset["name"]) for asset in assets})
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@functools.cache
|
|
65
|
+
def get_session_data_assets(
|
|
66
|
+
session: str | npc_session.SessionRecord,
|
|
67
|
+
) -> tuple[DataAsset, ...]:
|
|
68
|
+
session = npc_session.SessionRecord(session)
|
|
69
|
+
assets = get_subject_data_assets(session.subject)
|
|
70
|
+
return tuple(
|
|
71
|
+
asset
|
|
72
|
+
for asset in assets
|
|
73
|
+
if re.match(
|
|
74
|
+
f"ecephys_{session.subject}_{session.date}_{parsing._TIME}", asset["name"]
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@functools.cache
|
|
80
|
+
def get_raw_data_root(session: str | npc_session.SessionRecord) -> upath.UPath:
|
|
81
|
+
"""
|
|
82
|
+
>>> get_raw_data_root('668759_20230711')
|
|
83
|
+
S3Path('s3://aind-ephys-data/ecephys_668759_2023-07-11_13-07-32')
|
|
84
|
+
"""
|
|
85
|
+
session = npc_session.SessionRecord(session)
|
|
86
|
+
raw_assets = tuple(
|
|
87
|
+
asset
|
|
88
|
+
for asset in get_session_data_assets(session)
|
|
89
|
+
if asset["custom_metadata"].get("data level") == "raw data"
|
|
90
|
+
)
|
|
91
|
+
if len(raw_assets) < session.idx:
|
|
92
|
+
raise ValueError(
|
|
93
|
+
f"Number of paths raw sessions on s3 {len(raw_assets)} is less than {session.idx = }"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
raw_asset = raw_assets[session.idx]
|
|
97
|
+
bucket_info = raw_asset["sourceBucket"]
|
|
98
|
+
roots = {"aws": "s3", "gcs": "gs"}
|
|
99
|
+
if bucket_info["origin"] not in roots:
|
|
100
|
+
raise RuntimeError(
|
|
101
|
+
f"Unknown bucket origin - not sure how to create UPath: {bucket_info = }"
|
|
102
|
+
)
|
|
103
|
+
return upath.UPath(
|
|
104
|
+
f"{roots[bucket_info['origin']]}://{bucket_info['bucket']}/{bucket_info['prefix']}"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@functools.cache
|
|
109
|
+
def get_raw_data_top_level_paths(
|
|
110
|
+
session: str | npc_session.SessionRecord,
|
|
111
|
+
) -> tuple[upath.UPath, ...]:
|
|
112
|
+
"""
|
|
113
|
+
>>> files = get_raw_data_top_level_paths('668759_20230711')
|
|
114
|
+
>>> assert len(files) > 0
|
|
115
|
+
"""
|
|
116
|
+
raw_data_root = get_raw_data_root(session)
|
|
117
|
+
directories: Iterator = (
|
|
118
|
+
directory for directory in raw_data_root.iterdir() if directory.is_dir()
|
|
119
|
+
)
|
|
120
|
+
first_level_files_directories: Iterator = (
|
|
121
|
+
tuple(directory.iterdir()) for directory in directories
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
return functools.reduce(operator.add, first_level_files_directories)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
if __name__ == "__main__":
|
|
128
|
+
import doctest
|
|
129
|
+
|
|
130
|
+
doctest.testmod(
|
|
131
|
+
optionflags=(doctest.IGNORE_EXCEPTION_DETAIL | doctest.NORMALIZE_WHITESPACE)
|
|
132
|
+
)
|
npc_lims/py.typed
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Redis connection for persisting session state.
|
|
3
|
+
"""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import collections.abc
|
|
7
|
+
import contextlib
|
|
8
|
+
import logging
|
|
9
|
+
import os
|
|
10
|
+
from typing import ClassVar, Iterator, Union
|
|
11
|
+
|
|
12
|
+
import redis
|
|
13
|
+
from typing_extensions import TypeAlias
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
# AcceptedType will be coerced to RedisType before being stored in Redis
|
|
18
|
+
RedisType: TypeAlias = Union[str, int, float]
|
|
19
|
+
"""Can be stored in Redis directly, or returned from Redis."""
|
|
20
|
+
|
|
21
|
+
AcceptedType: TypeAlias = Union[RedisType, bool, None]
|
|
22
|
+
"""Can be stored in Redis after using `encode(value: AcceptedType)`."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class State(collections.abc.MutableMapping):
|
|
26
|
+
"""Get and set session state in Redis via a dict interface.
|
|
27
|
+
|
|
28
|
+
- dict interface provides `keys`, `get`, `setdefault`, `pop`, etc.
|
|
29
|
+
- accepted value types are str, int, float, bool, None
|
|
30
|
+
|
|
31
|
+
>>> test_id = 0
|
|
32
|
+
>>> state = State(test_id)
|
|
33
|
+
>>> state['test'] = 1.0
|
|
34
|
+
>>> state['test']
|
|
35
|
+
1.0
|
|
36
|
+
>>> state['test'] = 'test'
|
|
37
|
+
>>> state['test']
|
|
38
|
+
'test'
|
|
39
|
+
>>> all('test' in _ for _ in (state, state.keys(), state.values()))
|
|
40
|
+
True
|
|
41
|
+
>>> state.setdefault('test', True)
|
|
42
|
+
'test'
|
|
43
|
+
>>> state.pop('test')
|
|
44
|
+
'test'
|
|
45
|
+
>>> del state['test']
|
|
46
|
+
>>> state.get('test') is None
|
|
47
|
+
True
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
db: ClassVar[redis.Redis]
|
|
51
|
+
|
|
52
|
+
def __init__(self, id: int | str) -> None:
|
|
53
|
+
self.name = str(id)
|
|
54
|
+
try:
|
|
55
|
+
_ = self.db
|
|
56
|
+
except AttributeError:
|
|
57
|
+
self.__class__.connect()
|
|
58
|
+
|
|
59
|
+
def __repr__(self) -> str:
|
|
60
|
+
return self.data.__repr__()
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def connect(cls) -> None:
|
|
64
|
+
cls.db = redis.Redis(
|
|
65
|
+
host="redis-11357.c1.us-west-2-2.ec2.cloud.redislabs.com",
|
|
66
|
+
port=11357,
|
|
67
|
+
password=os.environ["REDIS_DEFAULT_USER_PASSWORD"],
|
|
68
|
+
)
|
|
69
|
+
if cls.db.ping():
|
|
70
|
+
logger.debug("Connected to Redis database: %s", cls.db)
|
|
71
|
+
else:
|
|
72
|
+
logger.error("Failed to connect to Redis database")
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def data(self) -> dict[str, AcceptedType]:
|
|
76
|
+
return {k.decode(): decode(v) for k, v in self.db.hgetall(self.name).items()}
|
|
77
|
+
|
|
78
|
+
def __getitem__(self, key: str) -> AcceptedType:
|
|
79
|
+
_ = decode(self.db.hget(self.name, key))
|
|
80
|
+
if _ is None:
|
|
81
|
+
raise KeyError(f"{key!r} not found in Redis db entry {self!r}")
|
|
82
|
+
return _
|
|
83
|
+
|
|
84
|
+
def __setitem__(self, key: str, value: AcceptedType) -> None:
|
|
85
|
+
self.db.hset(self.name, key, encode(value))
|
|
86
|
+
|
|
87
|
+
def __delitem__(self, key: str) -> None:
|
|
88
|
+
self.db.hdel(self.name, key)
|
|
89
|
+
|
|
90
|
+
def __iter__(self) -> Iterator[str]:
|
|
91
|
+
return iter(self.data)
|
|
92
|
+
|
|
93
|
+
def __len__(self) -> int:
|
|
94
|
+
return len(self.data)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def encode(value: AcceptedType) -> RedisType:
|
|
98
|
+
"""Redis can't store bools: convert to something compatible before entering."""
|
|
99
|
+
if isinstance(value, bool) or value is None:
|
|
100
|
+
return str(value)
|
|
101
|
+
if isinstance(value, (int, str, float)):
|
|
102
|
+
return value
|
|
103
|
+
raise TypeError(f"Cannot store {value!r} in Redis")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def decode(value: bytes | None) -> AcceptedType:
|
|
107
|
+
"""Redis stores everything as bytes: convert back to our original python datatype."""
|
|
108
|
+
if value is None:
|
|
109
|
+
return None
|
|
110
|
+
decoded_value: str = value.decode()
|
|
111
|
+
if decoded_value.isnumeric():
|
|
112
|
+
return int(decoded_value)
|
|
113
|
+
with contextlib.suppress(ValueError):
|
|
114
|
+
return float(decoded_value)
|
|
115
|
+
if decoded_value.capitalize() in (str(_) for _ in (True, False, None)):
|
|
116
|
+
return eval(decoded_value.capitalize())
|
|
117
|
+
return decoded_value
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
if __name__ == "__main__":
|
|
121
|
+
import doctest
|
|
122
|
+
|
|
123
|
+
doctest.testmod(
|
|
124
|
+
optionflags=(doctest.IGNORE_EXCEPTION_DETAIL | doctest.NORMALIZE_WHITESPACE)
|
|
125
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: npc-lims
|
|
3
|
+
Version: 0.1.8
|
|
4
|
+
Summary: Tools for fetch and updating paths, metadata and state for Mindscope Neuropixels sessions, in the cloud.
|
|
5
|
+
Author-email: Arjun Sridhar <arjun.sridhar@alleninstitute.org>, Ben Hardcastle <ben.hardcastle@alleninstitute.org>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/AllenInstitute/npc_lims
|
|
8
|
+
Project-URL: Issues, https://github.com/AllenInstitute/npc_lims/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: typing-extensions >=4.7.1
|
|
19
|
+
Requires-Dist: npc-session >=0.1.0
|
|
20
|
+
Requires-Dist: redis >=4.6.0
|
|
21
|
+
Requires-Dist: python-dateutil >=2.8.2
|
|
22
|
+
Requires-Dist: universal-pathlib >=0.1.1
|
|
23
|
+
Requires-Dist: aind-codeocean-api >=0.2.4
|
|
24
|
+
Requires-Dist: s3fs >=2023.6.0
|
|
25
|
+
Requires-Dist: types-redis >=4.6.0.3
|
|
26
|
+
|
|
27
|
+
# npc_lims
|
|
28
|
+
|
|
29
|
+
**n**euro**p**ixels **c**loud **lims**
|
|
30
|
+
|
|
31
|
+
Tools for fetch and updating paths, metadata and state for Mindscope Neuropixels sessions, in the cloud.
|
|
32
|
+
|
|
33
|
+
[](https://pypi.python.org/pypi/npc-lims/)
|
|
35
|
+
## quickstart
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install npc_lims
|
|
39
|
+
```
|
|
40
|
+
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
npc_lims/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
npc_lims/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
3
|
+
npc_lims/jobs/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
4
|
+
npc_lims/paths/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
npc_lims/paths/codeocean.py,sha256=a6Hu_Zftm_xeA3kg4hvzEkKTE05gAjCj50Qv5xLt-6k,3729
|
|
6
|
+
npc_lims/state/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
7
|
+
npc_lims/state/redis_state.py,sha256=iVukCDneEwv6vyxxDzeJwI35AP9V2uoq8uDpRDrBYe4,3675
|
|
8
|
+
npc_lims/status/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
npc_lims-0.1.8.dist-info/METADATA,sha256=9FiBPYIBmaYApyx7bi2FEAVgKBttwXJEtUiBzFBLc_0,1468
|
|
10
|
+
npc_lims-0.1.8.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
|
|
11
|
+
npc_lims-0.1.8.dist-info/top_level.txt,sha256=aO8-ldqqYMDYjIpiCJkrHRyR7N1O41A6itWODtM06Z0,9
|
|
12
|
+
npc_lims-0.1.8.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
npc_lims
|