npc_lims 0.1.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
npc_lims/__init__.py ADDED
File without changes
@@ -0,0 +1 @@
1
+
File without changes
@@ -0,0 +1,132 @@
1
+ from __future__ import annotations
2
+
3
+ import functools
4
+ import operator
5
+ import os
6
+ import re
7
+ from typing import Any, Iterator, Literal
8
+
9
+ import npc_session
10
+ import npc_session.parsing as parsing
11
+ import upath
12
+ from aind_codeocean_api import codeocean # type: ignore
13
+ from typing_extensions import TypeAlias
14
+
15
+ CODE_OCEAN_API_TOKEN = os.getenv("CODE_OCEAN_API_TOKEN")
16
+ CODE_OCEAN_DOMAIN = os.getenv("CODE_OCEAN_DOMAIN")
17
+
18
+ DataAsset: TypeAlias = dict[
19
+ Literal[
20
+ "created",
21
+ "custom_metadata",
22
+ "description",
23
+ "files",
24
+ "id",
25
+ "last_used",
26
+ "name",
27
+ "size",
28
+ "sourceBucket",
29
+ "state",
30
+ "tags",
31
+ "type",
32
+ ],
33
+ Any,
34
+ ]
35
+
36
+
37
+ @functools.cache
38
+ def get_subject_data_assets(subject: str | int) -> tuple[DataAsset, ...]:
39
+ """
40
+ >>> assets = get_subject_data_assets(668759)
41
+ >>> assert len(assets) > 0
42
+ """
43
+ codeocean_client = codeocean.CodeOceanClient(
44
+ domain=CODE_OCEAN_DOMAIN, token=CODE_OCEAN_API_TOKEN
45
+ )
46
+ response = codeocean_client.search_data_assets(
47
+ query=f"subject id: {npc_session.SubjectRecord(subject)}"
48
+ )
49
+ response.raise_for_status()
50
+ return response.json()["results"]
51
+
52
+
53
+ def get_sessions_with_data_assets(
54
+ subject: str | int,
55
+ ) -> tuple[npc_session.SessionRecord, ...]:
56
+ """
57
+ >>> sessions = get_sessions_with_data_assets(668759)
58
+ >>> assert len(sessions) > 0
59
+ """
60
+ assets = get_subject_data_assets(subject)
61
+ return tuple({npc_session.SessionRecord(asset["name"]) for asset in assets})
62
+
63
+
64
+ @functools.cache
65
+ def get_session_data_assets(
66
+ session: str | npc_session.SessionRecord,
67
+ ) -> tuple[DataAsset, ...]:
68
+ session = npc_session.SessionRecord(session)
69
+ assets = get_subject_data_assets(session.subject)
70
+ return tuple(
71
+ asset
72
+ for asset in assets
73
+ if re.match(
74
+ f"ecephys_{session.subject}_{session.date}_{parsing._TIME}", asset["name"]
75
+ )
76
+ )
77
+
78
+
79
+ @functools.cache
80
+ def get_raw_data_root(session: str | npc_session.SessionRecord) -> upath.UPath:
81
+ """
82
+ >>> get_raw_data_root('668759_20230711')
83
+ S3Path('s3://aind-ephys-data/ecephys_668759_2023-07-11_13-07-32')
84
+ """
85
+ session = npc_session.SessionRecord(session)
86
+ raw_assets = tuple(
87
+ asset
88
+ for asset in get_session_data_assets(session)
89
+ if asset["custom_metadata"].get("data level") == "raw data"
90
+ )
91
+ if len(raw_assets) < session.idx:
92
+ raise ValueError(
93
+ f"Number of paths raw sessions on s3 {len(raw_assets)} is less than {session.idx = }"
94
+ )
95
+
96
+ raw_asset = raw_assets[session.idx]
97
+ bucket_info = raw_asset["sourceBucket"]
98
+ roots = {"aws": "s3", "gcs": "gs"}
99
+ if bucket_info["origin"] not in roots:
100
+ raise RuntimeError(
101
+ f"Unknown bucket origin - not sure how to create UPath: {bucket_info = }"
102
+ )
103
+ return upath.UPath(
104
+ f"{roots[bucket_info['origin']]}://{bucket_info['bucket']}/{bucket_info['prefix']}"
105
+ )
106
+
107
+
108
+ @functools.cache
109
+ def get_raw_data_top_level_paths(
110
+ session: str | npc_session.SessionRecord,
111
+ ) -> tuple[upath.UPath, ...]:
112
+ """
113
+ >>> files = get_raw_data_top_level_paths('668759_20230711')
114
+ >>> assert len(files) > 0
115
+ """
116
+ raw_data_root = get_raw_data_root(session)
117
+ directories: Iterator = (
118
+ directory for directory in raw_data_root.iterdir() if directory.is_dir()
119
+ )
120
+ first_level_files_directories: Iterator = (
121
+ tuple(directory.iterdir()) for directory in directories
122
+ )
123
+
124
+ return functools.reduce(operator.add, first_level_files_directories)
125
+
126
+
127
+ if __name__ == "__main__":
128
+ import doctest
129
+
130
+ doctest.testmod(
131
+ optionflags=(doctest.IGNORE_EXCEPTION_DETAIL | doctest.NORMALIZE_WHITESPACE)
132
+ )
npc_lims/py.typed ADDED
@@ -0,0 +1 @@
1
+
@@ -0,0 +1 @@
1
+
@@ -0,0 +1,125 @@
1
+ """
2
+ Redis connection for persisting session state.
3
+ """
4
+ from __future__ import annotations
5
+
6
+ import collections.abc
7
+ import contextlib
8
+ import logging
9
+ import os
10
+ from typing import ClassVar, Iterator, Union
11
+
12
+ import redis
13
+ from typing_extensions import TypeAlias
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ # AcceptedType will be coerced to RedisType before being stored in Redis
18
+ RedisType: TypeAlias = Union[str, int, float]
19
+ """Can be stored in Redis directly, or returned from Redis."""
20
+
21
+ AcceptedType: TypeAlias = Union[RedisType, bool, None]
22
+ """Can be stored in Redis after using `encode(value: AcceptedType)`."""
23
+
24
+
25
+ class State(collections.abc.MutableMapping):
26
+ """Get and set session state in Redis via a dict interface.
27
+
28
+ - dict interface provides `keys`, `get`, `setdefault`, `pop`, etc.
29
+ - accepted value types are str, int, float, bool, None
30
+
31
+ >>> test_id = 0
32
+ >>> state = State(test_id)
33
+ >>> state['test'] = 1.0
34
+ >>> state['test']
35
+ 1.0
36
+ >>> state['test'] = 'test'
37
+ >>> state['test']
38
+ 'test'
39
+ >>> all('test' in _ for _ in (state, state.keys(), state.values()))
40
+ True
41
+ >>> state.setdefault('test', True)
42
+ 'test'
43
+ >>> state.pop('test')
44
+ 'test'
45
+ >>> del state['test']
46
+ >>> state.get('test') is None
47
+ True
48
+ """
49
+
50
+ db: ClassVar[redis.Redis]
51
+
52
+ def __init__(self, id: int | str) -> None:
53
+ self.name = str(id)
54
+ try:
55
+ _ = self.db
56
+ except AttributeError:
57
+ self.__class__.connect()
58
+
59
+ def __repr__(self) -> str:
60
+ return self.data.__repr__()
61
+
62
+ @classmethod
63
+ def connect(cls) -> None:
64
+ cls.db = redis.Redis(
65
+ host="redis-11357.c1.us-west-2-2.ec2.cloud.redislabs.com",
66
+ port=11357,
67
+ password=os.environ["REDIS_DEFAULT_USER_PASSWORD"],
68
+ )
69
+ if cls.db.ping():
70
+ logger.debug("Connected to Redis database: %s", cls.db)
71
+ else:
72
+ logger.error("Failed to connect to Redis database")
73
+
74
+ @property
75
+ def data(self) -> dict[str, AcceptedType]:
76
+ return {k.decode(): decode(v) for k, v in self.db.hgetall(self.name).items()}
77
+
78
+ def __getitem__(self, key: str) -> AcceptedType:
79
+ _ = decode(self.db.hget(self.name, key))
80
+ if _ is None:
81
+ raise KeyError(f"{key!r} not found in Redis db entry {self!r}")
82
+ return _
83
+
84
+ def __setitem__(self, key: str, value: AcceptedType) -> None:
85
+ self.db.hset(self.name, key, encode(value))
86
+
87
+ def __delitem__(self, key: str) -> None:
88
+ self.db.hdel(self.name, key)
89
+
90
+ def __iter__(self) -> Iterator[str]:
91
+ return iter(self.data)
92
+
93
+ def __len__(self) -> int:
94
+ return len(self.data)
95
+
96
+
97
+ def encode(value: AcceptedType) -> RedisType:
98
+ """Redis can't store bools: convert to something compatible before entering."""
99
+ if isinstance(value, bool) or value is None:
100
+ return str(value)
101
+ if isinstance(value, (int, str, float)):
102
+ return value
103
+ raise TypeError(f"Cannot store {value!r} in Redis")
104
+
105
+
106
+ def decode(value: bytes | None) -> AcceptedType:
107
+ """Redis stores everything as bytes: convert back to our original python datatype."""
108
+ if value is None:
109
+ return None
110
+ decoded_value: str = value.decode()
111
+ if decoded_value.isnumeric():
112
+ return int(decoded_value)
113
+ with contextlib.suppress(ValueError):
114
+ return float(decoded_value)
115
+ if decoded_value.capitalize() in (str(_) for _ in (True, False, None)):
116
+ return eval(decoded_value.capitalize())
117
+ return decoded_value
118
+
119
+
120
+ if __name__ == "__main__":
121
+ import doctest
122
+
123
+ doctest.testmod(
124
+ optionflags=(doctest.IGNORE_EXCEPTION_DETAIL | doctest.NORMALIZE_WHITESPACE)
125
+ )
File without changes
@@ -0,0 +1,40 @@
1
+ Metadata-Version: 2.1
2
+ Name: npc-lims
3
+ Version: 0.1.8
4
+ Summary: Tools for fetch and updating paths, metadata and state for Mindscope Neuropixels sessions, in the cloud.
5
+ Author-email: Arjun Sridhar <arjun.sridhar@alleninstitute.org>, Ben Hardcastle <ben.hardcastle@alleninstitute.org>
6
+ License: MIT
7
+ Project-URL: Repository, https://github.com/AllenInstitute/npc_lims
8
+ Project-URL: Issues, https://github.com/AllenInstitute/npc_lims/issues
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.9
11
+ Classifier: Programming Language :: Python :: 3.10
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Operating System :: Microsoft :: Windows
15
+ Classifier: Operating System :: POSIX :: Linux
16
+ Requires-Python: >=3.9
17
+ Description-Content-Type: text/markdown
18
+ Requires-Dist: typing-extensions >=4.7.1
19
+ Requires-Dist: npc-session >=0.1.0
20
+ Requires-Dist: redis >=4.6.0
21
+ Requires-Dist: python-dateutil >=2.8.2
22
+ Requires-Dist: universal-pathlib >=0.1.1
23
+ Requires-Dist: aind-codeocean-api >=0.2.4
24
+ Requires-Dist: s3fs >=2023.6.0
25
+ Requires-Dist: types-redis >=4.6.0.3
26
+
27
+ # npc_lims
28
+
29
+ **n**euro**p**ixels **c**loud **lims**
30
+
31
+ Tools for fetch and updating paths, metadata and state for Mindscope Neuropixels sessions, in the cloud.
32
+
33
+ [![Python
34
+ Versions](https://img.shields.io/pypi/pyversions/npc_lims.svg)](https://pypi.python.org/pypi/npc-lims/)
35
+ ## quickstart
36
+
37
+ ```bash
38
+ pip install npc_lims
39
+ ```
40
+
@@ -0,0 +1,12 @@
1
+ npc_lims/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ npc_lims/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
3
+ npc_lims/jobs/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
4
+ npc_lims/paths/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ npc_lims/paths/codeocean.py,sha256=a6Hu_Zftm_xeA3kg4hvzEkKTE05gAjCj50Qv5xLt-6k,3729
6
+ npc_lims/state/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
7
+ npc_lims/state/redis_state.py,sha256=iVukCDneEwv6vyxxDzeJwI35AP9V2uoq8uDpRDrBYe4,3675
8
+ npc_lims/status/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
+ npc_lims-0.1.8.dist-info/METADATA,sha256=9FiBPYIBmaYApyx7bi2FEAVgKBttwXJEtUiBzFBLc_0,1468
10
+ npc_lims-0.1.8.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
11
+ npc_lims-0.1.8.dist-info/top_level.txt,sha256=aO8-ldqqYMDYjIpiCJkrHRyR7N1O41A6itWODtM06Z0,9
12
+ npc_lims-0.1.8.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: bdist_wheel (0.41.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ npc_lims