ONE-api 3.0b5__py3-none-any.whl → 3.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- one/__init__.py +1 -1
- one/alf/cache.py +66 -16
- one/api.py +86 -46
- one/remote/globus.py +1 -1
- one/webclient.py +5 -1
- {ONE_api-3.0b5.dist-info → one_api-3.1.1.dist-info}/METADATA +3 -2
- {ONE_api-3.0b5.dist-info → one_api-3.1.1.dist-info}/RECORD +10 -10
- {ONE_api-3.0b5.dist-info → one_api-3.1.1.dist-info}/WHEEL +1 -1
- {ONE_api-3.0b5.dist-info → one_api-3.1.1.dist-info/licenses}/LICENSE +0 -0
- {ONE_api-3.0b5.dist-info → one_api-3.1.1.dist-info}/top_level.txt +0 -0
one/__init__.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""The Open Neurophysiology Environment (ONE) API."""
|
|
2
|
-
__version__ = '3.
|
|
2
|
+
__version__ = '3.1.1'
|
one/alf/cache.py
CHANGED
|
@@ -22,11 +22,12 @@ from functools import partial
|
|
|
22
22
|
from pathlib import Path
|
|
23
23
|
import warnings
|
|
24
24
|
import logging
|
|
25
|
+
from copy import deepcopy
|
|
25
26
|
|
|
26
27
|
import pandas as pd
|
|
27
28
|
import numpy as np
|
|
28
29
|
from packaging import version
|
|
29
|
-
from iblutil.util import Bunch
|
|
30
|
+
from iblutil.util import Bunch, ensure_list
|
|
30
31
|
from iblutil.io import parquet
|
|
31
32
|
from iblutil.io.hashfile import md5
|
|
32
33
|
|
|
@@ -35,8 +36,9 @@ from one.alf.io import iter_sessions
|
|
|
35
36
|
from one.alf.path import session_path_parts, get_alf_path
|
|
36
37
|
|
|
37
38
|
__all__ = [
|
|
38
|
-
'make_parquet_db', '
|
|
39
|
-
'
|
|
39
|
+
'make_parquet_db', 'load_tables', 'patch_tables', 'merge_tables',
|
|
40
|
+
'remove_table_files', 'remove_missing_datasets', 'default_cache',
|
|
41
|
+
'QC_TYPE', 'EMPTY_DATASETS_FRAME', 'EMPTY_SESSIONS_FRAME']
|
|
40
42
|
_logger = logging.getLogger(__name__)
|
|
41
43
|
|
|
42
44
|
# -------------------------------------------------------------------------------------------------
|
|
@@ -259,6 +261,36 @@ def _make_datasets_df(root_dir, hash_files=False) -> pd.DataFrame:
|
|
|
259
261
|
return pd.DataFrame(rows, columns=DATASETS_COLUMNS).astype(DATASETS_COLUMNS)
|
|
260
262
|
|
|
261
263
|
|
|
264
|
+
def default_cache(origin=''):
|
|
265
|
+
"""Returns an empty cache dictionary with the default tables.
|
|
266
|
+
|
|
267
|
+
Parameters
|
|
268
|
+
----------
|
|
269
|
+
origin : str, optional
|
|
270
|
+
The origin of the cache (e.g. a computer name or database name).
|
|
271
|
+
|
|
272
|
+
Returns
|
|
273
|
+
-------
|
|
274
|
+
Bunch
|
|
275
|
+
A Bunch object containing the loaded cache tables and associated metadata.
|
|
276
|
+
|
|
277
|
+
"""
|
|
278
|
+
table_meta = _metadata(origin)
|
|
279
|
+
# The origin is now a set, however we leave _metadata as Alyx relies on this and sets
|
|
280
|
+
# can't be serialized to JSON
|
|
281
|
+
table_meta['origin'] = set(filter(None, [origin]))
|
|
282
|
+
return Bunch({
|
|
283
|
+
'datasets': EMPTY_DATASETS_FRAME.copy(),
|
|
284
|
+
'sessions': EMPTY_SESSIONS_FRAME.copy(),
|
|
285
|
+
'_meta': {
|
|
286
|
+
'created_time': None,
|
|
287
|
+
'loaded_time': None,
|
|
288
|
+
'modified_time': None,
|
|
289
|
+
'saved_time': None,
|
|
290
|
+
'raw': {k: deepcopy(table_meta) for k in ('datasets', 'sessions')}}
|
|
291
|
+
})
|
|
292
|
+
|
|
293
|
+
|
|
262
294
|
def make_parquet_db(root_dir, out_dir=None, hash_ids=True, hash_files=False, lab=None):
|
|
263
295
|
"""Given a data directory, index the ALF datasets and save the generated cache tables.
|
|
264
296
|
|
|
@@ -375,17 +407,8 @@ def load_tables(tables_dir, glob_pattern='*.pqt'):
|
|
|
375
407
|
A Bunch object containing the loaded cache tables and associated metadata.
|
|
376
408
|
|
|
377
409
|
"""
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
'loaded_time': None,
|
|
381
|
-
'modified_time': None,
|
|
382
|
-
'saved_time': None,
|
|
383
|
-
'raw': {}
|
|
384
|
-
}
|
|
385
|
-
caches = Bunch({
|
|
386
|
-
'datasets': EMPTY_DATASETS_FRAME.copy(),
|
|
387
|
-
'sessions': EMPTY_SESSIONS_FRAME.copy(),
|
|
388
|
-
'_meta': meta})
|
|
410
|
+
caches = default_cache()
|
|
411
|
+
meta = caches['_meta']
|
|
389
412
|
INDEX_KEY = '.?id'
|
|
390
413
|
for cache_file in Path(tables_dir).glob(glob_pattern):
|
|
391
414
|
table = cache_file.stem
|
|
@@ -416,6 +439,10 @@ def load_tables(tables_dir, glob_pattern='*.pqt'):
|
|
|
416
439
|
if not cache.index.is_monotonic_increasing:
|
|
417
440
|
cache.sort_index(inplace=True)
|
|
418
441
|
|
|
442
|
+
# Ensure origin is a set (supports multiple origins)
|
|
443
|
+
meta['raw'][table]['origin'] = set(
|
|
444
|
+
filter(None, ensure_list(meta['raw'][table].get('origin', 'unknown'))))
|
|
445
|
+
|
|
419
446
|
caches[table] = cache
|
|
420
447
|
|
|
421
448
|
created = [datetime.datetime.fromisoformat(x['date_created'])
|
|
@@ -425,9 +452,12 @@ def load_tables(tables_dir, glob_pattern='*.pqt'):
|
|
|
425
452
|
return caches
|
|
426
453
|
|
|
427
454
|
|
|
428
|
-
def merge_tables(cache, strict=False, **kwargs):
|
|
455
|
+
def merge_tables(cache, strict=False, origin=None, **kwargs):
|
|
429
456
|
"""Update the cache tables with new records.
|
|
430
457
|
|
|
458
|
+
Note: A copy of the tables in cache may be returned if the original tables are immutable.
|
|
459
|
+
This can happen when tables are loaded from a parquet file.
|
|
460
|
+
|
|
431
461
|
Parameters
|
|
432
462
|
----------
|
|
433
463
|
dict
|
|
@@ -435,6 +465,8 @@ def merge_tables(cache, strict=False, **kwargs):
|
|
|
435
465
|
strict : bool
|
|
436
466
|
If not True, the columns don't need to match. Extra columns in input tables are
|
|
437
467
|
dropped and missing columns are added and filled with np.nan.
|
|
468
|
+
origin : str
|
|
469
|
+
The origin of the cache (e.g. a computer name or database name).
|
|
438
470
|
kwargs
|
|
439
471
|
pandas.DataFrame or pandas.Series to insert/update for each table.
|
|
440
472
|
|
|
@@ -488,13 +520,31 @@ def merge_tables(cache, strict=False, **kwargs):
|
|
|
488
520
|
records = records.astype(cache[table].dtypes)
|
|
489
521
|
# Update existing rows
|
|
490
522
|
to_update = records.index.isin(cache[table].index)
|
|
491
|
-
|
|
523
|
+
try:
|
|
524
|
+
cache[table].loc[records.index[to_update], :] = records[to_update]
|
|
525
|
+
except ValueError as e:
|
|
526
|
+
if 'assignment destination is read-only' in str(e):
|
|
527
|
+
# NB: nullable integer and categorical dtypes may be backed by immutable arrays
|
|
528
|
+
# after loading from parquet and therefore must be copied before assignment
|
|
529
|
+
cache[table] = cache[table].copy()
|
|
530
|
+
cache[table].loc[records.index[to_update], :] = records[to_update]
|
|
531
|
+
else:
|
|
532
|
+
raise e # pragma: no cover
|
|
533
|
+
|
|
492
534
|
# Assign new rows
|
|
493
535
|
to_assign = records[~to_update]
|
|
494
536
|
frames = [cache[table], to_assign]
|
|
495
537
|
# Concatenate and sort
|
|
496
538
|
cache[table] = pd.concat(frames).sort_index()
|
|
497
539
|
updated = datetime.datetime.now()
|
|
540
|
+
# Update the table metadata with the origin
|
|
541
|
+
if origin is not None:
|
|
542
|
+
table_meta = cache['_meta']['raw'].get(table, {})
|
|
543
|
+
if 'origin' not in table_meta:
|
|
544
|
+
table_meta['origin'] = set(origin)
|
|
545
|
+
else:
|
|
546
|
+
table_meta['origin'].add(origin)
|
|
547
|
+
cache['_meta']['raw'][table] = table_meta
|
|
498
548
|
cache['_meta']['modified_time'] = updated
|
|
499
549
|
return updated
|
|
500
550
|
|
one/api.py
CHANGED
|
@@ -31,7 +31,7 @@ import one.alf.exceptions as alferr
|
|
|
31
31
|
from one.alf.path import ALFPath
|
|
32
32
|
from .alf.cache import (
|
|
33
33
|
make_parquet_db, load_tables, remove_table_files, merge_tables,
|
|
34
|
-
|
|
34
|
+
default_cache, cast_index_object)
|
|
35
35
|
from .alf.spec import is_uuid, is_uuid_string, QC, to_alf
|
|
36
36
|
from . import __version__
|
|
37
37
|
from one.converters import ConversionMixin, session_record2path, ses2records, datasets2records
|
|
@@ -49,7 +49,7 @@ class One(ConversionMixin):
|
|
|
49
49
|
"""An API for searching and loading data on a local filesystem."""
|
|
50
50
|
|
|
51
51
|
_search_terms = (
|
|
52
|
-
'
|
|
52
|
+
'datasets', 'date_range', 'laboratory', 'number',
|
|
53
53
|
'projects', 'subject', 'task_protocol', 'dataset_qc_lte'
|
|
54
54
|
)
|
|
55
55
|
|
|
@@ -113,16 +113,7 @@ class One(ConversionMixin):
|
|
|
113
113
|
|
|
114
114
|
def _reset_cache(self):
|
|
115
115
|
"""Replace the cache object with a Bunch that contains the right fields."""
|
|
116
|
-
self._cache =
|
|
117
|
-
'datasets': EMPTY_DATASETS_FRAME.copy(),
|
|
118
|
-
'sessions': EMPTY_SESSIONS_FRAME.copy(),
|
|
119
|
-
'_meta': {
|
|
120
|
-
'created_time': None,
|
|
121
|
-
'loaded_time': None,
|
|
122
|
-
'modified_time': None,
|
|
123
|
-
'saved_time': None,
|
|
124
|
-
'raw': {}} # map of original table metadata
|
|
125
|
-
})
|
|
116
|
+
self._cache = default_cache()
|
|
126
117
|
|
|
127
118
|
def _remove_table_files(self, tables=None):
|
|
128
119
|
"""Delete cache tables on disk.
|
|
@@ -143,20 +134,25 @@ class One(ConversionMixin):
|
|
|
143
134
|
tables = tables or filter(lambda x: x[0] != '_', self._cache)
|
|
144
135
|
return remove_table_files(self._tables_dir, tables)
|
|
145
136
|
|
|
146
|
-
def load_cache(self, tables_dir=None, **kwargs):
|
|
137
|
+
def load_cache(self, tables_dir=None, clobber=True, **kwargs):
|
|
147
138
|
"""Load parquet cache files from a local directory.
|
|
148
139
|
|
|
149
140
|
Parameters
|
|
150
141
|
----------
|
|
151
142
|
tables_dir : str, pathlib.Path
|
|
152
143
|
An optional directory location of the parquet files, defaults to One._tables_dir.
|
|
144
|
+
clobber : bool
|
|
145
|
+
If true, the cache is loaded without merging with existing table files.
|
|
153
146
|
|
|
154
147
|
Returns
|
|
155
148
|
-------
|
|
156
149
|
datetime.datetime
|
|
157
150
|
A timestamp of when the cache was loaded.
|
|
158
151
|
"""
|
|
159
|
-
|
|
152
|
+
if clobber:
|
|
153
|
+
self._reset_cache()
|
|
154
|
+
else:
|
|
155
|
+
raise NotImplementedError('clobber=False not implemented yet')
|
|
160
156
|
self._tables_dir = Path(tables_dir or self._tables_dir or self.cache_dir)
|
|
161
157
|
self._cache = load_tables(self._tables_dir)
|
|
162
158
|
|
|
@@ -169,7 +165,7 @@ class One(ConversionMixin):
|
|
|
169
165
|
# prompt the user to delete them to improve load times
|
|
170
166
|
raw_meta = self._cache['_meta'].get('raw', {}).values() or [{}]
|
|
171
167
|
tagged = any(filter(None, flatten(x.get('database_tags') for x in raw_meta)))
|
|
172
|
-
origin = set(x
|
|
168
|
+
origin = set(filter(None, flatten(ensure_list(x.get('origin', [])) for x in raw_meta)))
|
|
173
169
|
older = (self._cache['_meta']['created_time'] or datetime.now()) < datetime(2025, 2, 13)
|
|
174
170
|
remote = not self.offline and self.mode == 'remote'
|
|
175
171
|
if remote and origin == {'alyx'} and older and not self._web_client.silent and not tagged:
|
|
@@ -216,11 +212,21 @@ class One(ConversionMixin):
|
|
|
216
212
|
caches = load_tables(save_dir)
|
|
217
213
|
merge_tables(
|
|
218
214
|
caches, **{k: v for k, v in self._cache.items() if not k.startswith('_')})
|
|
215
|
+
# Ensure we use the minimum created date for each table
|
|
216
|
+
for table in caches['_meta']['raw']:
|
|
217
|
+
raw_meta = [x['_meta']['raw'].get(table, {}) for x in (caches, self._cache)]
|
|
218
|
+
created = filter(None, (x.get('date_created') for x in raw_meta))
|
|
219
|
+
if any(created := list(created)):
|
|
220
|
+
created = min(map(datetime.fromisoformat, created))
|
|
221
|
+
created = created.isoformat(sep=' ', timespec='minutes')
|
|
222
|
+
meta['raw'][table]['date_created'] = created
|
|
219
223
|
|
|
220
224
|
with FileLock(save_dir, log=_logger, timeout=TIMEOUT, timeout_action='delete'):
|
|
221
225
|
_logger.info('Saving cache tables...')
|
|
222
226
|
for table in filter(lambda x: not x[0] == '_', caches.keys()):
|
|
223
227
|
metadata = meta['raw'].get(table, {})
|
|
228
|
+
if isinstance(metadata.get('origin'), set):
|
|
229
|
+
metadata['origin'] = list(metadata['origin'])
|
|
224
230
|
metadata['date_modified'] = modified.isoformat(sep=' ', timespec='minutes')
|
|
225
231
|
filename = save_dir.joinpath(f'{table}.pqt')
|
|
226
232
|
# Cast indices to str before saving
|
|
@@ -319,10 +325,8 @@ class One(ConversionMixin):
|
|
|
319
325
|
|
|
320
326
|
Parameters
|
|
321
327
|
----------
|
|
322
|
-
|
|
323
|
-
One or more dataset names. Returns sessions containing all these datasets.
|
|
324
|
-
A dataset matches if it contains the search string e.g. 'wheel.position' matches
|
|
325
|
-
'_ibl_wheel.position.npy'.
|
|
328
|
+
datasets : str, list
|
|
329
|
+
One or more (exact) dataset names. Returns sessions containing all of these datasets.
|
|
326
330
|
dataset_qc_lte : str, int, one.alf.spec.QC
|
|
327
331
|
A dataset QC value, returns sessions with datasets at or below this QC value, including
|
|
328
332
|
those with no QC set. If `dataset` not passed, sessions with any passing QC datasets
|
|
@@ -370,7 +374,9 @@ class One(ConversionMixin):
|
|
|
370
374
|
|
|
371
375
|
Search for sessions on a given date, in a given lab, containing trials and spike data.
|
|
372
376
|
|
|
373
|
-
>>> eids = one.search(
|
|
377
|
+
>>> eids = one.search(
|
|
378
|
+
... date='2023-01-01', lab='churchlandlab',
|
|
379
|
+
... datasets=['trials.table.pqt', 'spikes.times.npy'])
|
|
374
380
|
|
|
375
381
|
Search for sessions containing trials and spike data where QC for both are WARNING or less.
|
|
376
382
|
|
|
@@ -397,13 +403,14 @@ class One(ConversionMixin):
|
|
|
397
403
|
|
|
398
404
|
def all_present(x, dsets, exists=True):
|
|
399
405
|
"""Returns true if all datasets present in Series."""
|
|
400
|
-
|
|
406
|
+
name = x.str.rsplit('/', n=1, expand=True).iloc[:, -1]
|
|
407
|
+
return all(any(name.str.fullmatch(y) & exists) for y in dsets)
|
|
401
408
|
|
|
402
409
|
# Iterate over search filters, reducing the sessions table
|
|
403
410
|
sessions = self._cache['sessions']
|
|
404
411
|
|
|
405
412
|
# Ensure sessions filtered in a particular order, with datasets last
|
|
406
|
-
search_order = ('date_range', 'number', '
|
|
413
|
+
search_order = ('date_range', 'number', 'datasets')
|
|
407
414
|
|
|
408
415
|
def sort_fcn(itm):
|
|
409
416
|
return -1 if itm[0] not in search_order else search_order.index(itm[0])
|
|
@@ -430,12 +437,15 @@ class One(ConversionMixin):
|
|
|
430
437
|
query = ensure_list(value)
|
|
431
438
|
sessions = sessions[sessions[key].isin(map(int, query))]
|
|
432
439
|
# Dataset/QC check is biggest so this should be done last
|
|
433
|
-
elif key == '
|
|
440
|
+
elif key == 'datasets' or (key == 'dataset_qc_lte' and 'datasets' not in queries):
|
|
434
441
|
datasets = self._cache['datasets']
|
|
435
442
|
qc = QC.validate(queries.get('dataset_qc_lte', 'FAIL')).name # validate value
|
|
436
443
|
has_dset = sessions.index.isin(datasets.index.get_level_values('eid'))
|
|
444
|
+
if not has_dset.any():
|
|
445
|
+
sessions = sessions.iloc[0:0] # No datasets for any sessions
|
|
446
|
+
continue
|
|
437
447
|
datasets = datasets.loc[(sessions.index.values[has_dset], ), :]
|
|
438
|
-
query = ensure_list(value if key == '
|
|
448
|
+
query = ensure_list(value if key == 'datasets' else '')
|
|
439
449
|
# For each session check any dataset both contains query and exists
|
|
440
450
|
mask = (
|
|
441
451
|
(datasets
|
|
@@ -1025,7 +1035,7 @@ class One(ConversionMixin):
|
|
|
1025
1035
|
"""
|
|
1026
1036
|
query_type = query_type or self.mode
|
|
1027
1037
|
datasets = self.list_datasets(
|
|
1028
|
-
eid, details=True, query_type=query_type, keep_eid_index=True
|
|
1038
|
+
eid, details=True, query_type=query_type, keep_eid_index=True)
|
|
1029
1039
|
|
|
1030
1040
|
if len(datasets) == 0:
|
|
1031
1041
|
raise alferr.ALFObjectNotFound(obj)
|
|
@@ -1692,7 +1702,8 @@ class OneAlyx(One):
|
|
|
1692
1702
|
tag = tag or current_tags[0] # For refreshes take the current tag as default
|
|
1693
1703
|
different_tag = any(x != tag for x in current_tags)
|
|
1694
1704
|
if not (clobber or different_tag):
|
|
1695
|
-
|
|
1705
|
+
# Load any present cache
|
|
1706
|
+
super(OneAlyx, self).load_cache(tables_dir, clobber=True)
|
|
1696
1707
|
cache_meta = self._cache.get('_meta', {})
|
|
1697
1708
|
raw_meta = cache_meta.get('raw', {}).values() or [{}]
|
|
1698
1709
|
|
|
@@ -1711,7 +1722,13 @@ class OneAlyx(One):
|
|
|
1711
1722
|
remote_created = datetime.fromisoformat(cache_info['date_created'])
|
|
1712
1723
|
local_created = cache_meta.get('created_time', None)
|
|
1713
1724
|
fresh = local_created and (remote_created - local_created) < timedelta(minutes=1)
|
|
1714
|
-
|
|
1725
|
+
# The local cache may have been created locally more recently, but if it doesn't
|
|
1726
|
+
# contain the same tag or origin, we need to download the remote one.
|
|
1727
|
+
origin = cache_info.get('origin', 'unknown')
|
|
1728
|
+
local_origin = (x.get('origin', []) for x in raw_meta)
|
|
1729
|
+
local_origin = set(flatten(map(ensure_list, local_origin)))
|
|
1730
|
+
different_origin = origin not in local_origin
|
|
1731
|
+
if fresh and not (different_tag or different_origin):
|
|
1715
1732
|
_logger.info('No newer cache available')
|
|
1716
1733
|
return cache_meta['loaded_time']
|
|
1717
1734
|
|
|
@@ -1725,19 +1742,27 @@ class OneAlyx(One):
|
|
|
1725
1742
|
self._tables_dir = self._tables_dir or self.cache_dir
|
|
1726
1743
|
|
|
1727
1744
|
# Check if the origin has changed. This is to warn users if downloading from a
|
|
1728
|
-
# different database to the one currently loaded.
|
|
1729
|
-
|
|
1730
|
-
origin
|
|
1731
|
-
if
|
|
1745
|
+
# different database to the one currently loaded. When building the cache from
|
|
1746
|
+
# remote queries the origin is set to the Alyx database URL. If the cache info
|
|
1747
|
+
# origin name and URL are different, warn the user.
|
|
1748
|
+
if different_origin and local_origin and self.alyx.base_url not in local_origin:
|
|
1732
1749
|
warnings.warn(
|
|
1733
1750
|
'Downloading cache tables from another origin '
|
|
1734
|
-
f'("{origin}" instead of "{", ".join(
|
|
1751
|
+
f'("{origin}" instead of "{", ".join(local_origin)}")')
|
|
1735
1752
|
|
|
1736
1753
|
# Download the remote cache files
|
|
1737
1754
|
_logger.info('Downloading remote caches...')
|
|
1738
1755
|
files = self.alyx.download_cache_tables(cache_info.get('location'), self._tables_dir)
|
|
1739
1756
|
assert any(files)
|
|
1740
|
-
|
|
1757
|
+
# Reload cache after download
|
|
1758
|
+
loaded_time = super(OneAlyx, self).load_cache(self._tables_dir)
|
|
1759
|
+
# Add db URL to origin set so we know where the cache came from
|
|
1760
|
+
for raw_meta in self._cache['_meta']['raw'].values():
|
|
1761
|
+
table_origin = set(filter(None, ensure_list(raw_meta.get('origin', []))))
|
|
1762
|
+
if origin in table_origin:
|
|
1763
|
+
table_origin.add(self.alyx.base_url)
|
|
1764
|
+
raw_meta['origin'] = table_origin
|
|
1765
|
+
return loaded_time
|
|
1741
1766
|
except (requests.exceptions.HTTPError, wc.HTTPError, requests.exceptions.SSLError) as ex:
|
|
1742
1767
|
_logger.debug(ex)
|
|
1743
1768
|
_logger.error(f'{type(ex).__name__}: Failed to load the remote cache file')
|
|
@@ -1847,7 +1872,8 @@ class OneAlyx(One):
|
|
|
1847
1872
|
return self._cache['datasets'].iloc[0:0] if details else [] # Return empty
|
|
1848
1873
|
session, datasets = ses2records(self.alyx.rest('sessions', 'read', id=eid))
|
|
1849
1874
|
# Add to cache tables
|
|
1850
|
-
merge_tables(
|
|
1875
|
+
merge_tables(
|
|
1876
|
+
self._cache, sessions=session, datasets=datasets.copy(), origin=self.alyx.base_url)
|
|
1851
1877
|
if datasets is None or datasets.empty:
|
|
1852
1878
|
return self._cache['datasets'].iloc[0:0] if details else [] # Return empty
|
|
1853
1879
|
assert set(datasets.index.unique('eid')) == {eid}
|
|
@@ -1999,7 +2025,7 @@ class OneAlyx(One):
|
|
|
1999
2025
|
rec = self.alyx.rest('insertions', 'read', id=str(pid))
|
|
2000
2026
|
return UUID(rec['session']), rec['name']
|
|
2001
2027
|
|
|
2002
|
-
def eid2pid(self, eid, query_type=None, details=False):
|
|
2028
|
+
def eid2pid(self, eid, query_type=None, details=False, **kwargs) -> (UUID, str, list):
|
|
2003
2029
|
"""Given an experiment UUID (eID), return the probe IDs and labels (i.e. ALF collection).
|
|
2004
2030
|
|
|
2005
2031
|
NB: Requires a connection to the Alyx database.
|
|
@@ -2013,6 +2039,8 @@ class OneAlyx(One):
|
|
|
2013
2039
|
Query mode - options include 'remote', and 'refresh'.
|
|
2014
2040
|
details : bool
|
|
2015
2041
|
Additionally return the complete Alyx records from insertions endpoint.
|
|
2042
|
+
kwargs
|
|
2043
|
+
Additional parameters to filter insertions Alyx endpoint.
|
|
2016
2044
|
|
|
2017
2045
|
Returns
|
|
2018
2046
|
-------
|
|
@@ -2023,6 +2051,15 @@ class OneAlyx(One):
|
|
|
2023
2051
|
list of dict (optional)
|
|
2024
2052
|
If details is true, returns the Alyx records from insertions endpoint.
|
|
2025
2053
|
|
|
2054
|
+
Examples
|
|
2055
|
+
--------
|
|
2056
|
+
Get the probe IDs and details for a given session ID
|
|
2057
|
+
|
|
2058
|
+
>>> pids, labels, recs = one.eid2pid(eid, details=True)
|
|
2059
|
+
|
|
2060
|
+
Get the probe ID for a given session ID and label
|
|
2061
|
+
|
|
2062
|
+
>>> (pid,), _ = one.eid2pid(eid, details=False, name='probe00')
|
|
2026
2063
|
"""
|
|
2027
2064
|
query_type = query_type or self.mode
|
|
2028
2065
|
if query_type == 'local' and 'insertions' not in self._cache.keys():
|
|
@@ -2030,7 +2067,7 @@ class OneAlyx(One):
|
|
|
2030
2067
|
eid = self.to_eid(eid) # Ensure we have a UUID str
|
|
2031
2068
|
if not eid:
|
|
2032
2069
|
return (None,) * (3 if details else 2)
|
|
2033
|
-
recs = self.alyx.rest('insertions', 'list', session=eid)
|
|
2070
|
+
recs = self.alyx.rest('insertions', 'list', session=eid, **kwargs)
|
|
2034
2071
|
pids = [UUID(x['id']) for x in recs]
|
|
2035
2072
|
labels = [x['name'] for x in recs]
|
|
2036
2073
|
if details:
|
|
@@ -2173,7 +2210,8 @@ class OneAlyx(One):
|
|
|
2173
2210
|
# Build sessions table
|
|
2174
2211
|
session_records = (x['session_info'] for x in insertions_records)
|
|
2175
2212
|
sessions_df = pd.DataFrame(next(zip(*map(ses2records, session_records))))
|
|
2176
|
-
return merge_tables(
|
|
2213
|
+
return merge_tables(
|
|
2214
|
+
self._cache, insertions=df, sessions=sessions_df, origin=self.alyx.base_url)
|
|
2177
2215
|
|
|
2178
2216
|
def search(self, details=False, query_type=None, **kwargs):
|
|
2179
2217
|
"""Searches sessions matching the given criteria and returns a list of matching eids.
|
|
@@ -2192,10 +2230,8 @@ class OneAlyx(One):
|
|
|
2192
2230
|
|
|
2193
2231
|
Parameters
|
|
2194
2232
|
----------
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
A dataset matches if it contains the search string e.g. 'wheel.position' matches
|
|
2198
|
-
'_ibl_wheel.position.npy'. C.f. `datasets` argument.
|
|
2233
|
+
datasets : str, list
|
|
2234
|
+
One or more (exact) dataset names. Returns sessions containing all of these datasets.
|
|
2199
2235
|
date_range : str, list, datetime.datetime, datetime.date, pandas.timestamp
|
|
2200
2236
|
A single date to search or a list of 2 dates that define the range (inclusive). To
|
|
2201
2237
|
define only the upper or lower date bound, set the other element to None.
|
|
@@ -2222,11 +2258,12 @@ class OneAlyx(One):
|
|
|
2222
2258
|
A str or list of lab location (as per Alyx definition) name.
|
|
2223
2259
|
Note: this corresponds to the specific rig, not the lab geographical location per se.
|
|
2224
2260
|
dataset_types : str, list
|
|
2225
|
-
One or more of dataset_types.
|
|
2226
|
-
|
|
2227
|
-
|
|
2261
|
+
One or more of dataset_types. Unlike with `datasets`, the dataset types for the
|
|
2262
|
+
sessions returned may not be reachable (i.e. for recent sessions the datasets may not
|
|
2263
|
+
yet be available).
|
|
2228
2264
|
dataset_qc_lte : int, str, one.alf.spec.QC
|
|
2229
|
-
The maximum QC value for associated datasets.
|
|
2265
|
+
The maximum QC value for associated datasets. NB: Without `datasets`, not all
|
|
2266
|
+
associated datasets with the matching QC values are guarenteed to be reachable.
|
|
2230
2267
|
details : bool
|
|
2231
2268
|
If true also returns a dict of dataset details.
|
|
2232
2269
|
query_type : str, None
|
|
@@ -2271,6 +2308,9 @@ class OneAlyx(One):
|
|
|
2271
2308
|
- In default and local mode, when the one.wildcards flag is True (default), queries are
|
|
2272
2309
|
interpreted as regular expressions. To turn this off set one.wildcards to False.
|
|
2273
2310
|
- In remote mode regular expressions are only supported using the `django` argument.
|
|
2311
|
+
- In remote mode, only the `datasets` argument returns sessions where datasets are
|
|
2312
|
+
registered *and* exist. Using `dataset_types` or `dataset_qc_lte` without `datasets`
|
|
2313
|
+
will not check that the datasets are reachable.
|
|
2274
2314
|
|
|
2275
2315
|
"""
|
|
2276
2316
|
query_type = query_type or self.mode
|
|
@@ -2340,7 +2380,7 @@ class OneAlyx(One):
|
|
|
2340
2380
|
|
|
2341
2381
|
"""
|
|
2342
2382
|
df = pd.DataFrame(next(zip(*map(ses2records, session_records))))
|
|
2343
|
-
return merge_tables(self._cache, sessions=df)
|
|
2383
|
+
return merge_tables(self._cache, sessions=df, origin=self.alyx.base_url)
|
|
2344
2384
|
|
|
2345
2385
|
def _download_datasets(self, dsets, **kwargs) -> List[ALFPath]:
|
|
2346
2386
|
"""Download a single or multitude of datasets if stored on AWS.
|
one/remote/globus.py
CHANGED
|
@@ -1211,7 +1211,7 @@ class Globus(DownloadClient):
|
|
|
1211
1211
|
async def task_wait_async(self, task_id, polling_interval=10, timeout=10):
|
|
1212
1212
|
"""Asynchronously wait until a Task is complete or fails, with a time limit.
|
|
1213
1213
|
|
|
1214
|
-
If the task status is ACTIVE after
|
|
1214
|
+
If the task status is ACTIVE after timeout, returns False, otherwise returns True.
|
|
1215
1215
|
|
|
1216
1216
|
Parameters
|
|
1217
1217
|
----------
|
one/webclient.py
CHANGED
|
@@ -1155,7 +1155,8 @@ class AlyxClient:
|
|
|
1155
1155
|
assert endpoint_scheme[action]['action'] == 'get'
|
|
1156
1156
|
# add to url data if it is a string
|
|
1157
1157
|
if id:
|
|
1158
|
-
# this is a special case of the list where we query a uuid
|
|
1158
|
+
# this is a special case of the list where we query a uuid
|
|
1159
|
+
# usually read is better but list may return fewer data and therefore be faster
|
|
1159
1160
|
if 'django' in kwargs.keys():
|
|
1160
1161
|
kwargs['django'] = kwargs['django'] + ','
|
|
1161
1162
|
else:
|
|
@@ -1163,6 +1164,9 @@ class AlyxClient:
|
|
|
1163
1164
|
kwargs['django'] = f"{kwargs['django']}pk,{id}"
|
|
1164
1165
|
# otherwise, look for a dictionary of filter terms
|
|
1165
1166
|
if kwargs:
|
|
1167
|
+
# if django arg is present but is None, server will return a cryptic 500 status
|
|
1168
|
+
if 'django' in kwargs and kwargs['django'] is None:
|
|
1169
|
+
del kwargs['django']
|
|
1166
1170
|
# Convert all lists in query params to comma separated list
|
|
1167
1171
|
query_params = {k: ','.join(map(str, ensure_list(v))) for k, v in kwargs.items()}
|
|
1168
1172
|
url = update_url_params(url, query_params)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: ONE-api
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.1.1
|
|
4
4
|
Summary: Open Neurophysiology Environment
|
|
5
5
|
Author: IBL Staff
|
|
6
6
|
License: MIT
|
|
@@ -21,6 +21,7 @@ Requires-Dist: iblutil>=1.14.0
|
|
|
21
21
|
Requires-Dist: packaging
|
|
22
22
|
Requires-Dist: boto3
|
|
23
23
|
Requires-Dist: pyyaml
|
|
24
|
+
Dynamic: license-file
|
|
24
25
|
|
|
25
26
|
# Open Neurophysiology Environment
|
|
26
27
|
[](https://coveralls.io/github/int-brain-lab/ONE?branch=main)
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
one/__init__.py,sha256=
|
|
2
|
-
one/api.py,sha256=
|
|
1
|
+
one/__init__.py,sha256=w5ArMDExoOvziPE_Nix7Iafegwtcr75LLiks1Yl0Cl0,76
|
|
2
|
+
one/api.py,sha256=YypYLcHsGEJ-gY7TEx5YR3NNT4AF5uIpZ0-H05ydFlQ,134054
|
|
3
3
|
one/converters.py,sha256=icKlwPyxf3tJtyOFBj_SG06QDLIZLdTGalCSk1-cAvk,30733
|
|
4
4
|
one/params.py,sha256=zwR0Yq09ztROfH3fJsCUc-IDs_PBixT3WU3dm1vX728,15050
|
|
5
5
|
one/registration.py,sha256=cWvQFAzCF04wMZJjdOzBPJkYOJ3BO2KEgqtVG7qOlmA,36177
|
|
6
6
|
one/util.py,sha256=NUG_dTz3_4GXYG49qSql6mFCBkaVaq7_XdecRPRszJQ,20173
|
|
7
|
-
one/webclient.py,sha256=
|
|
7
|
+
one/webclient.py,sha256=s7O5S9DhGnxj6g2xMfq1NsyvoBXd_Zz9UEw74A9VnBE,50409
|
|
8
8
|
one/alf/__init__.py,sha256=DaFIi7PKlurp5HnyNOpJuIlY3pyhiottFpJfxR59VsY,70
|
|
9
|
-
one/alf/cache.py,sha256=
|
|
9
|
+
one/alf/cache.py,sha256=09NDLGIUA6GcCJgzbDxSFpgq9sU6gyCuHIF8c7zHAMM,24814
|
|
10
10
|
one/alf/exceptions.py,sha256=6Gw8_ZObLnuYUpY4i0UyU4IA0kBZMBnJBGQyzIcDlUM,3427
|
|
11
11
|
one/alf/io.py,sha256=7mmd1wJwh7qok0WKkpGzkLxEuw4WSnaQ6EkGVvaODLI,30798
|
|
12
12
|
one/alf/path.py,sha256=LpB0kdTPBzuEdAWGe5hFDWCKSq4FgCdiDk8LwVzv-T0,46793
|
|
@@ -14,7 +14,7 @@ one/alf/spec.py,sha256=eWW4UDMsyYO52gzadZ-l09X2adVbXwpRKUBtd7NlYMw,20711
|
|
|
14
14
|
one/remote/__init__.py,sha256=pasT1r9t2F2_-hbx-5EKlJ-yCM9ItucQbeDHUbFJcyA,40
|
|
15
15
|
one/remote/aws.py,sha256=DurvObYa8rgyYQMWeaq1ol7PsdJoE9bVT1NnH2rxRRE,10241
|
|
16
16
|
one/remote/base.py,sha256=cmS5TwCPjlSLuoN2UN995O9N-4Zr8ZHz3in_iRopMgs,4406
|
|
17
|
-
one/remote/globus.py,sha256=
|
|
17
|
+
one/remote/globus.py,sha256=GZTuORKFGKmuO9a71YcaURXIozmjIj_--mD5hqsZmLw,47511
|
|
18
18
|
one/tests/fixtures/datasets.pqt,sha256=oYfOoGJfT1HN_rj-zpAaNpzzJsneekvDqg4zQ6XOQgk,29918
|
|
19
19
|
one/tests/fixtures/sessions.pqt,sha256=KmBvSi_o0fhSnFhOjQOTq1BHemZsbn89BcSVw4Ecp60,6916
|
|
20
20
|
one/tests/fixtures/test_dbs.json,sha256=Dq6IBOjofB6YdsEJEZDSDe6Gh4WebKX4Erx7Hs2i8WQ,310
|
|
@@ -30,8 +30,8 @@ one/tests/fixtures/rest_responses/6dc96f7e9bcc6ac2e7581489b9580a6cd3f28293,sha25
|
|
|
30
30
|
one/tests/fixtures/rest_responses/db1731fb8df0208944ae85f76718430813a8bf50,sha256=Dki7cTDg1bhbtPkuiv9NPRsuhM7TuicxoLcGtRD4D8g,209
|
|
31
31
|
one/tests/fixtures/rest_responses/dcce48259bb929661f60a02a48563f70aa6185b3,sha256=skaKl6sPgTyrufCIGNVNJfccXM-jSjYvAdyqNS3HXuA,416
|
|
32
32
|
one/tests/fixtures/rest_responses/f530d6022f61cdc9e38cc66beb3cb71f3003c9a1,sha256=EOqhNIVcmZ7j7aF09g5StFY434f2xbxwZLHwfeM4tug,22530
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
33
|
+
one_api-3.1.1.dist-info/licenses/LICENSE,sha256=W6iRQJcr-tslNfY4gL98IWvPtpe0E3tcWCFOD3IFUqg,1087
|
|
34
|
+
one_api-3.1.1.dist-info/METADATA,sha256=DtcBaq4qRm80sgWhjnQKvIC2yle_bdSTope6U8iAv7M,4170
|
|
35
|
+
one_api-3.1.1.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
|
|
36
|
+
one_api-3.1.1.dist-info/top_level.txt,sha256=LIsI2lzmA5jh8Zrw5dzMdE3ydLgmq-WF6rpoxSVDSAY,4
|
|
37
|
+
one_api-3.1.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|