ONE-api 3.0b5__py3-none-any.whl → 3.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
one/__init__.py CHANGED
@@ -1,2 +1,2 @@
1
1
  """The Open Neurophysiology Environment (ONE) API."""
2
- __version__ = '3.0b5'
2
+ __version__ = '3.1.1'
one/alf/cache.py CHANGED
@@ -22,11 +22,12 @@ from functools import partial
22
22
  from pathlib import Path
23
23
  import warnings
24
24
  import logging
25
+ from copy import deepcopy
25
26
 
26
27
  import pandas as pd
27
28
  import numpy as np
28
29
  from packaging import version
29
- from iblutil.util import Bunch
30
+ from iblutil.util import Bunch, ensure_list
30
31
  from iblutil.io import parquet
31
32
  from iblutil.io.hashfile import md5
32
33
 
@@ -35,8 +36,9 @@ from one.alf.io import iter_sessions
35
36
  from one.alf.path import session_path_parts, get_alf_path
36
37
 
37
38
  __all__ = [
38
- 'make_parquet_db', 'patch_tables', 'merge_tables', 'QC_TYPE', 'remove_table_files',
39
- 'remove_missing_datasets', 'load_tables', 'EMPTY_DATASETS_FRAME', 'EMPTY_SESSIONS_FRAME']
39
+ 'make_parquet_db', 'load_tables', 'patch_tables', 'merge_tables',
40
+ 'remove_table_files', 'remove_missing_datasets', 'default_cache',
41
+ 'QC_TYPE', 'EMPTY_DATASETS_FRAME', 'EMPTY_SESSIONS_FRAME']
40
42
  _logger = logging.getLogger(__name__)
41
43
 
42
44
  # -------------------------------------------------------------------------------------------------
@@ -259,6 +261,36 @@ def _make_datasets_df(root_dir, hash_files=False) -> pd.DataFrame:
259
261
  return pd.DataFrame(rows, columns=DATASETS_COLUMNS).astype(DATASETS_COLUMNS)
260
262
 
261
263
 
264
+ def default_cache(origin=''):
265
+ """Returns an empty cache dictionary with the default tables.
266
+
267
+ Parameters
268
+ ----------
269
+ origin : str, optional
270
+ The origin of the cache (e.g. a computer name or database name).
271
+
272
+ Returns
273
+ -------
274
+ Bunch
275
+ A Bunch object containing the loaded cache tables and associated metadata.
276
+
277
+ """
278
+ table_meta = _metadata(origin)
279
+ # The origin is now a set, however we leave _metadata as Alyx relies on this and sets
280
+ # can't be serialized to JSON
281
+ table_meta['origin'] = set(filter(None, [origin]))
282
+ return Bunch({
283
+ 'datasets': EMPTY_DATASETS_FRAME.copy(),
284
+ 'sessions': EMPTY_SESSIONS_FRAME.copy(),
285
+ '_meta': {
286
+ 'created_time': None,
287
+ 'loaded_time': None,
288
+ 'modified_time': None,
289
+ 'saved_time': None,
290
+ 'raw': {k: deepcopy(table_meta) for k in ('datasets', 'sessions')}}
291
+ })
292
+
293
+
262
294
  def make_parquet_db(root_dir, out_dir=None, hash_ids=True, hash_files=False, lab=None):
263
295
  """Given a data directory, index the ALF datasets and save the generated cache tables.
264
296
 
@@ -375,17 +407,8 @@ def load_tables(tables_dir, glob_pattern='*.pqt'):
375
407
  A Bunch object containing the loaded cache tables and associated metadata.
376
408
 
377
409
  """
378
- meta = {
379
- 'created_time': None,
380
- 'loaded_time': None,
381
- 'modified_time': None,
382
- 'saved_time': None,
383
- 'raw': {}
384
- }
385
- caches = Bunch({
386
- 'datasets': EMPTY_DATASETS_FRAME.copy(),
387
- 'sessions': EMPTY_SESSIONS_FRAME.copy(),
388
- '_meta': meta})
410
+ caches = default_cache()
411
+ meta = caches['_meta']
389
412
  INDEX_KEY = '.?id'
390
413
  for cache_file in Path(tables_dir).glob(glob_pattern):
391
414
  table = cache_file.stem
@@ -416,6 +439,10 @@ def load_tables(tables_dir, glob_pattern='*.pqt'):
416
439
  if not cache.index.is_monotonic_increasing:
417
440
  cache.sort_index(inplace=True)
418
441
 
442
+ # Ensure origin is a set (supports multiple origins)
443
+ meta['raw'][table]['origin'] = set(
444
+ filter(None, ensure_list(meta['raw'][table].get('origin', 'unknown'))))
445
+
419
446
  caches[table] = cache
420
447
 
421
448
  created = [datetime.datetime.fromisoformat(x['date_created'])
@@ -425,9 +452,12 @@ def load_tables(tables_dir, glob_pattern='*.pqt'):
425
452
  return caches
426
453
 
427
454
 
428
- def merge_tables(cache, strict=False, **kwargs):
455
+ def merge_tables(cache, strict=False, origin=None, **kwargs):
429
456
  """Update the cache tables with new records.
430
457
 
458
+ Note: A copy of the tables in cache may be returned if the original tables are immutable.
459
+ This can happen when tables are loaded from a parquet file.
460
+
431
461
  Parameters
432
462
  ----------
433
463
  dict
@@ -435,6 +465,8 @@ def merge_tables(cache, strict=False, **kwargs):
435
465
  strict : bool
436
466
  If not True, the columns don't need to match. Extra columns in input tables are
437
467
  dropped and missing columns are added and filled with np.nan.
468
+ origin : str
469
+ The origin of the cache (e.g. a computer name or database name).
438
470
  kwargs
439
471
  pandas.DataFrame or pandas.Series to insert/update for each table.
440
472
 
@@ -488,13 +520,31 @@ def merge_tables(cache, strict=False, **kwargs):
488
520
  records = records.astype(cache[table].dtypes)
489
521
  # Update existing rows
490
522
  to_update = records.index.isin(cache[table].index)
491
- cache[table].loc[records.index[to_update], :] = records[to_update]
523
+ try:
524
+ cache[table].loc[records.index[to_update], :] = records[to_update]
525
+ except ValueError as e:
526
+ if 'assignment destination is read-only' in str(e):
527
+ # NB: nullable integer and categorical dtypes may be backed by immutable arrays
528
+ # after loading from parquet and therefore must be copied before assignment
529
+ cache[table] = cache[table].copy()
530
+ cache[table].loc[records.index[to_update], :] = records[to_update]
531
+ else:
532
+ raise e # pragma: no cover
533
+
492
534
  # Assign new rows
493
535
  to_assign = records[~to_update]
494
536
  frames = [cache[table], to_assign]
495
537
  # Concatenate and sort
496
538
  cache[table] = pd.concat(frames).sort_index()
497
539
  updated = datetime.datetime.now()
540
+ # Update the table metadata with the origin
541
+ if origin is not None:
542
+ table_meta = cache['_meta']['raw'].get(table, {})
543
+ if 'origin' not in table_meta:
544
+ table_meta['origin'] = set(origin)
545
+ else:
546
+ table_meta['origin'].add(origin)
547
+ cache['_meta']['raw'][table] = table_meta
498
548
  cache['_meta']['modified_time'] = updated
499
549
  return updated
500
550
 
one/api.py CHANGED
@@ -31,7 +31,7 @@ import one.alf.exceptions as alferr
31
31
  from one.alf.path import ALFPath
32
32
  from .alf.cache import (
33
33
  make_parquet_db, load_tables, remove_table_files, merge_tables,
34
- EMPTY_DATASETS_FRAME, EMPTY_SESSIONS_FRAME, cast_index_object)
34
+ default_cache, cast_index_object)
35
35
  from .alf.spec import is_uuid, is_uuid_string, QC, to_alf
36
36
  from . import __version__
37
37
  from one.converters import ConversionMixin, session_record2path, ses2records, datasets2records
@@ -49,7 +49,7 @@ class One(ConversionMixin):
49
49
  """An API for searching and loading data on a local filesystem."""
50
50
 
51
51
  _search_terms = (
52
- 'dataset', 'date_range', 'laboratory', 'number',
52
+ 'datasets', 'date_range', 'laboratory', 'number',
53
53
  'projects', 'subject', 'task_protocol', 'dataset_qc_lte'
54
54
  )
55
55
 
@@ -113,16 +113,7 @@ class One(ConversionMixin):
113
113
 
114
114
  def _reset_cache(self):
115
115
  """Replace the cache object with a Bunch that contains the right fields."""
116
- self._cache = Bunch({
117
- 'datasets': EMPTY_DATASETS_FRAME.copy(),
118
- 'sessions': EMPTY_SESSIONS_FRAME.copy(),
119
- '_meta': {
120
- 'created_time': None,
121
- 'loaded_time': None,
122
- 'modified_time': None,
123
- 'saved_time': None,
124
- 'raw': {}} # map of original table metadata
125
- })
116
+ self._cache = default_cache()
126
117
 
127
118
  def _remove_table_files(self, tables=None):
128
119
  """Delete cache tables on disk.
@@ -143,20 +134,25 @@ class One(ConversionMixin):
143
134
  tables = tables or filter(lambda x: x[0] != '_', self._cache)
144
135
  return remove_table_files(self._tables_dir, tables)
145
136
 
146
- def load_cache(self, tables_dir=None, **kwargs):
137
+ def load_cache(self, tables_dir=None, clobber=True, **kwargs):
147
138
  """Load parquet cache files from a local directory.
148
139
 
149
140
  Parameters
150
141
  ----------
151
142
  tables_dir : str, pathlib.Path
152
143
  An optional directory location of the parquet files, defaults to One._tables_dir.
144
+ clobber : bool
145
+ If true, the cache is loaded without merging with existing table files.
153
146
 
154
147
  Returns
155
148
  -------
156
149
  datetime.datetime
157
150
  A timestamp of when the cache was loaded.
158
151
  """
159
- self._reset_cache()
152
+ if clobber:
153
+ self._reset_cache()
154
+ else:
155
+ raise NotImplementedError('clobber=False not implemented yet')
160
156
  self._tables_dir = Path(tables_dir or self._tables_dir or self.cache_dir)
161
157
  self._cache = load_tables(self._tables_dir)
162
158
 
@@ -169,7 +165,7 @@ class One(ConversionMixin):
169
165
  # prompt the user to delete them to improve load times
170
166
  raw_meta = self._cache['_meta'].get('raw', {}).values() or [{}]
171
167
  tagged = any(filter(None, flatten(x.get('database_tags') for x in raw_meta)))
172
- origin = set(x['origin'] for x in raw_meta if 'origin' in x)
168
+ origin = set(filter(None, flatten(ensure_list(x.get('origin', [])) for x in raw_meta)))
173
169
  older = (self._cache['_meta']['created_time'] or datetime.now()) < datetime(2025, 2, 13)
174
170
  remote = not self.offline and self.mode == 'remote'
175
171
  if remote and origin == {'alyx'} and older and not self._web_client.silent and not tagged:
@@ -216,11 +212,21 @@ class One(ConversionMixin):
216
212
  caches = load_tables(save_dir)
217
213
  merge_tables(
218
214
  caches, **{k: v for k, v in self._cache.items() if not k.startswith('_')})
215
+ # Ensure we use the minimum created date for each table
216
+ for table in caches['_meta']['raw']:
217
+ raw_meta = [x['_meta']['raw'].get(table, {}) for x in (caches, self._cache)]
218
+ created = filter(None, (x.get('date_created') for x in raw_meta))
219
+ if any(created := list(created)):
220
+ created = min(map(datetime.fromisoformat, created))
221
+ created = created.isoformat(sep=' ', timespec='minutes')
222
+ meta['raw'][table]['date_created'] = created
219
223
 
220
224
  with FileLock(save_dir, log=_logger, timeout=TIMEOUT, timeout_action='delete'):
221
225
  _logger.info('Saving cache tables...')
222
226
  for table in filter(lambda x: not x[0] == '_', caches.keys()):
223
227
  metadata = meta['raw'].get(table, {})
228
+ if isinstance(metadata.get('origin'), set):
229
+ metadata['origin'] = list(metadata['origin'])
224
230
  metadata['date_modified'] = modified.isoformat(sep=' ', timespec='minutes')
225
231
  filename = save_dir.joinpath(f'{table}.pqt')
226
232
  # Cast indices to str before saving
@@ -319,10 +325,8 @@ class One(ConversionMixin):
319
325
 
320
326
  Parameters
321
327
  ----------
322
- dataset : str, list
323
- One or more dataset names. Returns sessions containing all these datasets.
324
- A dataset matches if it contains the search string e.g. 'wheel.position' matches
325
- '_ibl_wheel.position.npy'.
328
+ datasets : str, list
329
+ One or more (exact) dataset names. Returns sessions containing all of these datasets.
326
330
  dataset_qc_lte : str, int, one.alf.spec.QC
327
331
  A dataset QC value, returns sessions with datasets at or below this QC value, including
328
332
  those with no QC set. If `dataset` not passed, sessions with any passing QC datasets
@@ -370,7 +374,9 @@ class One(ConversionMixin):
370
374
 
371
375
  Search for sessions on a given date, in a given lab, containing trials and spike data.
372
376
 
373
- >>> eids = one.search(date='2023-01-01', lab='churchlandlab', dataset=['trials', 'spikes'])
377
+ >>> eids = one.search(
378
+ ... date='2023-01-01', lab='churchlandlab',
379
+ ... datasets=['trials.table.pqt', 'spikes.times.npy'])
374
380
 
375
381
  Search for sessions containing trials and spike data where QC for both are WARNING or less.
376
382
 
@@ -397,13 +403,14 @@ class One(ConversionMixin):
397
403
 
398
404
  def all_present(x, dsets, exists=True):
399
405
  """Returns true if all datasets present in Series."""
400
- return all(any(x.str.contains(y, regex=self.wildcards) & exists) for y in dsets)
406
+ name = x.str.rsplit('/', n=1, expand=True).iloc[:, -1]
407
+ return all(any(name.str.fullmatch(y) & exists) for y in dsets)
401
408
 
402
409
  # Iterate over search filters, reducing the sessions table
403
410
  sessions = self._cache['sessions']
404
411
 
405
412
  # Ensure sessions filtered in a particular order, with datasets last
406
- search_order = ('date_range', 'number', 'dataset')
413
+ search_order = ('date_range', 'number', 'datasets')
407
414
 
408
415
  def sort_fcn(itm):
409
416
  return -1 if itm[0] not in search_order else search_order.index(itm[0])
@@ -430,12 +437,15 @@ class One(ConversionMixin):
430
437
  query = ensure_list(value)
431
438
  sessions = sessions[sessions[key].isin(map(int, query))]
432
439
  # Dataset/QC check is biggest so this should be done last
433
- elif key == 'dataset' or (key == 'dataset_qc_lte' and 'dataset' not in queries):
440
+ elif key == 'datasets' or (key == 'dataset_qc_lte' and 'datasets' not in queries):
434
441
  datasets = self._cache['datasets']
435
442
  qc = QC.validate(queries.get('dataset_qc_lte', 'FAIL')).name # validate value
436
443
  has_dset = sessions.index.isin(datasets.index.get_level_values('eid'))
444
+ if not has_dset.any():
445
+ sessions = sessions.iloc[0:0] # No datasets for any sessions
446
+ continue
437
447
  datasets = datasets.loc[(sessions.index.values[has_dset], ), :]
438
- query = ensure_list(value if key == 'dataset' else '')
448
+ query = ensure_list(value if key == 'datasets' else '')
439
449
  # For each session check any dataset both contains query and exists
440
450
  mask = (
441
451
  (datasets
@@ -1025,7 +1035,7 @@ class One(ConversionMixin):
1025
1035
  """
1026
1036
  query_type = query_type or self.mode
1027
1037
  datasets = self.list_datasets(
1028
- eid, details=True, query_type=query_type, keep_eid_index=True, revision=revision)
1038
+ eid, details=True, query_type=query_type, keep_eid_index=True)
1029
1039
 
1030
1040
  if len(datasets) == 0:
1031
1041
  raise alferr.ALFObjectNotFound(obj)
@@ -1692,7 +1702,8 @@ class OneAlyx(One):
1692
1702
  tag = tag or current_tags[0] # For refreshes take the current tag as default
1693
1703
  different_tag = any(x != tag for x in current_tags)
1694
1704
  if not (clobber or different_tag):
1695
- super(OneAlyx, self).load_cache(tables_dir) # Load any present cache
1705
+ # Load any present cache
1706
+ super(OneAlyx, self).load_cache(tables_dir, clobber=True)
1696
1707
  cache_meta = self._cache.get('_meta', {})
1697
1708
  raw_meta = cache_meta.get('raw', {}).values() or [{}]
1698
1709
 
@@ -1711,7 +1722,13 @@ class OneAlyx(One):
1711
1722
  remote_created = datetime.fromisoformat(cache_info['date_created'])
1712
1723
  local_created = cache_meta.get('created_time', None)
1713
1724
  fresh = local_created and (remote_created - local_created) < timedelta(minutes=1)
1714
- if fresh and not different_tag:
1725
+ # The local cache may have been created locally more recently, but if it doesn't
1726
+ # contain the same tag or origin, we need to download the remote one.
1727
+ origin = cache_info.get('origin', 'unknown')
1728
+ local_origin = (x.get('origin', []) for x in raw_meta)
1729
+ local_origin = set(flatten(map(ensure_list, local_origin)))
1730
+ different_origin = origin not in local_origin
1731
+ if fresh and not (different_tag or different_origin):
1715
1732
  _logger.info('No newer cache available')
1716
1733
  return cache_meta['loaded_time']
1717
1734
 
@@ -1725,19 +1742,27 @@ class OneAlyx(One):
1725
1742
  self._tables_dir = self._tables_dir or self.cache_dir
1726
1743
 
1727
1744
  # Check if the origin has changed. This is to warn users if downloading from a
1728
- # different database to the one currently loaded.
1729
- prev_origin = list(set(filter(None, (x.get('origin') for x in raw_meta))))
1730
- origin = cache_info.get('origin', 'unknown')
1731
- if prev_origin and origin not in prev_origin:
1745
+ # different database to the one currently loaded. When building the cache from
1746
+ # remote queries the origin is set to the Alyx database URL. If the cache info
1747
+ # origin name and URL are different, warn the user.
1748
+ if different_origin and local_origin and self.alyx.base_url not in local_origin:
1732
1749
  warnings.warn(
1733
1750
  'Downloading cache tables from another origin '
1734
- f'("{origin}" instead of "{", ".join(prev_origin)}")')
1751
+ f'("{origin}" instead of "{", ".join(local_origin)}")')
1735
1752
 
1736
1753
  # Download the remote cache files
1737
1754
  _logger.info('Downloading remote caches...')
1738
1755
  files = self.alyx.download_cache_tables(cache_info.get('location'), self._tables_dir)
1739
1756
  assert any(files)
1740
- return super(OneAlyx, self).load_cache(self._tables_dir) # Reload cache after download
1757
+ # Reload cache after download
1758
+ loaded_time = super(OneAlyx, self).load_cache(self._tables_dir)
1759
+ # Add db URL to origin set so we know where the cache came from
1760
+ for raw_meta in self._cache['_meta']['raw'].values():
1761
+ table_origin = set(filter(None, ensure_list(raw_meta.get('origin', []))))
1762
+ if origin in table_origin:
1763
+ table_origin.add(self.alyx.base_url)
1764
+ raw_meta['origin'] = table_origin
1765
+ return loaded_time
1741
1766
  except (requests.exceptions.HTTPError, wc.HTTPError, requests.exceptions.SSLError) as ex:
1742
1767
  _logger.debug(ex)
1743
1768
  _logger.error(f'{type(ex).__name__}: Failed to load the remote cache file')
@@ -1847,7 +1872,8 @@ class OneAlyx(One):
1847
1872
  return self._cache['datasets'].iloc[0:0] if details else [] # Return empty
1848
1873
  session, datasets = ses2records(self.alyx.rest('sessions', 'read', id=eid))
1849
1874
  # Add to cache tables
1850
- merge_tables(self._cache, sessions=session, datasets=datasets.copy())
1875
+ merge_tables(
1876
+ self._cache, sessions=session, datasets=datasets.copy(), origin=self.alyx.base_url)
1851
1877
  if datasets is None or datasets.empty:
1852
1878
  return self._cache['datasets'].iloc[0:0] if details else [] # Return empty
1853
1879
  assert set(datasets.index.unique('eid')) == {eid}
@@ -1999,7 +2025,7 @@ class OneAlyx(One):
1999
2025
  rec = self.alyx.rest('insertions', 'read', id=str(pid))
2000
2026
  return UUID(rec['session']), rec['name']
2001
2027
 
2002
- def eid2pid(self, eid, query_type=None, details=False):
2028
+ def eid2pid(self, eid, query_type=None, details=False, **kwargs) -> (UUID, str, list):
2003
2029
  """Given an experiment UUID (eID), return the probe IDs and labels (i.e. ALF collection).
2004
2030
 
2005
2031
  NB: Requires a connection to the Alyx database.
@@ -2013,6 +2039,8 @@ class OneAlyx(One):
2013
2039
  Query mode - options include 'remote', and 'refresh'.
2014
2040
  details : bool
2015
2041
  Additionally return the complete Alyx records from insertions endpoint.
2042
+ kwargs
2043
+ Additional parameters to filter insertions Alyx endpoint.
2016
2044
 
2017
2045
  Returns
2018
2046
  -------
@@ -2023,6 +2051,15 @@ class OneAlyx(One):
2023
2051
  list of dict (optional)
2024
2052
  If details is true, returns the Alyx records from insertions endpoint.
2025
2053
 
2054
+ Examples
2055
+ --------
2056
+ Get the probe IDs and details for a given session ID
2057
+
2058
+ >>> pids, labels, recs = one.eid2pid(eid, details=True)
2059
+
2060
+ Get the probe ID for a given session ID and label
2061
+
2062
+ >>> (pid,), _ = one.eid2pid(eid, details=False, name='probe00')
2026
2063
  """
2027
2064
  query_type = query_type or self.mode
2028
2065
  if query_type == 'local' and 'insertions' not in self._cache.keys():
@@ -2030,7 +2067,7 @@ class OneAlyx(One):
2030
2067
  eid = self.to_eid(eid) # Ensure we have a UUID str
2031
2068
  if not eid:
2032
2069
  return (None,) * (3 if details else 2)
2033
- recs = self.alyx.rest('insertions', 'list', session=eid)
2070
+ recs = self.alyx.rest('insertions', 'list', session=eid, **kwargs)
2034
2071
  pids = [UUID(x['id']) for x in recs]
2035
2072
  labels = [x['name'] for x in recs]
2036
2073
  if details:
@@ -2173,7 +2210,8 @@ class OneAlyx(One):
2173
2210
  # Build sessions table
2174
2211
  session_records = (x['session_info'] for x in insertions_records)
2175
2212
  sessions_df = pd.DataFrame(next(zip(*map(ses2records, session_records))))
2176
- return merge_tables(self._cache, insertions=df, sessions=sessions_df)
2213
+ return merge_tables(
2214
+ self._cache, insertions=df, sessions=sessions_df, origin=self.alyx.base_url)
2177
2215
 
2178
2216
  def search(self, details=False, query_type=None, **kwargs):
2179
2217
  """Searches sessions matching the given criteria and returns a list of matching eids.
@@ -2192,10 +2230,8 @@ class OneAlyx(One):
2192
2230
 
2193
2231
  Parameters
2194
2232
  ----------
2195
- dataset : str
2196
- A (partial) dataset name. Returns sessions containing matching datasets.
2197
- A dataset matches if it contains the search string e.g. 'wheel.position' matches
2198
- '_ibl_wheel.position.npy'. C.f. `datasets` argument.
2233
+ datasets : str, list
2234
+ One or more (exact) dataset names. Returns sessions containing all of these datasets.
2199
2235
  date_range : str, list, datetime.datetime, datetime.date, pandas.timestamp
2200
2236
  A single date to search or a list of 2 dates that define the range (inclusive). To
2201
2237
  define only the upper or lower date bound, set the other element to None.
@@ -2222,11 +2258,12 @@ class OneAlyx(One):
2222
2258
  A str or list of lab location (as per Alyx definition) name.
2223
2259
  Note: this corresponds to the specific rig, not the lab geographical location per se.
2224
2260
  dataset_types : str, list
2225
- One or more of dataset_types.
2226
- datasets : str, list
2227
- One or more (exact) dataset names. Returns sessions containing all of these datasets.
2261
+ One or more of dataset_types. Unlike with `datasets`, the dataset types for the
2262
+ sessions returned may not be reachable (i.e. for recent sessions the datasets may not
2263
+ yet be available).
2228
2264
  dataset_qc_lte : int, str, one.alf.spec.QC
2229
- The maximum QC value for associated datasets.
2265
+ The maximum QC value for associated datasets. NB: Without `datasets`, not all
2266
+ associated datasets with the matching QC values are guarenteed to be reachable.
2230
2267
  details : bool
2231
2268
  If true also returns a dict of dataset details.
2232
2269
  query_type : str, None
@@ -2271,6 +2308,9 @@ class OneAlyx(One):
2271
2308
  - In default and local mode, when the one.wildcards flag is True (default), queries are
2272
2309
  interpreted as regular expressions. To turn this off set one.wildcards to False.
2273
2310
  - In remote mode regular expressions are only supported using the `django` argument.
2311
+ - In remote mode, only the `datasets` argument returns sessions where datasets are
2312
+ registered *and* exist. Using `dataset_types` or `dataset_qc_lte` without `datasets`
2313
+ will not check that the datasets are reachable.
2274
2314
 
2275
2315
  """
2276
2316
  query_type = query_type or self.mode
@@ -2340,7 +2380,7 @@ class OneAlyx(One):
2340
2380
 
2341
2381
  """
2342
2382
  df = pd.DataFrame(next(zip(*map(ses2records, session_records))))
2343
- return merge_tables(self._cache, sessions=df)
2383
+ return merge_tables(self._cache, sessions=df, origin=self.alyx.base_url)
2344
2384
 
2345
2385
  def _download_datasets(self, dsets, **kwargs) -> List[ALFPath]:
2346
2386
  """Download a single or multitude of datasets if stored on AWS.
one/remote/globus.py CHANGED
@@ -1211,7 +1211,7 @@ class Globus(DownloadClient):
1211
1211
  async def task_wait_async(self, task_id, polling_interval=10, timeout=10):
1212
1212
  """Asynchronously wait until a Task is complete or fails, with a time limit.
1213
1213
 
1214
- If the task status is ACTIVE after timout, returns False, otherwise returns True.
1214
+ If the task status is ACTIVE after timeout, returns False, otherwise returns True.
1215
1215
 
1216
1216
  Parameters
1217
1217
  ----------
one/webclient.py CHANGED
@@ -1155,7 +1155,8 @@ class AlyxClient:
1155
1155
  assert endpoint_scheme[action]['action'] == 'get'
1156
1156
  # add to url data if it is a string
1157
1157
  if id:
1158
- # this is a special case of the list where we query a uuid. Usually read is better
1158
+ # this is a special case of the list where we query a uuid
1159
+ # usually read is better but list may return fewer data and therefore be faster
1159
1160
  if 'django' in kwargs.keys():
1160
1161
  kwargs['django'] = kwargs['django'] + ','
1161
1162
  else:
@@ -1163,6 +1164,9 @@ class AlyxClient:
1163
1164
  kwargs['django'] = f"{kwargs['django']}pk,{id}"
1164
1165
  # otherwise, look for a dictionary of filter terms
1165
1166
  if kwargs:
1167
+ # if django arg is present but is None, server will return a cryptic 500 status
1168
+ if 'django' in kwargs and kwargs['django'] is None:
1169
+ del kwargs['django']
1166
1170
  # Convert all lists in query params to comma separated list
1167
1171
  query_params = {k: ','.join(map(str, ensure_list(v))) for k, v in kwargs.items()}
1168
1172
  url = update_url_params(url, query_params)
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: ONE-api
3
- Version: 3.0b5
3
+ Version: 3.1.1
4
4
  Summary: Open Neurophysiology Environment
5
5
  Author: IBL Staff
6
6
  License: MIT
@@ -21,6 +21,7 @@ Requires-Dist: iblutil>=1.14.0
21
21
  Requires-Dist: packaging
22
22
  Requires-Dist: boto3
23
23
  Requires-Dist: pyyaml
24
+ Dynamic: license-file
24
25
 
25
26
  # Open Neurophysiology Environment
26
27
  [![Coverage Status](https://coveralls.io/repos/github/int-brain-lab/ONE/badge.svg?branch=main)](https://coveralls.io/github/int-brain-lab/ONE?branch=main)
@@ -1,12 +1,12 @@
1
- one/__init__.py,sha256=nInf_OtC7ZRulDTsGaWoCYJCQdioyk1iYSfMN-_fbMU,76
2
- one/api.py,sha256=IWAjCBhw4DCLmfxcwVkOl0bgRlu2RXIV6Nyfey8HNas,131576
1
+ one/__init__.py,sha256=w5ArMDExoOvziPE_Nix7Iafegwtcr75LLiks1Yl0Cl0,76
2
+ one/api.py,sha256=YypYLcHsGEJ-gY7TEx5YR3NNT4AF5uIpZ0-H05ydFlQ,134054
3
3
  one/converters.py,sha256=icKlwPyxf3tJtyOFBj_SG06QDLIZLdTGalCSk1-cAvk,30733
4
4
  one/params.py,sha256=zwR0Yq09ztROfH3fJsCUc-IDs_PBixT3WU3dm1vX728,15050
5
5
  one/registration.py,sha256=cWvQFAzCF04wMZJjdOzBPJkYOJ3BO2KEgqtVG7qOlmA,36177
6
6
  one/util.py,sha256=NUG_dTz3_4GXYG49qSql6mFCBkaVaq7_XdecRPRszJQ,20173
7
- one/webclient.py,sha256=CfHKn6eZu-woed_VEeY8DV6WTzb49u41-lBTRe-EYnM,50132
7
+ one/webclient.py,sha256=s7O5S9DhGnxj6g2xMfq1NsyvoBXd_Zz9UEw74A9VnBE,50409
8
8
  one/alf/__init__.py,sha256=DaFIi7PKlurp5HnyNOpJuIlY3pyhiottFpJfxR59VsY,70
9
- one/alf/cache.py,sha256=NDN-Ii8E19C6IH1C7EQEIIgPFah3AtM9jhjLeHuBIjM,22735
9
+ one/alf/cache.py,sha256=09NDLGIUA6GcCJgzbDxSFpgq9sU6gyCuHIF8c7zHAMM,24814
10
10
  one/alf/exceptions.py,sha256=6Gw8_ZObLnuYUpY4i0UyU4IA0kBZMBnJBGQyzIcDlUM,3427
11
11
  one/alf/io.py,sha256=7mmd1wJwh7qok0WKkpGzkLxEuw4WSnaQ6EkGVvaODLI,30798
12
12
  one/alf/path.py,sha256=LpB0kdTPBzuEdAWGe5hFDWCKSq4FgCdiDk8LwVzv-T0,46793
@@ -14,7 +14,7 @@ one/alf/spec.py,sha256=eWW4UDMsyYO52gzadZ-l09X2adVbXwpRKUBtd7NlYMw,20711
14
14
  one/remote/__init__.py,sha256=pasT1r9t2F2_-hbx-5EKlJ-yCM9ItucQbeDHUbFJcyA,40
15
15
  one/remote/aws.py,sha256=DurvObYa8rgyYQMWeaq1ol7PsdJoE9bVT1NnH2rxRRE,10241
16
16
  one/remote/base.py,sha256=cmS5TwCPjlSLuoN2UN995O9N-4Zr8ZHz3in_iRopMgs,4406
17
- one/remote/globus.py,sha256=3a3r5n0j5KuwNs-khu9hRoNjFx4ODoZfuHgrgToEEgM,47510
17
+ one/remote/globus.py,sha256=GZTuORKFGKmuO9a71YcaURXIozmjIj_--mD5hqsZmLw,47511
18
18
  one/tests/fixtures/datasets.pqt,sha256=oYfOoGJfT1HN_rj-zpAaNpzzJsneekvDqg4zQ6XOQgk,29918
19
19
  one/tests/fixtures/sessions.pqt,sha256=KmBvSi_o0fhSnFhOjQOTq1BHemZsbn89BcSVw4Ecp60,6916
20
20
  one/tests/fixtures/test_dbs.json,sha256=Dq6IBOjofB6YdsEJEZDSDe6Gh4WebKX4Erx7Hs2i8WQ,310
@@ -30,8 +30,8 @@ one/tests/fixtures/rest_responses/6dc96f7e9bcc6ac2e7581489b9580a6cd3f28293,sha25
30
30
  one/tests/fixtures/rest_responses/db1731fb8df0208944ae85f76718430813a8bf50,sha256=Dki7cTDg1bhbtPkuiv9NPRsuhM7TuicxoLcGtRD4D8g,209
31
31
  one/tests/fixtures/rest_responses/dcce48259bb929661f60a02a48563f70aa6185b3,sha256=skaKl6sPgTyrufCIGNVNJfccXM-jSjYvAdyqNS3HXuA,416
32
32
  one/tests/fixtures/rest_responses/f530d6022f61cdc9e38cc66beb3cb71f3003c9a1,sha256=EOqhNIVcmZ7j7aF09g5StFY434f2xbxwZLHwfeM4tug,22530
33
- ONE_api-3.0b5.dist-info/LICENSE,sha256=W6iRQJcr-tslNfY4gL98IWvPtpe0E3tcWCFOD3IFUqg,1087
34
- ONE_api-3.0b5.dist-info/METADATA,sha256=V71vo5_2W9rsjtyhah07thKb0zXKS0VkM53Zw3eYRu8,4148
35
- ONE_api-3.0b5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
36
- ONE_api-3.0b5.dist-info/top_level.txt,sha256=LIsI2lzmA5jh8Zrw5dzMdE3ydLgmq-WF6rpoxSVDSAY,4
37
- ONE_api-3.0b5.dist-info/RECORD,,
33
+ one_api-3.1.1.dist-info/licenses/LICENSE,sha256=W6iRQJcr-tslNfY4gL98IWvPtpe0E3tcWCFOD3IFUqg,1087
34
+ one_api-3.1.1.dist-info/METADATA,sha256=DtcBaq4qRm80sgWhjnQKvIC2yle_bdSTope6U8iAv7M,4170
35
+ one_api-3.1.1.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
36
+ one_api-3.1.1.dist-info/top_level.txt,sha256=LIsI2lzmA5jh8Zrw5dzMdE3ydLgmq-WF6rpoxSVDSAY,4
37
+ one_api-3.1.1.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.8.0)
2
+ Generator: setuptools (80.8.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5