ONE-api 3.2.1__py3-none-any.whl → 3.4.0__py3-none-any.whl

one/__init__.py

@@ -1,2 +1,2 @@
 """The Open Neurophysiology Environment (ONE) API."""
-__version__ = '3.2.1'
+__version__ = '3.4.0'

one/alf/path.py

@@ -27,6 +27,7 @@ ALFPath properties return empty str instead of None if ALF part isn't present..
 """
 import os
 import pathlib
+from functools import cached_property
 from collections import OrderedDict
 from datetime import datetime
 from typing import Union, Optional, Iterable
@@ -844,17 +845,17 @@ class PureALFPath(pathlib.PurePath):  # py3.12 supports direct subclassing
         """
         return filename_parts(self.name, assert_valid=False, as_dict=as_dict)
 
-    @property
+    @cached_property
     def dataset_name_parts(self):
         """tuple of str: the dataset name parts, with empty strings for missing parts."""
         return tuple(p or '' for p in self.parse_alf_name(as_dict=False))
 
-    @property
+    @cached_property
     def session_parts(self):
         """tuple of str: the session path parts, with empty strings for missing parts."""
         return tuple(p or '' for p in session_path_parts(self, assert_valid=False))
 
-    @property
+    @cached_property
     def alf_parts(self):
         """tuple of str: the full ALF path parts, with empty strings for missing parts."""
         return tuple(p or '' for p in self.parse_alf_path(as_dict=False))

one/api.py

@@ -6,6 +6,7 @@ import logging
 from weakref import WeakMethod
 from datetime import datetime, timedelta
 from functools import lru_cache, partial
+from itertools import islice
 from inspect import unwrap
 from pathlib import Path, PurePosixPath
 from typing import Any, Union, Optional, List
@@ -28,7 +29,7 @@ import one.webclient as wc
 import one.alf.io as alfio
 import one.alf.path as alfiles
 import one.alf.exceptions as alferr
-from one.alf.path import ALFPath
+from one.alf.path import ALFPath, ensure_alf_path
 from .alf.cache import (
     make_parquet_db, load_tables, remove_table_files, merge_tables,
     default_cache, cast_index_object)
@@ -41,6 +42,8 @@ _logger = logging.getLogger(__name__)
 __all__ = ['ONE', 'One', 'OneAlyx']
 SAVE_ON_DELETE = (os.environ.get('ONE_SAVE_ON_DELETE') or '0').casefold() in ('true', '1')
 """bool: Whether to save modified cache tables on delete."""
+REVISION_LAST_BEFORE = os.environ.get('ONE_REVISION_LAST_BEFORE')
+"""str: If set, the revision string to use when loading data before a given date."""
 
 _logger.debug('ONE_SAVE_ON_DELETE: %s', SAVE_ON_DELETE)
 
@@ -1001,7 +1004,7 @@ class One(ConversionMixin):
             The dataset revision (typically an ISO date).  If no exact match, the previous
             revision (ordered lexicographically) is returned.  If None, the default revision is
             returned (usually the most recent revision).  Regular expressions/wildcards not
-            permitted.
+            permitted.  May be set with `ONE_REVISION_LAST_BEFORE` environment variable.
         query_type : str
             Query cache ('local') or Alyx database ('remote').
         download_only : bool
@@ -1041,6 +1044,7 @@ class One(ConversionMixin):
             raise alferr.ALFObjectNotFound(obj)
 
         dataset = {'object': obj, **kwargs}
+        revision = revision or REVISION_LAST_BEFORE  # Use env var if set
         datasets = util.filter_datasets(datasets, dataset, collection, revision,
                                         assert_unique=False, wildcards=self.wildcards)
 
@@ -1094,7 +1098,7 @@ class One(ConversionMixin):
             The dataset revision (typically an ISO date).  If no exact match, the previous
             revision (ordered lexicographically) is returned.  If None, the default revision is
             returned (usually the most recent revision).  Regular expressions/wildcards not
-            permitted.
+            permitted.  May be set with `ONE_REVISION_LAST_BEFORE` environment variable.
         query_type : str
             Query cache ('local') or Alyx database ('remote')
         download_only : bool
@@ -1150,6 +1154,7 @@ class One(ConversionMixin):
             dataset += '.*'
             _logger.debug('Appending extension wildcard: ' + dataset)
 
+        revision = revision or REVISION_LAST_BEFORE  # Use env var if set
         assert_unique = ('/' if isinstance(dataset, str) else 'collection') not in dataset
         # Check if wildcard was used (this is not an exhaustive check)
         if not assert_unique and isinstance(dataset, str) and '*' in dataset:
@@ -1204,7 +1209,7 @@ class One(ConversionMixin):
             The dataset revision (typically an ISO date).  If no exact match, the previous
             revision (ordered lexicographically) is returned.  If None, the default revision is
             returned (usually the most recent revision).  Regular expressions/wildcards not
-            permitted.
+            permitted.  May be set with `ONE_REVISION_LAST_BEFORE` environment variable.
         query_type : str
             Query cache ('local') or Alyx database ('remote')
         assert_present : bool
@@ -1281,9 +1286,11 @@ class One(ConversionMixin):
         # Check if rel paths have been used (e.g. the output of list_datasets)
         is_frame = isinstance(datasets, pd.DataFrame)
         if is_rel_paths := (is_frame or any('/' in x for x in datasets)):
-            if not (collections, revisions) == (None, None):
+            if not (collections, revisions) == (None, None) or REVISION_LAST_BEFORE:
                 raise ValueError(
-                    'collection and revision kwargs must be None when dataset is a relative path')
+                    'collection and revision kwargs must be None when dataset is a relative path '
+                    '(ONE_REVISION_LAST_BEFORE env var must also be unset)'
+                )
             if is_frame:
                 if 'eid' in datasets.index.names:
                     assert set(datasets.index.get_level_values('eid')) == {eid}
@@ -1322,6 +1329,7 @@ class One(ConversionMixin):
             ]
 
         # Check input args
+        revisions = revisions or REVISION_LAST_BEFORE
         collections, revisions = _verify_specifiers([collections, revisions])
 
         # If collections provided in datasets list, e.g. [collection/x.y.z], do not assert unique
@@ -1456,7 +1464,7 @@ class One(ConversionMixin):
             The dataset revision (typically an ISO date).  If no exact match, the previous
             revision (ordered lexicographically) is returned.  If None, the default revision is
             returned (usually the most recent revision).  Regular expressions/wildcards not
-            permitted.
+            permitted.  May be set with `ONE_REVISION_LAST_BEFORE` environment variable.
         query_type : str
             Query cache ('local') or Alyx database ('remote')
         download_only : bool
@@ -1495,6 +1503,7 @@ class One(ConversionMixin):
             raise alferr.ALFError(f'{collection} not found for session {eid}')
 
         dataset = {'object': object, **kwargs}
+        revision = revision or REVISION_LAST_BEFORE  # use env var if set
         datasets = util.filter_datasets(datasets, dataset, revision,
                                         assert_unique=False, wildcards=self.wildcards)
 
@@ -1815,7 +1824,7 @@ class OneAlyx(One):
 
         endpoint = endpoint or self._search_endpoint
         # Return search terms from REST schema
-        fields = self.alyx.rest_schemes[endpoint]['list']['fields']
+        fields = self.alyx.rest_schemes.fields(endpoint, action='list')
         excl = ('lab',)  # 'laboratory' already in search terms
         if endpoint != 'sessions':
             return tuple(x['name'] for x in fields)
@@ -1981,6 +1990,9 @@ class OneAlyx(One):
 
         >>> trials = one.load_aggregate('subjects', 'SP026', '_ibl_subjectTraining.table')
 
+        Notes
+        -----
+        Unlike other loading functions, this function loads datasets with a matching revision.
         """
         # If only two parts and wildcards are on, append ext wildcard
         if self.wildcards and isinstance(dataset, str) and len(dataset.split('.')) == 2:
@@ -2020,9 +2032,26 @@ class OneAlyx(One):
 
         """
         query_type = query_type or self.mode
-        if query_type == 'local' and 'insertions' not in self._cache.keys():
-            raise NotImplementedError('Converting probe IDs required remote connection')
-        rec = self.alyx.rest('insertions', 'read', id=str(pid))
+        if query_type == 'local':  # and 'insertions' not in self._cache.keys():
+            if 'insertions' not in self._cache.keys():
+                raise NotImplementedError('Converting probe IDs requires remote connection')
+            else:
+                # If local, use the cache table
+                pid = UUID(pid) if isinstance(pid, str) else pid
+                try:
+                    rec = self._cache['insertions'].loc[pd.IndexSlice[:, pid], 'name']
+                    (eid, _), name = next(rec.items())
+                    return eid, name
+                except KeyError:
+                    return None, None
+        try:
+            rec = self.alyx.rest('insertions', 'read', id=pid)
+        except requests.exceptions.HTTPError as ex:
+            if ex.response.status_code == 404:
+                _logger.error(f'Probe {pid} not found in Alyx')
+                return None, None
+            raise ex
+        self._update_insertions_table([rec])
         return UUID(rec['session']), rec['name']
 
     def eid2pid(self, eid, query_type=None, details=False, **kwargs) -> (UUID, str, list):
@@ -2063,15 +2092,33 @@ class OneAlyx(One):
         """
         query_type = query_type or self.mode
         if query_type == 'local' and 'insertions' not in self._cache.keys():
-            raise NotImplementedError('Converting probe IDs required remote connection')
+            raise NotImplementedError('Converting to probe ID requires remote connection')
         eid = self.to_eid(eid)  # Ensure we have a UUID str
         if not eid:
             return (None,) * (3 if details else 2)
-        recs = self.alyx.rest('insertions', 'list', session=eid, **kwargs)
-        pids = [UUID(x['id']) for x in recs]
-        labels = [x['name'] for x in recs]
+        if query_type == 'local':
+            try:  # If local, use the cache table
+                rec = self._cache['insertions'].loc[(eid,), :]
+                pids, names = map(list, zip(*rec.sort_values('name')['name'].items()))
+                if details:
+                    rec['session'] = str(eid)
+                    session_info = self._cache['sessions'].loc[eid].to_dict()
+                    session_info['date'] = session_info['date'].isoformat()
+                    session_info['projects'] = session_info['projects'].split(',')
+                    rec['session_info'] = session_info
+                    # Convert to list of dicts after casting UUIDs to strings
+                    recs = cast_index_object(rec, str).reset_index().to_dict('records')
+                    return pids, names, recs
+                return pids, names
+            except KeyError:
+                return (None,) * (3 if details else 2)
+
+        if recs := self.alyx.rest('insertions', 'list', session=eid, **kwargs):
+            self._update_insertions_table(recs)
+        pids = [UUID(x['id']) for x in recs] or None
+        labels = [x['name'] for x in recs] or None
         if details:
-            return pids, labels, recs
+            return pids, labels, recs or None
         else:
             return pids, labels
 
@@ -2325,12 +2372,6 @@ class OneAlyx(One):
             # check that the input matches one of the defined filters
             if field == 'date_range':
                 params[field] = [x.date().isoformat() for x in util.validate_date_range(value)]
-            elif field == 'dataset':
-                if not isinstance(value, str):
-                    raise TypeError(
-                        '"dataset" parameter must be a string. For lists use "datasets"')
-                query = f'data_dataset_session_related__name__icontains,{value}'
-                params['django'] += (',' if params['django'] else '') + query
             elif field == 'laboratory':
                 params['lab'] = value
             else:
@@ -2757,10 +2798,11 @@ class OneAlyx(One):
             return [unwrapped(self, e, query_type='remote') for e in eid]
 
         # if it wasn't successful, query Alyx
-        ses = self.alyx.rest('sessions', 'list', django=f'pk,{str(eid)}')
+        ses = self.alyx.rest('sessions', 'list', id=eid)
         if len(ses) == 0:
             return None
         else:
+            self._update_sessions_table(ses)
             return ALFPath(self.cache_dir).joinpath(
                 ses[0]['lab'], 'Subjects', ses[0]['subject'], ses[0]['start_time'][:10],
                 str(ses[0]['number']).zfill(3))
@@ -2788,7 +2830,7 @@ class OneAlyx(One):
                 eid_list.append(self.path2eid(p))
             return eid_list
         # else ensure the path ends with mouse, date, number
-        path_obj = ALFPath(path_obj)
+        path_obj = ensure_alf_path(path_obj)
 
         # try the cached info to possibly avoid hitting database
         mode = query_type or self.mode
@@ -2969,26 +3011,40 @@ class OneAlyx(One):
             [Errno 404] Remote session not found on Alyx.
 
         """
+        def process(d, root=self.cache_dir):
+            """Returns dict in similar format to One.search output."""
+            det_fields = ['subject', 'start_time', 'number', 'lab', 'projects',
+                          'url', 'task_protocol', 'local_path']
+            out = {k: v for k, v in d.items() if k in det_fields}
+            out['projects'] = ','.join(out['projects'])
+            out['date'] = datetime.fromisoformat(out['start_time']).date()
+            out['local_path'] = session_record2path(out, root)
+            return out
+
         if (query_type or self.mode) == 'local':
             return super().get_details(eid, full=full)
         # If eid is a list of eIDs recurse through list and return the results
-        if isinstance(eid, (list, util.LazyId)):
-            details_list = []
-            for p in eid:
-                details_list.append(self.get_details(p, full=full))
-            return details_list
-        # load all details
-        dets = self.alyx.rest('sessions', 'read', eid)
+        eids = ensure_list(eid)
+        details = dict.fromkeys(map(str, eids), None)  # create map to skip duplicates
         if full:
-            return dets
-        # If it's not full return the normal output like from a one.search
-        det_fields = ['subject', 'start_time', 'number', 'lab', 'projects',
-                      'url', 'task_protocol', 'local_path']
-        out = {k: v for k, v in dets.items() if k in det_fields}
-        out['projects'] = ','.join(out['projects'])
-        out.update({'local_path': self.eid2path(eid),
-                    'date': datetime.fromisoformat(out['start_time']).date()})
-        return out
+            for e in details:
+                # check for duplicates
+                details[e] = self.alyx.rest('sessions', 'read', id=e)
+                session, datasets = ses2records(details[e])
+                merge_tables(
+                    self._cache, sessions=session, datasets=datasets.copy(),
+                    origin=self.alyx.base_url)
+            details = [details[str(e)].copy() for e in eids]
+        else:
+            # batch to ensure the list is not too long for the GET request
+            iterator = iter(details.keys())
+            while batch := tuple(islice(iterator, 50)):
+                ret = self.alyx.rest('sessions', 'list', django=f'pk__in,{batch}')
+                details.update({d['id']: d for d in ret})
+            self._update_sessions_table(details.values())
+            details = [process(details[str(e)]) for e in eids]
+        # Return either a single dict or a list of dicts depending on the input type
+        return (details if isinstance(eid, (list, util.LazyId)) else details[0])
 
 
 def _setup(**kwargs):

one/remote/globus.py

@@ -642,7 +642,7 @@ class Globus(DownloadClient):
         recursive : bool
             If true, transfer the contents of nested directories (NB: all data_paths must be
             directories).
-        **kwargs
+        kwargs
             See Globus.transfer_data.
 
         Returns
@@ -715,7 +715,7 @@ class Globus(DownloadClient):
         ----------
         client_name : str
             Parameter profile name to set up e.g. 'default', 'admin'.
-        **kwargs
+        kwargs
             Optional Globus constructor arguments.
 
         Returns
@@ -937,7 +937,7 @@ class Globus(DownloadClient):
         recursive : bool
             If true, transfer the contents of nested directories (NB: all data_paths must be
             directories).
-        **kwargs
+        kwargs
             See globus_sdk.TransferData.
 
         Returns
@@ -996,7 +996,7 @@ class Globus(DownloadClient):
         recursive : bool
             If true, delete the contents of nested directories (NB: all data_paths must be
             directories).
-        **kwargs
+        kwargs
             See globus_sdk.DeleteData.
 
         Returns
@@ -1108,7 +1108,7 @@ class Globus(DownloadClient):
             is a UUID, the path must be absolute.
         timeout : int
             Maximum time in seconds to wait for the task to complete.
-        **kwargs
+        kwargs
             Optional arguments for globus_sdk.TransferData.
 
         Returns