ONE-api 3.1.0__tar.gz → 3.2.0__tar.gz

{one_api-3.1.0 → one_api-3.2.0/ONE_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ONE-api
-Version: 3.1.0
+Version: 3.2.0
 Summary: Open Neurophysiology Environment
 Author: IBL Staff
 License: MIT

{one_api-3.1.0/ONE_api.egg-info → one_api-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ONE-api
-Version: 3.1.0
+Version: 3.2.0
 Summary: Open Neurophysiology Environment
 Author: IBL Staff
 License: MIT

{one_api-3.1.0 → one_api-3.2.0}/one/__init__.py

@@ -1,2 +1,2 @@
 """The Open Neurophysiology Environment (ONE) API."""
-__version__ = '3.1.0'
+__version__ = '3.2.0'

{one_api-3.1.0 → one_api-3.2.0}/one/alf/cache.py

@@ -22,6 +22,7 @@ from functools import partial
 from pathlib import Path
 import warnings
 import logging
+from copy import deepcopy
 
 import pandas as pd
 import numpy as np
@@ -275,6 +276,9 @@ def default_cache(origin=''):
 
     """
     table_meta = _metadata(origin)
+    # The origin is now a set, however we leave _metadata as Alyx relies on this and sets
+    # can't be serialized to JSON
+    table_meta['origin'] = set(filter(None, [origin]))
     return Bunch({
             'datasets': EMPTY_DATASETS_FRAME.copy(),
             'sessions': EMPTY_SESSIONS_FRAME.copy(),
@@ -283,7 +287,7 @@ def default_cache(origin=''):
                 'loaded_time': None,
                 'modified_time': None,
                 'saved_time': None,
-                'raw': {k: table_meta.copy() for k in ('datasets', 'sessions')}}
+                'raw': {k: deepcopy(table_meta) for k in ('datasets', 'sessions')}}
         })
 
 
@@ -435,6 +439,10 @@ def load_tables(tables_dir, glob_pattern='*.pqt'):
         if not cache.index.is_monotonic_increasing:
             cache.sort_index(inplace=True)
 
+        # Ensure origin is a set (supports multiple origins)
+        meta['raw'][table]['origin'] = set(
+            filter(None, ensure_list(meta['raw'][table].get('origin', 'unknown'))))
+
         caches[table] = cache
 
     created = [datetime.datetime.fromisoformat(x['date_created'])
@@ -532,10 +540,10 @@ def merge_tables(cache, strict=False, origin=None, **kwargs):
         # Update the table metadata with the origin
         if origin is not None:
             table_meta = cache['_meta']['raw'].get(table, {})
-            if not table_meta.get('origin'):
-                table_meta['origin'] = origin
+            if 'origin' not in table_meta:
+                table_meta['origin'] = set(origin)
             else:
-                table_meta['origin'] = set((*ensure_list(table_meta['origin']), origin))
+                table_meta['origin'].add(origin)
             cache['_meta']['raw'][table] = table_meta
     cache['_meta']['modified_time'] = updated
     return updated

{one_api-3.1.0 → one_api-3.2.0}/one/alf/path.py

@@ -859,6 +859,36 @@ class PureALFPath(pathlib.PurePath):  # py3.12 supports direct subclassing
         """tuple of str: the full ALF path parts, with empty strings for missing parts."""
         return tuple(p or '' for p in self.parse_alf_path(as_dict=False))
 
+    @property
+    def lab(self):
+        """str: The lab part of the ALF path, or an empty str if not present."""
+        return self.session_parts[0]
+
+    @property
+    def subject(self):
+        """str: The subject part of the ALF path, or an empty str if not present."""
+        return self.session_parts[1]
+
+    @property
+    def date(self):
+        """str: The date part of the ALF path, or an empty str if not present."""
+        return self.session_parts[2]
+
+    @property
+    def sequence(self):
+        """str: The number part of the ALF path, or an empty str if not present."""
+        return self.session_parts[3]
+
+    @property
+    def collection(self):
+        """str: The collection part of the ALF path, or an empty str if not present."""
+        return self.alf_parts[4]
+
+    @property
+    def revision(self):
+        """str: The revision part of the ALF path, or an empty str if not present."""
+        return self.alf_parts[5]
+
     @property
     def namespace(self):
         """str: The namespace part of the ALF name, or and empty str if not present."""
@@ -884,6 +914,134 @@ class PureALFPath(pathlib.PurePath):  # py3.12 supports direct subclassing
         """str: The extra part of the ALF name, or and empty str if not present."""
         return self.dataset_name_parts[4]
 
+    def with_lab(self, lab, strict=False):
+        """Return a new path with the ALF lab changed.
+
+        Parameters
+        ----------
+        lab : str
+            An ALF lab name part to use.
+        strict : bool, optional
+            If True, the lab part must be present in the path, otherwise the lab/Subjects/ part
+            is added if not present.
+
+        Returns
+        -------
+        PureALFPath
+            The same file path but with the lab part replaced with the input.
+
+        Raises
+        ------
+        ValueError
+            The lab name is invalid.
+        ALFInvalid
+            The path is not a valid ALF session path, or the lab part is not present in the path
+            when strict is True.
+
+        """
+        if not (lab and spec.regex('^{lab}$').match(lab)):
+            raise ValueError(f'Invalid lab name: {lab}')
+        if not self.subject or (strict and not self.lab):  # FIXME check logic
+            raise ALFInvalid(str(self))
+
+        pattern = spec.regex(SESSION_SPEC)
+        repl = fr'{lab}/Subjects/\g<subject>/\g<date>/\g<number>'
+        return self.__class__(pattern.sub(repl, self.as_posix(), count=1))
+
+    def with_subject(self, subject):
+        """Return a new path with the ALF subject changed.
+
+        Parameters
+        ----------
+        subject : str
+            An ALF subject name part to use.
+
+        Returns
+        -------
+        PureALFPath
+            The same file path but with the subject part replaced with the input.
+
+        Raises
+        ------
+        ValueError
+            The subject name is invalid.
+        ALFInvalid
+            The path is not a valid ALF session path.
+
+        """
+        if not (subject and spec.regex('^{subject}$').match(subject)):
+            raise ValueError(f'Invalid subject name: {subject}')
+        if not self.subject:
+            raise ALFInvalid(str(self))
+
+        pattern = spec.regex('{subject}/{date}/{number}')
+        repl = fr'{subject}/\g<date>/\g<number>'
+        return self.__class__(pattern.sub(repl, self.as_posix()), count=1)
+
+    def with_date(self, date):
+        """Return a new path with the ALF date changed.
+
+        Parameters
+        ----------
+        date : str, datetime.datetime, datetime.date
+            An ALF date part to use, in YYYY-MM-DD format.
+
+        Returns
+        -------
+        PureALFPath
+            The same file path but with the date part replaced with the input.
+
+        Raises
+        ------
+        ValueError
+            The date is not in YYYY-MM-DD format.
+        ALFInvalid
+            The path is not a valid ALF session path.
+
+        """
+        if date and not isinstance(date, str):
+            date = str(date)[:10]
+        if not (date and spec.regex('^{date}$').match(date)):
+            raise ValueError(f'Invalid date: {date}')
+        if not self.date:
+            raise ALFInvalid(str(self))
+
+        pattern = spec.regex('{subject}/{date}/{number}')
+        repl = fr'\g<subject>/{date}/\g<number>'
+        return self.__class__(pattern.sub(repl, self.as_posix()), count=1)
+
+    def with_sequence(self, number):
+        """Return a new path with the ALF number changed.
+
+        Parameters
+        ----------
+        number : str, int
+            An ALF number part to use, as a string or integer.
+
+        Returns
+        -------
+        PureALFPath
+            The same file path but with the number part replaced with the input.
+
+        Raises
+        ------
+        ValueError
+            The number is not a valid ALF number.
+        ALFInvalid
+            The path is not a valid ALF session path.
+
+        """
+        if isinstance(number, str):
+            number = int(number.strip())
+        if number is None or not spec.regex('^{number}$').match(str(number)):
+            raise ValueError(f'Invalid number: {number}')
+        if not self.sequence:
+            raise ALFInvalid(str(self))
+
+        pattern = spec.regex('{subject}/{date}/{number}')
+        repl = fr'\g<subject>/\g<date>/{number:03d}'
+        return self.__class__(pattern.sub(repl, self.as_posix()), count=1)
+
     def with_object(self, obj):
         """Return a new path with the ALF object changed.
 
@@ -1084,6 +1242,58 @@ class PureALFPath(pathlib.PurePath):  # py3.12 supports direct subclassing
         """
         return padded_sequence(path)
 
+    def with_collection(self, collection):
+        """Return a new path with the ALF collection part added/changed.
+
+        NB: The ALFPath must include the session parts (subject/date/number) for this to work.
+
+        Parameters
+        ----------
+        collection : str
+            An ALF collection part to use.
+
+        Returns
+        -------
+        PureALFPath
+            The same file path but with the collection part added or replaced with the input.
+
+        Raises
+        ------
+        ValueError
+            The collection name is invalid.
+            The path does not include the session parts (subject/date/number).
+        ALFInvalid
+            The path is not a valid ALF session path.
+
+        """
+        collection = pathlib.PurePath(collection or '').as_posix().strip('/')
+        if not (collection and spec.regex('^{collection}$').match(collection)):
+            raise ValueError(f'Invalid collection name: {collection}')
+        # Check path contains session parts
+        if not self.session_path():
+            raise ValueError(
+                'Cannot add collection to a path without session parts, e.g. subject/date/number'
+            )
+        # If the path is a session path, simply append the collection to it
+        if self.is_session_path():
+            return self.joinpath(collection)
+        # Otherwise substitute the collection with regex
+        string = self.as_posix()
+        if not self.is_dataset():
+            cpat = spec.regex(f'^{COLLECTION_SPEC}$')
+            if cpat.match(self.relative_to_session().as_posix() + '/'):
+                string += '/'  # ensure trailing slash for matching folder paths
+            else:
+                raise ALFInvalid(str(self))
+        # Replace the collection part in the path
+        # NB: We don't use SESSION_SPEC here to avoid handling optional lab part
+        pattern = spec.regex('{subject}/{date}/{number}/' + COLLECTION_SPEC)
+        match = pattern.search(string)
+        repl = fr'\g<subject>/\g<date>/\g<number>/{collection}/'
+        if match.groupdict()['revision']:
+            repl += r'#\g<revision>#/'
+        return self.__class__(pattern.sub(repl, string), count=1)
+
     def with_revision(self, revision):
         """Return a new path with the ALF revision part added/changed.
 

{one_api-3.1.0 → one_api-3.2.0}/one/api.py

@@ -6,7 +6,6 @@ import logging
 from weakref import WeakMethod
 from datetime import datetime, timedelta
 from functools import lru_cache, partial
-from itertools import chain
 from inspect import unwrap
 from pathlib import Path, PurePosixPath
 from typing import Any, Union, Optional, List
@@ -40,7 +39,7 @@ from one import util
 
 _logger = logging.getLogger(__name__)
 __all__ = ['ONE', 'One', 'OneAlyx']
-SAVE_ON_DELETE = (os.environ.get('ONE_SAVE_ON_DELETE') or '1').casefold() in ('true', '1')
+SAVE_ON_DELETE = (os.environ.get('ONE_SAVE_ON_DELETE') or '0').casefold() in ('true', '1')
 """bool: Whether to save modified cache tables on delete."""
 
 _logger.debug('ONE_SAVE_ON_DELETE: %s', SAVE_ON_DELETE)
@@ -135,20 +134,25 @@ class One(ConversionMixin):
         tables = tables or filter(lambda x: x[0] != '_', self._cache)
         return remove_table_files(self._tables_dir, tables)
 
-    def load_cache(self, tables_dir=None, **kwargs):
+    def load_cache(self, tables_dir=None, clobber=True, **kwargs):
         """Load parquet cache files from a local directory.
 
         Parameters
         ----------
         tables_dir : str, pathlib.Path
             An optional directory location of the parquet files, defaults to One._tables_dir.
+        clobber : bool
+            If true, the cache is loaded without merging with existing table files.
 
         Returns
         -------
         datetime.datetime
             A timestamp of when the cache was loaded.
         """
-        self._reset_cache()
+        if clobber:
+            self._reset_cache()
+        else:
+            raise NotImplementedError('clobber=False not implemented yet')
         self._tables_dir = Path(tables_dir or self._tables_dir or self.cache_dir)
         self._cache = load_tables(self._tables_dir)
 
@@ -161,7 +165,7 @@ class One(ConversionMixin):
         # prompt the user to delete them to improve load times
         raw_meta = self._cache['_meta'].get('raw', {}).values() or [{}]
         tagged = any(filter(None, flatten(x.get('database_tags') for x in raw_meta)))
-        origin = set(x['origin'] for x in raw_meta if 'origin' in x)
+        origin = set(filter(None, flatten(ensure_list(x.get('origin', [])) for x in raw_meta)))
         older = (self._cache['_meta']['created_time'] or datetime.now()) < datetime(2025, 2, 13)
         remote = not self.offline and self.mode == 'remote'
         if remote and origin == {'alyx'} and older and not self._web_client.silent and not tagged:
@@ -217,10 +221,12 @@ class One(ConversionMixin):
                     created = created.isoformat(sep=' ', timespec='minutes')
                     meta['raw'][table]['date_created'] = created
 
-        with FileLock(save_dir, log=_logger, timeout=TIMEOUT, timeout_action='delete'):
+        with FileLock(save_dir / '.ONE', log=_logger, timeout=TIMEOUT, timeout_action='delete'):
             _logger.info('Saving cache tables...')
             for table in filter(lambda x: not x[0] == '_', caches.keys()):
                 metadata = meta['raw'].get(table, {})
+                if isinstance(metadata.get('origin'), set):
+                    metadata['origin'] = list(metadata['origin'])
                 metadata['date_modified'] = modified.isoformat(sep=' ', timespec='minutes')
                 filename = save_dir.joinpath(f'{table}.pqt')
                 # Cast indices to str before saving
@@ -1696,7 +1702,8 @@ class OneAlyx(One):
         tag = tag or current_tags[0]  # For refreshes take the current tag as default
         different_tag = any(x != tag for x in current_tags)
         if not (clobber or different_tag):
-            super(OneAlyx, self).load_cache(tables_dir)  # Load any present cache
+            # Load any present cache
+            super(OneAlyx, self).load_cache(tables_dir, clobber=True)
             cache_meta = self._cache.get('_meta', {})
             raw_meta = cache_meta.get('raw', {}).values() or [{}]
 
@@ -1719,7 +1726,7 @@ class OneAlyx(One):
             # contain the same tag or origin, we need to download the remote one.
             origin = cache_info.get('origin', 'unknown')
             local_origin = (x.get('origin', []) for x in raw_meta)
-            local_origin = set(chain.from_iterable(map(ensure_list, local_origin)))
+            local_origin = set(flatten(map(ensure_list, local_origin)))
             different_origin = origin not in local_origin
             if fresh and not (different_tag or different_origin):
                 _logger.info('No newer cache available')

{one_api-3.1.0 → one_api-3.2.0}/one/webclient.py

@@ -77,7 +77,7 @@ class _JSONEncoder(json.JSONEncoder):
 
 
 def _cache_response(method):
-    """Decorator for the generic request method for caching REST reponses.
+    """Decorator for the generic request method for caching REST responses.
 
     Caches the result of the query and on subsequent calls, returns cache instead of hitting the
     database.