ONE-api 3.0b4__py3-none-any.whl → 3.0b5__py3-none-any.whl

{ONE_api-3.0b4.dist-info → ONE_api-3.0b5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ONE-api
-Version: 3.0b4
+Version: 3.0b5
 Summary: Open Neurophysiology Environment
 Author: IBL Staff
 License: MIT

{ONE_api-3.0b4.dist-info → ONE_api-3.0b5.dist-info}/RECORD

@@ -1,10 +1,10 @@
-one/__init__.py,sha256=taw94M4t6gNVqH0F1Cu53S3s3ZaVsKg6cDxs3XDd7ek,76
-one/api.py,sha256=R3ozjKJglmtIgw6Wlwtv8gsZY8vbN1SPBmmx4yl2Hmw,129891
+one/__init__.py,sha256=nInf_OtC7ZRulDTsGaWoCYJCQdioyk1iYSfMN-_fbMU,76
+one/api.py,sha256=IWAjCBhw4DCLmfxcwVkOl0bgRlu2RXIV6Nyfey8HNas,131576
 one/converters.py,sha256=icKlwPyxf3tJtyOFBj_SG06QDLIZLdTGalCSk1-cAvk,30733
 one/params.py,sha256=zwR0Yq09ztROfH3fJsCUc-IDs_PBixT3WU3dm1vX728,15050
 one/registration.py,sha256=cWvQFAzCF04wMZJjdOzBPJkYOJ3BO2KEgqtVG7qOlmA,36177
 one/util.py,sha256=NUG_dTz3_4GXYG49qSql6mFCBkaVaq7_XdecRPRszJQ,20173
-one/webclient.py,sha256=SDIVvQsSUK_Ng4sz69HrgQgPnO4iSATTJPV2FVMZTAM,50064
+one/webclient.py,sha256=CfHKn6eZu-woed_VEeY8DV6WTzb49u41-lBTRe-EYnM,50132
 one/alf/__init__.py,sha256=DaFIi7PKlurp5HnyNOpJuIlY3pyhiottFpJfxR59VsY,70
 one/alf/cache.py,sha256=NDN-Ii8E19C6IH1C7EQEIIgPFah3AtM9jhjLeHuBIjM,22735
 one/alf/exceptions.py,sha256=6Gw8_ZObLnuYUpY4i0UyU4IA0kBZMBnJBGQyzIcDlUM,3427
@@ -30,8 +30,8 @@ one/tests/fixtures/rest_responses/6dc96f7e9bcc6ac2e7581489b9580a6cd3f28293,sha25
 one/tests/fixtures/rest_responses/db1731fb8df0208944ae85f76718430813a8bf50,sha256=Dki7cTDg1bhbtPkuiv9NPRsuhM7TuicxoLcGtRD4D8g,209
 one/tests/fixtures/rest_responses/dcce48259bb929661f60a02a48563f70aa6185b3,sha256=skaKl6sPgTyrufCIGNVNJfccXM-jSjYvAdyqNS3HXuA,416
 one/tests/fixtures/rest_responses/f530d6022f61cdc9e38cc66beb3cb71f3003c9a1,sha256=EOqhNIVcmZ7j7aF09g5StFY434f2xbxwZLHwfeM4tug,22530
-ONE_api-3.0b4.dist-info/LICENSE,sha256=W6iRQJcr-tslNfY4gL98IWvPtpe0E3tcWCFOD3IFUqg,1087
-ONE_api-3.0b4.dist-info/METADATA,sha256=dHcPJrlzYqqXB_xcGhju2T11-BaTDlC-BoBQW7cG5o4,4148
-ONE_api-3.0b4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ONE_api-3.0b4.dist-info/top_level.txt,sha256=LIsI2lzmA5jh8Zrw5dzMdE3ydLgmq-WF6rpoxSVDSAY,4
-ONE_api-3.0b4.dist-info/RECORD,,
+ONE_api-3.0b5.dist-info/LICENSE,sha256=W6iRQJcr-tslNfY4gL98IWvPtpe0E3tcWCFOD3IFUqg,1087
+ONE_api-3.0b5.dist-info/METADATA,sha256=V71vo5_2W9rsjtyhah07thKb0zXKS0VkM53Zw3eYRu8,4148
+ONE_api-3.0b5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+ONE_api-3.0b5.dist-info/top_level.txt,sha256=LIsI2lzmA5jh8Zrw5dzMdE3ydLgmq-WF6rpoxSVDSAY,4
+ONE_api-3.0b5.dist-info/RECORD,,

one/__init__.py

@@ -1,2 +1,2 @@
 """The Open Neurophysiology Environment (ONE) API."""
-__version__ = '3.0b4'
+__version__ = '3.0b5'

one/api.py

@@ -39,8 +39,10 @@ from one import util
 
 _logger = logging.getLogger(__name__)
 __all__ = ['ONE', 'One', 'OneAlyx']
-N_THREADS = os.environ.get('ONE_HTTP_DL_THREADS', 4)
-"""int: The number of download threads."""
+SAVE_ON_DELETE = (os.environ.get('ONE_SAVE_ON_DELETE') or '1').casefold() in ('true', '1')
+"""bool: Whether to save modified cache tables on delete."""
+
+_logger.debug('ONE_SAVE_ON_DELETE: %s', SAVE_ON_DELETE)
 
 
 class One(ConversionMixin):
@@ -97,7 +99,8 @@ class One(ConversionMixin):
 
     def __del__(self):
         """Save cache tables to disk before deleting the object."""
-        self.save_cache()
+        if SAVE_ON_DELETE:
+            self.save_cache()
 
     @property
     def offline(self):
@@ -162,6 +165,25 @@ class One(ConversionMixin):
             if self.offline:  # In online mode, the cache tables should be downloaded later
                 warnings.warn(f'No cache tables found in {self._tables_dir}')
 
+        # If in remote mode and loading old tables generated on Alyx,
+        # prompt the user to delete them to improve load times
+        raw_meta = self._cache['_meta'].get('raw', {}).values() or [{}]
+        tagged = any(filter(None, flatten(x.get('database_tags') for x in raw_meta)))
+        origin = set(x['origin'] for x in raw_meta if 'origin' in x)
+        older = (self._cache['_meta']['created_time'] or datetime.now()) < datetime(2025, 2, 13)
+        remote = not self.offline and self.mode == 'remote'
+        if remote and origin == {'alyx'} and older and not self._web_client.silent and not tagged:
+            message = ('Old Alyx cache tables detected on disk. '
+                      'It\'s recomended to remove these tables as they '
+                      'negatively affect performance.\nDelete these tables? [Y/n]: ')
+            if (input(message).casefold().strip() or 'y')[0] == 'y':
+                self._remove_table_files()
+                self._reset_cache()
+        elif len(self._cache.datasets) > 1e6:
+            warnings.warn(
+                'Large cache tables affect performance. '
+                'Consider removing them by calling the `_remove_table_files` method.')
+
         return self._cache['_meta']['loaded_time']
 
     def save_cache(self, save_dir=None, clobber=False):
@@ -1643,6 +1665,11 @@ class OneAlyx(One):
         tag : str
             An optional Alyx dataset tag for loading cache tables containing a subset of datasets.
 
+        Returns
+        -------
+        datetime.datetime
+            A timestamp of when the cache was loaded.
+
         Examples
         --------
         To load the cache tables for a given release tag
@@ -1666,6 +1693,8 @@ class OneAlyx(One):
         different_tag = any(x != tag for x in current_tags)
         if not (clobber or different_tag):
             super(OneAlyx, self).load_cache(tables_dir)  # Load any present cache
+            cache_meta = self._cache.get('_meta', {})
+            raw_meta = cache_meta.get('raw', {}).values() or [{}]
 
         try:
             # Determine whether a newer cache is available
@@ -1676,7 +1705,7 @@ class OneAlyx(One):
             min_version = packaging.version.parse(cache_info.get('min_api_version', '0.0.0'))
             if packaging.version.parse(one.__version__) < min_version:
                 warnings.warn(f'Newer cache tables require ONE version {min_version} or greater')
-                return
+                return cache_meta['loaded_time']
 
             # Check whether remote cache more recent
             remote_created = datetime.fromisoformat(cache_info['date_created'])
@@ -1684,7 +1713,7 @@ class OneAlyx(One):
             fresh = local_created and (remote_created - local_created) < timedelta(minutes=1)
             if fresh and not different_tag:
                 _logger.info('No newer cache available')
-                return
+                return cache_meta['loaded_time']
 
             # Set the cache table directory location
             if tables_dir:  # If tables directory specified, use that
@@ -1708,7 +1737,7 @@ class OneAlyx(One):
             _logger.info('Downloading remote caches...')
             files = self.alyx.download_cache_tables(cache_info.get('location'), self._tables_dir)
             assert any(files)
-            super(OneAlyx, self).load_cache(self._tables_dir)  # Reload cache after download
+            return super(OneAlyx, self).load_cache(self._tables_dir)  # Reload cache after download
         except (requests.exceptions.HTTPError, wc.HTTPError, requests.exceptions.SSLError) as ex:
             _logger.debug(ex)
             _logger.error(f'{type(ex).__name__}: Failed to load the remote cache file')
@@ -1725,6 +1754,7 @@ class OneAlyx(One):
                 'Please provide valid tables_dir / cache_dir kwargs '
                 'or run ONE.setup to update the default directory.'
             )
+        return cache_meta['loaded_time']
 
     @property
     def alyx(self):

one/webclient.py

@@ -34,6 +34,7 @@ from uuid import UUID
 import json
 import logging
 import math
+import os
 import re
 import functools
 import urllib.request
@@ -61,6 +62,8 @@ from iblutil.io.params import set_hidden
 from iblutil.util import ensure_list
 import concurrent.futures
 _logger = logging.getLogger(__name__)
+N_THREADS = int(os.environ.get('ONE_HTTP_DL_THREADS', 4))
+"""int: The number of download threads."""
 
 
 class _JSONEncoder(json.JSONEncoder):
@@ -355,7 +358,6 @@ def http_download_file_list(links_to_file_list, **kwargs):
 
     """
     links_to_file_list = list(links_to_file_list)  # In case generator was passed
-    n_threads = 4  # Max number of threads
     outputs = []
     target_dir = kwargs.pop('target_dir', None)
     # Ensure target dir the length of url list
@@ -364,7 +366,7 @@ def http_download_file_list(links_to_file_list, **kwargs):
     assert len(target_dir) == len(links_to_file_list)
     # using with statement to ensure threads are cleaned up promptly
     zipped = zip(links_to_file_list, target_dir)
-    with concurrent.futures.ThreadPoolExecutor(max_workers=n_threads) as executor:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=N_THREADS) as executor:
         # Multithreading load operations
         futures = [executor.submit(
             http_download_file, link, target_dir=target, **kwargs) for link, target in zipped]