ONE-api 3.0b4__tar.gz → 3.0b5__tar.gz

{one_api-3.0b4 → one_api-3.0b5/ONE_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ONE-api
-Version: 3.0b4
+Version: 3.0b5
 Summary: Open Neurophysiology Environment
 Author: IBL Staff
 License: MIT

{one_api-3.0b4/ONE_api.egg-info → one_api-3.0b5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ONE-api
-Version: 3.0b4
+Version: 3.0b5
 Summary: Open Neurophysiology Environment
 Author: IBL Staff
 License: MIT

{one_api-3.0b4 → one_api-3.0b5}/one/__init__.py

@@ -1,2 +1,2 @@
 """The Open Neurophysiology Environment (ONE) API."""
-__version__ = '3.0b4'
+__version__ = '3.0b5'

{one_api-3.0b4 → one_api-3.0b5}/one/api.py

@@ -39,8 +39,10 @@ from one import util
 
 _logger = logging.getLogger(__name__)
 __all__ = ['ONE', 'One', 'OneAlyx']
-N_THREADS = os.environ.get('ONE_HTTP_DL_THREADS', 4)
-"""int: The number of download threads."""
+SAVE_ON_DELETE = (os.environ.get('ONE_SAVE_ON_DELETE') or '1').casefold() in ('true', '1')
+"""bool: Whether to save modified cache tables on delete."""
+
+_logger.debug('ONE_SAVE_ON_DELETE: %s', SAVE_ON_DELETE)
 
 
 class One(ConversionMixin):
@@ -97,7 +99,8 @@ class One(ConversionMixin):
 
     def __del__(self):
         """Save cache tables to disk before deleting the object."""
-        self.save_cache()
+        if SAVE_ON_DELETE:
+            self.save_cache()
 
     @property
     def offline(self):
@@ -162,6 +165,25 @@ class One(ConversionMixin):
             if self.offline:  # In online mode, the cache tables should be downloaded later
                 warnings.warn(f'No cache tables found in {self._tables_dir}')
 
+        # If in remote mode and loading old tables generated on Alyx,
+        # prompt the user to delete them to improve load times
+        raw_meta = self._cache['_meta'].get('raw', {}).values() or [{}]
+        tagged = any(filter(None, flatten(x.get('database_tags') for x in raw_meta)))
+        origin = set(x['origin'] for x in raw_meta if 'origin' in x)
+        older = (self._cache['_meta']['created_time'] or datetime.now()) < datetime(2025, 2, 13)
+        remote = not self.offline and self.mode == 'remote'
+        if remote and origin == {'alyx'} and older and not self._web_client.silent and not tagged:
+            message = ('Old Alyx cache tables detected on disk. '
+                      'It\'s recomended to remove these tables as they '
+                      'negatively affect performance.\nDelete these tables? [Y/n]: ')
+            if (input(message).casefold().strip() or 'y')[0] == 'y':
+                self._remove_table_files()
+                self._reset_cache()
+        elif len(self._cache.datasets) > 1e6:
+            warnings.warn(
+                'Large cache tables affect performance. '
+                'Consider removing them by calling the `_remove_table_files` method.')
+
         return self._cache['_meta']['loaded_time']
 
     def save_cache(self, save_dir=None, clobber=False):
@@ -1643,6 +1665,11 @@ class OneAlyx(One):
         tag : str
             An optional Alyx dataset tag for loading cache tables containing a subset of datasets.
 
+        Returns
+        -------
+        datetime.datetime
+            A timestamp of when the cache was loaded.
+
         Examples
         --------
         To load the cache tables for a given release tag
@@ -1666,6 +1693,8 @@ class OneAlyx(One):
         different_tag = any(x != tag for x in current_tags)
         if not (clobber or different_tag):
             super(OneAlyx, self).load_cache(tables_dir)  # Load any present cache
+            cache_meta = self._cache.get('_meta', {})
+            raw_meta = cache_meta.get('raw', {}).values() or [{}]
 
         try:
             # Determine whether a newer cache is available
@@ -1676,7 +1705,7 @@ class OneAlyx(One):
             min_version = packaging.version.parse(cache_info.get('min_api_version', '0.0.0'))
             if packaging.version.parse(one.__version__) < min_version:
                 warnings.warn(f'Newer cache tables require ONE version {min_version} or greater')
-                return
+                return cache_meta['loaded_time']
 
             # Check whether remote cache more recent
             remote_created = datetime.fromisoformat(cache_info['date_created'])
@@ -1684,7 +1713,7 @@ class OneAlyx(One):
             fresh = local_created and (remote_created - local_created) < timedelta(minutes=1)
             if fresh and not different_tag:
                 _logger.info('No newer cache available')
-                return
+                return cache_meta['loaded_time']
 
             # Set the cache table directory location
             if tables_dir:  # If tables directory specified, use that
@@ -1708,7 +1737,7 @@ class OneAlyx(One):
             _logger.info('Downloading remote caches...')
             files = self.alyx.download_cache_tables(cache_info.get('location'), self._tables_dir)
             assert any(files)
-            super(OneAlyx, self).load_cache(self._tables_dir)  # Reload cache after download
+            return super(OneAlyx, self).load_cache(self._tables_dir)  # Reload cache after download
         except (requests.exceptions.HTTPError, wc.HTTPError, requests.exceptions.SSLError) as ex:
             _logger.debug(ex)
             _logger.error(f'{type(ex).__name__}: Failed to load the remote cache file')
@@ -1725,6 +1754,7 @@ class OneAlyx(One):
                 'Please provide valid tables_dir / cache_dir kwargs '
                 'or run ONE.setup to update the default directory.'
             )
+        return cache_meta['loaded_time']
 
     @property
     def alyx(self):

{one_api-3.0b4 → one_api-3.0b5}/one/webclient.py

@@ -34,6 +34,7 @@ from uuid import UUID
 import json
 import logging
 import math
+import os
 import re
 import functools
 import urllib.request
@@ -61,6 +62,8 @@ from iblutil.io.params import set_hidden
 from iblutil.util import ensure_list
 import concurrent.futures
 _logger = logging.getLogger(__name__)
+N_THREADS = int(os.environ.get('ONE_HTTP_DL_THREADS', 4))
+"""int: The number of download threads."""
 
 
 class _JSONEncoder(json.JSONEncoder):
@@ -355,7 +358,6 @@ def http_download_file_list(links_to_file_list, **kwargs):
 
     """
     links_to_file_list = list(links_to_file_list)  # In case generator was passed
-    n_threads = 4  # Max number of threads
     outputs = []
     target_dir = kwargs.pop('target_dir', None)
     # Ensure target dir the length of url list
@@ -364,7 +366,7 @@ def http_download_file_list(links_to_file_list, **kwargs):
     assert len(target_dir) == len(links_to_file_list)
     # using with statement to ensure threads are cleaned up promptly
     zipped = zip(links_to_file_list, target_dir)
-    with concurrent.futures.ThreadPoolExecutor(max_workers=n_threads) as executor:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=N_THREADS) as executor:
         # Multithreading load operations
         futures = [executor.submit(
             http_download_file, link, target_dir=target, **kwargs) for link, target in zipped]