pixeltable 0.2.26__py3-none-any.whl → 0.5.7__py3-none-any.whl

pixeltable/env.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import datetime
 import glob
 import http.server
@@ -7,68 +8,101 @@ import importlib
 import importlib.util
 import inspect
 import logging
+import math
 import os
+import platform
 import shutil
 import subprocess
 import sys
 import threading
-import uuid
+import types
+import typing
 import warnings
-from dataclasses import dataclass
+from contextlib import contextmanager
+from dataclasses import dataclass, field
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar
+from sys import stdout
+from typing import TYPE_CHECKING, Any, Callable, Iterator, TypeVar
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
+import nest_asyncio  # type: ignore[import-untyped]
 import pixeltable_pgserver
 import sqlalchemy as sql
-import toml
+import tzlocal
+from pillow_heif import register_heif_opener  # type: ignore[import-untyped]
+from sqlalchemy import orm
+from tenacity import retry, stop_after_attempt, wait_exponential_jitter
 from tqdm import TqdmWarning
 
-import pixeltable.exceptions as excs
-from pixeltable import metadata
+from pixeltable import exceptions as excs
+from pixeltable.config import Config
+from pixeltable.utils.console_output import ConsoleLogger, ConsoleMessageFilter, ConsoleOutputHandler, map_level
+from pixeltable.utils.dbms import CockroachDbms, Dbms, PostgresqlDbms
 from pixeltable.utils.http_server import make_server
+from pixeltable.utils.object_stores import ObjectPath
+from pixeltable.utils.sql import add_option_to_db_url
 
 if TYPE_CHECKING:
     import spacy
 
 
+_logger = logging.getLogger('pixeltable')
+
+T = TypeVar('T')
+
+
 class Env:
     """
-    Store for runtime globals.
+    Store runtime globals for both local and non-local environments.
+    For a local environment, Pixeltable uses an embedded PostgreSQL server that runs locally in a separate process.
+    For a non-local environment, Pixeltable uses a connection string to the externally managed database.
     """
 
-    _instance: Optional[Env] = None
+    SERIALIZABLE_ISOLATION_LEVEL = 'SERIALIZABLE'
+
+    _instance: Env | None = None
+    __initializing: bool = False
     _log_fmt_str = '%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d: %(message)s'
 
-    _home: Optional[Path]
-    _media_dir: Optional[Path]
-    _file_cache_dir: Optional[Path]  # cached media files with external URL
-    _dataset_cache_dir: Optional[Path]  # cached datasets (eg, pytorch or COCO)
-    _log_dir: Optional[Path]  # log files
-    _tmp_dir: Optional[Path]  # any tmp files
-    _sa_engine: Optional[sql.engine.base.Engine]
-    _pgdata_dir: Optional[Path]
-    _db_name: Optional[str]
-    _db_server: Optional[pixeltable_pgserver.PostgresServer]
-    _db_url: Optional[str]
-    _default_time_zone: Optional[ZoneInfo]
+    _media_dir: Path | None
+    _file_cache_dir: Path | None  # cached object files with external URL
+    _dataset_cache_dir: Path | None  # cached datasets (eg, pytorch or COCO)
+    _log_dir: Path | None  # log files
+    _tmp_dir: Path | None  # any tmp files
+    _sa_engine: sql.engine.base.Engine | None
+    _pgdata_dir: Path | None
+    _db_name: str | None
+    _db_server: pixeltable_pgserver.PostgresServer | None  # set only when running in local environment
+    _db_url: str | None
+    _default_time_zone: ZoneInfo | None
+    _verbosity: int
 
     # info about optional packages that are utilized by some parts of the code
     __optional_packages: dict[str, PackageInfo]
 
-    _spacy_nlp: Optional[spacy.Language]
-    _httpd: Optional[http.server.HTTPServer]
-    _http_address: Optional[str]
+    _spacy_nlp: spacy.Language | None
+    _httpd: http.server.HTTPServer | None
+    _http_address: str | None
     _logger: logging.Logger
     _default_log_level: int
-    _logfilename: Optional[str]
+    _logfilename: str | None
     _log_to_stdout: bool
     _module_log_level: dict[str, int]  # module name -> log level
-    _config_file: Optional[Path]
-    _config: Optional[Config]
+    _file_cache_size_g: float
+    _default_input_media_dest: str | None
+    _default_output_media_dest: str | None
+    _pxt_api_key: str | None
     _stdout_handler: logging.StreamHandler
+    _default_video_encoder: str | None
     _initialized: bool
 
+    _resource_pool_info: dict[str, Any]
+    _current_conn: sql.Connection | None
+    _current_session: orm.Session | None
+    _current_isolation_level: str | None
+    _dbms: Dbms | None
+    _event_loop: asyncio.AbstractEventLoop | None  # event loop for ExecNode
+
     @classmethod
     def get(cls) -> Env:
         if cls._instance is None:
@@ -77,15 +111,26 @@ class Env:
 
     @classmethod
     def _init_env(cls, reinit_db: bool = False) -> None:
+        assert not cls.__initializing, 'Circular env initialization detected.'
+        cls.__initializing = True
+        if cls._instance is not None:
+            cls._instance._clean_up()
+        cls._instance = None
         env = Env()
-        env._set_up(reinit_db=reinit_db)
-        env._upgrade_metadata()
-        cls._instance = env
+        try:
+            env._set_up(reinit_db=reinit_db)
+            env._upgrade_metadata()
+            cls._instance = env
+        finally:
+            # Reset the initializing flag, even if setup fails.
+            # This prevents the environment from being left in a broken state.
+            cls.__initializing = False
+
+    def __init__(self) -> None:
+        assert self._instance is None, 'Env is a singleton; use Env.get() to access the instance'
 
-    def __init__(self):
-        self._home = None
         self._media_dir = None  # computed media files
-        self._file_cache_dir = None  # cached media files with external URL
+        self._file_cache_dir = None  # cached object files with external URL
         self._dataset_cache_dir = None  # cached datasets (eg, pytorch or COCO)
         self._log_dir = None  # log files
         self._tmp_dir = None  # any tmp files
@@ -95,11 +140,11 @@ class Env:
         self._db_server = None
         self._db_url = None
         self._default_time_zone = None
-
         self.__optional_packages = {}
         self._spacy_nlp = None
         self._httpd = None
         self._http_address = None
+        self._default_video_encoder = None
 
         # logging-related state
         self._logger = logging.getLogger('pixeltable')
@@ -111,19 +156,42 @@ class Env:
         self._log_to_stdout = False
         self._module_log_level = {}  # module name -> log level
 
-        # config
-        self._config_file = None
-        self._config = None
-
         # create logging handler to also log to stdout
         self._stdout_handler = logging.StreamHandler(stream=sys.stdout)
         self._stdout_handler.setFormatter(logging.Formatter(self._log_fmt_str))
         self._initialized = False
 
+        self._resource_pool_info = {}
+        self._current_conn = None
+        self._current_session = None
+        self._current_isolation_level = None
+        self._dbms = None
+        self._event_loop = None
+
+    def _init_event_loop(self) -> None:
+        try:
+            # check if we are already in an event loop (eg, Jupyter's); if so, patch it to allow
+            # multiple run_until_complete()
+            running_loop = asyncio.get_running_loop()
+            self._event_loop = running_loop
+            _logger.debug('Patched running loop')
+        except RuntimeError:
+            self._event_loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._event_loop)
+            # we set a deliberately long duration to avoid warnings getting printed to the console in debug mode
+            self._event_loop.slow_callback_duration = 3600
+
+        # always allow nested event loops, we need that to run async udfs synchronously (eg, for SimilarityExpr);
+        # see run_coroutine_synchronously()
+        nest_asyncio.apply()
+        if _logger.isEnabledFor(logging.DEBUG):
+            self._event_loop.set_debug(True)
+
     @property
-    def config(self) -> Config:
-        assert self._config is not None
-        return self._config
+    def event_loop(self) -> asyncio.AbstractEventLoop:
+        if self._event_loop is None:
+            self._init_event_loop()
+        return self._event_loop
 
     @property
     def db_url(self) -> str:
@@ -136,25 +204,105 @@ class Env:
         return self._http_address
 
     @property
-    def default_time_zone(self) -> Optional[ZoneInfo]:
+    def user(self) -> str | None:
+        return Config.get().get_string_value('user')
+
+    @user.setter
+    def user(self, user: str | None) -> None:
+        if user is None:
+            if 'PIXELTABLE_USER' in os.environ:
+                del os.environ['PIXELTABLE_USER']
+        else:
+            os.environ['PIXELTABLE_USER'] = user
+
+    @property
+    def default_time_zone(self) -> ZoneInfo | None:
         return self._default_time_zone
 
     @default_time_zone.setter
-    def default_time_zone(self, tz: Optional[ZoneInfo]) -> None:
+    def default_time_zone(self, tz: ZoneInfo | None) -> None:
         """
         This is not a publicly visible setter; it is only for testing purposes.
         """
-        tz_name = None if tz is None else tz.key
+        if tz is None:
+            tz_name = self._get_tz_name()
+        else:
+            assert isinstance(tz, ZoneInfo)
+            tz_name = tz.key
         self.engine.dispose()
         self._create_engine(time_zone_name=tz_name)
 
+    @property
+    def verbosity(self) -> int:
+        return self._verbosity
+
+    @property
+    def conn(self) -> sql.Connection | None:
+        assert self._current_conn is not None
+        return self._current_conn
+
+    @property
+    def session(self) -> orm.Session | None:
+        assert self._current_session is not None
+        return self._current_session
+
+    @property
+    def dbms(self) -> Dbms | None:
+        assert self._dbms is not None
+        return self._dbms
+
+    @property
+    def is_using_cockroachdb(self) -> bool:
+        assert self._dbms is not None
+        return isinstance(self._dbms, CockroachDbms)
+
+    @property
+    def in_xact(self) -> bool:
+        return self._current_conn is not None
+
+    @property
+    def is_local(self) -> bool:
+        assert self._db_url is not None  # is_local should be called only after db initialization
+        return self._db_server is not None
+
+    @contextmanager
+    def begin_xact(self, *, for_write: bool = False) -> Iterator[sql.Connection]:
+        """
+        Call Catalog.begin_xact() instead, unless there is a specific reason to call this directly.
+
+        for_write: if True, uses serializable isolation; if False, uses repeatable_read
+
+        TODO: repeatable read is not available in Cockroachdb; instead, run queries against a snapshot TVP
+        that avoids tripping over any pending ops
+        """
+        if self._current_conn is None:
+            assert self._current_session is None
+            try:
+                self._current_isolation_level = self.SERIALIZABLE_ISOLATION_LEVEL
+                with (
+                    self.engine.connect().execution_options(isolation_level=self._current_isolation_level) as conn,
+                    orm.Session(conn) as session,
+                    conn.begin(),
+                ):
+                    self._current_conn = conn
+                    self._current_session = session
+                    yield conn
+            finally:
+                self._current_session = None
+                self._current_conn = None
+                self._current_isolation_level = None
+        else:
+            assert self._current_session is not None
+            assert self._current_isolation_level == self.SERIALIZABLE_ISOLATION_LEVEL or not for_write
+            yield self._current_conn
+
     def configure_logging(
         self,
         *,
-        to_stdout: Optional[bool] = None,
-        level: Optional[int] = None,
-        add: Optional[str] = None,
-        remove: Optional[str] = None,
+        to_stdout: bool | None = None,
+        level: int | None = None,
+        add: str | None = None,
+        remove: str | None = None,
     ) -> None:
         """Configure logging.
 
@@ -196,7 +344,7 @@ class Env:
     def set_log_level(self, level: int) -> None:
         self._default_log_level = level
 
-    def set_module_log_level(self, module: str, level: Optional[int]) -> None:
+    def set_module_log_level(self, module: str, level: int | None) -> None:
         if level is None:
             self._module_log_level.pop(module, None)
         else:
@@ -211,14 +359,37 @@ class Env:
             # accept log messages from a configured pixeltable module (at any level of the module hierarchy)
             path_parts = list(Path(record.pathname).parts)
             path_parts.reverse()
+            if 'pixeltable' not in path_parts:
+                return False
             max_idx = path_parts.index('pixeltable')
             for module_name in path_parts[:max_idx]:
                 if module_name in self._module_log_level and record.levelno >= self._module_log_level[module_name]:
                     return True
-        if record.levelno >= self._default_log_level:
-            return True
+        return record.levelno >= self._default_log_level
+
+    @property
+    def console_logger(self) -> ConsoleLogger:
+        return self._console_logger
+
+    def _get_tz_name(self) -> str:
+        """Get the time zone name from the configuration, or the system local time zone if not specified.
+
+        Returns:
+            str: The time zone name.
+        """
+        tz_name = Config.get().get_string_value('time_zone')
+        if tz_name is not None:
+            # Validate tzname
+            if not isinstance(tz_name, str):
+                self._logger.error('Invalid time zone specified in configuration.')
+            else:
+                try:
+                    _ = ZoneInfo(tz_name)
+                except ZoneInfoNotFoundError:
+                    self._logger.error(f'Invalid time zone specified in configuration: {tz_name}')
         else:
-            return False
+            tz_name = tzlocal.get_localzone_name()
+        return tz_name
 
     def _set_up(self, echo: bool = False, reinit_db: bool = False) -> None:
         if self._initialized:
@@ -226,55 +397,57 @@ class Env:
 
         os.environ['TOKENIZERS_PARALLELISM'] = 'false'
 
+        config = Config.get()
+
         self._initialized = True
-        home = Path(os.environ.get('PIXELTABLE_HOME', str(Path.home() / '.pixeltable')))
-        assert self._home is None or self._home == home
-        self._home = home
-        self._config_file = Path(os.environ.get('PIXELTABLE_CONFIG', str(self._home / 'config.toml')))
-        self._media_dir = self._home / 'media'
-        self._file_cache_dir = self._home / 'file_cache'
-        self._dataset_cache_dir = self._home / 'dataset_cache'
-        self._log_dir = self._home / 'logs'
-        self._tmp_dir = self._home / 'tmp'
-
-        if self._home.exists() and not self._home.is_dir():
-            raise RuntimeError(f'{self._home} is not a directory')
-
-        if not self._home.exists():
-            # we don't have our logger set up yet, so print to stdout
-            print(f'Creating a Pixeltable instance at: {self._home}')
-            self._home.mkdir()
-            # TODO (aaron-siegel) This is the existing behavior, but it seems scary. If something happens to
-            # self._home, it will cause the DB to be destroyed even if pgdata is in an alternate location.
-            # PROPOSAL: require `reinit_db` to be set explicitly to destroy the DB.
-            reinit_db = True
-
-        if not self._media_dir.exists():
-            self._media_dir.mkdir()
-        if not self._file_cache_dir.exists():
-            self._file_cache_dir.mkdir()
-        if not self._dataset_cache_dir.exists():
-            self._dataset_cache_dir.mkdir()
-        if not self._log_dir.exists():
-            self._log_dir.mkdir()
-        if not self._tmp_dir.exists():
-            self._tmp_dir.mkdir()
-
-        # Read in the config
-        self._config = Config.from_file(self._config_file)
-        self._file_cache_size_g = self._config.get_float_value('file_cache_size_g')
+
+        self._media_dir = Config.get().home / 'media'
+        self._file_cache_dir = Config.get().home / 'file_cache'
+        self._dataset_cache_dir = Config.get().home / 'dataset_cache'
+        self._log_dir = Config.get().home / 'logs'
+        self._tmp_dir = Config.get().home / 'tmp'
+
+        self._media_dir.mkdir(exist_ok=True)
+        self._file_cache_dir.mkdir(exist_ok=True)
+        self._dataset_cache_dir.mkdir(exist_ok=True)
+        self._log_dir.mkdir(exist_ok=True)
+        self._tmp_dir.mkdir(exist_ok=True)
+
+        self._file_cache_size_g = config.get_float_value('file_cache_size_g')
         if self._file_cache_size_g is None:
             raise excs.Error(
                 'pixeltable/file_cache_size_g is missing from configuration\n'
-                f'(either add a `file_cache_size_g` entry to the `pixeltable` section of {self._config_file},\n'
+                f'(either add a `file_cache_size_g` entry to the `pixeltable` section of {Config.get().config_file},\n'
                 'or set the PIXELTABLE_FILE_CACHE_SIZE_G environment variable)'
             )
 
+        self._default_input_media_dest = config.get_string_value('input_media_dest')
+        self._default_output_media_dest = config.get_string_value('output_media_dest')
+        for mode, uri in (('input', self._default_input_media_dest), ('output', self._default_output_media_dest)):
+            if uri is not None:
+                try:
+                    _ = ObjectPath.parse_object_storage_addr(uri, False)
+                except Exception as e:
+                    raise excs.Error(f'Invalid {mode} media destination URI: {uri}') from e
+
+        self._pxt_api_key = config.get_string_value('api_key')
+
         # Disable spurious warnings
         warnings.simplefilter('ignore', category=TqdmWarning)
-        if self._config.get_bool_value('hide_warnings'):
+        if config.get_bool_value('hide_warnings'):
             # Disable more warnings
             warnings.simplefilter('ignore', category=UserWarning)
+            warnings.simplefilter('ignore', category=FutureWarning)
+
+        # Set verbosity level for user visible console messages
+        self._verbosity = config.get_int_value('verbosity')
+        if self._verbosity is None:
+            self._verbosity = 1
+        stdout_handler = ConsoleOutputHandler(stream=stdout)
+        stdout_handler.setLevel(map_level(self._verbosity))
+        stdout_handler.addFilter(ConsoleMessageFilter())
+        self._logger.addHandler(stdout_handler)
+        self._console_logger = ConsoleLogger(self._logger)
 
         # configure _logger to log to a file
         self._logfilename = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + '.log'
@@ -304,33 +477,21 @@ class Env:
         http_logger.addHandler(http_fh)
         http_logger.propagate = False
 
-        # empty tmp dir
-        for path in glob.glob(f'{self._tmp_dir}/*'):
-            os.remove(path)
-
-        self._db_name = os.environ.get('PIXELTABLE_DB', 'pixeltable')
-        self._pgdata_dir = Path(os.environ.get('PIXELTABLE_PGDATA', str(self._home / 'pgdata')))
+        self.clear_tmp_dir()
+        tz_name = self._get_tz_name()
 
-        # in pixeltable_pgserver.get_server(): cleanup_mode=None will leave db on for debugging purposes
-        self._db_server = pixeltable_pgserver.get_server(self._pgdata_dir, cleanup_mode=None)
-        self._db_url = self._db_server.get_uri(database=self._db_name, driver='psycopg')
+        # configure pixeltable database
+        self._init_db(config)
 
-        tz_name = self.config.get_string_value('time_zone')
-        if tz_name is not None:
-            # Validate tzname
-            if not isinstance(tz_name, str):
-                self._logger.error(f'Invalid time zone specified in configuration.')
-            else:
-                try:
-                    _ = ZoneInfo(tz_name)
-                except ZoneInfoNotFoundError:
-                    self._logger.error(f'Invalid time zone specified in configuration: {tz_name}')
+        if reinit_db and not self.is_local:
+            raise excs.Error(
+                'Reinitializing pixeltable database is not supported when running in non-local environment'
+            )
 
         if reinit_db and self._store_db_exists():
             self._drop_store_db()
 
         create_db = not self._store_db_exists()
-
         if create_db:
             self._logger.info(f'creating database at: {self.db_url}')
             self._create_store_db()
@@ -340,38 +501,104 @@ class Env:
         # Create the SQLAlchemy engine. This will also set the default time zone.
         self._create_engine(time_zone_name=tz_name, echo=echo)
 
-        if create_db:
-            from pixeltable.metadata import schema
-            schema.base_metadata.create_all(self._sa_engine)
-            metadata.create_system_info(self._sa_engine)
+        # Create catalog tables and system metadata
+        self._init_metadata()
 
-        print(f'Connected to Pixeltable database at: {self.db_url}')
+        self.console_logger.info(f'Connected to Pixeltable database at: {self.db_url}')
 
         # we now have a home directory and db; start other services
         self._set_up_runtime()
         self.log_to_stdout(False)
 
-    def _create_engine(self, time_zone_name: Optional[str], echo: bool = False) -> None:
-        connect_args = {} if time_zone_name is None else {'options': f'-c timezone={time_zone_name}'}
+    def _init_db(self, config: Config) -> None:
+        """
+        Initialize the pixeltable database along with its associated DBMS.
+        """
+        db_connect_str = config.get_string_value('DB_CONNECT_STR')
+        if db_connect_str is not None:
+            try:
+                db_url = sql.make_url(db_connect_str)
+            except sql.exc.ArgumentError as e:
+                error = f'Invalid db connection string {db_connect_str}: {e}'
+                self._logger.error(error)
+                raise excs.Error(error) from e
+            self._db_url = db_url.render_as_string(hide_password=False)
+            self._db_name = db_url.database  # use the dbname given in connect string
+            dialect = db_url.get_dialect().name
+            if dialect == 'cockroachdb':
+                self._dbms = CockroachDbms(db_url)
+            else:
+                raise excs.Error(f'Unsupported DBMS {dialect}')
+            # Check if database exists
+            if not self._store_db_exists():
+                error = f'Database {self._db_name!r} does not exist'
+                self._logger.error(error)
+                raise excs.Error(error)
+            self._logger.info(f'Using database at: {self.db_url}')
+        else:
+            self._db_name = config.get_string_value('db') or 'pixeltable'
+            self._pgdata_dir = Path(os.environ.get('PIXELTABLE_PGDATA', str(Config.get().home / 'pgdata')))
+            # cleanup_mode=None will leave the postgres process running after Python exits
+            # cleanup_mode='stop' will terminate the postgres process when Python exits
+            # On Windows, we need cleanup_mode='stop' because child processes are killed automatically when the parent
+            # process (such as Terminal or VSCode) exits, potentially leaving it in an unusable state.
+            cleanup_mode = 'stop' if platform.system() == 'Windows' else None
+            self._db_server = pixeltable_pgserver.get_server(self._pgdata_dir, cleanup_mode=cleanup_mode)
+            self._db_url = self._db_server.get_uri(database=self._db_name, driver='psycopg')
+            self._dbms = PostgresqlDbms(sql.make_url(self._db_url))
+        assert self._dbms is not None
+        assert self._db_url is not None
+        assert self._db_name is not None
+
+    @retry(
+        stop=stop_after_attempt(3),  # Stop after 3 attempts
+        wait=wait_exponential_jitter(initial=0.2, max=1.0, jitter=0.2),  # Exponential backoff with jitter
+    )
+    def _init_metadata(self) -> None:
+        """
+        Create pixeltable metadata tables and system metadata.
+        This is an idempotent operation.
+
+        Retry logic handles race conditions when multiple Pixeltable processes
+        attempt to initialize metadata tables simultaneously. The first process may succeed
+        in creating tables while others encounter database constraints (e.g., "table already exists").
+        Exponential backoff with jitter reduces contention between competing processes.
+        """
+        assert self._sa_engine is not None
+        from pixeltable import metadata
+
+        self._logger.debug('Creating pixeltable metadata')
+        metadata.schema.base_metadata.create_all(self._sa_engine, checkfirst=True)
+        metadata.create_system_info(self._sa_engine)
+
+    def _create_engine(self, time_zone_name: str, echo: bool = False) -> None:
+        # Add timezone option to connection string
+        updated_url = add_option_to_db_url(self.db_url, f'-c timezone={time_zone_name}')
+
         self._sa_engine = sql.create_engine(
-            self.db_url,
-            echo=echo,
-            future=True,
-            isolation_level='AUTOCOMMIT',
-            connect_args=connect_args,
+            updated_url, echo=echo, isolation_level=self._dbms.transaction_isolation_level
         )
+
         self._logger.info(f'Created SQLAlchemy engine at: {self.db_url}')
+        self._logger.info(f'Engine dialect: {self._sa_engine.dialect.name}')
+        self._logger.info(f'Engine driver : {self._sa_engine.dialect.driver}')
+
         with self.engine.begin() as conn:
             tz_name = conn.execute(sql.text('SHOW TIME ZONE')).scalar()
             assert isinstance(tz_name, str)
             self._logger.info(f'Database time zone is now: {tz_name}')
             self._default_time_zone = ZoneInfo(tz_name)
+            if self.is_using_cockroachdb:
+                # This could be set when the database is created, but we set it now
+                conn.execute(sql.text('SET null_ordered_last = true;'))
+                null_ordered_last = conn.execute(sql.text('SHOW null_ordered_last')).scalar()
+                assert isinstance(null_ordered_last, str)
+                self._logger.info(f'Database null_ordered_last is now: {null_ordered_last}')
 
     def _store_db_exists(self) -> bool:
         assert self._db_name is not None
         # don't try to connect to self.db_name, it may not exist
-        db_url = self._db_server.get_uri(database='postgres', driver='psycopg')
-        engine = sql.create_engine(db_url, future=True)
+        engine = sql.create_engine(self._dbms.default_system_db_url(), future=True)
         try:
             with engine.begin() as conn:
                 stmt = f"SELECT COUNT(*) FROM pg_database WHERE datname = '{self._db_name}'"
@@ -384,53 +611,55 @@ class Env:
     def _create_store_db(self) -> None:
         assert self._db_name is not None
         # create the db
-        pg_db_url = self._db_server.get_uri(database='postgres', driver='psycopg')
-        engine = sql.create_engine(pg_db_url, future=True, isolation_level='AUTOCOMMIT')
+        engine = sql.create_engine(self._dbms.default_system_db_url(), future=True, isolation_level='AUTOCOMMIT')
         preparer = engine.dialect.identifier_preparer
         try:
             with engine.begin() as conn:
-                # use C collation to get standard C/Python-style sorting
-                stmt = (
-                    f"CREATE DATABASE {preparer.quote(self._db_name)} "
-                    "ENCODING 'utf-8' LC_COLLATE 'C' LC_CTYPE 'C' TEMPLATE template0"
-                )
+                stmt = self._dbms.create_db_stmt(preparer.quote(self._db_name))
                 conn.execute(sql.text(stmt))
         finally:
             engine.dispose()
 
         # enable pgvector
-        store_db_url = self._db_server.get_uri(database=self._db_name, driver='psycopg')
-        engine = sql.create_engine(store_db_url, future=True, isolation_level='AUTOCOMMIT')
+        engine = sql.create_engine(self.db_url, future=True, isolation_level='AUTOCOMMIT')
         try:
             with engine.begin() as conn:
                 conn.execute(sql.text('CREATE EXTENSION vector'))
         finally:
             engine.dispose()
 
+    def _pgserver_terminate_connections_stmt(self) -> str:
+        return f"""
+                SELECT pg_terminate_backend(pg_stat_activity.pid)
+                FROM pg_stat_activity
+                WHERE pg_stat_activity.datname = '{self._db_name}'
+                AND pid <> pg_backend_pid()
+            """
+
     def _drop_store_db(self) -> None:
         assert self._db_name is not None
-        db_url = self._db_server.get_uri(database='postgres', driver='psycopg')
-        engine = sql.create_engine(db_url, future=True, isolation_level='AUTOCOMMIT')
+        engine = sql.create_engine(self._dbms.default_system_db_url(), future=True, isolation_level='AUTOCOMMIT')
         preparer = engine.dialect.identifier_preparer
         try:
             with engine.begin() as conn:
                 # terminate active connections
-                stmt = (f"""
-                    SELECT pg_terminate_backend(pg_stat_activity.pid)
-                    FROM pg_stat_activity
-                    WHERE pg_stat_activity.datname = '{self._db_name}'
-                    AND pid <> pg_backend_pid()
-                """)
-                conn.execute(sql.text(stmt))
+                if self._db_server is not None:
+                    conn.execute(sql.text(self._pgserver_terminate_connections_stmt()))
                 # drop db
-                stmt = f'DROP DATABASE {preparer.quote(self._db_name)}'
+                stmt = self._dbms.drop_db_stmt(preparer.quote(self._db_name))
                 conn.execute(sql.text(stmt))
         finally:
             engine.dispose()
 
     def _upgrade_metadata(self) -> None:
+        from pixeltable import metadata
+
         metadata.upgrade_md(self._sa_engine)
 
+    @property
+    def pxt_api_key(self) -> str | None:
+        return self._pxt_api_key
+
     def get_client(self, name: str) -> Any:
         """
         Gets the client with the specified name, initializing it if necessary.
@@ -438,35 +667,51 @@ class Env:
         Args:
             - name: The name of the client
         """
-        cl = _registered_clients[name]
-        if cl.client_obj is not None:
-            return cl.client_obj  # Already initialized
-
-        # Construct a client, retrieving each parameter from config.
-
-        init_kwargs: dict[str, str] = {}
-        for param in cl.param_names:
-            arg = self._config.get_string_value(param, section=name)
-            if arg is not None and len(arg) > 0:
-                init_kwargs[param] = arg
-            else:
+        # Return the existing client if it has already been constructed
+        with _registered_clients_lock:
+            cl = _registered_clients[name]
+            if cl.client_obj is not None:
+                return cl.client_obj  # Already initialized
+
+        # Retrieve parameters required to construct the requested client.
+        init_kwargs: dict[str, Any] = {}
+        for param in cl.params.values():
+            # Determine the type of the parameter for proper config parsing.
+            pname = param.name
+            t = param.annotation
+            # Deference T | None
+            if typing.get_origin(t) in (typing.Union, types.UnionType):
+                args = typing.get_args(t)
+                if args[0] is type(None):
+                    t = args[1]
+                elif args[1] is type(None):
+                    t = args[0]
+            assert isinstance(t, type), t
+            arg: Any = Config.get().get_value(pname, t, section=name)
+            if arg is not None:
+                init_kwargs[pname] = arg
+            elif param.default is inspect.Parameter.empty:
                 raise excs.Error(
-                    f'`{name}` client not initialized: parameter `{param}` is not configured.\n'
-                    f'To fix this, specify the `{name.upper()}_{param.upper()}` environment variable, or put `{param.lower()}` in '
-                    f'the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
+                    f'`{name}` client not initialized: parameter `{pname}` is not configured.\n'
+                    f'To fix this, specify the `{name.upper()}_{pname.upper()}` environment variable, '
+                    f'or put `{pname.lower()}` in the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
                 )
 
-        cl.client_obj = cl.init_fn(**init_kwargs)
-        self._logger.info(f'Initialized `{name}` client.')
-        return cl.client_obj
+        # Construct the requested client
+        with _registered_clients_lock:
+            if cl.client_obj is not None:
+                return cl.client_obj  # Already initialized
+            cl.client_obj = cl.init_fn(**init_kwargs)
+            self._logger.info(f'Initialized `{name}` client with parameters: {init_kwargs}.')
+            return cl.client_obj
 
     def _start_web_server(self) -> None:
         """
         The http server root is the file system root.
         eg: /home/media/foo.mp4 is located at http://127.0.0.1:{port}/home/media/foo.mp4
-        in windows, the server will translate paths like http://127.0.0.1:{port}/c:/media/foo.mp4
-        This arrangement enables serving media hosted within _home,
-        as well as external media inserted into pixeltable or produced by pixeltable.
+        On Windows, the server will translate paths like http://127.0.0.1:{port}/c:/media/foo.mp4
+        This arrangement enables serving objects hosted within _home,
+        as well as external objects inserted into pixeltable or produced by pixeltable.
         The port is chosen dynamically to prevent conflicts.
         """
         # Port 0 means OS picks one for us.
@@ -474,7 +719,7 @@ class Env:
         port = self._httpd.server_address[1]
         self._http_address = f'http://127.0.0.1:{port}'
 
-        def run_server():
+        def run_server() -> None:
             logging.log(logging.INFO, f'running web server at {self._http_address}')
             self._httpd.serve_forever()
 
@@ -484,30 +729,77 @@ class Env:
 
     def _set_up_runtime(self) -> None:
         """Check for and start runtime services"""
+        register_heif_opener()
         self._start_web_server()
         self.__register_packages()
-        if self.is_installed_package('spacy'):
-            self.__init_spacy()
+
+    @property
+    def default_video_encoder(self) -> str | None:
+        if self._default_video_encoder is None:
+            self._default_video_encoder = self._determine_default_video_encoder()
+        return self._default_video_encoder
+
+    def _determine_default_video_encoder(self) -> str | None:
+        """
+        Returns the first available encoder from a list of candidates.
+
+        TODO:
+        - the user might prefer a hardware-accelerated encoder (eg, h264_nvenc or h264_videotoolbox)
+        - allow user override via a config option 'video_encoder'
+        """
+        # look for available encoders, in this order
+        candidates = [
+            'libx264',  # GPL, best quality
+            'libopenh264',  # BSD
+        ]
+
+        try:
+            # Get list of available encoders
+            result = subprocess.run(['ffmpeg', '-encoders'], capture_output=True, text=True, timeout=10, check=True)
+
+            if result.returncode == 0:
+                available_encoders = result.stdout
+                for encoder in candidates:
+                    # ffmpeg -encoders output format: " V..... encoder_name  description"
+                    if f' {encoder} ' in available_encoders:
+                        _logger.debug(f'Using H.264 encoder: {encoder}')
+                        return encoder
+        except Exception:
+            pass
+        return None
 
     def __register_packages(self) -> None:
         """Declare optional packages that are utilized by some parts of the code."""
+        self.__register_package('accelerate')
         self.__register_package('anthropic')
+        self.__register_package('azure.storage.blob', library_name='azure-storage-blob')
         self.__register_package('boto3')
         self.__register_package('datasets')
+        self.__register_package('diffusers')
         self.__register_package('fiftyone')
+        self.__register_package('twelvelabs')
+        self.__register_package('fal_client', library_name='fal-client')
         self.__register_package('fireworks', library_name='fireworks-ai')
+        self.__register_package('google.cloud.storage', library_name='google-cloud-storage')
+        self.__register_package('google.genai', library_name='google-genai')
+        self.__register_package('groq')
         self.__register_package('huggingface_hub', library_name='huggingface-hub')
         self.__register_package('label_studio_sdk', library_name='label-studio-sdk')
+        self.__register_package('librosa')
         self.__register_package('llama_cpp', library_name='llama-cpp-python')
+        self.__register_package('mcp')
         self.__register_package('mistralai')
         self.__register_package('mistune')
         self.__register_package('ollama')
         self.__register_package('openai')
         self.__register_package('openpyxl')
         self.__register_package('pyarrow')
+        self.__register_package('pydantic')
         self.__register_package('replicate')
+        self.__register_package('reve')
         self.__register_package('sentencepiece')
         self.__register_package('sentence_transformers', library_name='sentence-transformers')
+        self.__register_package('soundfile')
         self.__register_package('spacy')
         self.__register_package('tiktoken')
         self.__register_package('together')
@@ -515,17 +807,30 @@ class Env:
         self.__register_package('torchaudio')
         self.__register_package('torchvision')
         self.__register_package('transformers')
+        self.__register_package('voyageai')
         self.__register_package('whisper', library_name='openai-whisper')
         self.__register_package('whisperx')
-        self.__register_package('yolox', library_name='git+https://github.com/Megvii-BaseDetection/YOLOX@ac58e0a')
+        self.__register_package('yolox', library_name='pixeltable-yolox')
+        self.__register_package('lancedb')
+        self.__register_package('scenedetect')
 
-    def __register_package(self, package_name: str, library_name: Optional[str] = None) -> None:
+    def __register_package(self, package_name: str, library_name: str | None = None) -> None:
+        is_installed: bool
+        try:
+            is_installed = importlib.util.find_spec(package_name) is not None
+        except ModuleNotFoundError:
+            # This can happen if the parent of `package_name` is not installed.
+            is_installed = False
         self.__optional_packages[package_name] = PackageInfo(
-            is_installed=importlib.util.find_spec(package_name) is not None,
-            library_name=library_name or package_name  # defaults to package_name unless specified otherwise
+            is_installed=is_installed,
+            library_name=library_name or package_name,  # defaults to package_name unless specified otherwise
         )
 
-    def require_package(self, package_name: str, min_version: Optional[list[int]] = None) -> None:
+    def require_binary(self, binary_name: str) -> None:
+        if not shutil.which(binary_name):
+            raise excs.Error(f'{binary_name} is not installed or not in PATH. Please install it to use this feature.')
+
+    def require_package(self, package_name: str, min_version: list[int] | None = None) -> None:
         """
         Checks whether the specified optional package is available. If not, raises an exception
         with an error message informing the user how to install it.
@@ -542,7 +847,8 @@ class Env:
             if not package_info.is_installed:
                 # Still not found.
                 raise excs.Error(
-                    f'This feature requires the `{package_name}` package. To install it, run: `pip install -U {package_info.library_name}`'
+                    f'This feature requires the `{package_name}` package. To install it, run: '
+                    f'`pip install -U {package_info.library_name}`'
                 )
 
         if min_version is None:
@@ -555,56 +861,41 @@ class Env:
 
         if min_version > package_info.version:
             raise excs.Error(
-                f'The installed version of package `{package_name}` is {".".join(str(v) for v in package_info.version)}, '
+                f'The installed version of package `{package_name}` is '
+                f'{".".join(str(v) for v in package_info.version)}, '
                 f'but version >={".".join(str(v) for v in min_version)} is required. '
                 f'To fix this, run: `pip install -U {package_info.library_name}`'
             )
 
-    def __init_spacy(self) -> None:
-        """
-        spaCy relies on a pip-installed model to operate. In order to avoid requiring the model as a separate
-        dependency, we install it programmatically here. This should cause no problems, since the model packages
-        have no sub-dependencies (in fact, this is how spaCy normally manages its model resources).
-        """
-        import spacy
-        from spacy.cli.download import get_model_filename
-        spacy_model = 'en_core_web_sm'
-        spacy_model_version = '3.7.1'
-        filename = get_model_filename(spacy_model, spacy_model_version, sdist=False)
-        url = f'{spacy.about.__download_url__}/{filename}'
-        # Try to `pip install` the model. We set check=False; if the pip command fails, it's not necessarily
-        # a problem, because the model have been installed on a previous attempt.
-        self._logger.info(f'Ensuring spaCy model is installed: {filename}')
-        ret = subprocess.run([sys.executable, '-m', 'pip', 'install', '-qU', url], check=False)
-        if ret.returncode != 0:
-            self._logger.warn(f'pip install failed for spaCy model: {filename}')
-        try:
-            self._logger.info(f'Loading spaCy model: {spacy_model}')
-            self._spacy_nlp = spacy.load(spacy_model)
-        except Exception as exc:
-            self._logger.warn(f'Failed to load spaCy model: {spacy_model}', exc_info=exc)
-            warnings.warn(
-                f"Failed to load spaCy model '{spacy_model}'. spaCy features will not be available.",
-                excs.PixeltableWarning
-            )
-            self.__optional_packages['spacy'].is_installed = False
-
-    def num_tmp_files(self) -> int:
-        return len(glob.glob(f'{self._tmp_dir}/*'))
-
-    def create_tmp_path(self, extension: str = '') -> Path:
-        return self._tmp_dir / f'{uuid.uuid4()}{extension}'
+    def clear_tmp_dir(self) -> None:
+        for path in glob.glob(f'{self._tmp_dir}/*'):
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            else:
+                os.remove(path)
 
-    @property
-    def home(self) -> Path:
-        assert self._home is not None
-        return self._home
+    # def get_resource_pool_info(self, pool_id: str, pool_info_cls: Type[T] | None) -> T:
+    def get_resource_pool_info(self, pool_id: str, make_pool_info: Callable[[], T] | None = None) -> T:
+        """Returns the info object for the given id, creating it if necessary."""
+        info = self._resource_pool_info.get(pool_id)
+        if info is None and make_pool_info is not None:
+            info = make_pool_info()
+            self._resource_pool_info[pool_id] = info
+        return info
 
     @property
     def media_dir(self) -> Path:
         assert self._media_dir is not None
         return self._media_dir
 
+    @property
+    def default_input_media_dest(self) -> str | None:
+        return self._default_input_media_dest
+
+    @property
+    def default_output_media_dest(self) -> str | None:
+        return self._default_output_media_dest
+
     @property
     def file_cache_dir(self) -> Path:
         assert self._file_cache_dir is not None
@@ -628,9 +919,86 @@ class Env:
     @property
     def spacy_nlp(self) -> spacy.Language:
         Env.get().require_package('spacy')
+        if self._spacy_nlp is None:
+            self.__init_spacy()
         assert self._spacy_nlp is not None
         return self._spacy_nlp
 
+    def __init_spacy(self) -> None:
+        """
+        spaCy relies on a pip-installed model to operate. In order to avoid requiring the model as a separate
+        dependency, we install it programmatically here. This should cause no problems, since the model packages
+        have no sub-dependencies (in fact, this is how spaCy normally manages its model resources).
+        """
+        import spacy
+        from spacy.cli.download import download
+
+        spacy_model = 'en_core_web_sm'
+        self._logger.info(f'Ensuring spaCy model is installed: {spacy_model}')
+        download(spacy_model)
+        self._logger.info(f'Loading spaCy model: {spacy_model}')
+        try:
+            self._spacy_nlp = spacy.load(spacy_model)
+        except Exception as exc:
+            raise excs.Error(f'Failed to load spaCy model: {spacy_model}') from exc
+
+    def _clean_up(self) -> None:
+        """
+        Internal cleanup method that properly closes all resources and resets state.
+        This is called before destroying the singleton instance.
+        """
+        assert self._current_session is None
+        assert self._current_conn is None
+
+        # Stop HTTP server
+        if self._httpd is not None:
+            try:
+                self._httpd.shutdown()
+                self._httpd.server_close()
+            except Exception as e:
+                _logger.warning(f'Error stopping HTTP server: {e}')
+
+        # First terminate all connections to the database
+        if self._db_server is not None:
+            assert self._dbms is not None
+            assert self._db_name is not None
+            try:
+                temp_engine = sql.create_engine(self._dbms.default_system_db_url(), isolation_level='AUTOCOMMIT')
+                try:
+                    with temp_engine.begin() as conn:
+                        conn.execute(sql.text(self._pgserver_terminate_connections_stmt()))
+                        _logger.info(f"Terminated all connections to database '{self._db_name}'")
+                except Exception as e:
+                    _logger.warning(f'Error terminating database connections: {e}')
+                finally:
+                    temp_engine.dispose()
+            except Exception as e:
+                _logger.warning(f'Error stopping database server: {e}')
+
+        # Dispose of SQLAlchemy engine (after stopping db server)
+        if self._sa_engine is not None:
+            try:
+                self._sa_engine.dispose()
+            except Exception as e:
+                _logger.warning(f'Error disposing engine: {e}')
+
+        # Close event loop
+        if self._event_loop is not None:
+            try:
+                if self._event_loop.is_running():
+                    self._event_loop.stop()
+                self._event_loop.close()
+            except Exception as e:
+                _logger.warning(f'Error closing event loop: {e}')
+
+        # Remove logging handlers
+        for handler in self._logger.handlers[:]:
+            try:
+                handler.close()
+                self._logger.removeHandler(handler)
+            except Exception as e:
+                _logger.warning(f'Error removing handler: {e}')
+
 
 def register_client(name: str) -> Callable:
     """Decorator that registers a third-party API client for use by Pixeltable.
@@ -656,100 +1024,183 @@ def register_client(name: str) -> Callable:
     Args:
         - name (str): The name of the API client (e.g., 'openai' or 'label-studio').
     """
+
     def decorator(fn: Callable) -> None:
-        global _registered_clients
         sig = inspect.signature(fn)
-        param_names = list(sig.parameters.keys())
-        _registered_clients[name] = ApiClient(init_fn=fn, param_names=param_names)
+        params = dict(sig.parameters)
+        with _registered_clients_lock:
+            _registered_clients[name] = ApiClient(init_fn=fn, params=params)
 
     return decorator
 
 
-class Config:
+_registered_clients_lock: threading.Lock = threading.Lock()
+_registered_clients: dict[str, ApiClient] = {}
+
+
+@dataclass
+class ApiClient:
+    init_fn: Callable
+    params: dict[str, inspect.Parameter]
+    client_obj: Any | None = None
+
+
+@dataclass
+class PackageInfo:
+    is_installed: bool
+    library_name: str  # pypi library name (may be different from package name)
+    version: list[int] | None = None  # installed version, as a list of components (such as [3,0,2] for "3.0.2")
+
+
+TIME_FORMAT = '%H:%M.%S %f'
+# As far as rate limiting goes, we try not go lower than 5% of the capacity because we don't have perfect information
+# about the rate limits and the usage
+TARGET_RATE_LIMIT_RESOURCE_FRACT = 0.05
+
+
+@dataclass
+class RateLimitsInfo:
     """
-    The (global) Pixeltable configuration, as loaded from `config.toml`. Provides methods for retrieving
-    configuration values, which can be set in the config file or as environment variables.
+    Abstract base class for resource pools made up of rate limits for different resources.
+
+    Rate limits and currently remaining resources are periodically reported via record().
+
+    Subclasses provide operational customization via:
+    - get_retry_delay()
+    - get_request_resources(self, ...) -> dict[str, int]
+    with parameters that are a subset of those of the udf that creates the subclass's instance
     """
-    __config: dict[str, Any]
 
-    T = TypeVar('T')
+    # get_request_resources:
+    # - Returns estimated resources needed for a specific request (ie, a single udf call) as a dict (key: resource name)
+    # - parameters are a subset of those of the udf
+    # - this is not a class method because the signature depends on the instantiating udf
+    get_request_resources: Callable[..., dict[str, int]]
 
-    @classmethod
-    def from_file(cls, path: Path) -> Config:
-        """
-        Loads configuration from the specified TOML file. If the file does not exist, it will be
-        created and populated with the default configuration.
-        """
-        if os.path.isfile(path):
-            with open(path, 'r') as stream:
-                try:
-                    config_dict = toml.load(stream)
-                except Exception as exc:
-                    raise excs.Error(f'Could not read config file: {str(path)}') from exc
-        else:
-            config_dict = cls.__create_default_config(path)
-            with open(path, 'w') as stream:
-                try:
-                    toml.dump(config_dict, stream)
-                except Exception as exc:
-                    raise excs.Error(f'Could not write config file: {str(path)}') from exc
-            logging.getLogger('pixeltable').info(f'Created default config file at: {str(path)}')
-        return cls(config_dict)
+    resource_limits: dict[str, RateLimitInfo] = field(default_factory=dict)
+    has_exc: bool = False
 
-    @classmethod
-    def __create_default_config(cls, config_path: Path) -> dict[str, Any]:
-        free_disk_space_bytes = shutil.disk_usage(config_path.parent).free
-        # Default cache size is 1/5 of free disk space
-        file_cache_size_g = free_disk_space_bytes / 5 / (1 << 30)
-        return {
-            'pixeltable': {
-                'file_cache_size_g': round(file_cache_size_g, 1),
-                'hide_warnings': False,
-            }
-        }
-
-    def __init__(self, config: dict[str, Any]) -> None:
-        self.__config = config
-
-    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
-        env_var = f'{section.upper()}_{key.upper()}'
-        if env_var in os.environ:
-            value = os.environ[env_var]
-        elif section in self.__config and key in self.__config[section]:
-            value = self.__config[section][key]
-        else:
-            return None
+    def debug_str(self) -> str:
+        return ','.join(info.debug_str() for info in self.resource_limits.values())
 
-        try:
-            return expected_type(value)  # type: ignore[call-arg]
-        except ValueError:
-            raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}')
+    def is_initialized(self) -> bool:
+        return len(self.resource_limits) > 0
 
-    def get_string_value(self, key: str, section: str = 'pixeltable') -> Optional[str]:
-        return self.get_value(key, str, section)
+    def reset(self) -> None:
+        self.resource_limits.clear()
 
-    def get_int_value(self, key: str, section: str = 'pixeltable') -> Optional[int]:
-        return self.get_value(key, int, section)
+    def record(self, request_ts: datetime.datetime, reset_exc: bool = False, **kwargs: Any) -> None:
+        """Update self.resource_limits with the provided rate limit info.
+        Args:
+            - request_ts: time at which the request was made
+            - reset_exc: if True, reset the has_exc flag
+        """
+        if len(self.resource_limits) == 0:
+            self.resource_limits = {k: RateLimitInfo(k, request_ts, *v) for k, v in kwargs.items() if v is not None}
+            # TODO: remove
+            for info in self.resource_limits.values():
+                _logger.debug(f'Updated resource state: {info}')
+        else:
+            if self.has_exc and not reset_exc:
+                # ignore updates until we're asked to reset
+                _logger.debug(f'rate_limits.record(): ignoring update {kwargs}')
+                return
+            self.has_exc = False
+            for k, v in kwargs.items():
+                if v is not None:
+                    self.resource_limits[k].update(request_ts, *v)
+                    _logger.debug(f'Updated resource state: {self.resource_limits[k]}')
+
+    def record_exc(self, request_ts: datetime.datetime, exc: Exception) -> None:
+        """Update self.resource_limits based on the exception headers
+        Args:
+            - request_ts: time at which the request that caused the exception was made
+            - exc: the exception raised"""
+        self.has_exc = True
+
+    def get_retry_delay(self, exc: Exception, attempt: int) -> float | None:
+        """Returns number of seconds to wait before retry, or None if not retryable"""
+        # Find the highest wait until at least 5% availability of all resources
+        max_wait = 0.0
+        for limit_info in self.resource_limits.values():
+            time_until = limit_info.estimated_resource_refill_delay(
+                math.ceil(TARGET_RATE_LIMIT_RESOURCE_FRACT * limit_info.limit)
+            )
+            if time_until is not None:
+                max_wait = max(max_wait, time_until)
+        return max_wait if max_wait > 0 else None
 
-    def get_float_value(self, key: str, section: str = 'pixeltable') -> Optional[float]:
-        return self.get_value(key, float, section)
 
-    def get_bool_value(self, key: str, section: str = 'pixeltable') -> Optional[bool]:
-        return self.get_value(key, bool, section)
+@dataclass
+class RateLimitInfo:
+    """Container for rate limit-related information for a single resource."""
+
+    resource: str
+    request_start_ts: datetime.datetime
+    limit: int
+    remaining: int
+    reset_at: datetime.datetime
+
+    def debug_str(self) -> str:
+        return (
+            f'{self.resource}@{self.request_start_ts.strftime(TIME_FORMAT)}: '
+            f'{self.limit}/{self.remaining}/{self.reset_at.strftime(TIME_FORMAT)}'
+        )
 
+    def update(
+        self, request_start_ts: datetime.datetime, limit: int, remaining: int, reset_at: datetime.datetime
+    ) -> None:
+        # Responses can come out of order, especially for failed requests. We need to be careful not to overwrite
+        # the current state with less up-to-date information. We use request_start_ts as a proxy for rate limit info
+        # recency.
+        if self.request_start_ts > request_start_ts:
+            # The current state is more up-to-date than the update
+            _logger.debug(
+                f'Ignoring out-of-date update for {self.resource}. Current request_start_ts: '
+                f'{self.request_start_ts}, update: {request_start_ts}'
+            )
+            return
+        self.request_start_ts = request_start_ts
+        self.limit = limit
+        self.remaining = remaining
+        self.reset_at = reset_at
+
+    def estimated_resource_refill_delay(self, target_remaining: int) -> float | None:
+        """Estimate time in seconds until remaining resources reaches target_remaining.
+        Assumes linear replenishment of resources over time.
+        Returns None if unable to estimate.
+        """
+        if self.remaining >= target_remaining:
+            return 0
+        if self.request_start_ts >= self.reset_at:
+            return 0
+        if self.limit < target_remaining:
+            return None
 
-_registered_clients: dict[str, ApiClient] = {}
+        # Estimate resource refill rate based on the recorded state and timestamps. Assumes linear refill.
+        refill_rate = (self.limit - self.remaining) / (self.reset_at - self.request_start_ts).total_seconds()
+        assert refill_rate > 0, f'self={self}, target_remaining={target_remaining}'
 
+        now = datetime.datetime.now(tz=datetime.timezone.utc)
+        time_until = (target_remaining - self.remaining) / refill_rate - (now - self.request_start_ts).total_seconds()
+        return max(0, math.ceil(time_until))
 
-@dataclass
-class ApiClient:
-    init_fn: Callable
-    param_names: list[str]
-    client_obj: Optional[Any] = None
+    def __repr__(self) -> str:
+        return (
+            f'RateLimitInfo(resource={self.resource}, request_start_ts={self.request_start_ts}, '
+            f'remaining={self.remaining}/{self.limit} ({(100 * self.remaining / self.limit):.1f}%), '
+            f'reset_at={self.reset_at})'
+        )
 
 
 @dataclass
-class PackageInfo:
-    is_installed: bool
-    library_name: str  # pypi library name (may be different from package name)
-    version: Optional[list[int]] = None  # installed version, as a list of components (such as [3,0,2] for "3.0.2")
+class RuntimeCtx:
+    """
+    Container for runtime data provided by the execution system to udfs.
+
+    Udfs that accept the special _runtime_ctx parameter receive an instance of this class.
+    """
+
+    # Indicates a retry attempt following a rate limit error (error code: 429). Requires a 'rate-limits' resource pool.
+    # If True, call RateLimitsInfo.record() with reset_exc=True.
+    is_retry: bool = False