pixeltable 0.3.14__py3-none-any.whl → 0.5.7__py3-none-any.whl

pixeltable/env.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import datetime
 import glob
 import http.server
@@ -7,24 +8,30 @@ import importlib
 import importlib.util
 import inspect
 import logging
+import math
 import os
 import platform
 import shutil
 import subprocess
 import sys
 import threading
-import uuid
+import types
+import typing
 import warnings
-from abc import abstractmethod
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
 from sys import stdout
-from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, TypeVar
+from typing import TYPE_CHECKING, Any, Callable, Iterator, TypeVar
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
+import nest_asyncio  # type: ignore[import-untyped]
 import pixeltable_pgserver
 import sqlalchemy as sql
+import tzlocal
+from pillow_heif import register_heif_opener  # type: ignore[import-untyped]
+from sqlalchemy import orm
+from tenacity import retry, stop_after_attempt, wait_exponential_jitter
 from tqdm import TqdmWarning
 
 from pixeltable import exceptions as excs
@@ -32,6 +39,8 @@ from pixeltable.config import Config
 from pixeltable.utils.console_output import ConsoleLogger, ConsoleMessageFilter, ConsoleOutputHandler, map_level
 from pixeltable.utils.dbms import CockroachDbms, Dbms, PostgresqlDbms
 from pixeltable.utils.http_server import make_server
+from pixeltable.utils.object_stores import ObjectPath
+from pixeltable.utils.sql import add_option_to_db_url
 
 if TYPE_CHECKING:
     import spacy
@@ -49,42 +58,50 @@ class Env:
     For a non-local environment, Pixeltable uses a connection string to the externally managed database.
     """
 
-    _instance: Optional[Env] = None
+    SERIALIZABLE_ISOLATION_LEVEL = 'SERIALIZABLE'
+
+    _instance: Env | None = None
     __initializing: bool = False
     _log_fmt_str = '%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d: %(message)s'
 
-    _media_dir: Optional[Path]
-    _file_cache_dir: Optional[Path]  # cached media files with external URL
-    _dataset_cache_dir: Optional[Path]  # cached datasets (eg, pytorch or COCO)
-    _log_dir: Optional[Path]  # log files
-    _tmp_dir: Optional[Path]  # any tmp files
-    _sa_engine: Optional[sql.engine.base.Engine]
-    _pgdata_dir: Optional[Path]
-    _db_name: Optional[str]
-    _db_server: Optional[pixeltable_pgserver.PostgresServer]  # set only when running in local environment
-    _db_url: Optional[str]
-    _default_time_zone: Optional[ZoneInfo]
+    _media_dir: Path | None
+    _file_cache_dir: Path | None  # cached object files with external URL
+    _dataset_cache_dir: Path | None  # cached datasets (eg, pytorch or COCO)
+    _log_dir: Path | None  # log files
+    _tmp_dir: Path | None  # any tmp files
+    _sa_engine: sql.engine.base.Engine | None
+    _pgdata_dir: Path | None
+    _db_name: str | None
+    _db_server: pixeltable_pgserver.PostgresServer | None  # set only when running in local environment
+    _db_url: str | None
+    _default_time_zone: ZoneInfo | None
+    _verbosity: int
 
     # info about optional packages that are utilized by some parts of the code
     __optional_packages: dict[str, PackageInfo]
 
-    _spacy_nlp: Optional[spacy.Language]
-    _httpd: Optional[http.server.HTTPServer]
-    _http_address: Optional[str]
+    _spacy_nlp: spacy.Language | None
+    _httpd: http.server.HTTPServer | None
+    _http_address: str | None
     _logger: logging.Logger
     _default_log_level: int
-    _logfilename: Optional[str]
+    _logfilename: str | None
     _log_to_stdout: bool
     _module_log_level: dict[str, int]  # module name -> log level
     _file_cache_size_g: float
-    _pxt_api_key: Optional[str]
+    _default_input_media_dest: str | None
+    _default_output_media_dest: str | None
+    _pxt_api_key: str | None
     _stdout_handler: logging.StreamHandler
+    _default_video_encoder: str | None
     _initialized: bool
 
     _resource_pool_info: dict[str, Any]
-    _current_conn: Optional[sql.Connection]
-    _current_session: Optional[sql.orm.Session]
-    _dbms: Optional[Dbms]
+    _current_conn: sql.Connection | None
+    _current_session: orm.Session | None
+    _current_isolation_level: str | None
+    _dbms: Dbms | None
+    _event_loop: asyncio.AbstractEventLoop | None  # event loop for ExecNode
 
     @classmethod
     def get(cls) -> Env:
@@ -96,17 +113,24 @@ class Env:
     def _init_env(cls, reinit_db: bool = False) -> None:
         assert not cls.__initializing, 'Circular env initialization detected.'
         cls.__initializing = True
+        if cls._instance is not None:
+            cls._instance._clean_up()
+        cls._instance = None
         env = Env()
-        env._set_up(reinit_db=reinit_db)
-        env._upgrade_metadata()
-        cls._instance = env
-        cls.__initializing = False
+        try:
+            env._set_up(reinit_db=reinit_db)
+            env._upgrade_metadata()
+            cls._instance = env
+        finally:
+            # Reset the initializing flag, even if setup fails.
+            # This prevents the environment from being left in a broken state.
+            cls.__initializing = False
 
     def __init__(self) -> None:
         assert self._instance is None, 'Env is a singleton; use Env.get() to access the instance'
 
         self._media_dir = None  # computed media files
-        self._file_cache_dir = None  # cached media files with external URL
+        self._file_cache_dir = None  # cached object files with external URL
         self._dataset_cache_dir = None  # cached datasets (eg, pytorch or COCO)
         self._log_dir = None  # log files
         self._tmp_dir = None  # any tmp files
@@ -120,6 +144,7 @@ class Env:
         self._spacy_nlp = None
         self._httpd = None
         self._http_address = None
+        self._default_video_encoder = None
 
         # logging-related state
         self._logger = logging.getLogger('pixeltable')
@@ -139,7 +164,34 @@ class Env:
         self._resource_pool_info = {}
         self._current_conn = None
         self._current_session = None
+        self._current_isolation_level = None
         self._dbms = None
+        self._event_loop = None
+
+    def _init_event_loop(self) -> None:
+        try:
+            # check if we are already in an event loop (eg, Jupyter's); if so, patch it to allow
+            # multiple run_until_complete()
+            running_loop = asyncio.get_running_loop()
+            self._event_loop = running_loop
+            _logger.debug('Patched running loop')
+        except RuntimeError:
+            self._event_loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._event_loop)
+            # we set a deliberately long duration to avoid warnings getting printed to the console in debug mode
+            self._event_loop.slow_callback_duration = 3600
+
+        # always allow nested event loops, we need that to run async udfs synchronously (eg, for SimilarityExpr);
+        # see run_coroutine_synchronously()
+        nest_asyncio.apply()
+        if _logger.isEnabledFor(logging.DEBUG):
+            self._event_loop.set_debug(True)
+
+    @property
+    def event_loop(self) -> asyncio.AbstractEventLoop:
+        if self._event_loop is None:
+            self._init_event_loop()
+        return self._event_loop
 
     @property
     def db_url(self) -> str:
@@ -152,11 +204,11 @@ class Env:
         return self._http_address
 
     @property
-    def user(self) -> Optional[str]:
+    def user(self) -> str | None:
         return Config.get().get_string_value('user')
 
     @user.setter
-    def user(self, user: Optional[str]) -> None:
+    def user(self, user: str | None) -> None:
         if user is None:
             if 'PIXELTABLE_USER' in os.environ:
                 del os.environ['PIXELTABLE_USER']
@@ -164,33 +216,47 @@ class Env:
             os.environ['PIXELTABLE_USER'] = user
 
     @property
-    def default_time_zone(self) -> Optional[ZoneInfo]:
+    def default_time_zone(self) -> ZoneInfo | None:
         return self._default_time_zone
 
     @default_time_zone.setter
-    def default_time_zone(self, tz: Optional[ZoneInfo]) -> None:
+    def default_time_zone(self, tz: ZoneInfo | None) -> None:
         """
         This is not a publicly visible setter; it is only for testing purposes.
         """
-        tz_name = None if tz is None else tz.key
+        if tz is None:
+            tz_name = self._get_tz_name()
+        else:
+            assert isinstance(tz, ZoneInfo)
+            tz_name = tz.key
         self.engine.dispose()
         self._create_engine(time_zone_name=tz_name)
 
     @property
-    def conn(self) -> Optional[sql.Connection]:
+    def verbosity(self) -> int:
+        return self._verbosity
+
+    @property
+    def conn(self) -> sql.Connection | None:
         assert self._current_conn is not None
         return self._current_conn
 
     @property
-    def session(self) -> Optional[sql.orm.Session]:
+    def session(self) -> orm.Session | None:
         assert self._current_session is not None
         return self._current_session
 
     @property
-    def dbms(self) -> Optional[Dbms]:
+    def dbms(self) -> Dbms | None:
         assert self._dbms is not None
         return self._dbms
 
+    @property
+    def is_using_cockroachdb(self) -> bool:
+        assert self._dbms is not None
+        return isinstance(self._dbms, CockroachDbms)
+
+    @property
     def in_xact(self) -> bool:
         return self._current_conn is not None
 
@@ -200,32 +266,43 @@ class Env:
         return self._db_server is not None
 
     @contextmanager
-    def begin_xact(self) -> Iterator[sql.Connection]:
-        """Return a context manager that yields a connection to the database. Idempotent."""
+    def begin_xact(self, *, for_write: bool = False) -> Iterator[sql.Connection]:
+        """
+        Call Catalog.begin_xact() instead, unless there is a specific reason to call this directly.
+
+        for_write: if True, uses serializable isolation; if False, uses repeatable_read
+
+        TODO: repeatable read is not available in Cockroachdb; instead, run queries against a snapshot TVP
+        that avoids tripping over any pending ops
+        """
         if self._current_conn is None:
             assert self._current_session is None
             try:
-                with self.engine.begin() as conn, sql.orm.Session(conn) as session:
-                    # TODO: remove print() once we're done with debugging the concurrent update behavior
-                    # print(f'{datetime.datetime.now()}: start xact')
+                self._current_isolation_level = self.SERIALIZABLE_ISOLATION_LEVEL
+                with (
+                    self.engine.connect().execution_options(isolation_level=self._current_isolation_level) as conn,
+                    orm.Session(conn) as session,
+                    conn.begin(),
+                ):
                     self._current_conn = conn
                     self._current_session = session
                     yield conn
             finally:
                 self._current_session = None
                 self._current_conn = None
-                # print(f'{datetime.datetime.now()}: end xact')
+                self._current_isolation_level = None
         else:
             assert self._current_session is not None
+            assert self._current_isolation_level == self.SERIALIZABLE_ISOLATION_LEVEL or not for_write
             yield self._current_conn
 
     def configure_logging(
         self,
         *,
-        to_stdout: Optional[bool] = None,
-        level: Optional[int] = None,
-        add: Optional[str] = None,
-        remove: Optional[str] = None,
+        to_stdout: bool | None = None,
+        level: int | None = None,
+        add: str | None = None,
+        remove: str | None = None,
     ) -> None:
         """Configure logging.
 
@@ -267,7 +344,7 @@ class Env:
     def set_log_level(self, level: int) -> None:
         self._default_log_level = level
 
-    def set_module_log_level(self, module: str, level: Optional[int]) -> None:
+    def set_module_log_level(self, module: str, level: int | None) -> None:
         if level is None:
             self._module_log_level.pop(module, None)
         else:
@@ -282,6 +359,8 @@ class Env:
             # accept log messages from a configured pixeltable module (at any level of the module hierarchy)
             path_parts = list(Path(record.pathname).parts)
             path_parts.reverse()
+            if 'pixeltable' not in path_parts:
+                return False
             max_idx = path_parts.index('pixeltable')
             for module_name in path_parts[:max_idx]:
                 if module_name in self._module_log_level and record.levelno >= self._module_log_level[module_name]:
@@ -292,6 +371,26 @@ class Env:
     def console_logger(self) -> ConsoleLogger:
         return self._console_logger
 
+    def _get_tz_name(self) -> str:
+        """Get the time zone name from the configuration, or the system local time zone if not specified.
+
+        Returns:
+            str: The time zone name.
+        """
+        tz_name = Config.get().get_string_value('time_zone')
+        if tz_name is not None:
+            # Validate tzname
+            if not isinstance(tz_name, str):
+                self._logger.error('Invalid time zone specified in configuration.')
+            else:
+                try:
+                    _ = ZoneInfo(tz_name)
+                except ZoneInfoNotFoundError:
+                    self._logger.error(f'Invalid time zone specified in configuration: {tz_name}')
+        else:
+            tz_name = tzlocal.get_localzone_name()
+        return tz_name
+
     def _set_up(self, echo: bool = False, reinit_db: bool = False) -> None:
         if self._initialized:
             return
@@ -301,22 +400,18 @@ class Env:
         config = Config.get()
 
         self._initialized = True
+
         self._media_dir = Config.get().home / 'media'
         self._file_cache_dir = Config.get().home / 'file_cache'
         self._dataset_cache_dir = Config.get().home / 'dataset_cache'
         self._log_dir = Config.get().home / 'logs'
         self._tmp_dir = Config.get().home / 'tmp'
 
-        if not self._media_dir.exists():
-            self._media_dir.mkdir()
-        if not self._file_cache_dir.exists():
-            self._file_cache_dir.mkdir()
-        if not self._dataset_cache_dir.exists():
-            self._dataset_cache_dir.mkdir()
-        if not self._log_dir.exists():
-            self._log_dir.mkdir()
-        if not self._tmp_dir.exists():
-            self._tmp_dir.mkdir()
+        self._media_dir.mkdir(exist_ok=True)
+        self._file_cache_dir.mkdir(exist_ok=True)
+        self._dataset_cache_dir.mkdir(exist_ok=True)
+        self._log_dir.mkdir(exist_ok=True)
+        self._tmp_dir.mkdir(exist_ok=True)
 
         self._file_cache_size_g = config.get_float_value('file_cache_size_g')
         if self._file_cache_size_g is None:
@@ -325,6 +420,16 @@ class Env:
                 f'(either add a `file_cache_size_g` entry to the `pixeltable` section of {Config.get().config_file},\n'
                 'or set the PIXELTABLE_FILE_CACHE_SIZE_G environment variable)'
             )
+
+        self._default_input_media_dest = config.get_string_value('input_media_dest')
+        self._default_output_media_dest = config.get_string_value('output_media_dest')
+        for mode, uri in (('input', self._default_input_media_dest), ('output', self._default_output_media_dest)):
+            if uri is not None:
+                try:
+                    _ = ObjectPath.parse_object_storage_addr(uri, False)
+                except Exception as e:
+                    raise excs.Error(f'Invalid {mode} media destination URI: {uri}') from e
+
         self._pxt_api_key = config.get_string_value('api_key')
 
         # Disable spurious warnings
@@ -334,10 +439,12 @@ class Env:
             warnings.simplefilter('ignore', category=UserWarning)
             warnings.simplefilter('ignore', category=FutureWarning)
 
-        # Set verbose level for user visible console messages
-        verbosity = map_level(config.get_int_value('verbosity'))
+        # Set verbosity level for user visible console messages
+        self._verbosity = config.get_int_value('verbosity')
+        if self._verbosity is None:
+            self._verbosity = 1
         stdout_handler = ConsoleOutputHandler(stream=stdout)
-        stdout_handler.setLevel(verbosity)
+        stdout_handler.setLevel(map_level(self._verbosity))
         stdout_handler.addFilter(ConsoleMessageFilter())
         self._logger.addHandler(stdout_handler)
         self._console_logger = ConsoleLogger(self._logger)
@@ -371,6 +478,7 @@ class Env:
         http_logger.propagate = False
 
         self.clear_tmp_dir()
+        tz_name = self._get_tz_name()
 
         # configure pixeltable database
         self._init_db(config)
@@ -380,22 +488,10 @@ class Env:
                 'Reinitializing pixeltable database is not supported when running in non-local environment'
             )
 
-        tz_name = config.get_string_value('time_zone')
-        if tz_name is not None:
-            # Validate tzname
-            if not isinstance(tz_name, str):
-                self._logger.error('Invalid time zone specified in configuration.')
-            else:
-                try:
-                    _ = ZoneInfo(tz_name)
-                except ZoneInfoNotFoundError:
-                    self._logger.error(f'Invalid time zone specified in configuration: {tz_name}')
-
         if reinit_db and self._store_db_exists():
             self._drop_store_db()
 
         create_db = not self._store_db_exists()
-
         if create_db:
             self._logger.info(f'creating database at: {self.db_url}')
             self._create_store_db()
@@ -440,7 +536,7 @@ class Env:
                 raise excs.Error(error)
             self._logger.info(f'Using database at: {self.db_url}')
         else:
-            self._db_name = os.environ.get('PIXELTABLE_DB', 'pixeltable')
+            self._db_name = config.get_string_value('db') or 'pixeltable'
             self._pgdata_dir = Path(os.environ.get('PIXELTABLE_PGDATA', str(Config.get().home / 'pgdata')))
             # cleanup_mode=None will leave the postgres process running after Python exits
             # cleanup_mode='stop' will terminate the postgres process when Python exits
@@ -454,30 +550,50 @@ class Env:
         assert self._db_url is not None
         assert self._db_name is not None
 
+    @retry(
+        stop=stop_after_attempt(3),  # Stop after 3 attempts
+        wait=wait_exponential_jitter(initial=0.2, max=1.0, jitter=0.2),  # Exponential backoff with jitter
+    )
     def _init_metadata(self) -> None:
         """
         Create pixeltable metadata tables and system metadata.
         This is an idempotent operation.
+
+        Retry logic handles race conditions when multiple Pixeltable processes
+        attempt to initialize metadata tables simultaneously. The first process may succeed
+        in creating tables while others encounter database constraints (e.g., "table already exists").
+        Exponential backoff with jitter reduces contention between competing processes.
         """
         assert self._sa_engine is not None
         from pixeltable import metadata
 
+        self._logger.debug('Creating pixeltable metadata')
         metadata.schema.base_metadata.create_all(self._sa_engine, checkfirst=True)
         metadata.create_system_info(self._sa_engine)
 
-    def _create_engine(self, time_zone_name: Optional[str], echo: bool = False) -> None:
-        connect_args = {} if time_zone_name is None else {'options': f'-c timezone={time_zone_name}'}
+    def _create_engine(self, time_zone_name: str, echo: bool = False) -> None:
+        # Add timezone option to connection string
+        updated_url = add_option_to_db_url(self.db_url, f'-c timezone={time_zone_name}')
+
         self._sa_engine = sql.create_engine(
-            self.db_url, echo=echo, isolation_level=self._dbms.transaction_isolation_level, connect_args=connect_args
+            updated_url, echo=echo, isolation_level=self._dbms.transaction_isolation_level
         )
 
         self._logger.info(f'Created SQLAlchemy engine at: {self.db_url}')
+        self._logger.info(f'Engine dialect: {self._sa_engine.dialect.name}')
+        self._logger.info(f'Engine driver : {self._sa_engine.dialect.driver}')
 
         with self.engine.begin() as conn:
             tz_name = conn.execute(sql.text('SHOW TIME ZONE')).scalar()
             assert isinstance(tz_name, str)
             self._logger.info(f'Database time zone is now: {tz_name}')
             self._default_time_zone = ZoneInfo(tz_name)
+            if self.is_using_cockroachdb:
+                # This could be set when the database is created, but we set it now
+                conn.execute(sql.text('SET null_ordered_last = true;'))
+                null_ordered_last = conn.execute(sql.text('SHOW null_ordered_last')).scalar()
+                assert isinstance(null_ordered_last, str)
+                self._logger.info(f'Database null_ordered_last is now: {null_ordered_last}')
 
     def _store_db_exists(self) -> bool:
         assert self._db_name is not None
@@ -512,6 +628,14 @@ class Env:
         finally:
             engine.dispose()
 
+    def _pgserver_terminate_connections_stmt(self) -> str:
+        return f"""
+                SELECT pg_terminate_backend(pg_stat_activity.pid)
+                FROM pg_stat_activity
+                WHERE pg_stat_activity.datname = '{self._db_name}'
+                AND pid <> pg_backend_pid()
+            """
+
     def _drop_store_db(self) -> None:
         assert self._db_name is not None
         engine = sql.create_engine(self._dbms.default_system_db_url(), future=True, isolation_level='AUTOCOMMIT')
@@ -520,13 +644,7 @@ class Env:
             with engine.begin() as conn:
                 # terminate active connections
                 if self._db_server is not None:
-                    stmt = f"""
-                        SELECT pg_terminate_backend(pg_stat_activity.pid)
-                        FROM pg_stat_activity
-                        WHERE pg_stat_activity.datname = '{self._db_name}'
-                        AND pid <> pg_backend_pid()
-                    """
-                    conn.execute(sql.text(stmt))
+                    conn.execute(sql.text(self._pgserver_terminate_connections_stmt()))
                 # drop db
                 stmt = self._dbms.drop_db_stmt(preparer.quote(self._db_name))
                 conn.execute(sql.text(stmt))
@@ -539,12 +657,7 @@ class Env:
         metadata.upgrade_md(self._sa_engine)
 
     @property
-    def pxt_api_key(self) -> str:
-        if self._pxt_api_key is None:
-            raise excs.Error(
-                'No API key is configured. Set the PIXELTABLE_API_KEY environment variable, or add an entry to '
-                'config.toml as described here:\nhttps://pixeltable.github.io/pixeltable/config/'
-            )
+    def pxt_api_key(self) -> str | None:
         return self._pxt_api_key
 
     def get_client(self, name: str) -> Any:
@@ -554,35 +667,51 @@ class Env:
         Args:
             - name: The name of the client
         """
-        cl = _registered_clients[name]
-        if cl.client_obj is not None:
-            return cl.client_obj  # Already initialized
-
-        # Construct a client, retrieving each parameter from config.
-
-        init_kwargs: dict[str, str] = {}
-        for param in cl.param_names:
-            arg = Config.get().get_string_value(param, section=name)
-            if arg is not None and len(arg) > 0:
-                init_kwargs[param] = arg
-            else:
+        # Return the existing client if it has already been constructed
+        with _registered_clients_lock:
+            cl = _registered_clients[name]
+            if cl.client_obj is not None:
+                return cl.client_obj  # Already initialized
+
+        # Retrieve parameters required to construct the requested client.
+        init_kwargs: dict[str, Any] = {}
+        for param in cl.params.values():
+            # Determine the type of the parameter for proper config parsing.
+            pname = param.name
+            t = param.annotation
+            # Deference T | None
+            if typing.get_origin(t) in (typing.Union, types.UnionType):
+                args = typing.get_args(t)
+                if args[0] is type(None):
+                    t = args[1]
+                elif args[1] is type(None):
+                    t = args[0]
+            assert isinstance(t, type), t
+            arg: Any = Config.get().get_value(pname, t, section=name)
+            if arg is not None:
+                init_kwargs[pname] = arg
+            elif param.default is inspect.Parameter.empty:
                 raise excs.Error(
-                    f'`{name}` client not initialized: parameter `{param}` is not configured.\n'
-                    f'To fix this, specify the `{name.upper()}_{param.upper()}` environment variable, '
-                    f'or put `{param.lower()}` in the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
+                    f'`{name}` client not initialized: parameter `{pname}` is not configured.\n'
+                    f'To fix this, specify the `{name.upper()}_{pname.upper()}` environment variable, '
+                    f'or put `{pname.lower()}` in the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
                 )
 
-        cl.client_obj = cl.init_fn(**init_kwargs)
-        self._logger.info(f'Initialized `{name}` client.')
-        return cl.client_obj
+        # Construct the requested client
+        with _registered_clients_lock:
+            if cl.client_obj is not None:
+                return cl.client_obj  # Already initialized
+            cl.client_obj = cl.init_fn(**init_kwargs)
+            self._logger.info(f'Initialized `{name}` client with parameters: {init_kwargs}.')
+            return cl.client_obj
 
     def _start_web_server(self) -> None:
         """
         The http server root is the file system root.
         eg: /home/media/foo.mp4 is located at http://127.0.0.1:{port}/home/media/foo.mp4
-        in windows, the server will translate paths like http://127.0.0.1:{port}/c:/media/foo.mp4
-        This arrangement enables serving media hosted within _home,
-        as well as external media inserted into pixeltable or produced by pixeltable.
+        On Windows, the server will translate paths like http://127.0.0.1:{port}/c:/media/foo.mp4
+        This arrangement enables serving objects hosted within _home,
+        as well as external objects inserted into pixeltable or produced by pixeltable.
         The port is chosen dynamically to prevent conflicts.
         """
         # Port 0 means OS picks one for us.
@@ -600,20 +729,65 @@ class Env:
 
     def _set_up_runtime(self) -> None:
         """Check for and start runtime services"""
+        register_heif_opener()
         self._start_web_server()
         self.__register_packages()
 
+    @property
+    def default_video_encoder(self) -> str | None:
+        if self._default_video_encoder is None:
+            self._default_video_encoder = self._determine_default_video_encoder()
+        return self._default_video_encoder
+
+    def _determine_default_video_encoder(self) -> str | None:
+        """
+        Returns the first available encoder from a list of candidates.
+
+        TODO:
+        - the user might prefer a hardware-accelerated encoder (eg, h264_nvenc or h264_videotoolbox)
+        - allow user override via a config option 'video_encoder'
+        """
+        # look for available encoders, in this order
+        candidates = [
+            'libx264',  # GPL, best quality
+            'libopenh264',  # BSD
+        ]
+
+        try:
+            # Get list of available encoders
+            result = subprocess.run(['ffmpeg', '-encoders'], capture_output=True, text=True, timeout=10, check=True)
+
+            if result.returncode == 0:
+                available_encoders = result.stdout
+                for encoder in candidates:
+                    # ffmpeg -encoders output format: " V..... encoder_name  description"
+                    if f' {encoder} ' in available_encoders:
+                        _logger.debug(f'Using H.264 encoder: {encoder}')
+                        return encoder
+        except Exception:
+            pass
+        return None
+
     def __register_packages(self) -> None:
         """Declare optional packages that are utilized by some parts of the code."""
+        self.__register_package('accelerate')
         self.__register_package('anthropic')
+        self.__register_package('azure.storage.blob', library_name='azure-storage-blob')
         self.__register_package('boto3')
         self.__register_package('datasets')
+        self.__register_package('diffusers')
         self.__register_package('fiftyone')
+        self.__register_package('twelvelabs')
+        self.__register_package('fal_client', library_name='fal-client')
         self.__register_package('fireworks', library_name='fireworks-ai')
+        self.__register_package('google.cloud.storage', library_name='google-cloud-storage')
         self.__register_package('google.genai', library_name='google-genai')
+        self.__register_package('groq')
         self.__register_package('huggingface_hub', library_name='huggingface-hub')
         self.__register_package('label_studio_sdk', library_name='label-studio-sdk')
+        self.__register_package('librosa')
         self.__register_package('llama_cpp', library_name='llama-cpp-python')
+        self.__register_package('mcp')
         self.__register_package('mistralai')
         self.__register_package('mistune')
         self.__register_package('ollama')
@@ -622,8 +796,10 @@ class Env:
         self.__register_package('pyarrow')
         self.__register_package('pydantic')
         self.__register_package('replicate')
+        self.__register_package('reve')
         self.__register_package('sentencepiece')
         self.__register_package('sentence_transformers', library_name='sentence-transformers')
+        self.__register_package('soundfile')
         self.__register_package('spacy')
         self.__register_package('tiktoken')
         self.__register_package('together')
@@ -631,11 +807,14 @@ class Env:
         self.__register_package('torchaudio')
         self.__register_package('torchvision')
         self.__register_package('transformers')
+        self.__register_package('voyageai')
         self.__register_package('whisper', library_name='openai-whisper')
         self.__register_package('whisperx')
         self.__register_package('yolox', library_name='pixeltable-yolox')
+        self.__register_package('lancedb')
+        self.__register_package('scenedetect')
 
-    def __register_package(self, package_name: str, library_name: Optional[str] = None) -> None:
+    def __register_package(self, package_name: str, library_name: str | None = None) -> None:
         is_installed: bool
         try:
             is_installed = importlib.util.find_spec(package_name) is not None
@@ -647,7 +826,11 @@ class Env:
             library_name=library_name or package_name,  # defaults to package_name unless specified otherwise
         )
 
-    def require_package(self, package_name: str, min_version: Optional[list[int]] = None) -> None:
+    def require_binary(self, binary_name: str) -> None:
+        if not shutil.which(binary_name):
+            raise excs.Error(f'{binary_name} is not installed or not in PATH. Please install it to use this feature.')
+
+    def require_package(self, package_name: str, min_version: list[int] | None = None) -> None:
         """
         Checks whether the specified optional package is available. If not, raises an exception
         with an error message informing the user how to install it.
@@ -691,14 +874,8 @@ class Env:
             else:
                 os.remove(path)
 
-    def num_tmp_files(self) -> int:
-        return len(glob.glob(f'{self._tmp_dir}/*'))
-
-    def create_tmp_path(self, extension: str = '') -> Path:
-        return self._tmp_dir / f'{uuid.uuid4()}{extension}'
-
-    # def get_resource_pool_info(self, pool_id: str, pool_info_cls: Optional[Type[T]]) -> T:
-    def get_resource_pool_info(self, pool_id: str, make_pool_info: Optional[Callable[[], T]] = None) -> T:
+    # def get_resource_pool_info(self, pool_id: str, pool_info_cls: Type[T] | None) -> T:
+    def get_resource_pool_info(self, pool_id: str, make_pool_info: Callable[[], T] | None = None) -> T:
         """Returns the info object for the given id, creating it if necessary."""
         info = self._resource_pool_info.get(pool_id)
         if info is None and make_pool_info is not None:
@@ -711,6 +888,14 @@ class Env:
         assert self._media_dir is not None
         return self._media_dir
 
+    @property
+    def default_input_media_dest(self) -> str | None:
+        return self._default_input_media_dest
+
+    @property
+    def default_output_media_dest(self) -> str | None:
+        return self._default_output_media_dest
+
     @property
     def file_cache_dir(self) -> Path:
         assert self._file_cache_dir is not None
@@ -746,24 +931,74 @@ class Env:
         have no sub-dependencies (in fact, this is how spaCy normally manages its model resources).
         """
         import spacy
-        from spacy.cli.download import get_model_filename
+        from spacy.cli.download import download
 
         spacy_model = 'en_core_web_sm'
-        spacy_model_version = '3.7.1'
-        filename = get_model_filename(spacy_model, spacy_model_version, sdist=False)
-        url = f'{spacy.about.__download_url__}/{filename}'
-        # Try to `pip install` the model. We set check=False; if the pip command fails, it's not necessarily
-        # a problem, because the model might have been installed on a previous attempt.
-        self._logger.info(f'Ensuring spaCy model is installed: {filename}')
-        ret = subprocess.run([sys.executable, '-m', 'pip', 'install', '-qU', url], check=False)
-        if ret.returncode != 0:
-            self._logger.warning(f'pip install failed for spaCy model: {filename}')
+        self._logger.info(f'Ensuring spaCy model is installed: {spacy_model}')
+        download(spacy_model)
         self._logger.info(f'Loading spaCy model: {spacy_model}')
         try:
             self._spacy_nlp = spacy.load(spacy_model)
         except Exception as exc:
             raise excs.Error(f'Failed to load spaCy model: {spacy_model}') from exc
 
+    def _clean_up(self) -> None:
+        """
+        Internal cleanup method that properly closes all resources and resets state.
+        This is called before destroying the singleton instance.
+        """
+        assert self._current_session is None
+        assert self._current_conn is None
+
+        # Stop HTTP server
+        if self._httpd is not None:
+            try:
+                self._httpd.shutdown()
+                self._httpd.server_close()
+            except Exception as e:
+                _logger.warning(f'Error stopping HTTP server: {e}')
+
+        # First terminate all connections to the database
+        if self._db_server is not None:
+            assert self._dbms is not None
+            assert self._db_name is not None
+            try:
+                temp_engine = sql.create_engine(self._dbms.default_system_db_url(), isolation_level='AUTOCOMMIT')
+                try:
+                    with temp_engine.begin() as conn:
+                        conn.execute(sql.text(self._pgserver_terminate_connections_stmt()))
+                        _logger.info(f"Terminated all connections to database '{self._db_name}'")
+                except Exception as e:
+                    _logger.warning(f'Error terminating database connections: {e}')
+                finally:
+                    temp_engine.dispose()
+            except Exception as e:
+                _logger.warning(f'Error stopping database server: {e}')
+
+        # Dispose of SQLAlchemy engine (after stopping db server)
+        if self._sa_engine is not None:
+            try:
+                self._sa_engine.dispose()
+            except Exception as e:
+                _logger.warning(f'Error disposing engine: {e}')
+
+        # Close event loop
+        if self._event_loop is not None:
+            try:
+                if self._event_loop.is_running():
+                    self._event_loop.stop()
+                self._event_loop.close()
+            except Exception as e:
+                _logger.warning(f'Error closing event loop: {e}')
+
+        # Remove logging handlers
+        for handler in self._logger.handlers[:]:
+            try:
+                handler.close()
+                self._logger.removeHandler(handler)
+            except Exception as e:
+                _logger.warning(f'Error removing handler: {e}')
+
 
 def register_client(name: str) -> Callable:
     """Decorator that registers a third-party API client for use by Pixeltable.
@@ -792,30 +1027,35 @@ def register_client(name: str) -> Callable:
 
     def decorator(fn: Callable) -> None:
         sig = inspect.signature(fn)
-        param_names = list(sig.parameters.keys())
-        _registered_clients[name] = ApiClient(init_fn=fn, param_names=param_names)
+        params = dict(sig.parameters)
+        with _registered_clients_lock:
+            _registered_clients[name] = ApiClient(init_fn=fn, params=params)
 
     return decorator
 
 
+_registered_clients_lock: threading.Lock = threading.Lock()
 _registered_clients: dict[str, ApiClient] = {}
 
 
 @dataclass
 class ApiClient:
     init_fn: Callable
-    param_names: list[str]
-    client_obj: Optional[Any] = None
+    params: dict[str, inspect.Parameter]
+    client_obj: Any | None = None
 
 
 @dataclass
 class PackageInfo:
     is_installed: bool
     library_name: str  # pypi library name (may be different from package name)
-    version: Optional[list[int]] = None  # installed version, as a list of components (such as [3,0,2] for "3.0.2")
+    version: list[int] | None = None  # installed version, as a list of components (such as [3,0,2] for "3.0.2")
 
 
 TIME_FORMAT = '%H:%M.%S %f'
+# As far as rate limiting goes, we try not go lower than 5% of the capacity because we don't have perfect information
+# about the rate limits and the usage
+TARGET_RATE_LIMIT_RESOURCE_FRACT = 0.05
 
 
 @dataclass
@@ -838,6 +1078,10 @@ class RateLimitsInfo:
     get_request_resources: Callable[..., dict[str, int]]
 
     resource_limits: dict[str, RateLimitInfo] = field(default_factory=dict)
+    has_exc: bool = False
+
+    def debug_str(self) -> str:
+        return ','.join(info.debug_str() for info in self.resource_limits.values())
 
     def is_initialized(self) -> bool:
         return len(self.resource_limits) > 0
@@ -845,25 +1089,46 @@ class RateLimitsInfo:
     def reset(self) -> None:
         self.resource_limits.clear()
 
-    def record(self, **kwargs: Any) -> None:
-        now = datetime.datetime.now(tz=datetime.timezone.utc)
+    def record(self, request_ts: datetime.datetime, reset_exc: bool = False, **kwargs: Any) -> None:
+        """Update self.resource_limits with the provided rate limit info.
+        Args:
+            - request_ts: time at which the request was made
+            - reset_exc: if True, reset the has_exc flag
+        """
         if len(self.resource_limits) == 0:
-            self.resource_limits = {k: RateLimitInfo(k, now, *v) for k, v in kwargs.items() if v is not None}
+            self.resource_limits = {k: RateLimitInfo(k, request_ts, *v) for k, v in kwargs.items() if v is not None}
             # TODO: remove
             for info in self.resource_limits.values():
-                _logger.debug(
-                    f'Init {info.resource} rate limit: rem={info.remaining} '
-                    f'reset={info.reset_at.strftime(TIME_FORMAT)} delta={(info.reset_at - now).total_seconds()}'
-                )
+                _logger.debug(f'Updated resource state: {info}')
         else:
+            if self.has_exc and not reset_exc:
+                # ignore updates until we're asked to reset
+                _logger.debug(f'rate_limits.record(): ignoring update {kwargs}')
+                return
+            self.has_exc = False
             for k, v in kwargs.items():
                 if v is not None:
-                    self.resource_limits[k].update(now, *v)
+                    self.resource_limits[k].update(request_ts, *v)
+                    _logger.debug(f'Updated resource state: {self.resource_limits[k]}')
 
-    @abstractmethod
-    def get_retry_delay(self, exc: Exception) -> Optional[float]:
+    def record_exc(self, request_ts: datetime.datetime, exc: Exception) -> None:
+        """Update self.resource_limits based on the exception headers
+        Args:
+            - request_ts: time at which the request that caused the exception was made
+            - exc: the exception raised"""
+        self.has_exc = True
+
+    def get_retry_delay(self, exc: Exception, attempt: int) -> float | None:
         """Returns number of seconds to wait before retry, or None if not retryable"""
-        pass
+        # Find the highest wait until at least 5% availability of all resources
+        max_wait = 0.0
+        for limit_info in self.resource_limits.values():
+            time_until = limit_info.estimated_resource_refill_delay(
+                math.ceil(TARGET_RATE_LIMIT_RESOURCE_FRACT * limit_info.limit)
+            )
+            if time_until is not None:
+                max_wait = max(max_wait, time_until)
+        return max_wait if max_wait > 0 else None
 
 
 @dataclass
@@ -871,22 +1136,71 @@ class RateLimitInfo:
     """Container for rate limit-related information for a single resource."""
 
     resource: str
-    recorded_at: datetime.datetime
+    request_start_ts: datetime.datetime
     limit: int
     remaining: int
     reset_at: datetime.datetime
 
-    def update(self, recorded_at: datetime.datetime, limit: int, remaining: int, reset_at: datetime.datetime) -> None:
-        # we always update everything, even though responses may come back out-of-order: we can't use reset_at to
-        # determine order, because it doesn't increase monotonically (the reeset duration shortens as output_tokens
-        # are freed up - going from max to actual)
-        self.recorded_at = recorded_at
+    def debug_str(self) -> str:
+        return (
+            f'{self.resource}@{self.request_start_ts.strftime(TIME_FORMAT)}: '
+            f'{self.limit}/{self.remaining}/{self.reset_at.strftime(TIME_FORMAT)}'
+        )
+
+    def update(
+        self, request_start_ts: datetime.datetime, limit: int, remaining: int, reset_at: datetime.datetime
+    ) -> None:
+        # Responses can come out of order, especially for failed requests. We need to be careful not to overwrite
+        # the current state with less up-to-date information. We use request_start_ts as a proxy for rate limit info
+        # recency.
+        if self.request_start_ts > request_start_ts:
+            # The current state is more up-to-date than the update
+            _logger.debug(
+                f'Ignoring out-of-date update for {self.resource}. Current request_start_ts: '
+                f'{self.request_start_ts}, update: {request_start_ts}'
+            )
+            return
+        self.request_start_ts = request_start_ts
         self.limit = limit
         self.remaining = remaining
-        reset_delta = reset_at - self.reset_at
         self.reset_at = reset_at
-        # TODO: remove
-        _logger.debug(
-            f'Update {self.resource} rate limit: rem={self.remaining} reset={self.reset_at.strftime(TIME_FORMAT)} '
-            f'reset_delta={reset_delta.total_seconds()} recorded_delta={(self.reset_at - recorded_at).total_seconds()}'
+
+    def estimated_resource_refill_delay(self, target_remaining: int) -> float | None:
+        """Estimate time in seconds until remaining resources reaches target_remaining.
+        Assumes linear replenishment of resources over time.
+        Returns None if unable to estimate.
+        """
+        if self.remaining >= target_remaining:
+            return 0
+        if self.request_start_ts >= self.reset_at:
+            return 0
+        if self.limit < target_remaining:
+            return None
+
+        # Estimate resource refill rate based on the recorded state and timestamps. Assumes linear refill.
+        refill_rate = (self.limit - self.remaining) / (self.reset_at - self.request_start_ts).total_seconds()
+        assert refill_rate > 0, f'self={self}, target_remaining={target_remaining}'
+
+        now = datetime.datetime.now(tz=datetime.timezone.utc)
+        time_until = (target_remaining - self.remaining) / refill_rate - (now - self.request_start_ts).total_seconds()
+        return max(0, math.ceil(time_until))
+
+    def __repr__(self) -> str:
+        return (
+            f'RateLimitInfo(resource={self.resource}, request_start_ts={self.request_start_ts}, '
+            f'remaining={self.remaining}/{self.limit} ({(100 * self.remaining / self.limit):.1f}%), '
+            f'reset_at={self.reset_at})'
         )
+
+
+@dataclass
+class RuntimeCtx:
+    """
+    Container for runtime data provided by the execution system to udfs.
+
+    Udfs that accept the special _runtime_ctx parameter receive an instance of this class.
+    """
+
+    # Indicates a retry attempt following a rate limit error (error code: 429). Requires a 'rate-limits' resource pool.
+    # If True, call RateLimitsInfo.record() with reset_exc=True.
+    is_retry: bool = False