pixeltable 0.4.18__py3-none-any.whl → 0.4.19__py3-none-any.whl

pixeltable/env.py

@@ -21,7 +21,7 @@ from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
 from sys import stdout
-from typing import TYPE_CHECKING, Any, Callable, Iterator, Literal, Optional, TypeVar
+from typing import TYPE_CHECKING, Any, Callable, Iterator, TypeVar
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 import nest_asyncio  # type: ignore[import-untyped]
@@ -38,7 +38,7 @@ from pixeltable.config import Config
 from pixeltable.utils.console_output import ConsoleLogger, ConsoleMessageFilter, ConsoleOutputHandler, map_level
 from pixeltable.utils.dbms import CockroachDbms, Dbms, PostgresqlDbms
 from pixeltable.utils.http_server import make_server
-from pixeltable.utils.object_stores import ObjectPath, StorageObjectAddress
+from pixeltable.utils.object_stores import ObjectPath
 
 if TYPE_CHECKING:
     import spacy
@@ -56,47 +56,50 @@ class Env:
     For a non-local environment, Pixeltable uses a connection string to the externally managed database.
     """
 
-    _instance: Optional[Env] = None
+    SERIALIZABLE_ISOLATION_LEVEL = 'SERIALIZABLE'
+
+    _instance: Env | None = None
     __initializing: bool = False
     _log_fmt_str = '%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d: %(message)s'
 
-    _media_dir: Optional[Path]
-    _object_soa: Optional[StorageObjectAddress]
-    _file_cache_dir: Optional[Path]  # cached object files with external URL
-    _dataset_cache_dir: Optional[Path]  # cached datasets (eg, pytorch or COCO)
-    _log_dir: Optional[Path]  # log files
-    _tmp_dir: Optional[Path]  # any tmp files
-    _sa_engine: Optional[sql.engine.base.Engine]
-    _pgdata_dir: Optional[Path]
-    _db_name: Optional[str]
-    _db_server: Optional[pixeltable_pgserver.PostgresServer]  # set only when running in local environment
-    _db_url: Optional[str]
-    _default_time_zone: Optional[ZoneInfo]
+    _media_dir: Path | None
+    _file_cache_dir: Path | None  # cached object files with external URL
+    _dataset_cache_dir: Path | None  # cached datasets (eg, pytorch or COCO)
+    _log_dir: Path | None  # log files
+    _tmp_dir: Path | None  # any tmp files
+    _sa_engine: sql.engine.base.Engine | None
+    _pgdata_dir: Path | None
+    _db_name: str | None
+    _db_server: pixeltable_pgserver.PostgresServer | None  # set only when running in local environment
+    _db_url: str | None
+    _default_time_zone: ZoneInfo | None
     _verbosity: int
 
     # info about optional packages that are utilized by some parts of the code
     __optional_packages: dict[str, PackageInfo]
 
-    _spacy_nlp: Optional[spacy.Language]
-    _httpd: Optional[http.server.HTTPServer]
-    _http_address: Optional[str]
+    _spacy_nlp: spacy.Language | None
+    _httpd: http.server.HTTPServer | None
+    _http_address: str | None
     _logger: logging.Logger
     _default_log_level: int
-    _logfilename: Optional[str]
+    _logfilename: str | None
     _log_to_stdout: bool
     _module_log_level: dict[str, int]  # module name -> log level
     _file_cache_size_g: float
-    _pxt_api_key: Optional[str]
+    _default_input_media_dest: str | None
+    _default_output_media_dest: str | None
+    _pxt_api_key: str | None
     _stdout_handler: logging.StreamHandler
     _default_video_encoder: str | None
     _initialized: bool
 
     _resource_pool_info: dict[str, Any]
-    _current_conn: Optional[sql.Connection]
-    _current_session: Optional[orm.Session]
-    _current_isolation_level: Optional[Literal['REPEATABLE_READ', 'SERIALIZABLE']]
-    _dbms: Optional[Dbms]
-    _event_loop: Optional[asyncio.AbstractEventLoop]  # event loop for ExecNode
+    _current_conn: sql.Connection | None
+    _current_session: orm.Session | None
+    _current_isolation_level: str | None
+    _dbms: Dbms | None
+    _event_loop: asyncio.AbstractEventLoop | None  # event loop for ExecNode
 
     @classmethod
     def get(cls) -> Env:
@@ -125,7 +128,6 @@ class Env:
         assert self._instance is None, 'Env is a singleton; use Env.get() to access the instance'
 
         self._media_dir = None  # computed media files
-        self._object_soa = None  # computed object files in StorageObjectAddress format
         self._file_cache_dir = None  # cached object files with external URL
         self._dataset_cache_dir = None  # cached datasets (eg, pytorch or COCO)
         self._log_dir = None  # log files
@@ -200,11 +202,11 @@ class Env:
         return self._http_address
 
     @property
-    def user(self) -> Optional[str]:
+    def user(self) -> str | None:
         return Config.get().get_string_value('user')
 
     @user.setter
-    def user(self, user: Optional[str]) -> None:
+    def user(self, user: str | None) -> None:
         if user is None:
             if 'PIXELTABLE_USER' in os.environ:
                 del os.environ['PIXELTABLE_USER']
@@ -212,11 +214,11 @@ class Env:
             os.environ['PIXELTABLE_USER'] = user
 
     @property
-    def default_time_zone(self) -> Optional[ZoneInfo]:
+    def default_time_zone(self) -> ZoneInfo | None:
         return self._default_time_zone
 
     @default_time_zone.setter
-    def default_time_zone(self, tz: Optional[ZoneInfo]) -> None:
+    def default_time_zone(self, tz: ZoneInfo | None) -> None:
         """
         This is not a publicly visible setter; it is only for testing purposes.
         """
@@ -233,17 +235,17 @@ class Env:
         return self._verbosity
 
     @property
-    def conn(self) -> Optional[sql.Connection]:
+    def conn(self) -> sql.Connection | None:
         assert self._current_conn is not None
         return self._current_conn
 
     @property
-    def session(self) -> Optional[orm.Session]:
+    def session(self) -> orm.Session | None:
         assert self._current_session is not None
         return self._current_session
 
     @property
-    def dbms(self) -> Optional[Dbms]:
+    def dbms(self) -> Dbms | None:
         assert self._dbms is not None
         return self._dbms
 
@@ -274,7 +276,7 @@ class Env:
         if self._current_conn is None:
             assert self._current_session is None
             try:
-                self._current_isolation_level = 'SERIALIZABLE'
+                self._current_isolation_level = self.SERIALIZABLE_ISOLATION_LEVEL
                 with (
                     self.engine.connect().execution_options(isolation_level=self._current_isolation_level) as conn,
                     orm.Session(conn) as session,
@@ -289,16 +291,16 @@ class Env:
                 self._current_isolation_level = None
         else:
             assert self._current_session is not None
-            assert for_write == (self._current_isolation_level == 'serializable')
+            assert self._current_isolation_level == self.SERIALIZABLE_ISOLATION_LEVEL or not for_write
             yield self._current_conn
 
     def configure_logging(
         self,
         *,
-        to_stdout: Optional[bool] = None,
-        level: Optional[int] = None,
-        add: Optional[str] = None,
-        remove: Optional[str] = None,
+        to_stdout: bool | None = None,
+        level: int | None = None,
+        add: str | None = None,
+        remove: str | None = None,
     ) -> None:
         """Configure logging.
 
@@ -340,7 +342,7 @@ class Env:
     def set_log_level(self, level: int) -> None:
         self._default_log_level = level
 
-    def set_module_log_level(self, module: str, level: Optional[int]) -> None:
+    def set_module_log_level(self, module: str, level: int | None) -> None:
         if level is None:
             self._module_log_level.pop(module, None)
         else:
@@ -396,23 +398,18 @@ class Env:
         config = Config.get()
 
         self._initialized = True
+
         self._media_dir = Config.get().home / 'media'
         self._file_cache_dir = Config.get().home / 'file_cache'
         self._dataset_cache_dir = Config.get().home / 'dataset_cache'
         self._log_dir = Config.get().home / 'logs'
         self._tmp_dir = Config.get().home / 'tmp'
 
-        if not self._media_dir.exists():
-            self._media_dir.mkdir()
-        self._object_soa = ObjectPath.parse_object_storage_addr(str(self._media_dir), may_contain_object_name=False)
-        if not self._file_cache_dir.exists():
-            self._file_cache_dir.mkdir()
-        if not self._dataset_cache_dir.exists():
-            self._dataset_cache_dir.mkdir()
-        if not self._log_dir.exists():
-            self._log_dir.mkdir()
-        if not self._tmp_dir.exists():
-            self._tmp_dir.mkdir()
+        self._media_dir.mkdir(exist_ok=True)
+        self._file_cache_dir.mkdir(exist_ok=True)
+        self._dataset_cache_dir.mkdir(exist_ok=True)
+        self._log_dir.mkdir(exist_ok=True)
+        self._tmp_dir.mkdir(exist_ok=True)
 
         self._file_cache_size_g = config.get_float_value('file_cache_size_g')
         if self._file_cache_size_g is None:
@@ -421,6 +418,16 @@ class Env:
                 f'(either add a `file_cache_size_g` entry to the `pixeltable` section of {Config.get().config_file},\n'
                 'or set the PIXELTABLE_FILE_CACHE_SIZE_G environment variable)'
             )
+
+        self._default_input_media_dest = config.get_string_value('input_media_dest')
+        self._default_output_media_dest = config.get_string_value('output_media_dest')
+        for mode, uri in (('input', self._default_input_media_dest), ('output', self._default_output_media_dest)):
+            if uri is not None:
+                try:
+                    _ = ObjectPath.parse_object_storage_addr(uri, False)
+                except Exception as e:
+                    raise excs.Error(f'Invalid {mode} media destination URI: {uri}') from e
+
         self._pxt_api_key = config.get_string_value('api_key')
 
         # Disable spurious warnings
@@ -647,7 +654,7 @@ class Env:
         metadata.upgrade_md(self._sa_engine)
 
     @property
-    def pxt_api_key(self) -> Optional[str]:
+    def pxt_api_key(self) -> str | None:
         return self._pxt_api_key
 
     def get_client(self, name: str) -> Any:
@@ -669,7 +676,7 @@ class Env:
             # Determine the type of the parameter for proper config parsing.
             pname = param.name
             t = param.annotation
-            # Deference Optional[T]
+            # Deference T | None
             if typing.get_origin(t) in (typing.Union, types.UnionType):
                 args = typing.get_args(t)
                 if args[0] is type(None):
@@ -767,6 +774,7 @@ class Env:
         self.__register_package('datasets')
         self.__register_package('diffusers')
         self.__register_package('fiftyone')
+        self.__register_package('twelvelabs')
         self.__register_package('fireworks', library_name='fireworks-ai')
         self.__register_package('google.cloud.storage', library_name='google-cloud-storage')
         self.__register_package('google.genai', library_name='google-genai')
@@ -799,7 +807,7 @@ class Env:
         self.__register_package('yolox', library_name='pixeltable-yolox')
         self.__register_package('lancedb')
 
-    def __register_package(self, package_name: str, library_name: Optional[str] = None) -> None:
+    def __register_package(self, package_name: str, library_name: str | None = None) -> None:
         is_installed: bool
         try:
             is_installed = importlib.util.find_spec(package_name) is not None
@@ -815,7 +823,7 @@ class Env:
         if not shutil.which(binary_name):
             raise excs.Error(f'{binary_name} is not installed or not in PATH. Please install it to use this feature.')
 
-    def require_package(self, package_name: str, min_version: Optional[list[int]] = None) -> None:
+    def require_package(self, package_name: str, min_version: list[int] | None = None) -> None:
         """
         Checks whether the specified optional package is available. If not, raises an exception
         with an error message informing the user how to install it.
@@ -859,8 +867,8 @@ class Env:
             else:
                 os.remove(path)
 
-    # def get_resource_pool_info(self, pool_id: str, pool_info_cls: Optional[Type[T]]) -> T:
-    def get_resource_pool_info(self, pool_id: str, make_pool_info: Optional[Callable[[], T]] = None) -> T:
+    # def get_resource_pool_info(self, pool_id: str, pool_info_cls: Type[T] | None) -> T:
+    def get_resource_pool_info(self, pool_id: str, make_pool_info: Callable[[], T] | None = None) -> T:
         """Returns the info object for the given id, creating it if necessary."""
         info = self._resource_pool_info.get(pool_id)
         if info is None and make_pool_info is not None:
@@ -874,10 +882,12 @@ class Env:
         return self._media_dir
 
     @property
-    def object_soa(self) -> StorageObjectAddress:
-        assert self._media_dir is not None
-        assert self._object_soa is not None
-        return self._object_soa
+    def default_input_media_dest(self) -> str | None:
+        return self._default_input_media_dest
+
+    @property
+    def default_output_media_dest(self) -> str | None:
+        return self._default_output_media_dest
 
     @property
     def file_cache_dir(self) -> Path:
@@ -1025,14 +1035,14 @@ _registered_clients: dict[str, ApiClient] = {}
 class ApiClient:
     init_fn: Callable
     params: dict[str, inspect.Parameter]
-    client_obj: Optional[Any] = None
+    client_obj: Any | None = None
 
 
 @dataclass
 class PackageInfo:
     is_installed: bool
     library_name: str  # pypi library name (may be different from package name)
-    version: Optional[list[int]] = None  # installed version, as a list of components (such as [3,0,2] for "3.0.2")
+    version: list[int] | None = None  # installed version, as a list of components (such as [3,0,2] for "3.0.2")
 
 
 TIME_FORMAT = '%H:%M.%S %f'
@@ -1093,7 +1103,7 @@ class RateLimitsInfo:
         """Update self.resource_limits based on the exception headers"""
         self.has_exc = True
 
-    def get_retry_delay(self, exc: Exception) -> Optional[float]:
+    def get_retry_delay(self, exc: Exception) -> float | None:
         """Returns number of seconds to wait before retry, or None if not retryable"""
         if len(self.resource_limits) == 0:
             return 1.0

pixeltable/exec/aggregation_node.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import logging
 import sys
-from typing import Any, AsyncIterator, Iterable, Optional, cast
+from typing import Any, AsyncIterator, Iterable, cast
 
 from pixeltable import catalog, exceptions as excs, exprs
 
@@ -19,18 +19,18 @@ class AggregationNode(ExecNode):
     At the moment, this returns all results in a single DataRowBatch.
     """
 
-    group_by: Optional[list[exprs.Expr]]
+    group_by: list[exprs.Expr] | None
     input_exprs: list[exprs.Expr]
     agg_fn_eval_ctx: exprs.RowBuilder.EvalCtx
     agg_fn_calls: list[exprs.FunctionCall]
     output_batch: DataRowBatch
-    limit: Optional[int]
+    limit: int | None
 
     def __init__(
         self,
         tbl: catalog.TableVersionHandle,
         row_builder: exprs.RowBuilder,
-        group_by: Optional[list[exprs.Expr]],
+        group_by: list[exprs.Expr] | None,
         agg_fn_calls: list[exprs.FunctionCall],
         input_exprs: Iterable[exprs.Expr],
         input: ExecNode,
@@ -72,8 +72,8 @@ class AggregationNode(ExecNode):
                 raise excs.ExprEvalError(fn_call, expr_msg, exc, exc_tb, input_vals, row_num) from exc
 
     async def __aiter__(self) -> AsyncIterator[DataRowBatch]:
-        prev_row: Optional[exprs.DataRow] = None
-        current_group: Optional[list[Any]] = None  # the values of the group-by exprs
+        prev_row: exprs.DataRow | None = None
+        current_group: list[Any] | None = None  # the values of the group-by exprs
         num_input_rows = 0
         num_output_rows = 0
         async for row_batch in self.input:

pixeltable/exec/cache_prefetch_node.py

@@ -9,7 +9,7 @@ import urllib.request
 from collections import deque
 from concurrent import futures
 from pathlib import Path
-from typing import AsyncIterator, Iterator, Optional
+from typing import AsyncIterator, Iterator
 from uuid import UUID
 
 from pixeltable import exceptions as excs, exprs
@@ -43,18 +43,18 @@ class CachePrefetchNode(ExecNode):
 
     # ready_rows: rows that are ready to be returned, ordered by row idx;
     # the implied row idx of ready_rows[0] is num_returned_rows
-    ready_rows: deque[Optional[exprs.DataRow]]
+    ready_rows: deque[exprs.DataRow | None]
 
     in_flight_rows: dict[int, CachePrefetchNode.RowState]  # rows with in-flight urls; id(row) -> RowState
     in_flight_requests: dict[futures.Future, str]  # in-flight requests for urls; future -> URL
     in_flight_urls: dict[str, list[tuple[exprs.DataRow, exprs.ColumnSlotIdx]]]  # URL -> [(row, info)]
     input_finished: bool
-    row_idx: Iterator[Optional[int]]
+    row_idx: Iterator[int | None]
 
     @dataclasses.dataclass
     class RowState:
         row: exprs.DataRow
-        idx: Optional[int]  # position in input stream; None if we don't retain input order
+        idx: int | None  # position in input stream; None if we don't retain input order
         num_missing: int  # number of missing URLs in this row
 
     def __init__(
@@ -78,7 +78,7 @@ class CachePrefetchNode(ExecNode):
     def queued_work(self) -> int:
         return len(self.in_flight_requests)
 
-    async def get_input_batch(self, input_iter: AsyncIterator[DataRowBatch]) -> Optional[DataRowBatch]:
+    async def get_input_batch(self, input_iter: AsyncIterator[DataRowBatch]) -> DataRowBatch | None:
         """Get the next batch of input rows, or None if there are no more rows"""
         try:
             input_batch = await anext(input_iter)
@@ -127,7 +127,7 @@ class CachePrefetchNode(ExecNode):
             sum(int(row is not None) for row in itertools.islice(self.ready_rows, self.BATCH_SIZE)) == self.BATCH_SIZE
         )
 
-    def __add_ready_row(self, row: exprs.DataRow, row_idx: Optional[int]) -> None:
+    def __add_ready_row(self, row: exprs.DataRow, row_idx: int | None) -> None:
         if row_idx is None:
             self.ready_rows.append(row)
         else:
@@ -144,12 +144,12 @@ class CachePrefetchNode(ExecNode):
             tmp_path, exc = f.result()
             if exc is not None and not ignore_errors:
                 raise exc
-            local_path: Optional[Path] = None
+            local_path: Path | None = None
             if tmp_path is not None:
                 # register the file with the cache for the first column in which it's missing
                 assert url in self.in_flight_urls
                 _, info = self.in_flight_urls[url][0]
-                local_path = file_cache.add(info.col.tbl.id, info.col.id, url, tmp_path)
+                local_path = file_cache.add(info.col.get_tbl().id, info.col.id, url, tmp_path)
                 _logger.debug(f'cached {url} as {local_path}')
 
             # add the local path/exception to the slots that reference the url
@@ -174,7 +174,7 @@ class CachePrefetchNode(ExecNode):
         # the time it takes to get the next batch together
         cache_misses: list[str] = []
 
-        url_pos: dict[str, Optional[int]] = {}  # url -> row_idx; used for logging
+        url_pos: dict[str, int | None] = {}  # url -> row_idx; used for logging
         for row in input_batch:
             # identify missing local files in input batch, or fill in their paths if they're already cached
             num_missing = 0
@@ -213,7 +213,7 @@ class CachePrefetchNode(ExecNode):
             _logger.debug(f'submitted {url} for idx {url_pos[url]}')
             self.in_flight_requests[f] = url
 
-    def __fetch_url(self, url: str) -> tuple[Optional[Path], Optional[Exception]]:
+    def __fetch_url(self, url: str) -> tuple[Path | None, Exception | None]:
         """Fetches a remote URL into the TempStore and returns its path"""
         from pixeltable.utils.local_store import TempStore
 

pixeltable/exec/data_row_batch.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import logging
-from typing import Iterator, Optional
+from typing import Iterator
 
 from pixeltable import exprs
 
@@ -19,11 +19,11 @@ class DataRowBatch:
     row_builder: exprs.RowBuilder
     rows: list[exprs.DataRow]
 
-    def __init__(self, row_builder: exprs.RowBuilder, rows: Optional[list[exprs.DataRow]] = None):
+    def __init__(self, row_builder: exprs.RowBuilder, rows: list[exprs.DataRow] | None = None):
         self.row_builder = row_builder
         self.rows = [] if rows is None else rows
 
-    def add_row(self, row: Optional[exprs.DataRow]) -> exprs.DataRow:
+    def add_row(self, row: exprs.DataRow | None) -> exprs.DataRow:
         if row is None:
             row = self.row_builder.make_row()
         self.rows.append(row)

pixeltable/exec/exec_context.py

@@ -1,5 +1,4 @@
 import random
-from typing import Optional
 
 import sqlalchemy as sql
 
@@ -13,9 +12,9 @@ class ExecContext:
     profile: exprs.ExecProfile
     show_pbar: bool
     batch_size: int
-    num_rows: Optional[int]
-    conn: Optional[sql.engine.Connection]
-    pk_clause: Optional[list[sql.ClauseElement]]
+    num_rows: int | None
+    conn: sql.engine.Connection | None
+    pk_clause: list[sql.ClauseElement] | None
     num_computed_exprs: int
     ignore_errors: bool
     random_seed: int  # general-purpose source of randomness with execution scope
@@ -26,7 +25,7 @@ class ExecContext:
         *,
         show_pbar: bool = False,
         batch_size: int = 0,
-        pk_clause: Optional[list[sql.ClauseElement]] = None,
+        pk_clause: list[sql.ClauseElement] | None = None,
         num_computed_exprs: int = 0,
         ignore_errors: bool = False,
     ):

pixeltable/exec/exec_node.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import abc
 import logging
-from typing import AsyncIterator, Iterable, Iterator, Optional, TypeVar
+from typing import AsyncIterator, Iterable, Iterator, TypeVar
 
 from pixeltable import exprs
 from pixeltable.env import Env
@@ -18,16 +18,16 @@ class ExecNode(abc.ABC):
 
     output_exprs: Iterable[exprs.Expr]
     row_builder: exprs.RowBuilder
-    input: Optional[ExecNode]
+    input: ExecNode | None
     flushed_img_slots: list[int]  # idxs of image slots of our output_exprs dependencies
-    ctx: Optional[ExecContext]
+    ctx: ExecContext | None
 
     def __init__(
         self,
         row_builder: exprs.RowBuilder,
         output_exprs: Iterable[exprs.Expr],
         input_exprs: Iterable[exprs.Expr],
-        input: Optional[ExecNode] = None,
+        input: ExecNode | None = None,
     ):
         assert all(expr.is_valid for expr in output_exprs)
         self.output_exprs = output_exprs
@@ -85,7 +85,7 @@ class ExecNode(abc.ABC):
 
     T = TypeVar('T', bound='ExecNode')
 
-    def get_node(self, node_class: type[T]) -> Optional[T]:
+    def get_node(self, node_class: type[T]) -> T | None:
         if isinstance(self, node_class):
             return self
         if self.input is not None:

pixeltable/exec/expr_eval/evaluators.py

@@ -5,7 +5,7 @@ import datetime
 import itertools
 import logging
 import sys
-from typing import Any, Callable, Iterator, Optional, cast
+from typing import Any, Callable, Iterator, cast
 
 from pixeltable import exprs, func
 
@@ -64,11 +64,11 @@ class FnCallEvaluator(Evaluator):
 
     fn_call: exprs.FunctionCall
     fn: func.CallableFunction
-    scalar_py_fn: Optional[Callable]  # only set for non-batching CallableFunctions
+    scalar_py_fn: Callable | None  # only set for non-batching CallableFunctions
 
     # only set if fn.is_batched
-    call_args_queue: Optional[asyncio.Queue[FnCallArgs]]  # FnCallArgs waiting for execution
-    batch_size: Optional[int]
+    call_args_queue: asyncio.Queue[FnCallArgs] | None  # FnCallArgs waiting for execution
+    batch_size: int | None
 
     def __init__(self, fn_call: exprs.FunctionCall, dispatcher: Dispatcher, exec_ctx: ExecCtx):
         super().__init__(dispatcher, exec_ctx)
@@ -160,8 +160,8 @@ class FnCallEvaluator(Evaluator):
 
     def _create_batch_call_args(self, call_args: list[FnCallArgs]) -> FnCallArgs:
         """Roll call_args into a single batched FnCallArgs"""
-        batch_args: list[list[Optional[Any]]] = [[None] * len(call_args) for _ in range(len(self.fn_call.arg_idxs))]
-        batch_kwargs: dict[str, list[Optional[Any]]] = {k: [None] * len(call_args) for k in self.fn_call.kwarg_idxs}
+        batch_args: list[list[Any | None]] = [[None] * len(call_args) for _ in range(len(self.fn_call.arg_idxs))]
+        batch_kwargs: dict[str, list[Any | None]] = {k: [None] * len(call_args) for k in self.fn_call.kwarg_idxs}
         assert isinstance(self.fn, func.CallableFunction)
         for i, item in enumerate(call_args):
             for j in range(len(item.args)):

pixeltable/exec/expr_eval/expr_eval_node.py

@@ -4,7 +4,7 @@ import asyncio
 import logging
 import traceback
 from types import TracebackType
-from typing import AsyncIterator, Iterable, Optional
+from typing import AsyncIterator, Iterable
 
 import numpy as np
 
@@ -49,17 +49,17 @@ class ExprEvalNode(ExecNode):
     # execution state
     tasks: set[asyncio.Task]  # collects all running tasks to prevent them from getting gc'd
     exc_event: asyncio.Event  # set if an exception needs to be propagated
-    error: Optional[Exception]  # exception that needs to be propagated
+    error: Exception | None  # exception that needs to be propagated
     completed_rows: asyncio.Queue[exprs.DataRow]  # rows that have completed evaluation
     completed_event: asyncio.Event  # set when completed_rows is non-empty
     input_iter: AsyncIterator[DataRowBatch]
-    current_input_batch: Optional[DataRowBatch]  # batch from which we're currently consuming rows
+    current_input_batch: DataRowBatch | None  # batch from which we're currently consuming rows
     input_row_idx: int  # next row to consume from current_input_batch
-    next_input_batch: Optional[DataRowBatch]  # read-ahead input batch
+    next_input_batch: DataRowBatch | None  # read-ahead input batch
     avail_input_rows: int  # total number across both current_/next_input_batch
     input_complete: bool  # True if we've received all input batches
     num_in_flight: int  # number of dispatched rows that haven't completed
-    row_pos_map: Optional[dict[int, int]]  # id(row) -> position of row in input; only set if maintain_input_order
+    row_pos_map: dict[int, int] | None  # id(row) -> position of row in input; only set if maintain_input_order
     output_buffer: RowBuffer  # holds rows that are ready to be returned, in order
 
     # debugging
@@ -217,9 +217,10 @@ class ExprEvalNode(ExecNode):
 
         row: exprs.DataRow
         exc_event_aw = asyncio.create_task(self.exc_event.wait(), name='exc_event.wait()')
-        input_batch_aw: Optional[asyncio.Task] = None
-        completed_aw: Optional[asyncio.Task] = None
+        input_batch_aw: asyncio.Task | None = None
+        completed_aw: asyncio.Task | None = None
         closed_evaluators = False  # True after calling Evaluator.close()
+        exprs.Expr.prepare_list(self.exec_ctx.all_exprs)
 
         try:
             while True:

pixeltable/exec/expr_eval/globals.py

@@ -4,7 +4,7 @@ import abc
 import asyncio
 from dataclasses import dataclass
 from types import TracebackType
-from typing import Any, Iterable, Optional, Protocol
+from typing import Any, Iterable, Protocol
 
 import numpy as np
 
@@ -18,11 +18,11 @@ class FnCallArgs:
     fn_call: exprs.FunctionCall
     rows: list[exprs.DataRow]
     # single call
-    args: Optional[list[Any]] = None
-    kwargs: Optional[dict[str, Any]] = None
+    args: list[Any] | None = None
+    kwargs: dict[str, Any] | None = None
     # batch call
-    batch_args: Optional[list[list[Optional[Any]]]] = None
-    batch_kwargs: Optional[dict[str, list[Optional[Any]]]] = None
+    batch_args: list[list[Any | None]] | None = None
+    batch_kwargs: dict[str, list[Any | None]] | None = None
 
     @property
     def pxt_fn(self) -> func.CallableFunction:
@@ -56,7 +56,7 @@ class Scheduler(abc.ABC):
         request: FnCallArgs
         num_retries: int
         exec_ctx: ExecCtx
-        retry_after: Optional[float] = None  # time.monotonic()
+        retry_after: float | None = None  # time.monotonic()
 
         def __lt__(self, other: Scheduler.QueueItem) -> bool:
             # prioritize by number of retries (more retries = higher priority)