pixeltable 0.4.3__py3-none-any.whl → 0.4.5__py3-none-any.whl

pixeltable/catalog/table_version_path.py

@@ -76,7 +76,7 @@ class TableVersionPath:
         elif self._cached_tbl_version is not None:
             return
 
-        with Catalog.get().begin_xact(for_write=False):
+        with Catalog.get().begin_xact(tbl_id=self.tbl_version.id, for_write=False):
             self._cached_tbl_version = self.tbl_version.get()
 
     def clear_cached_md(self) -> None:

pixeltable/catalog/tbl_ops.py

@@ -0,0 +1,44 @@
+# This file contains all dataclasses related to schema.PendingTableOp:
+# - TableOp: the container for each log entry
+# - <>Op: the actual operation, which is performed by TableVersion.exec_op(); each <>Op class contains
+#   enough information for exec_op() to perform the operation without having to reference data outside of
+#   TableVersion
+
+import dataclasses
+from typing import Any, Optional
+
+
+@dataclasses.dataclass
+class CreateStoreTableOp:
+    pass
+
+
+@dataclasses.dataclass
+class LoadViewOp:
+    view_path: dict[str, Any]  # needed to create the view load plan
+
+
+@dataclasses.dataclass
+class DeleteTableMdOp:
+    pass
+
+
+@dataclasses.dataclass
+class DeleteTableMediaFilesOp:
+    pass
+
+
+@dataclasses.dataclass
+class DropStoreTableOp:
+    pass
+
+
+@dataclasses.dataclass
+class TableOp:
+    tbl_id: str  # uuid.UUID
+    op_sn: int  # sequence number within the update operation; [0, num_ops)
+    num_ops: int  # total number of ops forming the update operation
+    needs_xact: bool  # if True, op must be run as part of a transaction
+
+    create_store_table_op: Optional[CreateStoreTableOp] = None
+    load_view_op: Optional[LoadViewOp] = None

pixeltable/catalog/view.py

@@ -9,7 +9,6 @@ import pixeltable.exceptions as excs
 import pixeltable.metadata.schema as md_schema
 import pixeltable.type_system as ts
 from pixeltable import catalog, exprs, func
-from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 
 if TYPE_CHECKING:
@@ -19,9 +18,10 @@ if TYPE_CHECKING:
 from .column import Column
 from .globals import _POS_COLUMN_NAME, MediaValidation
 from .table import Table
-from .table_version import TableVersion
+from .table_version import TableVersion, TableVersionMd
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
+from .tbl_ops import CreateStoreTableOp, LoadViewOp, TableOp
 from .update_status import UpdateStatus
 
 if TYPE_CHECKING:
@@ -45,9 +45,18 @@ class View(Table):
         if not snapshot_only:
             self._tbl_version = tbl_version_path.tbl_version
 
-    @classmethod
-    def _display_name(cls) -> str:
-        return 'view'
+    def _display_name(self) -> str:
+        name: str
+        if self._tbl_version_path.is_snapshot():
+            name = 'snapshot'
+        elif self._tbl_version_path.is_view():
+            name = 'view'
+        else:
+            assert self._tbl_version_path.is_replica()
+            name = 'table'
+        if self._tbl_version_path.is_replica():
+            name = f'{name}-replica'
+        return name
 
     @classmethod
     def select_list_to_additional_columns(cls, select_list: list[tuple[exprs.Expr, Optional[str]]]) -> dict[str, dict]:
@@ -80,7 +89,7 @@ class View(Table):
         media_validation: MediaValidation,
         iterator_cls: Optional[type[ComponentIterator]],
         iterator_args: Optional[dict],
-    ) -> View:
+    ) -> tuple[TableVersionMd, Optional[list[TableOp]]]:
         from pixeltable.plan import SampleClause
 
         # Convert select_list to more additional_columns if present
@@ -167,11 +176,10 @@ class View(Table):
             for col in columns:
                 if col.name in iterator_col_names:
                     raise excs.Error(
-                        f'Duplicate name: column {col.name} is already present in the iterator output schema'
+                        f'Duplicate name: column {col.name!r} is already present in the iterator output schema'
                     )
             columns = iterator_cols + columns
 
-        session = Env.get().session
         from pixeltable.exprs import InlineDict
 
         iterator_args_expr: exprs.Expr = InlineDict(iterator_args) if iterator_args is not None else None
@@ -200,54 +208,26 @@ class View(Table):
             iterator_args=iterator_args_expr.as_dict() if iterator_args_expr is not None else None,
         )
 
-        id, tbl_version = TableVersion.create(
-            dir_id,
-            name,
-            columns,
-            num_retained_versions,
-            comment,
-            media_validation=media_validation,
-            # base_path=base_version_path,
-            view_md=view_md,
+        md = TableVersion.create_initial_md(
+            name, columns, num_retained_versions, comment, media_validation=media_validation, view_md=view_md
         )
-        if tbl_version is None:
-            # this is purely a snapshot: we use the base's tbl version path
-            view = cls(id, dir_id, name, base_version_path, snapshot_only=True)
-            _logger.info(f'created snapshot {name}')
+        if md.tbl_md.is_pure_snapshot:
+            # this is purely a snapshot: no store table to create or load
+            return md, None
         else:
-            view = cls(
-                id,
-                dir_id,
-                name,
-                TableVersionPath(
-                    TableVersionHandle(tbl_version.id, tbl_version.effective_version), base=base_version_path
-                ),
-                snapshot_only=False,
-            )
-            _logger.info(f'Created view `{name}`, id={tbl_version.id}')
-
-            from pixeltable.plan import Planner
-
-            try:
-                plan, _ = Planner.create_view_load_plan(view._tbl_version_path)
-                _, row_counts = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
-                status = UpdateStatus(row_count_stats=row_counts)
-                tbl_version._write_md_update_status(0, update_status=status)
-
-            except:
-                # we need to remove the orphaned TableVersion instance
-                del catalog.Catalog.get()._tbl_versions[tbl_version.id, tbl_version.effective_version]
-                base_tbl_version = base.tbl_version.get()
-                if tbl_version.effective_version is None and not base_tbl_version.is_snapshot:
-                    # also remove tbl_version from the base
-                    base_tbl_version.mutable_views.remove(TableVersionHandle.create(tbl_version))
-                raise
-            Env.get().console_logger.info(
-                f'Created view `{name}` with {status.num_rows} rows, {status.num_excs} exceptions.'
+            tbl_id = md.tbl_md.tbl_id
+            view_path = TableVersionPath(
+                TableVersionHandle(UUID(tbl_id), effective_version=0 if is_snapshot else None), base=base_version_path
             )
-
-        session.commit()
-        return view
+            ops = [
+                TableOp(
+                    tbl_id=tbl_id, op_sn=0, num_ops=2, needs_xact=False, create_store_table_op=CreateStoreTableOp()
+                ),
+                TableOp(
+                    tbl_id=tbl_id, op_sn=1, num_ops=2, needs_xact=True, load_view_op=LoadViewOp(view_path.as_dict())
+                ),
+            ]
+            return md, ops
 
     @classmethod
     def _verify_column(cls, col: Column) -> None:
@@ -275,13 +255,26 @@ class View(Table):
             base=cls._get_snapshot_path(tbl_version_path.base) if tbl_version_path.base is not None else None,
         )
 
+    def _is_anonymous_snapshot(self) -> bool:
+        """
+        Returns True if this is an unnamed snapshot (i.e., a snapshot that is not a separate schema object).
+        """
+        return self._snapshot_only and self._id == self._tbl_version_path.tbl_id
+
     def _get_metadata(self) -> dict[str, Any]:
         md = super()._get_metadata()
         md['is_view'] = True
         md['is_snapshot'] = self._tbl_version_path.is_snapshot()
+        if self._is_anonymous_snapshot():
+            # Update name and path with version qualifiers.
+            md['name'] = f'{self._name}:{self._tbl_version_path.version()}'
+            md['path'] = f'{self._path()}:{self._tbl_version_path.version()}'
         base_tbl = self._get_base_table()
-        base_version = self._effective_base_versions[0]
-        md['base'] = base_tbl._path() if base_version is None else f'{base_tbl._path()}:{base_version}'
+        if base_tbl is None:
+            md['base'] = None
+        else:
+            base_version = self._effective_base_versions[0]
+            md['base'] = base_tbl._path() if base_version is None else f'{base_tbl._path()}:{base_version}'
         return md
 
     def insert(
@@ -295,28 +288,33 @@ class View(Table):
         print_stats: bool = False,
         **kwargs: Any,
     ) -> UpdateStatus:
-        raise excs.Error(f'{self._display_name()} {self._name!r}: cannot insert into view')
+        raise excs.Error(f'{self._display_str()}: Cannot insert into a {self._display_name()}.')
 
     def delete(self, where: Optional[exprs.Expr] = None) -> UpdateStatus:
-        raise excs.Error(f'{self._display_name()} {self._name!r}: cannot delete from view')
+        raise excs.Error(f'{self._display_str()}: Cannot delete from a {self._display_name()}.')
 
     def _get_base_table(self) -> Optional['Table']:
-        # if this is a pure snapshot, our tbl_version_path only reflects the base (there is no TableVersion instance
-        # for the snapshot itself)
-        base_id = self._tbl_version_path.tbl_id if self._snapshot_only else self._tbl_version_path.base.tbl_id
-        return catalog.Catalog.get().get_table_by_id(base_id)
+        if self._tbl_version_path.tbl_id != self._id:
+            # _tbl_version_path represents a different schema object from this one. This can only happen if this is a
+            # named pure snapshot.
+            base_id = self._tbl_version_path.tbl_id
+        elif self._tbl_version_path.base is None:
+            return None
+        else:
+            base_id = self._tbl_version_path.base.tbl_id
+        with catalog.Catalog.get().begin_xact(tbl_id=base_id, for_write=False):
+            return catalog.Catalog.get().get_table_by_id(base_id)
 
     @property
     def _effective_base_versions(self) -> list[Optional[int]]:
         effective_versions = [tv.effective_version for tv in self._tbl_version_path.get_tbl_versions()]
-        if self._snapshot_only:
-            return effective_versions
+        if self._snapshot_only and not self._is_anonymous_snapshot():
+            return effective_versions  # Named pure snapshot
         else:
             return effective_versions[1:]
 
     def _table_descriptor(self) -> str:
-        display_name = 'Snapshot' if self._snapshot_only else 'View'
-        result = [f'{display_name} {self._path()!r}']
+        result = [self._display_str()]
         bases_descrs: list[str] = []
         for base, effective_version in zip(self._get_base_tables(), self._effective_base_versions):
             if effective_version is None:

pixeltable/config.py

@@ -8,7 +8,7 @@ from typing import Any, ClassVar, Optional, TypeVar
 
 import toml
 
-from pixeltable import exceptions as excs
+from pixeltable import env, exceptions as excs
 
 _logger = logging.getLogger('pixeltable')
 
@@ -82,7 +82,11 @@ class Config:
         return cls.__instance
 
     @classmethod
-    def init(cls, config_overrides: dict[str, Any]) -> None:
+    def init(cls, config_overrides: dict[str, Any], reinit: bool = False) -> None:
+        if reinit:
+            cls.__instance = None
+            for cl in env._registered_clients.values():
+                cl.client_obj = None
         if cls.__instance is None:
             cls.__instance = cls(config_overrides)
         elif len(config_overrides) > 0:
@@ -102,7 +106,7 @@ class Config:
         env_var = f'{section.upper()}_{key.upper()}'
         if override_var in self.__config_overrides:
             return self.__config_overrides[override_var]
-        if env_var in os.environ:
+        if env_var in os.environ and len(os.environ[env_var]) > 0:
             return os.environ[env_var]
         return default
 
@@ -157,7 +161,11 @@ KNOWN_CONFIG_OPTIONS = {
     'groq': {'api_key': 'Groq API key'},
     'label_studio': {'api_key': 'Label Studio API key', 'url': 'Label Studio server URL'},
     'mistral': {'api_key': 'Mistral API key'},
-    'openai': {'api_key': 'OpenAI API key'},
+    'openai': {
+        'api_key': 'OpenAI API key',
+        'base_url': 'OpenAI API base URL',
+        'api_version': 'API version if using Azure OpenAI',
+    },
     'replicate': {'api_token': 'Replicate API token'},
     'together': {'api_key': 'Together API key'},
     'pypi': {'api_key': 'PyPI API key (for internal use only)'},

pixeltable/dataframe.py

@@ -8,9 +8,22 @@ import json
 import logging
 import traceback
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, AsyncIterator, Callable, Hashable, Iterator, NoReturn, Optional, Sequence, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncIterator,
+    Callable,
+    Hashable,
+    Iterator,
+    NoReturn,
+    Optional,
+    Sequence,
+    TypeVar,
+    Union,
+)
 
 import pandas as pd
+import pydantic
 import sqlalchemy as sql
 
 from pixeltable import catalog, exceptions as excs, exec, exprs, plan, type_system as ts
@@ -32,6 +45,11 @@ _logger = logging.getLogger('pixeltable')
 
 
 class DataFrameResultSet:
+    _rows: list[list[Any]]
+    _col_names: list[str]
+    __schema: dict[str, ColumnType]
+    __formatter: Formatter
+
     def __init__(self, rows: list[list[Any]], schema: dict[str, ColumnType]):
         self._rows = rows
         self._col_names = list(schema.keys())
@@ -66,6 +84,44 @@ class DataFrameResultSet:
     def to_pandas(self) -> pd.DataFrame:
         return pd.DataFrame.from_records(self._rows, columns=self._col_names)
 
+    BaseModelT = TypeVar('BaseModelT', bound=pydantic.BaseModel)
+
+    def to_pydantic(self, model: type[BaseModelT]) -> Iterator[BaseModelT]:
+        """
+        Convert the DataFrameResultSet to a list of Pydantic model instances.
+
+        Args:
+            model: A Pydantic model class.
+
+        Returns:
+            An iterator over Pydantic model instances, one for each row in the result set.
+
+        Raises:
+            Error: If the row data doesn't match the model schema.
+        """
+        model_fields = model.model_fields
+        model_config = getattr(model, 'model_config', {})
+        forbid_extra_fields = model_config.get('extra') == 'forbid'
+
+        # schema validation
+        required_fields = {name for name, field in model_fields.items() if field.is_required()}
+        col_names = set(self._col_names)
+        missing_fields = required_fields - col_names
+        if len(missing_fields) > 0:
+            raise excs.Error(
+                f'Required model fields {missing_fields} are missing from result set columns {self._col_names}'
+            )
+        if forbid_extra_fields:
+            extra_fields = col_names - set(model_fields.keys())
+            if len(extra_fields) > 0:
+                raise excs.Error(f"Extra fields {extra_fields} are not allowed in model with extra='forbid'")
+
+        for row in self:
+            try:
+                yield model(**row)
+            except pydantic.ValidationError as e:
+                raise excs.Error(str(e)) from e
+
     def _row_to_dict(self, row_idx: int) -> dict[str, Any]:
         return {self._col_names[i]: self._rows[row_idx][i] for i in range(len(self._col_names))}
 
@@ -1185,7 +1241,7 @@ class DataFrame:
         """
         self._validate_mutable('delete', False)
         if not self._first_tbl.is_insertable():
-            raise excs.Error('Cannot delete from view')
+            raise excs.Error('Cannot use `delete` on a view.')
         with Catalog.get().begin_xact(tbl=self._first_tbl, for_write=True, lock_mutable_tree=True):
             return self._first_tbl.tbl_version.get().delete(where=self.where_clause)
 
@@ -1196,14 +1252,27 @@ class DataFrame:
             op_name: The name of the operation for which the test is being performed.
             allow_select: If True, allow a select() specification in the Dataframe.
         """
+        self._validate_mutable_op_sequence(op_name, allow_select)
+
+        # TODO: Reconcile these with Table.__check_mutable()
+        assert len(self._from_clause.tbls) == 1
+        if self._first_tbl.is_snapshot():
+            raise excs.Error(f'Cannot use `{op_name}` on a snapshot.')
+        if self._first_tbl.is_replica():
+            raise excs.Error(f'Cannot use `{op_name}` on a replica.')
+
+    def _validate_mutable_op_sequence(self, op_name: str, allow_select: bool) -> None:
+        """Tests whether the sequence of operations on this DataFrame is valid for a mutation operation."""
         if self.group_by_clause is not None or self.grouping_tbl is not None:
-            raise excs.Error(f'Cannot use `{op_name}` after `group_by`')
+            raise excs.Error(f'Cannot use `{op_name}` after `group_by`.')
         if self.order_by_clause is not None:
-            raise excs.Error(f'Cannot use `{op_name}` after `order_by`')
+            raise excs.Error(f'Cannot use `{op_name}` after `order_by`.')
         if self.select_list is not None and not allow_select:
-            raise excs.Error(f'Cannot use `{op_name}` after `select`')
+            raise excs.Error(f'Cannot use `{op_name}` after `select`.')
         if self.limit_val is not None:
-            raise excs.Error(f'Cannot use `{op_name}` after `limit`')
+            raise excs.Error(f'Cannot use `{op_name}` after `limit`.')
+        if self._has_joins():
+            raise excs.Error(f'Cannot use `{op_name}` after `join`.')
 
     def as_dict(self) -> dict[str, Any]:
         """

pixeltable/env.py

@@ -13,6 +13,8 @@ import platform
 import shutil
 import sys
 import threading
+import types
+import typing
 import uuid
 import warnings
 from abc import abstractmethod
@@ -20,7 +22,7 @@ from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
 from sys import stdout
-from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, TypeVar
+from typing import TYPE_CHECKING, Any, Callable, Iterator, Literal, Optional, TypeVar
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 import nest_asyncio  # type: ignore[import-untyped]
@@ -86,6 +88,7 @@ class Env:
     _resource_pool_info: dict[str, Any]
     _current_conn: Optional[sql.Connection]
     _current_session: Optional[sql.orm.Session]
+    _current_isolation_level: Optional[Literal['REPEATABLE_READ', 'SERIALIZABLE']]
     _dbms: Optional[Dbms]
     _event_loop: Optional[asyncio.AbstractEventLoop]  # event loop for ExecNode
 
@@ -99,6 +102,7 @@ class Env:
     def _init_env(cls, reinit_db: bool = False) -> None:
         assert not cls.__initializing, 'Circular env initialization detected.'
         cls.__initializing = True
+        cls._instance = None
         env = Env()
         env._set_up(reinit_db=reinit_db)
         env._upgrade_metadata()
@@ -142,6 +146,7 @@ class Env:
         self._resource_pool_info = {}
         self._current_conn = None
         self._current_session = None
+        self._current_isolation_level = None
         self._dbms = None
         self._event_loop = None
 
@@ -230,20 +235,34 @@ class Env:
         return self._db_server is not None
 
     @contextmanager
-    def begin_xact(self) -> Iterator[sql.Connection]:
-        """Call Catalog.begin_xact() instead, unless there is a specific reason to call this directly."""
+    def begin_xact(self, for_write: bool = False) -> Iterator[sql.Connection]:
+        """
+        Call Catalog.begin_xact() instead, unless there is a specific reason to call this directly.
+
+        for_write: if True, uses serializable isolation; if False, uses repeatable_read
+
+        TODO: repeatable read is not available in Cockroachdb; instead, run queries against a snapshot TVP
+        that avoids tripping over any pending ops
+        """
         if self._current_conn is None:
             assert self._current_session is None
             try:
-                with self.engine.begin() as conn, sql.orm.Session(conn) as session:
+                self._current_isolation_level = 'SERIALIZABLE' if for_write else 'REPEATABLE_READ'
+                with (
+                    self.engine.connect().execution_options(isolation_level=self._current_isolation_level) as conn,
+                    sql.orm.Session(conn) as session,
+                    conn.begin(),
+                ):
                     self._current_conn = conn
                     self._current_session = session
                     yield conn
             finally:
                 self._current_session = None
                 self._current_conn = None
+                self._current_isolation_level = None
         else:
             assert self._current_session is not None
+            assert for_write == (self._current_isolation_level == 'serializable')
             yield self._current_conn
 
     def configure_logging(
@@ -587,16 +606,26 @@ class Env:
 
         # Construct a client, retrieving each parameter from config.
 
-        init_kwargs: dict[str, str] = {}
-        for param in cl.param_names:
-            arg = Config.get().get_string_value(param, section=name)
-            if arg is not None and len(arg) > 0:
-                init_kwargs[param] = arg
-            else:
+        init_kwargs: dict[str, Any] = {}
+        for param in cl.params.values():
+            # Determine the type of the parameter for proper config parsing.
+            t = param.annotation
+            # Deference Optional[T]
+            if typing.get_origin(t) in (typing.Union, types.UnionType):
+                args = typing.get_args(t)
+                if args[0] is type(None):
+                    t = args[1]
+                elif args[1] is type(None):
+                    t = args[0]
+            assert isinstance(t, type), t
+            arg: Any = Config.get().get_value(param.name, t, section=name)
+            if arg is not None:
+                init_kwargs[param.name] = arg
+            elif param.default is inspect.Parameter.empty:
                 raise excs.Error(
-                    f'`{name}` client not initialized: parameter `{param}` is not configured.\n'
-                    f'To fix this, specify the `{name.upper()}_{param.upper()}` environment variable, '
-                    f'or put `{param.lower()}` in the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
+                    f'`{name}` client not initialized: parameter `{param.name}` is not configured.\n'
+                    f'To fix this, specify the `{name.upper()}_{param.name.upper()}` environment variable, '
+                    f'or put `{param.name.lower()}` in the `{name.lower()}` section of $PIXELTABLE_HOME/config.toml.'
                 )
 
         cl.client_obj = cl.init_fn(**init_kwargs)
@@ -607,7 +636,7 @@ class Env:
         """
         The http server root is the file system root.
         eg: /home/media/foo.mp4 is located at http://127.0.0.1:{port}/home/media/foo.mp4
-        in windows, the server will translate paths like http://127.0.0.1:{port}/c:/media/foo.mp4
+        On Windows, the server will translate paths like http://127.0.0.1:{port}/c:/media/foo.mp4
         This arrangement enables serving media hosted within _home,
         as well as external media inserted into pixeltable or produced by pixeltable.
         The port is chosen dynamically to prevent conflicts.
@@ -815,8 +844,8 @@ def register_client(name: str) -> Callable:
 
     def decorator(fn: Callable) -> None:
         sig = inspect.signature(fn)
-        param_names = list(sig.parameters.keys())
-        _registered_clients[name] = ApiClient(init_fn=fn, param_names=param_names)
+        params = dict(sig.parameters)
+        _registered_clients[name] = ApiClient(init_fn=fn, params=params)
 
     return decorator
 
@@ -827,7 +856,7 @@ _registered_clients: dict[str, ApiClient] = {}
 @dataclass
 class ApiClient:
     init_fn: Callable
-    param_names: list[str]
+    params: dict[str, inspect.Parameter]
     client_obj: Optional[Any] = None
 
 

pixeltable/exec/aggregation_node.py

@@ -45,7 +45,7 @@ class AggregationNode(ExecNode):
         # we need to make sure to refer to the same exprs that RowBuilder.eval() will use
         self.agg_fn_calls = [cast(exprs.FunctionCall, e) for e in self.agg_fn_eval_ctx.target_exprs]
         # create output_batch here, rather than in __iter__(), so we don't need to remember tbl and row_builder
-        self.output_batch = DataRowBatch(tbl, row_builder, 0)
+        self.output_batch = DataRowBatch(row_builder)
         self.limit = None
 
     def set_limit(self, limit: int) -> None:

pixeltable/exec/cache_prefetch_node.py

@@ -12,7 +12,7 @@ from pathlib import Path
 from typing import Any, AsyncIterator, Iterator, Optional
 from uuid import UUID
 
-from pixeltable import catalog, env, exceptions as excs, exprs
+from pixeltable import env, exceptions as excs, exprs
 from pixeltable.utils.filecache import FileCache
 
 from .data_row_batch import DataRowBatch
@@ -37,7 +37,6 @@ class CachePrefetchNode(ExecNode):
     boto_client_lock: threading.Lock
 
     # execution state
-    batch_tbl_version: Optional[catalog.TableVersionHandle]  # needed to construct output batches
     num_returned_rows: int
 
     # ready_rows: rows that are ready to be returned, ordered by row idx;
@@ -68,7 +67,6 @@ class CachePrefetchNode(ExecNode):
         self.boto_client = None
         self.boto_client_lock = threading.Lock()
 
-        self.batch_tbl_version = None
         self.num_returned_rows = 0
         self.ready_rows = deque()
         self.in_flight_rows = {}
@@ -95,7 +93,7 @@ class CachePrefetchNode(ExecNode):
 
                 if len(self.ready_rows) > 0:
                     # create DataRowBatch from the first BATCH_SIZE ready rows
-                    batch = DataRowBatch(self.batch_tbl_version, self.row_builder)
+                    batch = DataRowBatch(self.row_builder)
                     rows = [self.ready_rows.popleft() for _ in range(min(self.BATCH_SIZE, len(self.ready_rows)))]
                     for row in rows:
                         assert row is not None
@@ -173,8 +171,6 @@ class CachePrefetchNode(ExecNode):
         if input_batch is None:
             self.input_finished = True
             return
-        if self.batch_tbl_version is None:
-            self.batch_tbl_version = input_batch.tbl
 
         file_cache = FileCache.get()
 

pixeltable/exec/component_iteration_node.py

@@ -40,7 +40,7 @@ class ComponentIterationNode(ExecNode):
         }
 
     async def __aiter__(self) -> AsyncIterator[DataRowBatch]:
-        output_batch = DataRowBatch(self.view, self.row_builder)
+        output_batch = DataRowBatch(self.row_builder)
         async for input_batch in self.input:
             for input_row in input_batch:
                 self.row_builder.eval(input_row, self.iterator_args_ctx)
@@ -52,13 +52,14 @@ class ComponentIterationNode(ExecNode):
                 if self.__non_nullable_args_specified(iterator_args):
                     iterator = self.view.get().iterator_cls(**iterator_args)
                     for pos, component_dict in enumerate(iterator):
-                        output_row = output_batch.add_row()
+                        output_row = self.row_builder.make_row()
                         input_row.copy(output_row)
                         # we're expanding the input and need to add the iterator position to the pk
                         self.__populate_output_row(output_row, pos, component_dict)
+                        output_batch.add_row(output_row)
                         if len(output_batch) == self.__OUTPUT_BATCH_SIZE:
                             yield output_batch
-                            output_batch = DataRowBatch(self.view, self.row_builder)
+                            output_batch = DataRowBatch(self.row_builder)
 
         if len(output_batch) > 0:
             yield output_batch