pixeltable 0.0.0__py3-none-any.whl

pixeltable/exec/sql_scan_node.py

@@ -0,0 +1,226 @@
+from typing import List, Optional, Tuple, Iterable, Set
+from uuid import UUID
+import logging
+import warnings
+
+import sqlalchemy as sql
+
+from .data_row_batch import DataRowBatch
+from .exec_node import ExecNode
+import pixeltable.exprs as exprs
+import pixeltable.catalog as catalog
+
+
+_logger = logging.getLogger('pixeltable')
+
+class SqlScanNode(ExecNode):
+    """Materializes data from the store via SQL
+    """
+    def __init__(
+            self, tbl: catalog.TableVersionPath, row_builder: exprs.RowBuilder,
+            select_list: Iterable[exprs.Expr],
+            where_clause: Optional[exprs.Expr] = None, filter: Optional[exprs.Predicate] = None,
+            order_by_items: Optional[List[Tuple[exprs.Expr, bool]]] = None,
+            limit: int = 0, set_pk: bool = False, exact_version_only: Optional[List[catalog.TableVersion]] = None
+    ):
+        """
+        Args:
+            select_list: output of the query
+            sql_where_clause: SQL Where clause
+            filter: additional Where-clause predicate that can't be evaluated via SQL
+            limit: max number of rows to return: 0 = no limit
+            set_pk: if True, sets the primary for each DataRow
+            exact_version_only: tables for which we only want to see rows created at the current version
+        """
+        # create Select stmt
+        if order_by_items is None:
+            order_by_items = []
+        if exact_version_only is None:
+            exact_version_only = []
+        super().__init__(row_builder, [], [], None)
+        self.tbl = tbl
+        target = tbl.tbl_version  # the stored table we're scanning
+        self.sql_exprs = exprs.ExprSet(select_list)
+        # unstored iter columns: we also need to retrieve whatever is needed to materialize the iter args
+        for iter_arg in row_builder.unstored_iter_args.values():
+            sql_subexprs = iter_arg.subexprs(filter=lambda e: e.sql_expr() is not None, traverse_matches=False)
+            [self.sql_exprs.append(e) for e in sql_subexprs]
+        self.filter = filter
+        self.filter_eval_ctx = \
+            row_builder.create_eval_ctx([filter], exclude=select_list) if filter is not None else None
+        self.limit = limit
+
+        # change rowid refs against a base table to rowid refs against the target table, so that we minimize
+        # the number of tables that need to be joined to the target table
+        for rowid_ref in [e for e in self.sql_exprs if isinstance(e, exprs.RowidRef)]:
+            rowid_ref.set_tbl(tbl)
+
+        where_clause_tbl_ids = where_clause.tbl_ids() if where_clause is not None else set()
+        refd_tbl_ids = exprs.Expr.list_tbl_ids(self.sql_exprs) | where_clause_tbl_ids
+        sql_select_list = [e.sql_expr() for e in self.sql_exprs]
+        assert len(sql_select_list) == len(self.sql_exprs)
+        assert all([e is not None for e in sql_select_list])
+        self.set_pk = set_pk
+        self.num_pk_cols = 0
+        if set_pk:
+            # we also need to retrieve the pk columns
+            pk_columns = target.store_tbl.pk_columns()
+            self.num_pk_cols = len(pk_columns)
+            sql_select_list += pk_columns
+
+        self.stmt = sql.select(*sql_select_list)
+        self.stmt = self.create_from_clause(
+            tbl, self.stmt, refd_tbl_ids, exact_version_only={t.id for t in exact_version_only})
+
+        # change rowid refs against a base table to rowid refs against the target table, so that we minimize
+        # the number of tables that need to be joined to the target table
+        for rowid_ref in [e for e, _ in order_by_items if isinstance(e, exprs.RowidRef)]:
+            rowid_ref.set_tbl(tbl)
+        order_by_clause: List[sql.ClauseElement] = []
+        for e, asc in order_by_items:
+            if isinstance(e, exprs.SimilarityExpr):
+                order_by_clause.append(e.as_order_by_clause(asc))
+            else:
+                order_by_clause.append(e.sql_expr().desc() if not asc else e.sql_expr())
+
+        if where_clause is not None:
+            sql_where_clause = where_clause.sql_expr()
+            assert sql_where_clause is not None
+            self.stmt = self.stmt.where(sql_where_clause)
+        if len(order_by_clause) > 0:
+            self.stmt = self.stmt.order_by(*order_by_clause)
+        elif target.id in row_builder.unstored_iter_args:
+            # we are referencing unstored iter columns from this view and try to order by our primary key,
+            # which ensures that iterators will see monotonically increasing pos values
+            self.stmt = self.stmt.order_by(*self.tbl.store_tbl.rowid_columns())
+        if limit != 0 and self.filter is None:
+            # if we need to do post-SQL filtering, we can't use LIMIT
+            self.stmt = self.stmt.limit(limit)
+
+        self.result_cursor: Optional[sql.engine.CursorResult] = None
+
+        try:
+            # log stmt, if possible
+            stmt_str = str(self.stmt.compile(compile_kwargs={'literal_binds': True}))
+            _logger.debug(f'SqlScanNode stmt:\n{stmt_str}')
+        except Exception as e:
+            pass
+
+    @classmethod
+    def create_from_clause(
+            cls, tbl: catalog.TableVersionPath, stmt: sql.Select, refd_tbl_ids: Optional[Set[UUID]] = None,
+            exact_version_only: Optional[Set[UUID]] = None
+    ) -> sql.Select:
+        """Add From clause to stmt for tables/views referenced by materialized_exprs
+        Args:
+            tbl: root table of join chain
+            stmt: stmt to add From clause to
+            materialized_exprs: list of exprs that reference tables in the join chain; if empty, include only the root
+            exact_version_only: set of table ids for which we only want to see rows created at the current version
+        Returns:
+            augmented stmt
+        """
+        # we need to include at least the root
+        if refd_tbl_ids is None:
+            refd_tbl_ids = {}
+        if exact_version_only is None:
+            exact_version_only = {}
+        candidates = tbl.get_tbl_versions()
+        assert len(candidates) > 0
+        joined_tbls: List[catalog.TableVersion] = [candidates[0]]
+        for tbl in candidates[1:]:
+            if tbl.id in refd_tbl_ids:
+                joined_tbls.append(tbl)
+
+        first = True
+        for tbl in joined_tbls[::-1]:
+            if first:
+                stmt = stmt.select_from(tbl.store_tbl.sa_tbl)
+                first = False
+            else:
+                # join tbl to prev_tbl on prev_tbl's rowid cols
+                prev_tbl_rowid_cols = prev_tbl.store_tbl.rowid_columns()
+                tbl_rowid_cols = tbl.store_tbl.rowid_columns()
+                rowid_clauses = \
+                    [c1 == c2 for c1, c2 in zip(prev_tbl_rowid_cols, tbl_rowid_cols[:len(prev_tbl_rowid_cols)])]
+                stmt = stmt.join(tbl.store_tbl.sa_tbl, sql.and_(*rowid_clauses))
+            if tbl.id in exact_version_only:
+                stmt = stmt.where(tbl.store_tbl.v_min_col == tbl.version)
+            else:
+                stmt = stmt \
+                    .where(tbl.store_tbl.v_min_col <= tbl.version) \
+                    .where(tbl.store_tbl.v_max_col > tbl.version)
+            prev_tbl = tbl
+        return stmt
+
+    def _log_explain(self, conn: sql.engine.Connection) -> None:
+        try:
+            # don't set dialect=Env.get().engine.dialect: x % y turns into x %% y, which results in a syntax error
+            stmt_str = str(self.stmt.compile(compile_kwargs={'literal_binds': True}))
+            explain_result = self.ctx.conn.execute(sql.text(f'EXPLAIN {stmt_str}'))
+            explain_str = '\n'.join([str(row) for row in explain_result])
+            _logger.debug(f'SqlScanNode explain:\n{explain_str}')
+        except Exception as e:
+            _logger.warning(f'EXPLAIN failed')
+
+    def __next__(self) -> DataRowBatch:
+        if self.result_cursor is None:
+            # run the query; do this here rather than in _open(), exceptions are only expected during iteration
+            assert self.ctx.conn is not None
+            try:
+                self._log_explain(self.ctx.conn)
+                with warnings.catch_warnings(record=True) as w:
+                    self.result_cursor = self.ctx.conn.execute(self.stmt)
+                    for warning in w:
+                        pass
+                self.has_more_rows = True
+            except Exception as e:
+                self.has_more_rows = False
+                raise e
+
+        if not self.has_more_rows:
+            raise StopIteration
+
+        output_batch = DataRowBatch(self.tbl.tbl_version, self.row_builder)
+        needs_row = True
+        while self.ctx.batch_size == 0 or len(output_batch) < self.ctx.batch_size:
+            try:
+                sql_row = next(self.result_cursor)
+            except StopIteration:
+                self.has_more_rows = False
+                break
+
+            if needs_row:
+                output_row = output_batch.add_row()
+            if self.num_pk_cols > 0:
+                output_row.set_pk(tuple(sql_row[-self.num_pk_cols:]))
+            # copy the output of the SQL query into the output row
+            for i, e in enumerate(self.sql_exprs):
+                slot_idx = e.slot_idx
+                output_row[slot_idx] = sql_row[i]
+            if self.filter is not None:
+                self.row_builder.eval(output_row, self.filter_eval_ctx, profile=self.ctx.profile)
+                if output_row[self.filter.slot_idx]:
+                    needs_row = True
+                    if self.limit > 0 and len(output_batch) >= self.limit:
+                        self.has_more_rows = False
+                        break
+                else:
+                    # we re-use this row for the next sql row if it didn't pass the filter
+                    needs_row = False
+                    output_row.clear()
+
+        if not needs_row:
+            # the last row didn't pass the filter
+            assert self.filter is not None
+            output_batch.pop_row()
+
+        _logger.debug(f'SqlScanNode: returning {len(output_batch)} rows')
+        if len(output_batch) == 0:
+            raise StopIteration
+        return output_batch
+
+    def _close(self) -> None:
+        if self.result_cursor is not None:
+            self.result_cursor.close()
+

pixeltable/exprs/__init__.py

@@ -0,0 +1,25 @@
+from .arithmetic_expr import ArithmeticExpr
+from .array_slice import ArraySlice
+from .column_property_ref import ColumnPropertyRef
+from .column_ref import ColumnRef
+from .comparison import Comparison
+from .compound_predicate import CompoundPredicate
+from .data_row import DataRow
+from .expr import Expr
+from .expr_set import ExprSet
+from .function_call import FunctionCall
+from .image_member_access import ImageMemberAccess
+from .in_predicate import InPredicate
+from .inline_array import InlineArray
+from .inline_dict import InlineDict
+from .is_null import IsNull
+from .json_mapper import JsonMapper
+from .json_path import RELATIVE_PATH_ROOT, JsonPath
+from .literal import Literal
+from .object_ref import ObjectRef
+from .predicate import Predicate
+from .row_builder import RowBuilder, ColumnSlotIdx, ExecProfile
+from .rowid_ref import RowidRef
+from .similarity_expr import SimilarityExpr
+from .type_cast import TypeCast
+from .variable import Variable

pixeltable/exprs/arithmetic_expr.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+from typing import Optional, List, Any, Dict, Tuple
+
+import sqlalchemy as sql
+
+from .globals import ArithmeticOperator
+from .expr import Expr
+from .data_row import DataRow
+from .row_builder import RowBuilder
+import pixeltable.exceptions as excs
+import pixeltable.catalog as catalog
+import pixeltable.type_system as ts
+
+
+class ArithmeticExpr(Expr):
+    """
+    Allows arithmetic exprs on json paths
+    """
+    def __init__(self, operator: ArithmeticOperator, op1: Expr, op2: Expr):
+        # TODO: determine most specific common supertype
+        if op1.col_type.is_json_type() or op2.col_type.is_json_type():
+            # we assume it's a float
+            super().__init__(ts.FloatType())
+        else:
+            super().__init__(ts.ColumnType.supertype(op1.col_type, op2.col_type))
+        self.operator = operator
+        self.components = [op1, op2]
+
+        # do typechecking after initialization in order for __str__() to work
+        if not op1.col_type.is_numeric_type() and not op1.col_type.is_json_type():
+            raise excs.Error(f'{self}: {operator} requires numeric types, but {op1} has type {op1.col_type}')
+        if not op2.col_type.is_numeric_type() and not op2.col_type.is_json_type():
+            raise excs.Error(f'{self}: {operator} requires numeric types, but {op2} has type {op2.col_type}')
+
+        self.id = self._create_id()
+
+    def __str__(self) -> str:
+        # add parentheses around operands that are ArithmeticExprs to express precedence
+        op1_str = f'({self._op1})' if isinstance(self._op1, ArithmeticExpr) else str(self._op1)
+        op2_str = f'({self._op2})' if isinstance(self._op2, ArithmeticExpr) else str(self._op2)
+        return f'{op1_str} {str(self.operator)} {op2_str}'
+
+    def _equals(self, other: ArithmeticExpr) -> bool:
+        return self.operator == other.operator
+
+    def _id_attrs(self) -> List[Tuple[str, Any]]:
+        return super()._id_attrs() + [('operator', self.operator.value)]
+
+    @property
+    def _op1(self) -> Expr:
+        return self.components[0]
+
+    @property
+    def _op2(self) -> Expr:
+        return self.components[1]
+
+    def sql_expr(self) -> Optional[sql.ClauseElement]:
+        left = self._op1.sql_expr()
+        right = self._op2.sql_expr()
+        if left is None or right is None:
+            return None
+        if self.operator == ArithmeticOperator.ADD:
+            return left + right
+        if self.operator == ArithmeticOperator.SUB:
+            return left - right
+        if self.operator == ArithmeticOperator.MUL:
+            return left * right
+        if self.operator == ArithmeticOperator.DIV:
+            return left / right
+        if self.operator == ArithmeticOperator.MOD:
+            return left % right
+
+    def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
+        op1_val = data_row[self._op1.slot_idx]
+        op2_val = data_row[self._op2.slot_idx]
+        # check types if we couldn't do that prior to execution
+        if self._op1.col_type.is_json_type() and not isinstance(op1_val, int) and not isinstance(op1_val, float):
+            raise excs.Error(
+                f'{self.operator} requires numeric type, but {self._op1} has type {type(op1_val).__name__}')
+        if self._op2.col_type.is_json_type() and not isinstance(op2_val, int) and not isinstance(op2_val, float):
+            raise excs.Error(
+                f'{self.operator} requires numeric type, but {self._op2} has type {type(op2_val).__name__}')
+
+        if self.operator == ArithmeticOperator.ADD:
+            data_row[self.slot_idx] = op1_val + op2_val
+        elif self.operator == ArithmeticOperator.SUB:
+            data_row[self.slot_idx] = op1_val - op2_val
+        elif self.operator == ArithmeticOperator.MUL:
+            data_row[self.slot_idx] = op1_val * op2_val
+        elif self.operator == ArithmeticOperator.DIV:
+            data_row[self.slot_idx] = op1_val / op2_val
+        elif self.operator == ArithmeticOperator.MOD:
+            data_row[self.slot_idx] = op1_val % op2_val
+
+    def _as_dict(self) -> Dict:
+        return {'operator': self.operator.value, **super()._as_dict()}
+
+    @classmethod
+    def _from_dict(cls, d: Dict, components: List[Expr]) -> Expr:
+        assert 'operator' in d
+        assert len(components) == 2
+        return cls(ArithmeticOperator(d['operator']), components[0], components[1])

pixeltable/exprs/array_slice.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+from typing import Optional, List, Any, Dict, Tuple
+import copy
+
+import sqlalchemy as sql
+
+from .expr import Expr
+from .globals import print_slice
+from .data_row import DataRow
+from .row_builder import RowBuilder
+import pixeltable.catalog as catalog
+
+
+class ArraySlice(Expr):
+    """
+    Slice operation on an array, eg, t.array_col[:, 1:2].
+    """
+    def __init__(self, arr: Expr, index: Tuple):
+        assert arr.col_type.is_array_type()
+        # determine result type
+        super().__init__(arr.col_type)
+        self.components = [arr]
+        self.index = index
+        self.id = self._create_id()
+
+    def __str__(self) -> str:
+        index_strs: List[str] = []
+        for el in self.index:
+            if isinstance(el, int):
+                index_strs.append(str(el))
+            if isinstance(el, slice):
+                index_strs.append(print_slice(el))
+        return f'{self._array}[{", ".join(index_strs)}]'
+
+    @property
+    def _array(self) -> Expr:
+        return self.components[0]
+
+    def _equals(self, other: ArraySlice) -> bool:
+        return self.index == other.index
+
+    def _id_attrs(self) -> List[Tuple[str, Any]]:
+        return super()._id_attrs() + [('index', self.index)]
+
+    def sql_expr(self) -> Optional[sql.ClauseElement]:
+        return None
+
+    def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
+        val = data_row[self._array.slot_idx]
+        data_row[self.slot_idx] = val[self.index]
+
+    def _as_dict(self) -> Dict:
+        index = []
+        for el in self.index:
+            if isinstance(el, slice):
+                index.append([el.start, el.stop, el.step])
+            else:
+                index.append(el)
+        return {'index': index, **super()._as_dict()}
+
+    @classmethod
+    def _from_dict(cls, d: Dict, components: List[Expr]) -> Expr:
+        assert 'index' in d
+        index = []
+        for el in d['index']:
+            if isinstance(el, list):
+                index.append(slice(el[0], el[1], el[2]))
+            else:
+                index.append(el)
+        return cls(components[0], tuple(index))
+

pixeltable/exprs/column_property_ref.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+from typing import Optional, List, Any, Dict, Tuple
+import enum
+
+import sqlalchemy as sql
+
+from .expr import Expr
+from .column_ref import ColumnRef
+from .row_builder import RowBuilder
+from .data_row import DataRow
+import pixeltable.catalog as catalog
+import pixeltable.type_system as ts
+
+
+class ColumnPropertyRef(Expr):
+    """A reference to a property of a table column
+
+    The properties themselves are type-specific and may or may not need to reference the underlying column data.
+    """
+    class Property(enum.Enum):
+        ERRORTYPE = 0
+        ERRORMSG = 1
+        FILEURL = 2
+        LOCALPATH = 3
+
+    def __init__(self, col_ref: ColumnRef, prop: Property):
+        super().__init__(ts.StringType(nullable=True))
+        self.components = [col_ref]
+        self.prop = prop
+        self.id = self._create_id()
+
+    def default_column_name(self) -> Optional[str]:
+        return str(self).replace('.', '_')
+
+    def _equals(self, other: ColumnRef) -> bool:
+        return self.prop == other.prop
+
+    def _id_attrs(self) -> List[Tuple[str, Any]]:
+        return super()._id_attrs() + [('prop', self.prop.value)]
+
+    @property
+    def _col_ref(self) -> ColumnRef:
+        return self.components[0]
+
+    def __str__(self) -> str:
+        return f'{self._col_ref}.{self.prop.name.lower()}'
+
+    def sql_expr(self) -> Optional[sql.ClauseElement]:
+        if not self._col_ref.col.is_stored:
+            return None
+        if self.prop == self.Property.ERRORTYPE:
+            assert self._col_ref.col.sa_errortype_col is not None
+            return self._col_ref.col.sa_errortype_col
+        if self.prop == self.Property.ERRORMSG:
+            assert self._col_ref.col.sa_errormsg_col is not None
+            return self._col_ref.col.sa_errormsg_col
+        if self.prop == self.Property.FILEURL:
+            # the file url is stored as the column value
+            return self._col_ref.sql_expr()
+        return None
+
+    def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
+        assert self.prop == self.Property.FILEURL or self.prop == self.Property.LOCALPATH
+        assert data_row.has_val[self._col_ref.slot_idx]
+        if self.prop == self.Property.FILEURL:
+            data_row[self.slot_idx] = data_row.file_urls[self._col_ref.slot_idx]
+        if self.prop == self.Property.LOCALPATH:
+            data_row[self.slot_idx] = data_row.file_paths[self._col_ref.slot_idx]
+
+    def _as_dict(self) -> Dict:
+        return {'prop': self.prop.value, **super()._as_dict()}
+
+    @classmethod
+    def _from_dict(cls, d: Dict, components: List[Expr]) -> Expr:
+        assert 'prop' in d
+        return cls(components[0], cls.Property(d['prop']))
+

pixeltable/exprs/column_ref.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+from typing import Optional, List, Any, Dict, Tuple
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from .expr import Expr
+from .data_row import DataRow
+from .row_builder import RowBuilder
+import pixeltable.iterators as iters
+import pixeltable.exceptions as excs
+import pixeltable.catalog as catalog
+
+
+class ColumnRef(Expr):
+    """A reference to a table column
+
+    When this reference is created in the context of a view, it can also refer to a column of the view base.
+    For that reason, a ColumnRef needs to be serialized with the qualifying table id (column ids are only
+    unique in the context of a particular table).
+    """
+    def __init__(self, col: catalog.Column):
+        super().__init__(col.col_type)
+        assert col.tbl is not None
+        self.col = col
+        self.is_unstored_iter_col = \
+            col.tbl.is_component_view() and col.tbl.is_iterator_column(col) and not col.is_stored
+        self.iter_arg_ctx: Optional[RowBuilder.EvalCtx] = None
+        # number of rowid columns in the base table
+        self.base_rowid_len = col.tbl.base.num_rowid_columns() if self.is_unstored_iter_col else 0
+        self.base_rowid = [None] * self.base_rowid_len
+        self.iterator: Optional[iters.ComponentIterator] = None
+        # index of the position column in the view's primary key; don't try to reference tbl.store_tbl here
+        self.pos_idx: Optional[int] = col.tbl.num_rowid_columns() - 1 if self.is_unstored_iter_col else None
+        self.id = self._create_id()
+
+    def set_iter_arg_ctx(self, iter_arg_ctx: RowBuilder.EvalCtx) -> None:
+        self.iter_arg_ctx = iter_arg_ctx
+        assert len(self.iter_arg_ctx.target_slot_idxs) == 1  # a single inline dict
+
+    def _id_attrs(self) -> List[Tuple[str, Any]]:
+        return super()._id_attrs() + [('tbl_id', self.col.tbl.id), ('col_id', self.col.id)]
+
+    def __getattr__(self, name: str) -> Expr:
+        from .column_property_ref import ColumnPropertyRef
+        # resolve column properties
+        if name == ColumnPropertyRef.Property.ERRORTYPE.name.lower() \
+                or name == ColumnPropertyRef.Property.ERRORMSG.name.lower():
+            if not (self.col.is_computed and self.col.is_stored) and not self.col.col_type.is_media_type():
+                raise excs.Error(f'{name} only valid for a stored computed or media column: {self}')
+            return ColumnPropertyRef(self, ColumnPropertyRef.Property[name.upper()])
+        if name == ColumnPropertyRef.Property.FILEURL.name.lower() \
+                or name == ColumnPropertyRef.Property.LOCALPATH.name.lower():
+            if not self.col.col_type.is_media_type():
+                raise excs.Error(f'{name} only valid for image/video/audio/document columns: {self}')
+            if self.col.is_computed and not self.col.is_stored:
+                raise excs.Error(f'{name} not valid for computed unstored columns: {self}')
+            return ColumnPropertyRef(self, ColumnPropertyRef.Property[name.upper()])
+
+        if self.col_type.is_json_type():
+            from .json_path import JsonPath
+            return JsonPath(self, [name])
+
+        return super().__getattr__(name)
+
+    def similarity(self, other: Any) -> Expr:
+        if isinstance(other, Expr):
+            raise excs.Error(f'similarity(): requires a string or a PIL.Image.Image object, not an expression')
+        item = Expr.from_object(other)
+        if item is None or not(item.col_type.is_string_type() or item.col_type.is_image_type()):
+            raise excs.Error(f'similarity(): requires a string or a PIL.Image.Image object, not a {type(other)}')
+        from .similarity_expr import SimilarityExpr
+        return SimilarityExpr(self, item)
+
+    def default_column_name(self) -> Optional[str]:
+        return str(self)
+
+    def _equals(self, other: ColumnRef) -> bool:
+        return self.col == other.col
+
+    def __str__(self) -> str:
+        return self.col.name
+
+    def sql_expr(self) -> Optional[sql.ClauseElement]:
+        return self.col.sa_col
+
+    def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
+        if not self.is_unstored_iter_col:
+            assert data_row.has_val[self.slot_idx]
+            return
+
+        # if this is a new base row, we need to instantiate a new iterator
+        if self.base_rowid != data_row.pk[:self.base_rowid_len]:
+            row_builder.eval(data_row, self.iter_arg_ctx)
+            iterator_args = data_row[self.iter_arg_ctx.target_slot_idxs[0]]
+            self.iterator = self.col.tbl.iterator_cls(**iterator_args)
+            self.base_rowid = data_row.pk[:self.base_rowid_len]
+        self.iterator.set_pos(data_row.pk[self.pos_idx])
+        res = next(self.iterator)
+        data_row[self.slot_idx] = res[self.col.name]
+
+    def _as_dict(self) -> Dict:
+        tbl = self.col.tbl
+        version = tbl.version if tbl.is_snapshot else None
+        return {'tbl_id': str(tbl.id), 'tbl_version': version, 'col_id': self.col.id}
+
+    @classmethod
+    def _from_dict(cls, d: Dict, components: List[Expr]) -> Expr:
+        tbl_id, version, col_id = UUID(d['tbl_id']), d['tbl_version'], d['col_id']
+        tbl_version = catalog.Catalog.get().tbl_versions[(tbl_id, version)]
+        assert col_id in tbl_version.cols_by_id
+        col = tbl_version.cols_by_id[col_id]
+        return cls(col)
+