pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.2__py3-none-any.whl

pixeltable/catalog/view.py

@@ -41,6 +41,8 @@ class View(Table):
     def __init__(self, id: UUID, dir_id: UUID, name: str, tbl_version_path: TableVersionPath, snapshot_only: bool):
         super().__init__(id, dir_id, name, tbl_version_path)
         self._snapshot_only = snapshot_only
+        if not snapshot_only:
+            self._tbl_version = tbl_version_path.tbl_version
 
     @classmethod
     def _display_name(cls) -> str:
@@ -227,7 +229,7 @@ class View(Table):
 
             try:
                 plan, _ = Planner.create_view_load_plan(view._tbl_version_path)
-                num_rows, num_excs, _ = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
+                _, status = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
             except:
                 # we need to remove the orphaned TableVersion instance
                 del catalog.Catalog.get()._tbl_versions[tbl_version.id, tbl_version.effective_version]
@@ -236,7 +238,9 @@ class View(Table):
                     # also remove tbl_version from the base
                     base_tbl_version.mutable_views.remove(TableVersionHandle.create(tbl_version))
                 raise
-            Env.get().console_logger.info(f'Created view `{name}` with {num_rows} rows, {num_excs} exceptions.')
+            Env.get().console_logger.info(
+                f'Created view `{name}` with {status.num_rows} rows, {status.num_excs} exceptions.'
+            )
 
         session.commit()
         return view
@@ -267,17 +271,8 @@ class View(Table):
             base=cls._get_snapshot_path(tbl_version_path.base) if tbl_version_path.base is not None else None,
         )
 
-    def _drop(self) -> None:
-        if self._snapshot_only:
-            # there is not TableVersion to drop
-            self._check_is_dropped()
-            self.is_dropped = True
-            catalog.Catalog.get().delete_tbl_md(self._id)
-        else:
-            super()._drop()
-
-    def get_metadata(self) -> dict[str, Any]:
-        md = super().get_metadata()
+    def _get_metadata(self) -> dict[str, Any]:
+        md = super()._get_metadata()
         md['is_view'] = True
         md['is_snapshot'] = self._tbl_version_path.is_snapshot()
         return md
@@ -298,11 +293,10 @@ class View(Table):
     def delete(self, where: Optional[exprs.Expr] = None) -> UpdateStatus:
         raise excs.Error(f'{self._display_name()} {self._name!r}: cannot delete from view')
 
-    @property
-    def _base_table(self) -> Optional['Table']:
+    def _get_base_table(self) -> Optional['Table']:
         # if this is a pure snapshot, our tbl_version_path only reflects the base (there is no TableVersion instance
         # for the snapshot itself)
-        base_id = self._tbl_version.id if self._snapshot_only else self._tbl_version_path.base.tbl_version.id
+        base_id = self._tbl_version_path.tbl_id if self._snapshot_only else self._tbl_version_path.base.tbl_id
         return catalog.Catalog.get().get_table_by_id(base_id)
 
     @property
@@ -317,7 +311,7 @@ class View(Table):
         display_name = 'Snapshot' if self._snapshot_only else 'View'
         result = [f'{display_name} {self._path()!r}']
         bases_descrs: list[str] = []
-        for base, effective_version in zip(self._base_tables, self._effective_base_versions):
+        for base, effective_version in zip(self._get_base_tables(), self._effective_base_versions):
             if effective_version is None:
                 bases_descrs.append(f'{base._path()!r}')
             else:
@@ -325,8 +319,8 @@ class View(Table):
                 bases_descrs.append(f'{base_descr!r}')
         result.append(f' (of {", ".join(bases_descrs)})')
 
-        if self._tbl_version.get().predicate is not None:
-            result.append(f'\nWhere: {self._tbl_version.get().predicate!s}')
-        if self._tbl_version.get().sample_clause is not None:
+        if self._tbl_version_path.tbl_version.get().predicate is not None:
+            result.append(f'\nWhere: {self._tbl_version_path.tbl_version.get().predicate!s}')
+        if self._tbl_version_path.tbl_version.get().sample_clause is not None:
             result.append(f'\nSample: {self._tbl_version.get().sample_clause!s}')
         return ''.join(result)

pixeltable/config.py

@@ -86,6 +86,10 @@ class Config:
             return None
 
         try:
+            if expected_type is bool and isinstance(value, str):
+                if value.lower() not in ('true', 'false'):
+                    raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}')
+                return value.lower() == 'true'  # type: ignore[return-value]
             return expected_type(value)  # type: ignore[call-arg]
         except ValueError as exc:
             raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}') from exc

pixeltable/dataframe.py

@@ -475,7 +475,9 @@ class DataFrame:
         raise excs.Error(msg) from e
 
     def _output_row_iterator(self) -> Iterator[list]:
-        with Catalog.get().begin_xact(for_write=False):
+        # TODO: extend begin_xact() to accept multiple TVPs for joins
+        single_tbl = self._first_tbl if len(self._from_clause.tbls) == 1 else None
+        with Catalog.get().begin_xact(tbl=single_tbl, for_write=False):
             try:
                 for data_row in self._exec():
                     yield [data_row[e.slot_idx] for e in self._select_list_exprs]
@@ -507,7 +509,7 @@ class DataFrame:
 
         from pixeltable.plan import Planner
 
-        with Catalog.get().begin_xact(for_write=False) as conn:
+        with Catalog.get().begin_xact(tbl=self._first_tbl, for_write=False) as conn:
             stmt = Planner.create_count_stmt(self._first_tbl, self.where_clause)
             result: int = conn.execute(stmt).scalar_one()
             assert isinstance(result, int)
@@ -903,7 +905,7 @@ class DataFrame:
                 grouping_tbl = item if isinstance(item, catalog.TableVersion) else item._tbl_version.get()
                 # we need to make sure that the grouping table is a base of self.tbl
                 base = self._first_tbl.find_tbl_version(grouping_tbl.id)
-                if base is None or base.id == self._first_tbl.tbl_id():
+                if base is None or base.id == self._first_tbl.tbl_id:
                     raise excs.Error(
                         f'group_by(): {grouping_tbl.name} is not a base table of {self._first_tbl.tbl_name()}'
                     )
@@ -1161,8 +1163,7 @@ class DataFrame:
             >>> df = person.where(t.year == 2014).update({'age': 30})
         """
         self._validate_mutable('update', False)
-        tbl_id = self._first_tbl.tbl_id()
-        with Catalog.get().begin_xact(tbl_id=tbl_id, for_write=True):
+        with Catalog.get().begin_xact(tbl=self._first_tbl, for_write=True, lock_mutable_tree=True):
             return self._first_tbl.tbl_version.get().update(value_spec, where=self.where_clause, cascade=cascade)
 
     def delete(self) -> UpdateStatus:
@@ -1185,8 +1186,7 @@ class DataFrame:
         self._validate_mutable('delete', False)
         if not self._first_tbl.is_insertable():
             raise excs.Error('Cannot delete from view')
-        tbl_id = self._first_tbl.tbl_id()
-        with Catalog.get().begin_xact(tbl_id=tbl_id, for_write=True):
+        with Catalog.get().begin_xact(tbl=self._first_tbl, for_write=True, lock_mutable_tree=True):
             return self._first_tbl.tbl_version.get().delete(where=self.where_clause)
 
     def _validate_mutable(self, op_name: str, allow_select: bool) -> None:
@@ -1307,7 +1307,8 @@ class DataFrame:
             assert data_file_path.is_file()
             return data_file_path
         else:
-            with Catalog.get().begin_xact(for_write=False):
+            # TODO: extend begin_xact() to accept multiple TVPs for joins
+            with Catalog.get().begin_xact(tbl=self._first_tbl, for_write=False):
                 return write_coco_dataset(self, dest_path)
 
     def to_pytorch_dataset(self, image_format: str = 'pt') -> 'torch.utils.data.IterableDataset':
@@ -1352,7 +1353,7 @@ class DataFrame:
         if dest_path.exists():  # fast path: use cache
             assert dest_path.is_dir()
         else:
-            with Catalog.get().begin_xact(for_write=False):
+            with Catalog.get().begin_xact(tbl=self._first_tbl, for_write=False):
                 export_parquet(self, dest_path, inline_images=True)
 
         return PixeltablePytorchDataset(path=dest_path, image_format=image_format)

pixeltable/env.py

@@ -10,7 +10,6 @@ import logging
 import os
 import platform
 import shutil
-import subprocess
 import sys
 import threading
 import uuid
@@ -611,9 +610,11 @@ class Env:
         self.__register_package('fiftyone')
         self.__register_package('fireworks', library_name='fireworks-ai')
         self.__register_package('google.genai', library_name='google-genai')
+        self.__register_package('groq')
         self.__register_package('huggingface_hub', library_name='huggingface-hub')
         self.__register_package('label_studio_sdk', library_name='label-studio-sdk')
         self.__register_package('llama_cpp', library_name='llama-cpp-python')
+        self.__register_package('mcp')
         self.__register_package('mistralai')
         self.__register_package('mistune')
         self.__register_package('ollama')
@@ -746,18 +747,11 @@ class Env:
         have no sub-dependencies (in fact, this is how spaCy normally manages its model resources).
         """
         import spacy
-        from spacy.cli.download import get_model_filename
+        from spacy.cli.download import download
 
         spacy_model = 'en_core_web_sm'
-        spacy_model_version = '3.7.1'
-        filename = get_model_filename(spacy_model, spacy_model_version, sdist=False)
-        url = f'{spacy.about.__download_url__}/{filename}'
-        # Try to `pip install` the model. We set check=False; if the pip command fails, it's not necessarily
-        # a problem, because the model might have been installed on a previous attempt.
-        self._logger.info(f'Ensuring spaCy model is installed: {filename}')
-        ret = subprocess.run([sys.executable, '-m', 'pip', 'install', '-qU', url], check=False)
-        if ret.returncode != 0:
-            self._logger.warning(f'pip install failed for spaCy model: {filename}')
+        self._logger.info(f'Ensuring spaCy model is installed: {spacy_model}')
+        download(spacy_model)
         self._logger.info(f'Loading spaCy model: {spacy_model}')
         try:
             self._spacy_nlp = spacy.load(spacy_model)

pixeltable/exceptions.py

@@ -10,6 +10,12 @@ class Error(Exception):
 
 
 class ExprEvalError(Exception):
+    """
+    Used during query execution to signal expr evaluation failures.
+
+    NOT A USER-FACING EXCEPTION. All ExprEvalError instances need to be converted into Error instances.
+    """
+
     expr: 'exprs.Expr'
     expr_msg: str
     exc: Exception

pixeltable/exec/exec_node.py

@@ -73,6 +73,8 @@ class ExecNode(abc.ABC):
         except RuntimeError:
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
+            # we set a deliberately long duration to avoid warnings getting printed to the console in debug mode
+            loop.slow_callback_duration = 3600
 
         if _logger.isEnabledFor(logging.DEBUG):
             loop.set_debug(True)

pixeltable/exec/expr_eval/expr_eval_node.py

@@ -49,7 +49,7 @@ class ExprEvalNode(ExecNode):
     # execution state
     tasks: set[asyncio.Task]  # collects all running tasks to prevent them from getting gc'd
     exc_event: asyncio.Event  # set if an exception needs to be propagated
-    error: Optional[Union[excs.Error, excs.ExprEvalError]]  # exception that needs to be propagated
+    error: Optional[Union[Exception]]  # exception that needs to be propagated
     completed_rows: asyncio.Queue[exprs.DataRow]  # rows that have completed evaluation
     completed_event: asyncio.Event  # set when completed_rows is non-empty
     input_iter: AsyncIterator[DataRowBatch]
@@ -133,10 +133,10 @@ class ExprEvalNode(ExecNode):
         except StopAsyncIteration:
             self.input_complete = True
             _logger.debug(f'finished input: #input_rows={self.num_input_rows}, #avail={self.avail_input_rows}')
-        except excs.Error as err:
-            self.error = err
+        # make sure to pass DBAPIError through, so the transaction handling logic sees it
+        except Exception as exc:
+            self.error = exc
             self.exc_event.set()
-        # TODO: should we also handle Exception here and create an excs.Error from it?
 
     @property
     def total_buffered(self) -> int:

pixeltable/exec/sql_node.py

@@ -308,8 +308,7 @@ class SqlNode(ExecNode):
                 _logger.debug(f'SqlLookupNode stmt:\n{stmt_str}')
             except Exception:
                 # log something if we can't log the compiled stmt
-                stmt_str = repr(stmt)
-                _logger.debug(f'SqlLookupNode proto-stmt:\n{stmt_str}')
+                _logger.debug(f'SqlLookupNode proto-stmt:\n{stmt}')
             self._log_explain(stmt)
 
             conn = Env.get().conn
@@ -530,40 +529,39 @@ class SqlJoinNode(SqlNode):
 
 class SqlSampleNode(SqlNode):
     """
-    Returns rows from a stratified sample with N samples per strata.
+    Returns rows sampled from the input node.
     """
 
-    stratify_exprs: Optional[list[exprs.Expr]]
-    n_samples: Optional[int]
-    fraction_samples: Optional[float]
-    seed: int
     input_cte: Optional[sql.CTE]
     pk_count: int
+    stratify_exprs: Optional[list[exprs.Expr]]
+    sample_clause: 'SampleClause'
 
     def __init__(
         self,
         row_builder: exprs.RowBuilder,
         input: SqlNode,
         select_list: Iterable[exprs.Expr],
-        stratify_exprs: Optional[list[exprs.Expr]] = None,
-        sample_clause: Optional['SampleClause'] = None,
+        sample_clause: 'SampleClause',
+        stratify_exprs: list[exprs.Expr],
     ):
         """
         Args:
+            input: SqlNode to sample from
             select_list: can contain calls to AggregateFunctions
-            stratify_exprs: list of expressions to group by
-            n: number of samples per strata
+            sample_clause: specifies the sampling method
+            stratify_exprs: Analyzer processed list of expressions to stratify by.
         """
+        assert isinstance(input, SqlNode)
         self.input_cte, input_col_map = input.to_cte(keep_pk=True)
         self.pk_count = input.num_pk_cols
         assert self.pk_count > 1
         sql_elements = exprs.SqlElementCache(input_col_map)
+        assert sql_elements.contains_all(stratify_exprs)
         super().__init__(input.tbl, row_builder, select_list, sql_elements, set_pk=True)
         self.stratify_exprs = stratify_exprs
-        self.n_samples = sample_clause.n
-        self.n_per_stratum = sample_clause.n_per_stratum
-        self.fraction_samples = sample_clause.fraction
-        self.seed = sample_clause.seed if sample_clause.seed is not None else 0
+        self.sample_clause = sample_clause
+        assert isinstance(self.sample_clause.seed, int)
 
     @classmethod
     def key_sql_expr(cls, seed: sql.ColumnElement, sql_cols: Iterable[sql.ColumnElement]) -> sql.ColumnElement:
@@ -573,25 +571,44 @@ class SqlSampleNode(SqlNode):
         """
         sql_expr: sql.ColumnElement = sql.cast(seed, sql.Text)
         for e in sql_cols:
-            sql_expr = sql_expr + sql.literal_column("'___'") + sql.cast(e, sql.Text)
+            # Quotes are required below to guarantee that the string is properly presented in SQL
+            sql_expr = sql_expr + sql.literal_column("'___'", sql.Text) + sql.cast(e, sql.Text)
         sql_expr = sql.func.md5(sql_expr)
         return sql_expr
 
-    def _create_order_by(self, cte: sql.CTE) -> sql.ColumnElement:
+    def _create_key_sql(self, cte: sql.CTE) -> sql.ColumnElement:
         """Create an expression for randomly ordering rows with a given seed"""
         rowid_cols = [*cte.c[-self.pk_count : -1]]  # exclude the version column
         assert len(rowid_cols) > 0
-        return self.key_sql_expr(sql.literal_column(str(self.seed)), rowid_cols)
+        return self.key_sql_expr(sql.literal_column(str(self.sample_clause.seed)), rowid_cols)
 
     def _create_stmt(self) -> sql.Select:
-        if self.fraction_samples is not None:
-            return self._create_stmt_fraction(self.fraction_samples)
-        return self._create_stmt_n(self.n_samples, self.n_per_stratum)
+        from pixeltable.plan import SampleClause
+
+        if self.sample_clause.fraction is not None:
+            if len(self.stratify_exprs) == 0:
+                # If non-stratified sampling, construct a where clause, order_by, and limit clauses
+                s_key = self._create_key_sql(self.input_cte)
+
+                # Construct a suitable where clause
+                fraction_sql = sql.cast(SampleClause.fraction_to_md5_hex(float(self.sample_clause.fraction)), sql.Text)
+                order_by = self._create_key_sql(self.input_cte)
+                return sql.select(*self.input_cte.c).where(s_key < fraction_sql).order_by(order_by)
+
+            return self._create_stmt_stratified_fraction(self.sample_clause.fraction)
+        else:
+            if len(self.stratify_exprs) == 0:
+                # No stratification, just return n samples from the input CTE
+                order_by = self._create_key_sql(self.input_cte)
+                return sql.select(*self.input_cte.c).order_by(order_by).limit(self.sample_clause.n)
+
+            return self._create_stmt_stratified_n(self.sample_clause.n, self.sample_clause.n_per_stratum)
+
+    def _create_stmt_stratified_n(self, n: Optional[int], n_per_stratum: Optional[int]) -> sql.Select:
+        """Create a Select stmt that returns n samples across all strata or n_per_stratum samples per stratum"""
 
-    def _create_stmt_n(self, n: Optional[int], n_per_stratum: Optional[int]) -> sql.Select:
-        """Create a Select stmt that returns n samples across all strata"""
         sql_strata_exprs = [self.sql_elements.get(e) for e in self.stratify_exprs]
-        order_by = self._create_order_by(self.input_cte)
+        order_by = self._create_key_sql(self.input_cte)
 
         # Create a list of all columns plus the rank
         # Get all columns from the input CTE dynamically
@@ -605,15 +622,15 @@ class SqlSampleNode(SqlNode):
         if n_per_stratum is not None:
             return sql.select(*final_columns).filter(row_rank_cte.c.rank <= n_per_stratum)
         else:
-            secondary_order = self._create_order_by(row_rank_cte)
+            secondary_order = self._create_key_sql(row_rank_cte)
             return sql.select(*final_columns).order_by(row_rank_cte.c.rank, secondary_order).limit(n)
 
-    def _create_stmt_fraction(self, fraction_samples: float) -> sql.Select:
+    def _create_stmt_stratified_fraction(self, fraction_samples: float) -> sql.Select:
         """Create a Select stmt that returns a fraction of the rows per strata"""
 
         # Build the strata count CTE
         # Produces a table of the form:
-        #   ([stratify_exprs], s_s_size)
+        #   (*stratify_exprs, s_s_size)
         # where s_s_size is the number of samples to take from each stratum
         sql_strata_exprs = [self.sql_elements.get(e) for e in self.stratify_exprs]
         per_strata_count_cte = (
@@ -628,19 +645,19 @@ class SqlSampleNode(SqlNode):
 
         # Build a CTE that ranks the rows within each stratum
         # Include all columns from the input CTE dynamically
-        order_by = self._create_order_by(self.input_cte)
+        order_by = self._create_key_sql(self.input_cte)
         select_columns = [*self.input_cte.c]
         select_columns.append(
             sql.func.row_number().over(partition_by=sql_strata_exprs, order_by=order_by).label('rank')
         )
         row_rank_cte = sql.select(*select_columns).select_from(self.input_cte).cte('row_rank_cte')
 
-        # Build the join criterion dynamically to accommodate any number of group by columns
+        # Build the join criterion dynamically to accommodate any number of stratify_by expressions
         join_c = sql.true()
         for col in per_strata_count_cte.c[:-1]:
             join_c &= row_rank_cte.c[col.name].isnot_distinct_from(col)
 
-        # Join srcp with per_strata_count_cte to limit returns to the requested fraction of rows
+        # Join with per_strata_count_cte to limit returns to the requested fraction of rows
         final_columns = [*row_rank_cte.c[:-1]]  # exclude the rank column
         stmt = (
             sql.select(*final_columns)

pixeltable/exprs/column_property_ref.py

@@ -55,17 +55,9 @@ class ColumnPropertyRef(Expr):
         return self.prop in (self.Property.ERRORTYPE, self.Property.ERRORMSG)
 
     def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
-        if not self._col_ref.col.is_stored:
+        if not self._col_ref.col_handle.get().is_stored:
             return None
-
-        # we need to reestablish that we have the correct Column instance, there could have been a metadata
-        # reload since init()
-        # TODO: add an explicit prepare phase (ie, Expr.prepare()) that gives every subclass instance a chance to
-        # perform runtime checks and update state
-        tv = self._col_ref.tbl_version.get()
-        assert tv.is_validated
-        col = tv.cols_by_id[self._col_ref.col_id]
-        # TODO: check for column being dropped
+        col = self._col_ref.col_handle.get()
 
         # the errortype/-msg properties of a read-validated media column need to be extracted from the DataRow
         if (

pixeltable/exprs/column_ref.py

@@ -10,6 +10,7 @@ import pixeltable as pxt
 from pixeltable import catalog, exceptions as excs, iterators as iters
 
 from ..utils.description_helper import DescriptionHelper
+from ..utils.filecache import FileCache
 from .data_row import DataRow
 from .expr import Expr
 from .row_builder import RowBuilder
@@ -41,7 +42,8 @@ class ColumnRef(Expr):
     insert them into the EvalCtxs as needed
     """
 
-    col: catalog.Column
+    col: catalog.Column  # TODO: merge with col_handle
+    col_handle: catalog.ColumnHandle
     reference_tbl: Optional[catalog.TableVersionPath]
     is_unstored_iter_col: bool
     iter_arg_ctx: Optional[RowBuilder.EvalCtx]
@@ -52,10 +54,6 @@ class ColumnRef(Expr):
     id: int
     perform_validation: bool  # if True, performs media validation
 
-    # needed by sql_expr() to re-resolve Column instance after a metadata reload
-    tbl_version: catalog.TableVersionHandle
-    col_id: int
-
     def __init__(
         self,
         col: catalog.Column,
@@ -66,8 +64,7 @@ class ColumnRef(Expr):
         assert col.tbl is not None
         self.col = col
         self.reference_tbl = reference_tbl
-        self.tbl_version = catalog.TableVersionHandle(col.tbl.id, col.tbl.effective_version)
-        self.col_id = col.id
+        self.col_handle = catalog.ColumnHandle(col.tbl.handle, col.id)
 
         self.is_unstored_iter_col = col.tbl.is_component_view and col.tbl.is_iterator_column(col) and not col.is_stored
         self.iter_arg_ctx = None
@@ -170,6 +167,20 @@ class ColumnRef(Expr):
             idx_info = embedding_idx_info
         return idx_info
 
+    def recompute(self, *, cascade: bool = True, errors_only: bool = False) -> catalog.UpdateStatus:
+        cat = catalog.Catalog.get()
+        # lock_mutable_tree=True: we need to be able to see whether any transitive view has column dependents
+        with cat.begin_xact(tbl=self.reference_tbl, for_write=True, lock_mutable_tree=True):
+            tbl_version = self.col_handle.tbl_version.get()
+            if tbl_version.id != self.reference_tbl.tbl_id:
+                raise excs.Error('Cannot recompute column of a base.')
+            if tbl_version.is_snapshot:
+                raise excs.Error('Cannot recompute column of a snapshot.')
+            col_name = self.col_handle.get().name
+            status = tbl_version.recompute_columns([col_name], errors_only=errors_only, cascade=cascade)
+            FileCache.get().emit_eviction_warnings()
+            return status
+
     def similarity(self, item: Any, *, idx: Optional[str] = None) -> Expr:
         from .similarity_expr import SimilarityExpr
 
@@ -239,22 +250,9 @@ class ColumnRef(Expr):
         return helper
 
     def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
-        # return None if self.perform_validation else self.col.sa_col
         if self.perform_validation:
             return None
-        # we need to reestablish that we have the correct Column instance, there could have been a metadata
-        # reload since init()
-        # TODO: add an explicit prepare phase (ie, Expr.prepare()) that gives every subclass instance a chance to
-        # perform runtime checks and update state
-        tv = self.tbl_version.get()
-        assert tv.is_validated
-        self.col = tv.cols_by_id[self.col_id]
-        assert self.col.tbl is tv
-        # TODO: check for column being dropped
-        # print(
-        #     f'ColumnRef.sql_expr: tbl={tv.id}:{tv.effective_version} sa_tbl={id(self.col.tbl.store_tbl.sa_tbl):x} '
-        #     f'tv={id(tv):x}'
-        # )
+        self.col = self.col_handle.get()
         return self.col.sa_col
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
@@ -315,6 +313,11 @@ class ColumnRef(Expr):
             'perform_validation': self.perform_validation,
         }
 
+    @classmethod
+    def get_column_id(cls, d: dict) -> catalog.QColumnId:
+        tbl_id, col_id = UUID(d['tbl_id']), d['col_id']
+        return catalog.QColumnId(tbl_id, col_id)
+
     @classmethod
     def get_column(cls, d: dict) -> catalog.Column:
         tbl_id, version, col_id = UUID(d['tbl_id']), d['tbl_version'], d['col_id']

pixeltable/exprs/data_row.py

@@ -42,6 +42,10 @@ class DataRow:
     has_val: np.ndarray  # of bool
     excs: np.ndarray  # of object
 
+    # If `may_have_exc` is False, then we guarantee that no slot has an exception set. This is used to optimize
+    # exception handling under normal operation.
+    _may_have_exc: bool
+
     # expr evaluation state; indexed by slot idx
     missing_slots: np.ndarray  # of bool; number of missing dependencies
     missing_dependents: np.ndarray  # of int16; number of missing dependents
@@ -90,6 +94,7 @@ class DataRow:
         self.vals = np.full(num_slots, None, dtype=object)
         self.has_val = np.zeros(num_slots, dtype=bool)
         self.excs = np.full(num_slots, None, dtype=object)
+        self._may_have_exc = False
         self.missing_slots = np.zeros(num_slots, dtype=bool)
         self.missing_dependents = np.zeros(num_slots, dtype=np.int16)
         self.is_scheduled = np.zeros(num_slots, dtype=bool)
@@ -136,6 +141,9 @@ class DataRow:
         """
         Returns True if an exception has been set for the given slot index, or for any slot index if slot_idx is None
         """
+        if not self._may_have_exc:
+            return False
+
         if slot_idx is not None:
             return self.excs[slot_idx] is not None
         return (self.excs != None).any()
@@ -154,6 +162,7 @@ class DataRow:
     def set_exc(self, slot_idx: int, exc: Exception) -> None:
         assert self.excs[slot_idx] is None
         self.excs[slot_idx] = exc
+        self._may_have_exc = True
 
         # an exception means the value is None
         self.has_val[slot_idx] = True

pixeltable/exprs/expr.py

@@ -394,17 +394,17 @@ class Expr(abc.ABC):
         return {tbl_id for e in exprs_ for tbl_id in e.tbl_ids()}
 
     @classmethod
-    def get_refd_columns(cls, expr_dict: dict[str, Any]) -> list[catalog.Column]:
+    def get_refd_column_ids(cls, expr_dict: dict[str, Any]) -> set[catalog.QColumnId]:
         """Return Columns referenced by expr_dict."""
-        result: list[catalog.Column] = []
+        result: set[catalog.QColumnId] = set()
         assert '_classname' in expr_dict
         from .column_ref import ColumnRef
 
         if expr_dict['_classname'] == 'ColumnRef':
-            result.append(ColumnRef.get_column(expr_dict))
+            result.add(ColumnRef.get_column_id(expr_dict))
         if 'components' in expr_dict:
             for component_dict in expr_dict['components']:
-                result.extend(cls.get_refd_columns(component_dict))
+                result.update(cls.get_refd_column_ids(component_dict))
         return result
 
     def as_literal(self) -> Optional[Expr]: