pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.20__py3-none-any.whl

pixeltable/exprs/arithmetic_expr.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
-from pixeltable import exceptions as excs, type_system as ts
+from pixeltable import env, exceptions as excs, type_system as ts
 
 from .data_row import DataRow
 from .expr import Expr
@@ -58,29 +58,36 @@ class ArithmeticExpr(Expr):
     def _id_attrs(self) -> list[tuple[str, Any]]:
         return [*super()._id_attrs(), ('operator', self.operator.value)]
 
-    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
         assert self.col_type.is_int_type() or self.col_type.is_float_type() or self.col_type.is_json_type()
         left = sql_elements.get(self._op1)
         right = sql_elements.get(self._op2)
         if left is None or right is None:
             return None
-        if self.operator == ArithmeticOperator.ADD:
-            return left + right
-        if self.operator == ArithmeticOperator.SUB:
-            return left - right
-        if self.operator == ArithmeticOperator.MUL:
-            return left * right
+        if self.operator in (ArithmeticOperator.ADD, ArithmeticOperator.SUB, ArithmeticOperator.MUL):
+            if env.Env.get().is_using_cockroachdb and self._op1.col_type != self._op2.col_type:
+                if self._op1.col_type != self.col_type:
+                    left = sql.cast(left, self.col_type.to_sa_type())
+                if self._op2.col_type != self.col_type:
+                    right = sql.cast(right, self.col_type.to_sa_type())
+            if self.operator == ArithmeticOperator.ADD:
+                return left + right
+            if self.operator == ArithmeticOperator.SUB:
+                return left - right
+            if self.operator == ArithmeticOperator.MUL:
+                return left * right
         if self.operator == ArithmeticOperator.DIV:
             assert self.col_type.is_float_type()
-            # Avoid DivisionByZero: if right is 0, make this a NULL
+            # Avoid division by zero errors by converting any zero divisor to NULL.
             # TODO: Should we cast the NULLs to NaNs when they are retrieved back into Python?
-            nullif = sql.sql.func.nullif(right, 0)
-            # We have to cast to a `float`, or else we'll get a `Decimal`
-            return sql.sql.expression.cast(left / nullif, self.col_type.to_sa_type())
+            # These casts cause the computation to take place in float units, rather than DECIMAL.
+            nullif = sql.cast(sql.func.nullif(right, 0), self.col_type.to_sa_type())
+            return sql.cast(left, self.col_type.to_sa_type()) / nullif
         if self.operator == ArithmeticOperator.MOD:
             if self.col_type.is_int_type():
-                nullif = sql.sql.func.nullif(right, 0)
-                return left % nullif
+                # Avoid division by zero errors by converting any zero divisor to NULL.
+                nullif1 = sql.cast(sql.func.nullif(right, 0), self.col_type.to_sa_type())
+                return left % nullif1
             if self.col_type.is_float_type():
                 # Postgres does not support modulus for floats
                 return None
@@ -90,11 +97,9 @@ class ArithmeticExpr(Expr):
             # We need the behavior to be consistent, so that expressions will evaluate the same way
             # whether or not their operands can be translated to SQL. These SQL clauses should
             # mimic the behavior of Python's // operator.
-            nullif = sql.sql.func.nullif(right, 0)
-            if self.col_type.is_int_type():
-                return sql.sql.expression.cast(sql.func.floor(left / nullif), self.col_type.to_sa_type())
-            if self.col_type.is_float_type():
-                return sql.sql.expression.cast(sql.func.floor(left / nullif), self.col_type.to_sa_type())
+            # Avoid division by zero errors by converting any zero divisor to NULL.
+            nullif = sql.cast(sql.func.nullif(right, 0), self.col_type.to_sa_type())
+            return sql.func.floor(sql.cast(left, self.col_type.to_sa_type()) / nullif)
         raise AssertionError()
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
@@ -113,7 +118,7 @@ class ArithmeticExpr(Expr):
 
         data_row[self.slot_idx] = self.eval_nullable(op1_val, op2_val)
 
-    def eval_nullable(self, op1_val: Optional[float], op2_val: Optional[float]) -> Optional[float]:
+    def eval_nullable(self, op1_val: float | None, op2_val: float | None) -> float | None:
         """
         Return the result of evaluating the expression on two nullable int/float operands,
         None is interpreted as SQL NULL
@@ -139,7 +144,7 @@ class ArithmeticExpr(Expr):
         elif self.operator == ArithmeticOperator.FLOORDIV:
             return op1_val // op2_val
 
-    def as_literal(self) -> Optional[Literal]:
+    def as_literal(self) -> Literal | None:
         op1_lit = self._op1.as_literal()
         if op1_lit is None:
             return None

pixeltable/exprs/array_slice.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Optional, Union
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -16,7 +16,7 @@ class ArraySlice(Expr):
     Slice operation on an array, eg, t.array_col[:, 1:2].
     """
 
-    def __init__(self, arr: Expr, index: tuple[Union[int, slice], ...]):
+    def __init__(self, arr: Expr, index: tuple[int | slice, ...]):
         assert arr.col_type.is_array_type()
         # determine result type
         super().__init__(arr.col_type)
@@ -43,7 +43,7 @@ class ArraySlice(Expr):
     def _id_attrs(self) -> list[tuple[str, Any]]:
         return [*super()._id_attrs(), ('index', self.index)]
 
-    def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
         return None
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:

pixeltable/exprs/column_property_ref.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import enum
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -26,6 +26,7 @@ class ColumnPropertyRef(Expr):
         ERRORMSG = 1
         FILEURL = 2
         LOCALPATH = 3
+        CELLMD = 4  # JSON metadata for the cell, e.g. errortype, errormsg for media columns
 
     def __init__(self, col_ref: ColumnRef, prop: Property):
         super().__init__(ts.StringType(nullable=True))
@@ -33,7 +34,7 @@ class ColumnPropertyRef(Expr):
         self.prop = prop
         self.id = self._create_id()
 
-    def default_column_name(self) -> Optional[str]:
+    def default_column_name(self) -> str | None:
         return str(self).replace('.', '_')
 
     def _equals(self, other: ColumnPropertyRef) -> bool:
@@ -43,66 +44,69 @@ class ColumnPropertyRef(Expr):
         return [*super()._id_attrs(), ('prop', self.prop.value)]
 
     @property
-    def _col_ref(self) -> ColumnRef:
+    def col_ref(self) -> ColumnRef:
         col_ref = self.components[0]
         assert isinstance(col_ref, ColumnRef)
         return col_ref
 
     def __repr__(self) -> str:
-        return f'{self._col_ref}.{self.prop.name.lower()}'
+        return f'{self.col_ref}.{self.prop.name.lower()}'
 
-    def is_error_prop(self) -> bool:
-        return self.prop in (self.Property.ERRORTYPE, self.Property.ERRORMSG)
+    def is_cellmd_prop(self) -> bool:
+        return self.prop in (self.Property.ERRORTYPE, self.Property.ERRORMSG, self.Property.CELLMD)
 
-    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
-        if not self._col_ref.col.is_stored:
+    def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
+        if not self.col_ref.col_handle.get().is_stored:
             return None
-
-        # we need to reestablish that we have the correct Column instance, there could have been a metadata
-        # reload since init()
-        # TODO: add an explicit prepare phase (ie, Expr.prepare()) that gives every subclass instance a chance to
-        # perform runtime checks and update state
-        tv = self._col_ref.tbl_version.get()
-        assert tv.is_validated
-        col = tv.cols_by_id[self._col_ref.col_id]
-        # TODO: check for column being dropped
+        col = self.col_ref.col_handle.get()
 
         # the errortype/-msg properties of a read-validated media column need to be extracted from the DataRow
         if (
             col.col_type.is_media_type()
             and col.media_validation == catalog.MediaValidation.ON_READ
-            and self.is_error_prop()
+            and self.is_cellmd_prop()
         ):
             return None
 
         if self.prop == self.Property.ERRORTYPE:
-            assert col.sa_errortype_col is not None
-            return col.sa_errortype_col
+            return col.sa_cellmd_col.op('->>')('errortype')
         if self.prop == self.Property.ERRORMSG:
-            assert col.sa_errormsg_col is not None
-            return col.sa_errormsg_col
+            return col.sa_cellmd_col.op('->>')('errormsg')
+        if self.prop == self.Property.CELLMD:
+            assert col.sa_cellmd_col is not None
+            return col.sa_cellmd_col
         if self.prop == self.Property.FILEURL:
             # the file url is stored as the column value
-            return sql_elements.get(self._col_ref)
+            return sql_elements.get(self.col_ref)
         return None
 
+    @classmethod
+    def create_cellmd_exc(cls, exc: Exception) -> dict[str, str]:
+        """Create a cellmd value from an exception."""
+        return {'errortype': type(exc).__name__, 'errormsg': str(exc)}
+
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
         if self.prop == self.Property.FILEURL:
-            assert data_row.has_val[self._col_ref.slot_idx]
-            data_row[self.slot_idx] = data_row.file_urls[self._col_ref.slot_idx]
+            assert data_row.has_val[self.col_ref.slot_idx]
+            data_row[self.slot_idx] = data_row.file_urls[self.col_ref.slot_idx]
             return
         elif self.prop == self.Property.LOCALPATH:
-            assert data_row.has_val[self._col_ref.slot_idx]
-            data_row[self.slot_idx] = data_row.file_paths[self._col_ref.slot_idx]
+            assert data_row.has_val[self.col_ref.slot_idx]
+            data_row[self.slot_idx] = data_row.file_paths[self.col_ref.slot_idx]
             return
-        elif self.is_error_prop():
-            exc = data_row.get_exc(self._col_ref.slot_idx)
+        elif self.is_cellmd_prop():
+            exc = data_row.get_exc(self.col_ref.slot_idx)
             if exc is None:
                 data_row[self.slot_idx] = None
             elif self.prop == self.Property.ERRORTYPE:
                 data_row[self.slot_idx] = type(exc).__name__
-            else:
+            elif self.prop == self.Property.ERRORMSG:
                 data_row[self.slot_idx] = str(exc)
+            elif self.prop == self.Property.CELLMD:
+                data_row[self.slot_idx] = self.create_cellmd_exc(exc)
+            else:
+                raise AssertionError(f'Unknown property {self.prop}')
+            return
         else:
             raise AssertionError()
 

pixeltable/exprs/column_ref.py

@@ -1,20 +1,24 @@
 from __future__ import annotations
 
-import copy
-from typing import Any, Optional, Sequence
+from typing import TYPE_CHECKING, Any, Sequence, cast
 from uuid import UUID
 
 import sqlalchemy as sql
 
-import pixeltable as pxt
-from pixeltable import catalog, exceptions as excs, iterators as iters
+import pixeltable.catalog as catalog
+import pixeltable.exceptions as excs
+import pixeltable.iterators as iters
 
 from ..utils.description_helper import DescriptionHelper
+from ..utils.filecache import FileCache
 from .data_row import DataRow
 from .expr import Expr
 from .row_builder import RowBuilder
 from .sql_element_cache import SqlElementCache
 
+if TYPE_CHECKING:
+    from pixeltable.dataframe import DataFrame, DataFrameResultSet
+
 
 class ColumnRef(Expr):
     """A reference to a table column
@@ -41,42 +45,36 @@ class ColumnRef(Expr):
     insert them into the EvalCtxs as needed
     """
 
-    col: catalog.Column
-    reference_tbl: Optional[catalog.TableVersionPath]
+    col: catalog.Column  # TODO: merge with col_handle
+    col_handle: catalog.ColumnHandle
+    reference_tbl: catalog.TableVersionPath | None
     is_unstored_iter_col: bool
-    iter_arg_ctx: Optional[RowBuilder.EvalCtx]
-    base_rowid_len: int
-    base_rowid: Sequence[Optional[Any]]
-    iterator: Optional[iters.ComponentIterator]
-    pos_idx: Optional[int]
-    id: int
     perform_validation: bool  # if True, performs media validation
+    iter_arg_ctx: RowBuilder.EvalCtx | None
+    base_rowid_len: int  # number of rowid columns in the base table
 
-    # needed by sql_expr() to re-resolve Column instance after a metadata reload
-    tbl_version: catalog.TableVersionHandle
-    col_id: int
+    # execution state
+    base_rowid: Sequence[Any | None]
+    iterator: iters.ComponentIterator | None
+    pos_idx: int
 
     def __init__(
         self,
         col: catalog.Column,
-        reference_tbl: Optional[catalog.TableVersionPath] = None,
-        perform_validation: Optional[bool] = None,
+        reference_tbl: catalog.TableVersionPath | None = None,
+        perform_validation: bool | None = None,
     ):
         super().__init__(col.col_type)
-        assert col.tbl is not None
         self.col = col
         self.reference_tbl = reference_tbl
-        self.tbl_version = catalog.TableVersionHandle(col.tbl.id, col.tbl.effective_version)
-        self.col_id = col.id
+        self.col_handle = col.handle
 
-        self.is_unstored_iter_col = col.tbl.is_component_view and col.tbl.is_iterator_column(col) and not col.is_stored
+        self.is_unstored_iter_col = col.is_iterator_col and not col.is_stored
         self.iter_arg_ctx = None
-        # number of rowid columns in the base table
-        self.base_rowid_len = col.tbl.base.get().num_rowid_columns() if self.is_unstored_iter_col else 0
-        self.base_rowid = [None] * self.base_rowid_len
+        self.base_rowid_len = 0
+        self.base_rowid = []
         self.iterator = None
-        # index of the position column in the view's primary key; don't try to reference tbl.store_tbl here
-        self.pos_idx = col.tbl.num_rowid_columns() - 1 if self.is_unstored_iter_col else None
+        self.pos_idx = 0
 
         self.perform_validation = False
         if col.col_type.is_media_type():
@@ -102,14 +100,14 @@ class ColumnRef(Expr):
     def _id_attrs(self) -> list[tuple[str, Any]]:
         return [
             *super()._id_attrs(),
-            ('tbl_id', self.col.tbl.id),
+            ('tbl_id', self.col.tbl_handle.id),
             ('col_id', self.col.id),
             ('perform_validation', self.perform_validation),
         ]
 
     # override
     def _retarget(self, tbl_versions: dict[UUID, catalog.TableVersion]) -> ColumnRef:
-        target = tbl_versions[self.col.tbl.id]
+        target = tbl_versions[self.col.tbl_handle.id]
         assert self.col.id in target.cols_by_id
         col = target.cols_by_id[self.col.id]
         return ColumnRef(col, self.reference_tbl)
@@ -118,12 +116,16 @@ class ColumnRef(Expr):
         from .column_property_ref import ColumnPropertyRef
 
         # resolve column properties
+        if name == ColumnPropertyRef.Property.CELLMD.name.lower():
+            # This is not user accessible, but used internally to store cell metadata
+            return super().__getattr__(name)
+
         if (
             name == ColumnPropertyRef.Property.ERRORTYPE.name.lower()
             or name == ColumnPropertyRef.Property.ERRORMSG.name.lower()
         ):
-            property_is_present = self.col.is_stored and (self.col.is_computed or self.col_type.is_media_type())
-            if not property_is_present:
+            is_valid = (self.col.is_computed or self.col.col_type.is_media_type()) and self.col.is_stored
+            if not is_valid:
                 raise excs.Error(f'{name} only valid for a stored computed or media column: {self}')
             return ColumnPropertyRef(self, ColumnPropertyRef.Property[name.upper()])
         if (
@@ -143,76 +145,66 @@ class ColumnRef(Expr):
 
         return super().__getattr__(name)
 
-    def find_embedding_index(
-        self, idx_name: Optional[str], method_name: str
-    ) -> dict[str, catalog.TableVersion.IndexInfo]:
-        """Return IndexInfo for a column, with an optional given name"""
-        from pixeltable import index
-
-        # determine index to use
-        idx_info_dict = self.col.get_idx_info(self.reference_tbl)
-
-        embedding_idx_info = {
-            info: value for info, value in idx_info_dict.items() if isinstance(value.idx, index.EmbeddingIndex)
-        }
-        if len(embedding_idx_info) == 0:
-            raise excs.Error(f'No indices found for {method_name!r} on column {self.col.name!r}')
-        if idx_name is not None and idx_name not in embedding_idx_info:
-            raise excs.Error(f'Index {idx_name!r} not found for {method_name!r} on column {self.col.name!r}')
-        if len(embedding_idx_info) > 1:
-            if idx_name is None:
-                raise excs.Error(
-                    f'Column {self.col.name!r} has multiple indices; use the index name to disambiguate: '
-                    f'`{method_name}(..., idx=<index_name>)`'
-                )
-            idx_info = {idx_name: embedding_idx_info[idx_name]}
-        else:
-            idx_info = embedding_idx_info
-        return idx_info
-
-    def similarity(self, item: Any, *, idx: Optional[str] = None) -> Expr:
+    def recompute(self, *, cascade: bool = True, errors_only: bool = False) -> catalog.UpdateStatus:
+        cat = catalog.Catalog.get()
+        # lock_mutable_tree=True: we need to be able to see whether any transitive view has column dependents
+        with cat.begin_xact(tbl=self.reference_tbl, for_write=True, lock_mutable_tree=True):
+            tbl_version = self.col_handle.tbl_version.get()
+            if tbl_version.id != self.reference_tbl.tbl_id:
+                raise excs.Error('Cannot recompute column of a base.')
+            if tbl_version.is_snapshot:
+                raise excs.Error('Cannot recompute column of a snapshot.')
+            col_name = self.col_handle.get().name
+            status = tbl_version.recompute_columns([col_name], errors_only=errors_only, cascade=cascade)
+            FileCache.get().emit_eviction_warnings()
+            return status
+
+    def similarity(self, item: Any, *, idx: str | None = None) -> Expr:
         from .similarity_expr import SimilarityExpr
 
         return SimilarityExpr(self, item, idx_name=idx)
 
-    def embedding(self, *, idx: Optional[str] = None) -> ColumnRef:
-        idx_info = self.find_embedding_index(idx, 'embedding')
-        assert len(idx_info) == 1
-        col = copy.copy(next(iter(idx_info.values())).val_col)
-        col.name = f'{self.col.name}_embedding_{idx if idx is not None else ""}'
-        # col.create_sa_cols()
-        return ColumnRef(col)
+    def embedding(self, *, idx: str | None = None) -> ColumnRef:
+        from pixeltable.index import EmbeddingIndex
 
-    def default_column_name(self) -> Optional[str]:
+        idx_info = self.tbl.get().get_idx(self.col, idx, EmbeddingIndex)
+        return ColumnRef(idx_info.val_col)
+
+    @property
+    def tbl(self) -> catalog.TableVersionHandle:
+        return self.reference_tbl.tbl_version if self.reference_tbl is not None else self.col.tbl_handle
+
+    def default_column_name(self) -> str | None:
         return self.col.name if self.col is not None else None
 
     def _equals(self, other: ColumnRef) -> bool:
         return self.col == other.col and self.perform_validation == other.perform_validation
 
-    def _df(self) -> 'pxt.dataframe.DataFrame':
-        from pixeltable import plan
+    def _df(self) -> 'DataFrame':
+        import pixeltable.plan as plan
+        from pixeltable.dataframe import DataFrame
 
         if self.reference_tbl is None:
             # No reference table; use the current version of the table to which the column belongs
-            tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl.id)
+            tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl_handle.id)
             return tbl.select(self)
         else:
             # Explicit reference table; construct a DataFrame directly from it
-            return pxt.DataFrame(plan.FromClause([self.reference_tbl])).select(self)
+            return DataFrame(plan.FromClause([self.reference_tbl])).select(self)
 
-    def show(self, *args: Any, **kwargs: Any) -> 'pxt.dataframe.DataFrameResultSet':
+    def show(self, *args: Any, **kwargs: Any) -> 'DataFrameResultSet':
         return self._df().show(*args, **kwargs)
 
-    def head(self, *args: Any, **kwargs: Any) -> 'pxt.dataframe.DataFrameResultSet':
+    def head(self, *args: Any, **kwargs: Any) -> 'DataFrameResultSet':
         return self._df().head(*args, **kwargs)
 
-    def tail(self, *args: Any, **kwargs: Any) -> 'pxt.dataframe.DataFrameResultSet':
+    def tail(self, *args: Any, **kwargs: Any) -> 'DataFrameResultSet':
         return self._df().tail(*args, **kwargs)
 
     def count(self) -> int:
         return self._df().count()
 
-    def distinct(self) -> 'pxt.dataframe.DataFrame':
+    def distinct(self) -> 'DataFrame':
         """Return distinct values in this column."""
         return self._df().distinct()
 
@@ -229,7 +221,8 @@ class ColumnRef(Expr):
         return self._descriptors().to_html()
 
     def _descriptors(self) -> DescriptionHelper:
-        tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl.id)
+        with catalog.Catalog.get().begin_xact():
+            tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl_handle.id)
         helper = DescriptionHelper()
         helper.append(f'Column\n{self.col.name!r}\n(of table {tbl._path()!r})')
         helper.append(tbl._col_descriptor([self.col.name]))
@@ -238,23 +231,21 @@ class ColumnRef(Expr):
             helper.append(idxs)
         return helper
 
-    def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
-        # return None if self.perform_validation else self.col.sa_col
+    def prepare(self) -> None:
+        from pixeltable import store
+
+        if not self.is_unstored_iter_col:
+            return
+        col = self.col_handle.get()
+        self.base_rowid_len = col.get_tbl().base.get().num_rowid_columns()
+        self.base_rowid = [None] * self.base_rowid_len
+        assert isinstance(col.get_tbl().store_tbl, store.StoreComponentView)
+        self.pos_idx = cast(store.StoreComponentView, col.get_tbl().store_tbl).pos_col_idx
+
+    def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
         if self.perform_validation:
             return None
-        # we need to reestablish that we have the correct Column instance, there could have been a metadata
-        # reload since init()
-        # TODO: add an explicit prepare phase (ie, Expr.prepare()) that gives every subclass instance a chance to
-        # perform runtime checks and update state
-        tv = self.tbl_version.get()
-        assert tv.is_validated
-        self.col = tv.cols_by_id[self.col_id]
-        assert self.col.tbl is tv
-        # TODO: check for column being dropped
-        # print(
-        #     f'ColumnRef.sql_expr: tbl={tv.id}:{tv.effective_version} sa_tbl={id(self.col.tbl.store_tbl.sa_tbl):x} '
-        #     f'tv={id(tv):x}'
-        # )
+        self.col = self.col_handle.get()
         return self.col.sa_col
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
@@ -296,29 +287,34 @@ class ColumnRef(Expr):
         if self.base_rowid != data_row.pk[: self.base_rowid_len]:
             row_builder.eval(data_row, self.iter_arg_ctx)
             iterator_args = data_row[self.iter_arg_ctx.target_slot_idxs[0]]
-            self.iterator = self.col.tbl.iterator_cls(**iterator_args)
+            self.iterator = self.col.get_tbl().iterator_cls(**iterator_args)
             self.base_rowid = data_row.pk[: self.base_rowid_len]
         self.iterator.set_pos(data_row.pk[self.pos_idx])
         res = next(self.iterator)
         data_row[self.slot_idx] = res[self.col.name]
 
     def _as_dict(self) -> dict:
-        tbl = self.col.tbl
-        version = tbl.version if tbl.is_snapshot else None
+        tbl_handle = self.col.tbl_handle
         # we omit self.components, even if this is a validating ColumnRef, because init() will recreate the
         # non-validating component ColumnRef
         return {
-            'tbl_id': str(tbl.id),
-            'tbl_version': version,
+            'tbl_id': str(tbl_handle.id),
+            'tbl_version': tbl_handle.effective_version,
             'col_id': self.col.id,
             'reference_tbl': self.reference_tbl.as_dict() if self.reference_tbl is not None else None,
             'perform_validation': self.perform_validation,
         }
 
+    @classmethod
+    def get_column_id(cls, d: dict) -> catalog.QColumnId:
+        tbl_id, col_id = UUID(d['tbl_id']), d['col_id']
+        return catalog.QColumnId(tbl_id, col_id)
+
     @classmethod
     def get_column(cls, d: dict) -> catalog.Column:
         tbl_id, version, col_id = UUID(d['tbl_id']), d['tbl_version'], d['col_id']
-        tbl_version = catalog.Catalog.get().get_tbl_version(tbl_id, version)
+        # validate_initialized=False: this gets called as part of TableVersion.init()
+        tbl_version = catalog.Catalog.get().get_tbl_version(tbl_id, version, validate_initialized=False)
         # don't use tbl_version.cols_by_id here, this might be a snapshot reference to a column that was then dropped
         col = next(col for col in tbl_version.cols if col.id == col_id)
         return col

pixeltable/exprs/comparison.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Optional
+from typing import Any
 
 import sqlalchemy as sql
 
@@ -69,8 +69,8 @@ class Comparison(Expr):
     def _op2(self) -> Expr:
         return self.components[1]
 
-    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
-        from pixeltable import index
+    def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
+        import pixeltable.index as index
 
         if str(self._op1.col_type.to_sa_type()) != str(self._op2.col_type.to_sa_type()):
             # Comparing columns of different SQL types (e.g., string vs. json); this can only be done in Python
@@ -81,9 +81,9 @@ class Comparison(Expr):
         if self.is_search_arg_comparison:
             # reference the index value column if there is an index and this is not a snapshot
             # (indices don't apply to snapshots)
-            tbl = self._op1.col.tbl
+            tbl = self._op1.col.get_tbl()
             idx_info = [
-                info for info in self._op1.col.get_idx_info().values() if isinstance(info.idx, index.BtreeIndex)
+                info for info in tbl.idxs_by_col.get(self._op1.col.qid, []) if isinstance(info.idx, index.BtreeIndex)
             ]
             if len(idx_info) > 0 and not tbl.is_snapshot:
                 # there shouldn't be multiple B-tree indices on a column

pixeltable/exprs/compound_predicate.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import operator
-from typing import Any, Callable, Optional
+from typing import Any, Callable
 
 import sqlalchemy as sql
 
@@ -36,7 +36,8 @@ class CompoundPredicate(Expr):
         return f' {self.operator} '.join([f'({e})' for e in self.components])
 
     @classmethod
-    def make_conjunction(cls, operands: list[Expr]) -> Optional[Expr]:
+    def make_conjunction(cls, operands: list[Expr | None]) -> Expr | None:
+        operands = [e for e in operands if e is not None]
         if len(operands) == 0:
             return None
         if len(operands) == 1:
@@ -60,14 +61,14 @@ class CompoundPredicate(Expr):
     def _id_attrs(self) -> list[tuple[str, Any]]:
         return [*super()._id_attrs(), ('operator', self.operator.value)]
 
-    def split_conjuncts(self, condition: Callable[[Expr], bool]) -> tuple[list[Expr], Optional[Expr]]:
+    def split_conjuncts(self, condition: Callable[[Expr], bool]) -> tuple[list[Expr], Expr | None]:
         if self.operator in (LogicalOperator.OR, LogicalOperator.NOT):
             return super().split_conjuncts(condition)
         matches = [op for op in self.components if condition(op)]
         non_matches = [op for op in self.components if not condition(op)]
         return (matches, self.make_conjunction(non_matches))
 
-    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
         sql_exprs = [sql_elements.get(op) for op in self.components]
         if any(e is None for e in sql_exprs):
             return None