pixeltable 0.3.14__py3-none-any.whl → 0.4.0rc1__py3-none-any.whl

pixeltable/catalog/table_version_handle.py

@@ -41,8 +41,21 @@ class TableVersionHandle:
     def get(self) -> TableVersion:
         from .catalog import Catalog
 
-        if self._tbl_version is None:
-            self._tbl_version = Catalog.get().get_tbl_version(self.id, self.effective_version)
+        cat = Catalog.get()
+        if self._tbl_version is None or not self._tbl_version.is_validated:
+            if self.effective_version is not None and self._tbl_version is not None:
+                # this is a snapshot version; we need to make sure we refer to the instance cached
+                # in Catalog, in order to avoid mixing sa_tbl instances in the same transaction
+                # (which will lead to duplicates in the From clause generated in SqlNode.create_from_clause())
+                assert (self.id, self.effective_version) in cat._tbl_versions
+                self._tbl_version = cat._tbl_versions[self.id, self.effective_version]
+                self._tbl_version.is_validated = True
+            else:
+                self._tbl_version = Catalog.get().get_tbl_version(self.id, self.effective_version)
+        if self.effective_version is None:
+            # make sure we don't see a discarded instance of a live TableVersion
+            tvs = list(Catalog.get()._tbl_versions.values())
+            assert self._tbl_version in tvs
         return self._tbl_version
 
     def as_dict(self) -> dict:

pixeltable/catalog/table_version_path.py

@@ -4,9 +4,11 @@ import logging
 from typing import Optional
 from uuid import UUID
 
+from pixeltable.env import Env
 from pixeltable.metadata import schema
 
 from .column import Column
+from .table_version import TableVersion
 from .table_version_handle import TableVersionHandle
 
 _logger = logging.getLogger('pixeltable')
@@ -22,15 +24,28 @@ class TableVersionPath:
 
     TableVersionPath contains all metadata needed to execute queries and updates against a particular version of a
     table/view.
+
+    TableVersionPath supplies metadata needed for query construction (eg, column names), for which it uses a
+    cached TableVersion instance.
+    - when running inside a transaction, this instance is guaranteed to be validated
+    - when running outside a transaction, we use an unvalidated instance in order to avoid repeated validation
+      on every metadata-related method call (the instance won't stay validated, because TableVersionHandle.get()
+      runs a local transaction, at the end of which the instance is again invalidated)
+    - supplying metadata from an unvalidated instance is okay, because it needs to get revalidated anyway when a
+      query actually runs (at which point there is a transaction context) - there is no guarantee that in between
+      constructing a DataFrame and executing it, the underlying table schema hasn't changed (eg, a concurrent process
+      could have dropped a column referenced in the query).
     """
 
     tbl_version: TableVersionHandle
     base: Optional[TableVersionPath]
+    _cached_tbl_version: Optional[TableVersion]
 
     def __init__(self, tbl_version: TableVersionHandle, base: Optional[TableVersionPath] = None):
         assert tbl_version is not None
         self.tbl_version = tbl_version
         self.base = base
+        self._cached_tbl_version = None
 
     @classmethod
     def from_md(cls, path: schema.TableVersionPath) -> TableVersionPath:
@@ -47,17 +62,40 @@ class TableVersionPath:
             result.extend(self.base.as_md())
         return result
 
+    def refresh_cached_md(self) -> None:
+        from pixeltable.catalog import Catalog
+
+        if Env.get().in_xact:
+            # when we're running inside a transaction, we need to make sure to supply current metadata;
+            # mixing stale metadata with current metadata leads to query construction failures
+            # (multiple sqlalchemy Table instances for the same underlying table create corrupted From clauses)
+            if self._cached_tbl_version is not None and self._cached_tbl_version.is_validated:
+                # nothing to refresh
+                return
+        elif self._cached_tbl_version is not None:
+            return
+
+        with Catalog.get().begin_xact(for_write=False):
+            self._cached_tbl_version = self.tbl_version.get()
+
+    def clear_cached_md(self) -> None:
+        self._cached_tbl_version = None
+        if self.base is not None:
+            self.base.clear_cached_md()
+
     def tbl_id(self) -> UUID:
         """Return the id of the table/view that this path represents"""
         return self.tbl_version.id
 
     def version(self) -> int:
         """Return the version of the table/view that this path represents"""
-        return self.tbl_version.get().version
+        self.refresh_cached_md()
+        return self._cached_tbl_version.version
 
     def tbl_name(self) -> str:
         """Return the name of the table/view that this path represents"""
-        return self.tbl_version.get().name
+        self.refresh_cached_md()
+        return self._cached_tbl_version.name
 
     def path_len(self) -> int:
         """Return the length of the path"""
@@ -65,18 +103,22 @@ class TableVersionPath:
 
     def is_snapshot(self) -> bool:
         """Return True if this is a path of snapshot versions"""
-        if not self.tbl_version.get().is_snapshot:
+        self.refresh_cached_md()
+        if not self._cached_tbl_version.is_snapshot:
             return False
         return self.base.is_snapshot() if self.base is not None else True
 
     def is_view(self) -> bool:
-        return self.tbl_version.get().is_view
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_view
 
     def is_component_view(self) -> bool:
-        return self.tbl_version.get().is_component_view
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_component_view
 
     def is_insertable(self) -> bool:
-        return self.tbl_version.get().is_insertable()
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_insertable
 
     def get_tbl_versions(self) -> list[TableVersionHandle]:
         """Return all tbl versions"""
@@ -98,20 +140,14 @@ class TableVersionPath:
             return None
         return self.base.find_tbl_version(id)
 
-    @property
-    def ancestor_paths(self) -> list[TableVersionPath]:
-        if self.base is None:
-            return [self]
-        else:
-            return [self, *self.base.ancestor_paths]
-
     def columns(self) -> list[Column]:
         """Return all user columns visible in this tbl version path, including columns from bases"""
-        result = list(self.tbl_version.get().cols_by_name.values())
-        if self.base is not None and self.tbl_version.get().include_base_columns:
+        self.refresh_cached_md()
+        result = list(self._cached_tbl_version.cols_by_name.values())
+        if self.base is not None and self._cached_tbl_version.include_base_columns:
             base_cols = self.base.columns()
             # we only include base columns that don't conflict with one of our column names
-            result.extend(c for c in base_cols if c.name not in self.tbl_version.get().cols_by_name)
+            result.extend(c for c in base_cols if c.name not in self._cached_tbl_version.cols_by_name)
         return result
 
     def cols_by_name(self) -> dict[str, Column]:
@@ -126,19 +162,21 @@ class TableVersionPath:
 
     def get_column(self, name: str, include_bases: Optional[bool] = None) -> Optional[Column]:
         """Return the column with the given name, or None if not found"""
-        col = self.tbl_version.get().cols_by_name.get(name)
+        self.refresh_cached_md()
+        col = self._cached_tbl_version.cols_by_name.get(name)
         if col is not None:
             return col
-        elif self.base is not None and (include_bases or self.tbl_version.get().include_base_columns):
+        elif self.base is not None and (include_bases or self._cached_tbl_version.include_base_columns):
             return self.base.get_column(name)
         else:
             return None
 
     def get_column_by_id(self, tbl_id: UUID, col_id: int) -> Optional[Column]:
         """Return the column for the given tbl/col id"""
+        self.refresh_cached_md()
         if self.tbl_version.id == tbl_id:
-            assert col_id in self.tbl_version.get().cols_by_id
-            return self.tbl_version.get().cols_by_id[col_id]
+            assert col_id in self._cached_tbl_version.cols_by_id
+            return self._cached_tbl_version.cols_by_id[col_id]
         elif self.base is not None:
             return self.base.get_column_by_id(tbl_id, col_id)
         else:
@@ -146,11 +184,12 @@ class TableVersionPath:
 
     def has_column(self, col: Column, include_bases: bool = True) -> bool:
         """Return True if this table has the given column."""
+        self.refresh_cached_md()
         assert col.tbl is not None
         if (
             col.tbl.id == self.tbl_version.id
             and col.tbl.effective_version == self.tbl_version.effective_version
-            and col.id in self.tbl_version.get().cols_by_id
+            and col.id in self._cached_tbl_version.cols_by_id
         ):
             # the column is visible in this table version
             return True

pixeltable/catalog/view.py

@@ -204,8 +204,17 @@ class View(Table):
 
             from pixeltable.plan import Planner
 
-            plan, _ = Planner.create_view_load_plan(view._tbl_version_path)
-            num_rows, num_excs, _ = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
+            try:
+                plan, _ = Planner.create_view_load_plan(view._tbl_version_path)
+                num_rows, num_excs, _ = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
+            except:
+                # we need to remove the orphaned TableVersion instance
+                del catalog.Catalog.get()._tbl_versions[tbl_version.id, tbl_version.effective_version]
+                base_tbl_version = base.tbl_version.get()
+                if tbl_version.effective_version is None and not base_tbl_version.is_snapshot:
+                    # also remove tbl_version from the base
+                    base_tbl_version.mutable_views.remove(TableVersionHandle.create(tbl_version))
+                raise
             Env.get().console_logger.info(f'Created view `{name}` with {num_rows} rows, {num_excs} exceptions.')
 
         session.commit()
@@ -285,13 +294,13 @@ class View(Table):
 
     def _table_descriptor(self) -> str:
         display_name = 'Snapshot' if self._snapshot_only else 'View'
-        result = [f'{display_name} {self._path!r}']
+        result = [f'{display_name} {self._path()!r}']
         bases_descrs: list[str] = []
         for base, effective_version in zip(self._base_tables, self._effective_base_versions):
             if effective_version is None:
-                bases_descrs.append(f'{base._path!r}')
+                bases_descrs.append(f'{base._path()!r}')
             else:
-                base_descr = f'{base._path}:{effective_version}'
+                base_descr = f'{base._path()}:{effective_version}'
                 bases_descrs.append(f'{base_descr!r}')
         result.append(f' (of {", ".join(bases_descrs)})')
 

pixeltable/dataframe.py

@@ -14,7 +14,7 @@ import pandas as pd
 import sqlalchemy as sql
 
 from pixeltable import catalog, exceptions as excs, exec, exprs, plan, type_system as ts
-from pixeltable.catalog import is_valid_identifier
+from pixeltable.catalog import Catalog, is_valid_identifier
 from pixeltable.catalog.globals import UpdateStatus
 from pixeltable.env import Env
 from pixeltable.type_system import ColumnType
@@ -431,7 +431,7 @@ class DataFrame:
         raise excs.Error(msg) from e
 
     def _output_row_iterator(self) -> Iterator[list]:
-        with Env.get().begin_xact():
+        with Catalog.get().begin_xact(for_write=False):
             try:
                 for data_row in self._exec():
                     yield [data_row[e.slot_idx] for e in self._select_list_exprs]
@@ -463,8 +463,8 @@ class DataFrame:
 
         from pixeltable.plan import Planner
 
-        stmt = Planner.create_count_stmt(self._first_tbl, self.where_clause)
-        with Env.get().begin_xact() as conn:
+        with Catalog.get().begin_xact(for_write=False) as conn:
+            stmt = Planner.create_count_stmt(self._first_tbl, self.where_clause)
             result: int = conn.execute(stmt).scalar_one()
             assert isinstance(result, int)
             return result
@@ -988,7 +988,8 @@ class DataFrame:
             >>> df = person.where(t.year == 2014).update({'age': 30})
         """
         self._validate_mutable('update', False)
-        with Env.get().begin_xact():
+        tbl_id = self._first_tbl.tbl_id()
+        with Catalog.get().begin_xact(tbl_id=tbl_id, for_write=True):
             return self._first_tbl.tbl_version.get().update(value_spec, where=self.where_clause, cascade=cascade)
 
     def delete(self) -> UpdateStatus:
@@ -1011,7 +1012,8 @@ class DataFrame:
         self._validate_mutable('delete', False)
         if not self._first_tbl.is_insertable():
             raise excs.Error('Cannot delete from view')
-        with Env.get().begin_xact():
+        tbl_id = self._first_tbl.tbl_id()
+        with Catalog.get().begin_xact(tbl_id=tbl_id, for_write=True):
             return self._first_tbl.tbl_version.get().delete(where=self.where_clause)
 
     def _validate_mutable(self, op_name: str, allow_select: bool) -> None:
@@ -1059,7 +1061,7 @@ class DataFrame:
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> 'DataFrame':
         # we need to wrap the construction with a transaction, because it might need to load metadata
-        with Env.get().begin_xact():
+        with Catalog.get().begin_xact(for_write=False):
             tbls = [catalog.TableVersionPath.from_dict(tbl_dict) for tbl_dict in d['from_clause']['tbls']]
             join_clauses = [plan.JoinClause(**clause_dict) for clause_dict in d['from_clause']['join_clauses']]
             from_clause = plan.FromClause(tbls=tbls, join_clauses=join_clauses)
@@ -1129,7 +1131,7 @@ class DataFrame:
             assert data_file_path.is_file()
             return data_file_path
         else:
-            with Env.get().begin_xact():
+            with Catalog.get().begin_xact(for_write=False):
                 return write_coco_dataset(self, dest_path)
 
     def to_pytorch_dataset(self, image_format: str = 'pt') -> 'torch.utils.data.IterableDataset':
@@ -1174,7 +1176,7 @@ class DataFrame:
         if dest_path.exists():  # fast path: use cache
             assert dest_path.is_dir()
         else:
-            with Env.get().begin_xact():
+            with Catalog.get().begin_xact(for_write=False):
                 export_parquet(self, dest_path, inline_images=True)
 
         return PixeltablePytorchDataset(path=dest_path, image_format=image_format)

pixeltable/env.py

@@ -191,6 +191,7 @@ class Env:
         assert self._dbms is not None
         return self._dbms
 
+    @property
     def in_xact(self) -> bool:
         return self._current_conn is not None
 
@@ -201,20 +202,17 @@ class Env:
 
     @contextmanager
     def begin_xact(self) -> Iterator[sql.Connection]:
-        """Return a context manager that yields a connection to the database. Idempotent."""
+        """Call Catalog.begin_xact() instead, unless there is a specific reason to call this directly."""
         if self._current_conn is None:
             assert self._current_session is None
             try:
                 with self.engine.begin() as conn, sql.orm.Session(conn) as session:
-                    # TODO: remove print() once we're done with debugging the concurrent update behavior
-                    # print(f'{datetime.datetime.now()}: start xact')
                     self._current_conn = conn
                     self._current_session = session
                     yield conn
             finally:
                 self._current_session = None
                 self._current_conn = None
-                # print(f'{datetime.datetime.now()}: end xact')
         else:
             assert self._current_session is not None
             yield self._current_conn

pixeltable/exec/in_memory_data_node.py

@@ -38,7 +38,7 @@ class InMemoryDataNode(ExecNode):
         # we materialize the input slots
         output_exprs = list(row_builder.input_exprs)
         super().__init__(row_builder, output_exprs, [], None)
-        assert tbl.get().is_insertable()
+        assert tbl.get().is_insertable
         self.tbl = tbl
         self.input_rows = rows
         self.start_row_id = start_row_id

pixeltable/exec/sql_node.py

@@ -134,6 +134,11 @@ class SqlNode(ExecNode):
         self.where_clause_element = None
         self.order_by_clause = []
 
+        if self.tbl is not None:
+            tv = self.tbl.tbl_version._tbl_version
+            if tv is not None:
+                assert tv.is_validated
+
     def _create_stmt(self) -> sql.Select:
         """Create Select from local state"""
 
@@ -141,6 +146,7 @@ class SqlNode(ExecNode):
         sql_select_list = [self.sql_elements.get(e) for e in self.select_list]
         if self.set_pk:
             assert self.tbl is not None
+            assert self.tbl.tbl_version.get().is_validated
             sql_select_list += self.tbl.tbl_version.get().store_tbl.pk_columns()
         stmt = sql.select(*sql_select_list)
 
@@ -220,26 +226,29 @@ class SqlNode(ExecNode):
                 joined_tbls.append(t)
 
         first = True
-        prev_tbl: Optional[catalog.TableVersionHandle] = None
+        prev_tv: Optional[catalog.TableVersion] = None
         for t in joined_tbls[::-1]:
+            tv = t.get()
+            # _logger.debug(f'create_from_clause: tbl_id={tv.id} {id(tv.store_tbl.sa_tbl)}')
             if first:
-                stmt = stmt.select_from(t.get().store_tbl.sa_tbl)
+                stmt = stmt.select_from(tv.store_tbl.sa_tbl)
                 first = False
             else:
-                # join tbl to prev_tbl on prev_tbl's rowid cols
-                prev_tbl_rowid_cols = prev_tbl.get().store_tbl.rowid_columns()
-                tbl_rowid_cols = t.get().store_tbl.rowid_columns()
+                # join tv to prev_tv on prev_tv's rowid cols
+                prev_tbl_rowid_cols = prev_tv.store_tbl.rowid_columns()
+                tbl_rowid_cols = tv.store_tbl.rowid_columns()
                 rowid_clauses = [
                     c1 == c2 for c1, c2 in zip(prev_tbl_rowid_cols, tbl_rowid_cols[: len(prev_tbl_rowid_cols)])
                 ]
-                stmt = stmt.join(t.get().store_tbl.sa_tbl, sql.and_(*rowid_clauses))
+                stmt = stmt.join(tv.store_tbl.sa_tbl, sql.and_(*rowid_clauses))
+
             if t.id in exact_version_only:
-                stmt = stmt.where(t.get().store_tbl.v_min_col == t.get().version)
+                stmt = stmt.where(tv.store_tbl.v_min_col == tv.version)
             else:
-                stmt = stmt.where(t.get().store_tbl.v_min_col <= t.get().version).where(
-                    t.get().store_tbl.v_max_col > t.get().version
-                )
-            prev_tbl = t
+                stmt = stmt.where(tv.store_tbl.sa_tbl.c.v_min <= tv.version)
+                stmt = stmt.where(tv.store_tbl.sa_tbl.c.v_max > tv.version)
+            prev_tv = tv
+
         return stmt
 
     def set_where(self, where_clause: exprs.Expr) -> None:

pixeltable/exprs/column_property_ref.py

@@ -58,20 +58,29 @@ class ColumnPropertyRef(Expr):
         if not self._col_ref.col.is_stored:
             return None
 
+        # we need to reestablish that we have the correct Column instance, there could have been a metadata
+        # reload since init()
+        # TODO: add an explicit prepare phase (ie, Expr.prepare()) that gives every subclass instance a chance to
+        # perform runtime checks and update state
+        tv = self._col_ref.tbl_version.get()
+        assert tv.is_validated
+        col = tv.cols_by_id[self._col_ref.col_id]
+        # TODO: check for column being dropped
+
         # the errortype/-msg properties of a read-validated media column need to be extracted from the DataRow
         if (
-            self._col_ref.col.col_type.is_media_type()
-            and self._col_ref.col.media_validation == catalog.MediaValidation.ON_READ
+            col.col_type.is_media_type()
+            and col.media_validation == catalog.MediaValidation.ON_READ
             and self.is_error_prop()
         ):
             return None
 
         if self.prop == self.Property.ERRORTYPE:
-            assert self._col_ref.col.sa_errortype_col is not None
-            return self._col_ref.col.sa_errortype_col
+            assert col.sa_errortype_col is not None
+            return col.sa_errortype_col
         if self.prop == self.Property.ERRORMSG:
-            assert self._col_ref.col.sa_errormsg_col is not None
-            return self._col_ref.col.sa_errormsg_col
+            assert col.sa_errormsg_col is not None
+            return col.sa_errormsg_col
         if self.prop == self.Property.FILEURL:
             # the file url is stored as the column value
             return sql_elements.get(self._col_ref)

pixeltable/exprs/column_ref.py

@@ -52,6 +52,10 @@ class ColumnRef(Expr):
     id: int
     perform_validation: bool  # if True, performs media validation
 
+    # needed by sql_expr() to re-resolve Column instance after a metadata reload
+    tbl_version: catalog.TableVersionHandle
+    col_id: int
+
     def __init__(
         self,
         col: catalog.Column,
@@ -62,16 +66,17 @@ class ColumnRef(Expr):
         assert col.tbl is not None
         self.col = col
         self.reference_tbl = reference_tbl
-        self.is_unstored_iter_col = (
-            col.tbl.get().is_component_view and col.tbl.get().is_iterator_column(col) and not col.is_stored
-        )
+        self.tbl_version = catalog.TableVersionHandle(col.tbl.id, col.tbl.effective_version)
+        self.col_id = col.id
+
+        self.is_unstored_iter_col = col.tbl.is_component_view and col.tbl.is_iterator_column(col) and not col.is_stored
         self.iter_arg_ctx = None
         # number of rowid columns in the base table
-        self.base_rowid_len = col.tbl.get().base.get().num_rowid_columns() if self.is_unstored_iter_col else 0
+        self.base_rowid_len = col.tbl.base.get().num_rowid_columns() if self.is_unstored_iter_col else 0
         self.base_rowid = [None] * self.base_rowid_len
         self.iterator = None
         # index of the position column in the view's primary key; don't try to reference tbl.store_tbl here
-        self.pos_idx = col.tbl.get().num_rowid_columns() - 1 if self.is_unstored_iter_col else None
+        self.pos_idx = col.tbl.num_rowid_columns() - 1 if self.is_unstored_iter_col else None
 
         self.perform_validation = False
         if col.col_type.is_media_type():
@@ -175,7 +180,7 @@ class ColumnRef(Expr):
         assert len(idx_info) == 1
         col = copy.copy(next(iter(idx_info.values())).val_col)
         col.name = f'{self.col.name}_embedding_{idx if idx is not None else ""}'
-        col.create_sa_cols()
+        # col.create_sa_cols()
         return ColumnRef(col)
 
     def default_column_name(self) -> Optional[str]:
@@ -226,7 +231,7 @@ class ColumnRef(Expr):
     def _descriptors(self) -> DescriptionHelper:
         tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl.id)
         helper = DescriptionHelper()
-        helper.append(f'Column\n{self.col.name!r}\n(of table {tbl._path!r})')
+        helper.append(f'Column\n{self.col.name!r}\n(of table {tbl._path()!r})')
         helper.append(tbl._col_descriptor([self.col.name]))
         idxs = tbl._index_descriptor([self.col.name])
         if len(idxs) > 0:
@@ -234,7 +239,23 @@ class ColumnRef(Expr):
         return helper
 
     def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
-        return None if self.perform_validation else self.col.sa_col
+        # return None if self.perform_validation else self.col.sa_col
+        if self.perform_validation:
+            return None
+        # we need to reestablish that we have the correct Column instance, there could have been a metadata
+        # reload since init()
+        # TODO: add an explicit prepare phase (ie, Expr.prepare()) that gives every subclass instance a chance to
+        # perform runtime checks and update state
+        tv = self.tbl_version.get()
+        assert tv.is_validated
+        self.col = tv.cols_by_id[self.col_id]
+        assert self.col.tbl is tv
+        # TODO: check for column being dropped
+        # print(
+        #     f'ColumnRef.sql_expr: tbl={tv.id}:{tv.effective_version} sa_tbl={id(self.col.tbl.store_tbl.sa_tbl):x} '
+        #     f'tv={id(tv):x}'
+        # )
+        return self.col.sa_col
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
         if self.perform_validation:
@@ -275,7 +296,7 @@ class ColumnRef(Expr):
         if self.base_rowid != data_row.pk[: self.base_rowid_len]:
             row_builder.eval(data_row, self.iter_arg_ctx)
             iterator_args = data_row[self.iter_arg_ctx.target_slot_idxs[0]]
-            self.iterator = self.col.tbl.get().iterator_cls(**iterator_args)
+            self.iterator = self.col.tbl.iterator_cls(**iterator_args)
             self.base_rowid = data_row.pk[: self.base_rowid_len]
         self.iterator.set_pos(data_row.pk[self.pos_idx])
         res = next(self.iterator)
@@ -283,12 +304,12 @@ class ColumnRef(Expr):
 
     def _as_dict(self) -> dict:
         tbl = self.col.tbl
-        tbl_version = tbl.get().version if tbl.get().is_snapshot else None
+        version = tbl.version if tbl.is_snapshot else None
         # we omit self.components, even if this is a validating ColumnRef, because init() will recreate the
         # non-validating component ColumnRef
         return {
             'tbl_id': str(tbl.id),
-            'tbl_version': tbl_version,
+            'tbl_version': version,
             'col_id': self.col.id,
             'reference_tbl': self.reference_tbl.as_dict() if self.reference_tbl is not None else None,
             'perform_validation': self.perform_validation,

pixeltable/exprs/comparison.py

@@ -81,7 +81,7 @@ class Comparison(Expr):
         if self.is_search_arg_comparison:
             # reference the index value column if there is an index and this is not a snapshot
             # (indices don't apply to snapshots)
-            tbl = self._op1.col.tbl.get()
+            tbl = self._op1.col.tbl
             idx_info = [
                 info for info in self._op1.col.get_idx_info().values() if isinstance(info.idx, index.BtreeIndex)
             ]

pixeltable/exprs/row_builder.py

@@ -172,13 +172,11 @@ class RowBuilder:
 
         def refs_unstored_iter_col(col_ref: ColumnRef) -> bool:
             tbl = col_ref.col.tbl
-            return (
-                tbl.get().is_component_view and tbl.get().is_iterator_column(col_ref.col) and not col_ref.col.is_stored
-            )
+            return tbl.is_component_view and tbl.is_iterator_column(col_ref.col) and not col_ref.col.is_stored
 
         unstored_iter_col_refs = [col_ref for col_ref in col_refs if refs_unstored_iter_col(col_ref)]
         component_views = [col_ref.col.tbl for col_ref in unstored_iter_col_refs]
-        unstored_iter_args = {view.id: view.get().iterator_args.copy() for view in component_views}
+        unstored_iter_args = {view.id: view.iterator_args.copy() for view in component_views}
         self.unstored_iter_args = {
             id: self._record_unique_expr(arg, recursive=True) for id, arg in unstored_iter_args.items()
         }
@@ -450,9 +448,9 @@ class RowBuilder:
             else:
                 if col.col_type.is_image_type() and data_row.file_urls[slot_idx] is None:
                     # we have yet to store this image
-                    filepath = str(MediaStore.prepare_media_path(col.tbl.id, col.id, col.tbl.get().version))
+                    filepath = str(MediaStore.prepare_media_path(col.tbl.id, col.id, col.tbl.version))
                     data_row.flush_img(slot_idx, filepath)
-                val = data_row.get_stored_val(slot_idx, col.sa_col.type)
+                val = data_row.get_stored_val(slot_idx, col.get_sa_col_type())
                 table_row[col.store_name()] = val
                 # we unfortunately need to set these, even if there are no errors
                 table_row[col.errortype_store_name()] = None

pixeltable/exprs/rowid_ref.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 from typing import Any, Optional, cast
 from uuid import UUID
 
@@ -12,6 +13,8 @@ from .expr import Expr
 from .row_builder import RowBuilder
 from .sql_element_cache import SqlElementCache
 
+_logger = logging.getLogger('pixeltable')
+
 
 class RowidRef(Expr):
     """A reference to a part of a table rowid
@@ -97,10 +100,15 @@ class RowidRef(Expr):
 
     def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
         tbl = self.tbl.get() if self.tbl is not None else catalog.Catalog.get().get_tbl_version(self.tbl_id, None)
+        assert tbl.is_validated
         rowid_cols = tbl.store_tbl.rowid_columns()
         assert self.rowid_component_idx <= len(rowid_cols), (
             f'{self.rowid_component_idx} not consistent with {rowid_cols}'
         )
+        # _logger.debug(
+        #     f'RowidRef.sql_expr: tbl={tbl.id}{tbl.effective_version} sa_tbl={id(tbl.store_tbl.sa_tbl):x} '
+        #     f'tv={id(tbl):x}'
+        # )
         return rowid_cols[self.rowid_component_idx]
 
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:

pixeltable/exprs/similarity_expr.py

@@ -54,6 +54,7 @@ class SimilarityExpr(Expr):
         return 'similarity'
 
     def sql_expr(self, _: SqlElementCache) -> Optional[sql.ColumnElement]:
+        # TODO: validate that the index still exists
         if not isinstance(self.components[1], Literal):
             raise excs.Error('similarity(): requires a string or a PIL.Image.Image object, not an expression')
         item = self.components[1].val

pixeltable/func/query_template_function.py

@@ -162,7 +162,7 @@ def retrieval_udf(
     else:
         for param in parameters:
             if isinstance(param, str) and param not in table.columns:
-                raise excs.Error(f'The specified parameter {param!r} is not a column of the table {table._path!r}')
+                raise excs.Error(f'The specified parameter {param!r} is not a column of the table {table._path()!r}')
         col_refs = [table[param] if isinstance(param, str) else param for param in parameters]
 
     if len(col_refs) == 0: