pixeltable 0.4.17__py3-none-any.whl → 0.4.19__py3-none-any.whl

pixeltable/catalog/table_version.py

@@ -3,19 +3,21 @@ from __future__ import annotations
 import copy
 import dataclasses
 import importlib
+import itertools
 import logging
 import time
 import uuid
-from typing import TYPE_CHECKING, Any, Iterable, Iterator, Literal, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Literal
 from uuid import UUID
 
 import jsonschema.exceptions
 import sqlalchemy as sql
 from sqlalchemy import exc as sql_exc
 
-import pixeltable as pxt
 import pixeltable.exceptions as excs
-from pixeltable import exprs, index
+import pixeltable.exprs as exprs
+import pixeltable.index as index
+import pixeltable.type_system as ts
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 from pixeltable.metadata import schema
@@ -31,8 +33,12 @@ from .update_status import RowCountStats, UpdateStatus
 if TYPE_CHECKING:
     from pixeltable import exec, store
     from pixeltable.catalog.table_version_handle import TableVersionHandle
+    from pixeltable.dataframe import DataFrame
+    from pixeltable.io import ExternalStore
     from pixeltable.plan import SampleClause
 
+    from .table_version_path import TableVersionPath
+
 _logger = logging.getLogger('pixeltable')
 
 
@@ -77,14 +83,14 @@ class TableVersion:
     _version_md: schema.TableVersionMd
     _schema_version_md: schema.TableSchemaVersionMd
 
-    effective_version: Optional[int]
-    path: Optional[pxt.catalog.TableVersionPath]  # only set for live tables; needed to resolve computed cols
-    base: Optional[TableVersionHandle]  # only set for views
-    predicate: Optional[exprs.Expr]
-    sample_clause: Optional['SampleClause']
+    effective_version: int | None
+    path: 'TableVersionPath' | None  # only set for live tables; needed to resolve computed cols
+    base: TableVersionHandle | None  # only set for views
+    predicate: exprs.Expr | None
+    sample_clause: 'SampleClause' | None
 
-    iterator_cls: Optional[type[ComponentIterator]]
-    iterator_args: Optional[exprs.InlineDict]
+    iterator_cls: type[ComponentIterator] | None
+    iterator_args: exprs.InlineDict | None
     num_iterator_cols: int
 
     # target for data operation propagation (only set for non-snapshots, and only records non-snapshot views)
@@ -96,11 +102,19 @@ class TableVersion:
     cols_by_name: dict[str, Column]
     # contains only columns visible in this version, both system and user
     cols_by_id: dict[int, Column]
-    # contains only actively maintained indices
+
+    # True if this TableVersion instance can have indices:
+    # - live version of a mutable table
+    # - the most recent version of a replica
+    supports_idxs: bool
+
+    # only populated with indices visible in this TableVersion instance
+    idxs: dict[int, TableVersion.IndexInfo]  # key: index id
     idxs_by_name: dict[str, TableVersion.IndexInfo]
+    idxs_by_col: dict[QColumnId, list[TableVersion.IndexInfo]]
 
-    external_stores: dict[str, pxt.io.ExternalStore]
-    store_tbl: Optional['store.StoreBase']
+    external_stores: dict[str, ExternalStore]
+    store_tbl: 'store.StoreBase' | None
 
     is_initialized: bool  # True if init() has been called
 
@@ -123,11 +137,11 @@ class TableVersion:
         id: UUID,
         tbl_md: schema.TableMd,
         version_md: schema.TableVersionMd,
-        effective_version: Optional[int],
+        effective_version: int | None,
         schema_version_md: schema.TableSchemaVersionMd,
         mutable_views: list[TableVersionHandle],
-        base_path: Optional[pxt.catalog.TableVersionPath] = None,
-        base: Optional[TableVersionHandle] = None,
+        base_path: 'TableVersionPath' | None = None,
+        base: TableVersionHandle | None = None,
     ):
         self.is_validated = True  # a freshly constructed instance is always valid
         self.is_initialized = False
@@ -180,7 +194,12 @@ class TableVersion:
         self.cols = []
         self.cols_by_name = {}
         self.cols_by_id = {}
+        self.idxs = {}
         self.idxs_by_name = {}
+        self.idxs_by_col = {}
+        self.supports_idxs = self.effective_version is None or (
+            self.is_replica and self.effective_version == self.tbl_md.current_version
+        )
         self.external_stores = {}
 
     def __hash__(self) -> int:
@@ -219,19 +238,27 @@ class TableVersion:
         num_retained_versions: int,
         comment: str,
         media_validation: MediaValidation,
-        view_md: Optional[schema.ViewMd] = None,
+        create_default_idxs: bool,
+        view_md: schema.ViewMd | None = None,
     ) -> TableVersionMd:
+        from .table_version_handle import TableVersionHandle
+
         user = Env.get().user
         timestamp = time.time()
 
+        tbl_id = uuid.uuid4()
+        tbl_id_str = str(tbl_id)
+        tbl_handle = TableVersionHandle(tbl_id, None)
+        column_ids = itertools.count(0)
+        index_ids = itertools.count(0)
+
         # assign ids, create metadata
-        cols_by_name: dict[str, Column] = {}
         column_md: dict[int, schema.ColumnMd] = {}
         schema_col_md: dict[int, schema.SchemaColumn] = {}
         for pos, col in enumerate(cols):
-            col.id = pos
+            col.tbl_handle = tbl_handle
+            col.id = next(column_ids)
             col.schema_version_add = 0
-            cols_by_name[col.name] = col
             if col.is_computed:
                 col.check_value_expr()
             col_md, sch_md = col.to_md(pos)
@@ -239,8 +266,39 @@ class TableVersion:
             column_md[col.id] = col_md
             schema_col_md[col.id] = sch_md
 
-        tbl_id = uuid.uuid4()
-        tbl_id_str = str(tbl_id)
+        index_md: dict[int, schema.IndexMd] = {}
+        if create_default_idxs and (view_md is None or not view_md.is_snapshot):
+            index_cols: list[Column] = []
+            for col in (c for c in cols if cls._is_btree_indexable(c)):
+                idx = index.BtreeIndex()
+                val_col, undo_col = cls._create_index_columns(col, idx, 0, tbl_handle, id_cb=lambda: next(column_ids))
+                index_cols.extend([val_col, undo_col])
+
+                idx_id = next(index_ids)
+                idx_cls = type(idx)
+                md = schema.IndexMd(
+                    id=idx_id,
+                    name=f'idx{idx_id}',
+                    indexed_col_id=col.id,
+                    indexed_col_tbl_id=tbl_id_str,
+                    index_val_col_id=val_col.id,
+                    index_val_undo_col_id=undo_col.id,
+                    schema_version_add=0,
+                    schema_version_drop=None,
+                    class_fqn=idx_cls.__module__ + '.' + idx_cls.__name__,
+                    init_args=idx.as_dict(),
+                )
+                index_md[idx_id] = md
+
+            for col in index_cols:
+                col_md, _ = col.to_md()
+                column_md[col.id] = col_md
+
+            assert all(column_md[id].id == id for id in column_md)
+            assert all(index_md[id].id == id for id in index_md)
+
+            cols.extend(index_cols)
+
         tbl_md = schema.TableMd(
             tbl_id=tbl_id_str,
             name=name,
@@ -248,12 +306,12 @@ class TableVersion:
             is_replica=False,
             current_version=0,
             current_schema_version=0,
-            next_col_id=len(cols),
-            next_idx_id=0,
+            next_col_id=next(column_ids),
+            next_idx_id=next(index_ids),
             next_row_id=0,
             view_sn=0,
             column_md=column_md,
-            index_md={},
+            index_md=index_md,
             external_stores=[],
             view_md=view_md,
             additional_md={},
@@ -281,51 +339,15 @@ class TableVersion:
         )
         return TableVersionMd(tbl_md, table_version_md, schema_version_md)
 
-    @classmethod
-    def create(
-        cls,
-        dir_id: UUID,
-        name: str,
-        cols: list[Column],
-        num_retained_versions: int,
-        comment: str,
-        media_validation: MediaValidation,
-    ) -> tuple[UUID, Optional[TableVersion]]:
-        initial_md = cls.create_initial_md(name, cols, num_retained_versions, comment, media_validation, view_md=None)
-        cat = pxt.catalog.Catalog.get()
-
-        tbl_id = UUID(hex=initial_md.tbl_md.tbl_id)
-        assert (tbl_id, None) not in cat._tbl_versions
-        tbl_version = cls(tbl_id, initial_md.tbl_md, initial_md.version_md, None, initial_md.schema_version_md, [])
-
-        @cat.register_undo_action
-        def _() -> None:
-            if (tbl_id, None) in cat._tbl_versions:
-                del cat._tbl_versions[tbl_id, None]
-
-        # TODO: break this up, so that Catalog.create_table() registers tbl_version
-        cat._tbl_versions[tbl_id, None] = tbl_version
-        tbl_version.init()
-        tbl_version.store_tbl.create()
-        # add default indices, after creating the store table
-        for col in tbl_version.cols_by_name.values():
-            status = tbl_version._add_default_index(col)
-            assert status is None or status.num_excs == 0
-
-        cat.store_tbl_md(
-            tbl_id=tbl_id,
-            dir_id=dir_id,
-            tbl_md=tbl_version.tbl_md,
-            version_md=initial_md.version_md,
-            schema_version_md=initial_md.schema_version_md,
-        )
-        return tbl_id, tbl_version
-
     def exec_op(self, op: TableOp) -> None:
         if op.create_store_table_op is not None:
-            # don't use Catalog.begin_xact() here, to avoid accidental recursive calls to exec_op()
+            # this needs to be called outside of a transaction
+            self.store_tbl.create()
+
+        elif op.create_index_op is not None:
+            idx_info = self.idxs[op.create_index_op.idx_id]
             with Env.get().begin_xact():
-                self.store_tbl.create()
+                self.store_tbl.create_index(idx_info.id)
 
         elif op.load_view_op is not None:
             from pixeltable.catalog import Catalog
@@ -344,7 +366,7 @@ class TableVersion:
 
     @classmethod
     def create_replica(cls, md: schema.FullTableMd) -> TableVersion:
-        from .catalog import TableVersionPath
+        from .catalog import Catalog, TableVersionPath
 
         assert Env.get().in_xact
         assert md.tbl_md.is_replica
@@ -363,7 +385,7 @@ class TableVersion:
             base_path=base_path,
             base=base,
         )
-        cat = pxt.catalog.Catalog.get()
+        cat = Catalog.get()
         # We're creating a new TableVersion replica, so we should never have seen this particular
         # TableVersion instance before.
         # Actually this isn't true, because we might be re-creating a dropped replica.
@@ -373,10 +395,9 @@ class TableVersion:
         cat._tbl_versions[tbl_version.id, tbl_version.effective_version] = tbl_version
         tbl_version.init()
         tbl_version.store_tbl.create()
-        tbl_version.store_tbl.ensure_columns_exist(col for col in tbl_version.cols if col.is_stored)
         return tbl_version
 
-    def delete_media(self, tbl_version: Optional[int] = None) -> None:
+    def delete_media(self, tbl_version: int | None = None) -> None:
         # Assemble a set of column destinations and delete objects from all of them
         # None is a valid column destination which refers to the default object location
         destinations = {col.destination for col in self.cols if col.is_stored}
@@ -414,19 +435,21 @@ class TableVersion:
         self.is_initialized = True
 
     def _init_schema(self) -> None:
-        # create columns first, so the indices can reference them
-        self._init_cols()
-        self._init_idxs()
-
-        # create the sa schema only after creating the columns and indices
-        self._init_sa_schema()
+        from pixeltable.store import StoreComponentView, StoreTable, StoreView
 
-        # created value_exprs after everything else has been initialized
-        for col in self.cols_by_id.values():
-            col.init_value_expr()
+        # initialize IndexBase instances and collect sa_col_types
+        idxs: dict[int, index.IndexBase] = {}
+        val_col_idxs: dict[int, index.IndexBase] = {}  # key: id of value column
+        undo_col_idxs: dict[int, index.IndexBase] = {}  # key: id of undo column
+        for md in self.tbl_md.index_md.values():
+            cls_name = md.class_fqn.rsplit('.', 1)[-1]
+            cls = getattr(index, cls_name)
+            idx = cls.from_dict(md.init_args)
+            idxs[md.id] = idx
+            val_col_idxs[md.index_val_col_id] = idx
+            undo_col_idxs[md.index_val_undo_col_id] = idx
 
-    def _init_cols(self) -> None:
-        """Initialize self.cols with the columns visible in our effective version"""
+        # initialize Columns
         self.cols = []
         self.cols_by_name = {}
         self.cols_by_id = {}
@@ -434,77 +457,88 @@ class TableVersion:
         # point backward.
         sorted_column_md = sorted(self.tbl_md.column_md.values(), key=lambda item: item.id)
         for col_md in sorted_column_md:
+            col_type = ts.ColumnType.from_dict(col_md.col_type)
             schema_col_md = self.schema_version_md.columns.get(col_md.id)
-            col = Column.from_md(col_md, self, schema_col_md)
-            self.cols.append(col)
-
-            # populate the lookup structures before Expr.from_dict()
-            if col_md.schema_version_add > self.schema_version:
-                # column was added after this version
-                continue
-            if col_md.schema_version_drop is not None and col_md.schema_version_drop <= self.schema_version:
-                # column was dropped
-                continue
-            if col.name is not None:
-                self.cols_by_name[col.name] = col
-            self.cols_by_id[col.id] = col
-
-            # # make sure to traverse columns ordered by position = order in which cols were created;
-            # # this guarantees that references always point backwards
-            # if not self.is_snapshot and col_md.value_expr is not None:
-            #     self._record_refd_columns(col)
+            media_val = (
+                MediaValidation[schema_col_md.media_validation.upper()]
+                if schema_col_md is not None and schema_col_md.media_validation is not None
+                else None
+            )
 
-    def _init_idxs(self) -> None:
-        for md in self.tbl_md.index_md.values():
-            # Instantiate index object. This needs to be done for all indices, even those that are not active in this
-            # TableVersion, so that we can make appropriate adjustments to the SA schema.
-            cls_name = md.class_fqn.rsplit('.', 1)[-1]
-            cls = getattr(index, cls_name)
-            idx_col = self._lookup_column(QColumnId(UUID(md.indexed_col_tbl_id), md.indexed_col_id))
-            assert idx_col is not None
-            idx = cls.from_dict(idx_col, md.init_args)
-
-            # fix up the sa column type of the index value and undo columns
-            # we need to do this for all indices, not just those that are active in this TableVersion, to ensure we get
-            # the correct SA schema in the StoreTable.
-            val_col = next(col for col in self.cols if col.id == md.index_val_col_id)
-            val_col.sa_col_type = idx.index_sa_type()
-            undo_col = next(col for col in self.cols if col.id == md.index_val_undo_col_id)
-            undo_col.sa_col_type = idx.index_sa_type()
-            if not isinstance(idx, index.EmbeddingIndex):
-                # Historically, the intent has been not to store cellmd data, even for embedding indices. However,
-                # the cellmd columns get created anyway, even if stores_cellmd is set to `False` here, due to the
-                # timing of index column creation. In order to ensure that SA schemas align with what is actually in
-                # the physical tables, we keep this `True` for embedding indices.
-                # TODO: Decide whether index columns should store cellmd data.
-                #     - If not, set to `False`, fix the column creation timing issue, and add a migration script to
-                #       remedy existing cellmd columns.
-                #     - If so, remove this TODO.
-                val_col._stores_cellmd = False
-            undo_col._stores_cellmd = False
-
-            # The index is active in this TableVersion provided that:
-            # (i) the TableVersion supports indices (either it's not a snapshot, or it's a replica at
-            #     the head version); and
-            # (ii) the index was created on or before the schema version of this TableVersion; and
-            # (iii) the index was not dropped on or before the schema version of this TableVersion.
-            supports_idxs = self.effective_version is None or (
-                self.tbl_md.is_replica and self.effective_version == self.tbl_md.current_version
+            stores_cellmd: bool | None = None  # None: determined by the column properties (in the Column c'tor)
+            sa_col_type: sql.types.TypeEngine | None = None
+            if col_md.id in val_col_idxs:
+                idx = val_col_idxs[col_md.id]
+                # for index value columns, the index gets to override the default
+                stores_cellmd = idx.records_value_errors()
+                sa_col_type = idx.get_index_sa_type(col_type)
+            elif col_md.id in undo_col_idxs:
+                idx = undo_col_idxs[col_md.id]
+                # for index undo columns, we never store cellmd
+                stores_cellmd = False
+                sa_col_type = idx.get_index_sa_type(col_type)
+
+            col = Column(
+                col_id=col_md.id,
+                name=schema_col_md.name if schema_col_md is not None else None,
+                col_type=col_type,
+                is_pk=col_md.is_pk,
+                is_iterator_col=self.is_component_view and col_md.id < self.num_iterator_cols + 1,
+                stored=col_md.stored,
+                media_validation=media_val,
+                sa_col_type=sa_col_type,
+                schema_version_add=col_md.schema_version_add,
+                schema_version_drop=col_md.schema_version_drop,
+                stores_cellmd=stores_cellmd,
+                value_expr_dict=col_md.value_expr,
+                tbl_handle=self.handle,
+                destination=col_md.destination,
             )
-            if (
-                supports_idxs
-                and md.schema_version_add <= self.schema_version
-                and (md.schema_version_drop is None or md.schema_version_drop > self.schema_version)
+
+            self.cols.append(col)
+            # populate lookup structures before Expr.from_dict()
+            if col_md.schema_version_add <= self.schema_version and (
+                col_md.schema_version_drop is None or col_md.schema_version_drop > self.schema_version
             ):
-                # Since the index is present in this TableVersion, its associated columns must be as well.
-                # Sanity-check this.
-                assert md.indexed_col_id in self.cols_by_id
-                assert md.index_val_col_id in self.cols_by_id
-                assert md.index_val_undo_col_id in self.cols_by_id
-                idx_info = self.IndexInfo(
-                    id=md.id, name=md.name, idx=idx, col=idx_col, val_col=val_col, undo_col=undo_col
+                if col.name is not None:
+                    self.cols_by_name[col.name] = col
+                self.cols_by_id[col.id] = col
+
+        if self.supports_idxs:
+            # create IndexInfo for indices visible in current_version
+            visible_idxs = [
+                md
+                for md in self.tbl_md.index_md.values()
+                if md.schema_version_add <= self.schema_version
+                and (md.schema_version_drop is None or md.schema_version_drop > self.schema_version)
+            ]
+            for md in visible_idxs:
+                idx = idxs[md.id]
+                indexed_col_id = QColumnId(UUID(md.indexed_col_tbl_id), md.indexed_col_id)
+                idx_col = self._lookup_column(indexed_col_id)
+                info = self.IndexInfo(
+                    id=md.id,
+                    name=md.name,
+                    idx=idx,
+                    col=idx_col,
+                    val_col=self.cols_by_id[md.index_val_col_id],
+                    undo_col=self.cols_by_id[md.index_val_undo_col_id],
                 )
-                self.idxs_by_name[md.name] = idx_info
+                self.idxs[md.id] = info
+                self.idxs_by_name[md.name] = info
+                self.idxs_by_col.setdefault(indexed_col_id, []).append(info)
+
+        # create value exprs, now that we have all lookup structures in place
+        for col in self.cols_by_id.values():
+            col.init_value_expr()
+
+        # create the sqlalchemy schema, after instantiating all Columns
+        if self.is_component_view:
+            self.store_tbl = StoreComponentView(self)
+        elif self.is_view:
+            self.store_tbl = StoreView(self)
+        else:
+            self.store_tbl = StoreTable(self)
 
     def _lookup_column(self, id: QColumnId) -> Column | None:
         """
@@ -547,7 +581,7 @@ class TableVersion:
         """Return name of index in the store, which needs to be globally unique"""
         return f'idx_{self.id.hex}_{idx_id}'
 
-    def add_index(self, col: Column, idx_name: Optional[str], idx: index.IndexBase) -> UpdateStatus:
+    def add_index(self, col: Column, idx_name: str | None, idx: index.IndexBase) -> UpdateStatus:
         # we're creating a new schema version
         self.bump_version(bump_schema_version=True)
         status = self._add_index(col, idx_name, idx)
@@ -555,12 +589,13 @@ class TableVersion:
         _logger.info(f'Added index {idx_name} on column {col.name} to table {self.name}')
         return status
 
-    def _is_btree_indexable(self, col: Column) -> bool:
+    @classmethod
+    def _is_btree_indexable(cls, col: Column) -> bool:
         if not col.stored:
             # if the column is intentionally not stored, we want to avoid the overhead of an index
             return False
         # Skip index for stored media columns produced by an iterator
-        if col.col_type.is_media_type() and self.is_iterator_column(col):
+        if col.col_type.is_media_type() and col.is_iterator_col:
             return False
         if not col.col_type.is_scalar_type() and not (col.col_type.is_media_type() and not col.is_computed):
             # wrong type for a B-tree
@@ -570,53 +605,58 @@ class TableVersion:
             return False
         return True
 
-    def _add_default_index(self, col: Column) -> Optional[UpdateStatus]:
+    def _add_default_index(self, col: Column) -> UpdateStatus | None:
         """Add a B-tree index on this column if it has a compatible type"""
         if not self._is_btree_indexable(col):
             return None
-        status = self._add_index(col, idx_name=None, idx=index.BtreeIndex(col))
+        status = self._add_index(col, idx_name=None, idx=index.BtreeIndex())
         return status
 
-    def _create_index_columns(self, idx: index.IndexBase) -> Tuple[Column, Column]:
+    @classmethod
+    def _create_index_columns(
+        cls,
+        col: Column,
+        idx: index.IndexBase,
+        schema_version: int,
+        tbl_handle: TableVersionHandle,
+        id_cb: Callable[[], int],
+    ) -> tuple[Column, Column]:
         """Create value and undo columns for the given index.
         Args:
             idx:  index for which columns will be created.
         Returns:
-            A tuple containing the value column and the undo column.
+            A tuple containing the value column and the undo column, both of which are nullable.
         """
-        assert not self.is_snapshot
-        # add the index value and undo columns (which need to be nullable)
+        value_expr = idx.create_value_expr(col)
         val_col = Column(
-            col_id=self.next_col_id,
+            col_id=id_cb(),
             name=None,
-            computed_with=idx.index_value_expr(),
-            sa_col_type=idx.index_sa_type(),
+            computed_with=value_expr,
+            sa_col_type=idx.get_index_sa_type(value_expr.col_type),
             stored=True,
-            schema_version_add=self.schema_version,
-            schema_version_drop=None,
             stores_cellmd=idx.records_value_errors(),
+            schema_version_add=schema_version,
+            schema_version_drop=None,
         )
-        val_col.tbl = self
         val_col.col_type = val_col.col_type.copy(nullable=True)
-        self.next_col_id += 1
+        val_col.tbl_handle = tbl_handle
 
         undo_col = Column(
-            col_id=self.next_col_id,
+            col_id=id_cb(),
             name=None,
             col_type=val_col.col_type,
             sa_col_type=val_col.sa_col_type,
             stored=True,
-            schema_version_add=self.schema_version,
-            schema_version_drop=None,
             stores_cellmd=False,
+            schema_version_add=schema_version,
+            schema_version_drop=None,
         )
-        undo_col.tbl = self
         undo_col.col_type = undo_col.col_type.copy(nullable=True)
-        self.next_col_id += 1
+        undo_col.tbl_handle = tbl_handle
         return val_col, undo_col
 
     def _create_index(
-        self, col: Column, val_col: Column, undo_col: Column, idx_name: Optional[str], idx: index.IndexBase
+        self, col: Column, val_col: Column, undo_col: Column, idx_name: str | None, idx: index.IndexBase
     ) -> None:
         """Create the given index along with index md"""
         idx_id = self.next_idx_id
@@ -632,7 +672,7 @@ class TableVersion:
             id=idx_id,
             name=idx_name,
             indexed_col_id=col.id,
-            indexed_col_tbl_id=str(col.tbl.id),
+            indexed_col_tbl_id=str(col.get_tbl().id),
             index_val_col_id=val_col.id,
             index_val_undo_col_id=undo_col.id,
             schema_version_add=self.schema_version,
@@ -642,17 +682,21 @@ class TableVersion:
         )
         idx_info = self.IndexInfo(id=idx_id, name=idx_name, idx=idx, col=col, val_col=val_col, undo_col=undo_col)
         self._tbl_md.index_md[idx_id] = idx_md
+        self.idxs[idx_id] = idx_info
         self.idxs_by_name[idx_name] = idx_info
-        idx.create_index(self._store_idx_name(idx_id), val_col)
+        self.idxs_by_col.setdefault(col.qid, []).append(idx_info)
+        self.store_tbl.create_index(idx_id)
 
-    def _add_index(self, col: Column, idx_name: Optional[str], idx: index.IndexBase) -> UpdateStatus:
-        val_col, undo_vol = self._create_index_columns(idx)
+    def _add_index(self, col: Column, idx_name: str | None, idx: index.IndexBase) -> UpdateStatus:
+        val_col, undo_col = self._create_index_columns(
+            col, idx, self.schema_version, self.handle, id_cb=self.next_col_id
+        )
         # add the columns and update the metadata
         # TODO support on_error='abort' for indices; it's tricky because of the way metadata changes are entangled
         # with the database operations
-        status = self._add_columns([val_col, undo_vol], print_stats=False, on_error='ignore')
+        status = self._add_columns([val_col, undo_col], print_stats=False, on_error='ignore')
         # now create the index structure
-        self._create_index(col, val_col, undo_vol, idx_name, idx)
+        self._create_index(col, val_col, undo_col, idx_name, idx)
         return status
 
     def drop_index(self, idx_id: int) -> None:
@@ -668,7 +712,10 @@ class TableVersion:
         # remove this index entry from the active indexes (in memory)
         # and the index metadata (in persistent table metadata)
         # TODO: this is wrong, it breaks revert()
+        del self.idxs[idx_id]
         del self.idxs_by_name[idx_md.name]
+        if idx_info.col.qid in self.idxs_by_col:
+            self.idxs_by_col[idx_info.col.qid].remove(idx_info)
         del self._tbl_md.index_md[idx_id]
 
         self._drop_columns([idx_info.val_col, idx_info.undo_col])
@@ -684,9 +731,8 @@ class TableVersion:
         assert all(col.stored is not None for col in cols)
         assert all(col.name not in self.cols_by_name for col in cols if col.name is not None)
         for col in cols:
-            col.tbl = self
-            col.id = self.next_col_id
-            self.next_col_id += 1
+            col.tbl_handle = self.handle
+            col.id = self.next_col_id()
 
         # we're creating a new schema version
         self.bump_version(bump_schema_version=True)
@@ -695,8 +741,10 @@ class TableVersion:
         for col in cols:
             all_cols.append(col)
             if col.name is not None and self._is_btree_indexable(col):
-                idx = index.BtreeIndex(col)
-                val_col, undo_col = self._create_index_columns(idx)
+                idx = index.BtreeIndex()
+                val_col, undo_col = self._create_index_columns(
+                    col, idx, self.schema_version, self.handle, id_cb=self.next_col_id
+                )
                 index_cols[col] = (idx, val_col, undo_col)
                 all_cols.append(val_col)
                 all_cols.append(undo_col)
@@ -728,7 +776,7 @@ class TableVersion:
 
         row_count = self.store_tbl.count()
         for col in cols_to_add:
-            assert col.tbl is self
+            assert col.tbl_handle.id == self.id
             if not col.col_type.nullable and not col.is_computed and row_count > 0:
                 raise excs.Error(
                     f'Cannot add non-nullable column {col.name!r} to table {self.name!r} with existing rows'
@@ -738,7 +786,7 @@ class TableVersion:
         num_excs = 0
         cols_with_excs: list[Column] = []
         for col in cols_to_add:
-            assert col.id is not None, 'Column id must be set before adding the column'
+            assert col.id is not None
             excs_per_col = 0
             col.schema_version_add = self.schema_version
             # add the column to the lookup structures now, rather than after the store changes executed successfully,
@@ -792,7 +840,7 @@ class TableVersion:
             upd_rows=row_count, num_excs=num_excs, computed_values=computed_values
         )  # add_columns
         return UpdateStatus(
-            cols_with_excs=[f'{col.tbl.name}.{col.name}' for col in cols_with_excs if col.name is not None],
+            cols_with_excs=[f'{col.get_tbl().name}.{col.name}' for col in cols_with_excs if col.name is not None],
             row_count_stats=row_counts,
         )
 
@@ -806,7 +854,7 @@ class TableVersion:
 
         # drop this column and all dependent index columns and indices
         dropped_cols = [col]
-        dropped_idx_names: list[str] = []
+        dropped_idx_info: list[TableVersion.IndexInfo] = []
         for idx_info in self.idxs_by_name.values():
             if idx_info.col != col:
                 continue
@@ -814,11 +862,14 @@ class TableVersion:
             idx_md = self._tbl_md.index_md[idx_info.id]
             idx_md.schema_version_drop = self.schema_version
             assert idx_md.name in self.idxs_by_name
-            dropped_idx_names.append(idx_md.name)
+            dropped_idx_info.append(idx_info)
 
-        # update idxs_by_name
-        for idx_name in dropped_idx_names:
-            del self.idxs_by_name[idx_name]
+        # update index lookup structures
+        for info in dropped_idx_info:
+            del self.idxs[info.id]
+            del self.idxs_by_name[info.name]
+        if col.qid in self.idxs_by_col:
+            del self.idxs_by_col[col.qid]
 
         self._drop_columns(dropped_cols)
         self._write_md(new_version=True, new_schema_version=True)
@@ -826,6 +877,8 @@ class TableVersion:
 
     def _drop_columns(self, cols: Iterable[Column]) -> None:
         """Mark columns as dropped"""
+        from pixeltable.catalog import Catalog
+
         assert self.is_mutable
 
         for col in cols:
@@ -845,7 +898,7 @@ class TableVersion:
             schema_col.pos = pos
 
         self.store_tbl.create_sa_tbl()
-        pxt.catalog.Catalog.get().record_column_dependencies(self)
+        Catalog.get().record_column_dependencies(self)
 
     def rename_column(self, old_name: str, new_name: str) -> None:
         """Rename a column."""
@@ -854,12 +907,12 @@ class TableVersion:
         col = self.path.get_column(old_name)
         if col is None:
             raise excs.Error(f'Unknown column: {old_name}')
-        if col.tbl.id != self.id:
+        if col.get_tbl().id != self.id:
             raise excs.Error(f'Cannot rename base table column {col.name!r}')
         if not is_valid_identifier(new_name):
-            raise excs.Error(f"Invalid column name: '{new_name}'")
+            raise excs.Error(f'Invalid column name: {new_name}')
         if new_name in self.cols_by_name:
-            raise excs.Error(f'Column {new_name} already exists')
+            raise excs.Error(f'Column {new_name!r} already exists')
         del self.cols_by_name[old_name]
         col.name = new_name
         self.cols_by_name[new_name] = col
@@ -871,7 +924,7 @@ class TableVersion:
         self._write_md(new_version=True, new_schema_version=True)
         _logger.info(f'Renamed column {old_name} to {new_name} in table {self.name}, new version: {self.version}')
 
-    def set_comment(self, new_comment: Optional[str]) -> None:
+    def set_comment(self, new_comment: str | None) -> None:
         _logger.info(f'[{self.name}] Updating comment: {new_comment}')
         self.comment = new_comment
         self._create_schema_version()
@@ -892,8 +945,8 @@ class TableVersion:
 
     def insert(
         self,
-        rows: Optional[list[dict[str, Any]]],
-        df: Optional[pxt.DataFrame],
+        rows: list[dict[str, Any]] | None,
+        df: DataFrame | None,
         print_stats: bool = False,
         fail_on_exception: bool = True,
     ) -> UpdateStatus:
@@ -927,7 +980,7 @@ class TableVersion:
         exec_plan: 'exec.ExecNode',
         timestamp: float,
         *,
-        rowids: Optional[Iterator[int]] = None,
+        rowids: Iterator[int] | None = None,
         print_stats: bool = False,
         abort_on_exc: bool = False,
     ) -> UpdateStatus:
@@ -958,9 +1011,7 @@ class TableVersion:
         _logger.info(f'TableVersion {self.name}: new version {self.version}')
         return result
 
-    def update(
-        self, value_spec: dict[str, Any], where: Optional[exprs.Expr] = None, cascade: bool = True
-    ) -> UpdateStatus:
+    def update(self, value_spec: dict[str, Any], where: exprs.Expr | None = None, cascade: bool = True) -> UpdateStatus:
         """Update rows in this TableVersionPath.
         Args:
             value_spec: a list of (column, value) pairs specifying the columns to update and their new values.
@@ -976,11 +1027,11 @@ class TableVersion:
         update_spec = self._validate_update_spec(value_spec, allow_pk=False, allow_exprs=True, allow_media=True)
         if where is not None:
             if not isinstance(where, exprs.Expr):
-                raise excs.Error(f"'where' argument must be a predicate, got {type(where)}")
+                raise excs.Error(f'`where` argument must be a valid Pixeltable expression; got `{type(where)}`')
             analysis_info = Planner.analyze(self.path, where)
             # for now we require that the updated rows can be identified via SQL, rather than via a Python filter
             if analysis_info.filter is not None:
-                raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')
+                raise excs.Error(f'Filter not expressible in SQL: {analysis_info.filter}')
 
         plan, updated_cols, recomputed_cols = Planner.create_update_plan(self.path, update_spec, [], where, cascade)
 
@@ -1037,7 +1088,7 @@ class TableVersion:
         update_targets: dict[Column, exprs.Expr] = {}
         for col_name, val in value_spec.items():
             if not isinstance(col_name, str):
-                raise excs.Error(f'Update specification: dict key must be column name, got {col_name!r}')
+                raise excs.Error(f'Update specification: dict key must be column name; got {col_name!r}')
             if col_name == _ROWID_COLUMN_NAME:
                 # a valid rowid is a list of ints, one per rowid column
                 assert len(val) == len(self.store_tbl.rowid_columns())
@@ -1046,15 +1097,15 @@ class TableVersion:
                 continue
             col = self.path.get_column(col_name)
             if col is None:
-                raise excs.Error(f'Column {col_name} unknown')
-            if col.tbl.id != self.id:
+                raise excs.Error(f'Unknown column: {col_name}')
+            if col.get_tbl().id != self.id:
                 raise excs.Error(f'Column {col.name!r} is a base table column and cannot be updated')
             if col.is_computed:
-                raise excs.Error(f'Column {col_name} is computed and cannot be updated')
+                raise excs.Error(f'Column {col_name!r} is computed and cannot be updated')
             if col.is_pk and not allow_pk:
-                raise excs.Error(f'Column {col_name} is a primary key column and cannot be updated')
+                raise excs.Error(f'Column {col_name!r} is a primary key column and cannot be updated')
             if col.col_type.is_media_type() and not allow_media:
-                raise excs.Error(f'Column {col_name} is a media column and cannot be updated')
+                raise excs.Error(f'Column {col_name!r} is a media column and cannot be updated')
 
             # make sure that the value is compatible with the column type
             value_expr: exprs.Expr
@@ -1064,19 +1115,19 @@ class TableVersion:
             except (TypeError, jsonschema.exceptions.ValidationError) as exc:
                 if not allow_exprs:
                     raise excs.Error(
-                        f'Column {col_name}: value {val!r} is not a valid literal for this column '
-                        f'(expected {col.col_type})'
+                        f'Column {col_name!r}: value is not a valid literal for this column '
+                        f'(expected `{col.col_type}`): {val!r}'
                     ) from exc
                 # it's not a literal, let's try to create an expr from it
                 value_expr = exprs.Expr.from_object(val)
                 if value_expr is None:
                     raise excs.Error(
-                        f'Column {col_name}: value {val!r} is not a recognized literal or expression'
+                        f'Column {col_name!r}: value is not a recognized literal or expression: {val!r}'
                     ) from exc
                 if not col.col_type.is_supertype_of(value_expr.col_type, ignore_nullable=True):
                     raise excs.Error(
-                        f'Type of value {val!r} ({value_expr.col_type}) is not compatible with the type of column '
-                        f'{col_name} ({col.col_type})'
+                        f'Type `{value_expr.col_type}` of value {val!r} is not compatible with the type '
+                        f'`{col.col_type}` of column {col_name!r}'
                     ) from exc
             update_targets[col] = value_expr
 
@@ -1094,9 +1145,9 @@ class TableVersion:
         assert len(col_names) == 1 or not errors_only
 
         target_columns = [self.cols_by_name[name] for name in col_names]
-        where_clause: Optional[exprs.Expr] = None
+        where_clause: exprs.Expr | None = None
         if where is not None:
-            self._validate_where_clause(where, error_prefix="'where' argument")
+            self._validate_where_clause(where, error_prefix='`where` argument')
             where_clause = where
         if errors_only:
             errortype_pred = (
@@ -1122,10 +1173,10 @@ class TableVersion:
 
     def propagate_update(
         self,
-        plan: Optional[exec.ExecNode],
-        where_clause: Optional[sql.ColumnElement],
+        plan: exec.ExecNode | None,
+        where_clause: sql.ColumnElement | None,
         recomputed_view_cols: list[Column],
-        base_versions: list[Optional[int]],
+        base_versions: list[int | None],
         timestamp: float,
         cascade: bool,
         show_progress: bool = True,
@@ -1153,7 +1204,7 @@ class TableVersion:
             base_versions = [None if plan is None else self.version, *base_versions]  # don't update in place
             # propagate to views
             for view in self.mutable_views:
-                recomputed_cols = [col for col in recomputed_view_cols if col.tbl.id == view.id]
+                recomputed_cols = [col for col in recomputed_view_cols if col.get_tbl().id == view.id]
                 plan = None
                 if len(recomputed_cols) > 0:
                     plan = Planner.create_view_update_plan(view.get().path, recompute_targets=recomputed_cols)
@@ -1173,21 +1224,21 @@ class TableVersion:
         from pixeltable.plan import Planner
 
         if not isinstance(pred, Expr):
-            raise excs.Error(f'{error_prefix} must be a predicate, got {type(pred)}')
+            raise excs.Error(f'{error_prefix} must be a valid Pixeltable expression; got `{type(pred)}`')
         analysis_info = Planner.analyze(self.path, pred)
         # for now we require that the updated rows can be identified via SQL, rather than via a Python filter
         if analysis_info.filter is not None:
-            raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')
+            raise excs.Error(f'Filter not expressible in SQL: {analysis_info.filter}')
 
     def delete(self, where: exprs.Expr | None = None) -> UpdateStatus:
         assert self.is_insertable
         if where is not None:
-            self._validate_where_clause(where, error_prefix="'where' argument")
+            self._validate_where_clause(where, error_prefix='`where` argument')
         status = self.propagate_delete(where, base_versions=[], timestamp=time.time())
         return status
 
     def propagate_delete(
-        self, where: Optional[exprs.Expr], base_versions: list[Optional[int]], timestamp: float
+        self, where: exprs.Expr | None, base_versions: list[int | None], timestamp: float
     ) -> UpdateStatus:
         """Delete rows in this table and propagate to views"""
         from pixeltable.catalog import Catalog
@@ -1252,7 +1303,7 @@ class TableVersion:
             names = [row[1] for row in result]
             raise excs.Error(
                 (
-                    f'Current version is needed for {len(result)} snapshot{"s" if len(result) > 1 else ""} '
+                    f'Current version is needed for {len(result)} snapshot{"s" if len(result) > 1 else ""}: '
                     f'({", ".join(names)})'
                 )
             )
@@ -1261,7 +1312,7 @@ class TableVersion:
 
         # revert new deletions
         set_clause: dict[sql.Column, Any] = {self.store_tbl.sa_tbl.c.v_max: schema.Table.MAX_VERSION}
-        for index_info in self.idxs_by_name.values():
+        for index_info in self.idxs.values():
             # copy the index value back from the undo column and reset the undo column to NULL
             set_clause[index_info.val_col.sa_col] = index_info.undo_col.sa_col
             set_clause[index_info.undo_col.sa_col] = None
@@ -1335,16 +1386,18 @@ class TableVersion:
         # Do this at the end, after all DB operations have completed.
         # TODO: The transaction could still fail. Really this should be done via PendingTableOps.
         self.delete_media(tbl_version=old_version)
-        _logger.info(f'TableVersion {self.name}: reverted to version {self.version}')
+        _logger.info(f'TableVersion {self.name!r}: reverted to version {self.version}')
 
     def _init_external_stores(self) -> None:
+        from pixeltable.io.external_store import ExternalStore
+
         for store_md in self.tbl_md.external_stores:
             store_cls = resolve_symbol(store_md['class'])
-            assert isinstance(store_cls, type) and issubclass(store_cls, pxt.io.ExternalStore)
+            assert isinstance(store_cls, type) and issubclass(store_cls, ExternalStore)
             store = store_cls.from_dict(store_md['md'])
             self.external_stores[store.name] = store
 
-    def link_external_store(self, store: pxt.io.ExternalStore) -> None:
+    def link_external_store(self, store: ExternalStore) -> None:
         self.bump_version(bump_schema_version=True)
 
         self.external_stores[store.name] = store
@@ -1353,7 +1406,7 @@ class TableVersion:
         )
         self._write_md(new_version=True, new_schema_version=True)
 
-    def unlink_external_store(self, store: pxt.io.ExternalStore) -> None:
+    def unlink_external_store(self, store: ExternalStore) -> None:
         del self.external_stores[store.name]
         self.bump_version(bump_schema_version=True)
         idx = next(i for i, store_md in enumerate(self._tbl_md.external_stores) if store_md['md']['name'] == store.name)
@@ -1373,7 +1426,7 @@ class TableVersion:
         return self._schema_version_md
 
     @property
-    def view_md(self) -> Optional[schema.ViewMd]:
+    def view_md(self) -> schema.ViewMd | None:
         return self._tbl_md.view_md
 
     @property
@@ -1381,7 +1434,7 @@ class TableVersion:
         return self._tbl_md.name
 
     @property
-    def user(self) -> Optional[str]:
+    def user(self) -> str | None:
         return self._tbl_md.user
 
     @property
@@ -1419,7 +1472,7 @@ class TableVersion:
     def schema_version(self) -> int:
         return self._schema_version_md.schema_version
 
-    def bump_version(self, timestamp: Optional[float] = None, *, bump_schema_version: bool) -> None:
+    def bump_version(self, timestamp: float | None = None, *, bump_schema_version: bool) -> None:
         """
         Increments the table version and adjusts all associated metadata. This will *not* trigger a database action;
         _write_md() must be called separately to persist the changes.
@@ -1456,11 +1509,11 @@ class TableVersion:
             self._schema_version_md.schema_version = new_version
 
     @property
-    def preceding_schema_version(self) -> Optional[int]:
+    def preceding_schema_version(self) -> int | None:
         return self._schema_version_md.preceding_schema_version
 
     @property
-    def update_status(self) -> Optional[UpdateStatus]:
+    def update_status(self) -> UpdateStatus | None:
         return self._version_md.update_status
 
     @update_status.setter
@@ -1472,14 +1525,10 @@ class TableVersion:
     def media_validation(self) -> MediaValidation:
         return MediaValidation[self._schema_version_md.media_validation.upper()]
 
-    @property
     def next_col_id(self) -> int:
-        return self._tbl_md.next_col_id
-
-    @next_col_id.setter
-    def next_col_id(self, id: int) -> None:
-        assert self.effective_version is None
-        self._tbl_md.next_col_id = id
+        val = self._tbl_md.next_col_id
+        self._tbl_md.next_col_id += 1
+        return val
 
     @property
     def next_idx_id(self) -> int:
@@ -1558,15 +1607,35 @@ class TableVersion:
         return names
 
     def get_idx_val_columns(self, cols: Iterable[Column]) -> set[Column]:
-        result = {info.val_col for col in cols for info in col.get_idx_info().values()}
-        return result
+        # assumes that the indexed columns are all in this table
+        assert all(col.get_tbl().id == self.id for col in cols)
+        col_ids = {col.id for col in cols}
+        return {info.val_col for info in self.idxs.values() if info.col.id in col_ids}
+
+    def get_idx(self, col: Column, idx_name: str | None, idx_cls: type[index.IndexBase]) -> TableVersion.IndexInfo:
+        if not self.supports_idxs:
+            raise excs.Error('Snapshot does not support indices')
+        if col.qid not in self.idxs_by_col:
+            raise excs.Error(f'Column {col.name!r} does not have a {idx_cls.display_name()} index')
+        candidates = [info for info in self.idxs_by_col[col.qid] if isinstance(info.idx, idx_cls)]
+        if len(candidates) == 0:
+            raise excs.Error(f'No {idx_cls.display_name()} index found for column {col.name!r}')
+        if len(candidates) > 1 and idx_name is None:
+            raise excs.Error(
+                f'Column {col.name!r} has multiple {idx_cls.display_name()} indices; specify `idx_name` instead'
+            )
+        if idx_name is not None and idx_name not in [info.name for info in candidates]:
+            raise excs.Error(f'Index {idx_name!r} not found for column {col.name!r}')
+        return candidates[0] if idx_name is None else next(info for info in candidates if info.name == idx_name)
 
     def get_dependent_columns(self, cols: Iterable[Column]) -> set[Column]:
         """
         Return the set of columns that transitively depend on any of the given ones.
         """
-        cat = pxt.catalog.Catalog.get()
-        result = set().union(*[cat.get_column_dependents(col.tbl.id, col.id) for col in cols])
+        from pixeltable.catalog import Catalog
+
+        cat = Catalog.get()
+        result = set().union(*[cat.get_column_dependents(col.get_tbl().id, col.id) for col in cols])
         if len(result) > 0:
             result.update(self.get_dependent_columns(result))
         return result
@@ -1578,7 +1647,7 @@ class TableVersion:
         return 1
 
     @classmethod
-    def _create_stores_md(cls, stores: Iterable[pxt.io.ExternalStore]) -> list[dict[str, Any]]:
+    def _create_stores_md(cls, stores: Iterable[ExternalStore]) -> list[dict[str, Any]]:
         return [
             {'class': f'{type(store).__module__}.{type(store).__qualname__}', 'md': store.as_dict()} for store in stores
         ]