pixeltable 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

pixeltable/catalog/table_version.py

@@ -13,7 +13,9 @@ import sqlalchemy.orm as orm
 
 import pixeltable
 import pixeltable.func as func
-from pixeltable import exceptions as excs
+import pixeltable.type_system as ts
+import pixeltable.exceptions as excs
+import pixeltable.index as index
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 from pixeltable.metadata import schema
@@ -26,7 +28,8 @@ _logger = logging.getLogger('pixeltable')
 
 class TableVersion:
     """
-    TableVersion represents a particular version of a table/view along with its store table:
+    TableVersion represents a particular version of a table/view along with its physical representation:
+    - the physical representation is a store table with indices
     - the version can be mutable or a snapshot
     - tables and their recursive views form a tree, and a mutable TableVersion also records its own
       mutable views in order to propagate updates
@@ -37,6 +40,14 @@ class TableVersion:
         have TableVersions reference those
     - mutable TableVersions record their TableVersionPath, which is needed for expr evaluation in updates
     """
+    @dataclasses.dataclass
+    class IndexInfo:
+        id: int
+        idx: index.IndexBase
+        col: Column
+        val_col: Column
+        undo_col: Column
+
 
     def __init__(
             self, id: UUID, tbl_md: schema.TableMd, version: int, schema_version_md: schema.TableSchemaVersionMd,
@@ -67,12 +78,13 @@ class TableVersion:
         self.base = base_path.tbl_version if base_path is not None else base
         if self.is_snapshot:
             self.next_col_id = -1
+            self.next_idx_id = -1  # TODO: can snapshots have separate indices?
             self.next_rowid = -1
         else:
             assert tbl_md.current_version == self.version
             self.next_col_id = tbl_md.next_col_id
+            self.next_idx_id = tbl_md.next_idx_id
             self.next_rowid = tbl_md.next_row_id
-        self.column_history = tbl_md.column_history
 
         # view-specific initialization
         from pixeltable import exprs
@@ -101,8 +113,13 @@ class TableVersion:
         cat = catalog.Catalog.get()
         cat.tbl_versions[(self.id, self.effective_version)] = self
 
-        # do this after we determined whether we're a component view, and before we create the store table
-        self._init_schema(schema_version_md)
+        # init schema after we determined whether we're a component view, and before we create the store table
+        self.cols: List[Column] = []  # contains complete history of columns, incl dropped ones
+        self.cols_by_name: dict[str, Column] = {}  # contains only user-facing (named) columns visible in this version
+        self.cols_by_id: dict[int, Column] = {}  # contains only columns visible in this version
+        self.idx_md = tbl_md.index_md  # needed for _create_tbl_md()
+        self.idxs_by_name: dict[str, TableVersion.IndexInfo] = {}  # contains only actively maintained indices
+        self._init_schema(tbl_md, schema_version_md)
 
     def __hash__(self) -> int:
         return hash(self.id)
@@ -111,19 +128,21 @@ class TableVersion:
         """Create a snapshot copy of this TableVersion"""
         assert not self.is_snapshot
         return TableVersion(
-            self.id, self._create_md(), self.version,
+            self.id, self._create_tbl_md(), self.version,
             self._create_schema_version_md(preceding_schema_version=0),  # preceding_schema_version: dummy value
             is_snapshot=True, base=self.base)
 
     @classmethod
     def create(
-            cls, session: orm.Session, dir_id: UUID, name: str, cols: List[Column], num_retained_versions: int, comment: str,
-            base_path: Optional['pixeltable.catalog.TableVersionPath'] = None, view_md: Optional[schema.ViewMd] = None
+            cls, session: orm.Session, dir_id: UUID, name: str, cols: List[Column], num_retained_versions: int,
+            comment: str, base_path: Optional['pixeltable.catalog.TableVersionPath'] = None,
+            view_md: Optional[schema.ViewMd] = None
     ) -> Tuple[UUID, Optional[TableVersion]]:
         # assign ids
         cols_by_name: Dict[str, Column] = {}
         for pos, col in enumerate(cols):
             col.id = pos
+            col.schema_version_add = 0
             cols_by_name[col.name] = col
             if col.value_expr is None and col.compute_func is not None:
                 cls._create_value_expr(col, base_path)
@@ -132,14 +151,11 @@ class TableVersion:
 
         ts = time.time()
         # create schema.Table
-        column_history = {
-            col.id: schema.ColumnHistory(col_id=col.id, schema_version_add=0, schema_version_drop=None)
-            for col in cols
-        }
+        # Column.dependent_cols for existing cols is wrong at this point, but init() will set it correctly
+        column_md = cls._create_column_md(cols)
         table_md = schema.TableMd(
             name=name, current_version=0, current_schema_version=0,
-            next_col_id=len(cols), next_row_id=0, column_history=column_history,
-            view_md=view_md)
+            next_col_id=len(cols), next_idx_id=0, next_row_id=0, column_md=column_md, index_md={}, view_md=view_md)
         tbl_record = schema.Table(dir_id=dir_id, md=dataclasses.asdict(table_md))
         session.add(tbl_record)
         session.flush()  # sets tbl_record.id
@@ -152,16 +168,10 @@ class TableVersion:
         session.add(tbl_version_record)
 
         # create schema.TableSchemaVersion
-        column_md: Dict[int, schema.SchemaColumn] = {}
-        for pos, col in enumerate(cols):
-            # Column.dependent_cols for existing cols is wrong at this point, but init() will set it correctly
-            value_expr_dict = col.value_expr.as_dict() if col.value_expr is not None else None
-            column_md[col.id] = schema.SchemaColumn(
-                pos=pos, name=col.name, col_type=col.col_type.as_dict(),
-                is_pk=col.primary_key, value_expr=value_expr_dict, stored=col.stored, is_indexed=col.is_indexed)
+        schema_col_md = {col.id: schema.SchemaColumn(pos=pos, name=col.name) for pos, col in enumerate(cols)}
 
         schema_version_md = schema.TableSchemaVersionMd(
-            schema_version=0, preceding_schema_version=None, columns=column_md,
+            schema_version=0, preceding_schema_version=None, columns=schema_col_md,
             num_retained_versions=num_retained_versions, comment=comment)
         schema_version_record = schema.TableSchemaVersion(
             tbl_id=tbl_record.id, schema_version=0, md=dataclasses.asdict(schema_version_md))
@@ -202,21 +212,70 @@ class TableVersion:
         del cat.tbl_versions[(self.id, self.effective_version)]
         # TODO: remove from tbl_dependents
 
-    def _init_schema(self, schema_version_md: schema.TableSchemaVersionMd) -> None:
-        """Initialize self.cols as well as self.store_tbl"""
-        self.cols = [Column.from_md(col_id, col_md, self) for col_id, col_md in schema_version_md.columns.items()]
-        self.cols_by_name = {col.name: col for col in self.cols}
-        self.cols_by_id = {col.id: col for col in self.cols}
-
-        # make sure to traverse columns ordered by position = order in which cols were created;
-        # this guarantees that references always point backwards
-        from pixeltable import exprs
-        for col, col_md in zip(self.cols, schema_version_md.columns.values()):
+    def _init_schema(self, tbl_md: schema.TableMd, schema_version_md: schema.TableSchemaVersionMd) -> None:
+        # create columns first, so the indices can reference them
+        self._init_cols(tbl_md, schema_version_md)
+        self._init_idxs(tbl_md)
+        # create the sa schema only after creating the columns and indices
+        self._init_sa_schema()
+
+    def _init_cols(self, tbl_md: schema.TableMd, schema_version_md: schema.TableSchemaVersionMd) -> None:
+        """Initialize self.cols with the columns visible in our effective version"""
+        import pixeltable.exprs as exprs
+        self.cols = []
+        self.cols_by_name = {}
+        self.cols_by_id = {}
+        for col_md in tbl_md.column_md.values():
+            col_name = schema_version_md.columns[col_md.id].name if col_md.id in schema_version_md.columns else None
+            col = Column(
+                col_id=col_md.id, name=col_name, col_type=ts.ColumnType.from_dict(col_md.col_type),
+                is_pk=col_md.is_pk, stored=col_md.stored,
+                schema_version_add=col_md.schema_version_add, schema_version_drop=col_md.schema_version_drop)
             col.tbl = self
+            self.cols.append(col)
+
+            # populate the lookup structures before Expr.from_dict()
+            if col_md.schema_version_add > self.schema_version:
+                # column was added after this version
+                continue
+            if col_md.schema_version_drop is not None and col_md.schema_version_drop <= self.schema_version:
+                # column was dropped
+                continue
+            if col.name is not None:
+                self.cols_by_name[col.name] = col
+            self.cols_by_id[col.id] = col
+
+            # make sure to traverse columns ordered by position = order in which cols were created;
+            # this guarantees that references always point backwards
             if col_md.value_expr is not None:
                 col.value_expr = exprs.Expr.from_dict(col_md.value_expr)
                 self._record_value_expr(col)
 
+    def _init_idxs(self, tbl_md: schema.TableMd) -> None:
+        self.idx_md = tbl_md.index_md
+        self.idxs_by_name = {}
+        import pixeltable.index as index_module
+        for md in tbl_md.index_md.values():
+            if md.schema_version_add > self.schema_version \
+                    or md.schema_version_drop is not None and md.schema_version_drop <= self.schema_version:
+                # column not visible in this schema version
+                continue
+
+            # instantiate index object
+            cls_name = md.class_fqn.rsplit('.', 1)[-1]
+            cls = getattr(index_module, cls_name)
+            idx_col = self.cols_by_id[md.indexed_col_id]
+            idx = cls.from_dict(idx_col, md.init_args)
+
+            # fix up the sa column type of the index value and undo columns
+            val_col = self.cols_by_id[md.index_val_col_id]
+            val_col.sa_col_type = idx.index_sa_type()
+            undo_col = self.cols_by_id[md.index_val_undo_col_id]
+            undo_col.sa_col_type = idx.index_sa_type()
+            idx_info = self.IndexInfo(id=md.id, idx=idx, col=idx_col, val_col=val_col, undo_col=undo_col)
+            self.idxs_by_name[md.name] = idx_info
+
+    def _init_sa_schema(self) -> None:
         # create the sqlalchemy schema; do this after instantiating columns, in order to determine whether they
         # need to record errors
         from pixeltable.store import StoreBase, StoreTable, StoreView, StoreComponentView
@@ -227,8 +286,7 @@ class TableVersion:
         else:
             self.store_tbl: StoreBase = StoreTable(self)
 
-    def _update_md(
-            self, ts: float, preceding_schema_version: Optional[int], conn: sql.engine.Connection) -> None:
+    def _update_md(self, ts: float, preceding_schema_version: Optional[int], conn: sql.engine.Connection) -> None:
         """Update all recorded metadata in response to a data or schema change.
         Args:
             ts: timestamp of the change
@@ -236,8 +294,9 @@ class TableVersion:
         """
         conn.execute(
             sql.update(schema.Table.__table__)
-                .values({schema.Table.md: dataclasses.asdict(self._create_md())})
+                .values({schema.Table.md: dataclasses.asdict(self._create_tbl_md())})
                 .where(schema.Table.id == self.id))
+
         version_md = self._create_version_md(ts)
         conn.execute(
             sql.insert(schema.TableVersion.__table__)
@@ -250,6 +309,80 @@ class TableVersion:
                     tbl_id=self.id, schema_version=self.schema_version,
                     md=dataclasses.asdict(schema_version_md)))
 
+    def _store_idx_name(self, idx_id: int) -> str:
+        """Return name of index in the store, which needs to be globally unique"""
+        return f'idx_{self.id.hex}_{idx_id}'
+
+    def add_index(self, col: Column, idx_name: Optional[str], idx: index.IndexBase) -> UpdateStatus:
+        assert not self.is_snapshot
+        idx_id = self.next_idx_id
+        self.next_idx_id += 1
+        if idx_name is None:
+            idx_name = f'idx{idx_id}'
+        else:
+            assert is_valid_identifier(idx_name)
+            assert idx_name not in [i.name for i in self.idx_md.values()]
+
+        # we're creating a new schema version
+        self.version += 1
+        preceding_schema_version = self.schema_version
+        self.schema_version = self.version
+        with Env.get().engine.begin() as conn:
+            # add the index value and undo columns (which need to be nullable);
+            # we don't create a new schema version, because indices aren't part of the logical schema
+            val_col = Column(
+                col_id=self.next_col_id, name=None, computed_with=idx.index_value_expr(),
+                sa_col_type=idx.index_sa_type(), stored=True,
+                schema_version_add=self.schema_version, schema_version_drop=None)
+            val_col.tbl = self
+            val_col.col_type.nullable = True
+            self.next_col_id += 1
+
+            undo_col = Column(
+                col_id=self.next_col_id, name=None, col_type=val_col.col_type,
+                sa_col_type=val_col.sa_col_type, stored=True,
+                schema_version_add=self.schema_version, schema_version_drop=None)
+            undo_col.tbl = self
+            undo_col.col_type.nullable = True
+            self.next_col_id += 1
+
+            # create and register the index metadata
+            idx_cls = type(idx)
+            idx_md = schema.IndexMd(
+                id=idx_id, name=idx_name,
+                indexed_col_id=col.id, index_val_col_id=val_col.id, index_val_undo_col_id=undo_col.id,
+                schema_version_add=self.schema_version, schema_version_drop=None,
+                class_fqn=idx_cls.__module__ + '.' + idx_cls.__name__, init_args=idx.as_dict())
+            idx_info = self.IndexInfo(id=idx_id, idx=idx, col=col, val_col=val_col, undo_col=undo_col)
+            self.idx_md[idx_id] = idx_md
+            self.idxs_by_name[idx_name] = idx_info
+
+            # add the columns and update the metadata
+            status = self._add_columns([val_col, undo_col], conn, preceding_schema_version=preceding_schema_version)
+            # now create the index structure
+            idx.create_index(self._store_idx_name(idx_id), val_col, conn)
+
+        _logger.info(f'Added index {idx_name} on column {col.name} to table {self.name}')
+        return status
+
+    def drop_index(self, idx_id: int) -> None:
+        assert not self.is_snapshot
+        assert idx_id in self.idx_md
+
+        # we're creating a new schema version
+        self.version += 1
+        preceding_schema_version = self.schema_version
+        self.schema_version = self.version
+        idx_md = self.idx_md[idx_id]
+        idx_md.schema_version_drop = self.schema_version
+        assert idx_md.name in self.idxs_by_name
+        idx_info = self.idxs_by_name[idx_md.name]
+        del self.idxs_by_name[idx_md.name]
+
+        with Env.get().engine.begin() as conn:
+            self._drop_columns([idx_info.val_col, idx_info.undo_col], conn, preceding_schema_version)
+            _logger.info(f'Dropped index {idx_md.name} on table {self.name}')
+
     def add_column(self, col: Column, print_stats: bool = False) -> UpdateStatus:
         """Adds a column to the table.
         """
@@ -268,60 +401,86 @@ class TableVersion:
             col.check_value_expr()
             self._record_value_expr(col)
 
-        row_count = self.store_tbl.count()
-        if row_count > 0 and not col.col_type.nullable and not col.is_computed:
-            raise excs.Error(f'Cannot add non-nullable column "{col.name}" to table {self.name} with existing rows')
-
         # we're creating a new schema version
-        ts = time.time()
         self.version += 1
         preceding_schema_version = self.schema_version
         self.schema_version = self.version
+        with Env.get().engine.begin() as conn:
+            status = self._add_columns([col], conn, preceding_schema_version, print_stats=print_stats)
+        _logger.info(f'Added column {col.name} to table {self.name}, new version: {self.version}')
 
-        self.cols.append(col)
-        self.cols_by_name[col.name] = col
-        self.cols_by_id[col.id] = col
-        self.column_history[col.id] = schema.ColumnHistory(col.id, self.schema_version, None)
+        msg = (
+            f'Added {status.num_rows} column value{"" if status.num_rows == 1 else "s"} '
+            f'with {status.num_excs} error{"" if status.num_excs == 1 else "s"}.'
+        )
+        print(msg)
+        _logger.info(f'Column {col.name}: {msg}')
+        return status
+
+    def _add_columns(
+            self, cols: List[Column], conn: sql.engine.Connection, preceding_schema_version: Optional[int] = None,
+            print_stats: bool = False
+    ) -> UpdateStatus:
+        """Add and populate columns within the current transaction"""
+        ts = time.time()
+
+        row_count = self.store_tbl.count(conn=conn)
+        for col in cols:
+            if not col.col_type.nullable and not col.is_computed:
+                if row_count > 0:
+                    raise excs.Error(
+                        f'Cannot add non-nullable column "{col.name}" to table {self.name} with existing rows')
+
+        num_excs = 0
+        cols_with_excs: List[Column] = []
+        for col in cols:
+            col.schema_version_add = self.schema_version
+            # add the column to the lookup structures now, rather than after the store changes executed successfully,
+            # because it might be referenced by the next column's value_expr
+            self.cols.append(col)
+            if col.name is not None:
+                self.cols_by_name[col.name] = col
+            self.cols_by_id[col.id] = col
 
-        with Env.get().engine.begin() as conn:
-            self._update_md(ts, preceding_schema_version, conn)
-            _logger.info(f'Added column {col.name} to table {self.name}, new version: {self.version}')
             if col.is_stored:
                 self.store_tbl.add_column(col, conn)
 
-        print(f'Added column `{col.name}` to table `{self.name}`.')
-        if row_count == 0:
-            return UpdateStatus()
-        if (not col.is_computed or not col.is_stored) and not col.is_indexed:
-            return UpdateStatus(num_rows=row_count)
-        # compute values for the existing rows and compute embeddings, if this column is indexed;
-        # for some reason, it's not possible to run the following updates in the same transaction as the one
-        # that we just used to create the metadata (sqlalchemy hangs when exec() tries to run the query)
-        from pixeltable.plan import Planner
-        plan, value_expr_slot_idx, embedding_slot_idx = Planner.create_add_column_plan(self.path, col)
-        plan.ctx.num_rows = row_count
-        # TODO: create pgvector index, if col is indexed
+            if not col.is_computed or not col.is_stored or row_count == 0:
+                continue
+
+            # populate the column
+            from pixeltable.plan import Planner
+            plan, value_expr_slot_idx = Planner.create_add_column_plan(self.path, col)
+            plan.ctx.num_rows = row_count
 
-        try:
-            # TODO: do this in the same transaction as the metadata update
-            with Env.get().engine.begin() as conn:
+            try:
                 plan.ctx.conn = conn
                 plan.open()
-                num_excs = self.store_tbl.load_column(col, plan, value_expr_slot_idx, embedding_slot_idx, conn)
-        except sql.exc.DBAPIError as e:
-            self.drop_column(col.name)
-            raise excs.Error(f'Error during SQL execution:\n{e}')
-        finally:
-            plan.close()
-
-        msg = f'Added {row_count} column value{"" if row_count == 1 else "s"} with {num_excs} error{"" if num_excs == 1 else "s"}.'
-        print(msg)
-        _logger.info(f'Column {col.name}: {msg}')
+                num_excs = self.store_tbl.load_column(col, plan, value_expr_slot_idx, conn)
+                if num_excs > 0:
+                    cols_with_excs.append(col)
+            except sql.exc.DBAPIError as e:
+                self.cols.pop()
+                for col in cols:
+                    # remove columns that we already added
+                    if col.id not in self.cols_by_id:
+                        continue
+                    if col.name is not None:
+                        del self.cols_by_name[col.name]
+                    del self.cols_by_id[col.id]
+                # we need to re-initialize the sqlalchemy schema
+                self.store_tbl.create_sa_tbl()
+                raise excs.Error(f'Error during SQL execution:\n{e}')
+            finally:
+                plan.close()
+
+        self._update_md(ts, preceding_schema_version, conn)
         if print_stats:
             plan.ctx.profile.print(num_rows=row_count)
+        # TODO(mkornacker): what to do about system columns with exceptions?
         return UpdateStatus(
             num_rows=row_count, num_computed_values=row_count, num_excs=num_excs,
-            cols_with_excs=[f'{self.name}.{col.name}'] if num_excs > 0 else [])
+            cols_with_excs=[f'{col.tbl.name}.{col.name}'for col in cols_with_excs if col.name is not None])
 
     def drop_column(self, name: str) -> None:
         """Drop a column from the table.
@@ -330,35 +489,58 @@ class TableVersion:
         if name not in self.cols_by_name:
             raise excs.Error(f'Unknown column: {name}')
         col = self.cols_by_name[name]
-        if len(col.dependent_cols) > 0:
+        dependent_user_cols = [c for c in col.dependent_cols if c.name is not None]
+        if len(dependent_user_cols) > 0:
             raise excs.Error(
                 f'Cannot drop column {name} because the following columns depend on it:\n',
-                f'{", ".join([c.name for c in col.dependent_cols])}')
-
-        if col.value_expr is not None:
-            # update Column.dependent_cols
-            for c in self.cols:
-                if c == col:
-                    break
-                c.dependent_cols.discard(col)
+                f'{", ".join([c.name for c in dependent_user_cols])}')
 
         # we're creating a new schema version
-        ts = time.time()
         self.version += 1
         preceding_schema_version = self.schema_version
         self.schema_version = self.version
 
-        self.cols.remove(col)
-        del self.cols_by_name[name]
-        del self.cols_by_id[col.id]
-        self.column_history[col.id].schema_version_drop = self.schema_version
-
         with Env.get().engine.begin() as conn:
-            self._update_md(ts, preceding_schema_version, conn)
-        if col.is_stored:
-            self.store_tbl.drop_column()
+            # drop this column and all dependent index columns and indices
+            dropped_cols = [col]
+            dropped_idx_names: List[str] = []
+            for idx_info in self.idxs_by_name.values():
+                if idx_info.col != col:
+                    continue
+                dropped_cols.extend([idx_info.val_col, idx_info.undo_col])
+                idx_md = self.idx_md[idx_info.id]
+                idx_md.schema_version_drop = self.schema_version
+                assert idx_md.name in self.idxs_by_name
+                dropped_idx_names.append(idx_md.name)
+            # update idxs_by_name
+            for idx_name in dropped_idx_names:
+                del self.idxs_by_name[idx_name]
+            self._drop_columns(dropped_cols, conn, preceding_schema_version)
         _logger.info(f'Dropped column {name} from table {self.name}, new version: {self.version}')
 
+    def _drop_columns(self, cols: list[Column], conn: sql.engine.Connection, preceding_schema_version: int) -> None:
+        """Mark columns as dropped"""
+        assert not self.is_snapshot
+
+        ts = time.time()
+        for col in cols:
+            if col.value_expr is not None:
+                # update Column.dependent_cols
+                for c in self.cols:
+                    if c == col:
+                        break
+                    c.dependent_cols.discard(col)
+
+            col.schema_version_drop = self.schema_version
+            if col.name is not None:
+                assert col.name in self.cols_by_name
+                del self.cols_by_name[col.name]
+            assert col.id in self.cols_by_id
+            del self.cols_by_id[col.id]
+
+        self._update_md(ts, preceding_schema_version, conn)
+        self.store_tbl.create_sa_tbl()
+
     def rename_column(self, old_name: str, new_name: str) -> None:
         """Rename a column.
         """
@@ -387,14 +569,14 @@ class TableVersion:
     def set_comment(self, new_comment: Optional[str]):
         _logger.info(f'[{self.name}] Updating comment: {new_comment}')
         self.comment = new_comment
-        self._commit_new_schema_version()
+        self._create_schema_version()
 
     def set_num_retained_versions(self, new_num_retained_versions: int):
         _logger.info(f'[{self.name}] Updating num_retained_versions: {new_num_retained_versions} (was {self.num_retained_versions})')
         self.num_retained_versions = new_num_retained_versions
-        self._commit_new_schema_version()
+        self._create_schema_version()
 
-    def _commit_new_schema_version(self):
+    def _create_schema_version(self):
         # we're creating a new schema version
         ts = time.time()
         self.version += 1
@@ -448,7 +630,67 @@ class TableVersion:
         return result
 
     def update(
-            self, update_targets: Optional[List[Tuple[Column, 'pixeltable.exprs.Expr']]] = None,
+            self, update_targets: dict[Column, 'pixeltable.exprs.Expr'],
+            where_clause: Optional['pixeltable.exprs.Predicate'] = None, cascade: bool = True
+    ) -> UpdateStatus:
+        with Env.get().engine.begin() as conn:
+            return self._update(conn, update_targets, where_clause, cascade)
+
+    def batch_update(
+            self, batch: list[dict[Column, 'pixeltable.exprs.Expr']], rowids: list[Tuple[int, ...]],
+            cascade: bool = True
+    ) -> UpdateStatus:
+        """Update rows in batch.
+        Args:
+            batch: one dict per row, each mapping Columns to LiteralExprs representing the new values
+            rowids: if not empty, one tuple per row, each containing the rowid values for the corresponding row in batch
+        """
+        # if we do lookups of rowids, we must have one for each row in the batch
+        assert len(rowids) == 0 or len(rowids) == len(batch)
+        import pixeltable.exprs as exprs
+        result_status = UpdateStatus()
+        cols_with_excs: set[str] = set()
+        updated_cols: set[str] = set()
+        pk_cols = self.primary_key_columns()
+        use_rowids = len(rowids) > 0
+
+        with Env.get().engine.begin() as conn:
+            for i, row in enumerate(batch):
+                where_clause: Optional[exprs.Expr] = None
+                if use_rowids:
+                    # construct Where clause to match rowid
+                    num_rowid_cols = len(self.store_tbl.rowid_columns())
+                    for col_idx in range(num_rowid_cols):
+                        assert len(rowids[i]) == num_rowid_cols
+                        clause = exprs.RowidRef(self, col_idx) == rowids[i][col_idx]
+                        if where_clause is None:
+                            where_clause = clause
+                        else:
+                            where_clause = where_clause & clause
+                else:
+                    # construct Where clause for primary key columns
+                    for col in pk_cols:
+                        assert col in row
+                        clause = exprs.ColumnRef(col) == row[col]
+                        if where_clause is None:
+                            where_clause = clause
+                        else:
+                            where_clause = where_clause & clause
+
+                update_targets = {col: row[col] for col in row if col not in pk_cols}
+                status = self._update(conn, update_targets, where_clause, cascade)
+                result_status.num_rows += status.num_rows
+                result_status.num_excs += status.num_excs
+                result_status.num_computed_values += status.num_computed_values
+                cols_with_excs.update(status.cols_with_excs)
+                updated_cols.update(status.updated_cols)
+
+            result_status.cols_with_excs = list(cols_with_excs)
+            result_status.updated_cols = list(updated_cols)
+            return result_status
+
+    def _update(
+            self, conn: sql.engine.Connection, update_targets: dict[Column, 'pixeltable.exprs.Expr'],
             where_clause: Optional['pixeltable.exprs.Predicate'] = None, cascade: bool = True
     ) -> UpdateStatus:
         """Update rows in this table.
@@ -458,21 +700,18 @@ class TableVersion:
             cascade: if True, also update all computed columns that transitively depend on the updated columns,
                 including within views.
         """
-        if update_targets is None:
-            update_targets = []
         assert not self.is_snapshot
         from pixeltable.plan import Planner
         plan, updated_cols, recomputed_cols = \
             Planner.create_update_plan(self.path, update_targets, [], where_clause, cascade)
-        with Env.get().engine.begin() as conn:
-            ts = time.time()
-            result = self._update(
-                plan, where_clause.sql_expr() if where_clause is not None else None, recomputed_cols,
-                base_versions=[], conn=conn, ts=ts, cascade=cascade)
-            result.updated_cols = updated_cols
-            return result
+        ts = time.time()
+        result = self._propagate_update(
+            plan, where_clause.sql_expr() if where_clause is not None else None, recomputed_cols,
+            base_versions=[], conn=conn, ts=ts, cascade=cascade)
+        result.updated_cols = updated_cols
+        return result
 
-    def _update(
+    def _propagate_update(
             self, plan: Optional[exec.ExecNode], where_clause: Optional[sql.ClauseElement],
             recomputed_view_cols: List[Column], base_versions: List[Optional[int]], conn: sql.engine.Connection,
             ts: float, cascade: bool
@@ -497,7 +736,7 @@ class TableVersion:
                 if len(recomputed_cols) > 0:
                     from pixeltable.plan import Planner
                     plan = Planner.create_view_update_plan(view.path, recompute_targets=recomputed_cols)
-                status = view._update(
+                status = view._propagate_update(
                     plan, None, recomputed_view_cols, base_versions=base_versions, conn=conn, ts=ts, cascade=True)
                 result.num_rows += status.num_rows
                 result.num_excs += status.num_excs
@@ -554,6 +793,15 @@ class TableVersion:
             self._revert(session)
             session.commit()
 
+    def _delete_column(self, col: Column, conn: sql.engine.Connection) -> None:
+        """Physically remove the column from the schema and the store table"""
+        if col.is_stored:
+            self.store_tbl.drop_column(col, conn)
+        self.cols.remove(col)
+        if col.name is not None:
+            del self.cols_by_name[col.name]
+        del self.cols_by_id[col.id]
+
     def _revert(self, session: orm.Session) -> None:
         """Reverts this table version and propagates to views"""
         conn = session.connection()
@@ -577,28 +825,47 @@ class TableVersion:
         # delete newly-added data
         MediaStore.delete(self.id, version=self.version)
         conn.execute(sql.delete(self.store_tbl.sa_tbl).where(self.store_tbl.sa_tbl.c.v_min == self.version))
-        # revert new deletions
-        conn.execute(
-            sql.update(self.store_tbl.sa_tbl) \
-                .values({self.store_tbl.sa_tbl.c.v_max: schema.Table.MAX_VERSION})
-                .where(self.store_tbl.sa_tbl.c.v_max == self.version))
 
+        # revert new deletions
+        set_clause = {self.store_tbl.sa_tbl.c.v_max: schema.Table.MAX_VERSION}
+        for index_info in self.idxs_by_name.values():
+            # copy the index value back from the undo column and reset the undo column to NULL
+            set_clause[index_info.val_col.sa_col] = index_info.undo_col.sa_col
+            set_clause[index_info.undo_col.sa_col] = None
+        stmt = sql.update(self.store_tbl.sa_tbl) \
+            .values(set_clause) \
+            .where(self.store_tbl.sa_tbl.c.v_max == self.version)
+        conn.execute(stmt)
+
+        # revert schema changes
         if self.version == self.schema_version:
-            # the current version involved a schema change:
-            # if the schema change was to add a column, we now need to drop it
-            added_col_ids = [
-                col_history.col_id for col_history in self.column_history.values()
-                if col_history.schema_version_add == self.schema_version
-            ]
-            assert len(added_col_ids) <= 1
-            added_col: Optional[Column] = None
-            if len(added_col_ids) == 1:
-                added_col_id = added_col_ids[0]
-                # drop this newly-added column and its ColumnHistory record
-                c = self.cols_by_id[added_col_id]
-                if c.is_stored:
-                    added_col = c
-                del self.column_history[c.id]
+            # delete newly-added columns
+            added_cols = [col for col in self.cols if col.schema_version_add == self.schema_version]
+            if len(added_cols) > 0:
+                next_col_id = min(col.id for col in added_cols)
+                for col in added_cols:
+                    self._delete_column(col, conn)
+                self.next_col_id = next_col_id
+
+            # remove newly-added indices from the lookup structures
+            # (the value and undo columns got removed in the preceding step)
+            added_idx_md = [md for md in self.idx_md.values() if md.schema_version_add == self.schema_version]
+            if len(added_idx_md) > 0:
+                next_idx_id = min(md.id for md in added_idx_md)
+                for md in added_idx_md:
+                    del self.idx_md[md.id]
+                    del self.idxs_by_name[md.name]
+                self.next_idx_id = next_idx_id
+
+            # make newly-dropped columns visible again
+            dropped_cols = [col for col in self.cols if col.schema_version_drop == self.schema_version]
+            for col in dropped_cols:
+                col.schema_version_drop = None
+
+            # make newly-dropped indices visible again
+            dropped_idx_md = [md for md in self.idx_md.values() if md.schema_version_drop == self.schema_version]
+            for md in dropped_idx_md:
+                md.schema_version_drop = None
 
             # we need to determine the preceding schema version and reload the schema
             schema_version_md_dict = session.query(schema.TableSchemaVersion.md) \
@@ -612,11 +879,8 @@ class TableVersion:
                 .scalar()
             preceding_schema_version_md = schema.md_from_dict(
                 schema.TableSchemaVersionMd, preceding_schema_version_md_dict)
-            self._init_schema(preceding_schema_version_md)
-
-            # physically drop the column, but only after we have re-created the schema
-            if added_col is not None:
-                self.store_tbl.drop_column(added_col, conn)
+            tbl_md = self._create_tbl_md()
+            self._init_schema(tbl_md, preceding_schema_version_md)
 
             conn.execute(
                 sql.delete(schema.TableSchemaVersion.__table__)
@@ -634,7 +898,7 @@ class TableVersion:
         self.version -= 1
         conn.execute(
             sql.update(schema.Table.__table__)
-                .values({schema.Table.md: dataclasses.asdict(self._create_md())})
+                .values({schema.Table.md: dataclasses.asdict(self._create_tbl_md())})
                 .where(schema.Table.id == self.id))
 
         # propagate to views
@@ -667,6 +931,10 @@ class TableVersion:
         """Return all non-system columns"""
         return [c for c in self.cols if not self.is_system_column(c)]
 
+    def primary_key_columns(self) -> List[Column]:
+        """Return all non-system columns"""
+        return [c for c in self.cols if c.is_pk]
+
     def get_required_col_names(self) -> List[str]:
         """Return the names of all columns for which values must be specified in insert()"""
         assert not self.is_view()
@@ -727,22 +995,30 @@ class TableVersion:
             return 1 + self.base.num_rowid_columns()
         return 1
 
-    def _create_md(self) -> schema.TableMd:
+    @classmethod
+    def _create_column_md(cls, cols: List[Column]) -> dict[int, schema.ColumnMd]:
+        column_md: Dict[int, schema.ColumnMd] = {}
+        for col in cols:
+            value_expr_dict = col.value_expr.as_dict() if col.value_expr is not None else None
+            column_md[col.id] = schema.ColumnMd(
+                id=col.id, col_type=col.col_type.as_dict(), is_pk=col.is_pk,
+                schema_version_add=col.schema_version_add, schema_version_drop=col.schema_version_drop,
+                value_expr=value_expr_dict, stored=col.stored)
+        return column_md
+
+    def _create_tbl_md(self) -> schema.TableMd:
         return schema.TableMd(
             name=self.name, current_version=self.version, current_schema_version=self.schema_version,
-            next_col_id=self.next_col_id, next_row_id=self.next_rowid, column_history=self.column_history,
-            view_md=self.view_md)
+            next_col_id=self.next_col_id, next_idx_id=self.next_idx_id, next_row_id=self.next_rowid,
+            column_md=self._create_column_md(self.cols), index_md=self.idx_md, view_md=self.view_md)
 
     def _create_version_md(self, ts: float) -> schema.TableVersionMd:
         return schema.TableVersionMd(created_at=ts, version=self.version, schema_version=self.schema_version)
 
     def _create_schema_version_md(self, preceding_schema_version: int) -> schema.TableSchemaVersionMd:
         column_md: Dict[int, schema.SchemaColumn] = {}
-        for pos, col in enumerate(self.cols):
-            value_expr_dict = col.value_expr.as_dict() if col.value_expr is not None else None
-            column_md[col.id] = schema.SchemaColumn(
-                pos=pos, name=col.name, col_type=col.col_type.as_dict(),
-                is_pk=col.primary_key, value_expr=value_expr_dict, stored=col.stored, is_indexed=col.is_indexed)
+        for pos, col in enumerate(self.cols_by_name.values()):
+            column_md[col.id] = schema.SchemaColumn(pos=pos, name=col.name)
         # preceding_schema_version to be set by the caller
         return schema.TableSchemaVersionMd(
             schema_version=self.schema_version, preceding_schema_version=preceding_schema_version,