pixeltable 0.3.15__py3-none-any.whl → 0.4.0rc1__py3-none-any.whl

pixeltable/catalog/table_version.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import dataclasses
 import importlib
 import logging
@@ -50,43 +51,44 @@ class TableVersion:
 
     Instances of TableVersion should not be stored as member variables (ie, used across transaction boundaries).
     Use a TableVersionHandle instead.
+
+    Only TableVersion and Catalog interact directly with stored metadata. Everything else needs to go through these
+    two classes.
     """
 
     id: UUID
-    name: str
-    user: Optional[str]
+
+    # record metadata stored in catalog
+    _tbl_md: schema.TableMd
+    _schema_version_md: schema.TableSchemaVersionMd
+
     effective_version: Optional[int]
-    is_replica: bool
-    version: int
-    comment: str
-    media_validation: MediaValidation
-    num_retained_versions: int
-    schema_version: int
-    view_md: Optional[schema.ViewMd]
     path: Optional[pxt.catalog.TableVersionPath]  # only set for live tables; needed to resolve computed cols
     base: Optional[TableVersionHandle]  # only set for views
-    next_col_id: int
-    next_idx_id: int
-    next_rowid: int
     predicate: Optional[exprs.Expr]
-    mutable_views: list[TableVersionHandle]  # target for data operation propagation (only set for live tables)
     iterator_cls: Optional[type[ComponentIterator]]
     iterator_args: Optional[exprs.InlineDict]
     num_iterator_cols: int
 
+    # target for data operation propagation (only set for non-snapshots, and only records non-snapshot views)
+    mutable_views: set[TableVersionHandle]
+
     # contains complete history of columns, incl dropped ones
     cols: list[Column]
     # contains only user-facing (named) columns visible in this version
     cols_by_name: dict[str, Column]
     # contains only columns visible in this version, both system and user
     cols_by_id: dict[int, Column]
-    # needed for _create_tbl_md()
-    idx_md: dict[int, schema.IndexMd]
     # contains only actively maintained indices
     idxs_by_name: dict[str, TableVersion.IndexInfo]
 
     external_stores: dict[str, pxt.io.ExternalStore]
-    store_tbl: 'store.StoreBase'
+    store_tbl: Optional['store.StoreBase']
+
+    # used by Catalog to invalidate cached instances at the end of a transaction;
+    # True if this instance reflects the state of stored metadata in the context of this transaction and
+    # it is the instance cached in Catalog
+    is_validated: bool
 
     @dataclasses.dataclass
     class IndexInfo:
@@ -106,21 +108,15 @@ class TableVersion:
         mutable_views: list[TableVersionHandle],
         base_path: Optional[pxt.catalog.TableVersionPath] = None,
         base: Optional[TableVersionHandle] = None,
-        # base_store_tbl: Optional['store.StoreBase'] = None,
     ):
+        self.is_validated = True  # a freshly constructed instance is always valid
         self.id = id
-        self.name = tbl_md.name
-        self.user = tbl_md.user
+        self._tbl_md = copy.deepcopy(tbl_md)
+        self._schema_version_md = copy.deepcopy(schema_version_md)
         self.effective_version = effective_version
-        self.version = tbl_md.current_version if effective_version is None else effective_version
-        self.is_replica = tbl_md.is_replica
-        self.comment = schema_version_md.comment
-        self.num_retained_versions = schema_version_md.num_retained_versions
-        self.schema_version = schema_version_md.schema_version
-        self.view_md = tbl_md.view_md  # save this as-is, it's needed for _create_md()
-        self.media_validation = MediaValidation[schema_version_md.media_validation.upper()]
         assert not (self.is_view and base is None)
         self.base = base
+        self.store_tbl = None
 
         # mutable tables need their TableVersionPath for expr eval during updates
         from .table_version_handle import TableVersionHandle
@@ -134,22 +130,11 @@ class TableVersion:
                 assert base_path is not None
             self.path = TableVersionPath(self_handle, base=base_path)
 
-        if self.is_snapshot:
-            self.next_col_id = -1
-            self.next_idx_id = -1  # TODO: can snapshots have separate indices?
-            self.next_rowid = -1
-        else:
-            assert tbl_md.current_version == self.version
-            self.next_col_id = tbl_md.next_col_id
-            self.next_idx_id = tbl_md.next_idx_id
-            self.next_rowid = tbl_md.next_row_id
-
         # view-specific initialization
         from pixeltable import exprs
 
         predicate_dict = None if self.view_md is None or self.view_md.predicate is None else self.view_md.predicate
         self.predicate = exprs.Expr.from_dict(predicate_dict) if predicate_dict is not None else None
-        self.mutable_views = mutable_views
 
         # component view-specific initialization
         self.iterator_cls = None
@@ -164,22 +149,26 @@ class TableVersion:
             self.num_iterator_cols = len(output_schema)
             assert tbl_md.view_md.iterator_args is not None
 
-        # register this table version now so that it's available when we're re-creating value exprs
-        cat = pxt.catalog.Catalog.get()
-        cat.add_tbl_version(self)
+        self.mutable_views = set(mutable_views)
+        assert self.is_mutable or len(self.mutable_views) == 0
 
-        # init schema after we determined whether we're a component view, and before we create the store table
         self.cols = []
         self.cols_by_name = {}
         self.cols_by_id = {}
-        self.idx_md = tbl_md.index_md
         self.idxs_by_name = {}
         self.external_stores = {}
 
-        self._init_schema(tbl_md, schema_version_md)
+    def init(self) -> None:
+        """
+        Initialize schema-related in-memory metadata separately, now that this TableVersion instance is visible
+        in Catalog.
+        """
+        from .catalog import Catalog
 
-        # Init external stores (this needs to happen after the schema is created)
-        self._init_external_stores(tbl_md)
+        assert (self.id, self.effective_version) in Catalog.get()._tbl_versions
+        self._init_schema()
+        # init external stores; this needs to happen after the schema is created
+        self._init_external_stores()
 
     def __hash__(self) -> int:
         return hash(self.id)
@@ -188,19 +177,7 @@ class TableVersion:
         """Create a snapshot copy of this TableVersion"""
         assert not self.is_snapshot
         base = self.path.base.tbl_version if self.is_view else None
-        return TableVersion(
-            self.id,
-            self._create_tbl_md(),
-            self.version,
-            self._create_schema_version_md(preceding_schema_version=0),  # preceding_schema_version: dummy value
-            mutable_views=[],
-            base=base,
-        )
-
-    def create_handle(self) -> TableVersionHandle:
-        from .table_version_handle import TableVersionHandle
-
-        return TableVersionHandle(self.id, self.effective_version, tbl_version=self)
+        return TableVersion(self.id, self.tbl_md, self.version, self.schema_version_md, mutable_views=[], base=base)
 
     @property
     def versioned_name(self) -> str:
@@ -306,8 +283,19 @@ class TableVersion:
         tbl_version = cls(
             tbl_record.id, table_md, effective_version, schema_version_md, [], base_path=base_path, base=base
         )
-
+        # TODO: break this up, so that Catalog.create_table() registers tbl_version
+        cat = pxt.catalog.Catalog.get()
+        cat._tbl_versions[tbl_record.id, effective_version] = tbl_version
+        tbl_version.init()
         tbl_version.store_tbl.create()
+        is_mutable = not is_snapshot and not table_md.is_replica
+        if base is not None and base.get().is_mutable and is_mutable:
+            from .table_version_handle import TableVersionHandle
+
+            handle = TableVersionHandle(tbl_version.id, effective_version)
+            assert handle not in base.get().mutable_views
+            base.get().mutable_views.add(handle)
+
         if view_md is None or not view_md.is_snapshot:
             # add default indices, after creating the store table
             for col in tbl_version.cols_by_name.values():
@@ -315,7 +303,7 @@ class TableVersion:
                 assert status is None or status.num_excs == 0
 
         # we re-create the tbl_record here, now that we have new index metadata
-        tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(tbl_version._create_tbl_md()))
+        tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(tbl_version.tbl_md))
         session.add(tbl_record)
         session.add(tbl_version_record)
         session.add(schema_version_record)
@@ -331,6 +319,9 @@ class TableVersion:
         tbl_version = cls(
             tbl_id, md.tbl_md, md.version_md.version, md.schema_version_md, [], base_path=base_path, base=base
         )
+        cat = pxt.catalog.Catalog.get()
+        cat._tbl_versions[tbl_version.id, tbl_version.effective_version] = tbl_version
+        tbl_version.init()
         tbl_version.store_tbl.create()
         tbl_version.store_tbl.ensure_columns_exist(col for col in tbl_version.cols if col.is_stored)
         return tbl_version
@@ -338,6 +329,14 @@ class TableVersion:
     def drop(self) -> None:
         from .catalog import Catalog
 
+        if self.is_view and self.is_mutable:
+            # update mutable_views
+            from .table_version_handle import TableVersionHandle
+
+            assert self.base is not None
+            if self.base.get().is_mutable:
+                self.base.get().mutable_views.remove(TableVersionHandle.create(self))
+
         cat = Catalog.get()
         # delete this table and all associated data
         MediaStore.delete(self.id)
@@ -347,24 +346,24 @@ class TableVersion:
         # de-register table version from catalog
         cat.remove_tbl_version(self)
 
-    def _init_schema(self, tbl_md: schema.TableMd, schema_version_md: schema.TableSchemaVersionMd) -> None:
+    def _init_schema(self) -> None:
         # create columns first, so the indices can reference them
-        self._init_cols(tbl_md, schema_version_md)
+        self._init_cols()
         if not self.is_snapshot:
-            self._init_idxs(tbl_md)
+            self._init_idxs()
         # create the sa schema only after creating the columns and indices
         self._init_sa_schema()
 
-    def _init_cols(self, tbl_md: schema.TableMd, schema_version_md: schema.TableSchemaVersionMd) -> None:
+    def _init_cols(self) -> None:
         """Initialize self.cols with the columns visible in our effective version"""
         self.cols = []
         self.cols_by_name = {}
         self.cols_by_id = {}
         # Sort columns in column_md by the position specified in col_md.id to guarantee that all references
         # point backward.
-        sorted_column_md = sorted(tbl_md.column_md.values(), key=lambda item: item.id)
+        sorted_column_md = sorted(self.tbl_md.column_md.values(), key=lambda item: item.id)
         for col_md in sorted_column_md:
-            schema_col_md = schema_version_md.columns.get(col_md.id)
+            schema_col_md = self.schema_version_md.columns.get(col_md.id)
             col_name = schema_col_md.name if schema_col_md is not None else None
             media_val = (
                 MediaValidation[schema_col_md.media_validation.upper()]
@@ -382,7 +381,7 @@ class TableVersion:
                 schema_version_drop=col_md.schema_version_drop,
                 value_expr_dict=col_md.value_expr,
             )
-            col.tbl = self.create_handle()
+            col.tbl = self
             self.cols.append(col)
 
             # populate the lookup structures before Expr.from_dict()
@@ -401,12 +400,12 @@ class TableVersion:
             if not self.is_snapshot and col_md.value_expr is not None:
                 self._record_refd_columns(col)
 
-    def _init_idxs(self, tbl_md: schema.TableMd) -> None:
-        self.idx_md = tbl_md.index_md
+    def _init_idxs(self) -> None:
+        # self.idx_md = tbl_md.index_md
         self.idxs_by_name = {}
         import pixeltable.index as index_module
 
-        for md in tbl_md.index_md.values():
+        for md in self.tbl_md.index_md.values():
             if md.schema_version_add > self.schema_version or (
                 md.schema_version_drop is not None and md.schema_version_drop <= self.schema_version
             ):
@@ -441,28 +440,32 @@ class TableVersion:
         else:
             self.store_tbl = StoreTable(self)
 
-    def _update_md(
-        self, timestamp: float, update_tbl_version: bool = True, preceding_schema_version: Optional[int] = None
-    ) -> None:
+    def _write_md(self, new_version: bool, new_version_ts: float, new_schema_version: bool) -> None:
         """Writes table metadata to the database.
 
         Args:
             timestamp: timestamp of the change
-            conn: database connection to use
             update_tbl_version: if `True`, will also write `TableVersion` metadata
             preceding_schema_version: if specified, will also write `TableSchemaVersion` metadata, recording the
                 specified preceding schema version
         """
-        assert update_tbl_version or preceding_schema_version is None
         from pixeltable.catalog import Catalog
 
-        tbl_md = self._create_tbl_md()
-        version_md = self._create_version_md(timestamp) if update_tbl_version else None
-        schema_version_md = (
-            self._create_schema_version_md(preceding_schema_version) if preceding_schema_version is not None else None
+        version_md: Optional[schema.TableVersionMd] = (
+            schema.TableVersionMd(
+                tbl_id=str(self.id),
+                created_at=new_version_ts,
+                version=self.version,
+                schema_version=self.schema_version,
+                additional_md={},
+            )
+            if new_version
+            else None
         )
 
-        Catalog.get().store_tbl_md(self.id, tbl_md, version_md, schema_version_md)
+        Catalog.get().store_tbl_md(
+            self.id, self._tbl_md, version_md, self._schema_version_md if new_schema_version else None
+        )
 
     def ensure_md_loaded(self) -> None:
         """Ensure that table metadata is loaded."""
@@ -476,10 +479,10 @@ class TableVersion:
     def add_index(self, col: Column, idx_name: Optional[str], idx: index.IndexBase) -> UpdateStatus:
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
         status = self._add_index(col, idx_name, idx)
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Added index {idx_name} on column {col.name} to table {self.name}')
         return status
 
@@ -524,7 +527,7 @@ class TableVersion:
             schema_version_drop=None,
             records_errors=idx.records_value_errors(),
         )
-        val_col.tbl = self.create_handle()
+        val_col.tbl = self
         val_col.col_type = val_col.col_type.copy(nullable=True)
         self.next_col_id += 1
 
@@ -538,7 +541,7 @@ class TableVersion:
             schema_version_drop=None,
             records_errors=False,
         )
-        undo_col.tbl = self.create_handle()
+        undo_col.tbl = self
         undo_col.col_type = undo_col.col_type.copy(nullable=True)
         self.next_col_id += 1
         return val_col, undo_col
@@ -553,7 +556,7 @@ class TableVersion:
             idx_name = f'idx{idx_id}'
         else:
             assert is_valid_identifier(idx_name)
-            assert idx_name not in [i.name for i in self.idx_md.values()]
+            assert idx_name not in [i.name for i in self._tbl_md.index_md.values()]
         # create and register the index metadata
         idx_cls = type(idx)
         idx_md = schema.IndexMd(
@@ -569,7 +572,7 @@ class TableVersion:
             init_args=idx.as_dict(),
         )
         idx_info = self.IndexInfo(id=idx_id, name=idx_name, idx=idx, col=col, val_col=val_col, undo_col=undo_col)
-        self.idx_md[idx_id] = idx_md
+        self._tbl_md.index_md[idx_id] = idx_md
         self.idxs_by_name[idx_name] = idx_info
         try:
             idx.create_index(self._store_idx_name(idx_id), val_col)
@@ -578,7 +581,7 @@ class TableVersion:
             def cleanup_index() -> None:
                 """Delete the newly added in-memory index structure"""
                 del self.idxs_by_name[idx_name]
-                del self.idx_md[idx_id]
+                del self._tbl_md.index_md[idx_id]
                 self.next_idx_id = idx_id
 
             # Run cleanup only if there has been an exception; otherwise, skip cleanup.
@@ -596,47 +599,48 @@ class TableVersion:
 
     def drop_index(self, idx_id: int) -> None:
         assert not self.is_snapshot
-        assert idx_id in self.idx_md
+        assert idx_id in self._tbl_md.index_md
 
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
-        idx_md = self.idx_md[idx_id]
+        idx_md = self._tbl_md.index_md[idx_id]
         idx_md.schema_version_drop = self.schema_version
         assert idx_md.name in self.idxs_by_name
         idx_info = self.idxs_by_name[idx_md.name]
         # remove this index entry from the active indexes (in memory)
         # and the index metadata (in persistent table metadata)
+        # TODO: this is wrong, it breaks revert()
         del self.idxs_by_name[idx_md.name]
-        del self.idx_md[idx_id]
+        del self._tbl_md.index_md[idx_id]
 
         self._drop_columns([idx_info.val_col, idx_info.undo_col])
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Dropped index {idx_md.name} on table {self.name}')
 
     def add_columns(
         self, cols: Iterable[Column], print_stats: bool, on_error: Literal['abort', 'ignore']
     ) -> UpdateStatus:
-        """Adds a column to the table."""
+        """Adds columns to the table."""
         assert not self.is_snapshot
-        assert all(is_valid_identifier(col.name) for col in cols)
+        assert all(is_valid_identifier(col.name) for col in cols if col.name is not None)
         assert all(col.stored is not None for col in cols)
-        assert all(col.name not in self.cols_by_name for col in cols)
+        assert all(col.name not in self.cols_by_name for col in cols if col.name is not None)
         for col in cols:
-            col.tbl = self.create_handle()
+            col.tbl = self
             col.id = self.next_col_id
             self.next_col_id += 1
 
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
         index_cols: dict[Column, tuple[index.BtreeIndex, Column, Column]] = {}
         all_cols: list[Column] = []
         for col in cols:
             all_cols.append(col)
-            if self._is_btree_indexable(col):
+            if col.name is not None and self._is_btree_indexable(col):
                 idx = index.BtreeIndex(col)
                 val_col, undo_col = self._create_index_columns(idx)
                 index_cols[col] = (idx, val_col, undo_col)
@@ -644,10 +648,10 @@ class TableVersion:
                 all_cols.append(undo_col)
         # Add all columns
         status = self._add_columns(all_cols, print_stats=print_stats, on_error=on_error)
-        # Create indices and their mds
+        # Create indices and their md records
         for col, (idx, val_col, undo_col) in index_cols.items():
             self._create_index(col, val_col, undo_col, idx_name=None, idx=idx)
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Added columns {[col.name for col in cols]} to table {self.name}, new version: {self.version}')
 
         msg = (
@@ -685,6 +689,23 @@ class TableVersion:
                 col.check_value_expr()
                 self._record_refd_columns(col)
 
+            # also add to stored md
+            self._tbl_md.column_md[col.id] = schema.ColumnMd(
+                id=col.id,
+                col_type=col.col_type.as_dict(),
+                is_pk=col.is_pk,
+                schema_version_add=col.schema_version_add,
+                schema_version_drop=col.schema_version_drop,
+                value_expr=col.value_expr.as_dict() if col.value_expr is not None else None,
+                stored=col.stored,
+            )
+            if col.name is not None:
+                self._schema_version_md.columns[col.id] = schema.SchemaColumn(
+                    name=col.name,
+                    pos=len(self.cols_by_name),
+                    media_validation=col._media_validation.name.lower() if col._media_validation is not None else None,
+                )
+
             if col.is_stored:
                 self.store_tbl.add_column(col)
 
@@ -731,7 +752,7 @@ class TableVersion:
             num_rows=row_count,
             num_computed_values=row_count,
             num_excs=num_excs,
-            cols_with_excs=[f'{col.tbl.get().name}.{col.name}' for col in cols_with_excs if col.name is not None],
+            cols_with_excs=[f'{col.tbl.name}.{col.name}' for col in cols_with_excs if col.name is not None],
         )
 
     def drop_column(self, col: Column) -> None:
@@ -741,7 +762,7 @@ class TableVersion:
 
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
 
         # drop this column and all dependent index columns and indices
@@ -751,15 +772,17 @@ class TableVersion:
             if idx_info.col != col:
                 continue
             dropped_cols.extend([idx_info.val_col, idx_info.undo_col])
-            idx_md = self.idx_md[idx_info.id]
+            idx_md = self._tbl_md.index_md[idx_info.id]
             idx_md.schema_version_drop = self.schema_version
             assert idx_md.name in self.idxs_by_name
             dropped_idx_names.append(idx_md.name)
+
         # update idxs_by_name
         for idx_name in dropped_idx_names:
             del self.idxs_by_name[idx_name]
+
         self._drop_columns(dropped_cols)
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Dropped column {col.name} from table {self.name}, new version: {self.version}')
 
     def _drop_columns(self, cols: Iterable[Column]) -> None:
@@ -780,6 +803,14 @@ class TableVersion:
                 del self.cols_by_name[col.name]
             assert col.id in self.cols_by_id
             del self.cols_by_id[col.id]
+            # update stored md
+            self._tbl_md.column_md[col.id].schema_version_drop = col.schema_version_drop
+            if col.name is not None:
+                del self._schema_version_md.columns[col.id]
+
+        # update positions
+        for pos, schema_col in enumerate(self._schema_version_md.columns.values()):
+            schema_col.pos = pos
 
         self.store_tbl.create_sa_tbl()
 
@@ -796,13 +827,14 @@ class TableVersion:
         del self.cols_by_name[old_name]
         col.name = new_name
         self.cols_by_name[new_name] = col
+        self._schema_version_md.columns[col.id].name = new_name
 
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
 
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Renamed column {old_name} to {new_name} in table {self.name}, new version: {self.version}')
 
     def set_comment(self, new_comment: Optional[str]) -> None:
@@ -821,9 +853,9 @@ class TableVersion:
     def _create_schema_version(self) -> None:
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'[{self.name}] Updating table schema to version: {self.version}')
 
     def insert(
@@ -838,7 +870,7 @@ class TableVersion:
         """
         from pixeltable.plan import Planner
 
-        assert self.is_insertable()
+        assert self.is_insertable
         assert (rows is None) != (df is None)  # Exactly one must be specified
         if rows is not None:
             plan = Planner.create_insert_plan(self, rows, ignore_errors=not fail_on_exception)
@@ -848,8 +880,8 @@ class TableVersion:
         # this is a base table; we generate rowids during the insert
         def rowids() -> Iterator[int]:
             while True:
-                rowid = self.next_rowid
-                self.next_rowid += 1
+                rowid = self.next_row_id
+                self.next_row_id += 1
                 yield rowid
 
         return self._insert(plan, time.time(), print_stats=print_stats, rowids=rowids(), abort_on_exc=fail_on_exception)
@@ -874,7 +906,7 @@ class TableVersion:
         result.num_excs = num_excs
         result.num_computed_values += exec_plan.ctx.num_computed_exprs * num_rows
         result.cols_with_excs = [f'{self.name}.{self.cols_by_id[cid].name}' for cid in cols_with_excs]
-        self._update_md(timestamp)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=False)
 
         # update views
         for view in self.mutable_views:
@@ -1038,13 +1070,13 @@ class TableVersion:
             self.store_tbl.delete_rows(
                 self.version, base_versions=base_versions, match_on_vmin=True, where_clause=where_clause
             )
-            self._update_md(timestamp)
+            self._write_md(new_version=True, new_version_ts=timestamp, new_schema_version=False)
 
         if cascade:
             base_versions = [None if plan is None else self.version, *base_versions]  # don't update in place
             # propagate to views
             for view in self.mutable_views:
-                recomputed_cols = [col for col in recomputed_view_cols if col.tbl == view]
+                recomputed_cols = [col for col in recomputed_view_cols if col.tbl.id == view.id]
                 plan = None
                 if len(recomputed_cols) > 0:
                     from pixeltable.plan import Planner
@@ -1065,7 +1097,7 @@ class TableVersion:
         Args:
             where: a predicate to filter rows to delete.
         """
-        assert self.is_insertable()
+        assert self.is_insertable
         from pixeltable.exprs import Expr
         from pixeltable.plan import Planner
 
@@ -1093,14 +1125,22 @@ class TableVersion:
         Returns:
             number of deleted rows
         """
+        # print(f'calling sql_expr()')
         sql_where_clause = where.sql_expr(exprs.SqlElementCache()) if where is not None else None
+        # #print(f'sql_where_clause={str(sql_where_clause) if sql_where_clause is not None else None}')
+        # sql_cols: list[sql.Column] = []
+        # def collect_cols(col) -> None:
+        #     sql_cols.append(col)
+        # sql.sql.visitors.traverse(sql_where_clause, {}, {'column': collect_cols})
+        # x = [f'{str(c)}:{hash(c)}:{id(c.table)}' for c in sql_cols]
+        # print(f'where_clause cols: {x}')
         num_rows = self.store_tbl.delete_rows(
             self.version + 1, base_versions=base_versions, match_on_vmin=False, where_clause=sql_where_clause
         )
         if num_rows > 0:
             # we're creating a new version
             self.version += 1
-            self._update_md(timestamp)
+            self._write_md(new_version=True, new_version_ts=timestamp, new_schema_version=False)
         for view in self.mutable_views:
             num_rows += view.get().propagate_delete(
                 where=None, base_versions=[self.version, *base_versions], timestamp=timestamp
@@ -1114,17 +1154,13 @@ class TableVersion:
             raise excs.Error('Cannot revert version 0')
         self._revert()
 
-    def _delete_column(self, col: Column) -> None:
-        """Physically remove the column from the schema and the store table"""
-        if col.is_stored:
-            self.store_tbl.drop_column(col)
-        self.cols.remove(col)
-        if col.name is not None:
-            del self.cols_by_name[col.name]
-        del self.cols_by_id[col.id]
-
     def _revert(self) -> None:
-        """Reverts this table version and propagates to views"""
+        """
+        Reverts the stored metadata for this table version and propagates to views.
+
+        Doesn't attempt to revert the in-memory metadata, but instead invalidates this TableVersion instance
+        and relies on Catalog to reload it
+        """
         conn = Env.get().conn
         # make sure we don't have a snapshot referencing this version
         # (unclear how to express this with sqlalchemy)
@@ -1157,109 +1193,206 @@ class TableVersion:
         stmt = sql.update(self.store_tbl.sa_tbl).values(set_clause).where(self.store_tbl.sa_tbl.c.v_max == self.version)
         conn.execute(stmt)
 
-        # revert schema changes
+        # revert schema changes:
+        # - undo changes to self._tbl_md and write that back
+        # - delete newly-added TableVersion/TableSchemaVersion records
         if self.version == self.schema_version:
-            # delete newly-added columns
+            # physically delete newly-added columns and remove them from the stored md
             added_cols = [col for col in self.cols if col.schema_version_add == self.schema_version]
             if len(added_cols) > 0:
-                next_col_id = min(col.id for col in added_cols)
+                self._tbl_md.next_col_id = min(col.id for col in added_cols)
                 for col in added_cols:
-                    self._delete_column(col)
-                self.next_col_id = next_col_id
+                    if col.is_stored:
+                        self.store_tbl.drop_column(col)
+                    del self._tbl_md.column_md[col.id]
 
             # remove newly-added indices from the lookup structures
             # (the value and undo columns got removed in the preceding step)
-            added_idx_md = [md for md in self.idx_md.values() if md.schema_version_add == self.schema_version]
+            added_idx_md = [md for md in self._tbl_md.index_md.values() if md.schema_version_add == self.schema_version]
             if len(added_idx_md) > 0:
-                next_idx_id = min(md.id for md in added_idx_md)
+                self._tbl_md.next_idx_id = min(md.id for md in added_idx_md)
                 for md in added_idx_md:
-                    del self.idx_md[md.id]
-                    del self.idxs_by_name[md.name]
-                self.next_idx_id = next_idx_id
+                    # TODO: drop the index
+                    del self._tbl_md.index_md[md.id]
 
             # make newly-dropped columns visible again
-            dropped_cols = [col for col in self.cols if col.schema_version_drop == self.schema_version]
-            for col in dropped_cols:
-                col.schema_version_drop = None
+            dropped_col_md = [
+                md for md in self._tbl_md.column_md.values() if md.schema_version_drop == self.schema_version
+            ]
+            for col_md in dropped_col_md:
+                col_md.schema_version_drop = None
 
             # make newly-dropped indices visible again
-            dropped_idx_md = [md for md in self.idx_md.values() if md.schema_version_drop == self.schema_version]
-            for md in dropped_idx_md:
-                md.schema_version_drop = None
-
-            session = Env.get().session
-            # we need to determine the preceding schema version and reload the schema
-            schema_version_md_dict = (
-                session.query(schema.TableSchemaVersion.md)
-                .where(schema.TableSchemaVersion.tbl_id == self.id)
-                .where(schema.TableSchemaVersion.schema_version == self.schema_version)
-                .scalar()
-            )
-            preceding_schema_version = schema_version_md_dict['preceding_schema_version']
-            preceding_schema_version_md_dict = (
-                session.query(schema.TableSchemaVersion.md)
-                .where(schema.TableSchemaVersion.tbl_id == self.id)
-                .where(schema.TableSchemaVersion.schema_version == preceding_schema_version)
-                .scalar()
-            )
-            preceding_schema_version_md = schema.md_from_dict(
-                schema.TableSchemaVersionMd, preceding_schema_version_md_dict
-            )
-            tbl_md = self._create_tbl_md()
-            self._init_schema(tbl_md, preceding_schema_version_md)
+            dropped_idx_md = [
+                md for md in self._tbl_md.index_md.values() if md.schema_version_drop == self.schema_version
+            ]
+            for idx_md in dropped_idx_md:
+                idx_md.schema_version_drop = None
 
             conn.execute(
                 sql.delete(schema.TableSchemaVersion.__table__)
                 .where(schema.TableSchemaVersion.tbl_id == self.id)
                 .where(schema.TableSchemaVersion.schema_version == self.schema_version)
             )
-            self.schema_version = preceding_schema_version
-            self.comment = preceding_schema_version_md.comment
-            self.num_retained_versions = preceding_schema_version_md.num_retained_versions
+            self._tbl_md.current_schema_version = self._schema_version_md.preceding_schema_version
 
         conn.execute(
             sql.delete(schema.TableVersion.__table__)
             .where(schema.TableVersion.tbl_id == self.id)
             .where(schema.TableVersion.version == self.version)
         )
+
         self.version -= 1
-        conn.execute(
-            sql.update(schema.Table.__table__)
-            .values({schema.Table.md: dataclasses.asdict(self._create_tbl_md())})
-            .where(schema.Table.id == self.id)
-        )
+        self._write_md(new_version=False, new_version_ts=0, new_schema_version=False)
 
         # propagate to views
+        views_str = ', '.join([str(v.id) for v in self.mutable_views])
+        print(f'revert(): mutable_views={views_str}')
         for view in self.mutable_views:
             view.get()._revert()
+
+        # force reload on next operation
+        self.is_validated = False
+        pxt.catalog.Catalog.get().remove_tbl_version(self)
         _logger.info(f'TableVersion {self.name}: reverted to version {self.version}')
 
-    def _init_external_stores(self, tbl_md: schema.TableMd) -> None:
-        for store_md in tbl_md.external_stores:
+    def _init_external_stores(self) -> None:
+        for store_md in self.tbl_md.external_stores:
             store_cls = resolve_symbol(store_md['class'])
             assert isinstance(store_cls, type) and issubclass(store_cls, pxt.io.ExternalStore)
             store = store_cls.from_dict(store_md['md'])
             self.external_stores[store.name] = store
 
     def link_external_store(self, store: pxt.io.ExternalStore) -> None:
-        store.link(self)  # May result in additional metadata changes
+        self.version += 1
+        self.preceding_schema_version = self.schema_version
+        self.schema_version = self.version
+
         self.external_stores[store.name] = store
-        self._update_md(time.time(), update_tbl_version=False)
+        self._tbl_md.external_stores.append(
+            {'class': f'{type(store).__module__}.{type(store).__qualname__}', 'md': store.as_dict()}
+        )
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
+
+    def unlink_external_store(self, store: pxt.io.ExternalStore) -> None:
+        del self.external_stores[store.name]
+        self.version += 1
+        self.preceding_schema_version = self.schema_version
+        self.schema_version = self.version
+        idx = next(i for i, store_md in enumerate(self._tbl_md.external_stores) if store_md['md']['name'] == store.name)
+        self._tbl_md.external_stores.pop(idx)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
+
+    @property
+    def tbl_md(self) -> schema.TableMd:
+        return self._tbl_md
+
+    @property
+    def schema_version_md(self) -> schema.TableSchemaVersionMd:
+        return self._schema_version_md
+
+    @property
+    def view_md(self) -> Optional[schema.ViewMd]:
+        return self._tbl_md.view_md
+
+    @property
+    def name(self) -> str:
+        return self._tbl_md.name
+
+    @property
+    def user(self) -> Optional[str]:
+        return self._tbl_md.user
+
+    @property
+    def is_replica(self) -> bool:
+        return self._tbl_md.is_replica
+
+    @property
+    def comment(self) -> str:
+        return self._schema_version_md.comment
+
+    @comment.setter
+    def comment(self, c: str) -> None:
+        assert self.effective_version is None
+        self._schema_version_md.comment = c
+
+    @property
+    def num_retained_versions(self) -> int:
+        return self._schema_version_md.num_retained_versions
 
-    def unlink_external_store(self, store_name: str, delete_external_data: bool) -> None:
-        assert store_name in self.external_stores
-        store = self.external_stores[store_name]
-        store.unlink(self)  # May result in additional metadata changes
-        del self.external_stores[store_name]
-        self._update_md(time.time(), update_tbl_version=False)
+    @num_retained_versions.setter
+    def num_retained_versions(self, n: int) -> None:
+        assert self.effective_version is None
+        self._schema_version_md.num_retained_versions = n
 
-        if delete_external_data and isinstance(store, pxt.io.external_store.Project):
-            store.delete()
+    @property
+    def version(self) -> int:
+        # if this is a snapshot instance, we need to ignore current_version
+        return self._tbl_md.current_version if self.effective_version is None else self.effective_version
+
+    @version.setter
+    def version(self, version: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.current_version = version
+
+    @property
+    def schema_version(self) -> int:
+        return self._schema_version_md.schema_version
+
+    @schema_version.setter
+    def schema_version(self, version: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.current_schema_version = version
+        self._schema_version_md.schema_version = version
+
+    @property
+    def preceding_schema_version(self) -> int:
+        return self._schema_version_md.preceding_schema_version
+
+    @preceding_schema_version.setter
+    def preceding_schema_version(self, v: int) -> None:
+        assert self.effective_version is None
+        self._schema_version_md.preceding_schema_version = v
+
+    @property
+    def media_validation(self) -> MediaValidation:
+        return MediaValidation[self._schema_version_md.media_validation.upper()]
+
+    @property
+    def next_col_id(self) -> int:
+        return self._tbl_md.next_col_id
+
+    @next_col_id.setter
+    def next_col_id(self, id: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.next_col_id = id
+
+    @property
+    def next_idx_id(self) -> int:
+        return self._tbl_md.next_idx_id
+
+    @next_idx_id.setter
+    def next_idx_id(self, id: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.next_idx_id = id
+
+    @property
+    def next_row_id(self) -> int:
+        return self._tbl_md.next_row_id
+
+    @next_row_id.setter
+    def next_row_id(self, id: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.next_row_id = id
 
     @property
     def is_snapshot(self) -> bool:
         return self.effective_version is not None
 
+    @property
+    def is_mutable(self) -> bool:
+        return not self.is_snapshot and not self.is_replica
+
     @property
     def is_view(self) -> bool:
         return self.view_md is not None
@@ -1272,6 +1405,7 @@ class TableVersion:
     def is_component_view(self) -> bool:
         return self.iterator_cls is not None
 
+    @property
     def is_insertable(self) -> bool:
         """Returns True if this corresponds to an InsertableTable"""
         return not self.is_snapshot and not self.is_view
@@ -1363,24 +1497,6 @@ class TableVersion:
             {'class': f'{type(store).__module__}.{type(store).__qualname__}', 'md': store.as_dict()} for store in stores
         ]
 
-    def _create_tbl_md(self) -> schema.TableMd:
-        return schema.TableMd(
-            tbl_id=str(self.id),
-            name=self.name,
-            user=self.user,
-            is_replica=self.is_replica,
-            current_version=self.version,
-            current_schema_version=self.schema_version,
-            next_col_id=self.next_col_id,
-            next_idx_id=self.next_idx_id,
-            next_row_id=self.next_rowid,
-            column_md=self._create_column_md(self.cols),
-            index_md=self.idx_md,
-            external_stores=self._create_stores_md(self.external_stores.values()),
-            view_md=self.view_md,
-            additional_md={},
-        )
-
     def _create_version_md(self, timestamp: float) -> schema.TableVersionMd:
         return schema.TableVersionMd(
             tbl_id=str(self.id),