pixeltable 0.3.14__py3-none-any.whl → 0.4.0__py3-none-any.whl

pixeltable/catalog/table_version.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import copy
 import dataclasses
 import importlib
 import logging
@@ -22,6 +23,10 @@ from pixeltable.utils.exception_handler import run_cleanup_on_exception
 from pixeltable.utils.filecache import FileCache
 from pixeltable.utils.media_store import MediaStore
 
+if TYPE_CHECKING:
+    from pixeltable.plan import SampleClause
+
+
 from ..func.globals import resolve_symbol
 from .column import Column
 from .globals import _POS_COLUMN_NAME, _ROWID_COLUMN_NAME, MediaValidation, UpdateStatus, is_valid_identifier
@@ -50,43 +55,46 @@ class TableVersion:
 
     Instances of TableVersion should not be stored as member variables (ie, used across transaction boundaries).
     Use a TableVersionHandle instead.
+
+    Only TableVersion and Catalog interact directly with stored metadata. Everything else needs to go through these
+    two classes.
     """
 
     id: UUID
-    name: str
-    user: Optional[str]
+
+    # record metadata stored in catalog
+    _tbl_md: schema.TableMd
+    _schema_version_md: schema.TableSchemaVersionMd
+
     effective_version: Optional[int]
-    is_replica: bool
-    version: int
-    comment: str
-    media_validation: MediaValidation
-    num_retained_versions: int
-    schema_version: int
-    view_md: Optional[schema.ViewMd]
     path: Optional[pxt.catalog.TableVersionPath]  # only set for live tables; needed to resolve computed cols
     base: Optional[TableVersionHandle]  # only set for views
-    next_col_id: int
-    next_idx_id: int
-    next_rowid: int
     predicate: Optional[exprs.Expr]
-    mutable_views: list[TableVersionHandle]  # target for data operation propagation (only set for live tables)
+    sample_clause: Optional['SampleClause']
+
     iterator_cls: Optional[type[ComponentIterator]]
     iterator_args: Optional[exprs.InlineDict]
     num_iterator_cols: int
 
+    # target for data operation propagation (only set for non-snapshots, and only records non-snapshot views)
+    mutable_views: set[TableVersionHandle]
+
     # contains complete history of columns, incl dropped ones
     cols: list[Column]
     # contains only user-facing (named) columns visible in this version
     cols_by_name: dict[str, Column]
     # contains only columns visible in this version, both system and user
     cols_by_id: dict[int, Column]
-    # needed for _create_tbl_md()
-    idx_md: dict[int, schema.IndexMd]
     # contains only actively maintained indices
     idxs_by_name: dict[str, TableVersion.IndexInfo]
 
     external_stores: dict[str, pxt.io.ExternalStore]
-    store_tbl: 'store.StoreBase'
+    store_tbl: Optional['store.StoreBase']
+
+    # used by Catalog to invalidate cached instances at the end of a transaction;
+    # True if this instance reflects the state of stored metadata in the context of this transaction and
+    # it is the instance cached in Catalog
+    is_validated: bool
 
     @dataclasses.dataclass
     class IndexInfo:
@@ -106,21 +114,15 @@ class TableVersion:
         mutable_views: list[TableVersionHandle],
         base_path: Optional[pxt.catalog.TableVersionPath] = None,
         base: Optional[TableVersionHandle] = None,
-        # base_store_tbl: Optional['store.StoreBase'] = None,
     ):
+        self.is_validated = True  # a freshly constructed instance is always valid
         self.id = id
-        self.name = tbl_md.name
-        self.user = tbl_md.user
+        self._tbl_md = copy.deepcopy(tbl_md)
+        self._schema_version_md = copy.deepcopy(schema_version_md)
         self.effective_version = effective_version
-        self.version = tbl_md.current_version if effective_version is None else effective_version
-        self.is_replica = tbl_md.is_replica
-        self.comment = schema_version_md.comment
-        self.num_retained_versions = schema_version_md.num_retained_versions
-        self.schema_version = schema_version_md.schema_version
-        self.view_md = tbl_md.view_md  # save this as-is, it's needed for _create_md()
-        self.media_validation = MediaValidation[schema_version_md.media_validation.upper()]
         assert not (self.is_view and base is None)
         self.base = base
+        self.store_tbl = None
 
         # mutable tables need their TableVersionPath for expr eval during updates
         from .table_version_handle import TableVersionHandle
@@ -134,22 +136,14 @@ class TableVersion:
                 assert base_path is not None
             self.path = TableVersionPath(self_handle, base=base_path)
 
-        if self.is_snapshot:
-            self.next_col_id = -1
-            self.next_idx_id = -1  # TODO: can snapshots have separate indices?
-            self.next_rowid = -1
-        else:
-            assert tbl_md.current_version == self.version
-            self.next_col_id = tbl_md.next_col_id
-            self.next_idx_id = tbl_md.next_idx_id
-            self.next_rowid = tbl_md.next_row_id
-
         # view-specific initialization
         from pixeltable import exprs
+        from pixeltable.plan import SampleClause
 
         predicate_dict = None if self.view_md is None or self.view_md.predicate is None else self.view_md.predicate
         self.predicate = exprs.Expr.from_dict(predicate_dict) if predicate_dict is not None else None
-        self.mutable_views = mutable_views
+        sample_dict = None if self.view_md is None or self.view_md.sample_clause is None else self.view_md.sample_clause
+        self.sample_clause = SampleClause.from_dict(sample_dict) if sample_dict is not None else None
 
         # component view-specific initialization
         self.iterator_cls = None
@@ -164,23 +158,15 @@ class TableVersion:
             self.num_iterator_cols = len(output_schema)
             assert tbl_md.view_md.iterator_args is not None
 
-        # register this table version now so that it's available when we're re-creating value exprs
-        cat = pxt.catalog.Catalog.get()
-        cat.add_tbl_version(self)
+        self.mutable_views = set(mutable_views)
+        assert self.is_mutable or len(self.mutable_views) == 0
 
-        # init schema after we determined whether we're a component view, and before we create the store table
         self.cols = []
         self.cols_by_name = {}
         self.cols_by_id = {}
-        self.idx_md = tbl_md.index_md
         self.idxs_by_name = {}
         self.external_stores = {}
 
-        self._init_schema(tbl_md, schema_version_md)
-
-        # Init external stores (this needs to happen after the schema is created)
-        self._init_external_stores(tbl_md)
-
     def __hash__(self) -> int:
         return hash(self.id)
 
@@ -188,19 +174,7 @@ class TableVersion:
         """Create a snapshot copy of this TableVersion"""
         assert not self.is_snapshot
         base = self.path.base.tbl_version if self.is_view else None
-        return TableVersion(
-            self.id,
-            self._create_tbl_md(),
-            self.version,
-            self._create_schema_version_md(preceding_schema_version=0),  # preceding_schema_version: dummy value
-            mutable_views=[],
-            base=base,
-        )
-
-    def create_handle(self) -> TableVersionHandle:
-        from .table_version_handle import TableVersionHandle
-
-        return TableVersionHandle(self.id, self.effective_version, tbl_version=self)
+        return TableVersion(self.id, self.tbl_md, self.version, self.schema_version_md, mutable_views=[], base=base)
 
     @property
     def versioned_name(self) -> str:
@@ -248,6 +222,7 @@ class TableVersion:
             next_col_id=len(cols),
             next_idx_id=0,
             next_row_id=0,
+            view_sn=0,
             column_md=column_md,
             index_md={},
             external_stores=[],
@@ -292,7 +267,13 @@ class TableVersion:
 
         # if this is purely a snapshot (it doesn't require any additional storage for columns and it doesn't have a
         # predicate to apply at runtime), we don't create a physical table and simply use the base's table version path
-        if view_md is not None and view_md.is_snapshot and view_md.predicate is None and len(cols) == 0:
+        if (
+            view_md is not None
+            and view_md.is_snapshot
+            and view_md.predicate is None
+            and view_md.sample_clause is None
+            and len(cols) == 0
+        ):
             session.add(tbl_record)
             session.add(tbl_version_record)
             session.add(schema_version_record)
@@ -306,8 +287,19 @@ class TableVersion:
         tbl_version = cls(
             tbl_record.id, table_md, effective_version, schema_version_md, [], base_path=base_path, base=base
         )
-
+        # TODO: break this up, so that Catalog.create_table() registers tbl_version
+        cat = pxt.catalog.Catalog.get()
+        cat._tbl_versions[tbl_record.id, effective_version] = tbl_version
+        tbl_version.init()
         tbl_version.store_tbl.create()
+        is_mutable = not is_snapshot and not table_md.is_replica
+        if base is not None and base.get().is_mutable and is_mutable:
+            from .table_version_handle import TableVersionHandle
+
+            handle = TableVersionHandle(tbl_version.id, effective_version)
+            assert handle not in base.get().mutable_views
+            base.get().mutable_views.add(handle)
+
         if view_md is None or not view_md.is_snapshot:
             # add default indices, after creating the store table
             for col in tbl_version.cols_by_name.values():
@@ -315,7 +307,7 @@ class TableVersion:
                 assert status is None or status.num_excs == 0
 
         # we re-create the tbl_record here, now that we have new index metadata
-        tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(tbl_version._create_tbl_md()))
+        tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(tbl_version.tbl_md))
         session.add(tbl_record)
         session.add(tbl_version_record)
         session.add(schema_version_record)
@@ -324,45 +316,77 @@ class TableVersion:
     @classmethod
     def create_replica(cls, md: schema.FullTableMd) -> TableVersion:
         tbl_id = UUID(md.tbl_md.tbl_id)
+        _logger.info(f'Creating replica table version {tbl_id}:{md.version_md.version}.')
         view_md = md.tbl_md.view_md
         base_path = pxt.catalog.TableVersionPath.from_md(view_md.base_versions) if view_md is not None else None
         base = base_path.tbl_version if base_path is not None else None
         tbl_version = cls(
             tbl_id, md.tbl_md, md.version_md.version, md.schema_version_md, [], base_path=base_path, base=base
         )
+        cat = pxt.catalog.Catalog.get()
+        cat._tbl_versions[tbl_version.id, tbl_version.effective_version] = tbl_version
+        tbl_version.init()
         tbl_version.store_tbl.create()
+        tbl_version.store_tbl.ensure_columns_exist(col for col in tbl_version.cols if col.is_stored)
         return tbl_version
 
     def drop(self) -> None:
-        from .catalog import Catalog
+        if self.is_view and self.is_mutable:
+            # update mutable_views
+            # TODO: invalidate base to force reload
+            from .table_version_handle import TableVersionHandle
 
-        cat = Catalog.get()
+            assert self.base is not None
+            if self.base.get().is_mutable:
+                self.base.get().mutable_views.remove(TableVersionHandle.create(self))
+
+        # cat = Catalog.get()
         # delete this table and all associated data
         MediaStore.delete(self.id)
         FileCache.get().clear(tbl_id=self.id)
-        cat.delete_tbl_md(self.id)
+        # cat.delete_tbl_md(self.id)
         self.store_tbl.drop()
         # de-register table version from catalog
-        cat.remove_tbl_version(self)
+        # cat.remove_tbl_version(self)
+
+    def init(self) -> None:
+        """
+        Initialize schema-related in-memory metadata separately, now that this TableVersion instance is visible
+        in Catalog.
+        """
+        from .catalog import Catalog
+
+        cat = Catalog.get()
+        assert (self.id, self.effective_version) in cat._tbl_versions
+        self._init_schema()
+        if not self.is_snapshot:
+            cat.record_column_dependencies(self)
 
-    def _init_schema(self, tbl_md: schema.TableMd, schema_version_md: schema.TableSchemaVersionMd) -> None:
+        # init external stores; this needs to happen after the schema is created
+        self._init_external_stores()
+
+    def _init_schema(self) -> None:
         # create columns first, so the indices can reference them
-        self._init_cols(tbl_md, schema_version_md)
+        self._init_cols()
         if not self.is_snapshot:
-            self._init_idxs(tbl_md)
+            self._init_idxs()
         # create the sa schema only after creating the columns and indices
         self._init_sa_schema()
 
-    def _init_cols(self, tbl_md: schema.TableMd, schema_version_md: schema.TableSchemaVersionMd) -> None:
+        # created value_exprs after everything else has been initialized
+        for col in self.cols_by_id.values():
+            col.init_value_expr()
+
+    def _init_cols(self) -> None:
         """Initialize self.cols with the columns visible in our effective version"""
         self.cols = []
         self.cols_by_name = {}
         self.cols_by_id = {}
         # Sort columns in column_md by the position specified in col_md.id to guarantee that all references
         # point backward.
-        sorted_column_md = sorted(tbl_md.column_md.values(), key=lambda item: item.id)
+        sorted_column_md = sorted(self.tbl_md.column_md.values(), key=lambda item: item.id)
         for col_md in sorted_column_md:
-            schema_col_md = schema_version_md.columns.get(col_md.id)
+            schema_col_md = self.schema_version_md.columns.get(col_md.id)
             col_name = schema_col_md.name if schema_col_md is not None else None
             media_val = (
                 MediaValidation[schema_col_md.media_validation.upper()]
@@ -379,8 +403,9 @@ class TableVersion:
                 schema_version_add=col_md.schema_version_add,
                 schema_version_drop=col_md.schema_version_drop,
                 value_expr_dict=col_md.value_expr,
+                tbl=self,
             )
-            col.tbl = self.create_handle()
+            col.tbl = self
             self.cols.append(col)
 
             # populate the lookup structures before Expr.from_dict()
@@ -394,17 +419,17 @@ class TableVersion:
                 self.cols_by_name[col.name] = col
             self.cols_by_id[col.id] = col
 
-            # make sure to traverse columns ordered by position = order in which cols were created;
-            # this guarantees that references always point backwards
-            if not self.is_snapshot and col_md.value_expr is not None:
-                self._record_refd_columns(col)
+            # # make sure to traverse columns ordered by position = order in which cols were created;
+            # # this guarantees that references always point backwards
+            # if not self.is_snapshot and col_md.value_expr is not None:
+            #     self._record_refd_columns(col)
 
-    def _init_idxs(self, tbl_md: schema.TableMd) -> None:
-        self.idx_md = tbl_md.index_md
+    def _init_idxs(self) -> None:
+        # self.idx_md = tbl_md.index_md
         self.idxs_by_name = {}
         import pixeltable.index as index_module
 
-        for md in tbl_md.index_md.values():
+        for md in self.tbl_md.index_md.values():
             if md.schema_version_add > self.schema_version or (
                 md.schema_version_drop is not None and md.schema_version_drop <= self.schema_version
             ):
@@ -439,33 +464,32 @@ class TableVersion:
         else:
             self.store_tbl = StoreTable(self)
 
-    def _update_md(
-        self, timestamp: float, update_tbl_version: bool = True, preceding_schema_version: Optional[int] = None
-    ) -> None:
+    def _write_md(self, new_version: bool, new_version_ts: float, new_schema_version: bool) -> None:
         """Writes table metadata to the database.
 
         Args:
             timestamp: timestamp of the change
-            conn: database connection to use
             update_tbl_version: if `True`, will also write `TableVersion` metadata
             preceding_schema_version: if specified, will also write `TableSchemaVersion` metadata, recording the
                 specified preceding schema version
         """
-        assert update_tbl_version or preceding_schema_version is None
         from pixeltable.catalog import Catalog
 
-        tbl_md = self._create_tbl_md()
-        version_md = self._create_version_md(timestamp) if update_tbl_version else None
-        schema_version_md = (
-            self._create_schema_version_md(preceding_schema_version) if preceding_schema_version is not None else None
+        version_md: Optional[schema.TableVersionMd] = (
+            schema.TableVersionMd(
+                tbl_id=str(self.id),
+                created_at=new_version_ts,
+                version=self.version,
+                schema_version=self.schema_version,
+                additional_md={},
+            )
+            if new_version
+            else None
         )
 
-        Catalog.get().store_tbl_md(self.id, tbl_md, version_md, schema_version_md)
-
-    def ensure_md_loaded(self) -> None:
-        """Ensure that table metadata is loaded."""
-        for col in self.cols_by_id.values():
-            _ = col.value_expr
+        Catalog.get().store_tbl_md(
+            self.id, self._tbl_md, version_md, self._schema_version_md if new_schema_version else None
+        )
 
     def _store_idx_name(self, idx_id: int) -> str:
         """Return name of index in the store, which needs to be globally unique"""
@@ -474,10 +498,10 @@ class TableVersion:
     def add_index(self, col: Column, idx_name: Optional[str], idx: index.IndexBase) -> UpdateStatus:
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
         status = self._add_index(col, idx_name, idx)
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Added index {idx_name} on column {col.name} to table {self.name}')
         return status
 
@@ -522,7 +546,7 @@ class TableVersion:
             schema_version_drop=None,
             records_errors=idx.records_value_errors(),
         )
-        val_col.tbl = self.create_handle()
+        val_col.tbl = self
         val_col.col_type = val_col.col_type.copy(nullable=True)
         self.next_col_id += 1
 
@@ -536,7 +560,7 @@ class TableVersion:
             schema_version_drop=None,
             records_errors=False,
         )
-        undo_col.tbl = self.create_handle()
+        undo_col.tbl = self
         undo_col.col_type = undo_col.col_type.copy(nullable=True)
         self.next_col_id += 1
         return val_col, undo_col
@@ -551,7 +575,7 @@ class TableVersion:
             idx_name = f'idx{idx_id}'
         else:
             assert is_valid_identifier(idx_name)
-            assert idx_name not in [i.name for i in self.idx_md.values()]
+            assert idx_name not in [i.name for i in self._tbl_md.index_md.values()]
         # create and register the index metadata
         idx_cls = type(idx)
         idx_md = schema.IndexMd(
@@ -567,7 +591,7 @@ class TableVersion:
             init_args=idx.as_dict(),
         )
         idx_info = self.IndexInfo(id=idx_id, name=idx_name, idx=idx, col=col, val_col=val_col, undo_col=undo_col)
-        self.idx_md[idx_id] = idx_md
+        self._tbl_md.index_md[idx_id] = idx_md
         self.idxs_by_name[idx_name] = idx_info
         try:
             idx.create_index(self._store_idx_name(idx_id), val_col)
@@ -576,7 +600,7 @@ class TableVersion:
             def cleanup_index() -> None:
                 """Delete the newly added in-memory index structure"""
                 del self.idxs_by_name[idx_name]
-                del self.idx_md[idx_id]
+                del self._tbl_md.index_md[idx_id]
                 self.next_idx_id = idx_id
 
             # Run cleanup only if there has been an exception; otherwise, skip cleanup.
@@ -594,47 +618,48 @@ class TableVersion:
 
     def drop_index(self, idx_id: int) -> None:
         assert not self.is_snapshot
-        assert idx_id in self.idx_md
+        assert idx_id in self._tbl_md.index_md
 
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
-        idx_md = self.idx_md[idx_id]
+        idx_md = self._tbl_md.index_md[idx_id]
         idx_md.schema_version_drop = self.schema_version
         assert idx_md.name in self.idxs_by_name
         idx_info = self.idxs_by_name[idx_md.name]
         # remove this index entry from the active indexes (in memory)
         # and the index metadata (in persistent table metadata)
+        # TODO: this is wrong, it breaks revert()
         del self.idxs_by_name[idx_md.name]
-        del self.idx_md[idx_id]
+        del self._tbl_md.index_md[idx_id]
 
         self._drop_columns([idx_info.val_col, idx_info.undo_col])
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Dropped index {idx_md.name} on table {self.name}')
 
     def add_columns(
         self, cols: Iterable[Column], print_stats: bool, on_error: Literal['abort', 'ignore']
     ) -> UpdateStatus:
-        """Adds a column to the table."""
+        """Adds columns to the table."""
         assert not self.is_snapshot
-        assert all(is_valid_identifier(col.name) for col in cols)
+        assert all(is_valid_identifier(col.name) for col in cols if col.name is not None)
         assert all(col.stored is not None for col in cols)
-        assert all(col.name not in self.cols_by_name for col in cols)
+        assert all(col.name not in self.cols_by_name for col in cols if col.name is not None)
         for col in cols:
-            col.tbl = self.create_handle()
+            col.tbl = self
             col.id = self.next_col_id
             self.next_col_id += 1
 
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
         index_cols: dict[Column, tuple[index.BtreeIndex, Column, Column]] = {}
         all_cols: list[Column] = []
         for col in cols:
             all_cols.append(col)
-            if self._is_btree_indexable(col):
+            if col.name is not None and self._is_btree_indexable(col):
                 idx = index.BtreeIndex(col)
                 val_col, undo_col = self._create_index_columns(idx)
                 index_cols[col] = (idx, val_col, undo_col)
@@ -642,10 +667,10 @@ class TableVersion:
                 all_cols.append(undo_col)
         # Add all columns
         status = self._add_columns(all_cols, print_stats=print_stats, on_error=on_error)
-        # Create indices and their mds
+        # Create indices and their md records
         for col, (idx, val_col, undo_col) in index_cols.items():
             self._create_index(col, val_col, undo_col, idx_name=None, idx=idx)
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Added columns {[col.name for col in cols]} to table {self.name}, new version: {self.version}')
 
         msg = (
@@ -679,9 +704,23 @@ class TableVersion:
             if col.name is not None:
                 self.cols_by_name[col.name] = col
             self.cols_by_id[col.id] = col
-            if col.value_expr is not None:
-                col.check_value_expr()
-                self._record_refd_columns(col)
+
+            # also add to stored md
+            self._tbl_md.column_md[col.id] = schema.ColumnMd(
+                id=col.id,
+                col_type=col.col_type.as_dict(),
+                is_pk=col.is_pk,
+                schema_version_add=col.schema_version_add,
+                schema_version_drop=col.schema_version_drop,
+                value_expr=col.value_expr.as_dict() if col.value_expr is not None else None,
+                stored=col.stored,
+            )
+            if col.name is not None:
+                self._schema_version_md.columns[col.id] = schema.SchemaColumn(
+                    name=col.name,
+                    pos=len(self.cols_by_name),
+                    media_validation=col._media_validation.name.lower() if col._media_validation is not None else None,
+                )
 
             if col.is_stored:
                 self.store_tbl.add_column(col)
@@ -722,14 +761,16 @@ class TableVersion:
                 run_cleanup_on_exception(cleanup_on_error)
                 plan.close()
 
+        pxt.catalog.Catalog.get().record_column_dependencies(self)
+
         if print_stats:
             plan.ctx.profile.print(num_rows=row_count)
-        # TODO(mkornacker): what to do about system columns with exceptions?
+        # TODO: what to do about system columns with exceptions?
         return UpdateStatus(
             num_rows=row_count,
             num_computed_values=row_count,
             num_excs=num_excs,
-            cols_with_excs=[f'{col.tbl.get().name}.{col.name}' for col in cols_with_excs if col.name is not None],
+            cols_with_excs=[f'{col.tbl.name}.{col.name}' for col in cols_with_excs if col.name is not None],
         )
 
     def drop_column(self, col: Column) -> None:
@@ -739,7 +780,7 @@ class TableVersion:
 
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
 
         # drop this column and all dependent index columns and indices
@@ -749,15 +790,17 @@ class TableVersion:
             if idx_info.col != col:
                 continue
             dropped_cols.extend([idx_info.val_col, idx_info.undo_col])
-            idx_md = self.idx_md[idx_info.id]
+            idx_md = self._tbl_md.index_md[idx_info.id]
             idx_md.schema_version_drop = self.schema_version
             assert idx_md.name in self.idxs_by_name
             dropped_idx_names.append(idx_md.name)
+
         # update idxs_by_name
         for idx_name in dropped_idx_names:
             del self.idxs_by_name[idx_name]
+
         self._drop_columns(dropped_cols)
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Dropped column {col.name} from table {self.name}, new version: {self.version}')
 
     def _drop_columns(self, cols: Iterable[Column]) -> None:
@@ -765,21 +808,23 @@ class TableVersion:
         assert not self.is_snapshot
 
         for col in cols:
-            if col.value_expr is not None:
-                # update Column.dependent_cols
-                for c in self.cols:
-                    if c == col:
-                        break
-                    c.dependent_cols.discard(col)
-
             col.schema_version_drop = self.schema_version
             if col.name is not None:
                 assert col.name in self.cols_by_name
                 del self.cols_by_name[col.name]
             assert col.id in self.cols_by_id
             del self.cols_by_id[col.id]
+            # update stored md
+            self._tbl_md.column_md[col.id].schema_version_drop = col.schema_version_drop
+            if col.name is not None:
+                del self._schema_version_md.columns[col.id]
+
+        # update positions
+        for pos, schema_col in enumerate(self._schema_version_md.columns.values()):
+            schema_col.pos = pos
 
         self.store_tbl.create_sa_tbl()
+        pxt.catalog.Catalog.get().record_column_dependencies(self)
 
     def rename_column(self, old_name: str, new_name: str) -> None:
         """Rename a column."""
@@ -794,13 +839,14 @@ class TableVersion:
         del self.cols_by_name[old_name]
         col.name = new_name
         self.cols_by_name[new_name] = col
+        self._schema_version_md.columns[col.id].name = new_name
 
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
 
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'Renamed column {old_name} to {new_name} in table {self.name}, new version: {self.version}')
 
     def set_comment(self, new_comment: Optional[str]) -> None:
@@ -819,9 +865,9 @@ class TableVersion:
     def _create_schema_version(self) -> None:
         # we're creating a new schema version
         self.version += 1
-        preceding_schema_version = self.schema_version
+        self.preceding_schema_version = self.schema_version
         self.schema_version = self.version
-        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
         _logger.info(f'[{self.name}] Updating table schema to version: {self.version}')
 
     def insert(
@@ -836,7 +882,7 @@ class TableVersion:
         """
         from pixeltable.plan import Planner
 
-        assert self.is_insertable()
+        assert self.is_insertable
         assert (rows is None) != (df is None)  # Exactly one must be specified
         if rows is not None:
             plan = Planner.create_insert_plan(self, rows, ignore_errors=not fail_on_exception)
@@ -846,8 +892,8 @@ class TableVersion:
         # this is a base table; we generate rowids during the insert
         def rowids() -> Iterator[int]:
             while True:
-                rowid = self.next_rowid
-                self.next_rowid += 1
+                rowid = self.next_row_id
+                self.next_row_id += 1
                 yield rowid
 
         return self._insert(plan, time.time(), print_stats=print_stats, rowids=rowids(), abort_on_exc=fail_on_exception)
@@ -872,7 +918,7 @@ class TableVersion:
         result.num_excs = num_excs
         result.num_computed_values += exec_plan.ctx.num_computed_exprs * num_rows
         result.cols_with_excs = [f'{self.name}.{self.cols_by_id[cid].name}' for cid in cols_with_excs]
-        self._update_md(timestamp)
+        self._write_md(new_version=True, new_version_ts=timestamp, new_schema_version=False)
 
         # update views
         for view in self.mutable_views:
@@ -1036,13 +1082,13 @@ class TableVersion:
             self.store_tbl.delete_rows(
                 self.version, base_versions=base_versions, match_on_vmin=True, where_clause=where_clause
             )
-            self._update_md(timestamp)
+            self._write_md(new_version=True, new_version_ts=timestamp, new_schema_version=False)
 
         if cascade:
             base_versions = [None if plan is None else self.version, *base_versions]  # don't update in place
             # propagate to views
             for view in self.mutable_views:
-                recomputed_cols = [col for col in recomputed_view_cols if col.tbl == view]
+                recomputed_cols = [col for col in recomputed_view_cols if col.tbl.id == view.id]
                 plan = None
                 if len(recomputed_cols) > 0:
                     from pixeltable.plan import Planner
@@ -1063,7 +1109,7 @@ class TableVersion:
         Args:
             where: a predicate to filter rows to delete.
         """
-        assert self.is_insertable()
+        assert self.is_insertable
         from pixeltable.exprs import Expr
         from pixeltable.plan import Planner
 
@@ -1091,14 +1137,22 @@ class TableVersion:
         Returns:
             number of deleted rows
         """
+        # print(f'calling sql_expr()')
         sql_where_clause = where.sql_expr(exprs.SqlElementCache()) if where is not None else None
+        # #print(f'sql_where_clause={str(sql_where_clause) if sql_where_clause is not None else None}')
+        # sql_cols: list[sql.Column] = []
+        # def collect_cols(col) -> None:
+        #     sql_cols.append(col)
+        # sql.sql.visitors.traverse(sql_where_clause, {}, {'column': collect_cols})
+        # x = [f'{str(c)}:{hash(c)}:{id(c.table)}' for c in sql_cols]
+        # print(f'where_clause cols: {x}')
         num_rows = self.store_tbl.delete_rows(
             self.version + 1, base_versions=base_versions, match_on_vmin=False, where_clause=sql_where_clause
         )
         if num_rows > 0:
             # we're creating a new version
             self.version += 1
-            self._update_md(timestamp)
+            self._write_md(new_version=True, new_version_ts=timestamp, new_schema_version=False)
         for view in self.mutable_views:
             num_rows += view.get().propagate_delete(
                 where=None, base_versions=[self.version, *base_versions], timestamp=timestamp
@@ -1112,17 +1166,13 @@ class TableVersion:
             raise excs.Error('Cannot revert version 0')
         self._revert()
 
-    def _delete_column(self, col: Column) -> None:
-        """Physically remove the column from the schema and the store table"""
-        if col.is_stored:
-            self.store_tbl.drop_column(col)
-        self.cols.remove(col)
-        if col.name is not None:
-            del self.cols_by_name[col.name]
-        del self.cols_by_id[col.id]
-
     def _revert(self) -> None:
-        """Reverts this table version and propagates to views"""
+        """
+        Reverts the stored metadata for this table version and propagates to views.
+
+        Doesn't attempt to revert the in-memory metadata, but instead invalidates this TableVersion instance
+        and relies on Catalog to reload it
+        """
         conn = Env.get().conn
         # make sure we don't have a snapshot referencing this version
         # (unclear how to express this with sqlalchemy)
@@ -1155,109 +1205,206 @@ class TableVersion:
         stmt = sql.update(self.store_tbl.sa_tbl).values(set_clause).where(self.store_tbl.sa_tbl.c.v_max == self.version)
         conn.execute(stmt)
 
-        # revert schema changes
+        # revert schema changes:
+        # - undo changes to self._tbl_md and write that back
+        # - delete newly-added TableVersion/TableSchemaVersion records
         if self.version == self.schema_version:
-            # delete newly-added columns
+            # physically delete newly-added columns and remove them from the stored md
             added_cols = [col for col in self.cols if col.schema_version_add == self.schema_version]
             if len(added_cols) > 0:
-                next_col_id = min(col.id for col in added_cols)
+                self._tbl_md.next_col_id = min(col.id for col in added_cols)
                 for col in added_cols:
-                    self._delete_column(col)
-                self.next_col_id = next_col_id
+                    if col.is_stored:
+                        self.store_tbl.drop_column(col)
+                    del self._tbl_md.column_md[col.id]
 
             # remove newly-added indices from the lookup structures
             # (the value and undo columns got removed in the preceding step)
-            added_idx_md = [md for md in self.idx_md.values() if md.schema_version_add == self.schema_version]
+            added_idx_md = [md for md in self._tbl_md.index_md.values() if md.schema_version_add == self.schema_version]
             if len(added_idx_md) > 0:
-                next_idx_id = min(md.id for md in added_idx_md)
+                self._tbl_md.next_idx_id = min(md.id for md in added_idx_md)
                 for md in added_idx_md:
-                    del self.idx_md[md.id]
-                    del self.idxs_by_name[md.name]
-                self.next_idx_id = next_idx_id
+                    # TODO: drop the index
+                    del self._tbl_md.index_md[md.id]
 
             # make newly-dropped columns visible again
-            dropped_cols = [col for col in self.cols if col.schema_version_drop == self.schema_version]
-            for col in dropped_cols:
-                col.schema_version_drop = None
+            dropped_col_md = [
+                md for md in self._tbl_md.column_md.values() if md.schema_version_drop == self.schema_version
+            ]
+            for col_md in dropped_col_md:
+                col_md.schema_version_drop = None
 
             # make newly-dropped indices visible again
-            dropped_idx_md = [md for md in self.idx_md.values() if md.schema_version_drop == self.schema_version]
-            for md in dropped_idx_md:
-                md.schema_version_drop = None
-
-            session = Env.get().session
-            # we need to determine the preceding schema version and reload the schema
-            schema_version_md_dict = (
-                session.query(schema.TableSchemaVersion.md)
-                .where(schema.TableSchemaVersion.tbl_id == self.id)
-                .where(schema.TableSchemaVersion.schema_version == self.schema_version)
-                .scalar()
-            )
-            preceding_schema_version = schema_version_md_dict['preceding_schema_version']
-            preceding_schema_version_md_dict = (
-                session.query(schema.TableSchemaVersion.md)
-                .where(schema.TableSchemaVersion.tbl_id == self.id)
-                .where(schema.TableSchemaVersion.schema_version == preceding_schema_version)
-                .scalar()
-            )
-            preceding_schema_version_md = schema.md_from_dict(
-                schema.TableSchemaVersionMd, preceding_schema_version_md_dict
-            )
-            tbl_md = self._create_tbl_md()
-            self._init_schema(tbl_md, preceding_schema_version_md)
+            dropped_idx_md = [
+                md for md in self._tbl_md.index_md.values() if md.schema_version_drop == self.schema_version
+            ]
+            for idx_md in dropped_idx_md:
+                idx_md.schema_version_drop = None
 
             conn.execute(
                 sql.delete(schema.TableSchemaVersion.__table__)
                 .where(schema.TableSchemaVersion.tbl_id == self.id)
                 .where(schema.TableSchemaVersion.schema_version == self.schema_version)
             )
-            self.schema_version = preceding_schema_version
-            self.comment = preceding_schema_version_md.comment
-            self.num_retained_versions = preceding_schema_version_md.num_retained_versions
+            self._tbl_md.current_schema_version = self._schema_version_md.preceding_schema_version
 
         conn.execute(
             sql.delete(schema.TableVersion.__table__)
             .where(schema.TableVersion.tbl_id == self.id)
             .where(schema.TableVersion.version == self.version)
         )
+
         self.version -= 1
-        conn.execute(
-            sql.update(schema.Table.__table__)
-            .values({schema.Table.md: dataclasses.asdict(self._create_tbl_md())})
-            .where(schema.Table.id == self.id)
-        )
+        self._write_md(new_version=False, new_version_ts=0, new_schema_version=False)
 
         # propagate to views
+        views_str = ', '.join([str(v.id) for v in self.mutable_views])
+        print(f'revert(): mutable_views={views_str}')
         for view in self.mutable_views:
             view.get()._revert()
+
+        # force reload on next operation
+        self.is_validated = False
+        pxt.catalog.Catalog.get().remove_tbl_version(self)
         _logger.info(f'TableVersion {self.name}: reverted to version {self.version}')
 
-    def _init_external_stores(self, tbl_md: schema.TableMd) -> None:
-        for store_md in tbl_md.external_stores:
+    def _init_external_stores(self) -> None:
+        for store_md in self.tbl_md.external_stores:
             store_cls = resolve_symbol(store_md['class'])
             assert isinstance(store_cls, type) and issubclass(store_cls, pxt.io.ExternalStore)
             store = store_cls.from_dict(store_md['md'])
             self.external_stores[store.name] = store
 
     def link_external_store(self, store: pxt.io.ExternalStore) -> None:
-        store.link(self)  # May result in additional metadata changes
+        self.version += 1
+        self.preceding_schema_version = self.schema_version
+        self.schema_version = self.version
+
         self.external_stores[store.name] = store
-        self._update_md(time.time(), update_tbl_version=False)
+        self._tbl_md.external_stores.append(
+            {'class': f'{type(store).__module__}.{type(store).__qualname__}', 'md': store.as_dict()}
+        )
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
+
+    def unlink_external_store(self, store: pxt.io.ExternalStore) -> None:
+        del self.external_stores[store.name]
+        self.version += 1
+        self.preceding_schema_version = self.schema_version
+        self.schema_version = self.version
+        idx = next(i for i, store_md in enumerate(self._tbl_md.external_stores) if store_md['md']['name'] == store.name)
+        self._tbl_md.external_stores.pop(idx)
+        self._write_md(new_version=True, new_version_ts=time.time(), new_schema_version=True)
+
+    @property
+    def tbl_md(self) -> schema.TableMd:
+        return self._tbl_md
+
+    @property
+    def schema_version_md(self) -> schema.TableSchemaVersionMd:
+        return self._schema_version_md
+
+    @property
+    def view_md(self) -> Optional[schema.ViewMd]:
+        return self._tbl_md.view_md
+
+    @property
+    def name(self) -> str:
+        return self._tbl_md.name
+
+    @property
+    def user(self) -> Optional[str]:
+        return self._tbl_md.user
+
+    @property
+    def is_replica(self) -> bool:
+        return self._tbl_md.is_replica
+
+    @property
+    def comment(self) -> str:
+        return self._schema_version_md.comment
+
+    @comment.setter
+    def comment(self, c: str) -> None:
+        assert self.effective_version is None
+        self._schema_version_md.comment = c
+
+    @property
+    def num_retained_versions(self) -> int:
+        return self._schema_version_md.num_retained_versions
+
+    @num_retained_versions.setter
+    def num_retained_versions(self, n: int) -> None:
+        assert self.effective_version is None
+        self._schema_version_md.num_retained_versions = n
+
+    @property
+    def version(self) -> int:
+        # if this is a snapshot instance, we need to ignore current_version
+        return self._tbl_md.current_version if self.effective_version is None else self.effective_version
+
+    @version.setter
+    def version(self, version: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.current_version = version
+
+    @property
+    def schema_version(self) -> int:
+        return self._schema_version_md.schema_version
+
+    @schema_version.setter
+    def schema_version(self, version: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.current_schema_version = version
+        self._schema_version_md.schema_version = version
+
+    @property
+    def preceding_schema_version(self) -> int:
+        return self._schema_version_md.preceding_schema_version
+
+    @preceding_schema_version.setter
+    def preceding_schema_version(self, v: int) -> None:
+        assert self.effective_version is None
+        self._schema_version_md.preceding_schema_version = v
+
+    @property
+    def media_validation(self) -> MediaValidation:
+        return MediaValidation[self._schema_version_md.media_validation.upper()]
+
+    @property
+    def next_col_id(self) -> int:
+        return self._tbl_md.next_col_id
+
+    @next_col_id.setter
+    def next_col_id(self, id: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.next_col_id = id
 
-    def unlink_external_store(self, store_name: str, delete_external_data: bool) -> None:
-        assert store_name in self.external_stores
-        store = self.external_stores[store_name]
-        store.unlink(self)  # May result in additional metadata changes
-        del self.external_stores[store_name]
-        self._update_md(time.time(), update_tbl_version=False)
+    @property
+    def next_idx_id(self) -> int:
+        return self._tbl_md.next_idx_id
+
+    @next_idx_id.setter
+    def next_idx_id(self, id: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.next_idx_id = id
+
+    @property
+    def next_row_id(self) -> int:
+        return self._tbl_md.next_row_id
 
-        if delete_external_data and isinstance(store, pxt.io.external_store.Project):
-            store.delete()
+    @next_row_id.setter
+    def next_row_id(self, id: int) -> None:
+        assert self.effective_version is None
+        self._tbl_md.next_row_id = id
 
     @property
     def is_snapshot(self) -> bool:
         return self.effective_version is not None
 
+    @property
+    def is_mutable(self) -> bool:
+        return not self.is_snapshot and not self.is_replica
+
     @property
     def is_view(self) -> bool:
         return self.view_md is not None
@@ -1270,6 +1417,7 @@ class TableVersion:
     def is_component_view(self) -> bool:
         return self.iterator_cls is not None
 
+    @property
     def is_insertable(self) -> bool:
         """Returns True if this corresponds to an InsertableTable"""
         return not self.is_snapshot and not self.is_view
@@ -1307,18 +1455,6 @@ class TableVersion:
         names = [c.name for c in self.cols_by_name.values() if c.is_computed]
         return names
 
-    def _record_refd_columns(self, col: Column) -> None:
-        """Update Column.dependent_cols for all cols referenced in col.value_expr."""
-        from pixeltable import exprs
-
-        if col.value_expr_dict is not None:
-            # if we have a value_expr_dict, use that instead of instantiating the value_expr
-            refd_cols = exprs.Expr.get_refd_columns(col.value_expr_dict)
-        else:
-            refd_cols = [e.col for e in col.value_expr.subexprs(expr_class=exprs.ColumnRef)]
-        for refd_col in refd_cols:
-            refd_col.dependent_cols.add(col)
-
     def get_idx_val_columns(self, cols: Iterable[Column]) -> set[Column]:
         result = {info.val_col for col in cols for info in col.get_idx_info().values()}
         return result
@@ -1327,7 +1463,8 @@ class TableVersion:
         """
         Return the set of columns that transitively depend on any of the given ones.
         """
-        result = {dependent_col for col in cols for dependent_col in col.dependent_cols}
+        cat = pxt.catalog.Catalog.get()
+        result = set().union(*[cat.get_column_dependents(col.tbl.id, col.id) for col in cols])
         if len(result) > 0:
             result.update(self.get_dependent_columns(result))
         return result
@@ -1361,24 +1498,6 @@ class TableVersion:
             {'class': f'{type(store).__module__}.{type(store).__qualname__}', 'md': store.as_dict()} for store in stores
         ]
 
-    def _create_tbl_md(self) -> schema.TableMd:
-        return schema.TableMd(
-            tbl_id=str(self.id),
-            name=self.name,
-            user=self.user,
-            is_replica=self.is_replica,
-            current_version=self.version,
-            current_schema_version=self.schema_version,
-            next_col_id=self.next_col_id,
-            next_idx_id=self.next_idx_id,
-            next_row_id=self.next_rowid,
-            column_md=self._create_column_md(self.cols),
-            index_md=self.idx_md,
-            external_stores=self._create_stores_md(self.external_stores.values()),
-            view_md=self.view_md,
-            additional_md={},
-        )
-
     def _create_version_md(self, timestamp: float) -> schema.TableVersionMd:
         return schema.TableVersionMd(
             tbl_id=str(self.id),