pixeltable 0.3.6__py3-none-any.whl → 0.3.8__py3-none-any.whl

pixeltable/catalog/table_version.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import dataclasses
 import importlib
-import inspect
 import logging
 import time
 import uuid
@@ -11,12 +10,10 @@ from uuid import UUID
 
 import jsonschema.exceptions
 import sqlalchemy as sql
-import sqlalchemy.orm as orm
 
 import pixeltable as pxt
 import pixeltable.exceptions as excs
 import pixeltable.exprs as exprs
-import pixeltable.func as func
 import pixeltable.index as index
 import pixeltable.type_system as ts
 from pixeltable.env import Env
@@ -32,6 +29,8 @@ from .globals import _POS_COLUMN_NAME, _ROWID_COLUMN_NAME, MediaValidation, Upda
 if TYPE_CHECKING:
     from pixeltable import exec, store
 
+    from .table_version_handle import TableVersionHandle
+
 _logger = logging.getLogger('pixeltable')
 
 
@@ -48,26 +47,27 @@ class TableVersion:
       * TODO: create a separate hierarchy of objects that records the version-independent tree of tables/views, and
         have TableVersions reference those
     - mutable TableVersions record their TableVersionPath, which is needed for expr evaluation in updates
+
+    Instances of TableVersion should not be stored as member variables (ie, used across transaction boundaries).
+    Use a TableVersionHandle instead.
     """
 
     id: UUID
     name: str
+    effective_version: Optional[int]
     version: int
     comment: str
     media_validation: MediaValidation
     num_retained_versions: int
     schema_version: int
     view_md: Optional[schema.ViewMd]
-    is_snapshot: bool
-    include_base_columns: bool
-    effective_version: Optional[int]
-    path: Optional[pxt.catalog.TableVersionPath]
-    base: Optional[TableVersion]
+    path: Optional[pxt.catalog.TableVersionPath]  # only set for live tables; needed to resolve computed cols
+    base: Optional[TableVersionHandle]  # only set for views
     next_col_id: int
     next_idx_id: int
     next_rowid: int
     predicate: Optional[exprs.Expr]
-    mutable_views: list[TableVersion]
+    mutable_views: list[TableVersionHandle]  # target for data operation propagation (only set for live tables)
     iterator_cls: Optional[type[ComponentIterator]]
     iterator_args: Optional[exprs.InlineDict]
     num_iterator_cols: int
@@ -99,37 +99,37 @@ class TableVersion:
         self,
         id: UUID,
         tbl_md: schema.TableMd,
-        version: int,
+        effective_version: Optional[int],
         schema_version_md: schema.TableSchemaVersionMd,
-        base: Optional[TableVersion] = None,
+        mutable_views: list[TableVersionHandle],
         base_path: Optional[pxt.catalog.TableVersionPath] = None,
-        is_snapshot: Optional[bool] = None,
+        base: Optional[TableVersionHandle] = None,
+        # base_store_tbl: Optional['store.StoreBase'] = None,
     ):
-        # only one of base and base_path can be non-None
-        assert base is None or base_path is None
         self.id = id
         self.name = tbl_md.name
-        self.version = version
+        self.effective_version = effective_version
+        self.version = tbl_md.current_version if effective_version is None else effective_version
         self.comment = schema_version_md.comment
         self.num_retained_versions = schema_version_md.num_retained_versions
         self.schema_version = schema_version_md.schema_version
         self.view_md = tbl_md.view_md  # save this as-is, it's needed for _create_md()
-        is_view = tbl_md.view_md is not None
-        self.is_snapshot = (is_view and tbl_md.view_md.is_snapshot) or bool(is_snapshot)
-        self.include_base_columns = not is_view or tbl_md.view_md.include_base_columns
         self.media_validation = MediaValidation[schema_version_md.media_validation.upper()]
-        # a mutable TableVersion doesn't have a static version
-        self.effective_version = self.version if self.is_snapshot else None
+        assert not (self.is_view and base is None)
+        self.base = base
 
         # mutable tables need their TableVersionPath for expr eval during updates
+        from .table_version_handle import TableVersionHandle
         from .table_version_path import TableVersionPath
 
         if self.is_snapshot:
             self.path = None
         else:
-            self.path = TableVersionPath(self, base=base_path) if base_path is not None else TableVersionPath(self)
+            self_handle = TableVersionHandle(id, self.effective_version)
+            if self.is_view:
+                assert base_path is not None
+            self.path = TableVersionPath(self_handle, base=base_path)
 
-        self.base = base_path.tbl_version if base_path is not None else base
         if self.is_snapshot:
             self.next_col_id = -1
             self.next_idx_id = -1  # TODO: can snapshots have separate indices?
@@ -143,17 +143,15 @@ class TableVersion:
         # view-specific initialization
         from pixeltable import exprs
 
-        predicate_dict = None if not is_view or tbl_md.view_md.predicate is None else tbl_md.view_md.predicate
+        predicate_dict = None if self.view_md is None or self.view_md.predicate is None else self.view_md.predicate
         self.predicate = exprs.Expr.from_dict(predicate_dict) if predicate_dict is not None else None
-        self.mutable_views = []  # targets for update propagation
-        if self.base is not None and not self.base.is_snapshot and not self.is_snapshot:
-            self.base.mutable_views.append(self)
+        self.mutable_views = mutable_views
 
         # component view-specific initialization
         self.iterator_cls = None
         self.iterator_args = None
         self.num_iterator_cols = 0
-        if is_view and tbl_md.view_md.iterator_class_fqn is not None:
+        if self.view_md is not None and self.view_md.iterator_class_fqn is not None:
             module_name, class_name = tbl_md.view_md.iterator_class_fqn.rsplit('.', 1)
             module = importlib.import_module(module_name)
             self.iterator_cls = getattr(module, class_name)
@@ -164,7 +162,7 @@ class TableVersion:
 
         # register this table version now so that it's available when we're re-creating value exprs
         cat = pxt.catalog.Catalog.get()
-        cat.tbl_versions[(self.id, self.effective_version)] = self
+        cat.add_tbl_version(self)
 
         # init schema after we determined whether we're a component view, and before we create the store table
         self.cols = []
@@ -179,42 +177,45 @@ class TableVersion:
         # Init external stores (this needs to happen after the schema is created)
         self._init_external_stores(tbl_md)
 
+        # Force column metadata to load, in order to surface any invalid metadata now (as warnings)
+        for col in self.cols_by_id.values():
+            _ = col.value_expr
+
     def __hash__(self) -> int:
         return hash(self.id)
 
-    def _get_column(self, tbl_id: UUID, col_id: int) -> Column:
-        if self.id == tbl_id:
-            return self.cols_by_id[col_id]
-        else:
-            if self.base is None:
-                raise excs.Error(f'Unknown table id: {tbl_id}')
-            return self.base._get_column(tbl_id, col_id)
-
     def create_snapshot_copy(self) -> TableVersion:
         """Create a snapshot copy of this TableVersion"""
         assert not self.is_snapshot
+        base = self.path.base.tbl_version if self.is_view else None
         return TableVersion(
             self.id,
             self._create_tbl_md(),
             self.version,
             self._create_schema_version_md(preceding_schema_version=0),  # preceding_schema_version: dummy value
-            is_snapshot=True,
-            base=self.base,
+            mutable_views=[],
+            base=base,
         )
 
+    def create_handle(self) -> TableVersionHandle:
+        from .table_version_handle import TableVersionHandle
+
+        return TableVersionHandle(self.id, self.effective_version, tbl_version=self)
+
     @classmethod
     def create(
         cls,
-        session: orm.Session,
         dir_id: UUID,
         name: str,
         cols: list[Column],
         num_retained_versions: int,
         comment: str,
         media_validation: MediaValidation,
-        base_path: Optional[pxt.catalog.TableVersionPath] = None,
+        # base_path: Optional[pxt.catalog.TableVersionPath] = None,
         view_md: Optional[schema.ViewMd] = None,
     ) -> tuple[UUID, Optional[TableVersion]]:
+        session = Env.get().session
+
         # assign ids
         cols_by_name: dict[str, Column] = {}
         for pos, col in enumerate(cols):
@@ -276,7 +277,7 @@ class TableVersion:
             tbl_id=tbl_record.id, schema_version=0, md=dataclasses.asdict(schema_version_md)
         )
 
-        # if this is purely a snapshot (it doesn't require any additional storage for columns and it # doesn't have a
+        # if this is purely a snapshot (it doesn't require any additional storage for columns and it doesn't have a
         # predicate to apply at runtime), we don't create a physical table and simply use the base's table version path
         if view_md is not None and view_md.is_snapshot and view_md.predicate is None and len(cols) == 0:
             session.add(tbl_record)
@@ -284,17 +285,20 @@ class TableVersion:
             session.add(schema_version_record)
             return tbl_record.id, None
 
-        assert (base_path is not None) == (view_md is not None)
-        base = base_path.tbl_version if base_path is not None and view_md.is_snapshot else None
-        base_path = base_path if base_path is not None and not view_md.is_snapshot else None
-        tbl_version = cls(tbl_record.id, table_md, 0, schema_version_md, base=base, base_path=base_path)
+        # assert (base_path is not None) == (view_md is not None)
+        is_snapshot = view_md is not None and view_md.is_snapshot
+        effective_version = 0 if is_snapshot else None
+        base_path = pxt.catalog.TableVersionPath.from_md(view_md.base_versions) if view_md is not None else None
+        base = base_path.tbl_version if base_path is not None else None
+        tbl_version = cls(
+            tbl_record.id, table_md, effective_version, schema_version_md, [], base_path=base_path, base=base
+        )
 
-        conn = session.connection()
-        tbl_version.store_tbl.create(conn)
+        tbl_version.store_tbl.create()
         if view_md is None or not view_md.is_snapshot:
             # add default indices, after creating the store table
             for col in tbl_version.cols_by_name.values():
-                status = tbl_version._add_default_index(col, conn=conn)
+                status = tbl_version._add_default_index(col)
                 assert status is None or status.num_excs == 0
 
         # we re-create the tbl_record here, now that we have new index metadata
@@ -305,30 +309,30 @@ class TableVersion:
         return tbl_record.id, tbl_version
 
     @classmethod
-    def delete_md(cls, tbl_id: UUID, conn: sql.Connection) -> None:
+    def delete_md(cls, tbl_id: UUID) -> None:
+        conn = Env.get().conn
         conn.execute(sql.delete(schema.TableSchemaVersion.__table__).where(schema.TableSchemaVersion.tbl_id == tbl_id))
         conn.execute(sql.delete(schema.TableVersion.__table__).where(schema.TableVersion.tbl_id == tbl_id))
         conn.execute(sql.delete(schema.Table.__table__).where(schema.Table.id == tbl_id))
 
     def drop(self) -> None:
-        with Env.get().engine.begin() as conn:
-            # delete this table and all associated data
-            MediaStore.delete(self.id)
-            FileCache.get().clear(tbl_id=self.id)
-            self.delete_md(self.id, conn)
-            self.store_tbl.drop(conn)
+        # delete this table and all associated data
+        MediaStore.delete(self.id)
+        FileCache.get().clear(tbl_id=self.id)
+        self.delete_md(self.id)
+        self.store_tbl.drop()
 
         # de-register table version from catalog
         from .catalog import Catalog
 
         cat = Catalog.get()
-        del cat.tbl_versions[(self.id, self.effective_version)]
-        # TODO: remove from tbl_dependents
+        cat.remove_tbl_version(self)
 
     def _init_schema(self, tbl_md: schema.TableMd, schema_version_md: schema.TableSchemaVersionMd) -> None:
         # create columns first, so the indices can reference them
         self._init_cols(tbl_md, schema_version_md)
-        self._init_idxs(tbl_md)
+        if not self.is_snapshot:
+            self._init_idxs(tbl_md)
         # create the sa schema only after creating the columns and indices
         self._init_sa_schema()
 
@@ -356,7 +360,7 @@ class TableVersion:
                 schema_version_drop=col_md.schema_version_drop,
                 value_expr_dict=col_md.value_expr,
             )
-            col.tbl = self
+            col.tbl = self.create_handle()
             self.cols.append(col)
 
             # populate the lookup structures before Expr.from_dict()
@@ -373,7 +377,6 @@ class TableVersion:
             # make sure to traverse columns ordered by position = order in which cols were created;
             # this guarantees that references always point backwards
             if col_md.value_expr is not None:
-                refd_cols = exprs.Expr.get_refd_columns(col_md.value_expr)
                 self._record_refd_columns(col)
 
     def _init_idxs(self, tbl_md: schema.TableMd) -> None:
@@ -393,7 +396,7 @@ class TableVersion:
             # instantiate index object
             cls_name = md.class_fqn.rsplit('.', 1)[-1]
             cls = getattr(index_module, cls_name)
-            idx_col = self._get_column(UUID(md.indexed_col_tbl_id), md.indexed_col_id)
+            idx_col = self.path.get_column_by_id(UUID(md.indexed_col_tbl_id), md.indexed_col_id)
             idx = cls.from_dict(idx_col, md.init_args)
 
             # fix up the sa column type of the index value and undo columns
@@ -411,19 +414,15 @@ class TableVersion:
         # need to record errors
         from pixeltable.store import StoreComponentView, StoreTable, StoreView
 
-        if self.is_component_view():
+        if self.is_component_view:
             self.store_tbl = StoreComponentView(self)
-        elif self.is_view():
+        elif self.is_view:
             self.store_tbl = StoreView(self)
         else:
             self.store_tbl = StoreTable(self)
 
     def _update_md(
-        self,
-        timestamp: float,
-        conn: sql.engine.Connection,
-        update_tbl_version: bool = True,
-        preceding_schema_version: Optional[int] = None,
+        self, timestamp: float, update_tbl_version: bool = True, preceding_schema_version: Optional[int] = None
     ) -> None:
         """Writes table metadata to the database.
 
@@ -436,6 +435,7 @@ class TableVersion:
         """
         assert update_tbl_version or preceding_schema_version is None
 
+        conn = Env.get().conn
         conn.execute(
             sql.update(schema.Table.__table__)
             .values({schema.Table.md: dataclasses.asdict(self._create_tbl_md())})
@@ -467,13 +467,12 @@ class TableVersion:
         self.version += 1
         preceding_schema_version = self.schema_version
         self.schema_version = self.version
-        with Env.get().engine.begin() as conn:
-            status = self._add_index(col, idx_name, idx, conn)
-            self._update_md(time.time(), conn, preceding_schema_version=preceding_schema_version)
-            _logger.info(f'Added index {idx_name} on column {col.name} to table {self.name}')
-            return status
+        status = self._add_index(col, idx_name, idx)
+        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        _logger.info(f'Added index {idx_name} on column {col.name} to table {self.name}')
+        return status
 
-    def _add_default_index(self, col: Column, conn: sql.engine.Connection) -> Optional[UpdateStatus]:
+    def _add_default_index(self, col: Column) -> Optional[UpdateStatus]:
         """Add a B-tree index on this column if it has a compatible type"""
         if not col.stored:
             # if the column is intentionally not stored, we want to avoid the overhead of an index
@@ -487,12 +486,10 @@ class TableVersion:
         if col.col_type.is_bool_type():
             # B-trees on bools aren't useful
             return None
-        status = self._add_index(col, idx_name=None, idx=index.BtreeIndex(col), conn=conn)
+        status = self._add_index(col, idx_name=None, idx=index.BtreeIndex(col))
         return status
 
-    def _add_index(
-        self, col: Column, idx_name: Optional[str], idx: index.IndexBase, conn: sql.engine.Connection
-    ) -> UpdateStatus:
+    def _add_index(self, col: Column, idx_name: Optional[str], idx: index.IndexBase) -> UpdateStatus:
         assert not self.is_snapshot
         idx_id = self.next_idx_id
         self.next_idx_id += 1
@@ -513,7 +510,7 @@ class TableVersion:
             schema_version_drop=None,
             records_errors=idx.records_value_errors(),
         )
-        val_col.tbl = self
+        val_col.tbl = self.create_handle()
         val_col.col_type = val_col.col_type.copy(nullable=True)
         self.next_col_id += 1
 
@@ -527,7 +524,7 @@ class TableVersion:
             schema_version_drop=None,
             records_errors=False,
         )
-        undo_col.tbl = self
+        undo_col.tbl = self.create_handle()
         undo_col.col_type = undo_col.col_type.copy(nullable=True)
         self.next_col_id += 1
 
@@ -552,9 +549,9 @@ class TableVersion:
         # add the columns and update the metadata
         # TODO support on_error='abort' for indices; it's tricky because of the way metadata changes are entangled
         # with the database operations
-        status = self._add_columns([val_col, undo_col], conn, print_stats=False, on_error='ignore')
+        status = self._add_columns([val_col, undo_col], print_stats=False, on_error='ignore')
         # now create the index structure
-        idx.create_index(self._store_idx_name(idx_id), val_col, conn)
+        idx.create_index(self._store_idx_name(idx_id), val_col)
 
         return status
 
@@ -575,10 +572,9 @@ class TableVersion:
         del self.idxs_by_name[idx_md.name]
         del self.idx_md[idx_id]
 
-        with Env.get().engine.begin() as conn:
-            self._drop_columns([idx_info.val_col, idx_info.undo_col])
-            self._update_md(time.time(), conn, preceding_schema_version=preceding_schema_version)
-            _logger.info(f'Dropped index {idx_md.name} on table {self.name}')
+        self._drop_columns([idx_info.val_col, idx_info.undo_col])
+        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+        _logger.info(f'Dropped index {idx_md.name} on table {self.name}')
 
     def add_columns(
         self, cols: Iterable[Column], print_stats: bool, on_error: Literal['abort', 'ignore']
@@ -589,7 +585,7 @@ class TableVersion:
         assert all(col.stored is not None for col in cols)
         assert all(col.name not in self.cols_by_name for col in cols)
         for col in cols:
-            col.tbl = self
+            col.tbl = self.create_handle()
             col.id = self.next_col_id
             self.next_col_id += 1
 
@@ -597,11 +593,10 @@ class TableVersion:
         self.version += 1
         preceding_schema_version = self.schema_version
         self.schema_version = self.version
-        with Env.get().engine.begin() as conn:
-            status = self._add_columns(cols, conn, print_stats=print_stats, on_error=on_error)
-            for col in cols:
-                _ = self._add_default_index(col, conn)
-            self._update_md(time.time(), conn, preceding_schema_version=preceding_schema_version)
+        status = self._add_columns(cols, print_stats=print_stats, on_error=on_error)
+        for col in cols:
+            _ = self._add_default_index(col)
+        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
         _logger.info(f'Added columns {[col.name for col in cols]} to table {self.name}, new version: {self.version}')
 
         msg = (
@@ -613,15 +608,11 @@ class TableVersion:
         return status
 
     def _add_columns(
-        self,
-        cols: Iterable[Column],
-        conn: sql.engine.Connection,
-        print_stats: bool,
-        on_error: Literal['abort', 'ignore'],
+        self, cols: Iterable[Column], print_stats: bool, on_error: Literal['abort', 'ignore']
     ) -> UpdateStatus:
         """Add and populate columns within the current transaction"""
         cols = list(cols)
-        row_count = self.store_tbl.count(conn=conn)
+        row_count = self.store_tbl.count()
         for col in cols:
             if not col.col_type.nullable and not col.is_computed:
                 if row_count > 0:
@@ -644,7 +635,7 @@ class TableVersion:
                 self._record_refd_columns(col)
 
             if col.is_stored:
-                self.store_tbl.add_column(col, conn)
+                self.store_tbl.add_column(col)
 
             if not col.is_computed or not col.is_stored or row_count == 0:
                 continue
@@ -656,10 +647,9 @@ class TableVersion:
             plan.ctx.num_rows = row_count
 
             try:
-                plan.ctx.set_conn(conn)
                 plan.open()
                 try:
-                    num_excs = self.store_tbl.load_column(col, plan, value_expr_slot_idx, conn, on_error)
+                    num_excs = self.store_tbl.load_column(col, plan, value_expr_slot_idx, on_error)
                 except sql.exc.DBAPIError as exc:
                     # Wrap the DBAPIError in an excs.Error to unify processing in the subsequent except block
                     raise excs.Error(f'SQL error during execution of computed column `{col.name}`:\n{exc}') from exc
@@ -687,12 +677,11 @@ class TableVersion:
             num_rows=row_count,
             num_computed_values=row_count,
             num_excs=num_excs,
-            cols_with_excs=[f'{col.tbl.name}.{col.name}' for col in cols_with_excs if col.name is not None],
+            cols_with_excs=[f'{col.tbl.get().name}.{col.name}' for col in cols_with_excs if col.name is not None],
         )
 
     def drop_column(self, col: Column) -> None:
         """Drop a column from the table."""
-        from pixeltable.catalog import Catalog
 
         assert not self.is_snapshot
 
@@ -701,23 +690,22 @@ class TableVersion:
         preceding_schema_version = self.schema_version
         self.schema_version = self.version
 
-        with Env.get().engine.begin() as conn:
-            # drop this column and all dependent index columns and indices
-            dropped_cols = [col]
-            dropped_idx_names: list[str] = []
-            for idx_info in self.idxs_by_name.values():
-                if idx_info.col != col:
-                    continue
-                dropped_cols.extend([idx_info.val_col, idx_info.undo_col])
-                idx_md = self.idx_md[idx_info.id]
-                idx_md.schema_version_drop = self.schema_version
-                assert idx_md.name in self.idxs_by_name
-                dropped_idx_names.append(idx_md.name)
-            # update idxs_by_name
-            for idx_name in dropped_idx_names:
-                del self.idxs_by_name[idx_name]
-            self._drop_columns(dropped_cols)
-            self._update_md(time.time(), conn, preceding_schema_version=preceding_schema_version)
+        # drop this column and all dependent index columns and indices
+        dropped_cols = [col]
+        dropped_idx_names: list[str] = []
+        for idx_info in self.idxs_by_name.values():
+            if idx_info.col != col:
+                continue
+            dropped_cols.extend([idx_info.val_col, idx_info.undo_col])
+            idx_md = self.idx_md[idx_info.id]
+            idx_md.schema_version_drop = self.schema_version
+            assert idx_md.name in self.idxs_by_name
+            dropped_idx_names.append(idx_md.name)
+        # update idxs_by_name
+        for idx_name in dropped_idx_names:
+            del self.idxs_by_name[idx_name]
+        self._drop_columns(dropped_cols)
+        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
         _logger.info(f'Dropped column {col.name} from table {self.name}, new version: {self.version}')
 
     def _drop_columns(self, cols: Iterable[Column]) -> None:
@@ -760,8 +748,7 @@ class TableVersion:
         preceding_schema_version = self.schema_version
         self.schema_version = self.version
 
-        with Env.get().engine.begin() as conn:
-            self._update_md(time.time(), conn, preceding_schema_version=preceding_schema_version)
+        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
         _logger.info(f'Renamed column {old_name} to {new_name} in table {self.name}, new version: {self.version}')
 
     def set_comment(self, new_comment: Optional[str]):
@@ -781,15 +768,13 @@ class TableVersion:
         self.version += 1
         preceding_schema_version = self.schema_version
         self.schema_version = self.version
-        with Env.get().engine.begin() as conn:
-            self._update_md(time.time(), conn, preceding_schema_version=preceding_schema_version)
+        self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
         _logger.info(f'[{self.name}] Updating table schema to version: {self.version}')
 
     def insert(
         self,
         rows: Optional[list[dict[str, Any]]],
         df: Optional[pxt.DataFrame],
-        conn: Optional[sql.engine.Connection] = None,
         print_stats: bool = False,
         fail_on_exception: bool = True,
     ) -> UpdateStatus:
@@ -812,20 +797,11 @@ class TableVersion:
                 self.next_rowid += 1
                 yield rowid
 
-        if conn is None:
-            with Env.get().engine.begin() as conn:
-                return self._insert(
-                    plan, conn, time.time(), print_stats=print_stats, rowids=rowids(), abort_on_exc=fail_on_exception
-                )
-        else:
-            return self._insert(
-                plan, conn, time.time(), print_stats=print_stats, rowids=rowids(), abort_on_exc=fail_on_exception
-            )
+        return self._insert(plan, time.time(), print_stats=print_stats, rowids=rowids(), abort_on_exc=fail_on_exception)
 
     def _insert(
         self,
         exec_plan: 'exec.ExecNode',
-        conn: sql.engine.Connection,
         timestamp: float,
         *,
         rowids: Optional[Iterator[int]] = None,
@@ -837,20 +813,20 @@ class TableVersion:
         self.version += 1
         result = UpdateStatus()
         num_rows, num_excs, cols_with_excs = self.store_tbl.insert_rows(
-            exec_plan, conn, v_min=self.version, rowids=rowids, abort_on_exc=abort_on_exc
+            exec_plan, v_min=self.version, rowids=rowids, abort_on_exc=abort_on_exc
         )
         result.num_rows = num_rows
         result.num_excs = num_excs
         result.num_computed_values += exec_plan.ctx.num_computed_exprs * num_rows
         result.cols_with_excs = [f'{self.name}.{self.cols_by_id[cid].name}' for cid in cols_with_excs]
-        self._update_md(timestamp, conn)
+        self._update_md(timestamp)
 
         # update views
         for view in self.mutable_views:
             from pixeltable.plan import Planner
 
-            plan, _ = Planner.create_view_load_plan(view.path, propagates_insert=True)
-            status = view._insert(plan, conn, timestamp, print_stats=print_stats)
+            plan, _ = Planner.create_view_load_plan(view.get().path, propagates_insert=True)
+            status = view.get()._insert(plan, timestamp, print_stats=print_stats)
             result.num_rows += status.num_rows
             result.num_excs += status.num_excs
             result.num_computed_values += status.num_computed_values
@@ -886,22 +862,20 @@ class TableVersion:
             if analysis_info.filter is not None:
                 raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')
 
-        with Env.get().engine.begin() as conn:
-            plan, updated_cols, recomputed_cols = Planner.create_update_plan(self.path, update_spec, [], where, cascade)
-            from pixeltable.exprs import SqlElementCache
-
-            result = self.propagate_update(
-                plan,
-                where.sql_expr(SqlElementCache()) if where is not None else None,
-                recomputed_cols,
-                base_versions=[],
-                conn=conn,
-                timestamp=time.time(),
-                cascade=cascade,
-                show_progress=True,
-            )
-            result.updated_cols = updated_cols
-            return result
+        plan, updated_cols, recomputed_cols = Planner.create_update_plan(self.path, update_spec, [], where, cascade)
+        from pixeltable.exprs import SqlElementCache
+
+        result = self.propagate_update(
+            plan,
+            where.sql_expr(SqlElementCache()) if where is not None else None,
+            recomputed_cols,
+            base_versions=[],
+            timestamp=time.time(),
+            cascade=cascade,
+            show_progress=True,
+        )
+        result.updated_cols = updated_cols
+        return result
 
     def batch_update(
         self,
@@ -920,33 +894,24 @@ class TableVersion:
         assert len(rowids) == 0 or len(rowids) == len(batch)
         cols_with_excs: set[str] = set()
 
-        with Env.get().engine.begin() as conn:
-            from pixeltable.plan import Planner
+        from pixeltable.plan import Planner
 
-            plan, row_update_node, delete_where_clause, updated_cols, recomputed_cols = (
-                Planner.create_batch_update_plan(self.path, batch, rowids, cascade=cascade)
-            )
-            result = self.propagate_update(
-                plan,
-                delete_where_clause,
-                recomputed_cols,
-                base_versions=[],
-                conn=conn,
-                timestamp=time.time(),
-                cascade=cascade,
-            )
-            result.updated_cols = [c.qualified_name for c in updated_cols]
-
-            unmatched_rows = row_update_node.unmatched_rows()
-            if len(unmatched_rows) > 0:
-                if error_if_not_exists:
-                    raise excs.Error(f'batch_update(): {len(unmatched_rows)} row(s) not found')
-                if insert_if_not_exists:
-                    insert_status = self.insert(
-                        unmatched_rows, None, conn=conn, print_stats=False, fail_on_exception=False
-                    )
-                    result += insert_status
-            return result
+        plan, row_update_node, delete_where_clause, updated_cols, recomputed_cols = Planner.create_batch_update_plan(
+            self.path, batch, rowids, cascade=cascade
+        )
+        result = self.propagate_update(
+            plan, delete_where_clause, recomputed_cols, base_versions=[], timestamp=time.time(), cascade=cascade
+        )
+        result.updated_cols = [c.qualified_name for c in updated_cols]
+
+        unmatched_rows = row_update_node.unmatched_rows()
+        if len(unmatched_rows) > 0:
+            if error_if_not_exists:
+                raise excs.Error(f'batch_update(): {len(unmatched_rows)} row(s) not found')
+            if insert_if_not_exists:
+                insert_status = self.insert(unmatched_rows, None, print_stats=False, fail_on_exception=False)
+                result += insert_status
+        return result
 
     def _validate_update_spec(
         self, value_spec: dict[str, Any], allow_pk: bool, allow_exprs: bool
@@ -1000,7 +965,6 @@ class TableVersion:
         where_clause: Optional[sql.ColumnElement],
         recomputed_view_cols: list[Column],
         base_versions: list[Optional[int]],
-        conn: sql.engine.Connection,
         timestamp: float,
         cascade: bool,
         show_progress: bool = True,
@@ -1010,32 +974,26 @@ class TableVersion:
             # we're creating a new version
             self.version += 1
             result.num_rows, result.num_excs, cols_with_excs = self.store_tbl.insert_rows(
-                plan, conn, v_min=self.version, show_progress=show_progress
+                plan, v_min=self.version, show_progress=show_progress
             )
             result.cols_with_excs = [f'{self.name}.{self.cols_by_id[cid].name}' for cid in cols_with_excs]
             self.store_tbl.delete_rows(
-                self.version, base_versions=base_versions, match_on_vmin=True, where_clause=where_clause, conn=conn
+                self.version, base_versions=base_versions, match_on_vmin=True, where_clause=where_clause
             )
-            self._update_md(timestamp, conn)
+            self._update_md(timestamp)
 
         if cascade:
             base_versions = [None if plan is None else self.version] + base_versions  # don't update in place
             # propagate to views
             for view in self.mutable_views:
-                recomputed_cols = [col for col in recomputed_view_cols if col.tbl is view]
+                recomputed_cols = [col for col in recomputed_view_cols if col.tbl == view]
                 plan = None
                 if len(recomputed_cols) > 0:
                     from pixeltable.plan import Planner
 
-                    plan = Planner.create_view_update_plan(view.path, recompute_targets=recomputed_cols)
-                status = view.propagate_update(
-                    plan,
-                    None,
-                    recomputed_view_cols,
-                    base_versions=base_versions,
-                    conn=conn,
-                    timestamp=timestamp,
-                    cascade=True,
+                    plan = Planner.create_view_update_plan(view.get().path, recompute_targets=recomputed_cols)
+                status = view.get().propagate_update(
+                    plan, None, recomputed_view_cols, base_versions=base_versions, timestamp=timestamp, cascade=True
                 )
                 result.num_rows += status.num_rows
                 result.num_excs += status.num_excs
@@ -1063,18 +1021,13 @@ class TableVersion:
                 raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')
             sql_where_clause = analysis_info.sql_where_clause
 
-        with Env.get().engine.begin() as conn:
-            num_rows = self.propagate_delete(sql_where_clause, base_versions=[], conn=conn, timestamp=time.time())
+        num_rows = self.propagate_delete(sql_where_clause, base_versions=[], timestamp=time.time())
 
         status = UpdateStatus(num_rows=num_rows)
         return status
 
     def propagate_delete(
-        self,
-        where: Optional[exprs.Expr],
-        base_versions: list[Optional[int]],
-        conn: sql.engine.Connection,
-        timestamp: float,
+        self, where: Optional[exprs.Expr], base_versions: list[Optional[int]], timestamp: float
     ) -> int:
         """Delete rows in this table and propagate to views.
         Args:
@@ -1084,17 +1037,17 @@ class TableVersion:
         """
         sql_where_clause = where.sql_expr(exprs.SqlElementCache()) if where is not None else None
         num_rows = self.store_tbl.delete_rows(
-            self.version + 1, base_versions=base_versions, match_on_vmin=False, where_clause=sql_where_clause, conn=conn
+            self.version + 1, base_versions=base_versions, match_on_vmin=False, where_clause=sql_where_clause
         )
         if num_rows > 0:
             # we're creating a new version
             self.version += 1
-            self._update_md(timestamp, conn)
+            self._update_md(timestamp)
         else:
             pass
         for view in self.mutable_views:
-            num_rows += view.propagate_delete(
-                where=None, base_versions=[self.version] + base_versions, conn=conn, timestamp=timestamp
+            num_rows += view.get().propagate_delete(
+                where=None, base_versions=[self.version] + base_versions, timestamp=timestamp
             )
         return num_rows
 
@@ -1103,22 +1056,20 @@ class TableVersion:
         assert not self.is_snapshot
         if self.version == 0:
             raise excs.Error('Cannot revert version 0')
-        with orm.Session(Env.get().engine, future=True) as session:
-            self._revert(session)
-            session.commit()
+        self._revert()
 
-    def _delete_column(self, col: Column, conn: sql.engine.Connection) -> None:
+    def _delete_column(self, col: Column) -> None:
         """Physically remove the column from the schema and the store table"""
         if col.is_stored:
-            self.store_tbl.drop_column(col, conn)
+            self.store_tbl.drop_column(col)
         self.cols.remove(col)
         if col.name is not None:
             del self.cols_by_name[col.name]
         del self.cols_by_id[col.id]
 
-    def _revert(self, session: orm.Session) -> None:
+    def _revert(self) -> None:
         """Reverts this table version and propagates to views"""
-        conn = session.connection()
+        conn = Env.get().conn
         # make sure we don't have a snapshot referencing this version
         # (unclear how to express this with sqlalchemy)
         query = (
@@ -1137,7 +1088,6 @@ class TableVersion:
                 )
             )
 
-        conn = session.connection()
         # delete newly-added data
         MediaStore.delete(self.id, version=self.version)
         conn.execute(sql.delete(self.store_tbl.sa_tbl).where(self.store_tbl.sa_tbl.c.v_min == self.version))
@@ -1158,7 +1108,7 @@ class TableVersion:
             if len(added_cols) > 0:
                 next_col_id = min(col.id for col in added_cols)
                 for col in added_cols:
-                    self._delete_column(col, conn)
+                    self._delete_column(col)
                 self.next_col_id = next_col_id
 
             # remove newly-added indices from the lookup structures
@@ -1181,6 +1131,7 @@ class TableVersion:
             for md in dropped_idx_md:
                 md.schema_version_drop = None
 
+            session = Env.get().session
             # we need to determine the preceding schema version and reload the schema
             schema_version_md_dict = (
                 session.query(schema.TableSchemaVersion.md)
@@ -1224,7 +1175,7 @@ class TableVersion:
 
         # propagate to views
         for view in self.mutable_views:
-            view._revert(session)
+            view.get()._revert()
         _logger.info(f'TableVersion {self.name}: reverted to version {self.version}')
 
     def _init_external_stores(self, tbl_md: schema.TableMd) -> None:
@@ -1235,40 +1186,48 @@ class TableVersion:
             self.external_stores[store.name] = store
 
     def link_external_store(self, store: pxt.io.ExternalStore) -> None:
-        with Env.get().engine.begin() as conn:
-            store.link(self, conn)  # May result in additional metadata changes
-            self.external_stores[store.name] = store
-            self._update_md(time.time(), conn, update_tbl_version=False)
+        store.link(self)  # May result in additional metadata changes
+        self.external_stores[store.name] = store
+        self._update_md(time.time(), update_tbl_version=False)
 
     def unlink_external_store(self, store_name: str, delete_external_data: bool) -> None:
         assert store_name in self.external_stores
         store = self.external_stores[store_name]
-        with Env.get().engine.begin() as conn:
-            store.unlink(self, conn)  # May result in additional metadata changes
-            del self.external_stores[store_name]
-            self._update_md(time.time(), conn, update_tbl_version=False)
+        store.unlink(self)  # May result in additional metadata changes
+        del self.external_stores[store_name]
+        self._update_md(time.time(), update_tbl_version=False)
 
         if delete_external_data and isinstance(store, pxt.io.external_store.Project):
             store.delete()
 
+    @property
+    def is_snapshot(self) -> bool:
+        return self.effective_version is not None
+
+    @property
     def is_view(self) -> bool:
-        return self.base is not None
+        return self.view_md is not None
+
+    @property
+    def include_base_columns(self) -> bool:
+        return self.view_md is not None and self.view_md.include_base_columns
 
+    @property
     def is_component_view(self) -> bool:
         return self.iterator_cls is not None
 
     def is_insertable(self) -> bool:
         """Returns True if this corresponds to an InsertableTable"""
-        return not self.is_snapshot and not self.is_view()
+        return not self.is_snapshot and not self.is_view
 
     def is_iterator_column(self, col: Column) -> bool:
         """Returns True if col is produced by an iterator"""
         # the iterator columns directly follow the pos column
-        return self.is_component_view() and col.id > 0 and col.id < self.num_iterator_cols + 1
+        return self.is_component_view and col.id > 0 and col.id < self.num_iterator_cols + 1
 
     def is_system_column(self, col: Column) -> bool:
         """Return True if column was created by Pixeltable"""
-        if col.name == _POS_COLUMN_NAME and self.is_component_view():
+        if col.name == _POS_COLUMN_NAME and self.is_component_view:
             return True
         return False
 
@@ -1282,7 +1241,7 @@ class TableVersion:
 
     def get_required_col_names(self) -> list[str]:
         """Return the names of all columns for which values must be specified in insert()"""
-        assert not self.is_view()
+        assert not self.is_view
         names = [c.name for c in self.cols_by_name.values() if not c.is_computed and not c.col_type.nullable]
         return names
 
@@ -1318,8 +1277,8 @@ class TableVersion:
 
     def num_rowid_columns(self) -> int:
         """Return the number of columns of the rowids, without accessing store_tbl"""
-        if self.is_component_view():
-            return 1 + self.base.num_rowid_columns()
+        if self.is_component_view:
+            return 1 + self.base.get().num_rowid_columns()
         return 1
 
     @classmethod
@@ -1393,4 +1352,4 @@ class TableVersion:
 
         id = UUID(d['id'])
         effective_version = d['effective_version']
-        return catalog.Catalog.get().tbl_versions[(id, effective_version)]
+        return catalog.Catalog.get().get_tbl_version(id, effective_version)