pixeltable 0.3.14__py3-none-any.whl → 0.5.7__py3-none-any.whl

pixeltable/catalog/catalog.py

@@ -3,39 +3,48 @@ from __future__ import annotations
 import dataclasses
 import functools
 import logging
+import random
 import time
-from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar
+from collections import defaultdict
+from contextlib import contextmanager
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Callable, Iterator, TypeVar
 from uuid import UUID
 
 import psycopg
 import sqlalchemy as sql
+import sqlalchemy.exc as sql_exc
 
 from pixeltable import exceptions as excs
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 from pixeltable.metadata import schema
+from pixeltable.utils.exception_handler import run_cleanup
 
+from .column import Column
 from .dir import Dir
-from .globals import IfExistsParam, IfNotExistsParam, MediaValidation
+from .globals import IfExistsParam, IfNotExistsParam, MediaValidation, QColumnId
 from .insertable_table import InsertableTable
 from .path import Path
 from .schema_object import SchemaObject
 from .table import Table
-from .table_version import TableVersion
+from .table_version import TableVersion, TableVersionKey, TableVersionMd
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
+from .tbl_ops import TableOp
+from .update_status import UpdateStatus
 from .view import View
 
 if TYPE_CHECKING:
-    from .. import DataFrame, exprs
+    from pixeltable.plan import SampleClause
+
+    from .. import exprs
 
 
 _logger = logging.getLogger('pixeltable')
 
 
-def _unpack_row(
-    row: Optional[sql.engine.Row], entities: list[type[sql.orm.decl_api.DeclarativeBase]]
-) -> Optional[list[Any]]:
+def _unpack_row(row: sql.engine.Row | None, entities: list[type[sql.orm.decl_api.DeclarativeBase]]) -> list[Any] | None:
     """Convert a Row result into a list of entity instances.
 
     Assumes that the query contains a select() of exactly those entities.
@@ -56,49 +65,137 @@ def _unpack_row(
     return result
 
 
-_MAX_RETRIES = 3
+def md_dict_factory(data: list[tuple[str, Any]]) -> dict:
+    """Use this to serialize TableMd instances with asdict()"""
+    # serialize enums to their values
+    return {k: v.value if isinstance(v, Enum) else v for k, v in data}
+
+
+# -1: unlimited
+# for now, we don't limit the number of retries, because we haven't seen situations where the actual number of retries
+# grows uncontrollably
+_MAX_RETRIES = -1
+
 T = TypeVar('T')
 
 
-def _retry_loop(op: Callable[..., T]) -> Callable[..., T]:
-    @functools.wraps(op)
-    def loop(*args: Any, **kwargs: Any) -> T:
-        num_remaining_retries = _MAX_RETRIES
-        while True:
-            try:
-                # in order for retry to work, we need to make sure that there aren't any prior db updates
-                # that are part of an ongoing transaction
-                assert not Env.get().in_xact()
-                with Env.get().begin_xact():
-                    return op(*args, **kwargs)
-            except sql.exc.DBAPIError as e:
-                if isinstance(e.orig, psycopg.errors.SerializationFailure):
-                    if num_remaining_retries > 0:
-                        num_remaining_retries -= 1
-                        # print(f'serialization failure:\n{e}')
-                        # print('retrying ************************************************************')
-                        time.sleep(1)
+def retry_loop(
+    *, tbl: TableVersionPath | None = None, for_write: bool, lock_mutable_tree: bool = False
+) -> Callable[[Callable[..., T]], Callable[..., T]]:
+    def decorator(op: Callable[..., T]) -> Callable[..., T]:
+        @functools.wraps(op)
+        def loop(*args: Any, **kwargs: Any) -> T:
+            cat = Catalog.get()
+            # retry_loop() is reentrant
+            if cat._in_retry_loop:
+                return op(*args, **kwargs)
+
+            num_retries = 0
+            while True:
+                cat._in_retry_loop = True
+                try:
+                    # in order for retry to work, we need to make sure that there aren't any prior db updates
+                    # that are part of an ongoing transaction
+                    assert not Env.get().in_xact
+                    with Catalog.get().begin_xact(
+                        tbl=tbl,
+                        for_write=for_write,
+                        convert_db_excs=False,
+                        lock_mutable_tree=lock_mutable_tree,
+                        finalize_pending_ops=True,
+                    ):
+                        return op(*args, **kwargs)
+                except PendingTableOpsError as e:
+                    Env.get().console_logger.debug(f'retry_loop(): finalizing pending ops for {e.tbl_id}')
+                    Catalog.get()._finalize_pending_ops(e.tbl_id)
+                except (sql_exc.DBAPIError, sql_exc.OperationalError) as e:
+                    # TODO: what other exceptions should we be looking for?
+                    if isinstance(
+                        # TODO: Investigate whether DeadlockDetected points to a bug in our locking protocol,
+                        #     which is supposed to be deadlock-free.
+                        e.orig,
+                        (
+                            psycopg.errors.SerializationFailure,
+                            psycopg.errors.LockNotAvailable,
+                            psycopg.errors.DeadlockDetected,
+                        ),
+                    ):
+                        if num_retries < _MAX_RETRIES or _MAX_RETRIES == -1:
+                            num_retries += 1
+                            _logger.debug(f'Retrying ({num_retries}) after {type(e.orig)}')
+                            time.sleep(random.uniform(0.1, 0.5))
+                        else:
+                            raise excs.Error(f'Serialization retry limit ({_MAX_RETRIES}) exceeded') from e
                     else:
-                        raise excs.Error(f'Serialization retry limit ({_MAX_RETRIES}) exceeded') from e
-                else:
+                        raise
+                except Exception as e:
+                    # for informational/debugging purposes
+                    _logger.debug(f'retry_loop(): passing along {e}')
                     raise
+                finally:
+                    cat._in_retry_loop = False
+
+        return loop
+
+    return decorator
+
 
-    return loop
+class PendingTableOpsError(Exception):
+    tbl_id: UUID
+
+    def __init__(self, tbl_id: UUID) -> None:
+        self.tbl_id = tbl_id
 
 
 class Catalog:
     """The functional interface to getting access to catalog objects
 
-    All interface functions must be called in the context of a transaction, started with Env.begin().
+    All interface functions must be called in the context of a transaction, started with Catalog.begin_xact() or
+    via retry_loop().
+
+    When calling functions that involve Table or TableVersion instances, the catalog needs to get a chance to finalize
+    pending ops against those tables. To that end,
+    - use begin_xact(tbl) or begin_xact(tbl_id) if only accessing a single table
+    - use retry_loop() when accessing multiple tables (eg, pxt.ls())
+
+    Caching and invalidation of metadata:
+    - Catalog caches TableVersion instances in order to avoid excessive metadata loading
+    - for any specific table version (ie, combination of id and effective version) there can be only a single
+      Tableversion instance in circulation; the reason is that each TV instance has its own store_tbl.sa_tbl, and
+      mixing multiple instances of sqlalchemy Table objects in the same query (for the same underlying table) leads to
+      duplicate references to that table in the From clause (ie, incorrect Cartesian products)
+    - in order to allow multiple concurrent Python processes to perform updates (data and/or schema) against a shared
+      Pixeltable instance, Catalog needs to reload metadata from the store when there are changes
+    - concurrent changes are detected by comparing TableVersion.version/view_sn with the stored current version
+      (TableMd.current_version/view_sn)
+    - cached live TableVersion instances (those with effective_version == None) are validated against the stored
+      metadata on transaction boundaries; this is recorded in TableVersion.is_validated
+    - metadata validation is only needed for live TableVersion instances (snapshot instances are immutable)
     """
 
-    _instance: Optional[Catalog] = None
+    _instance: Catalog | None = None
 
-    # key: [id, version]
+    # cached TableVersion instances; key: [id, version, anchor_tbl_id]
     # - mutable version of a table: version == None (even though TableVersion.version is set correctly)
     # - snapshot versions: records the version of the snapshot
-    _tbl_versions: dict[tuple[UUID, Optional[int]], TableVersion]
-    _tbls: dict[UUID, Table]
+    # - anchored versions: records the tbl_id of the anchor table (used when the table is a replica)
+    _tbl_versions: dict[TableVersionKey, TableVersion]
+    _tbls: dict[tuple[UUID, int | None], Table]
+    _in_write_xact: bool  # True if we're in a write transaction
+    _x_locked_tbl_ids: set[UUID]  # non-empty for write transactions
+    _modified_tvs: set[TableVersionHandle]  # TableVersion instances modified in the current transaction
+    _roll_forward_ids: set[UUID]  # ids of Tables that have pending TableOps
+    _undo_actions: list[Callable[[], None]]
+    _in_retry_loop: bool
+
+    # cached column dependencies
+    # - key: table id, value: mapping from column id to its dependencies
+    # - only maintained for dependencies between non-snapshot table versions
+    # - can contain stale entries (stemming from invalidated TV instances)
+    _column_dependencies: dict[UUID, dict[QColumnId, set[QColumnId]]]
+
+    # column dependents are recomputed at the beginning of every write transaction and only reflect the locked tree
+    _column_dependents: dict[QColumnId, set[QColumnId]] | None
 
     @classmethod
     def get(cls) -> Catalog:
@@ -109,22 +206,601 @@ class Catalog:
     @classmethod
     def clear(cls) -> None:
         """Remove the instance. Used for testing."""
+        if cls._instance is not None:
+            # invalidate all existing instances to force reloading of metadata
+            for tbl_version in cls._instance._tbl_versions.values():
+                tbl_version.is_validated = False
         cls._instance = None
 
     def __init__(self) -> None:
         self._tbl_versions = {}
         self._tbls = {}  # don't use a defaultdict here, it doesn't cooperate with the debugger
+        self._in_write_xact = False
+        self._x_locked_tbl_ids = set()
+        self._modified_tvs = set()
+        self._roll_forward_ids = set()
+        self._undo_actions = []
+        self._in_retry_loop = False
+        self._column_dependencies = {}
+        self._column_dependents = None
         self._init_store()
 
-    @classmethod
-    def _lock_dir(cls, parent_id: Optional[UUID], dir_id: Optional[UUID], dir_name: Optional[str]) -> None:
-        """Update directory record(s) to sequentialize thread access. Lock is released when transaction commits.
+    def _active_tbl_clause(
+        self, *, tbl_id: UUID | None = None, dir_id: UUID | None = None, tbl_name: str | None = None
+    ) -> sql.ColumnElement[bool]:
+        """Create a clause that filters out dropped tables in addition to the specified conditions."""
+        # avoid tables that are in the process of getting dropped
+        clause = sql.func.coalesce(schema.Table.md['pending_stmt'].astext, '-1') != str(
+            schema.TableStatement.DROP_TABLE.value
+        )
+        if tbl_id is not None:
+            clause = sql.and_(schema.Table.id == tbl_id, clause)
+        if dir_id is not None:
+            clause = sql.and_(schema.Table.dir_id == dir_id, clause)
+        if tbl_name is not None:
+            clause = sql.and_(schema.Table.md['name'].astext == tbl_name, clause)
+        return clause
+
+    def _dropped_tbl_error_msg(self, tbl_id: UUID) -> str:
+        return f'Table was dropped (no record found for {tbl_id})'
+
+    def validate(self) -> None:
+        """Validate structural consistency of cached metadata"""
+        for (tbl_id, effective_version, anchor_tbl_id), tbl_version in self._tbl_versions.items():
+            assert tbl_id == tbl_version.id, f'{tbl_id} != {tbl_version.id}'
+            assert effective_version is None or anchor_tbl_id is None
+            assert tbl_version.effective_version == tbl_version.version or tbl_version.effective_version is None, (
+                f'{tbl_version.effective_version} != {tbl_version.version} for id {tbl_id}'
+            )
+            assert effective_version == tbl_version.effective_version, (
+                f'{effective_version} != {tbl_version.effective_version} for id {tbl_id}'
+            )
+            assert len(tbl_version.mutable_views) == 0 or tbl_version.is_mutable, (
+                f'snapshot_id={tbl_version.id} mutable_views={tbl_version.mutable_views}'
+            )
+
+            assert anchor_tbl_id is None or tbl_version.is_replica
+
+            if tbl_version.is_view and tbl_version.is_mutable and tbl_version.is_validated:
+                # make sure this mutable view is recorded in a mutable base
+                base = tbl_version.base
+                assert base is not None
+                if base.effective_version is None:
+                    key = TableVersionKey(base.id, None, None)
+                    assert key in self._tbl_versions
+                    base_tv = self._tbl_versions[key]
+                    if not base_tv.is_validated:
+                        continue
+                    mutable_view_ids = ', '.join(str(tv.id) for tv in self._tbl_versions[key].mutable_views)
+                    mutable_view_names = ', '.join(
+                        tv._tbl_version.name
+                        for tv in self._tbl_versions[key].mutable_views
+                        if tv._tbl_version is not None
+                    )
+                    assert tbl_version.handle in self._tbl_versions[key].mutable_views, (
+                        f'{tbl_version.name} ({tbl_version.id}) missing in {mutable_view_ids} ({mutable_view_names})'
+                    )
+
+            if len(tbl_version.mutable_views) > 0:
+                # make sure we also loaded mutable view metadata, which is needed to detect column dependencies
+                for v in tbl_version.mutable_views:
+                    assert v.effective_version is None, f'{v.id}:{v.effective_version}'
+
+    def mark_modified_tvs(self, *handle: TableVersionHandle) -> None:
+        """Record that the given TableVersion instances were modified in the current transaction"""
+        assert Env.get().in_xact
+        self._modified_tvs.update(handle)
+
+    @contextmanager
+    def begin_xact(
+        self,
+        *,
+        tbl: TableVersionPath | None = None,
+        tbl_id: UUID | None = None,
+        for_write: bool = False,
+        lock_mutable_tree: bool = False,
+        convert_db_excs: bool = True,
+        finalize_pending_ops: bool = True,
+    ) -> Iterator[sql.Connection]:
+        """
+        Return a context manager that yields a connection to the database. Idempotent.
+
+        It is mandatory to call this method, not Env.begin_xact(), if the transaction accesses any table data
+        or metadata.
+
+        If tbl != None, follows this locking protocol:
+        - validates/reloads the TableVersion instances of tbl's ancestors (in the hope that this reduces potential
+          SerializationErrors later on)
+        - if for_write == True, x-locks Table record (by updating Table.lock_dummy; see _acquire_tbl_lock())
+        - if for_write == False, validates TableVersion instance
+        - if lock_mutable_tree == True, also x-locks all mutable views of the table
+        - this needs to be done in a retry loop, because Postgres can decide to abort the transaction
+          (SerializationFailure, LockNotAvailable)
+        - for that reason, we do all lock acquisition prior to doing any real work (eg, compute column values),
+          to minimize the probability of losing that work due to a forced abort
+
+        If convert_db_excs == True, converts DBAPIErrors into excs.Errors.
+        """
+        assert tbl is None or tbl_id is None  # at most one can be specified
+        if Env.get().in_xact:
+            # make sure that we requested the required table lock at the beginning of the transaction
+            if for_write:
+                if tbl is not None:
+                    assert tbl.tbl_id in self._x_locked_tbl_ids, f'{tbl.tbl_id} not in {self._x_locked_tbl_ids}'
+                elif tbl_id is not None:
+                    assert tbl_id in self._x_locked_tbl_ids, f'{tbl_id} not in {self._x_locked_tbl_ids}'
+            yield Env.get().conn
+            return
+
+        # tv_msg = '\n'.join(
+        #     [
+        #         f'{tv.id}:{tv.effective_version} : tv={id(tv):x} sa_tbl={id(tv.store_tbl.sa_tbl):x}'
+        #         for tv in self._tbl_versions.values()
+        #     ]
+        # )
+        # _logger.debug(f'begin_xact(): {tv_msg}')
+        num_retries = 0
+        pending_ops_tbl_id: UUID | None = None
+        has_exc = False  # True if we exited the 'with ...begin_xact()' block with an exception
+        while True:
+            if pending_ops_tbl_id is not None:
+                Env.get().console_logger.debug(f'begin_xact(): finalizing pending ops for {pending_ops_tbl_id}')
+                self._finalize_pending_ops(pending_ops_tbl_id)
+                pending_ops_tbl_id = None
+
+            try:
+                self._in_write_xact = for_write
+                self._x_locked_tbl_ids = set()
+                self._modified_tvs = set()
+                self._column_dependents = None
+                has_exc = False
+
+                assert not self._undo_actions
+                with Env.get().begin_xact(for_write=for_write) as conn:
+                    if tbl is not None or tbl_id is not None:
+                        try:
+                            target: TableVersionHandle | None = None
+                            if tbl is not None:
+                                if self._acquire_path_locks(
+                                    tbl=tbl,
+                                    for_write=for_write,
+                                    lock_mutable_tree=lock_mutable_tree,
+                                    check_pending_ops=finalize_pending_ops,
+                                ):
+                                    target = tbl.tbl_version
+                            else:
+                                target = self._acquire_tbl_lock(
+                                    tbl_id=tbl_id,
+                                    for_write=for_write,
+                                    lock_mutable_tree=lock_mutable_tree,
+                                    raise_if_not_exists=True,
+                                    check_pending_ops=finalize_pending_ops,
+                                )
+
+                            if target is None:
+                                # didn't get the write lock
+                                for_write = False
+                            elif for_write:
+                                # we know at this point that target is mutable because we got the X-lock
+                                if lock_mutable_tree and not target.is_snapshot:
+                                    self._x_locked_tbl_ids = self._get_mutable_tree(target.id)
+                                    self._compute_column_dependents(self._x_locked_tbl_ids)
+                                else:
+                                    self._x_locked_tbl_ids = {target.id}
+                                if _logger.isEnabledFor(logging.DEBUG):
+                                    # validate only when we don't see errors
+                                    self.validate()
+
+                        except PendingTableOpsError as e:
+                            has_exc = True
+                            if finalize_pending_ops:
+                                # we remember which table id to finalize
+                                pending_ops_tbl_id = e.tbl_id
+                            # raise to abort the transaction
+                            raise
+
+                        except (sql_exc.DBAPIError, sql_exc.OperationalError) as e:
+                            has_exc = True
+                            if isinstance(
+                                e.orig, (psycopg.errors.SerializationFailure, psycopg.errors.LockNotAvailable)
+                            ) and (num_retries < _MAX_RETRIES or _MAX_RETRIES == -1):
+                                num_retries += 1
+                                _logger.debug(f'Retrying ({num_retries}) after {type(e.orig)}')
+                                time.sleep(random.uniform(0.1, 0.5))
+                                assert not self._undo_actions  # We should not have any undo actions at this point
+                                continue
+                            else:
+                                raise
+
+                    assert not self._undo_actions
+                    yield conn
+                    return
+
+            except PendingTableOpsError:
+                has_exc = True
+                if pending_ops_tbl_id is not None:
+                    # the next iteration of the loop will deal with pending ops for this table id
+                    continue
+                else:
+                    # we got this exception after getting the initial table locks and therefore need to abort
+                    raise
+
+            except (sql_exc.DBAPIError, sql_exc.OperationalError, sql_exc.InternalError) as e:
+                has_exc = True
+                self.convert_sql_exc(e, tbl_id, tbl.tbl_version if tbl is not None else None, convert_db_excs)
+                raise  # re-raise the error if it didn't convert to a pxt.Error
+
+            except (Exception, KeyboardInterrupt) as e:
+                has_exc = True
+                _logger.debug(f'Caught {e.__class__}')
+                raise
+
+            finally:
+                self._in_write_xact = False
+                self._x_locked_tbl_ids.clear()
+                self._column_dependents = None
+
+                # invalidate cached current TableVersion instances
+                for tv in self._tbl_versions.values():
+                    if tv.effective_version is None:
+                        _logger.debug(f'invalidating table version {tv} (0x{id(tv):x})')
+                        tv.is_validated = False
+
+                if has_exc:
+                    # Execute undo actions in reverse order (LIFO)
+                    for hook in reversed(self._undo_actions):
+                        run_cleanup(hook, raise_error=False)
+                    # purge all modified TableVersion instances; we can't guarantee they are still consistent with the
+                    # stored metadata
+                    for handle in self._modified_tvs:
+                        self._clear_tv_cache(handle.key)
+                    # Clear potentially corrupted cached metadata
+                    if tbl is not None:
+                        tbl.clear_cached_md()
+
+                self._undo_actions.clear()
+                self._modified_tvs.clear()
+
+    def register_undo_action(self, func: Callable[[], None]) -> Callable[[], None]:
+        """Registers a function to be called if the current transaction fails.
+
+        The function is called only if the current transaction fails due to an exception.
+
+        Rollback functions are called in reverse order of registration (LIFO).
+
+        The function should not raise exceptions; if it does, they are logged and ignored.
+        """
+        assert self.in_write_xact
+        self._undo_actions.append(func)
+        return func
+
+    def convert_sql_exc(
+        self,
+        e: sql_exc.StatementError,
+        tbl_id: UUID | None = None,
+        tbl: TableVersionHandle | None = None,
+        convert_db_excs: bool = True,
+    ) -> None:
+        # we got some db error during the actual operation (not just while trying to get locks on the metadata
+        # records); we convert these into pxt.Error exceptions if appropriate
+
+        # we always convert UndefinedTable exceptions (they can't be retried)
+        if isinstance(e.orig, psycopg.errors.UndefinedTable) and tbl is not None:
+            # the table got dropped in the middle of the operation
+            tbl_name = tbl.get().name
+            _logger.debug(f'Exception: undefined table {tbl_name!r}: Caught {type(e.orig)}: {e!r}')
+            raise excs.Error(f'Table was dropped: {tbl_name}') from None
+        elif (
+            # TODO: Investigate whether DeadlockDetected points to a bug in our locking protocol,
+            #     which is supposed to be deadlock-free.
+            isinstance(
+                e.orig,
+                (
+                    psycopg.errors.SerializationFailure,  # serialization error despite getting x-locks
+                    psycopg.errors.InFailedSqlTransaction,  # can happen after tx fails for another reason
+                    psycopg.errors.DuplicateColumn,  # if a different process added a column concurrently
+                    psycopg.errors.DeadlockDetected,  # locking protocol contention
+                ),
+            )
+            and convert_db_excs
+        ):
+            msg: str
+            if tbl is not None:
+                msg = f'{tbl.get().name} ({tbl.id})'
+            elif tbl_id is not None:
+                msg = f'{tbl_id}'
+            else:
+                msg = ''
+            _logger.debug(f'Exception: {e.orig.__class__}: {msg} ({e})')
+            # Suppress the underlying SQL exception unless DEBUG is enabled
+            raise_from = e if _logger.isEnabledFor(logging.DEBUG) else None
+            raise excs.Error(
+                'That Pixeltable operation could not be completed because it conflicted with another '
+                'operation that was run on a different process.\n'
+                'Please re-run the operation.'
+            ) from raise_from
+
+    @property
+    def in_write_xact(self) -> bool:
+        return self._in_write_xact
+
+    def _acquire_path_locks(
+        self,
+        *,
+        tbl: TableVersionPath,
+        for_write: bool = False,
+        lock_mutable_tree: bool = False,
+        check_pending_ops: bool = True,
+    ) -> bool:
+        """
+        Path locking protocol:
+        - refresh cached TableVersions of ancestors (we need those even during inserts, for computed columns that
+          reference the base tables)
+        - refresh cached TableVersion of tbl or get X-lock, depending on for_write
+        - if lock_mutable_tree, also X-lock all mutable views of tbl
+
+        Raises Error if tbl doesn't exist.
+        Return False if the lock couldn't be acquired (X-lock on a non-mutable table), True otherwise.
+        """
+        path_handles = tbl.get_tbl_versions()
+        read_handles = path_handles[:0:-1] if for_write else path_handles[::-1]
+        for handle in read_handles:
+            # update cache
+            _ = self.get_tbl_version(handle.key, validate_initialized=True)
+        if not for_write:
+            return True  # nothing left to lock
+        handle = self._acquire_tbl_lock(
+            tbl_id=tbl.tbl_id,
+            for_write=True,
+            lock_mutable_tree=lock_mutable_tree,
+            raise_if_not_exists=True,
+            check_pending_ops=check_pending_ops,
+        )
+        # update cache
+        _ = self.get_tbl_version(path_handles[0].key, validate_initialized=True)
+        return handle is not None
+
+    def _acquire_tbl_lock(
+        self,
+        *,
+        for_write: bool,
+        tbl_id: UUID | None = None,
+        dir_id: UUID | None = None,
+        tbl_name: str | None = None,
+        lock_mutable_tree: bool = False,
+        raise_if_not_exists: bool = True,
+        check_pending_ops: bool = True,
+    ) -> TableVersionHandle | None:
+        """
+        For writes: force acquisition of an X-lock on a Table record via a blind update.
+
+        Either tbl_id or dir_id/tbl_name need to be specified.
+        Returns True if the table was locked, False if it was a snapshot or not found.
+        If lock_mutable_tree, recursively locks all mutable views of the table.
+
+        Returns a handle to what was locked, None if the lock couldn't be acquired (eg, X-lock on a non-mutable table).
+        """
+        assert (tbl_id is not None) != (dir_id is not None and tbl_name is not None)
+        assert (dir_id is None) == (tbl_name is None)
+        where_clause: sql.ColumnElement
+        if tbl_id is not None:
+            where_clause = schema.Table.id == tbl_id
+        else:
+            where_clause = sql.and_(schema.Table.dir_id == dir_id, schema.Table.md['name'].astext == tbl_name)
+            user = Env.get().user
+            if user is not None:
+                where_clause = sql.and_(where_clause, schema.Table.md['user'].astext == Env.get().user)
+
+        conn = Env.get().conn
+        q = sql.select(schema.Table).where(where_clause)
+        if for_write:
+            q = q.with_for_update(nowait=True)
+        row = conn.execute(q).one_or_none()
+        if row is None:
+            if raise_if_not_exists:
+                raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
+            return None  # nothing to lock
+        tbl_md = schema.md_from_dict(schema.TableMd, row.md)
+        if for_write and tbl_md.is_mutable:
+            conn.execute(sql.update(schema.Table).values(lock_dummy=1).where(where_clause))
+
+        if check_pending_ops:
+            # check for pending ops after getting table lock
+            pending_ops_q = sql.select(sql.func.count()).where(schema.PendingTableOp.tbl_id == row.id)
+            has_pending_ops = conn.execute(pending_ops_q).scalar() > 0
+            if has_pending_ops:
+                raise PendingTableOpsError(row.id)
+
+        # TODO: properly handle concurrency for replicas with live views (once they are supported)
+        if for_write and not tbl_md.is_mutable:
+            return None  # nothing to lock
+
+        key = TableVersionKey(tbl_id, tbl_md.current_version if tbl_md.is_snapshot else None, None)
+        if tbl_md.is_mutable and lock_mutable_tree:
+            # also lock mutable views
+            tv = self.get_tbl_version(key, validate_initialized=True)
+            for view in tv.mutable_views:
+                self._acquire_tbl_lock(
+                    for_write=for_write,
+                    tbl_id=view.id,
+                    lock_mutable_tree=lock_mutable_tree,
+                    raise_if_not_exists=raise_if_not_exists,
+                    check_pending_ops=check_pending_ops,
+                )
+        return TableVersionHandle(key)
+
+    def _roll_forward(self) -> None:
+        """Finalize pending ops for all tables in self._roll_forward_ids."""
+        for tbl_id in self._roll_forward_ids:
+            self._finalize_pending_ops(tbl_id)
+            # TODO: handle replicas
+            self._clear_tv_cache(TableVersionKey(tbl_id, None, None))
+
+    def _finalize_pending_ops(self, tbl_id: UUID) -> None:
+        """Finalizes all pending ops for the given table."""
+        num_retries = 0
+        while True:
+            try:
+                tbl_version: int
+                op: TableOp | None = None
+                delete_next_op_stmt: sql.Delete
+                reset_state_stmt: sql.Update
+                with self.begin_xact(
+                    tbl_id=tbl_id, for_write=True, convert_db_excs=False, finalize_pending_ops=False
+                ) as conn:
+                    q = (
+                        sql.select(schema.Table.md, schema.PendingTableOp)
+                        .select_from(schema.Table)
+                        .join(schema.PendingTableOp)
+                        .where(schema.Table.id == tbl_id)
+                        .where(schema.PendingTableOp.tbl_id == tbl_id)
+                        .order_by(schema.PendingTableOp.op_sn)
+                        .limit(1)
+                        .with_for_update()
+                    )
+                    row = conn.execute(q).one_or_none()
+                    if row is None:
+                        return
+                    view_md = row.md.get('view_md')
+                    is_snapshot = False if view_md is None else view_md.get('is_snapshot')
+                    assert is_snapshot is not None
+                    tbl_version = row.md.get('current_version') if is_snapshot else None
+                    op = schema.md_from_dict(TableOp, row.op)
+                    delete_next_op_stmt = sql.delete(schema.PendingTableOp).where(
+                        schema.PendingTableOp.tbl_id == tbl_id, schema.PendingTableOp.op_sn == row.op_sn
+                    )
+                    reset_state_stmt = (
+                        sql.update(schema.Table)
+                        .where(schema.Table.id == tbl_id)
+                        .values(
+                            md=schema.Table.md.op('||')(
+                                {'tbl_state': schema.TableState.LIVE.value, 'pending_stmt': None}
+                            )
+                        )
+                    )
+                    _logger.debug(f'finalize_pending_ops({tbl_id}): finalizing op {op!s}')
+
+                    if op.needs_xact:
+                        if op.delete_table_md_op is not None:
+                            self.delete_tbl_md(tbl_id)
+                        else:
+                            tv = self.get_tbl_version(
+                                TableVersionKey(tbl_id, tbl_version, None),
+                                check_pending_ops=False,
+                                validate_initialized=True,
+                            )
+                            # TODO: The above TableVersionKey instance will need to be updated if we see a replica here.
+                            # For now, just assert that we don't.
+                            assert not tv.is_replica
+                            tv.exec_op(op)
+
+                        conn.execute(delete_next_op_stmt)
+                        if op.op_sn == op.num_ops - 1:
+                            conn.execute(reset_state_stmt)
+                            return
+                        continue
+
+                # this op runs outside of a transaction
+                tv = self.get_tbl_version(
+                    TableVersionKey(tbl_id, tbl_version, None), check_pending_ops=False, validate_initialized=True
+                )
+                tv.exec_op(op)
+                with self.begin_xact(
+                    tbl_id=tbl_id, for_write=True, convert_db_excs=False, finalize_pending_ops=False
+                ) as conn:
+                    conn.execute(delete_next_op_stmt)
+                    if op.op_sn == op.num_ops - 1:
+                        conn.execute(reset_state_stmt)
+                        return
+
+            except (sql_exc.DBAPIError, sql_exc.OperationalError) as e:
+                # TODO: why are we still seeing these here, instead of them getting taken care of by the retry
+                # logic of begin_xact()?
+                if isinstance(e.orig, (psycopg.errors.SerializationFailure, psycopg.errors.LockNotAvailable)):
+                    num_retries += 1
+                    log_msg: str
+                    if op is not None:
+                        log_msg = f'finalize_pending_ops(): retrying ({num_retries}) op {op!s} after {type(e.orig)}'
+                    else:
+                        log_msg = f'finalize_pending_ops(): retrying ({num_retries}) after {type(e.orig)}'
+                    Env.get().console_logger.debug(log_msg)
+                    time.sleep(random.uniform(0.1, 0.5))
+                    continue
+                else:
+                    raise
+            except Exception as e:
+                Env.get().console_logger.debug(f'finalize_pending_ops(): caught {e}')
+                raise
+
+            num_retries = 0
+
+    def _debug_str(self) -> str:
+        tv_str = '\n'.join(str(k) for k in self._tbl_versions)
+        tbl_str = '\n'.join(str(k) for k in self._tbls)
+        return f'tbl_versions:\n{tv_str}\ntbls:\n{tbl_str}'
+
+    def _get_mutable_tree(self, tbl_id: UUID) -> set[UUID]:
+        """Returns ids of all tables that form the tree of mutable views starting at tbl_id; includes the root."""
+        key = TableVersionKey(tbl_id, None, None)
+        assert key in self._tbl_versions, f'{key} not in {self._tbl_versions.keys()}\n{self._debug_str()}'
+        tv = self.get_tbl_version(key, validate_initialized=True)
+        assert not tv.is_replica
+        result: set[UUID] = {tv.id}
+        for view in tv.mutable_views:
+            result.update(self._get_mutable_tree(view.id))
+        return result
+
+    def _compute_column_dependents(self, mutable_tree: set[UUID]) -> None:
+        """Populate self._column_dependents for all tables in mutable_tree"""
+        assert self._column_dependents is None
+        self._column_dependents = defaultdict(set)
+        for tbl_id in mutable_tree:
+            assert tbl_id in self._column_dependencies, (
+                f'{tbl_id} not in {self._column_dependencies.keys()}\n{self._debug_str()}'
+            )
+            for col, dependencies in self._column_dependencies[tbl_id].items():
+                for dependency in dependencies:
+                    if dependency.tbl_id not in mutable_tree:
+                        continue
+                    dependents = self._column_dependents[dependency]
+                    dependents.add(col)
+
+    def record_column_dependencies(self, tbl_version: TableVersion) -> None:
+        """Update self._column_dependencies. Only valid for mutable versions."""
+        from pixeltable.exprs import Expr
+
+        assert tbl_version.is_mutable
+        dependencies: dict[QColumnId, set[QColumnId]] = {}
+        for col in tbl_version.cols_by_id.values():
+            if col.value_expr_dict is None:
+                continue
+            dependencies[QColumnId(tbl_version.id, col.id)] = Expr.get_refd_column_ids(col.value_expr_dict)
+        self._column_dependencies[tbl_version.id] = dependencies
+
+    def get_column_dependents(self, tbl_id: UUID, col_id: int) -> set[Column]:
+        """Return all Columns that transitively depend on the given column."""
+        assert self._column_dependents is not None
+        dependents = self._column_dependents[QColumnId(tbl_id, col_id)]
+        result: set[Column] = set()
+        for dependent in dependents:
+            tv = self.get_tbl_version(TableVersionKey(dependent.tbl_id, None, None), validate_initialized=True)
+            col = tv.cols_by_id[dependent.col_id]
+            result.add(col)
+        return result
+
+    def _acquire_dir_xlock(
+        self, *, parent_id: UUID | None = None, dir_id: UUID | None = None, dir_name: str | None = None
+    ) -> None:
+        """Force acquisition of an X-lock on a Dir record via a blind update.
+
         If dir_id is present, then all other conditions are ignored.
         Note that (parent_id==None) is a valid where condition.
         If dir_id is not specified, the user from the environment is added to the directory filters.
         """
+        assert (dir_name is None) != (dir_id is None)
+        assert not (parent_id is not None and dir_name is None)
         user = Env.get().user
-        conn = Env.get().conn
+        assert self._in_write_xact
         q = sql.update(schema.Dir).values(lock_dummy=1)
         if dir_id is not None:
             q = q.where(schema.Dir.id == dir_id)
@@ -134,10 +810,11 @@ class Catalog:
                 q = q.where(schema.Dir.md['name'].astext == dir_name)
             if user is not None:
                 q = q.where(schema.Dir.md['user'].astext == user)
-        conn.execute(q)
+        Env.get().conn.execute(q)
 
     def get_dir_path(self, dir_id: UUID) -> Path:
         """Return path for directory with given id"""
+        assert isinstance(dir_id, UUID)
         conn = Env.get().conn
         names: list[str] = []
         while True:
@@ -148,15 +825,15 @@ class Catalog:
                 break
             names.insert(0, dir.md['name'])
             dir_id = dir.parent_id
-        return Path('.'.join(names), empty_is_valid=True, allow_system_paths=True)
+        return Path.parse('.'.join(names), allow_empty_path=True, allow_system_path=True)
 
     @dataclasses.dataclass
     class DirEntry:
-        dir: Optional[schema.Dir]
+        dir: schema.Dir | None
         dir_entries: dict[str, Catalog.DirEntry]
-        table: Optional[schema.Table]
+        table: schema.Table | None
 
-    @_retry_loop
+    @retry_loop(for_write=False)
     def get_dir_contents(self, dir_path: Path, recursive: bool = False) -> dict[str, DirEntry]:
         dir = self._get_schema_object(dir_path, expected=Dir, raise_if_not_exists=True)
         return self._get_dir_contents(dir._id, recursive=recursive)
@@ -175,7 +852,7 @@ class Catalog:
                 dir_contents = self._get_dir_contents(dir.id, recursive=True)
             result[dir.md['name']] = self.DirEntry(dir=dir, dir_entries=dir_contents, table=None)
 
-        q = sql.select(schema.Table).where(schema.Table.dir_id == dir_id)
+        q = sql.select(schema.Table).where(self._active_tbl_clause(dir_id=dir_id))
         rows = conn.execute(q).all()
         for row in rows:
             tbl = schema.Table(**row._mapping)
@@ -183,31 +860,37 @@ class Catalog:
 
         return result
 
-    @_retry_loop
-    def move(self, path: Path, new_path: Path) -> None:
-        self._move(path, new_path)
+    @retry_loop(for_write=True)
+    def move(self, path: Path, new_path: Path, if_exists: IfExistsParam, if_not_exists: IfNotExistsParam) -> None:
+        self._move(path, new_path, if_exists, if_not_exists)
 
-    def _move(self, path: Path, new_path: Path) -> None:
-        _, dest_dir, src_obj = self._prepare_dir_op(
+    def _move(self, path: Path, new_path: Path, if_exists: IfExistsParam, if_not_exists: IfNotExistsParam) -> None:
+        dest_obj, dest_dir, src_obj = self._prepare_dir_op(
             add_dir_path=new_path.parent,
             add_name=new_path.name,
             drop_dir_path=path.parent,
             drop_name=path.name,
-            raise_if_exists=True,
-            raise_if_not_exists=True,
+            raise_if_exists=(if_exists == IfExistsParam.ERROR),
+            raise_if_not_exists=(if_not_exists == IfNotExistsParam.ERROR),
         )
-        src_obj._move(new_path.name, dest_dir._id)
+        assert dest_obj is None or if_exists == IfExistsParam.IGNORE
+        assert src_obj is not None or if_not_exists == IfNotExistsParam.IGNORE
+        if dest_obj is None and src_obj is not None:
+            # If dest_obj is not None, it means `if_exists='ignore'` and the destination already exists.
+            # If src_obj is None, it means `if_not_exists='ignore'` and the source doesn't exist.
+            # If dest_obj is None and src_obj is not None, then we can proceed with the move.
+            src_obj._move(new_path.name, dest_dir._id)
 
     def _prepare_dir_op(
         self,
-        add_dir_path: Optional[Path] = None,
-        add_name: Optional[str] = None,
-        drop_dir_path: Optional[Path] = None,
-        drop_name: Optional[str] = None,
-        drop_expected: Optional[type[SchemaObject]] = None,
+        add_dir_path: Path | None = None,
+        add_name: str | None = None,
+        drop_dir_path: Path | None = None,
+        drop_name: str | None = None,
+        drop_expected: type[SchemaObject] | None = None,
         raise_if_exists: bool = False,
         raise_if_not_exists: bool = False,
-    ) -> tuple[Optional[SchemaObject], Optional[SchemaObject], Optional[SchemaObject]]:
+    ) -> tuple[SchemaObject | None, Dir | None, SchemaObject | None]:
         """
         Validates paths and acquires locks needed for a directory operation, ie, add/drop/rename (add + drop) of a
         directory entry.
@@ -225,6 +908,7 @@ class Catalog:
         - if both add and drop (= two directories are involved), lock the directories in a pre-determined order
           (in this case, by name) in order to prevent deadlocks between concurrent directory modifications
         """
+        assert drop_expected in (None, Table, Dir), drop_expected
         assert (add_dir_path is None) == (add_name is None)
         assert (drop_dir_path is None) == (drop_name is None)
         dir_paths: set[Path] = set()
@@ -233,46 +917,50 @@ class Catalog:
         if drop_dir_path is not None:
             dir_paths.add(drop_dir_path)
 
-        add_dir: Optional[schema.Dir] = None
-        drop_dir: Optional[schema.Dir] = None
+        add_dir: schema.Dir | None = None
+        drop_dir: schema.Dir | None = None
         for p in sorted(dir_paths):
-            dir = self._get_dir(p, for_update=True)
+            dir = self._get_dir(p, lock_dir=True)
             if dir is None:
-                raise excs.Error(f'Directory {str(p)!r} does not exist.')
+                # Dir does not exist; raise an appropriate error.
+                if add_dir_path is not None or add_name is not None:
+                    raise excs.Error(f'Directory {p!r} does not exist. Create it first with:\npxt.create_dir({p!r})')
+                else:
+                    raise excs.Error(f'Directory {p!r} does not exist.')
             if p == add_dir_path:
                 add_dir = dir
             if p == drop_dir_path:
                 drop_dir = dir
 
-        add_obj: Optional[SchemaObject] = None
+        add_obj: SchemaObject | None = None
         if add_dir is not None:
-            add_obj = self._get_dir_entry(add_dir.id, add_name, for_update=True)
+            add_obj = self._get_dir_entry(add_dir.id, add_name, lock_entry=True)
             if add_obj is not None and raise_if_exists:
                 add_path = add_dir_path.append(add_name)
-                raise excs.Error(f'Path {str(add_path)!r} already exists.')
+                raise excs.Error(f'Path {add_path!r} already exists.')
 
-        drop_obj: Optional[SchemaObject] = None
+        drop_obj: SchemaObject | None = None
         if drop_dir is not None:
             drop_path = drop_dir_path.append(drop_name)
-            drop_obj = self._get_dir_entry(drop_dir.id, drop_name, for_update=True)
+            drop_obj = self._get_dir_entry(drop_dir.id, drop_name, lock_entry=True)
             if drop_obj is None and raise_if_not_exists:
-                raise excs.Error(f'Path {str(drop_path)!r} does not exist.')
+                raise excs.Error(f'Path {drop_path!r} does not exist.')
             if drop_obj is not None and drop_expected is not None and not isinstance(drop_obj, drop_expected):
-                raise excs.Error(
-                    f'{str(drop_path)!r} needs to be a {drop_expected._display_name()} '
-                    f'but is a {type(drop_obj)._display_name()}'
-                )
+                expected_name = 'table' if drop_expected is Table else 'directory'
+                raise excs.Error(f'{drop_path!r} needs to be a {expected_name} but is a {drop_obj._display_name()}')
 
         add_dir_obj = Dir(add_dir.id, add_dir.parent_id, add_dir.md['name']) if add_dir is not None else None
         return add_obj, add_dir_obj, drop_obj
 
-    def _get_dir_entry(self, dir_id: UUID, name: str, for_update: bool = False) -> Optional[SchemaObject]:
+    def _get_dir_entry(
+        self, dir_id: UUID, name: str, version: int | None = None, lock_entry: bool = False
+    ) -> SchemaObject | None:
         user = Env.get().user
         conn = Env.get().conn
 
         # check for subdirectory
-        if for_update:
-            self._lock_dir(dir_id, None, name)
+        if lock_entry:
+            self._acquire_dir_xlock(parent_id=dir_id, dir_id=None, dir_name=name)
         q = sql.select(schema.Dir).where(
             schema.Dir.parent_id == dir_id, schema.Dir.md['name'].astext == name, schema.Dir.md['user'].astext == user
         )
@@ -286,293 +974,385 @@ class Catalog:
             return Dir(dir_record.id, dir_record.parent_id, name)
 
         # check for table
+        if lock_entry:
+            self._acquire_tbl_lock(for_write=True, dir_id=dir_id, raise_if_not_exists=False, tbl_name=name)
         q = sql.select(schema.Table.id).where(
-            schema.Table.dir_id == dir_id,
-            schema.Table.md['name'].astext == name,
-            schema.Table.md['user'].astext == user,
+            self._active_tbl_clause(dir_id=dir_id, tbl_name=name), schema.Table.md['user'].astext == user
         )
-        if for_update:
-            q = q.with_for_update()
-        tbl_id = conn.execute(q).scalar_one_or_none()
-        if tbl_id is not None:
-            if tbl_id not in self._tbls:
-                self._tbls[tbl_id] = self._load_tbl(tbl_id)
-            return self._tbls[tbl_id]
+        tbl_id = conn.execute(q).scalars().all()
+        assert len(tbl_id) <= 1, name
+        if len(tbl_id) == 1:
+            return self.get_table_by_id(tbl_id[0], version)
 
         return None
 
     def _get_schema_object(
         self,
         path: Path,
-        expected: Optional[type[SchemaObject]] = None,
+        expected: type[SchemaObject] | None = None,
         raise_if_exists: bool = False,
         raise_if_not_exists: bool = False,
-        for_update: bool = False,
-    ) -> Optional[SchemaObject]:
+        lock_parent: bool = False,
+        lock_obj: bool = False,
+    ) -> SchemaObject | None:
         """Return the schema object at the given path, or None if it doesn't exist.
 
         Raises Error if
-        - the parent directory doesn't exist'
+        - the parent directory doesn't exist
         - raise_if_exists is True and the path exists
         - raise_if_not_exists is True and the path does not exist
         - expected is not None and the existing object has a different type
         """
+        assert expected in (None, Table, Dir), expected
+
         if path.is_root:
             # the root dir
             if expected is not None and expected is not Dir:
-                raise excs.Error(
-                    f'{str(path)!r} needs to be a {expected._display_name()} but is a {Dir._display_name()}'
-                )
-            dir = self._get_dir(path, for_update=for_update)
+                raise excs.Error(f'{path!r} needs to be a table but is a dir')
+            dir = self._get_dir(path, lock_dir=lock_obj)
             if dir is None:
                 raise excs.Error(f'Unknown user: {Env.get().user}')
             return Dir(dir.id, dir.parent_id, dir.md['name'])
 
         parent_path = path.parent
-        parent_dir = self._get_dir(parent_path, for_update=False)
+        parent_dir = self._get_dir(parent_path, lock_dir=lock_parent)
         if parent_dir is None:
-            raise excs.Error(f'Directory {str(parent_path)!r} does not exist.')
-        obj = self._get_dir_entry(parent_dir.id, path.name, for_update=for_update)
+            raise excs.Error(f'Directory {parent_path!r} does not exist.')
+        obj = self._get_dir_entry(parent_dir.id, path.name, path.version, lock_entry=lock_obj)
 
         if obj is None and raise_if_not_exists:
-            raise excs.Error(f'Path {str(path)!r} does not exist.')
+            raise excs.Error(f'Path {path!r} does not exist.')
         elif obj is not None and raise_if_exists:
-            raise excs.Error(f'Path {str(path)!r} is an existing {type(obj)._display_name()}.')
+            raise excs.Error(f'Path {path!r} is an existing {obj._display_name()}.')
         elif obj is not None and expected is not None and not isinstance(obj, expected):
-            raise excs.Error(
-                f'{str(path)!r} needs to be a {expected._display_name()} but is a {type(obj)._display_name()}.'
-            )
+            expected_name = 'table' if expected is Table else 'directory'
+            raise excs.Error(f'{path!r} needs to be a {expected_name} but is a {obj._display_name()}.')
         return obj
 
-    def get_table_by_id(self, tbl_id: UUID) -> Optional[Table]:
-        if tbl_id not in self._tbls:
-            tbl = self._load_tbl(tbl_id)
-            if tbl is None:
-                return None
-            self._tbls[tbl_id] = tbl
-        return self._tbls[tbl_id]
+    def get_table_by_id(
+        self, tbl_id: UUID, version: int | None = None, ignore_if_dropped: bool = False
+    ) -> Table | None:
+        """Must be executed inside a transaction. Might raise PendingTableOpsError."""
+        if (tbl_id, version) not in self._tbls:
+            if version is None:
+                return self._load_tbl(tbl_id, ignore_pending_drop=ignore_if_dropped)
+            else:
+                return self._load_tbl_at_version(tbl_id, version)
+        return self._tbls.get((tbl_id, version))
 
-    @_retry_loop
     def create_table(
         self,
         path: Path,
         schema: dict[str, Any],
-        df: 'DataFrame',
         if_exists: IfExistsParam,
-        primary_key: Optional[list[str]],
+        primary_key: list[str] | None,
         num_retained_versions: int,
         comment: str,
         media_validation: MediaValidation,
-    ) -> Table:
-        existing = self._handle_path_collision(path, InsertableTable, False, if_exists)
-        if existing is not None:
-            assert isinstance(existing, Table)
-            return existing
+        create_default_idxs: bool,
+    ) -> tuple[Table, bool]:
+        """
+        Creates a new InsertableTable at the given path.
 
-        dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
-        assert dir is not None
+        If `if_exists == IfExistsParam.IGNORE` and a table `t` already exists at the given path, returns `t, False`.
 
-        tbl = InsertableTable._create(
-            dir._id,
-            path.name,
-            schema,
-            df,
-            primary_key=primary_key,
-            num_retained_versions=num_retained_versions,
-            comment=comment,
-            media_validation=media_validation,
-        )
-        self._tbls[tbl._id] = tbl
-        return tbl
+        Otherwise, creates a new table `t` and returns `t, True` (or raises an exception if the operation fails).
+        """
+
+        @retry_loop(for_write=True)
+        def create_fn() -> tuple[UUID, bool]:
+            import pixeltable.metadata.schema
+
+            existing = self._handle_path_collision(path, InsertableTable, False, if_exists)
+            if existing is not None:
+                assert isinstance(existing, Table)
+                return existing._id, False
+
+            dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
+            assert dir is not None
+
+            md, ops = InsertableTable._create(
+                path.name,
+                schema,
+                primary_key=primary_key,
+                num_retained_versions=num_retained_versions,
+                comment=comment,
+                media_validation=media_validation,
+                create_default_idxs=create_default_idxs,
+            )
+            tbl_id = UUID(md.tbl_md.tbl_id)
+            md.tbl_md.pending_stmt = pixeltable.metadata.schema.TableStatement.CREATE_TABLE
+            self.write_tbl_md(tbl_id, dir._id, md.tbl_md, md.version_md, md.schema_version_md, ops)
+            return tbl_id, True
+
+        self._roll_forward_ids.clear()
+        tbl_id, is_created = create_fn()
+        self._roll_forward()
+        with self.begin_xact(tbl_id=tbl_id, for_write=True):
+            tbl = self.get_table_by_id(tbl_id)
+            _logger.info(f'Created table {tbl._name!r}, id={tbl._id}')
+            Env.get().console_logger.info(f'Created table {tbl._name!r}.')
+            return tbl, is_created
 
-    @_retry_loop
     def create_view(
         self,
         path: Path,
         base: TableVersionPath,
-        select_list: Optional[list[tuple[exprs.Expr, Optional[str]]]],
-        where: Optional[exprs.Expr],
-        additional_columns: Optional[dict[str, Any]],
+        select_list: list[tuple[exprs.Expr, str | None]] | None,
+        where: exprs.Expr | None,
+        sample_clause: 'SampleClause' | None,
+        additional_columns: dict[str, Any] | None,
         is_snapshot: bool,
-        iterator: Optional[tuple[type[ComponentIterator], dict[str, Any]]],
+        create_default_idxs: bool,
+        iterator: tuple[type[ComponentIterator], dict[str, Any]] | None,
         num_retained_versions: int,
         comment: str,
         media_validation: MediaValidation,
         if_exists: IfExistsParam,
     ) -> Table:
-        from pixeltable.utils.filecache import FileCache
-
-        existing = self._handle_path_collision(path, View, is_snapshot, if_exists)
-        if existing is not None:
-            assert isinstance(existing, View)
-            return existing
+        @retry_loop(for_write=True)
+        def create_fn() -> UUID:
+            if not is_snapshot and base.is_mutable():
+                # this is a mutable view of a mutable base; X-lock the base and advance its view_sn before adding
+                # the view
+                self._acquire_tbl_lock(tbl_id=base.tbl_id, for_write=True)
+                base_tv = self.get_tbl_version(TableVersionKey(base.tbl_id, None, None), validate_initialized=True)
+                base_tv.tbl_md.view_sn += 1
+                result = Env.get().conn.execute(
+                    sql.update(schema.Table)
+                    .values({schema.Table.md: dataclasses.asdict(base_tv.tbl_md, dict_factory=md_dict_factory)})
+                    .where(schema.Table.id == base.tbl_id)
+                )
+                assert result.rowcount == 1, result.rowcount
 
-        dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
-        assert dir is not None
-        if iterator is None:
-            iterator_class, iterator_args = None, None
-        else:
-            iterator_class, iterator_args = iterator
-        view = View._create(
-            dir._id,
-            path.name,
-            base=base,
-            select_list=select_list,
-            additional_columns=additional_columns,
-            predicate=where,
-            is_snapshot=is_snapshot,
-            iterator_cls=iterator_class,
-            iterator_args=iterator_args,
-            num_retained_versions=num_retained_versions,
-            comment=comment,
-            media_validation=media_validation,
-        )
-        FileCache.get().emit_eviction_warnings()
-        self._tbls[view._id] = view
-        return view
+            existing = self._handle_path_collision(path, View, is_snapshot, if_exists, base=base)
+            if existing is not None:
+                assert isinstance(existing, View)
+                return existing._id
 
-    @_retry_loop
-    def create_replica(
-        self, path: Path, md: list[schema.FullTableMd], if_exists: IfExistsParam = IfExistsParam.ERROR
-    ) -> Table:
+            dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
+            assert dir is not None
+            if iterator is None:
+                iterator_class, iterator_args = None, None
+            else:
+                iterator_class, iterator_args = iterator
+            md, ops = View._create(
+                dir._id,
+                path.name,
+                base=base,
+                select_list=select_list,
+                additional_columns=additional_columns,
+                predicate=where,
+                sample_clause=sample_clause,
+                is_snapshot=is_snapshot,
+                create_default_idxs=create_default_idxs,
+                iterator_cls=iterator_class,
+                iterator_args=iterator_args,
+                num_retained_versions=num_retained_versions,
+                comment=comment,
+                media_validation=media_validation,
+            )
+            tbl_id = UUID(md.tbl_md.tbl_id)
+            md.tbl_md.pending_stmt = schema.TableStatement.CREATE_VIEW
+            self.write_tbl_md(tbl_id, dir._id, md.tbl_md, md.version_md, md.schema_version_md, ops)
+            return tbl_id
+
+        self._roll_forward_ids.clear()
+        view_id = create_fn()
+        if not is_snapshot and base.is_mutable():
+            # invalidate base's TableVersion instance, so that it gets reloaded with the new mutable view
+            self._clear_tv_cache(base.tbl_version.key)
+            # base_tv = self.get_tbl_version(base.tbl_id, base.tbl_version.effective_version, validate_initialized=True)
+            # view_handle = TableVersionHandle(view_id, effective_version=None)
+            # base_tv.mutable_views.add(view_handle)
+
+        self._roll_forward()
+        with self.begin_xact(tbl_id=view_id, for_write=True):
+            return self.get_table_by_id(view_id)
+
+    def _clear_tv_cache(self, key: TableVersionKey) -> None:
+        if key in self._tbl_versions:
+            tv = self._tbl_versions[key]
+            tv.is_validated = False
+            del self._tbl_versions[key]
+
+    def create_replica(self, path: Path, md: list[TableVersionMd], create_store_tbls: bool = True) -> None:
         """
         Creates table, table_version, and table_schema_version records for a replica with the given metadata.
         The metadata should be presented in standard "ancestor order", with the table being replicated at
         list position 0 and the (root) base table at list position -1.
         """
+        assert self.in_write_xact
+
+        # Acquire locks for any tables in the ancestor hierarchy that might already exist (base table first).
+        for ancestor_md in md[::-1]:  # base table first
+            self._acquire_tbl_lock(for_write=True, tbl_id=UUID(ancestor_md.tbl_md.tbl_id), raise_if_not_exists=False)
+
         tbl_id = UUID(md[0].tbl_md.tbl_id)
 
-        # First handle path collisions (if_exists='ignore' or 'replace' or etc).
-        existing = self._handle_path_collision(path, View, False, if_exists)
-        if existing is not None:
-            if existing._id != tbl_id:
-                raise excs.Error(
-                    f"An attempt was made to create a replica table at {path!r} with if_exists='ignore', "
-                    'but a different table already exists at that location.'
-                )
-            assert isinstance(existing, View)
-            return existing
+        existing = self._handle_path_collision(path, Table, False, if_exists=IfExistsParam.IGNORE)  # type: ignore[type-abstract]
+        if existing is not None and existing._id != tbl_id:
+            raise excs.Error(
+                f'An attempt was made to create a replica table at {path!r}, '
+                'but a different table already exists at that location.'
+            )
 
         # Ensure that the system directory exists.
-        self._create_dir(Path('_system', allow_system_paths=True), if_exists=IfExistsParam.IGNORE, parents=False)
+        self.__ensure_system_dir_exists()
 
         # Now check to see if this table already exists in the catalog.
-        # TODO: Handle concurrency in create_replica()
-        existing = Catalog.get().get_table_by_id(tbl_id)
+        existing = self.get_table_by_id(tbl_id)
         if existing is not None:
-            existing_path = Path(existing._path, allow_system_paths=True)
-            # It does exist. If it's a non-system table, that's an error: it's already been replicated.
-            if not existing_path.is_system_path:
+            existing_path = Path.parse(existing._path(), allow_system_path=True)
+            if existing_path != path and not existing_path.is_system_path:
+                # It does exist, under a different path from the specified one.
                 raise excs.Error(
-                    f'That table has already been replicated as {existing._path!r}. \n'
+                    f'That table has already been replicated as {existing_path!r}.\n'
                     f'Drop the existing replica if you wish to re-create it.'
                 )
-            # If it's a system table, then this means it was created at some point as the ancestor of some other
-            # table (a snapshot-over-snapshot scenario). In that case, we simply move it to the new (named) location.
-            self._move(existing_path, path)
-
-        # Now store the metadata for this replica. In the case where the table already exists (and was just moved
-        # into a named location), this will be a no-op, but it still serves to validate that the newly received
-        # metadata is identical to what's in the catalog.
-        self.__store_replica_md(path, md[0])
 
-        # Now store the metadata for all of this table's proper ancestors. If one or more proper ancestors
+        # Now store the metadata for this replica's proper ancestors. If one or more proper ancestors
         # do not yet exist in the store, they will be created as anonymous system tables.
-        for ancestor_md in md[1:]:
+        # We instantiate the ancestors starting with the base table and ending with the immediate parent of the
+        # table being replicated.
+        for ancestor_md in md[:0:-1]:
             ancestor_id = UUID(ancestor_md.tbl_md.tbl_id)
-            replica = Catalog.get().get_table_by_id(ancestor_id)
+            replica = self.get_table_by_id(ancestor_id)
             replica_path: Path
             if replica is None:
                 # We've never seen this table before. Create a new anonymous system table for it.
-                replica_path = Path(f'_system.replica_{ancestor_id.hex}', allow_system_paths=True)
+                replica_path = Path.parse(f'_system.replica_{ancestor_id.hex}', allow_system_path=True)
             else:
                 # The table already exists in the catalog. The existing path might be a system path (if the table
                 # was created as an anonymous base table of some other table), or it might not (if it's a snapshot
                 # that was directly replicated by the user at some point). In either case, use the existing path.
-                replica_path = Path(replica._path, allow_system_paths=True)
+                replica_path = Path.parse(replica._path(), allow_system_path=True)
 
-            # Store the metadata; it could be a new version (in which case a new record will be created) or a
-            # known version (in which case the newly received metadata will be validated as identical).
+            # Store the metadata; it could be a new version (in which case a new record will be created), or a known
+            # version (in which case the newly received metadata will be validated as identical).
+            # If it's a new version, this will result in a new TableVersion record being created.
             self.__store_replica_md(replica_path, ancestor_md)
 
-        # Update the catalog (as a final step, after all DB operations completed successfully).
-        # Only the table being replicated is actually made visible in the catalog.
-        self._tbls[tbl_id] = self._load_tbl(tbl_id)
-        return self._tbls[tbl_id]
-
-    def __store_replica_md(self, path: Path, md: schema.FullTableMd) -> None:
+            # Now we must clear cached metadata for the ancestor table, to force the next table operation to pick up
+            # the new TableVersion instance. This is necessary because computed columns of descendant tables might
+            # reference columns of the ancestor table that only exist in the new version.
+            replica = self.get_table_by_id(ancestor_id)
+            # assert replica is not None  # If it didn't exist before, it must have been created by now.
+            if replica is not None:
+                replica._tbl_version_path.clear_cached_md()
+
+        # Store the metadata for the table being replicated; as before, it could be a new version or a known version.
+        # If it's a new version, then a TableVersion record will be created, unless the table being replicated
+        # is a pure snapshot.
+        self.__store_replica_md(path, md[0], create_store_tbls)
+
+        # Finally, it's possible that the table already exists in the catalog, but as an anonymous system table that
+        # was hidden the last time we checked (and that just became visible when the replica was imported). In this
+        # case, we need to make the existing table visible by moving it to the specified path.
+        # We need to do this at the end, since `existing_path` needs to first have a non-fragment table version in
+        # order to be instantiated as a schema object.
+        existing = self.get_table_by_id(tbl_id)
+        assert existing is not None
+        existing_path = Path.parse(existing._path(), allow_system_path=True)
+        if existing_path != path:
+            assert existing_path.is_system_path
+            self._move(existing_path, path, IfExistsParam.ERROR, IfNotExistsParam.ERROR)
+
+    def __ensure_system_dir_exists(self) -> Dir:
+        system_path = Path.parse('_system', allow_system_path=True)
+        return self._create_dir(system_path, if_exists=IfExistsParam.IGNORE, parents=False)
+
+    def __store_replica_md(self, path: Path, md: TableVersionMd, create_store_tbl: bool = True) -> None:
         _logger.info(f'Creating replica table at {path!r} with ID: {md.tbl_md.tbl_id}')
-        # TODO: Handle concurrency
         dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
         assert dir is not None
+        assert self._in_write_xact
 
         conn = Env.get().conn
         tbl_id = md.tbl_md.tbl_id
 
-        new_tbl_md: Optional[schema.TableMd] = None
-        new_version_md: Optional[schema.TableVersionMd] = None
-        new_schema_version_md: Optional[schema.TableSchemaVersionMd] = None
+        new_tbl_md: schema.TableMd | None = None
+        new_version_md: schema.VersionMd | None = None
+        new_schema_version_md: schema.SchemaVersionMd | None = None
+        is_new_tbl_version: bool = False
 
         # We need to ensure that the table metadata in the catalog always reflects the latest observed version of
         # this table. (In particular, if this is a base table, then its table metadata need to be consistent
         # with the latest version of this table having a replicated view somewhere in the catalog.)
+        # TODO: handle concurrent drop() of an existing replica; if we just ignore that Table record here, we can end
+        # up with a duplicate key violation; in principle, we should wait for the concurrent drop() to finish
         q: sql.Executable = sql.select(schema.Table.md).where(schema.Table.id == tbl_id)
         existing_md_row = conn.execute(q).one_or_none()
 
+        # Update md with the given name, current user, and is_replica flag.
+        md = dataclasses.replace(
+            md, tbl_md=dataclasses.replace(md.tbl_md, name=path.name, user=Env.get().user, is_replica=True)
+        )
         if existing_md_row is None:
             # No existing table, so create a new record.
             q = sql.insert(schema.Table.__table__).values(
-                id=tbl_id,
-                dir_id=dir._id,
-                md=dataclasses.asdict(
-                    dataclasses.replace(md.tbl_md, name=path.name, user=Env.get().user, is_replica=True)
-                ),
+                id=tbl_id, dir_id=dir._id, md=dataclasses.asdict(md.tbl_md, dict_factory=md_dict_factory)
             )
             conn.execute(q)
-        else:
-            assert existing_md_row.md['is_replica']
-            if md.tbl_md.current_version > existing_md_row.md['current_version']:
-                # New metadata is more recent than the metadata currently stored in the DB; we'll update the record
-                # in place in the DB.
-                new_tbl_md = dataclasses.replace(md.tbl_md, name=path.name, user=Env.get().user, is_replica=True)
+        elif not existing_md_row.md['is_replica']:
+            raise excs.Error(
+                'An attempt was made to replicate a view whose base table already exists in the local catalog '
+                'in its original form.\n'
+                'If this is intentional, you must first drop the existing base table:\n'
+                f'  pxt.drop_table({str(path)!r})'
+            )
+        elif md.tbl_md.current_version > existing_md_row.md['current_version']:
+            # New metadata is more recent than the metadata currently stored in the DB; we'll update the record
+            # in place in the DB.
+            new_tbl_md = md.tbl_md
 
         # Now see if a TableVersion record already exists in the DB for this table version. If not, insert it. If
         # it already exists, check that the existing record is identical to the new one.
         q = (
             sql.select(schema.TableVersion.md)
             .where(schema.TableVersion.tbl_id == tbl_id)
-            .where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {md.version_md.version}"))
+            .where(schema.TableVersion.md['version'].cast(sql.Integer) == md.version_md.version)
         )
         existing_version_md_row = conn.execute(q).one_or_none()
         if existing_version_md_row is None:
             new_version_md = md.version_md
+            is_new_tbl_version = True
         else:
-            existing_version_md = schema.md_from_dict(schema.TableVersionMd, existing_version_md_row.md)
-            if existing_version_md != md.version_md:
+            existing_version_md = schema.md_from_dict(schema.VersionMd, existing_version_md_row.md)
+            # Validate that the existing metadata are identical to the new metadata, except is_fragment
+            # and additional_md which may differ.
+            if (
+                dataclasses.replace(
+                    existing_version_md,
+                    is_fragment=md.version_md.is_fragment,
+                    additional_md=md.version_md.additional_md,
+                )
+                != md.version_md
+            ):
                 raise excs.Error(
                     f'The version metadata for the replica {path!r}:{md.version_md.version} is inconsistent with '
                     'the metadata recorded from a prior replica.\n'
                     'This is likely due to data corruption in the replicated table.'
                 )
+            if existing_version_md.is_fragment and not md.version_md.is_fragment:
+                # This version exists in the DB as a fragment, but we're importing a complete copy of the same version;
+                # set the is_fragment flag to False in the DB.
+                new_version_md = md.version_md
 
         # Do the same thing for TableSchemaVersion.
         q = (
             sql.select(schema.TableSchemaVersion.md)
             .where(schema.TableSchemaVersion.tbl_id == tbl_id)
             .where(
-                sql.text(
-                    f"({schema.TableSchemaVersion.__table__}.md->>'schema_version')::int = "
-                    f'{md.schema_version_md.schema_version}'
-                )
+                schema.TableSchemaVersion.md['schema_version'].cast(sql.Integer) == md.schema_version_md.schema_version
             )
         )
         existing_schema_version_md_row = conn.execute(q).one_or_none()
         if existing_schema_version_md_row is None:
             new_schema_version_md = md.schema_version_md
         else:
-            existing_schema_version_md = schema.md_from_dict(
-                schema.TableSchemaVersionMd, existing_schema_version_md_row.md
-            )
+            existing_schema_version_md = schema.md_from_dict(schema.SchemaVersionMd, existing_schema_version_md_row.md)
+            # Validate that the existing metadata are identical to the new metadata.
             if existing_schema_version_md != md.schema_version_md:
                 raise excs.Error(
                     f'The schema version metadata for the replica {path!r}:{md.schema_version_md.schema_version} '
@@ -580,65 +1360,216 @@ class Catalog:
                     'This is likely due to data corruption in the replicated table.'
                 )
 
-        self.store_tbl_md(UUID(tbl_id), new_tbl_md, new_version_md, new_schema_version_md)
+        self.write_tbl_md(UUID(tbl_id), None, new_tbl_md, new_version_md, new_schema_version_md)
+
+        if is_new_tbl_version and not md.is_pure_snapshot:
+            # It's a new version of a table that has a physical store, so we need to create a TableVersion instance.
+            TableVersion.create_replica(md, create_store_tbl)
+
+    def get_additional_md(self, tbl_id: UUID) -> dict[str, Any]:
+        """Return the additional_md field of the given table."""
+        assert Env.get().in_xact
+        conn = Env.get().conn
+        q = sql.select(schema.Table.additional_md).where(self._active_tbl_clause(tbl_id=tbl_id))
+        # TODO: handle concurrent drop()
+        row = conn.execute(q).one()
+        assert isinstance(row[0], dict)
+        return row[0]
+
+    def update_additional_md(self, tbl_id: UUID, additional_md: dict[str, Any]) -> None:
+        """
+        Update the additional_md field of the given table. The new additional_md is merged with the
+        existing one via a JSON dictionary merge, giving preference to the new values.
+        """
+        assert self._in_write_xact
+        conn = Env.get().conn
+        q = (
+            sql.update(schema.Table)
+            .where(schema.Table.id == str(tbl_id))
+            .values({schema.Table.additional_md: schema.Table.additional_md.op('||')(additional_md)})
+        )
+        result = conn.execute(q)
+        assert result.rowcount == 1, result.rowcount
+
+    @retry_loop(for_write=False)
+    def get_table(self, path: Path, if_not_exists: IfNotExistsParam) -> Table | None:
+        obj = Catalog.get()._get_schema_object(
+            path, expected=Table, raise_if_not_exists=(if_not_exists == IfNotExistsParam.ERROR)
+        )
+        if obj is None:
+            _logger.info(f'Skipped table {path!r} (does not exist).')
+            return None
 
-    @_retry_loop
-    def get_table(self, path: Path) -> Table:
-        obj = Catalog.get()._get_schema_object(path, expected=Table, raise_if_not_exists=True)
         assert isinstance(obj, Table)
-        obj._tbl_version.get().ensure_md_loaded()
+        # We need to clear cached metadata from tbl_version_path, in case the schema has been changed
+        # by another process.
+        obj._tbl_version_path.clear_cached_md()
         return obj
 
-    @_retry_loop
     def drop_table(self, path: Path, if_not_exists: IfNotExistsParam, force: bool) -> None:
-        _, _, src_obj = self._prepare_dir_op(
-            drop_dir_path=path.parent,
-            drop_name=path.name,
-            drop_expected=Table,
-            raise_if_not_exists=if_not_exists == IfNotExistsParam.ERROR and not force,
-        )
-        if src_obj is None:
-            _logger.info(f'Skipped table {str(path)!r} (does not exist).')
-            return
-        assert isinstance(src_obj, Table)
-        self._drop_tbl(src_obj, force=force, is_replace=False)
+        @retry_loop(for_write=True)
+        def drop_fn() -> None:
+            tbl = self._get_schema_object(
+                path,
+                expected=Table,
+                raise_if_not_exists=(if_not_exists == IfNotExistsParam.ERROR and not force),
+                lock_parent=True,
+                lock_obj=False,
+            )
+            if tbl is None:
+                _logger.info(f'Skipped table {path!r} (does not exist).')
+                return
+            assert isinstance(tbl, Table)
+
+            if isinstance(tbl, View) and tbl._tbl_version_path.is_mutable() and tbl._tbl_version_path.base.is_mutable():
+                # this is a mutable view of a mutable base;
+                # lock the base before the view, in order to avoid deadlocks with concurrent inserts/updates
+                base_id = tbl._tbl_version_path.base.tbl_id
+                self._acquire_tbl_lock(tbl_id=base_id, for_write=True, lock_mutable_tree=False)
+
+            self._drop_tbl(tbl, force=force, is_replace=False)
 
-    def _drop_tbl(self, tbl: Table, force: bool, is_replace: bool) -> None:
+        self._roll_forward_ids.clear()
+        drop_fn()
+        self._roll_forward()
+
+    def _drop_tbl(self, tbl: Table | TableVersionPath, force: bool, is_replace: bool) -> None:
         """
         Drop the table (and recursively its views, if force == True).
 
+        `tbl` can be an instance of `Table` for a user table, or `TableVersionPath` for a hidden (system) table.
+
+        Returns:
+            List of table ids that were dropped.
+
         Locking protocol:
         - X-lock base before X-locking any view
         - deadlock-free wrt to TableVersion.insert() (insert propagation also proceeds top-down)
         - X-locks parent dir prior to calling TableVersion.drop(): prevent concurrent creation of another SchemaObject
-          in the same directory with the same name (which could lead to duplicate names if we get rolled back)
+          in the same directory with the same name (which could lead to duplicate names if we get aborted)
         """
-        view_ids = self.get_view_ids(tbl._id, for_update=True)
+        is_pure_snapshot: bool
+        if isinstance(tbl, TableVersionPath):
+            tvp = tbl
+            tbl_id = tvp.tbl_id
+            tbl = None
+            is_pure_snapshot = False
+        else:
+            tvp = tbl._tbl_version_path
+            tbl_id = tbl._id
+            is_pure_snapshot = tbl._tbl_version is None
+
+        if tbl is not None:
+            self._acquire_dir_xlock(dir_id=tbl._dir_id)
+        self._acquire_tbl_lock(tbl_id=tbl_id, for_write=True, lock_mutable_tree=False)
+
+        view_ids = self.get_view_ids(tbl_id, for_update=True)
+        is_replica = tvp.is_replica()
+        do_drop = True
+
+        _logger.debug(f'Preparing to drop table {tbl_id} (force={force!r}, is_replica={is_replica}).')
+
         if len(view_ids) > 0:
-            if not force:
-                is_snapshot = tbl._tbl_version_path.is_snapshot()
-                obj_type_str = 'Snapshot' if is_snapshot else tbl._display_name().capitalize()
+            if force:
+                # recursively drop views first
+                for view_id in view_ids:
+                    view = self.get_table_by_id(view_id, ignore_if_dropped=True)
+                    if view is not None:
+                        self._drop_tbl(view, force=force, is_replace=is_replace)
+
+            elif is_replica:
+                # Dropping a replica with dependents and no 'force': just rename it to be a hidden table;
+                # the actual table will not be dropped.
+                assert tbl is not None  # can only occur for a user table
+                system_dir = self.__ensure_system_dir_exists()
+                new_name = f'replica_{tbl_id.hex}'
+                _logger.debug(f'{tbl._path()!r} is a replica with dependents; renaming to {new_name!r}.')
+                tbl._move(new_name, system_dir._id)
+                do_drop = False  # don't actually clear the catalog for this table
+
+            else:
+                # It has dependents but is not a replica and no 'force', so it's an error to drop it.
+                assert tbl is not None  # can only occur for a user table
                 msg: str
                 if is_replace:
                     msg = (
-                        f'{obj_type_str} {tbl._path} already exists and has dependents. '
+                        f'{tbl._display_str()} already exists and has dependents. '
                         "Use `if_exists='replace_force'` to replace it."
                     )
                 else:
-                    msg = f'{obj_type_str} {tbl._path} has dependents.'
+                    msg = f'{tbl._display_str()} has dependents.'
                 raise excs.Error(msg)
 
-            for view_id in view_ids:
-                view = self.get_table_by_id(view_id)
-                self._drop_tbl(view, force=force, is_replace=is_replace)
+        # if this is a mutable view of a mutable base, advance the base's view_sn
+        if isinstance(tbl, View) and tvp.is_mutable() and tvp.base.is_mutable():
+            base_id = tvp.base.tbl_id
+            base_tv = self.get_tbl_version(TableVersionKey(base_id, None, None), validate_initialized=True)
+            base_tv.tbl_md.view_sn += 1
+            result = Env.get().conn.execute(
+                sql.update(schema.Table.__table__)
+                .values({schema.Table.md: dataclasses.asdict(base_tv.tbl_md, dict_factory=md_dict_factory)})
+                .where(schema.Table.id == base_id)
+            )
+            assert result.rowcount == 1, result.rowcount
+            # force reload of base TV instance in order to make its state consistent with the stored metadata
+            self._clear_tv_cache(base_tv.key)
+
+        if do_drop:
+            if is_pure_snapshot:
+                # there is no physical table, but we still need to delete the Table record; we can do that right now
+                # as part of the current transaction
+                self.delete_tbl_md(tbl_id)
+            else:
+                # invalidate the TableVersion instance when we're done so that existing references to it can find out it
+                # has been dropped
+                self.mark_modified_tvs(tvp.tbl_version)
+
+                # write TableOps to execute the drop, plus the updated Table record
+                tv = tvp.tbl_version.get()
+                tv.tbl_md.pending_stmt = schema.TableStatement.DROP_TABLE
+                drop_ops = tv.drop()
+                self.write_tbl_md(
+                    tv.id,
+                    dir_id=None,
+                    tbl_md=tv.tbl_md,
+                    version_md=None,
+                    schema_version_md=None,
+                    pending_ops=drop_ops,
+                    remove_from_dir=True,
+                )
 
-        _ = self.get_dir(tbl._dir_id, for_update=True)  # X-lock the parent directory
-        tbl._drop()
-        assert tbl._id in self._tbls
-        del self._tbls[tbl._id]
-        _logger.info(f'Dropped table `{tbl._path}`.')
+            tvp.clear_cached_md()
 
-    @_retry_loop
+        assert (
+            is_replica
+            or (tbl_id, None) in self._tbls  # non-replica tables must have an entry with effective_version=None
+        )
+
+        # Remove visible Table references (we do this even for a replica that was just renamed).
+        versions = [version for id, version in self._tbls if id == tbl_id]
+        for version in versions:
+            del self._tbls[tbl_id, version]
+
+        _logger.info(f'Dropped table {tbl_id if tbl is None else repr(tbl._path())}.')
+
+        if (
+            is_replica  # if this is a replica,
+            and do_drop  # and it was actually dropped (not just renamed),
+            and tvp.base is not None  # and it has a base table,
+        ):
+            base_tbl = self.get_table_by_id(tvp.base.tbl_id)
+            base_tbl_path = None if base_tbl is None else Path.parse(base_tbl._path(), allow_system_path=True)
+            if (
+                (base_tbl_path is None or base_tbl_path.is_system_path)  # and the base table is hidden,
+                and len(self.get_view_ids(tvp.base.tbl_id, for_update=True)) == 0  # and has no other dependents,
+            ):
+                # then drop the base table as well (possibly recursively).
+                _logger.debug(f'Dropping hidden base table {tvp.base.tbl_id} of dropped replica {tbl_id}.')
+                # we just dropped the anchor on `tvp.base`; we need to clear the anchor so that we can actually
+                # load the TableVersion instance in order to drop it
+                self._drop_tbl(tvp.base.anchor_to(None), force=False, is_replace=False)
+
+    @retry_loop(for_write=True)
     def create_dir(self, path: Path, if_exists: IfExistsParam, parents: bool) -> Dir:
         return self._create_dir(path, if_exists, parents)
 
@@ -651,12 +1582,12 @@ class Catalog:
         # parent = self._get_schema_object(path.parent)
         # assert parent is not None
         # dir = Dir._create(parent._id, path.name)
-        # Env.get().console_logger.info(f'Created directory {str(path)!r}.')
+        # Env.get().console_logger.info(f'Created directory {path!r}.')
         # return dir
 
         if parents:
             # start walking down from the root
-            last_parent: Optional[SchemaObject] = None
+            last_parent: SchemaObject | None = None
             for ancestor in path.ancestors():
                 ancestor_obj = self._get_schema_object(ancestor, expected=Dir)
                 assert ancestor_obj is not None or last_parent is not None
@@ -670,21 +1601,26 @@ class Catalog:
             return existing
         assert parent is not None
         dir = Dir._create(parent._id, path.name)
-        Env.get().console_logger.info(f'Created directory {str(path)!r}.')
+        Env.get().console_logger.info(f'Created directory {path!r}.')
         return dir
 
-    @_retry_loop
     def drop_dir(self, path: Path, if_not_exists: IfNotExistsParam, force: bool) -> None:
-        _, _, schema_obj = self._prepare_dir_op(
-            drop_dir_path=path.parent,
-            drop_name=path.name,
-            drop_expected=Dir,
-            raise_if_not_exists=if_not_exists == IfNotExistsParam.ERROR and not force,
-        )
-        if schema_obj is None:
-            _logger.info(f'Directory {str(path)!r} does not exist; skipped drop_dir().')
-            return
-        self._drop_dir(schema_obj._id, path, force=force)
+        @retry_loop(for_write=True)
+        def drop_fn() -> None:
+            _, _, schema_obj = self._prepare_dir_op(
+                drop_dir_path=path.parent,
+                drop_name=path.name,
+                drop_expected=Dir,
+                raise_if_not_exists=if_not_exists == IfNotExistsParam.ERROR and not force,
+            )
+            if schema_obj is None:
+                _logger.info(f'Directory {path!r} does not exist; skipped drop_dir().')
+                return
+            self._drop_dir(schema_obj._id, path, force=force)
+
+        self._roll_forward_ids.clear()
+        drop_fn()
+        self._roll_forward()
 
     def _drop_dir(self, dir_id: UUID, dir_path: Path, force: bool = False) -> None:
         conn = Env.get().conn
@@ -692,60 +1628,145 @@ class Catalog:
             # check for existing entries
             q = sql.select(sql.func.count()).select_from(schema.Dir).where(schema.Dir.parent_id == dir_id)
             num_subdirs = conn.execute(q).scalar()
-            q = sql.select(sql.func.count()).select_from(schema.Table).where(schema.Table.dir_id == dir_id)
+            q = sql.select(sql.func.count()).select_from(schema.Table).where(self._active_tbl_clause(dir_id=dir_id))
             num_tbls = conn.execute(q).scalar()
             if num_subdirs + num_tbls > 0:
-                raise excs.Error(f'Directory {str(dir_path)!r} is not empty.')
+                raise excs.Error(f'Directory {dir_path!r} is not empty.')
 
         # drop existing subdirs
-        self._lock_dir(dir_id, None, None)
+        self._acquire_dir_xlock(dir_id=dir_id)
         dir_q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id)
         for row in conn.execute(dir_q).all():
             self._drop_dir(row.id, dir_path.append(row.md['name']), force=True)
 
         # drop existing tables
-        tbl_q = sql.select(schema.Table).where(schema.Table.dir_id == dir_id).with_for_update()
+        tbl_q = sql.select(schema.Table).where(self._active_tbl_clause(dir_id=dir_id)).with_for_update()
         for row in conn.execute(tbl_q).all():
-            tbl = self.get_table_by_id(row.id)
+            tbl = self.get_table_by_id(row.id, ignore_if_dropped=True)
             # this table would have been dropped already if it's a view of a base we dropped earlier
             if tbl is not None:
                 self._drop_tbl(tbl, force=True, is_replace=False)
 
         # self.drop_dir(dir_id)
         conn.execute(sql.delete(schema.Dir).where(schema.Dir.id == dir_id))
-        _logger.info(f'Removed directory {str(dir_path)!r}.')
+        _logger.info(f'Removed directory {dir_path!r}.')
 
     def get_view_ids(self, tbl_id: UUID, for_update: bool = False) -> list[UUID]:
         """Return the ids of views that directly reference the given table"""
         conn = Env.get().conn
-        q = sql.select(schema.Table.id).where(sql.text(f"md->'view_md'->'base_versions'->0->>0 = {tbl_id.hex!r}"))
+        # check whether this table still exists
+        q = sql.select(sql.func.count()).select_from(schema.Table).where(self._active_tbl_clause(tbl_id=tbl_id))
+        tbl_count = conn.execute(q).scalar()
+        if tbl_count == 0:
+            raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
+        q = (
+            sql.select(schema.Table.id)
+            .where(schema.Table.md['view_md']['base_versions'][0][0].astext == tbl_id.hex)
+            .where(self._active_tbl_clause())
+        )
         if for_update:
             q = q.with_for_update()
         result = [r[0] for r in conn.execute(q).all()]
         return result
 
-    def get_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
-        if (tbl_id, effective_version) not in self._tbl_versions:
-            self._tbl_versions[tbl_id, effective_version] = self._load_tbl_version(tbl_id, effective_version)
-        return self._tbl_versions[tbl_id, effective_version]
+    def get_tbl_version(
+        self, key: TableVersionKey, *, check_pending_ops: bool = True, validate_initialized: bool = False
+    ) -> TableVersion | None:
+        """
+        Returns the TableVersion instance for the given table and version and updates the cache.
+
+        If present in the cache and the instance isn't validated, validates version and view_sn against the stored
+        metadata.
+        """
+        # we need a transaction here, if we're not already in one; if this starts a new transaction,
+        # the returned TableVersion instance will not be validated
+        with self.begin_xact(for_write=False) as conn:
+            tv = self._tbl_versions.get(key)
+            if tv is None:
+                tv = self._load_tbl_version(key, check_pending_ops=check_pending_ops)
+            elif not tv.is_validated:
+                # only live instances are invalidated
+                assert key.effective_version is None
+                # _logger.debug(f'validating metadata for table {tbl_id}:{tv.version} ({id(tv):x})')
+                where_clause: sql.ColumnElement[bool]
+                if check_pending_ops:
+                    # if we don't want to see pending ops, we also don't want to see dropped tables
+                    where_clause = self._active_tbl_clause(tbl_id=key.tbl_id)
+                else:
+                    where_clause = schema.Table.id == key.tbl_id
+                q = sql.select(schema.Table.md).where(where_clause)
+                row = conn.execute(q).one_or_none()
+                if row is None:
+                    raise excs.Error(self._dropped_tbl_error_msg(key.tbl_id))
+
+                reload = False
+
+                if tv.anchor_tbl_id is None:
+                    # live non-replica table; compare our cached TableMd.current_version/view_sn to what's stored
+                    q = sql.select(schema.Table.md).where(where_clause)
+                    row = conn.execute(q).one_or_none()
+                    if row is None:
+                        raise excs.Error(self._dropped_tbl_error_msg(key.tbl_id))
+                    current_version, view_sn = row.md['current_version'], row.md['view_sn']
+                    if current_version != tv.version or view_sn != tv.tbl_md.view_sn:
+                        _logger.debug(
+                            f'reloading metadata for live table {key.tbl_id} '
+                            f'(cached/current version: {tv.version}/{current_version}, '
+                            f'cached/current view_sn: {tv.tbl_md.view_sn}/{view_sn})'
+                        )
+                        reload = True
+
+                else:
+                    # live replica table; use the anchored version
+                    anchor_tbl_version_md = self.head_version_md(tv.anchor_tbl_id)
+                    assert anchor_tbl_version_md is not None
+                    q = sql.select(schema.TableVersion.md)
+                    if check_pending_ops:
+                        q = q.join(schema.Table, schema.Table.id == schema.TableVersion.tbl_id).where(
+                            self._active_tbl_clause(tbl_id=key.tbl_id)
+                        )
+                    q = (
+                        q.where(schema.TableVersion.tbl_id == key.tbl_id)
+                        .where(schema.TableVersion.md['created_at'].cast(sql.Float) <= anchor_tbl_version_md.created_at)
+                        .order_by(schema.TableVersion.md['created_at'].cast(sql.Float).desc())
+                        .limit(1)
+                    )
+                    row = conn.execute(q).one_or_none()
+                    if row is None:
+                        raise excs.Error(self._dropped_tbl_error_msg(key.tbl_id))
+                    version = row.md['version']
+                    if version != tv.version:  # TODO: How will view_sn work for replicas?
+                        _logger.debug(
+                            f'reloading metadata for replica table {key.tbl_id} (anchor {key.anchor_tbl_id}) '
+                            f'(cached/anchored version: {tv.version}/{version})'
+                        )
+                        reload = True
+
+                # the stored version can be behind TableVersion.version, because we don't roll back the in-memory
+                # metadata changes after a failed update operation
+                if reload:
+                    # the cached metadata is invalid
+                    tv = self._load_tbl_version(key, check_pending_ops=check_pending_ops)
+                else:
+                    # the cached metadata is valid
+                    tv.is_validated = True
 
-    def add_tbl_version(self, tbl_version: TableVersion) -> None:
-        """Explicitly add a TableVersion"""
-        self._tbl_versions[tbl_version.id, tbl_version.effective_version] = tbl_version
-        # if this is a mutable view, also record it in the base
-        if tbl_version.is_view and tbl_version.effective_version is None:
-            base = tbl_version.base.get()
-            base.mutable_views.append(TableVersionHandle(tbl_version.id, tbl_version.effective_version))
+            assert tv.anchor_tbl_id == key.anchor_tbl_id
+            assert tv.is_validated, f'{key} not validated\n{tv.__dict__}\n{self._debug_str()}'
+            if validate_initialized:
+                assert tv.is_initialized, f'{key} not initialized\n{tv.__dict__}\n{self._debug_str()}'
+            return tv
 
-    def remove_tbl_version(self, tbl_version: TableVersion) -> None:
-        assert (tbl_version.id, tbl_version.effective_version) in self._tbl_versions
-        del self._tbl_versions[tbl_version.id, tbl_version.effective_version]
+    def remove_tbl_version(self, key: TableVersionKey) -> None:
+        assert isinstance(key, TableVersionKey)
+        assert key in self._tbl_versions
+        del self._tbl_versions[key]
 
-    def get_dir(self, dir_id: UUID, for_update: bool = False) -> Optional[Dir]:
+    def get_dir(self, dir_id: UUID, for_update: bool = False) -> Dir | None:
         """Return the Dir with the given id, or None if it doesn't exist"""
         conn = Env.get().conn
         if for_update:
-            self._lock_dir(None, dir_id, None)
+            self._acquire_dir_xlock(dir_id=dir_id)
         q = sql.select(schema.Dir).where(schema.Dir.id == dir_id)
         row = conn.execute(q).one_or_none()
         if row is None:
@@ -753,24 +1774,24 @@ class Catalog:
         dir_record = schema.Dir(**row._mapping)
         return Dir(dir_record.id, dir_record.parent_id, dir_record.md['name'])
 
-    def _get_dir(self, path: Path, for_update: bool = False) -> Optional[schema.Dir]:
+    def _get_dir(self, path: Path, lock_dir: bool = False) -> schema.Dir | None:
         """
-        Locking protocol: X locks on all ancestors
+        lock_dir: if True, X-locks target (but not the ancestors)
         """
         user = Env.get().user
         conn = Env.get().conn
         if path.is_root:
-            if for_update:
-                self._lock_dir(parent_id=None, dir_id=None, dir_name='')
+            if lock_dir:
+                self._acquire_dir_xlock(dir_name='')
             q = sql.select(schema.Dir).where(schema.Dir.parent_id.is_(None), schema.Dir.md['user'].astext == user)
             row = conn.execute(q).one_or_none()
             return schema.Dir(**row._mapping) if row is not None else None
         else:
-            parent_dir = self._get_dir(path.parent, for_update=False)
+            parent_dir = self._get_dir(path.parent, lock_dir=False)
             if parent_dir is None:
                 return None
-            if for_update:
-                self._lock_dir(parent_id=parent_dir.id, dir_id=None, dir_name=path.name)
+            if lock_dir:
+                self._acquire_dir_xlock(parent_id=parent_dir.id, dir_name=path.name)
             q = sql.select(schema.Dir).where(
                 schema.Dir.parent_id == parent_dir.id,
                 schema.Dir.md['name'].astext == path.name,
@@ -779,94 +1800,283 @@ class Catalog:
             row = conn.execute(q).one_or_none()
             return schema.Dir(**row._mapping) if row is not None else None
 
-    def _load_tbl(self, tbl_id: UUID) -> Optional[Table]:
-        _logger.info(f'Loading table {tbl_id}')
+    def _load_tbl(self, tbl_id: UUID, ignore_pending_drop: bool = False) -> Table | None:
+        """Loads metadata for the table with the given id and caches it."""
         from .insertable_table import InsertableTable
         from .view import View
 
+        assert tbl_id is not None
+        _logger.info(f'Loading table {tbl_id}')
+
         conn = Env.get().conn
+
+        if ignore_pending_drop:
+            # check whether this table is in the process of being dropped
+            q: sql.Executable = sql.select(schema.Table.md).where(schema.Table.id == tbl_id)
+            row = conn.execute(q).one()
+            if row.md['pending_stmt'] == schema.TableStatement.DROP_TABLE.value:
+                return None
+
+        # check for pending ops
+        q = sql.select(sql.func.count()).where(schema.PendingTableOp.tbl_id == tbl_id)
+        has_pending_ops = conn.execute(q).scalar() > 0
+        if has_pending_ops:
+            raise PendingTableOpsError(tbl_id)
+
         q = (
             sql.select(schema.Table, schema.TableSchemaVersion)
             .join(schema.TableSchemaVersion)
             .where(schema.Table.id == schema.TableSchemaVersion.tbl_id)
-            # Table.md['current_schema_version'] == TableSchemaVersion.schema_version
             .where(
-                sql.text(
-                    f"({schema.Table.__table__}.md->>'current_schema_version')::int = "
-                    f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
-                )
+                schema.Table.md['current_schema_version'].cast(sql.Integer) == schema.TableSchemaVersion.schema_version
             )
             .where(schema.Table.id == tbl_id)
         )
         row = conn.execute(q).one_or_none()
         if row is None:
             return None
-        tbl_record, schema_version_record = _unpack_row(row, [schema.Table, schema.TableSchemaVersion])
+        tbl_record, _ = _unpack_row(row, [schema.Table, schema.TableSchemaVersion])
 
         tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
         view_md = tbl_md.view_md
-        if view_md is None:
-            # this is a base table
-            if (tbl_id, None) not in self._tbl_versions:
-                self._tbl_versions[tbl_id, None] = self._load_tbl_version(tbl_id, None)
-            tbl = InsertableTable(tbl_record.dir_id, TableVersionHandle(tbl_id, None))
+
+        if view_md is None and not tbl_md.is_replica:
+            # this is a base, non-replica table
+            key = TableVersionKey(tbl_id, None, None)
+            if key not in self._tbl_versions:
+                _ = self._load_tbl_version(key)
+            tbl = InsertableTable(tbl_record.dir_id, TableVersionHandle(key))
+            self._tbls[tbl_id, None] = tbl
             return tbl
 
         # this is a view; determine the sequence of TableVersions to load
-        tbl_version_path: list[tuple[UUID, Optional[int]]] = []
-        schema_version_md = schema.md_from_dict(schema.TableSchemaVersionMd, schema_version_record.md)
-        pure_snapshot = view_md.is_snapshot and view_md.predicate is None and len(schema_version_md.columns) == 0
-        if pure_snapshot:
+        tbl_version_path: list[tuple[UUID, int | None]] = []
+        anchor_tbl_id = UUID(tbl_md.tbl_id) if tbl_md.is_replica else None
+        if tbl_md.is_pure_snapshot:
             # this is a pure snapshot, without a physical table backing it; we only need the bases
             pass
         else:
-            effective_version = 0 if view_md.is_snapshot else None  # snapshots only have version 0
+            effective_version = (
+                0 if view_md is not None and view_md.is_snapshot else None
+            )  # snapshots only have version 0
             tbl_version_path.append((tbl_id, effective_version))
-        tbl_version_path.extend((UUID(tbl_id), version) for tbl_id, version in view_md.base_versions)
+
+        if view_md is not None:
+            tbl_version_path.extend((UUID(ancestor_id), version) for ancestor_id, version in view_md.base_versions)
+
+        if anchor_tbl_id is not None and self.head_version_md(anchor_tbl_id) is None:
+            return None
 
         # load TableVersions, starting at the root
-        base_path: Optional[TableVersionPath] = None
-        view_path: Optional[TableVersionPath] = None
+        base_path: TableVersionPath | None = None
+        view_path: TableVersionPath | None = None
         for id, effective_version in tbl_version_path[::-1]:
-            if (id, effective_version) not in self._tbl_versions:
-                self._tbl_versions[id, effective_version] = self._load_tbl_version(id, effective_version)
-            view_path = TableVersionPath(TableVersionHandle(id, effective_version), base=base_path)
+            # anchor the path elements that have effective_version == None
+            key = TableVersionKey(id, effective_version, None if effective_version is not None else anchor_tbl_id)
+            if key not in self._tbl_versions:
+                _ = self._load_tbl_version(key)
+            view_path = TableVersionPath(TableVersionHandle(key), base=base_path)
             base_path = view_path
-        view = View(tbl_id, tbl_record.dir_id, tbl_md.name, view_path, snapshot_only=pure_snapshot)
-        # TODO: also load mutable views
+        view = View(tbl_id, tbl_record.dir_id, tbl_md.name, view_path, snapshot_only=tbl_md.is_pure_snapshot)
+        self._tbls[tbl_id, None] = view
         return view
 
-    def load_tbl_md(self, tbl_id: UUID, effective_version: Optional[int]) -> schema.FullTableMd:
+    def _load_tbl_at_version(self, tbl_id: UUID, version: int) -> Table | None:
+        from .view import View
+
+        # Load the specified TableMd and TableVersionMd records from the db.
+        conn = Env.get().conn
+        q: sql.Executable = (
+            sql.select(schema.Table, schema.TableVersion)
+            .join(schema.TableVersion)
+            .where(schema.Table.id == tbl_id)
+            .where(schema.Table.id == schema.TableVersion.tbl_id)
+            .where(schema.TableVersion.version == version)
+        )
+        row = conn.execute(q).one_or_none()
+        if row is None:
+            return None
+        tbl_record, version_record = _unpack_row(row, [schema.Table, schema.TableVersion])
+        tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
+        version_md = schema.md_from_dict(schema.VersionMd, version_record.md)
+        tvp = self.construct_tvp(tbl_id, version, tbl_md.ancestors, version_md.created_at)
+
+        view = View(tbl_id, tbl_record.dir_id, tbl_md.name, tvp, snapshot_only=True)
+        self._tbls[tbl_id, version] = view
+        return view
+
+    def construct_tvp(
+        self, tbl_id: UUID, version: int, ancestors_of_live_tbl: schema.TableVersionPath, created_at: float
+    ) -> TableVersionPath:
+        """
+        Construct the TableVersionPath for the specified version of the given table. Here `live_ancestors` is the
+        list of ancestor table IDs and fixed versions (if any) from the table's metadata. The constructed
+        TableVersionPath will preserve any fixed versions from `live_ancestors` (corresponding to a view-over-snapshot
+        scenario), while "filling in" the implied versions for any `None` versions.
+        """
+        # TODO: Currently, we reconstruct the ancestors by inspecting the created_at timestamps of the table and its
+        #     ancestors' versions. In the future, we should store the relevant TableVersionPaths in the database, so
+        #     that we don't need to rely on timestamps (which might be nondeterministic in distributed execution
+        #     scenarios).
+
+        assert Env.get().conn is not None
+
+        # Build the list of ancestor versions, starting with the given table and traversing back to the base table.
+        # For each proper ancestor,
+        # - If it's an ancestor with a fixed version (view-over-snapshot scenario), we keep the given fixed version.
+        # - If it's an ancestor with a live (floating) version, we use the version whose created_at timestamp equals
+        #     or most nearly precedes the given TableVersion's created_at timestamp.
+        ancestors: list[tuple[UUID, int]] = [(tbl_id, version)]
+        for ancestor_id, ancestor_version in ancestors_of_live_tbl:
+            if ancestor_version is not None:
+                # fixed version; just use it
+                ancestors.append((UUID(ancestor_id), ancestor_version))
+                continue
+
+            q = (
+                sql.select(schema.TableVersion)
+                .where(schema.TableVersion.tbl_id == ancestor_id)
+                .where(schema.TableVersion.md['created_at'].cast(sql.Float) <= created_at)
+                .order_by(schema.TableVersion.md['created_at'].cast(sql.Float).desc())
+                .limit(1)
+            )
+            row = Env.get().conn.execute(q).one_or_none()
+            if row is None:
+                # This can happen if an ancestor version is garbage collected; it can also happen in
+                # rare circumstances involving table versions created specifically with Pixeltable 0.4.3.
+                _logger.info(f'Ancestor {ancestor_id} not found for table {tbl_id}:{version}')
+                raise excs.Error('The specified table version is no longer valid and cannot be retrieved.')
+            ancestor_version_record = _unpack_row(row, [schema.TableVersion])[0]
+            ancestor_version_md = schema.md_from_dict(schema.VersionMd, ancestor_version_record.md)
+            assert ancestor_version_md.created_at <= created_at
+            ancestors.append((UUID(ancestor_id), ancestor_version_md.version))
+
+        # Force any ancestors to be loaded (base table first).
+        for anc_id, anc_version in ancestors[::-1]:
+            key = TableVersionKey(anc_id, anc_version, None)
+            if key not in self._tbl_versions:
+                _ = self._load_tbl_version(key)
+
+        # Now reconstruct the relevant TableVersionPath instance from the ancestor versions.
+        tvp: TableVersionPath | None = None
+        for anc_id, anc_version in ancestors[::-1]:
+            tvp = TableVersionPath(TableVersionHandle(TableVersionKey(anc_id, anc_version, None)), base=tvp)
+
+        return tvp
+
+    @retry_loop(for_write=False)
+    def collect_tbl_history(self, tbl_id: UUID, n: int | None) -> list[TableVersionMd]:
+        return self._collect_tbl_history(tbl_id, n)
+
+    def _collect_tbl_history(self, tbl_id: UUID, n: int | None) -> list[TableVersionMd]:
+        """
+        Returns the history of up to n versions of the table with the given UUID.
+
+        Args:
+            tbl_id: the UUID of the table to collect history for.
+            n: Optional limit on the maximum number of versions returned.
+
+        Returns:
+            A sequence of rows, ordered by version number
+            Each row contains a TableVersion and a TableSchemaVersion object.
+        """
+        q = (
+            sql.select(schema.Table, schema.TableVersion, schema.TableSchemaVersion)
+            .where(self._active_tbl_clause(tbl_id=tbl_id))
+            .join(schema.TableVersion)
+            .where(schema.TableVersion.tbl_id == tbl_id)
+            .join(schema.TableSchemaVersion)
+            .where(schema.TableSchemaVersion.tbl_id == tbl_id)
+            .where(
+                schema.TableVersion.md['schema_version'].cast(sql.Integer) == schema.TableSchemaVersion.schema_version
+            )
+            .order_by(schema.TableVersion.version.desc())
+        )
+        if n is not None:
+            q = q.limit(n)
+        src_rows = Env.get().session.execute(q).fetchall()
+        return [
+            TableVersionMd(
+                tbl_md=schema.md_from_dict(schema.TableMd, row.Table.md),
+                version_md=schema.md_from_dict(schema.VersionMd, row.TableVersion.md),
+                schema_version_md=schema.md_from_dict(schema.SchemaVersionMd, row.TableSchemaVersion.md),
+            )
+            for row in src_rows
+        ]
+
+    def head_version_md(self, tbl_id: UUID) -> schema.VersionMd | None:
+        """
+        Returns the TableVersionMd for the most recent non-fragment version of the given table.
+        """
+        conn = Env.get().conn
+
+        q = (
+            sql.select(schema.TableVersion.md)
+            .where(schema.TableVersion.tbl_id == tbl_id)
+            .where(schema.TableVersion.md['is_fragment'].astext == 'false')
+            .order_by(schema.TableVersion.md['version'].cast(sql.Integer).desc())
+            .limit(1)
+        )
+        row = conn.execute(q).one_or_none()
+        if row is None:
+            return None
+        assert isinstance(row[0], dict)
+        return schema.md_from_dict(schema.VersionMd, row[0])
+
+    def load_tbl_md(self, key: TableVersionKey) -> TableVersionMd:
         """
         Loads metadata from the store for a given table UUID and version.
         """
-        _logger.info(f'Loading metadata for table version: {tbl_id}:{effective_version}')
+        anchor_timestamp: float | None = None
+        if key.anchor_tbl_id is not None:
+            anchored_version_md = self.head_version_md(key.anchor_tbl_id)
+            # `anchor_tbl_id` must exist and have at least one non-fragment version, or else this isn't
+            # a valid TableVersion specification.
+            assert anchored_version_md is not None
+            anchor_timestamp = anchored_version_md.created_at
+
+        # _logger.info(f'Loading metadata for table version: {tbl_id}:{effective_version}')
         conn = Env.get().conn
 
         q = (
             sql.select(schema.Table, schema.TableVersion, schema.TableSchemaVersion)
             .select_from(schema.Table)
-            .where(schema.Table.id == tbl_id)
+            .where(schema.Table.id == key.tbl_id)
             .join(schema.TableVersion)
-            .where(schema.TableVersion.tbl_id == tbl_id)
+            .where(schema.TableVersion.tbl_id == key.tbl_id)
             .join(schema.TableSchemaVersion)
-            .where(schema.TableSchemaVersion.tbl_id == tbl_id)
+            .where(schema.TableSchemaVersion.tbl_id == key.tbl_id)
         )
 
-        if effective_version is not None:
+        if key.effective_version is not None:
             # we are loading a specific version
             # SELECT *
             # FROM Table t
             # JOIN TableVersion tv ON (tv.tbl_id = tbl_id AND tv.version = effective_version)
             # JOIN TableSchemaVersion tsv ON (tsv.tbl_id = tbl_id AND tv.md.schema_version = tsv.schema_version)
             # WHERE t.id = tbl_id
-            q = q.where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {effective_version}")).where(
-                sql.text(
-                    (
-                        f"({schema.TableVersion.__table__}.md->>'schema_version')::int = "
-                        f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
-                    )
+            q = q.where(
+                schema.TableVersion.md['version'].cast(sql.Integer) == key.effective_version,
+                schema.TableVersion.md['schema_version'].cast(sql.Integer) == schema.TableSchemaVersion.schema_version,
+            )
+        elif anchor_timestamp is not None:
+            # we are loading the version that is anchored to the head version of another table (see TableVersion
+            # docstring for details)
+            # SELECT *
+            # FROM Table t
+            # JOIN TableVersion tv ON (tv.tbl_id = tbl_id)
+            # JOIN TableSchemaVersion tsv ON (tsv.tbl_id = tbl_id AND tv.md.schema_version = tsv.schema_version)
+            # WHERE t.id = tbl_id AND tv.md.created_at <= anchor_timestamp
+            # ORDER BY tv.md.created_at DESC
+            # LIMIT 1
+            q = (
+                q.where(
+                    schema.TableVersion.md['created_at'].cast(sql.Float) <= anchor_timestamp,
+                    schema.TableVersion.md['schema_version'].cast(sql.Integer)
+                    == schema.TableSchemaVersion.schema_version,
                 )
+                .order_by(schema.TableVersion.md['created_at'].cast(sql.Float).desc())
+                .limit(1)
             )
         else:
             # we are loading the current version
@@ -876,97 +2086,184 @@ class Catalog:
             # JOIN TableSchemaVersion tsv ON (tsv.tbl_id = tbl_id AND t.current_schema_version = tsv.schema_version)
             # WHERE t.id = tbl_id
             q = q.where(
-                sql.text(
-                    f"({schema.Table.__table__}.md->>'current_version')::int = "
-                    f'{schema.TableVersion.__table__}.{schema.TableVersion.version.name}'
-                )
-            ).where(
-                sql.text(
-                    (
-                        f"({schema.Table.__table__}.md->>'current_schema_version')::int = "
-                        f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
-                    )
-                )
+                schema.Table.md['current_version'].cast(sql.Integer) == schema.TableVersion.version,
+                schema.Table.md['current_schema_version'].cast(sql.Integer) == schema.TableSchemaVersion.schema_version,
             )
 
         row = conn.execute(q).one_or_none()
-        assert row is not None, f'Table record not found: {tbl_id}:{effective_version}'
+        if row is None:
+            raise excs.Error(self._dropped_tbl_error_msg(key.tbl_id))
         tbl_record, version_record, schema_version_record = _unpack_row(
             row, [schema.Table, schema.TableVersion, schema.TableSchemaVersion]
         )
-        assert tbl_record.id == tbl_id
+        assert tbl_record.id == key.tbl_id
         tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
-        version_md = schema.md_from_dict(schema.TableVersionMd, version_record.md)
-        schema_version_md = schema.md_from_dict(schema.TableSchemaVersionMd, schema_version_record.md)
+        version_md = schema.md_from_dict(schema.VersionMd, version_record.md)
+        schema_version_md = schema.md_from_dict(schema.SchemaVersionMd, schema_version_record.md)
 
-        return schema.FullTableMd(tbl_md, version_md, schema_version_md)
+        return TableVersionMd(tbl_md, version_md, schema_version_md)
 
-    def store_tbl_md(
+    def write_tbl_md(
         self,
         tbl_id: UUID,
-        tbl_md: Optional[schema.TableMd],
-        version_md: Optional[schema.TableVersionMd],
-        schema_version_md: Optional[schema.TableSchemaVersionMd],
+        dir_id: UUID | None,
+        tbl_md: schema.TableMd | None,
+        version_md: schema.VersionMd | None,
+        schema_version_md: schema.SchemaVersionMd | None,
+        pending_ops: list[TableOp] | None = None,
+        remove_from_dir: bool = False,
     ) -> None:
         """
-        Stores metadata to the DB. If specified, `tbl_md` will be updated in place (only one such record can exist
-        per UUID); `version_md` and `schema_version_md` will be inserted as new records.
+        Stores metadata to the DB and adds tbl_id to self._roll_forward_ids if pending_ops is specified.
+
+        Args:
+            tbl_id: UUID of the table to store metadata for.
+            dir_id: If specified, the tbl_md will be added to the given directory; if None, the table must already exist
+            tbl_md: If specified, `tbl_md` will be inserted, or updated (only one such record can exist per UUID)
+            version_md: inserted as a new record if present
+            schema_version_md: will be inserted as a new record if present
 
         If inserting `version_md` or `schema_version_md` would be a primary key violation, an exception will be raised.
         """
-        conn = Env.get().conn
+        assert self._in_write_xact
+        assert version_md is None or version_md.created_at > 0.0
+        assert pending_ops is None or len(pending_ops) > 0
+        assert pending_ops is None or tbl_md is not None  # if we write pending ops, we must also write new tbl_md
+        session = Env.get().session
 
+        # Construct and insert or update table record if requested.
         if tbl_md is not None:
-            result = conn.execute(
-                sql.update(schema.Table.__table__)
-                .values({schema.Table.md: dataclasses.asdict(tbl_md)})
-                .where(schema.Table.id == tbl_id)
-            )
-            assert result.rowcount == 1, result.rowcount
+            assert tbl_md.tbl_id == str(tbl_id)
+            if version_md is not None:
+                assert tbl_md.current_version == version_md.version
+                assert tbl_md.current_schema_version == version_md.schema_version
+            if schema_version_md is not None:
+                assert tbl_md.current_schema_version == schema_version_md.schema_version
+            if pending_ops is not None:
+                assert tbl_md.pending_stmt is not None
+                assert all(op.tbl_id == str(tbl_id) for op in pending_ops)
+                assert all(op.op_sn == i for i, op in enumerate(pending_ops))
+                assert all(op.num_ops == len(pending_ops) for op in pending_ops)
+                tbl_md.tbl_state = schema.TableState.ROLLFORWARD
+                self._roll_forward_ids.add(tbl_id)
+
+            if dir_id is not None:
+                # We are inserting a record while creating a new table.
+                tbl_record = schema.Table(
+                    id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(tbl_md, dict_factory=md_dict_factory)
+                )
+                session.add(tbl_record)
+            else:
+                # Update the existing table record.
+                values: dict[Any, Any] = {schema.Table.md: dataclasses.asdict(tbl_md, dict_factory=md_dict_factory)}
+                if remove_from_dir:
+                    values.update({schema.Table.dir_id: None})
+                result = session.execute(
+                    sql.update(schema.Table.__table__).values(values).where(schema.Table.id == tbl_id)
+                )
+                assert isinstance(result, sql.CursorResult)
+                assert result.rowcount == 1, result.rowcount
 
+        # Construct and insert new table version record if requested.
         if version_md is not None:
-            conn.execute(
-                sql.insert(schema.TableVersion.__table__).values(
+            assert version_md.tbl_id == str(tbl_id)
+            if schema_version_md is not None:
+                assert version_md.schema_version == schema_version_md.schema_version
+            version_rows = (
+                session.query(schema.TableVersion)
+                .filter(schema.TableVersion.tbl_id == tbl_id, schema.TableVersion.version == version_md.version)
+                .all()
+            )
+            if len(version_rows) == 0:
+                # It's a new table version; insert a new record in the DB for it.
+                tbl_version_record = schema.TableVersion(
                     tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
                 )
-            )
+                session.add(tbl_version_record)
+            else:
+                # This table version already exists; update it.
+                assert len(version_rows) == 1  # must be unique
+                version_record = version_rows[0]
+                # Validate that the only fields that can change are 'is_fragment' and 'additional_md'.
+                assert version_record.md == dataclasses.asdict(
+                    dataclasses.replace(
+                        version_md,
+                        is_fragment=version_record.md['is_fragment'],
+                        additional_md=version_record.md['additional_md'],
+                    )
+                )
+                result = session.execute(
+                    sql.update(schema.TableVersion.__table__)
+                    .values({schema.TableVersion.md: dataclasses.asdict(version_md)})
+                    .where(schema.TableVersion.tbl_id == tbl_id, schema.TableVersion.version == version_md.version)
+                )
+                assert isinstance(result, sql.CursorResult)
+                assert result.rowcount == 1, result.rowcount
 
+        # Construct and insert a new schema version record if requested.
         if schema_version_md is not None:
-            conn.execute(
-                sql.insert(schema.TableSchemaVersion.__table__).values(
-                    tbl_id=tbl_id,
-                    schema_version=schema_version_md.schema_version,
-                    md=dataclasses.asdict(schema_version_md),
-                )
+            assert schema_version_md.tbl_id == str(tbl_id)
+            schema_version_record = schema.TableSchemaVersion(
+                tbl_id=tbl_id, schema_version=schema_version_md.schema_version, md=dataclasses.asdict(schema_version_md)
             )
+            session.add(schema_version_record)
+
+        # make sure we don't have any pending ops
+        assert session.query(schema.PendingTableOp).filter(schema.PendingTableOp.tbl_id == tbl_id).count() == 0
+
+        if pending_ops is not None:
+            for op in pending_ops:
+                op_record = schema.PendingTableOp(tbl_id=tbl_id, op_sn=op.op_sn, op=dataclasses.asdict(op))
+                session.add(op_record)
+
+        session.flush()  # Inform SQLAlchemy that we want to write these changes to the DB.
+
+    def store_update_status(self, tbl_id: UUID, version: int, status: UpdateStatus) -> None:
+        """Update the TableVersion.md.update_status field"""
+        assert self._in_write_xact
+        conn = Env.get().conn
+
+        stmt = (
+            sql.update(schema.TableVersion)
+            .where(schema.TableVersion.tbl_id == tbl_id, schema.TableVersion.version == version)
+            .values(md=schema.TableVersion.md.op('||')({'update_status': dataclasses.asdict(status)}))
+        )
+
+        res = conn.execute(stmt)
+        assert res.rowcount == 1, res.rowcount
 
     def delete_tbl_md(self, tbl_id: UUID) -> None:
         """
         Deletes all table metadata from the store for the given table UUID.
         """
         conn = Env.get().conn
+        _logger.info(f'delete_tbl_md({tbl_id})')
         conn.execute(sql.delete(schema.TableSchemaVersion.__table__).where(schema.TableSchemaVersion.tbl_id == tbl_id))
         conn.execute(sql.delete(schema.TableVersion.__table__).where(schema.TableVersion.tbl_id == tbl_id))
+        conn.execute(sql.delete(schema.PendingTableOp.__table__).where(schema.PendingTableOp.tbl_id == tbl_id))
         conn.execute(sql.delete(schema.Table.__table__).where(schema.Table.id == tbl_id))
 
-    def load_replica_md(self, tbl: Table) -> list[schema.FullTableMd]:
+    def load_replica_md(self, tbl: Table) -> list[TableVersionMd]:
         """
         Load metadata for the given table along with all its ancestors. The values of TableMd.current_version and
         TableMd.current_schema_version will be adjusted to ensure that the metadata represent a valid (internally
         consistent) table state.
         """
         # TODO: First acquire X-locks for all relevant metadata entries
+        # TODO: handle concurrent drop()
 
         # Load metadata for every table in the TableVersionPath for `tbl`.
-        md = [self.load_tbl_md(tv.id, tv.effective_version) for tv in tbl._tbl_version_path.get_tbl_versions()]
+        md = [self.load_tbl_md(tv.key) for tv in tbl._tbl_version_path.get_tbl_versions()]
 
         # If `tbl` is a named pure snapshot, we're not quite done, since the snapshot metadata won't appear in the
         # TableVersionPath. We need to prepend it separately.
-        if tbl._id != tbl._tbl_version.id:
-            snapshot_md = self.load_tbl_md(tbl._id, 0)
+        if isinstance(tbl, View) and tbl._is_named_pure_snapshot():
+            snapshot_md = self.load_tbl_md(TableVersionKey(tbl._id, 0, None))
             md = [snapshot_md, *md]
 
-        for ancestor_md in md[1:]:
+        for ancestor_md in md:
+            # Set the `is_replica` flag on every ancestor's TableMd.
+            ancestor_md.tbl_md.is_replica = True
             # For replica metadata, we guarantee that the current_version and current_schema_version of TableMd
             # match the corresponding values in TableVersionMd and TableSchemaVersionMd. This is to ensure that,
             # when the metadata is later stored in the catalog of a different Pixeltable instance, the values of
@@ -975,53 +2272,87 @@ class Catalog:
             ancestor_md.tbl_md.current_version = ancestor_md.version_md.version
             ancestor_md.tbl_md.current_schema_version = ancestor_md.schema_version_md.schema_version
 
+        for ancestor_md in md[1:]:
+            # Also, the table version of every proper ancestor is emphemeral; it does not represent a queryable
+            # table version (the data might be incomplete, since we have only retrieved one of its views, not
+            # the table itself).
+            ancestor_md.version_md.is_fragment = True
+
         return md
 
-    def _load_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
-        tbl_md, _, schema_version_md = self.load_tbl_md(tbl_id, effective_version)
+    def _load_tbl_version(self, key: TableVersionKey, *, check_pending_ops: bool = True) -> TableVersion | None:
+        """Creates TableVersion instance from stored metadata and registers it in _tbl_versions."""
+        tv_md = self.load_tbl_md(key)
+        tbl_md = tv_md.tbl_md
+        version_md = tv_md.version_md
+        schema_version_md = tv_md.schema_version_md
         view_md = tbl_md.view_md
 
-        _logger.info(f'Loading table version: {tbl_id}:{effective_version}')
         conn = Env.get().conn
 
-        # load mutable view ids
-        q = sql.select(schema.Table.id).where(
-            sql.text(
-                f"md->'view_md'->'base_versions'->0->>0 = {tbl_id.hex!r} "
-                "AND md->'view_md'->'base_versions'->0->1 IS NULL"
+        if check_pending_ops:
+            # if we care about pending ops, we also care whether the table is in the process of getting dropped
+            if tbl_md.pending_stmt == schema.TableStatement.DROP_TABLE:
+                raise excs.Error(self._dropped_tbl_error_msg(key.tbl_id))
+
+            pending_ops_q = (
+                sql.select(sql.func.count())
+                .select_from(schema.Table)
+                .join(schema.PendingTableOp)
+                .where(schema.PendingTableOp.tbl_id == key.tbl_id)
+                .where(schema.Table.id == key.tbl_id)
             )
-        )
-        mutable_view_ids = [r[0] for r in conn.execute(q).all()]
-        mutable_views = [TableVersionHandle(id, None) for id in mutable_view_ids]
+            if key.effective_version is not None:
+                # we only care about pending ops if the requested version is the current version
+                pending_ops_q = pending_ops_q.where(
+                    sql.text(f"({schema.Table.__table__}.md->>'current_version')::int = {key.effective_version}")
+                )
+            has_pending_ops = conn.execute(pending_ops_q).scalar() > 0
+            if has_pending_ops:
+                raise PendingTableOpsError(key.tbl_id)
+
+        # load mutable view ids for mutable TableVersions
+        mutable_view_ids: list[UUID] = []
+        if key.effective_version is None and key.anchor_tbl_id is None and not tbl_md.is_replica:
+            q = (
+                sql.select(schema.Table.id)
+                .where(schema.Table.md['view_md']['base_versions'][0][0].astext == key.tbl_id.hex)
+                .where(schema.Table.md['view_md']['base_versions'][0][1].astext == None)
+            )
+            mutable_view_ids = [r[0] for r in conn.execute(q).all()]
 
+        mutable_views = [TableVersionHandle(TableVersionKey(id, None, None)) for id in mutable_view_ids]
+
+        tbl_version: TableVersion
         if view_md is None:
             # this is a base table
+            tbl_version = TableVersion(key, tbl_md, version_md, schema_version_md, mutable_views)
+        else:
+            assert len(view_md.base_versions) > 0  # a view needs to have a base
+            assert (
+                not tv_md.is_pure_snapshot
+            )  # a pure snapshot doesn't have a physical table backing it, no point in loading it
+
+            base: TableVersionHandle
+            base_path: TableVersionPath | None = None  # needed for live view
+            if view_md.is_snapshot:
+                base = TableVersionHandle(
+                    TableVersionKey(UUID(view_md.base_versions[0][0]), view_md.base_versions[0][1], key.anchor_tbl_id)
+                )
+            else:
+                base_path = TableVersionPath.from_md(tbl_md.view_md.base_versions)
+                base = base_path.tbl_version
+
             tbl_version = TableVersion(
-                tbl_id, tbl_md, effective_version, schema_version_md, mutable_views=mutable_views
+                key, tbl_md, version_md, schema_version_md, mutable_views, base_path=base_path, base=base
             )
-            return tbl_version
 
-        assert len(view_md.base_versions) > 0  # a view needs to have a base
-        pure_snapshot = view_md.is_snapshot and view_md.predicate is None and len(schema_version_md.columns) == 0
-        assert not pure_snapshot  # a pure snapshot doesn't have a physical table backing it, no point in loading it
-
-        base: TableVersionHandle
-        base_path: Optional[TableVersionPath] = None  # needed for live view
-        if view_md.is_snapshot:
-            base = TableVersionHandle(UUID(view_md.base_versions[0][0]), view_md.base_versions[0][1])
-        else:
-            base_path = TableVersionPath.from_md(tbl_md.view_md.base_versions)
-            base = base_path.tbl_version
-
-        tbl_version = TableVersion(
-            tbl_id,
-            tbl_md,
-            effective_version,
-            schema_version_md,
-            base_path=base_path,
-            base=base,
-            mutable_views=mutable_views,
-        )
+        # register the instance before init()
+        self._tbl_versions[key] = tbl_version
+        # register this instance as modified, so that it gets purged if the transaction fails, it may not be
+        # fully initialized
+        self.mark_modified_tvs(tbl_version.handle)
+        tbl_version.init()
         return tbl_version
 
     def _init_store(self) -> None:
@@ -1029,7 +2360,7 @@ class Catalog:
         self.create_user(None)
         _logger.info('Initialized catalog.')
 
-    def create_user(self, user: Optional[str]) -> None:
+    def create_user(self, user: str | None) -> None:
         """
         Creates a catalog record (root directory) for the specified user, if one does not already exist.
         """
@@ -1047,19 +2378,31 @@ class Catalog:
             _logger.info(f'Added root directory record for user: {user!r}')
 
     def _handle_path_collision(
-        self, path: Path, expected_obj_type: type[SchemaObject], expected_snapshot: bool, if_exists: IfExistsParam
-    ) -> Optional[SchemaObject]:
+        self,
+        path: Path,
+        expected_obj_type: type[SchemaObject],
+        expected_snapshot: bool,
+        if_exists: IfExistsParam,
+        *,
+        base: TableVersionPath | None = None,
+    ) -> SchemaObject | None:
         obj, _, _ = self._prepare_dir_op(add_dir_path=path.parent, add_name=path.name)
 
         if if_exists == IfExistsParam.ERROR and obj is not None:
-            raise excs.Error(f'Path {str(path)!r} is an existing {type(obj)._display_name()}')
+            raise excs.Error(f'Path {path!r} is an existing {obj._display_name()}')
         else:
             is_snapshot = isinstance(obj, View) and obj._tbl_version_path.is_snapshot()
             if obj is not None and (not isinstance(obj, expected_obj_type) or (expected_snapshot and not is_snapshot)):
-                obj_type_str = 'snapshot' if expected_snapshot else expected_obj_type._display_name()
+                if expected_obj_type is Dir:
+                    obj_type_str = 'directory'
+                elif expected_obj_type is InsertableTable:
+                    obj_type_str = 'table'
+                elif expected_obj_type is View:
+                    obj_type_str = 'snapshot' if expected_snapshot else 'view'
+                else:
+                    raise AssertionError()
                 raise excs.Error(
-                    f'Path {str(path)!r} already exists but is not a {obj_type_str}. '
-                    f'Cannot {if_exists.name.lower()} it.'
+                    f'Path {path!r} already exists but is not a {obj_type_str}. Cannot {if_exists.name.lower()} it.'
                 )
 
         if obj is None:
@@ -1067,12 +2410,22 @@ class Catalog:
         if if_exists == IfExistsParam.IGNORE:
             return obj
 
+        assert if_exists in (IfExistsParam.REPLACE, IfExistsParam.REPLACE_FORCE)
+
+        # Check for circularity
+        if obj is not None and base is not None:
+            assert isinstance(obj, Table)  # or else it would have been caught above
+            if obj._id in tuple(version.id for version in base.get_tbl_versions()):
+                raise excs.Error(
+                    "Cannot use if_exists='replace' with the same name as one of the view's own ancestors."
+                )
+
         # drop the existing schema object
         if isinstance(obj, Dir):
             dir_contents = self._get_dir_contents(obj._id)
             if len(dir_contents) > 0 and if_exists == IfExistsParam.REPLACE:
                 raise excs.Error(
-                    f'Directory {str(path)!r} already exists and is not empty. '
+                    f'Directory {path!r} already exists and is not empty. '
                     'Use `if_exists="replace_force"` to replace it.'
                 )
             self._drop_dir(obj._id, path, force=True)