pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.20__py3-none-any.whl

pixeltable/catalog/catalog.py

@@ -5,41 +5,45 @@ import functools
 import logging
 import random
 import time
+from collections import defaultdict
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, TypeVar
+from typing import TYPE_CHECKING, Any, Callable, Iterator, TypeVar
 from uuid import UUID
 
 import psycopg
 import sqlalchemy as sql
+import sqlalchemy.exc as sql_exc
 
 from pixeltable import exceptions as excs
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 from pixeltable.metadata import schema
+from pixeltable.utils.exception_handler import run_cleanup
 
-if TYPE_CHECKING:
-    from pixeltable.plan import SampleClause
+from .column import Column
 from .dir import Dir
-from .globals import IfExistsParam, IfNotExistsParam, MediaValidation
+from .globals import IfExistsParam, IfNotExistsParam, MediaValidation, QColumnId
 from .insertable_table import InsertableTable
 from .path import Path
 from .schema_object import SchemaObject
 from .table import Table
-from .table_version import TableVersion
+from .table_version import TableVersion, TableVersionCompleteMd
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
+from .tbl_ops import TableOp
+from .update_status import UpdateStatus
 from .view import View
 
 if TYPE_CHECKING:
-    from .. import DataFrame, exprs
+    from pixeltable.plan import SampleClause
+
+    from .. import exprs
 
 
 _logger = logging.getLogger('pixeltable')
 
 
-def _unpack_row(
-    row: Optional[sql.engine.Row], entities: list[type[sql.orm.decl_api.DeclarativeBase]]
-) -> Optional[list[Any]]:
+def _unpack_row(row: sql.engine.Row | None, entities: list[type[sql.orm.decl_api.DeclarativeBase]]) -> list[Any] | None:
     """Convert a Row result into a list of entity instances.
 
     Assumes that the query contains a select() of exactly those entities.
@@ -60,46 +64,92 @@ def _unpack_row(
     return result
 
 
+# -1: unlimited
 # for now, we don't limit the number of retries, because we haven't seen situations where the actual number of retries
 # grows uncontrollably
-_MAX_RETRIES = 0
+_MAX_RETRIES = -1
 
 T = TypeVar('T')
 
 
-def _retry_loop(*, for_write: bool) -> Callable[[Callable[..., T]], Callable[..., T]]:
+def retry_loop(
+    *, tbl: TableVersionPath | None = None, for_write: bool, lock_mutable_tree: bool = False
+) -> Callable[[Callable[..., T]], Callable[..., T]]:
     def decorator(op: Callable[..., T]) -> Callable[..., T]:
         @functools.wraps(op)
         def loop(*args: Any, **kwargs: Any) -> T:
-            num_remaining_retries = _MAX_RETRIES
+            cat = Catalog.get()
+            # retry_loop() is reentrant
+            if cat._in_retry_loop:
+                return op(*args, **kwargs)
+
+            num_retries = 0
             while True:
+                cat._in_retry_loop = True
                 try:
                     # in order for retry to work, we need to make sure that there aren't any prior db updates
                     # that are part of an ongoing transaction
                     assert not Env.get().in_xact
-                    with Catalog.get().begin_xact(for_write=for_write):
+                    with Catalog.get().begin_xact(
+                        tbl=tbl,
+                        for_write=for_write,
+                        convert_db_excs=False,
+                        lock_mutable_tree=lock_mutable_tree,
+                        finalize_pending_ops=True,
+                    ):
                         return op(*args, **kwargs)
-                except sql.exc.DBAPIError as e:
+                except PendingTableOpsError as e:
+                    Env.get().console_logger.debug(f'retry_loop(): finalizing pending ops for {e.tbl_id}')
+                    Catalog.get()._finalize_pending_ops(e.tbl_id)
+                except (sql_exc.DBAPIError, sql_exc.OperationalError) as e:
                     # TODO: what other exceptions should we be looking for?
-                    if isinstance(e.orig, psycopg.errors.SerializationFailure):
-                        if num_remaining_retries > 0:
-                            num_remaining_retries -= 1
-                            _logger.debug(f'Serialization failure, retrying ({num_remaining_retries} retries left)')
+                    if isinstance(
+                        # TODO: Investigate whether DeadlockDetected points to a bug in our locking protocol,
+                        #     which is supposed to be deadlock-free.
+                        e.orig,
+                        (
+                            psycopg.errors.SerializationFailure,
+                            psycopg.errors.LockNotAvailable,
+                            psycopg.errors.DeadlockDetected,
+                        ),
+                    ):
+                        if num_retries < _MAX_RETRIES or _MAX_RETRIES == -1:
+                            num_retries += 1
+                            _logger.debug(f'Retrying ({num_retries}) after {type(e.orig)}')
                             time.sleep(random.uniform(0.1, 0.5))
                         else:
                             raise excs.Error(f'Serialization retry limit ({_MAX_RETRIES}) exceeded') from e
                     else:
                         raise
+                except Exception as e:
+                    # for informational/debugging purposes
+                    _logger.debug(f'retry_loop(): passing along {e}')
+                    raise
+                finally:
+                    cat._in_retry_loop = False
 
         return loop
 
     return decorator
 
 
+class PendingTableOpsError(Exception):
+    tbl_id: UUID
+
+    def __init__(self, tbl_id: UUID) -> None:
+        self.tbl_id = tbl_id
+
+
 class Catalog:
     """The functional interface to getting access to catalog objects
 
-    All interface functions must be called in the context of a transaction, started with Catalog.begin_xact().
+    All interface functions must be called in the context of a transaction, started with Catalog.begin_xact() or
+    via retry_loop().
+
+    When calling functions that involve Table or TableVersion instances, the catalog needs to get a chance to finalize
+    pending ops against those tables. To that end,
+    - use begin_xact(tbl) or begin_xact(tbl_id) if only accessing a single table
+    - use retry_loop() when accessing multiple tables (eg, pxt.ls())
 
     Caching and invalidation of metadata:
     - Catalog caches TableVersion instances in order to avoid excessive metadata loading
@@ -109,22 +159,34 @@ class Catalog:
       duplicate references to that table in the From clause (ie, incorrect Cartesian products)
     - in order to allow multiple concurrent Python processes to perform updates (data and/or schema) against a shared
       Pixeltable instance, Catalog needs to reload metadata from the store when there are changes
-    - concurrent changes are detected by comparing TableVersion.version with the stored current version
-      (TableMd.current_version)
+    - concurrent changes are detected by comparing TableVersion.version/view_sn with the stored current version
+      (TableMd.current_version/view_sn)
     - cached live TableVersion instances (those with effective_version == None) are validated against the stored
       metadata on transaction boundaries; this is recorded in TableVersion.is_validated
     - metadata validation is only needed for live TableVersion instances (snapshot instances are immutable)
     """
 
-    _instance: Optional[Catalog] = None
+    _instance: Catalog | None = None
 
-    # key: [id, version]
+    # cached TableVersion instances; key: [id, version]
     # - mutable version of a table: version == None (even though TableVersion.version is set correctly)
     # - snapshot versions: records the version of the snapshot
-    _tbl_versions: dict[tuple[UUID, Optional[int]], TableVersion]
-    _tbls: dict[UUID, Table]
+    _tbl_versions: dict[tuple[UUID, int | None], TableVersion]
+    _tbls: dict[tuple[UUID, int | None], Table]
     _in_write_xact: bool  # True if we're in a write transaction
-    _x_locked_tbl_id: Optional[UUID]  # set if begin_xact() was asked to write-lock a table
+    _x_locked_tbl_ids: set[UUID]  # non-empty for write transactions
+    _modified_tvs: set[TableVersionHandle]  # TableVersion instances modified in the current transaction
+    _undo_actions: list[Callable[[], None]]
+    _in_retry_loop: bool
+
+    # cached column dependencies
+    # - key: table id, value: mapping from column id to its dependencies
+    # - only maintained for dependencies between non-snapshot table versions
+    # - can contain stale entries (stemming from invalidated TV instances)
+    _column_dependencies: dict[UUID, dict[QColumnId, set[QColumnId]]]
+
+    # column dependents are recomputed at the beginning of every write transaction and only reflect the locked tree
+    _column_dependents: dict[QColumnId, set[QColumnId]] | None
 
     @classmethod
     def get(cls) -> Catalog:
@@ -135,21 +197,27 @@ class Catalog:
     @classmethod
     def clear(cls) -> None:
         """Remove the instance. Used for testing."""
-        # invalidate all existing instances to force reloading of metadata
-        for tbl_version in cls._instance._tbl_versions.values():
-            # _logger.debug(
-            #     f'Invalidating table version {tbl_version.id}:{tbl_version.effective_version} ({id(tbl_version):x})'
-            # )
-            tbl_version.is_validated = False
+        if cls._instance is not None:
+            # invalidate all existing instances to force reloading of metadata
+            for tbl_version in cls._instance._tbl_versions.values():
+                tbl_version.is_validated = False
         cls._instance = None
 
     def __init__(self) -> None:
         self._tbl_versions = {}
         self._tbls = {}  # don't use a defaultdict here, it doesn't cooperate with the debugger
         self._in_write_xact = False
-        self._x_locked_tbl_id = None
+        self._x_locked_tbl_ids = set()
+        self._modified_tvs = set()
+        self._undo_actions = []
+        self._in_retry_loop = False
+        self._column_dependencies = {}
+        self._column_dependents = None
         self._init_store()
 
+    def _dropped_tbl_error_msg(self, tbl_id: UUID) -> str:
+        return f'Table was dropped (no record found for {tbl_id})'
+
     def validate(self) -> None:
         """Validate structural consistency of cached metadata"""
         for (tbl_id, effective_version), tbl_version in self._tbl_versions.items():
@@ -164,38 +232,73 @@ class Catalog:
                 f'snapshot_id={tbl_version.id} mutable_views={tbl_version.mutable_views}'
             )
 
-            if tbl_version.is_view and tbl_version.is_mutable:
+            if tbl_version.is_view and tbl_version.is_mutable and tbl_version.is_validated:
                 # make sure this mutable view is recorded in a mutable base
                 base = tbl_version.base
                 assert base is not None
                 if base.effective_version is None:
                     assert (base.id, None) in self._tbl_versions
-                    assert TableVersionHandle.create(tbl_version) in self._tbl_versions[base.id, None].mutable_views
+                    base_tv = self._tbl_versions[base.id, None]
+                    if not base_tv.is_validated:
+                        continue
+                    mutable_view_ids = ', '.join(str(tv.id) for tv in self._tbl_versions[base.id, None].mutable_views)
+                    mutable_view_names = ', '.join(
+                        tv._tbl_version.name
+                        for tv in self._tbl_versions[base.id, None].mutable_views
+                        if tv._tbl_version is not None
+                    )
+                    assert TableVersionHandle.create(tbl_version) in self._tbl_versions[base.id, None].mutable_views, (
+                        f'{tbl_version.name} ({tbl_version.id}) missing in {mutable_view_ids} ({mutable_view_names})'
+                    )
 
             if len(tbl_version.mutable_views) > 0:
                 # make sure we also loaded mutable view metadata, which is needed to detect column dependencies
                 for v in tbl_version.mutable_views:
                     assert v.effective_version is None, f'{v.id}:{v.effective_version}'
 
+    def mark_modified_tvs(self, *handle: TableVersionHandle) -> None:
+        """Record that the given TableVersion instances were modified in the current transaction"""
+        assert Env.get().in_xact
+        self._modified_tvs.update(handle)
+
     @contextmanager
-    def begin_xact(self, *, tbl_id: Optional[UUID] = None, for_write: bool = False) -> Iterator[sql.Connection]:
+    def begin_xact(
+        self,
+        *,
+        tbl: TableVersionPath | None = None,
+        tbl_id: UUID | None = None,
+        for_write: bool = False,
+        lock_mutable_tree: bool = False,
+        convert_db_excs: bool = True,
+        finalize_pending_ops: bool = True,
+    ) -> Iterator[sql.Connection]:
         """
         Return a context manager that yields a connection to the database. Idempotent.
 
         It is mandatory to call this method, not Env.begin_xact(), if the transaction accesses any table data
         or metadata.
 
-        Lock acquisition:
-        - x-locks Table records by updating Table.lock_dummy
+        If tbl != None, follows this locking protocol:
+        - validates/reloads the TableVersion instances of tbl's ancestors (in the hope that this reduces potential
+          SerializationErrors later on)
+        - if for_write == True, x-locks Table record (by updating Table.lock_dummy; see _acquire_tbl_lock())
+        - if for_write == False, validates TableVersion instance
+        - if lock_mutable_tree == True, also x-locks all mutable views of the table
         - this needs to be done in a retry loop, because Postgres can decide to abort the transaction
           (SerializationFailure, LockNotAvailable)
         - for that reason, we do all lock acquisition prior to doing any real work (eg, compute column values),
-          to minimize (maybe avoid altogether) loosing that work
+          to minimize the probability of losing that work due to a forced abort
+
+        If convert_db_excs == True, converts DBAPIErrors into excs.Errors.
         """
+        assert tbl is None or tbl_id is None  # at most one can be specified
         if Env.get().in_xact:
-            if tbl_id is not None and for_write:
-                # make sure that we requested the required table lock at the beginning of the transaction
-                assert tbl_id == self._x_locked_tbl_id, f'{tbl_id} != {self._x_locked_tbl_id}'
+            # make sure that we requested the required table lock at the beginning of the transaction
+            if for_write:
+                if tbl is not None:
+                    assert tbl.tbl_id in self._x_locked_tbl_ids, f'{tbl.tbl_id} not in {self._x_locked_tbl_ids}'
+                elif tbl_id is not None:
+                    assert tbl_id in self._x_locked_tbl_ids, f'{tbl_id} not in {self._x_locked_tbl_ids}'
             yield Env.get().conn
             return
 
@@ -207,32 +310,105 @@ class Catalog:
         # )
         # _logger.debug(f'begin_xact(): {tv_msg}')
         num_retries = 0
+        pending_ops_tbl_id: UUID | None = None
+        has_exc = False  # True if we exited the 'with ...begin_xact()' block with an exception
         while True:
-            try:
-                with Env.get().begin_xact() as conn:
-                    if tbl_id is not None and for_write:
-                        # X-lock Table record
-                        conn.execute(
-                            sql.select(schema.Table).where(schema.Table.id == tbl_id).with_for_update(nowait=True)
-                        )
-                        conn.execute(sql.update(schema.Table).values(lock_dummy=1).where(schema.Table.id == tbl_id))
-                        self._x_locked_tbl_id = tbl_id
+            if pending_ops_tbl_id is not None:
+                Env.get().console_logger.debug(f'begin_xact(): finalizing pending ops for {pending_ops_tbl_id}')
+                self._finalize_pending_ops(pending_ops_tbl_id)
+                pending_ops_tbl_id = None
 
-                    self._in_write_xact = for_write
+            try:
+                self._in_write_xact = for_write
+                self._x_locked_tbl_ids = set()
+                self._modified_tvs = set()
+                self._column_dependents = None
+                has_exc = False
+
+                assert not self._undo_actions
+                with Env.get().begin_xact(for_write=for_write) as conn:
+                    if tbl is not None or tbl_id is not None:
+                        try:
+                            target: TableVersionHandle | None = None
+                            if tbl is not None:
+                                if self._acquire_path_locks(
+                                    tbl=tbl,
+                                    for_write=for_write,
+                                    lock_mutable_tree=lock_mutable_tree,
+                                    check_pending_ops=finalize_pending_ops,
+                                ):
+                                    target = tbl.tbl_version
+                            else:
+                                target = self._acquire_tbl_lock(
+                                    tbl_id=tbl_id,
+                                    for_write=for_write,
+                                    lock_mutable_tree=lock_mutable_tree,
+                                    raise_if_not_exists=True,
+                                    check_pending_ops=finalize_pending_ops,
+                                )
+
+                            if target is None:
+                                # didn't get the write lock
+                                for_write = False
+                            elif for_write:
+                                # we know at this point that target is mutable because we got the X-lock
+                                if lock_mutable_tree and not target.is_snapshot:
+                                    self._x_locked_tbl_ids = self._get_mutable_tree(target.id)
+                                    self._compute_column_dependents(self._x_locked_tbl_ids)
+                                else:
+                                    self._x_locked_tbl_ids = {target.id}
+                                if _logger.isEnabledFor(logging.DEBUG):
+                                    # validate only when we don't see errors
+                                    self.validate()
+
+                        except PendingTableOpsError as e:
+                            has_exc = True
+                            if finalize_pending_ops:
+                                # we remember which table id to finalize
+                                pending_ops_tbl_id = e.tbl_id
+                            # raise to abort the transaction
+                            raise
+
+                        except (sql_exc.DBAPIError, sql_exc.OperationalError) as e:
+                            has_exc = True
+                            if isinstance(
+                                e.orig, (psycopg.errors.SerializationFailure, psycopg.errors.LockNotAvailable)
+                            ) and (num_retries < _MAX_RETRIES or _MAX_RETRIES == -1):
+                                num_retries += 1
+                                _logger.debug(f'Retrying ({num_retries}) after {type(e.orig)}')
+                                time.sleep(random.uniform(0.1, 0.5))
+                                assert not self._undo_actions  # We should not have any undo actions at this point
+                                continue
+                            else:
+                                raise
+
+                    assert not self._undo_actions
                     yield conn
                     return
-            except sql.exc.DBAPIError as e:
-                if isinstance(e.orig, (psycopg.errors.SerializationFailure, psycopg.errors.LockNotAvailable)) and (
-                    num_retries < _MAX_RETRIES or _MAX_RETRIES == 0
-                ):
-                    num_retries += 1
-                    _logger.debug(f'Retrying ({num_retries}) after {type(e.orig)}')
-                    time.sleep(random.uniform(0.1, 0.5))
+
+            except PendingTableOpsError:
+                has_exc = True
+                if pending_ops_tbl_id is not None:
+                    # the next iteration of the loop will deal with pending ops for this table id
+                    continue
                 else:
+                    # we got this exception after getting the initial table locks and therefore need to abort
                     raise
+
+            except (sql_exc.DBAPIError, sql_exc.OperationalError, sql_exc.InternalError) as e:
+                has_exc = True
+                self.convert_sql_exc(e, tbl_id, tbl.tbl_version if tbl is not None else None, convert_db_excs)
+                raise  # re-raise the error if it didn't convert to a pxt.Error
+
+            except (Exception, KeyboardInterrupt) as e:
+                has_exc = True
+                _logger.debug(f'Caught {e.__class__}')
+                raise
+
             finally:
                 self._in_write_xact = False
-                self._x_locked_tbl_id = None
+                self._x_locked_tbl_ids.clear()
+                self._column_dependents = None
 
                 # invalidate cached current TableVersion instances
                 for tv in self._tbl_versions.values():
@@ -240,20 +416,333 @@ class Catalog:
                         _logger.debug(f'invalidating table version {tv.id}:None (tv={id(tv):x})')
                         tv.is_validated = False
 
-                if _logger.isEnabledFor(logging.DEBUG):
-                    self.validate()
+                if has_exc:
+                    # Execute undo actions in reverse order (LIFO)
+                    for hook in reversed(self._undo_actions):
+                        run_cleanup(hook, raise_error=False)
+                    # purge all modified TableVersion instances; we can't guarantee they are still consistent with the
+                    # stored metadata
+                    for handle in self._modified_tvs:
+                        self._clear_tv_cache(handle.id, handle.effective_version)
+                    # Clear potentially corrupted cached metadata
+                    if tbl is not None:
+                        tbl.clear_cached_md()
+
+                self._undo_actions.clear()
+                self._modified_tvs.clear()
+
+    def register_undo_action(self, func: Callable[[], None]) -> Callable[[], None]:
+        """Registers a function to be called if the current transaction fails.
+
+        The function is called only if the current transaction fails due to an exception.
+
+        Rollback functions are called in reverse order of registration (LIFO).
+
+        The function should not raise exceptions; if it does, they are logged and ignored.
+        """
+        assert self.in_write_xact
+        self._undo_actions.append(func)
+        return func
+
+    def convert_sql_exc(
+        self,
+        e: sql_exc.StatementError,
+        tbl_id: UUID | None = None,
+        tbl: TableVersionHandle | None = None,
+        convert_db_excs: bool = True,
+    ) -> None:
+        # we got some db error during the actual operation (not just while trying to get locks on the metadata
+        # records); we convert these into pxt.Error exceptions if appropriate
+
+        # we always convert UndefinedTable exceptions (they can't be retried)
+        if isinstance(e.orig, psycopg.errors.UndefinedTable) and tbl is not None:
+            # the table got dropped in the middle of the operation
+            tbl_name = tbl.get().name
+            _logger.debug(f'Exception: undefined table {tbl_name!r}: Caught {type(e.orig)}: {e!r}')
+            raise excs.Error(f'Table was dropped: {tbl_name}') from None
+        elif (
+            # TODO: Investigate whether DeadlockDetected points to a bug in our locking protocol,
+            #     which is supposed to be deadlock-free.
+            isinstance(
+                e.orig,
+                (
+                    psycopg.errors.SerializationFailure,  # serialization error despite getting x-locks
+                    psycopg.errors.InFailedSqlTransaction,  # can happen after tx fails for another reason
+                    psycopg.errors.DuplicateColumn,  # if a different process added a column concurrently
+                    psycopg.errors.DeadlockDetected,  # locking protocol contention
+                ),
+            )
+            and convert_db_excs
+        ):
+            msg: str
+            if tbl is not None:
+                msg = f'{tbl.get().name} ({tbl.id})'
+            elif tbl_id is not None:
+                msg = f'{tbl_id}'
+            else:
+                msg = ''
+            _logger.debug(f'Exception: {e.orig.__class__}: {msg} ({e})')
+            # Suppress the underlying SQL exception unless DEBUG is enabled
+            raise_from = e if _logger.isEnabledFor(logging.DEBUG) else None
+            raise excs.Error(
+                'That Pixeltable operation could not be completed because it conflicted with another '
+                'operation that was run on a different process.\n'
+                'Please re-run the operation.'
+            ) from raise_from
 
     @property
     def in_write_xact(self) -> bool:
         return self._in_write_xact
 
-    def _acquire_dir_xlock(self, parent_id: Optional[UUID], dir_id: Optional[UUID], dir_name: Optional[str]) -> None:
+    def _acquire_path_locks(
+        self,
+        *,
+        tbl: TableVersionPath,
+        for_write: bool = False,
+        lock_mutable_tree: bool = False,
+        check_pending_ops: bool | None = None,
+    ) -> bool:
+        """
+        Path locking protocol:
+        - refresh cached TableVersions of ancestors (we need those even during inserts, for computed columns that
+          reference the base tables)
+        - refresh cached TableVersion of tbl or get X-lock, depending on for_write
+        - if lock_mutable_tree, also X-lock all mutable views of tbl
+
+        Raises Error if tbl doesn't exist.
+        Return False if the lock couldn't be acquired (X-lock on a non-mutable table), True otherwise.
+        """
+        path_handles = tbl.get_tbl_versions()
+        read_handles = path_handles[:0:-1] if for_write else path_handles[::-1]
+        for handle in read_handles:
+            # update cache
+            _ = self.get_tbl_version(handle.id, handle.effective_version, validate_initialized=True)
+        if not for_write:
+            return True  # nothing left to lock
+        handle = self._acquire_tbl_lock(
+            tbl_id=tbl.tbl_id,
+            for_write=True,
+            lock_mutable_tree=lock_mutable_tree,
+            raise_if_not_exists=True,
+            check_pending_ops=check_pending_ops,
+        )
+        # update cache
+        _ = self.get_tbl_version(path_handles[0].id, path_handles[0].effective_version, validate_initialized=True)
+        return handle is not None
+
+    def _acquire_tbl_lock(
+        self,
+        *,
+        for_write: bool,
+        tbl_id: UUID | None = None,
+        dir_id: UUID | None = None,
+        tbl_name: str | None = None,
+        lock_mutable_tree: bool = False,
+        raise_if_not_exists: bool = True,
+        check_pending_ops: bool | None = None,
+    ) -> TableVersionHandle | None:
+        """
+        For writes: force acquisition of an X-lock on a Table record via a blind update.
+
+        Either tbl_id or dir_id/tbl_name need to be specified.
+        Returns True if the table was locked, False if it was a snapshot or not found.
+        If lock_mutable_tree, recursively locks all mutable views of the table.
+
+        Returns a handle to what was locked, None if the lock couldn't be acquired (eg, X-lock on a non-mutable table).
+        """
+        assert (tbl_id is not None) != (dir_id is not None and tbl_name is not None)
+        assert (dir_id is None) == (tbl_name is None)
+        where_clause: sql.ColumnElement
+        if tbl_id is not None:
+            where_clause = schema.Table.id == tbl_id
+        else:
+            where_clause = sql.and_(schema.Table.dir_id == dir_id, schema.Table.md['name'].astext == tbl_name)
+            user = Env.get().user
+            if user is not None:
+                where_clause = sql.and_(where_clause, schema.Table.md['user'].astext == Env.get().user)
+
+        conn = Env.get().conn
+        q = sql.select(schema.Table).where(where_clause)
+        if for_write:
+            q = q.with_for_update(nowait=True)
+        row = conn.execute(q).one_or_none()
+        if row is None:
+            if raise_if_not_exists:
+                raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
+            return None  # nothing to lock
+        tbl_md = schema.md_from_dict(schema.TableMd, row.md)
+        if for_write and tbl_md.is_mutable:
+            conn.execute(sql.update(schema.Table).values(lock_dummy=1).where(where_clause))
+
+        if check_pending_ops:
+            # check for pending ops after getting table lock
+            pending_ops_q = sql.select(sql.func.count()).where(schema.PendingTableOp.tbl_id == row.id)
+            has_pending_ops = conn.execute(pending_ops_q).scalar() > 0
+            if has_pending_ops:
+                raise PendingTableOpsError(row.id)
+
+        if for_write and not tbl_md.is_mutable:
+            return None  # nothing to lock
+
+        effective_version = tbl_md.current_version if tbl_md.is_snapshot else None
+        if tbl_md.is_mutable and lock_mutable_tree:
+            # also lock mutable views
+            tv = self.get_tbl_version(tbl_id, effective_version, validate_initialized=True)
+            for view in tv.mutable_views:
+                self._acquire_tbl_lock(
+                    for_write=for_write,
+                    tbl_id=view.id,
+                    lock_mutable_tree=lock_mutable_tree,
+                    raise_if_not_exists=raise_if_not_exists,
+                    check_pending_ops=check_pending_ops,
+                )
+        return TableVersionHandle(tbl_id, effective_version)
+
+    def _finalize_pending_ops(self, tbl_id: UUID) -> None:
+        """Finalizes all pending ops for the given table."""
+        num_retries = 0
+        while True:
+            try:
+                tbl_version: int
+                op: TableOp | None = None
+                delete_next_op_stmt: sql.Delete
+                reset_has_pending_stmt: sql.Update
+                with self.begin_xact(
+                    tbl_id=tbl_id, for_write=True, convert_db_excs=False, finalize_pending_ops=False
+                ) as conn:
+                    q = (
+                        sql.select(schema.Table.md, schema.PendingTableOp)
+                        .select_from(schema.Table)
+                        .join(schema.PendingTableOp)
+                        .where(schema.Table.id == tbl_id)
+                        .where(schema.PendingTableOp.tbl_id == tbl_id)
+                        .order_by(schema.PendingTableOp.op_sn)
+                        .limit(1)
+                        .with_for_update()
+                    )
+                    row = conn.execute(q).one_or_none()
+                    if row is None:
+                        return
+                    view_md = row.md.get('view_md')
+                    is_snapshot = False if view_md is None else view_md.get('is_snapshot')
+                    assert is_snapshot is not None
+                    tbl_version = row.md.get('current_version') if is_snapshot else None
+                    op = schema.md_from_dict(TableOp, row.op)
+                    delete_next_op_stmt = sql.delete(schema.PendingTableOp).where(
+                        schema.PendingTableOp.tbl_id == tbl_id, schema.PendingTableOp.op_sn == row.op_sn
+                    )
+                    reset_has_pending_stmt = (
+                        sql.update(schema.Table)
+                        .where(schema.Table.id == tbl_id)
+                        .values(md=schema.Table.md.op('||')({'has_pending_ops': False}))
+                    )
+
+                    if op.needs_xact:
+                        tv = self.get_tbl_version(
+                            tbl_id, tbl_version, check_pending_ops=False, validate_initialized=True
+                        )
+                        tv.exec_op(op)
+                        conn.execute(delete_next_op_stmt)
+                        if op.op_sn == op.num_ops - 1:
+                            conn.execute(reset_has_pending_stmt)
+                        continue
+
+                # this op runs outside of a transaction
+                tv = self.get_tbl_version(tbl_id, tbl_version, check_pending_ops=False, validate_initialized=True)
+                tv.exec_op(op)
+                with self.begin_xact(
+                    tbl_id=tbl_id, for_write=True, convert_db_excs=False, finalize_pending_ops=False
+                ) as conn:
+                    conn.execute(delete_next_op_stmt)
+                    if op.op_sn == op.num_ops - 1:
+                        conn.execute(reset_has_pending_stmt)
+
+            except (sql_exc.DBAPIError, sql_exc.OperationalError) as e:
+                # TODO: why are we still seeing these here, instead of them getting taken care of by the retry
+                # logic of begin_xact()?
+                if isinstance(e.orig, (psycopg.errors.SerializationFailure, psycopg.errors.LockNotAvailable)):
+                    num_retries += 1
+                    log_msg: str
+                    if op is not None:
+                        log_msg = f'finalize_pending_ops(): retrying ({num_retries}) op {op!s} after {type(e.orig)}'
+                    else:
+                        log_msg = f'finalize_pending_ops(): retrying ({num_retries}) after {type(e.orig)}'
+                    Env.get().console_logger.debug(log_msg)
+                    time.sleep(random.uniform(0.1, 0.5))
+                    continue
+                else:
+                    raise
+            except Exception as e:
+                Env.get().console_logger.debug(f'finalize_pending_ops(): caught {e}')
+                raise
+
+            num_retries = 0
+
+    def _debug_str(self) -> str:
+        tv_str = '\n'.join(str(k) for k in self._tbl_versions)
+        tbl_str = '\n'.join(str(k) for k in self._tbls)
+        return f'tbl_versions:\n{tv_str}\ntbls:\n{tbl_str}'
+
+    def _get_mutable_tree(self, tbl_id: UUID) -> set[UUID]:
+        """Returns ids of all tables that form the tree of mutable views starting at tbl_id; includes the root."""
+        assert (tbl_id, None) in self._tbl_versions, (
+            f'({tbl_id}, None) not in {self._tbl_versions.keys()}\n{self._debug_str()}'
+        )
+        tv = self.get_tbl_version(tbl_id, None, validate_initialized=True)
+        result: set[UUID] = {tv.id}
+        for view in tv.mutable_views:
+            result.update(self._get_mutable_tree(view.id))
+        return result
+
+    def _compute_column_dependents(self, mutable_tree: set[UUID]) -> None:
+        """Populate self._column_dependents for all tables in mutable_tree"""
+        assert self._column_dependents is None
+        self._column_dependents = defaultdict(set)
+        for tbl_id in mutable_tree:
+            assert tbl_id in self._column_dependencies, (
+                f'{tbl_id} not in {self._column_dependencies.keys()}\n{self._debug_str()}'
+            )
+            for col, dependencies in self._column_dependencies[tbl_id].items():
+                for dependency in dependencies:
+                    if dependency.tbl_id not in mutable_tree:
+                        continue
+                    dependents = self._column_dependents[dependency]
+                    dependents.add(col)
+
+    def record_column_dependencies(self, tbl_version: TableVersion) -> None:
+        """Update self._column_dependencies. Only valid for mutable versions."""
+        from pixeltable.exprs import Expr
+
+        assert tbl_version.is_mutable
+        dependencies: dict[QColumnId, set[QColumnId]] = {}
+        for col in tbl_version.cols_by_id.values():
+            if col.value_expr_dict is None:
+                continue
+            dependencies[QColumnId(tbl_version.id, col.id)] = Expr.get_refd_column_ids(col.value_expr_dict)
+        self._column_dependencies[tbl_version.id] = dependencies
+
+    def get_column_dependents(self, tbl_id: UUID, col_id: int) -> set[Column]:
+        """Return all Columns that transitively depend on the given column."""
+        assert self._column_dependents is not None
+        dependents = self._column_dependents[QColumnId(tbl_id, col_id)]
+        result: set[Column] = set()
+        for dependent in dependents:
+            tv = self.get_tbl_version(dependent.tbl_id, None, validate_initialized=True)
+            col = tv.cols_by_id[dependent.col_id]
+            result.add(col)
+        return result
+
+    def _acquire_dir_xlock(
+        self, *, parent_id: UUID | None = None, dir_id: UUID | None = None, dir_name: str | None = None
+    ) -> None:
         """Force acquisition of an X-lock on a Dir record via a blind update.
 
         If dir_id is present, then all other conditions are ignored.
         Note that (parent_id==None) is a valid where condition.
         If dir_id is not specified, the user from the environment is added to the directory filters.
         """
+        assert (dir_name is None) != (dir_id is None)
+        assert not (parent_id is not None and dir_name is None)
         user = Env.get().user
         assert self._in_write_xact
         q = sql.update(schema.Dir).values(lock_dummy=1)
@@ -269,6 +758,7 @@ class Catalog:
 
     def get_dir_path(self, dir_id: UUID) -> Path:
         """Return path for directory with given id"""
+        assert isinstance(dir_id, UUID)
         conn = Env.get().conn
         names: list[str] = []
         while True:
@@ -279,15 +769,15 @@ class Catalog:
                 break
             names.insert(0, dir.md['name'])
             dir_id = dir.parent_id
-        return Path('.'.join(names), empty_is_valid=True, allow_system_paths=True)
+        return Path.parse('.'.join(names), allow_empty_path=True, allow_system_path=True)
 
     @dataclasses.dataclass
     class DirEntry:
-        dir: Optional[schema.Dir]
+        dir: schema.Dir | None
         dir_entries: dict[str, Catalog.DirEntry]
-        table: Optional[schema.Table]
+        table: schema.Table | None
 
-    @_retry_loop(for_write=False)
+    @retry_loop(for_write=False)
     def get_dir_contents(self, dir_path: Path, recursive: bool = False) -> dict[str, DirEntry]:
         dir = self._get_schema_object(dir_path, expected=Dir, raise_if_not_exists=True)
         return self._get_dir_contents(dir._id, recursive=recursive)
@@ -314,31 +804,37 @@ class Catalog:
 
         return result
 
-    @_retry_loop(for_write=True)
-    def move(self, path: Path, new_path: Path) -> None:
-        self._move(path, new_path)
+    @retry_loop(for_write=True)
+    def move(self, path: Path, new_path: Path, if_exists: IfExistsParam, if_not_exists: IfNotExistsParam) -> None:
+        self._move(path, new_path, if_exists, if_not_exists)
 
-    def _move(self, path: Path, new_path: Path) -> None:
-        _, dest_dir, src_obj = self._prepare_dir_op(
+    def _move(self, path: Path, new_path: Path, if_exists: IfExistsParam, if_not_exists: IfNotExistsParam) -> None:
+        dest_obj, dest_dir, src_obj = self._prepare_dir_op(
             add_dir_path=new_path.parent,
             add_name=new_path.name,
             drop_dir_path=path.parent,
             drop_name=path.name,
-            raise_if_exists=True,
-            raise_if_not_exists=True,
+            raise_if_exists=(if_exists == IfExistsParam.ERROR),
+            raise_if_not_exists=(if_not_exists == IfNotExistsParam.ERROR),
         )
-        src_obj._move(new_path.name, dest_dir._id)
+        assert dest_obj is None or if_exists == IfExistsParam.IGNORE
+        assert src_obj is not None or if_not_exists == IfNotExistsParam.IGNORE
+        if dest_obj is None and src_obj is not None:
+            # If dest_obj is not None, it means `if_exists='ignore'` and the destination already exists.
+            # If src_obj is None, it means `if_not_exists='ignore'` and the source doesn't exist.
+            # If dest_obj is None and src_obj is not None, then we can proceed with the move.
+            src_obj._move(new_path.name, dest_dir._id)
 
     def _prepare_dir_op(
         self,
-        add_dir_path: Optional[Path] = None,
-        add_name: Optional[str] = None,
-        drop_dir_path: Optional[Path] = None,
-        drop_name: Optional[str] = None,
-        drop_expected: Optional[type[SchemaObject]] = None,
+        add_dir_path: Path | None = None,
+        add_name: str | None = None,
+        drop_dir_path: Path | None = None,
+        drop_name: str | None = None,
+        drop_expected: type[SchemaObject] | None = None,
         raise_if_exists: bool = False,
         raise_if_not_exists: bool = False,
-    ) -> tuple[Optional[SchemaObject], Optional[SchemaObject], Optional[SchemaObject]]:
+    ) -> tuple[SchemaObject | None, Dir | None, SchemaObject | None]:
         """
         Validates paths and acquires locks needed for a directory operation, ie, add/drop/rename (add + drop) of a
         directory entry.
@@ -356,6 +852,7 @@ class Catalog:
         - if both add and drop (= two directories are involved), lock the directories in a pre-determined order
           (in this case, by name) in order to prevent deadlocks between concurrent directory modifications
         """
+        assert drop_expected in (None, Table, Dir), drop_expected
         assert (add_dir_path is None) == (add_name is None)
         assert (drop_dir_path is None) == (drop_name is None)
         dir_paths: set[Path] = set()
@@ -364,46 +861,50 @@ class Catalog:
         if drop_dir_path is not None:
             dir_paths.add(drop_dir_path)
 
-        add_dir: Optional[schema.Dir] = None
-        drop_dir: Optional[schema.Dir] = None
+        add_dir: schema.Dir | None = None
+        drop_dir: schema.Dir | None = None
         for p in sorted(dir_paths):
-            dir = self._get_dir(p, for_update=True)
+            dir = self._get_dir(p, lock_dir=True)
             if dir is None:
-                raise excs.Error(f'Directory {str(p)!r} does not exist.')
+                # Dir does not exist; raise an appropriate error.
+                if add_dir_path is not None or add_name is not None:
+                    raise excs.Error(f'Directory {p!r} does not exist. Create it first with:\npxt.create_dir({p!r})')
+                else:
+                    raise excs.Error(f'Directory {p!r} does not exist.')
             if p == add_dir_path:
                 add_dir = dir
             if p == drop_dir_path:
                 drop_dir = dir
 
-        add_obj: Optional[SchemaObject] = None
+        add_obj: SchemaObject | None = None
         if add_dir is not None:
-            add_obj = self._get_dir_entry(add_dir.id, add_name, for_update=True)
+            add_obj = self._get_dir_entry(add_dir.id, add_name, lock_entry=True)
             if add_obj is not None and raise_if_exists:
                 add_path = add_dir_path.append(add_name)
-                raise excs.Error(f'Path {str(add_path)!r} already exists.')
+                raise excs.Error(f'Path {add_path!r} already exists.')
 
-        drop_obj: Optional[SchemaObject] = None
+        drop_obj: SchemaObject | None = None
         if drop_dir is not None:
             drop_path = drop_dir_path.append(drop_name)
-            drop_obj = self._get_dir_entry(drop_dir.id, drop_name, for_update=True)
+            drop_obj = self._get_dir_entry(drop_dir.id, drop_name, lock_entry=True)
             if drop_obj is None and raise_if_not_exists:
-                raise excs.Error(f'Path {str(drop_path)!r} does not exist.')
+                raise excs.Error(f'Path {drop_path!r} does not exist.')
             if drop_obj is not None and drop_expected is not None and not isinstance(drop_obj, drop_expected):
-                raise excs.Error(
-                    f'{str(drop_path)!r} needs to be a {drop_expected._display_name()} '
-                    f'but is a {type(drop_obj)._display_name()}'
-                )
+                expected_name = 'table' if drop_expected is Table else 'directory'
+                raise excs.Error(f'{drop_path!r} needs to be a {expected_name} but is a {drop_obj._display_name()}')
 
         add_dir_obj = Dir(add_dir.id, add_dir.parent_id, add_dir.md['name']) if add_dir is not None else None
         return add_obj, add_dir_obj, drop_obj
 
-    def _get_dir_entry(self, dir_id: UUID, name: str, for_update: bool = False) -> Optional[SchemaObject]:
+    def _get_dir_entry(
+        self, dir_id: UUID, name: str, version: int | None = None, lock_entry: bool = False
+    ) -> SchemaObject | None:
         user = Env.get().user
         conn = Env.get().conn
 
         # check for subdirectory
-        if for_update:
-            self._acquire_dir_xlock(dir_id, None, name)
+        if lock_entry:
+            self._acquire_dir_xlock(parent_id=dir_id, dir_id=None, dir_name=name)
         q = sql.select(schema.Dir).where(
             schema.Dir.parent_id == dir_id, schema.Dir.md['name'].astext == name, schema.Dir.md['user'].astext == user
         )
@@ -417,228 +918,285 @@ class Catalog:
             return Dir(dir_record.id, dir_record.parent_id, name)
 
         # check for table
+        if lock_entry:
+            self._acquire_tbl_lock(for_write=True, dir_id=dir_id, raise_if_not_exists=False, tbl_name=name)
         q = sql.select(schema.Table.id).where(
             schema.Table.dir_id == dir_id,
             schema.Table.md['name'].astext == name,
             schema.Table.md['user'].astext == user,
         )
-        if for_update:
-            q = q.with_for_update()
-        tbl_id = conn.execute(q).scalar_one_or_none()
-        if tbl_id is not None:
-            if tbl_id not in self._tbls:
-                _ = self._load_tbl(tbl_id)
-            return self._tbls[tbl_id]
+        tbl_id = conn.execute(q).scalars().all()
+        assert len(tbl_id) <= 1, name
+        if len(tbl_id) == 1:
+            return self.get_table_by_id(tbl_id[0], version)
 
         return None
 
     def _get_schema_object(
         self,
         path: Path,
-        expected: Optional[type[SchemaObject]] = None,
+        expected: type[SchemaObject] | None = None,
         raise_if_exists: bool = False,
         raise_if_not_exists: bool = False,
-        for_update: bool = False,
-    ) -> Optional[SchemaObject]:
+        lock_parent: bool = False,
+        lock_obj: bool = False,
+    ) -> SchemaObject | None:
         """Return the schema object at the given path, or None if it doesn't exist.
 
         Raises Error if
-        - the parent directory doesn't exist'
+        - the parent directory doesn't exist
         - raise_if_exists is True and the path exists
         - raise_if_not_exists is True and the path does not exist
         - expected is not None and the existing object has a different type
         """
+        assert expected in (None, Table, Dir), expected
+
         if path.is_root:
             # the root dir
             if expected is not None and expected is not Dir:
-                raise excs.Error(
-                    f'{str(path)!r} needs to be a {expected._display_name()} but is a {Dir._display_name()}'
-                )
-            dir = self._get_dir(path, for_update=for_update)
+                raise excs.Error(f'{path!r} needs to be a table but is a dir')
+            dir = self._get_dir(path, lock_dir=lock_obj)
             if dir is None:
                 raise excs.Error(f'Unknown user: {Env.get().user}')
             return Dir(dir.id, dir.parent_id, dir.md['name'])
 
         parent_path = path.parent
-        parent_dir = self._get_dir(parent_path, for_update=False)
+        parent_dir = self._get_dir(parent_path, lock_dir=lock_parent)
         if parent_dir is None:
-            raise excs.Error(f'Directory {str(parent_path)!r} does not exist.')
-        obj = self._get_dir_entry(parent_dir.id, path.name, for_update=for_update)
+            raise excs.Error(f'Directory {parent_path!r} does not exist.')
+        obj = self._get_dir_entry(parent_dir.id, path.name, path.version, lock_entry=lock_obj)
 
         if obj is None and raise_if_not_exists:
-            raise excs.Error(f'Path {str(path)!r} does not exist.')
+            raise excs.Error(f'Path {path!r} does not exist.')
         elif obj is not None and raise_if_exists:
-            raise excs.Error(f'Path {str(path)!r} is an existing {type(obj)._display_name()}.')
+            raise excs.Error(f'Path {path!r} is an existing {obj._display_name()}.')
         elif obj is not None and expected is not None and not isinstance(obj, expected):
-            raise excs.Error(
-                f'{str(path)!r} needs to be a {expected._display_name()} but is a {type(obj)._display_name()}.'
-            )
+            expected_name = 'table' if expected is Table else 'directory'
+            raise excs.Error(f'{path!r} needs to be a {expected_name} but is a {obj._display_name()}.')
         return obj
 
-    def get_table_by_id(self, tbl_id: UUID) -> Optional[Table]:
-        if tbl_id not in self._tbls:
-            tbl = self._load_tbl(tbl_id)
-            if tbl is None:
-                return None
-            # if this is a mutable table, we also need to have its mutable views loaded, in order to track column
-            # dependencies
-            tbl_version = tbl._tbl_version.get()
-            if tbl_version.is_mutable:
-                for v in tbl_version.mutable_views:
-                    _ = self.get_table_by_id(v.id)
-        return self._tbls[tbl_id]
+    def get_table_by_id(self, tbl_id: UUID, version: int | None = None) -> Table | None:
+        """Must be executed inside a transaction. Might raise PendingTableOpsError."""
+        if (tbl_id, version) not in self._tbls:
+            if version is None:
+                return self._load_tbl(tbl_id)
+            else:
+                return self._load_tbl_at_version(tbl_id, version)
+        return self._tbls.get((tbl_id, version))
 
-    @_retry_loop(for_write=True)
     def create_table(
         self,
         path: Path,
         schema: dict[str, Any],
-        df: 'DataFrame',
         if_exists: IfExistsParam,
-        primary_key: Optional[list[str]],
+        primary_key: list[str] | None,
         num_retained_versions: int,
         comment: str,
         media_validation: MediaValidation,
-    ) -> Table:
-        existing = self._handle_path_collision(path, InsertableTable, False, if_exists)
-        if existing is not None:
-            assert isinstance(existing, Table)
-            return existing
+        create_default_idxs: bool,
+    ) -> tuple[Table, bool]:
+        """
+        Creates a new InsertableTable at the given path.
 
-        dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
-        assert dir is not None
+        If `if_exists == IfExistsParam.IGNORE` and a table `t` already exists at the given path, returns `t, False`.
 
-        tbl = InsertableTable._create(
-            dir._id,
-            path.name,
-            schema,
-            df,
-            primary_key=primary_key,
-            num_retained_versions=num_retained_versions,
-            comment=comment,
-            media_validation=media_validation,
-        )
-        self._tbls[tbl._id] = tbl
-        return tbl
+        Otherwise, creates a new table `t` and returns `t, True` (or raises an exception if the operation fails).
+        """
+
+        @retry_loop(for_write=True)
+        def create_fn() -> tuple[UUID, bool]:
+            existing = self._handle_path_collision(path, InsertableTable, False, if_exists)
+            if existing is not None:
+                assert isinstance(existing, Table)
+                return existing._id, False
+
+            dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
+            assert dir is not None
+
+            md, ops = InsertableTable._create(
+                path.name,
+                schema,
+                primary_key=primary_key,
+                num_retained_versions=num_retained_versions,
+                comment=comment,
+                media_validation=media_validation,
+                create_default_idxs=create_default_idxs,
+            )
+            tbl_id = UUID(md.tbl_md.tbl_id)
+            self.store_tbl_md(tbl_id, dir._id, md.tbl_md, md.version_md, md.schema_version_md, ops)
+            return tbl_id, True
+
+        tbl_id, is_created = create_fn()
+        # finalize pending ops
+        with self.begin_xact(tbl_id=tbl_id, for_write=True, finalize_pending_ops=True):
+            tbl = self.get_table_by_id(tbl_id)
+            _logger.info(f'Created table {tbl._name!r}, id={tbl._id}')
+            Env.get().console_logger.info(f'Created table {tbl._name!r}.')
+            return tbl, is_created
 
-    @_retry_loop(for_write=True)
     def create_view(
         self,
         path: Path,
         base: TableVersionPath,
-        select_list: Optional[list[tuple[exprs.Expr, Optional[str]]]],
-        where: Optional[exprs.Expr],
-        sample_clause: Optional['SampleClause'],
-        additional_columns: Optional[dict[str, Any]],
+        select_list: list[tuple[exprs.Expr, str | None]] | None,
+        where: exprs.Expr | None,
+        sample_clause: 'SampleClause' | None,
+        additional_columns: dict[str, Any] | None,
         is_snapshot: bool,
-        iterator: Optional[tuple[type[ComponentIterator], dict[str, Any]]],
+        create_default_idxs: bool,
+        iterator: tuple[type[ComponentIterator], dict[str, Any]] | None,
         num_retained_versions: int,
         comment: str,
         media_validation: MediaValidation,
         if_exists: IfExistsParam,
     ) -> Table:
-        from pixeltable.utils.filecache import FileCache
-
-        existing = self._handle_path_collision(path, View, is_snapshot, if_exists)
-        if existing is not None:
-            assert isinstance(existing, View)
-            return existing
+        @retry_loop(for_write=True)
+        def create_fn() -> UUID:
+            if not is_snapshot and base.is_mutable():
+                # this is a mutable view of a mutable base; X-lock the base and advance its view_sn before adding
+                # the view
+                self._acquire_tbl_lock(tbl_id=base.tbl_id, for_write=True)
+                base_tv = self.get_tbl_version(base.tbl_id, None, validate_initialized=True)
+                base_tv.tbl_md.view_sn += 1
+                result = Env.get().conn.execute(
+                    sql.update(schema.Table)
+                    .values({schema.Table.md: dataclasses.asdict(base_tv.tbl_md)})
+                    .where(schema.Table.id == base.tbl_id)
+                )
+                assert result.rowcount == 1, result.rowcount
 
-        dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
-        assert dir is not None
-        if iterator is None:
-            iterator_class, iterator_args = None, None
-        else:
-            iterator_class, iterator_args = iterator
-        view = View._create(
-            dir._id,
-            path.name,
-            base=base,
-            select_list=select_list,
-            additional_columns=additional_columns,
-            predicate=where,
-            sample_clause=sample_clause,
-            is_snapshot=is_snapshot,
-            iterator_cls=iterator_class,
-            iterator_args=iterator_args,
-            num_retained_versions=num_retained_versions,
-            comment=comment,
-            media_validation=media_validation,
-        )
-        FileCache.get().emit_eviction_warnings()
-        self._tbls[view._id] = view
-        return view
+            existing = self._handle_path_collision(path, View, is_snapshot, if_exists)
+            if existing is not None:
+                assert isinstance(existing, View)
+                return existing._id
 
-    @_retry_loop(for_write=True)
-    def create_replica(
-        self, path: Path, md: list[schema.FullTableMd], if_exists: IfExistsParam = IfExistsParam.ERROR
-    ) -> None:
+            dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
+            assert dir is not None
+            if iterator is None:
+                iterator_class, iterator_args = None, None
+            else:
+                iterator_class, iterator_args = iterator
+            md, ops = View._create(
+                dir._id,
+                path.name,
+                base=base,
+                select_list=select_list,
+                additional_columns=additional_columns,
+                predicate=where,
+                sample_clause=sample_clause,
+                is_snapshot=is_snapshot,
+                create_default_idxs=create_default_idxs,
+                iterator_cls=iterator_class,
+                iterator_args=iterator_args,
+                num_retained_versions=num_retained_versions,
+                comment=comment,
+                media_validation=media_validation,
+            )
+            tbl_id = UUID(md.tbl_md.tbl_id)
+            self.store_tbl_md(tbl_id, dir._id, md.tbl_md, md.version_md, md.schema_version_md, ops)
+            return tbl_id
+
+        view_id = create_fn()
+        if not is_snapshot and base.is_mutable():
+            # invalidate base's TableVersion instance, so that it gets reloaded with the new mutable view
+            self._clear_tv_cache(base.tbl_id, base.tbl_version.effective_version)
+            # base_tv = self.get_tbl_version(base.tbl_id, base.tbl_version.effective_version, validate_initialized=True)
+            # view_handle = TableVersionHandle(view_id, effective_version=None)
+            # base_tv.mutable_views.add(view_handle)
+
+        # finalize pending ops
+        with self.begin_xact(tbl_id=view_id, for_write=True, finalize_pending_ops=True):
+            return self.get_table_by_id(view_id)
+
+    def _clear_tv_cache(self, tbl_id: UUID, effective_version: int | None) -> None:
+        if (tbl_id, effective_version) in self._tbl_versions:
+            tv = self._tbl_versions[tbl_id, effective_version]
+            tv.is_validated = False
+            del self._tbl_versions[tbl_id, effective_version]
+
+    def create_replica(self, path: Path, md: list[TableVersionCompleteMd]) -> None:
         """
         Creates table, table_version, and table_schema_version records for a replica with the given metadata.
         The metadata should be presented in standard "ancestor order", with the table being replicated at
         list position 0 and the (root) base table at list position -1.
-
-        TODO: create_replica() also needs to create the store tables and populate them in order to make
-        replica creation atomic.
         """
+        assert self.in_write_xact
+
         tbl_id = UUID(md[0].tbl_md.tbl_id)
 
-        # First handle path collisions (if_exists='ignore' or 'replace' or etc).
-        existing = self._handle_path_collision(path, View, False, if_exists)
-        if existing is not None:
-            if existing._id != tbl_id:
-                raise excs.Error(
-                    f"An attempt was made to create a replica table at {path!r} with if_exists='ignore', "
-                    'but a different table already exists at that location.'
-                )
-            assert isinstance(existing, View)
-            return
+        existing = self._handle_path_collision(path, Table, False, if_exists=IfExistsParam.IGNORE)  # type: ignore[type-abstract]
+        if existing is not None and existing._id != tbl_id:
+            raise excs.Error(
+                f'An attempt was made to create a replica table at {path!r}, '
+                'but a different table already exists at that location.'
+            )
 
         # Ensure that the system directory exists.
-        self._create_dir(Path('_system', allow_system_paths=True), if_exists=IfExistsParam.IGNORE, parents=False)
+        self.__ensure_system_dir_exists()
 
         # Now check to see if this table already exists in the catalog.
-        existing = Catalog.get().get_table_by_id(tbl_id)
+        existing = self.get_table_by_id(tbl_id)
         if existing is not None:
-            existing_path = Path(existing._path(), allow_system_paths=True)
-            # It does exist. If it's a non-system table, that's an error: it's already been replicated.
-            if not existing_path.is_system_path:
+            existing_path = Path.parse(existing._path(), allow_system_path=True)
+            if existing_path != path and not existing_path.is_system_path:
+                # It does exist, under a different path from the specified one.
                 raise excs.Error(
-                    f'That table has already been replicated as {existing._path()!r}. \n'
+                    f'That table has already been replicated as {existing_path!r}.\n'
                     f'Drop the existing replica if you wish to re-create it.'
                 )
-            # If it's a system table, then this means it was created at some point as the ancestor of some other
-            # table (a snapshot-over-snapshot scenario). In that case, we simply move it to the new (named) location.
-            self._move(existing_path, path)
 
-        # Now store the metadata for this replica. In the case where the table already exists (and was just moved
-        # into a named location), this will be a no-op, but it still serves to validate that the newly received
-        # metadata is identical to what's in the catalog.
-        self.__store_replica_md(path, md[0])
-
-        # Now store the metadata for all of this table's proper ancestors. If one or more proper ancestors
+        # Now store the metadata for this replica's proper ancestors. If one or more proper ancestors
         # do not yet exist in the store, they will be created as anonymous system tables.
-        for ancestor_md in md[1:]:
+        # We instantiate the ancestors starting with the base table and ending with the immediate parent of the
+        # table being replicated.
+        for ancestor_md in md[:0:-1]:
             ancestor_id = UUID(ancestor_md.tbl_md.tbl_id)
-            replica = Catalog.get().get_table_by_id(ancestor_id)
+            replica = self.get_table_by_id(ancestor_id)
             replica_path: Path
             if replica is None:
                 # We've never seen this table before. Create a new anonymous system table for it.
-                replica_path = Path(f'_system.replica_{ancestor_id.hex}', allow_system_paths=True)
+                replica_path = Path.parse(f'_system.replica_{ancestor_id.hex}', allow_system_path=True)
             else:
                 # The table already exists in the catalog. The existing path might be a system path (if the table
                 # was created as an anonymous base table of some other table), or it might not (if it's a snapshot
                 # that was directly replicated by the user at some point). In either case, use the existing path.
-                replica_path = Path(replica._path(), allow_system_paths=True)
+                replica_path = Path.parse(replica._path(), allow_system_path=True)
 
-            # Store the metadata; it could be a new version (in which case a new record will be created) or a
-            # known version (in which case the newly received metadata will be validated as identical).
+            # Store the metadata; it could be a new version (in which case a new record will be created), or a known
+            # version (in which case the newly received metadata will be validated as identical).
+            # If it's a new version, this will result in a new TableVersion record being created.
             self.__store_replica_md(replica_path, ancestor_md)
 
-        # don't create TableVersion instances at this point, they would be superseded by calls to TV.create_replica()
-        # in TableRestorer.restore()
+            # Now we must clear cached metadata for the ancestor table, to force the next table operation to pick up
+            # the new TableVersion instance. This is necessary because computed columns of descendant tables might
+            # reference columns of the ancestor table that only exist in the new version.
+            replica = Catalog.get().get_table_by_id(ancestor_id)
+            # assert replica is not None  # If it didn't exist before, it must have been created by now.
+            if replica is not None:
+                replica._tbl_version_path.clear_cached_md()
+
+        # Store the metadata for the table being replicated; as before, it could be a new version or a known version.
+        # If it's a new version, then a TableVersion record will be created, unless the table being replicated
+        # is a pure snapshot.
+        self.__store_replica_md(path, md[0])
 
-    def __store_replica_md(self, path: Path, md: schema.FullTableMd) -> None:
+        # Finally, it's possible that the table already exists in the catalog, but as an anonymous system table that
+        # was hidden the last time we checked (and that just became visible when the replica was imported). In this
+        # case, we need to make the existing table visible by moving it to the specified path.
+        # We need to do this at the end, since `existing_path` needs to first have a non-fragment table version in
+        # order to be instantiated as a schema object.
+        existing = self.get_table_by_id(tbl_id)
+        assert existing is not None
+        existing_path = Path.parse(existing._path(), allow_system_path=True)
+        if existing_path != path:
+            assert existing_path.is_system_path
+            self._move(existing_path, path, IfExistsParam.ERROR, IfNotExistsParam.ERROR)
+
+    def __ensure_system_dir_exists(self) -> Dir:
+        system_path = Path.parse('_system', allow_system_path=True)
+        return self._create_dir(system_path, if_exists=IfExistsParam.IGNORE, parents=False)
+
+    def __store_replica_md(self, path: Path, md: TableVersionCompleteMd) -> None:
         _logger.info(f'Creating replica table at {path!r} with ID: {md.tbl_md.tbl_id}')
         dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
         assert dir is not None
@@ -647,9 +1205,10 @@ class Catalog:
         conn = Env.get().conn
         tbl_id = md.tbl_md.tbl_id
 
-        new_tbl_md: Optional[schema.TableMd] = None
-        new_version_md: Optional[schema.TableVersionMd] = None
-        new_schema_version_md: Optional[schema.TableSchemaVersionMd] = None
+        new_tbl_md: schema.TableMd | None = None
+        new_version_md: schema.TableVersionMd | None = None
+        new_schema_version_md: schema.TableSchemaVersionMd | None = None
+        is_new_tbl_version: bool = False
 
         # We need to ensure that the table metadata in the catalog always reflects the latest observed version of
         # this table. (In particular, if this is a base table, then its table metadata need to be consistent
@@ -673,35 +1232,45 @@ class Catalog:
                 # New metadata is more recent than the metadata currently stored in the DB; we'll update the record
                 # in place in the DB.
                 new_tbl_md = dataclasses.replace(md.tbl_md, name=path.name, user=Env.get().user, is_replica=True)
-
         # Now see if a TableVersion record already exists in the DB for this table version. If not, insert it. If
         # it already exists, check that the existing record is identical to the new one.
         q = (
             sql.select(schema.TableVersion.md)
             .where(schema.TableVersion.tbl_id == tbl_id)
-            .where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {md.version_md.version}"))
+            .where(schema.TableVersion.md['version'].cast(sql.Integer) == md.version_md.version)
         )
         existing_version_md_row = conn.execute(q).one_or_none()
         if existing_version_md_row is None:
             new_version_md = md.version_md
+            is_new_tbl_version = True
         else:
             existing_version_md = schema.md_from_dict(schema.TableVersionMd, existing_version_md_row.md)
-            if existing_version_md != md.version_md:
+            # Validate that the existing metadata are identical to the new metadata, except is_fragment
+            # and additional_md which may differ.
+            if (
+                dataclasses.replace(
+                    existing_version_md,
+                    is_fragment=md.version_md.is_fragment,
+                    additional_md=md.version_md.additional_md,
+                )
+                != md.version_md
+            ):
                 raise excs.Error(
                     f'The version metadata for the replica {path!r}:{md.version_md.version} is inconsistent with '
                     'the metadata recorded from a prior replica.\n'
                     'This is likely due to data corruption in the replicated table.'
                 )
+            if existing_version_md.is_fragment and not md.version_md.is_fragment:
+                # This version exists in the DB as a fragment, but we're importing a complete copy of the same version;
+                # set the is_fragment flag to False in the DB.
+                new_version_md = md.version_md
 
         # Do the same thing for TableSchemaVersion.
         q = (
             sql.select(schema.TableSchemaVersion.md)
             .where(schema.TableSchemaVersion.tbl_id == tbl_id)
             .where(
-                sql.text(
-                    f"({schema.TableSchemaVersion.__table__}.md->>'schema_version')::int = "
-                    f'{md.schema_version_md.schema_version}'
-                )
+                schema.TableSchemaVersion.md['schema_version'].cast(sql.Integer) == md.schema_version_md.schema_version
             )
         )
         existing_schema_version_md_row = conn.execute(q).one_or_none()
@@ -711,6 +1280,7 @@ class Catalog:
             existing_schema_version_md = schema.md_from_dict(
                 schema.TableSchemaVersionMd, existing_schema_version_md_row.md
             )
+            # Validate that the existing metadata are identical to the new metadata.
             if existing_schema_version_md != md.schema_version_md:
                 raise excs.Error(
                     f'The schema version metadata for the replica {path!r}:{md.schema_version_md.schema_version} '
@@ -718,75 +1288,167 @@ class Catalog:
                     'This is likely due to data corruption in the replicated table.'
                 )
 
-        self.store_tbl_md(UUID(tbl_id), new_tbl_md, new_version_md, new_schema_version_md)
+        self.store_tbl_md(UUID(tbl_id), None, new_tbl_md, new_version_md, new_schema_version_md)
 
-    @_retry_loop(for_write=False)
-    def get_table(self, path: Path) -> Table:
-        obj = self._get_table(path)
-        return obj
+        if is_new_tbl_version and not md.is_pure_snapshot:
+            # It's a new version of a table that has a physical store, so we need to create a TableVersion instance.
+            TableVersion.create_replica(md)
+
+    @retry_loop(for_write=False)
+    def get_table(self, path: Path, if_not_exists: IfNotExistsParam) -> Table | None:
+        obj = Catalog.get()._get_schema_object(
+            path, expected=Table, raise_if_not_exists=(if_not_exists == IfNotExistsParam.ERROR)
+        )
+        if obj is None:
+            _logger.info(f'Skipped table {path!r} (does not exist).')
+            return None
 
-    def _get_table(self, path: Path) -> Table:
-        obj = Catalog.get()._get_schema_object(path, expected=Table, raise_if_not_exists=True)
         assert isinstance(obj, Table)
-        tbl_version = obj._tbl_version.get()
-        # TODO: instead of calling this here, move the logic into TableVersion.init(), which is called after
-        # registering the instance in _tbl_versions
-        tbl_version.ensure_md_loaded()
-        # if this table has mutable views, we need to load those as well, in order to record column dependencies
-        for v in tbl_version.mutable_views:
-            self.get_table_by_id(v.id)
+        # We need to clear cached metadata from tbl_version_path, in case the schema has been changed
+        # by another process.
+        obj._tbl_version_path.clear_cached_md()
         return obj
 
-    @_retry_loop(for_write=True)
+    @retry_loop(for_write=True)
     def drop_table(self, path: Path, if_not_exists: IfNotExistsParam, force: bool) -> None:
-        _, _, src_obj = self._prepare_dir_op(
-            drop_dir_path=path.parent,
-            drop_name=path.name,
-            drop_expected=Table,
-            raise_if_not_exists=if_not_exists == IfNotExistsParam.ERROR and not force,
+        tbl = self._get_schema_object(
+            path,
+            expected=Table,
+            raise_if_not_exists=(if_not_exists == IfNotExistsParam.ERROR and not force),
+            lock_parent=True,
+            lock_obj=False,
         )
-        if src_obj is None:
-            _logger.info(f'Skipped table {str(path)!r} (does not exist).')
+        if tbl is None:
+            _logger.info(f'Skipped table {path!r} (does not exist).')
             return
-        assert isinstance(src_obj, Table)
-        self._drop_tbl(src_obj, force=force, is_replace=False)
+        assert isinstance(tbl, Table)
+
+        if isinstance(tbl, View) and tbl._tbl_version_path.is_mutable() and tbl._tbl_version_path.base.is_mutable():
+            # this is a mutable view of a mutable base;
+            # lock the base before the view, in order to avoid deadlocks with concurrent inserts/updates
+            base_id = tbl._tbl_version_path.base.tbl_id
+            self._acquire_tbl_lock(tbl_id=base_id, for_write=True, lock_mutable_tree=False)
 
-    def _drop_tbl(self, tbl: Table, force: bool, is_replace: bool) -> None:
+        self._drop_tbl(tbl, force=force, is_replace=False)
+
+    def _drop_tbl(self, tbl: Table | TableVersionPath, force: bool, is_replace: bool) -> None:
         """
         Drop the table (and recursively its views, if force == True).
 
+        `tbl` can be an instance of `Table` for a user table, or `TableVersionPath` for a hidden (system) table.
+
         Locking protocol:
         - X-lock base before X-locking any view
         - deadlock-free wrt to TableVersion.insert() (insert propagation also proceeds top-down)
         - X-locks parent dir prior to calling TableVersion.drop(): prevent concurrent creation of another SchemaObject
-          in the same directory with the same name (which could lead to duplicate names if we get rolled back)
+          in the same directory with the same name (which could lead to duplicate names if we get aborted)
         """
-        view_ids = self.get_view_ids(tbl._id, for_update=True)
+        is_pure_snapshot: bool
+        if isinstance(tbl, TableVersionPath):
+            tvp = tbl
+            tbl_id = tvp.tbl_id
+            tbl = None
+            is_pure_snapshot = False
+        else:
+            tvp = tbl._tbl_version_path
+            tbl_id = tbl._id
+            is_pure_snapshot = tbl._tbl_version is None
+
+        if tbl is not None:
+            self._acquire_dir_xlock(dir_id=tbl._dir_id)
+        self._acquire_tbl_lock(tbl_id=tbl_id, for_write=True, lock_mutable_tree=False)
+
+        view_ids = self.get_view_ids(tbl_id, for_update=True)
+        is_replica = tvp.is_replica()
+        do_drop = True
+
+        _logger.debug(f'Preparing to drop table {tbl_id} (force={force!r}, is_replica={is_replica}).')
+
         if len(view_ids) > 0:
-            if not force:
-                is_snapshot = tbl._tbl_version_path.is_snapshot()
-                obj_type_str = 'Snapshot' if is_snapshot else tbl._display_name().capitalize()
+            if force:
+                # recursively drop views first
+                for view_id in view_ids:
+                    view = self.get_table_by_id(view_id)
+                    self._drop_tbl(view, force=force, is_replace=is_replace)
+
+            elif is_replica:
+                # Dropping a replica with dependents and no 'force': just rename it to be a hidden table;
+                # the actual table will not be dropped.
+                assert tbl is not None  # can only occur for a user table
+                system_dir = self.__ensure_system_dir_exists()
+                new_name = f'replica_{tbl_id.hex}'
+                _logger.debug(f'{tbl._path()!r} is a replica with dependents; renaming to {new_name!r}.')
+                tbl._move(new_name, system_dir._id)
+                do_drop = False  # don't actually clear the catalog for this table
+
+            else:
+                # It has dependents but is not a replica and no 'force', so it's an error to drop it.
+                assert tbl is not None  # can only occur for a user table
                 msg: str
                 if is_replace:
                     msg = (
-                        f'{obj_type_str} {tbl._path()} already exists and has dependents. '
+                        f'{tbl._display_str()} already exists and has dependents. '
                         "Use `if_exists='replace_force'` to replace it."
                     )
                 else:
-                    msg = f'{obj_type_str} {tbl._path()} has dependents.'
+                    msg = f'{tbl._display_str()} has dependents.'
                 raise excs.Error(msg)
 
-            for view_id in view_ids:
-                view = self.get_table_by_id(view_id)
-                self._drop_tbl(view, force=force, is_replace=is_replace)
+        # if this is a mutable view of a mutable base, advance the base's view_sn
+        if isinstance(tbl, View) and tvp.is_mutable() and tvp.base.is_mutable():
+            base_id = tvp.base.tbl_id
+            base_tv = self.get_tbl_version(base_id, None, validate_initialized=True)
+            base_tv.tbl_md.view_sn += 1
+            self.mark_modified_tvs(base_tv.handle)
+            result = Env.get().conn.execute(
+                sql.update(schema.Table.__table__)
+                .values({schema.Table.md: dataclasses.asdict(base_tv.tbl_md)})
+                .where(schema.Table.id == base_id)
+            )
+            assert result.rowcount == 1, result.rowcount
 
-        _ = self.get_dir(tbl._dir_id, for_update=True)  # X-lock the parent directory
-        tbl._drop()
-        assert tbl._id in self._tbls
-        del self._tbls[tbl._id]
-        _logger.info(f'Dropped table `{tbl._path()}`.')
+        if do_drop:
+            if not is_pure_snapshot:
+                # invalidate the TableVersion instance when we're done so that existing references to it can find out it
+                # has been dropped
+                self.mark_modified_tvs(tvp.tbl_version)
+            tv = tvp.tbl_version.get() if tvp.tbl_version is not None else None
+            if not is_pure_snapshot:
+                # drop the store table before deleting the Table record
+                tv = tvp.tbl_version.get()
+                tv.drop()
+
+            self.delete_tbl_md(tbl_id)
+            tvp.clear_cached_md()
+
+        assert (
+            is_replica
+            or (tbl_id, None) in self._tbls  # non-replica tables must have an entry with effective_version=None
+        )
 
-    @_retry_loop(for_write=True)
+        # Remove visible Table references (we do this even for a replica that was just renamed).
+        versions = [version for id, version in self._tbls if id == tbl_id]
+        for version in versions:
+            del self._tbls[tbl_id, version]
+
+        _logger.info(f'Dropped table {tbl_id if tbl is None else repr(tbl._path())}.')
+
+        if (
+            is_replica  # if this is a replica,
+            and do_drop  # and it was actually dropped (not just renamed),
+            and tvp.base is not None  # and it has a base table,
+        ):
+            base_tbl = self.get_table_by_id(tvp.base.tbl_id)
+            base_tbl_path = None if base_tbl is None else Path.parse(base_tbl._path(), allow_system_path=True)
+            if (
+                (base_tbl_path is None or base_tbl_path.is_system_path)  # and the base table is hidden,
+                and len(self.get_view_ids(tvp.base.tbl_id, for_update=True)) == 0  # and has no other dependents,
+            ):
+                # then drop the base table as well (possibly recursively).
+                _logger.debug(f'Dropping hidden base table {tvp.base.tbl_id} of dropped replica {tbl_id}.')
+                self._drop_tbl(tvp.base, force=False, is_replace=False)
+
+    @retry_loop(for_write=True)
     def create_dir(self, path: Path, if_exists: IfExistsParam, parents: bool) -> Dir:
         return self._create_dir(path, if_exists, parents)
 
@@ -799,12 +1461,12 @@ class Catalog:
         # parent = self._get_schema_object(path.parent)
         # assert parent is not None
         # dir = Dir._create(parent._id, path.name)
-        # Env.get().console_logger.info(f'Created directory {str(path)!r}.')
+        # Env.get().console_logger.info(f'Created directory {path!r}.')
         # return dir
 
         if parents:
             # start walking down from the root
-            last_parent: Optional[SchemaObject] = None
+            last_parent: SchemaObject | None = None
             for ancestor in path.ancestors():
                 ancestor_obj = self._get_schema_object(ancestor, expected=Dir)
                 assert ancestor_obj is not None or last_parent is not None
@@ -818,10 +1480,10 @@ class Catalog:
             return existing
         assert parent is not None
         dir = Dir._create(parent._id, path.name)
-        Env.get().console_logger.info(f'Created directory {str(path)!r}.')
+        Env.get().console_logger.info(f'Created directory {path!r}.')
         return dir
 
-    @_retry_loop(for_write=True)
+    @retry_loop(for_write=True)
     def drop_dir(self, path: Path, if_not_exists: IfNotExistsParam, force: bool) -> None:
         _, _, schema_obj = self._prepare_dir_op(
             drop_dir_path=path.parent,
@@ -830,7 +1492,7 @@ class Catalog:
             raise_if_not_exists=if_not_exists == IfNotExistsParam.ERROR and not force,
         )
         if schema_obj is None:
-            _logger.info(f'Directory {str(path)!r} does not exist; skipped drop_dir().')
+            _logger.info(f'Directory {path!r} does not exist; skipped drop_dir().')
             return
         self._drop_dir(schema_obj._id, path, force=force)
 
@@ -843,10 +1505,10 @@ class Catalog:
             q = sql.select(sql.func.count()).select_from(schema.Table).where(schema.Table.dir_id == dir_id)
             num_tbls = conn.execute(q).scalar()
             if num_subdirs + num_tbls > 0:
-                raise excs.Error(f'Directory {str(dir_path)!r} is not empty.')
+                raise excs.Error(f'Directory {dir_path!r} is not empty.')
 
         # drop existing subdirs
-        self._acquire_dir_xlock(dir_id, None, None)
+        self._acquire_dir_xlock(dir_id=dir_id)
         dir_q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id)
         for row in conn.execute(dir_q).all():
             self._drop_dir(row.id, dir_path.append(row.md['name']), force=True)
@@ -861,59 +1523,82 @@ class Catalog:
 
         # self.drop_dir(dir_id)
         conn.execute(sql.delete(schema.Dir).where(schema.Dir.id == dir_id))
-        _logger.info(f'Removed directory {str(dir_path)!r}.')
+        _logger.info(f'Removed directory {dir_path!r}.')
 
     def get_view_ids(self, tbl_id: UUID, for_update: bool = False) -> list[UUID]:
         """Return the ids of views that directly reference the given table"""
         conn = Env.get().conn
-        q = sql.select(schema.Table.id).where(sql.text(f"md->'view_md'->'base_versions'->0->>0 = {tbl_id.hex!r}"))
+        # check whether this table still exists
+        q = sql.select(sql.func.count()).select_from(schema.Table).where(schema.Table.id == tbl_id)
+        tbl_count = conn.execute(q).scalar()
+        if tbl_count == 0:
+            raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
+        q = sql.select(schema.Table.id).where(schema.Table.md['view_md']['base_versions'][0][0].astext == tbl_id.hex)
         if for_update:
             q = q.with_for_update()
         result = [r[0] for r in conn.execute(q).all()]
         return result
 
-    def get_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
+    def get_tbl_version(
+        self,
+        tbl_id: UUID,
+        effective_version: int | None,
+        check_pending_ops: bool | None = None,
+        validate_initialized: bool = False,
+    ) -> TableVersion | None:
+        """
+        Returns the TableVersion instance for the given table and version and updates the cache.
+
+        If present in the cache and the instance isn't validated, validates version and view_sn against the stored
+        metadata.
+        """
         # we need a transaction here, if we're not already in one; if this starts a new transaction,
         # the returned TableVersion instance will not be validated
-        with self.begin_xact(tbl_id=tbl_id, for_write=False) as conn:
+        with self.begin_xact(for_write=False) as conn:
             tv = self._tbl_versions.get((tbl_id, effective_version))
             if tv is None:
-                tv = self._load_tbl_version(tbl_id, effective_version)
+                tv = self._load_tbl_version(tbl_id, effective_version, check_pending_ops=check_pending_ops)
             elif not tv.is_validated:
                 # only live instances are invalidated
                 assert effective_version is None
-                # we validate live instances by comparing our cached version number to the stored current version
+                # we validate live instances by comparing our cached TableMd.current_version/view_sn to what's stored
                 # _logger.debug(f'validating metadata for table {tbl_id}:{tv.version} ({id(tv):x})')
                 q = sql.select(schema.Table.md).where(schema.Table.id == tbl_id)
-                row = conn.execute(q).one()
-                current_version = row.md['current_version']
+                row = conn.execute(q).one_or_none()
+                if row is None:
+                    raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
+                current_version, view_sn = row.md['current_version'], row.md['view_sn']
 
                 # the stored version can be behind TableVersion.version, because we don't roll back the in-memory
                 # metadata changes after a failed update operation
-                if current_version != tv.version:
+                if current_version != tv.version or view_sn != tv.tbl_md.view_sn:
                     # the cached metadata is invalid
                     _logger.debug(
                         f'reloading metadata for table {tbl_id} '
-                        f'(cached version: {tv.version}, current version: {current_version}'
-                        # f', id: {id(tv):x})'
+                        f'(cached/current version: {tv.version}/{current_version}, '
+                        f'cached/current view_sn: {tv.tbl_md.view_sn}/{view_sn})'
                     )
-                    tv = self._load_tbl_version(tbl_id, None)
+                    tv = self._load_tbl_version(tbl_id, None, check_pending_ops=check_pending_ops)
                 else:
                     # the cached metadata is valid
                     tv.is_validated = True
 
-            assert tv.is_validated
+            assert tv.is_validated, f'{tbl_id}:{effective_version} not validated\n{tv.__dict__}\n{self._debug_str()}'
+            if validate_initialized:
+                assert tv.is_initialized, (
+                    f'{tbl_id}:{effective_version} not initialized\n{tv.__dict__}\n{self._debug_str()}'
+                )
             return tv
 
     def remove_tbl_version(self, tbl_version: TableVersion) -> None:
         assert (tbl_version.id, tbl_version.effective_version) in self._tbl_versions
         del self._tbl_versions[tbl_version.id, tbl_version.effective_version]
 
-    def get_dir(self, dir_id: UUID, for_update: bool = False) -> Optional[Dir]:
+    def get_dir(self, dir_id: UUID, for_update: bool = False) -> Dir | None:
         """Return the Dir with the given id, or None if it doesn't exist"""
         conn = Env.get().conn
         if for_update:
-            self._acquire_dir_xlock(None, dir_id, None)
+            self._acquire_dir_xlock(dir_id=dir_id)
         q = sql.select(schema.Dir).where(schema.Dir.id == dir_id)
         row = conn.execute(q).one_or_none()
         if row is None:
@@ -921,24 +1606,24 @@ class Catalog:
         dir_record = schema.Dir(**row._mapping)
         return Dir(dir_record.id, dir_record.parent_id, dir_record.md['name'])
 
-    def _get_dir(self, path: Path, for_update: bool = False) -> Optional[schema.Dir]:
+    def _get_dir(self, path: Path, lock_dir: bool = False) -> schema.Dir | None:
         """
-        Locking protocol: X locks on all ancestors
+        lock_dir: if True, X-locks target (but not the ancestors)
         """
         user = Env.get().user
         conn = Env.get().conn
         if path.is_root:
-            if for_update:
-                self._acquire_dir_xlock(parent_id=None, dir_id=None, dir_name='')
+            if lock_dir:
+                self._acquire_dir_xlock(dir_name='')
             q = sql.select(schema.Dir).where(schema.Dir.parent_id.is_(None), schema.Dir.md['user'].astext == user)
             row = conn.execute(q).one_or_none()
             return schema.Dir(**row._mapping) if row is not None else None
         else:
-            parent_dir = self._get_dir(path.parent, for_update=False)
+            parent_dir = self._get_dir(path.parent, lock_dir=False)
             if parent_dir is None:
                 return None
-            if for_update:
-                self._acquire_dir_xlock(parent_id=parent_dir.id, dir_id=None, dir_name=path.name)
+            if lock_dir:
+                self._acquire_dir_xlock(parent_id=parent_dir.id, dir_name=path.name)
             q = sql.select(schema.Dir).where(
                 schema.Dir.parent_id == parent_dir.id,
                 schema.Dir.md['name'].astext == path.name,
@@ -947,66 +1632,198 @@ class Catalog:
             row = conn.execute(q).one_or_none()
             return schema.Dir(**row._mapping) if row is not None else None
 
-    def _load_tbl(self, tbl_id: UUID) -> Optional[Table]:
+    def _load_tbl(self, tbl_id: UUID) -> Table | None:
         """Loads metadata for the table with the given id and caches it."""
         _logger.info(f'Loading table {tbl_id}')
         from .insertable_table import InsertableTable
         from .view import View
 
         conn = Env.get().conn
-        q = (
+
+        # check for pending ops
+        q = sql.select(sql.func.count()).where(schema.PendingTableOp.tbl_id == tbl_id)
+        has_pending_ops = conn.execute(q).scalar() > 0
+        if has_pending_ops:
+            raise PendingTableOpsError(tbl_id)
+
+        q: sql.Executable = (
             sql.select(schema.Table, schema.TableSchemaVersion)
             .join(schema.TableSchemaVersion)
             .where(schema.Table.id == schema.TableSchemaVersion.tbl_id)
-            # Table.md['current_schema_version'] == TableSchemaVersion.schema_version
             .where(
-                sql.text(
-                    f"({schema.Table.__table__}.md->>'current_schema_version')::int = "
-                    f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
-                )
+                schema.Table.md['current_schema_version'].cast(sql.Integer) == schema.TableSchemaVersion.schema_version
             )
             .where(schema.Table.id == tbl_id)
         )
         row = conn.execute(q).one_or_none()
         if row is None:
             return None
-        tbl_record, schema_version_record = _unpack_row(row, [schema.Table, schema.TableSchemaVersion])
+        tbl_record, _ = _unpack_row(row, [schema.Table, schema.TableSchemaVersion])
 
         tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
         view_md = tbl_md.view_md
-        if view_md is None:
-            # this is a base table
+
+        if tbl_md.is_replica and not tbl_md.is_snapshot:
+            # If this is a non-snapshot replica, we have to load it as a specific version handle. This is because:
+            # (1) the head version might be a version fragment that isn't user-accessible, and
+            # (2) the cached data in view_md.base_versions is not reliable, since the replicated version does not
+            #     necessarily track the head version of the originally shared table.
+
+            # Query for the latest non-fragment table version.
+            q = (
+                sql.select(schema.TableVersion.version)
+                .where(schema.TableVersion.tbl_id == tbl_id)
+                .where(schema.TableVersion.md['is_fragment'].astext == 'false')
+                .order_by(schema.TableVersion.md['version'].cast(sql.Integer).desc())
+                .limit(1)
+            )
+            row = conn.execute(q).one_or_none()
+            if row is not None:
+                version = row[0]
+                return self._load_tbl_at_version(tbl_id, version)
+            return None
+
+        if view_md is None and not tbl_md.is_replica:
+            # this is a base, non-replica table
             if (tbl_id, None) not in self._tbl_versions:
                 _ = self._load_tbl_version(tbl_id, None)
             tbl = InsertableTable(tbl_record.dir_id, TableVersionHandle(tbl_id, None))
-            self._tbls[tbl_id] = tbl
+            self._tbls[tbl_id, None] = tbl
             return tbl
 
         # this is a view; determine the sequence of TableVersions to load
-        tbl_version_path: list[tuple[UUID, Optional[int]]] = []
-        schema_version_md = schema.md_from_dict(schema.TableSchemaVersionMd, schema_version_record.md)
-        pure_snapshot = view_md.is_snapshot and view_md.predicate is None and len(schema_version_md.columns) == 0
-        if pure_snapshot:
+        tbl_version_path: list[tuple[UUID, int | None]] = []
+        if tbl_md.is_pure_snapshot:
             # this is a pure snapshot, without a physical table backing it; we only need the bases
             pass
         else:
-            effective_version = 0 if view_md.is_snapshot else None  # snapshots only have version 0
+            effective_version = (
+                0 if view_md is not None and view_md.is_snapshot else None
+            )  # snapshots only have version 0
             tbl_version_path.append((tbl_id, effective_version))
-        tbl_version_path.extend((UUID(tbl_id), version) for tbl_id, version in view_md.base_versions)
+        if view_md is not None:
+            tbl_version_path.extend((UUID(tbl_id), version) for tbl_id, version in view_md.base_versions)
 
         # load TableVersions, starting at the root
-        base_path: Optional[TableVersionPath] = None
-        view_path: Optional[TableVersionPath] = None
+        base_path: TableVersionPath | None = None
+        view_path: TableVersionPath | None = None
         for id, effective_version in tbl_version_path[::-1]:
             if (id, effective_version) not in self._tbl_versions:
                 _ = self._load_tbl_version(id, effective_version)
             view_path = TableVersionPath(TableVersionHandle(id, effective_version), base=base_path)
             base_path = view_path
-        view = View(tbl_id, tbl_record.dir_id, tbl_md.name, view_path, snapshot_only=pure_snapshot)
-        self._tbls[tbl_id] = view
+        view = View(tbl_id, tbl_record.dir_id, tbl_md.name, view_path, snapshot_only=tbl_md.is_pure_snapshot)
+        self._tbls[tbl_id, None] = view
         return view
 
-    def load_tbl_md(self, tbl_id: UUID, effective_version: Optional[int]) -> schema.FullTableMd:
+    def _load_tbl_at_version(self, tbl_id: UUID, version: int) -> Table | None:
+        from .view import View
+
+        # Load the specified TableMd and TableVersionMd records from the db.
+        conn = Env.get().conn
+        q: sql.Executable = (
+            sql.select(schema.Table, schema.TableVersion)
+            .join(schema.TableVersion)
+            .where(schema.Table.id == tbl_id)
+            .where(schema.Table.id == schema.TableVersion.tbl_id)
+            .where(schema.TableVersion.version == version)
+        )
+        row = conn.execute(q).one_or_none()
+        if row is None:
+            return None
+        tbl_record, version_record = _unpack_row(row, [schema.Table, schema.TableVersion])
+        tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
+        version_md = schema.md_from_dict(schema.TableVersionMd, version_record.md)
+        tvp = self.construct_tvp(tbl_id, version, tbl_md.ancestor_ids, version_md.created_at)
+
+        view = View(tbl_id, tbl_record.dir_id, tbl_md.name, tvp, snapshot_only=True)
+        self._tbls[tbl_id, version] = view
+        return view
+
+    def construct_tvp(self, tbl_id: UUID, version: int, ancestor_ids: list[str], created_at: float) -> TableVersionPath:
+        # Construct the TableVersionPath for the specified TableVersion. We do this by examining the created_at
+        # timestamps of this table and all its ancestors.
+        # TODO: Store the relevant TableVersionPaths in the database, so that we don't need to rely on timestamps
+        #     (which might be nondeterministic in the future).
+
+        assert Env.get().conn is not None
+
+        # Build the list of ancestor versions, starting with the given table and traversing back to the base table.
+        # For each proper ancestor, we use the version whose created_at timestamp equals or most nearly precedes the
+        # given TableVersion's created_at timestamp.
+        ancestors: list[tuple[UUID, int]] = [(tbl_id, version)]
+        for ancestor_id in ancestor_ids:
+            q = (
+                sql.select(schema.TableVersion)
+                .where(schema.TableVersion.tbl_id == ancestor_id)
+                .where(schema.TableVersion.md['created_at'].cast(sql.Float) <= created_at)
+                .order_by(schema.TableVersion.md['created_at'].cast(sql.Float).desc())
+                .limit(1)
+            )
+            row = Env.get().conn.execute(q).one_or_none()
+            if row is None:
+                # This can happen if an ancestor version is garbage collected; it can also happen in
+                # rare circumstances involving table versions created specifically with Pixeltable 0.4.3.
+                _logger.info(f'Ancestor {ancestor_id} not found for table {tbl_id}:{version}')
+                raise excs.Error('The specified table version is no longer valid and cannot be retrieved.')
+            ancestor_version_record = _unpack_row(row, [schema.TableVersion])[0]
+            ancestor_version_md = schema.md_from_dict(schema.TableVersionMd, ancestor_version_record.md)
+            assert ancestor_version_md.created_at <= created_at
+            ancestors.append((UUID(ancestor_id), ancestor_version_md.version))
+
+        # Force any ancestors to be loaded (base table first).
+        for anc_id, anc_version in ancestors[::-1]:
+            if (anc_id, anc_version) not in self._tbl_versions:
+                _ = self._load_tbl_version(anc_id, anc_version)
+
+        # Now reconstruct the relevant TableVersionPath instance from the ancestor versions.
+        tvp: TableVersionPath | None = None
+        for anc_id, anc_version in ancestors[::-1]:
+            tvp = TableVersionPath(TableVersionHandle(anc_id, anc_version), base=tvp)
+
+        return tvp
+
+    @retry_loop(for_write=False)
+    def collect_tbl_history(self, tbl_id: UUID, n: int | None) -> list[TableVersionCompleteMd]:
+        return self._collect_tbl_history(tbl_id, n)
+
+    def _collect_tbl_history(self, tbl_id: UUID, n: int | None) -> list[TableVersionCompleteMd]:
+        """
+        Returns the history of up to n versions of the table with the given UUID.
+
+        Args:
+            tbl_id: the UUID of the table to collect history for.
+            n: Optional limit on the maximum number of versions returned.
+
+        Returns:
+            A sequence of rows, ordered by version number
+            Each row contains a TableVersion and a TableSchemaVersion object.
+        """
+        q = (
+            sql.select(schema.Table, schema.TableVersion, schema.TableSchemaVersion)
+            .where(schema.Table.id == tbl_id)
+            .join(schema.TableVersion)
+            .where(schema.TableVersion.tbl_id == tbl_id)
+            .join(schema.TableSchemaVersion)
+            .where(schema.TableSchemaVersion.tbl_id == tbl_id)
+            .where(
+                schema.TableVersion.md['schema_version'].cast(sql.Integer) == schema.TableSchemaVersion.schema_version
+            )
+            .order_by(schema.TableVersion.version.desc())
+        )
+        if n is not None:
+            q = q.limit(n)
+        src_rows = Env.get().session.execute(q).fetchall()
+        return [
+            TableVersionCompleteMd(
+                tbl_md=schema.md_from_dict(schema.TableMd, row.Table.md),
+                version_md=schema.md_from_dict(schema.TableVersionMd, row.TableVersion.md),
+                schema_version_md=schema.md_from_dict(schema.TableSchemaVersionMd, row.TableSchemaVersion.md),
+            )
+            for row in src_rows
+        ]
+
+    def load_tbl_md(self, tbl_id: UUID, effective_version: int | None) -> TableVersionCompleteMd:
         """
         Loads metadata from the store for a given table UUID and version.
         """
@@ -1030,13 +1847,9 @@ class Catalog:
             # JOIN TableVersion tv ON (tv.tbl_id = tbl_id AND tv.version = effective_version)
             # JOIN TableSchemaVersion tsv ON (tsv.tbl_id = tbl_id AND tv.md.schema_version = tsv.schema_version)
             # WHERE t.id = tbl_id
-            q = q.where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {effective_version}")).where(
-                sql.text(
-                    (
-                        f"({schema.TableVersion.__table__}.md->>'schema_version')::int = "
-                        f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
-                    )
-                )
+            q = q.where(
+                schema.TableVersion.md['version'].cast(sql.Integer) == effective_version,
+                schema.TableVersion.md['schema_version'].cast(sql.Integer) == schema.TableSchemaVersion.schema_version,
             )
         else:
             # we are loading the current version
@@ -1046,21 +1859,13 @@ class Catalog:
             # JOIN TableSchemaVersion tsv ON (tsv.tbl_id = tbl_id AND t.current_schema_version = tsv.schema_version)
             # WHERE t.id = tbl_id
             q = q.where(
-                sql.text(
-                    f"({schema.Table.__table__}.md->>'current_version')::int = "
-                    f'{schema.TableVersion.__table__}.{schema.TableVersion.version.name}'
-                )
-            ).where(
-                sql.text(
-                    (
-                        f"({schema.Table.__table__}.md->>'current_schema_version')::int = "
-                        f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
-                    )
-                )
+                schema.Table.md['current_version'].cast(sql.Integer) == schema.TableVersion.version,
+                schema.Table.md['current_schema_version'].cast(sql.Integer) == schema.TableSchemaVersion.schema_version,
             )
 
         row = conn.execute(q).one_or_none()
-        assert row is not None, f'Table record not found: {tbl_id}:{effective_version}'
+        if row is None:
+            raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
         tbl_record, version_record, schema_version_record = _unpack_row(
             row, [schema.Table, schema.TableVersion, schema.TableSchemaVersion]
         )
@@ -1069,24 +1874,36 @@ class Catalog:
         version_md = schema.md_from_dict(schema.TableVersionMd, version_record.md)
         schema_version_md = schema.md_from_dict(schema.TableSchemaVersionMd, schema_version_record.md)
 
-        return schema.FullTableMd(tbl_md, version_md, schema_version_md)
+        return TableVersionCompleteMd(tbl_md, version_md, schema_version_md)
 
     def store_tbl_md(
         self,
         tbl_id: UUID,
-        tbl_md: Optional[schema.TableMd],
-        version_md: Optional[schema.TableVersionMd],
-        schema_version_md: Optional[schema.TableSchemaVersionMd],
+        dir_id: UUID | None,
+        tbl_md: schema.TableMd | None,
+        version_md: schema.TableVersionMd | None,
+        schema_version_md: schema.TableSchemaVersionMd | None,
+        pending_ops: list[TableOp] | None = None,
     ) -> None:
         """
-        Stores metadata to the DB. If specified, `tbl_md` will be updated in place (only one such record can exist
-        per UUID); `version_md` and `schema_version_md` will be inserted as new records.
+        Stores metadata to the DB.
+
+        Args:
+            tbl_id: UUID of the table to store metadata for.
+            dir_id: If specified, the tbl_md will be added to the given directory; if None, the table must already exist
+            tbl_md: If specified, `tbl_md` will be inserted, or updated (only one such record can exist per UUID)
+            version_md: inserted as a new record if present
+            schema_version_md: will be inserted as a new record if present
 
         If inserting `version_md` or `schema_version_md` would be a primary key violation, an exception will be raised.
         """
-        conn = Env.get().conn
         assert self._in_write_xact
+        assert version_md is None or version_md.created_at > 0.0
+        assert pending_ops is None or len(pending_ops) > 0
+        assert pending_ops is None or tbl_md is not None  # if we write pending ops, we must also write new tbl_md
+        session = Env.get().session
 
+        # Construct and insert or update table record if requested.
         if tbl_md is not None:
             assert tbl_md.tbl_id == str(tbl_id)
             if version_md is not None:
@@ -1094,32 +1911,88 @@ class Catalog:
                 assert tbl_md.current_schema_version == version_md.schema_version
             if schema_version_md is not None:
                 assert tbl_md.current_schema_version == schema_version_md.schema_version
-            result = conn.execute(
-                sql.update(schema.Table.__table__)
-                .values({schema.Table.md: dataclasses.asdict(tbl_md)})
-                .where(schema.Table.id == tbl_id)
-            )
-            assert result.rowcount == 1, result.rowcount
+            if pending_ops is not None:
+                tbl_md.has_pending_ops = True
+
+            if dir_id is not None:
+                # We are inserting a record while creating a new table.
+                tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(tbl_md))
+                session.add(tbl_record)
+            else:
+                # Update the existing table record.
+                result = session.execute(
+                    sql.update(schema.Table.__table__)
+                    .values({schema.Table.md: dataclasses.asdict(tbl_md)})
+                    .where(schema.Table.id == tbl_id)
+                )
+                assert isinstance(result, sql.CursorResult)
+                assert result.rowcount == 1, result.rowcount
 
+        # Construct and insert new table version record if requested.
         if version_md is not None:
             assert version_md.tbl_id == str(tbl_id)
             if schema_version_md is not None:
                 assert version_md.schema_version == schema_version_md.schema_version
-            conn.execute(
-                sql.insert(schema.TableVersion.__table__).values(
+            tv_rows = (
+                session.query(schema.TableVersion)
+                .filter(schema.TableVersion.tbl_id == tbl_id, schema.TableVersion.version == version_md.version)
+                .all()
+            )
+            if len(tv_rows) == 0:
+                # It's a new table version; insert a new record in the DB for it.
+                tbl_version_record = schema.TableVersion(
                     tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
                 )
-            )
+                session.add(tbl_version_record)
+            else:
+                # This table version already exists; update it.
+                assert len(tv_rows) == 1  # must be unique
+                tv = tv_rows[0]
+                # Validate that the only fields that can change are 'is_fragment' and 'additional_md'.
+                assert tv.md == dataclasses.asdict(
+                    dataclasses.replace(
+                        version_md, is_fragment=tv.md['is_fragment'], additional_md=tv.md['additional_md']
+                    )
+                )
+                result = session.execute(
+                    sql.update(schema.TableVersion.__table__)
+                    .values({schema.TableVersion.md: dataclasses.asdict(version_md)})
+                    .where(schema.TableVersion.tbl_id == tbl_id, schema.TableVersion.version == version_md.version)
+                )
+                assert isinstance(result, sql.CursorResult)
+                assert result.rowcount == 1, result.rowcount
 
+        # Construct and insert a new schema version record if requested.
         if schema_version_md is not None:
             assert schema_version_md.tbl_id == str(tbl_id)
-            conn.execute(
-                sql.insert(schema.TableSchemaVersion.__table__).values(
-                    tbl_id=tbl_id,
-                    schema_version=schema_version_md.schema_version,
-                    md=dataclasses.asdict(schema_version_md),
-                )
+            schema_version_record = schema.TableSchemaVersion(
+                tbl_id=tbl_id, schema_version=schema_version_md.schema_version, md=dataclasses.asdict(schema_version_md)
             )
+            session.add(schema_version_record)
+
+        # make sure we don't have any pending ops
+        assert session.query(schema.PendingTableOp).filter(schema.PendingTableOp.tbl_id == tbl_id).count() == 0
+
+        if pending_ops is not None:
+            for op in pending_ops:
+                op_record = schema.PendingTableOp(tbl_id=tbl_id, op_sn=op.op_sn, op=dataclasses.asdict(op))
+                session.add(op_record)
+
+        session.flush()  # Inform SQLAlchemy that we want to write these changes to the DB.
+
+    def store_update_status(self, tbl_id: UUID, version: int, status: UpdateStatus) -> None:
+        """Update the TableVersion.md.update_status field"""
+        assert self._in_write_xact
+        conn = Env.get().conn
+
+        stmt = (
+            sql.update(schema.TableVersion)
+            .where(schema.TableVersion.tbl_id == tbl_id, schema.TableVersion.version == version)
+            .values(md=schema.TableVersion.md.op('||')({'update_status': dataclasses.asdict(status)}))
+        )
+
+        res = conn.execute(stmt)
+        assert res.rowcount == 1, res.rowcount
 
     def delete_tbl_md(self, tbl_id: UUID) -> None:
         """
@@ -1128,9 +2001,10 @@ class Catalog:
         conn = Env.get().conn
         conn.execute(sql.delete(schema.TableSchemaVersion.__table__).where(schema.TableSchemaVersion.tbl_id == tbl_id))
         conn.execute(sql.delete(schema.TableVersion.__table__).where(schema.TableVersion.tbl_id == tbl_id))
+        conn.execute(sql.delete(schema.PendingTableOp.__table__).where(schema.PendingTableOp.tbl_id == tbl_id))
         conn.execute(sql.delete(schema.Table.__table__).where(schema.Table.id == tbl_id))
 
-    def load_replica_md(self, tbl: Table) -> list[schema.FullTableMd]:
+    def load_replica_md(self, tbl: Table) -> list[TableVersionCompleteMd]:
         """
         Load metadata for the given table along with all its ancestors. The values of TableMd.current_version and
         TableMd.current_schema_version will be adjusted to ensure that the metadata represent a valid (internally
@@ -1143,57 +2017,87 @@ class Catalog:
 
         # If `tbl` is a named pure snapshot, we're not quite done, since the snapshot metadata won't appear in the
         # TableVersionPath. We need to prepend it separately.
-        if tbl._id != tbl._tbl_version.id:
+        if isinstance(tbl, View) and tbl._is_named_pure_snapshot():
             snapshot_md = self.load_tbl_md(tbl._id, 0)
             md = [snapshot_md, *md]
 
-        for ancestor_md in md[1:]:
+        for ancestor_md in md:
+            # Set the `is_replica` flag on every ancestor's TableMd.
+            ancestor_md.tbl_md.is_replica = True
             # For replica metadata, we guarantee that the current_version and current_schema_version of TableMd
-            # match the corresponding values in TableVersionMd and TableSchemaVersionMd. This is to ensure that,
+            # match the corresponding values in TableVersionCompleteMd and TableSchemaVersionMd. This is to ensure that,
             # when the metadata is later stored in the catalog of a different Pixeltable instance, the values of
             # current_version and current_schema_version will always point to versions that are known to the
             # destination catalog.
             ancestor_md.tbl_md.current_version = ancestor_md.version_md.version
             ancestor_md.tbl_md.current_schema_version = ancestor_md.schema_version_md.schema_version
 
+        for ancestor_md in md[1:]:
+            # Also, the table version of every proper ancestor is emphemeral; it does not represent a queryable
+            # table version (the data might be incomplete, since we have only retrieved one of its views, not
+            # the table itself).
+            ancestor_md.version_md.is_fragment = True
+
         return md
 
-    def _load_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
+    def _load_tbl_version(
+        self, tbl_id: UUID, effective_version: int | None, check_pending_ops: bool = True
+    ) -> TableVersion | None:
         """Creates TableVersion instance from stored metadata and registers it in _tbl_versions."""
-        tbl_md, _, schema_version_md = self.load_tbl_md(tbl_id, effective_version)
+        table_version_md = self.load_tbl_md(tbl_id, effective_version)
+        tbl_md = table_version_md.tbl_md
+        version_md = table_version_md.version_md
+        schema_version_md = table_version_md.schema_version_md
         view_md = tbl_md.view_md
 
         conn = Env.get().conn
 
+        if check_pending_ops:
+            pending_ops_q = (
+                sql.select(sql.func.count())
+                .select_from(schema.Table)
+                .join(schema.PendingTableOp)
+                .where(schema.PendingTableOp.tbl_id == tbl_id)
+                .where(schema.Table.id == tbl_id)
+            )
+            if effective_version is not None:
+                # we only care about pending ops if the requested version is the current version
+                pending_ops_q = pending_ops_q.where(
+                    sql.text(f"({schema.Table.__table__}.md->>'current_version')::int = {effective_version}")
+                )
+            has_pending_ops = conn.execute(pending_ops_q).scalar() > 0
+            if has_pending_ops:
+                raise PendingTableOpsError(tbl_id)
+
         # load mutable view ids for mutable TableVersions
         mutable_view_ids: list[UUID] = []
-        # If this is a replica, effective_version should not be None. We see this today, because
-        # the replica's TV instance's Column instances contain value_expr_dicts that reference the live version.
-        # This is presumably a source of bugs, because it ignores schema version changes (eg, column renames).
-        # TODO: retarget the value_expr_dict when instantiating Columns for a particular TV instance.
         if effective_version is None and not tbl_md.is_replica:
-            q = sql.select(schema.Table.id).where(
-                sql.text(
-                    f"md->'view_md'->'base_versions'->0->>0 = {tbl_id.hex!r} "
-                    "AND md->'view_md'->'base_versions'->0->>1 IS NULL"
-                )
+            q = (
+                sql.select(schema.Table.id)
+                .where(schema.Table.md['view_md']['base_versions'][0][0].astext == tbl_id.hex)
+                .where(schema.Table.md['view_md']['base_versions'][0][1].astext == None)
             )
             mutable_view_ids = [r[0] for r in conn.execute(q).all()]
+
         mutable_views = [TableVersionHandle(id, None) for id in mutable_view_ids]
 
         tbl_version: TableVersion
         if view_md is None:
             # this is a base table
-            tbl_version = TableVersion(
-                tbl_id, tbl_md, effective_version, schema_version_md, mutable_views=mutable_views
-            )
+            tbl_version = TableVersion(tbl_id, tbl_md, version_md, effective_version, schema_version_md, mutable_views)
         else:
             assert len(view_md.base_versions) > 0  # a view needs to have a base
-            pure_snapshot = view_md.is_snapshot and view_md.predicate is None and len(schema_version_md.columns) == 0
+            # TODO: add TableVersionCompleteMd.is_pure_snapshot() and use that
+            pure_snapshot = (
+                view_md.is_snapshot
+                and view_md.predicate is None
+                and view_md.sample_clause is None
+                and len(schema_version_md.columns) == 0
+            )
             assert not pure_snapshot  # a pure snapshot doesn't have a physical table backing it, no point in loading it
 
             base: TableVersionHandle
-            base_path: Optional[TableVersionPath] = None  # needed for live view
+            base_path: TableVersionPath | None = None  # needed for live view
             if view_md.is_snapshot:
                 base = TableVersionHandle(UUID(view_md.base_versions[0][0]), view_md.base_versions[0][1])
             else:
@@ -1203,16 +2107,20 @@ class Catalog:
             tbl_version = TableVersion(
                 tbl_id,
                 tbl_md,
+                version_md,
                 effective_version,
                 schema_version_md,
+                mutable_views,
                 base_path=base_path,
                 base=base,
-                mutable_views=mutable_views,
             )
 
+        # register the instance before init()
         self._tbl_versions[tbl_id, effective_version] = tbl_version
+        # register this instance as modified, so that it gets purged if the transaction fails, it may not be
+        # fully initialized
+        self.mark_modified_tvs(tbl_version.handle)
         tbl_version.init()
-
         return tbl_version
 
     def _init_store(self) -> None:
@@ -1220,7 +2128,7 @@ class Catalog:
         self.create_user(None)
         _logger.info('Initialized catalog.')
 
-    def create_user(self, user: Optional[str]) -> None:
+    def create_user(self, user: str | None) -> None:
         """
         Creates a catalog record (root directory) for the specified user, if one does not already exist.
         """
@@ -1239,18 +2147,24 @@ class Catalog:
 
     def _handle_path_collision(
         self, path: Path, expected_obj_type: type[SchemaObject], expected_snapshot: bool, if_exists: IfExistsParam
-    ) -> Optional[SchemaObject]:
+    ) -> SchemaObject | None:
         obj, _, _ = self._prepare_dir_op(add_dir_path=path.parent, add_name=path.name)
 
         if if_exists == IfExistsParam.ERROR and obj is not None:
-            raise excs.Error(f'Path {str(path)!r} is an existing {type(obj)._display_name()}')
+            raise excs.Error(f'Path {path!r} is an existing {obj._display_name()}')
         else:
             is_snapshot = isinstance(obj, View) and obj._tbl_version_path.is_snapshot()
             if obj is not None and (not isinstance(obj, expected_obj_type) or (expected_snapshot and not is_snapshot)):
-                obj_type_str = 'snapshot' if expected_snapshot else expected_obj_type._display_name()
+                if expected_obj_type is Dir:
+                    obj_type_str = 'directory'
+                elif expected_obj_type is InsertableTable:
+                    obj_type_str = 'table'
+                elif expected_obj_type is View:
+                    obj_type_str = 'snapshot' if expected_snapshot else 'view'
+                else:
+                    raise AssertionError()
                 raise excs.Error(
-                    f'Path {str(path)!r} already exists but is not a {obj_type_str}. '
-                    f'Cannot {if_exists.name.lower()} it.'
+                    f'Path {path!r} already exists but is not a {obj_type_str}. Cannot {if_exists.name.lower()} it.'
                 )
 
         if obj is None:
@@ -1263,7 +2177,7 @@ class Catalog:
             dir_contents = self._get_dir_contents(obj._id)
             if len(dir_contents) > 0 and if_exists == IfExistsParam.REPLACE:
                 raise excs.Error(
-                    f'Directory {str(path)!r} already exists and is not empty. '
+                    f'Directory {path!r} already exists and is not empty. '
                     'Use `if_exists="replace_force"` to replace it.'
                 )
             self._drop_dir(obj._id, path, force=True)