pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.2__py3-none-any.whl

pixeltable/catalog/catalog.py

@@ -5,6 +5,7 @@ import functools
 import logging
 import random
 import time
+from collections import defaultdict
 from contextlib import contextmanager
 from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, TypeVar
 from uuid import UUID
@@ -17,10 +18,9 @@ from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 from pixeltable.metadata import schema
 
-if TYPE_CHECKING:
-    from pixeltable.plan import SampleClause
+from .column import Column
 from .dir import Dir
-from .globals import IfExistsParam, IfNotExistsParam, MediaValidation
+from .globals import IfExistsParam, IfNotExistsParam, MediaValidation, QColumnId
 from .insertable_table import InsertableTable
 from .path import Path
 from .schema_object import SchemaObject
@@ -31,6 +31,8 @@ from .table_version_path import TableVersionPath
 from .view import View
 
 if TYPE_CHECKING:
+    from pixeltable.plan import SampleClause
+
     from .. import DataFrame, exprs
 
 
@@ -60,9 +62,10 @@ def _unpack_row(
     return result
 
 
+# -1: unlimited
 # for now, we don't limit the number of retries, because we haven't seen situations where the actual number of retries
 # grows uncontrollably
-_MAX_RETRIES = 0
+_MAX_RETRIES = -1
 
 T = TypeVar('T')
 
@@ -71,25 +74,29 @@ def _retry_loop(*, for_write: bool) -> Callable[[Callable[..., T]], Callable[...
     def decorator(op: Callable[..., T]) -> Callable[..., T]:
         @functools.wraps(op)
         def loop(*args: Any, **kwargs: Any) -> T:
-            num_remaining_retries = _MAX_RETRIES
+            num_retries = 0
             while True:
                 try:
                     # in order for retry to work, we need to make sure that there aren't any prior db updates
                     # that are part of an ongoing transaction
                     assert not Env.get().in_xact
-                    with Catalog.get().begin_xact(for_write=for_write):
+                    with Catalog.get().begin_xact(for_write=for_write, convert_db_excs=False):
                         return op(*args, **kwargs)
                 except sql.exc.DBAPIError as e:
                     # TODO: what other exceptions should we be looking for?
-                    if isinstance(e.orig, psycopg.errors.SerializationFailure):
-                        if num_remaining_retries > 0:
-                            num_remaining_retries -= 1
-                            _logger.debug(f'Serialization failure, retrying ({num_remaining_retries} retries left)')
+                    if isinstance(e.orig, (psycopg.errors.SerializationFailure, psycopg.errors.LockNotAvailable)):
+                        if num_retries < _MAX_RETRIES or _MAX_RETRIES == -1:
+                            num_retries += 1
+                            _logger.debug(f'Retrying ({num_retries}) after {type(e.orig)}')
                             time.sleep(random.uniform(0.1, 0.5))
                         else:
                             raise excs.Error(f'Serialization retry limit ({_MAX_RETRIES}) exceeded') from e
                     else:
                         raise
+                except Exception as e:
+                    # for informational/debugging purposes
+                    _logger.debug(f'retry_loop(): passing along {e}')
+                    raise
 
         return loop
 
@@ -109,8 +116,8 @@ class Catalog:
       duplicate references to that table in the From clause (ie, incorrect Cartesian products)
     - in order to allow multiple concurrent Python processes to perform updates (data and/or schema) against a shared
       Pixeltable instance, Catalog needs to reload metadata from the store when there are changes
-    - concurrent changes are detected by comparing TableVersion.version with the stored current version
-      (TableMd.current_version)
+    - concurrent changes are detected by comparing TableVersion.version/view_sn with the stored current version
+      (TableMd.current_version/view_sn)
     - cached live TableVersion instances (those with effective_version == None) are validated against the stored
       metadata on transaction boundaries; this is recorded in TableVersion.is_validated
     - metadata validation is only needed for live TableVersion instances (snapshot instances are immutable)
@@ -118,13 +125,22 @@ class Catalog:
 
     _instance: Optional[Catalog] = None
 
-    # key: [id, version]
+    # cached TableVersion instances; key: [id, version]
     # - mutable version of a table: version == None (even though TableVersion.version is set correctly)
     # - snapshot versions: records the version of the snapshot
     _tbl_versions: dict[tuple[UUID, Optional[int]], TableVersion]
     _tbls: dict[UUID, Table]
     _in_write_xact: bool  # True if we're in a write transaction
-    _x_locked_tbl_id: Optional[UUID]  # set if begin_xact() was asked to write-lock a table
+    _x_locked_tbl_ids: set[UUID]  # non-empty for write transactions
+
+    # cached column dependencies
+    # - key: table id, value: mapping from column id to its dependencies
+    # - only maintained for dependencies between non-snapshot table versions
+    # - can contain stale entries (stemming from invalidated TV instances)
+    _column_dependencies: dict[UUID, dict[QColumnId, set[QColumnId]]]
+
+    # column dependents are recomputed at the beginning of every write transaction and only reflect the locked tree
+    _column_dependents: Optional[dict[QColumnId, set[QColumnId]]]
 
     @classmethod
     def get(cls) -> Catalog:
@@ -147,9 +163,14 @@ class Catalog:
         self._tbl_versions = {}
         self._tbls = {}  # don't use a defaultdict here, it doesn't cooperate with the debugger
         self._in_write_xact = False
-        self._x_locked_tbl_id = None
+        self._x_locked_tbl_ids = set()
+        self._column_dependencies = {}
+        self._column_dependents = None
         self._init_store()
 
+    def _dropped_tbl_error_msg(self, tbl_id: UUID) -> str:
+        return f'Table was dropped (no record found for {tbl_id})'
+
     def validate(self) -> None:
         """Validate structural consistency of cached metadata"""
         for (tbl_id, effective_version), tbl_version in self._tbl_versions.items():
@@ -164,13 +185,24 @@ class Catalog:
                 f'snapshot_id={tbl_version.id} mutable_views={tbl_version.mutable_views}'
             )
 
-            if tbl_version.is_view and tbl_version.is_mutable:
+            if tbl_version.is_view and tbl_version.is_mutable and tbl_version.is_validated:
                 # make sure this mutable view is recorded in a mutable base
                 base = tbl_version.base
                 assert base is not None
                 if base.effective_version is None:
                     assert (base.id, None) in self._tbl_versions
-                    assert TableVersionHandle.create(tbl_version) in self._tbl_versions[base.id, None].mutable_views
+                    base_tv = self._tbl_versions[base.id, None]
+                    if not base_tv.is_validated:
+                        continue
+                    mutable_view_ids = ', '.join(str(tv.id) for tv in self._tbl_versions[base.id, None].mutable_views)
+                    mutable_view_names = ', '.join(
+                        tv._tbl_version.name
+                        for tv in self._tbl_versions[base.id, None].mutable_views
+                        if tv._tbl_version is not None
+                    )
+                    assert TableVersionHandle.create(tbl_version) in self._tbl_versions[base.id, None].mutable_views, (
+                        f'{tbl_version.name} ({tbl_version.id}) missing in {mutable_view_ids} ({mutable_view_names})'
+                    )
 
             if len(tbl_version.mutable_views) > 0:
                 # make sure we also loaded mutable view metadata, which is needed to detect column dependencies
@@ -178,24 +210,37 @@ class Catalog:
                     assert v.effective_version is None, f'{v.id}:{v.effective_version}'
 
     @contextmanager
-    def begin_xact(self, *, tbl_id: Optional[UUID] = None, for_write: bool = False) -> Iterator[sql.Connection]:
+    def begin_xact(
+        self,
+        *,
+        tbl: Optional[TableVersionPath] = None,
+        for_write: bool = False,
+        lock_mutable_tree: bool = False,
+        convert_db_excs: bool = True,
+    ) -> Iterator[sql.Connection]:
         """
         Return a context manager that yields a connection to the database. Idempotent.
 
         It is mandatory to call this method, not Env.begin_xact(), if the transaction accesses any table data
         or metadata.
 
-        Lock acquisition:
-        - x-locks Table records by updating Table.lock_dummy
+        If tbl != None, follows this locking protocol:
+        - validates/reloads the TableVersion instances of tbl's ancestors (in the hope that this reduces potential
+          SerializationErrors later on)
+        - if for_write == True, x-locks Table record (by updating Table.lock_dummy; see _acquire_tbl_xlock())
+        - if for_write == False, validates TableVersion instance
+        - if lock_mutable_tree == True, also x-locks all mutable views of the table
         - this needs to be done in a retry loop, because Postgres can decide to abort the transaction
           (SerializationFailure, LockNotAvailable)
         - for that reason, we do all lock acquisition prior to doing any real work (eg, compute column values),
-          to minimize (maybe avoid altogether) loosing that work
+          to minimize the probability of loosing that work due to a forced abort
+
+        If convert_db_excs == True, converts DBAPIErrors into excs.Errors.
         """
         if Env.get().in_xact:
-            if tbl_id is not None and for_write:
+            if tbl is not None and for_write:
                 # make sure that we requested the required table lock at the beginning of the transaction
-                assert tbl_id == self._x_locked_tbl_id, f'{tbl_id} != {self._x_locked_tbl_id}'
+                assert tbl.tbl_id in self._x_locked_tbl_ids, f'{tbl.tbl_id} not in {self._x_locked_tbl_ids}'
             yield Env.get().conn
             return
 
@@ -209,30 +254,72 @@ class Catalog:
         num_retries = 0
         while True:
             try:
+                self._in_write_xact = False
+                self._x_locked_tbl_ids = set()
+                self._column_dependents = None
+
                 with Env.get().begin_xact() as conn:
-                    if tbl_id is not None and for_write:
-                        # X-lock Table record
-                        conn.execute(
-                            sql.select(schema.Table).where(schema.Table.id == tbl_id).with_for_update(nowait=True)
-                        )
-                        conn.execute(sql.update(schema.Table).values(lock_dummy=1).where(schema.Table.id == tbl_id))
-                        self._x_locked_tbl_id = tbl_id
+                    if tbl is not None:
+                        try:
+                            if not self._acquire_path_locks(
+                                tbl=tbl, for_write=for_write, lock_mutable_tree=lock_mutable_tree
+                            ):
+                                # this is a snapshot
+                                yield conn
+                                return
+
+                            if for_write:
+                                if lock_mutable_tree:
+                                    self._x_locked_tbl_ids = self._get_mutable_tree(tbl.tbl_id)
+                                    self._compute_column_dependents(self._x_locked_tbl_ids)
+                                else:
+                                    self._x_locked_tbl_ids = {tbl.tbl_id}
+                                if _logger.isEnabledFor(logging.DEBUG):
+                                    # validate only when we don't see errors
+                                    self.validate()
+
+                        except sql.exc.DBAPIError as e:
+                            if isinstance(
+                                e.orig, (psycopg.errors.SerializationFailure, psycopg.errors.LockNotAvailable)
+                            ) and (num_retries < _MAX_RETRIES or _MAX_RETRIES == -1):
+                                num_retries += 1
+                                _logger.debug(f'Retrying ({num_retries}) after {type(e.orig)}')
+                                time.sleep(random.uniform(0.1, 0.5))
+                                continue
+                            else:
+                                raise
 
                     self._in_write_xact = for_write
                     yield conn
                     return
+
             except sql.exc.DBAPIError as e:
-                if isinstance(e.orig, (psycopg.errors.SerializationFailure, psycopg.errors.LockNotAvailable)) and (
-                    num_retries < _MAX_RETRIES or _MAX_RETRIES == 0
-                ):
-                    num_retries += 1
-                    _logger.debug(f'Retrying ({num_retries}) after {type(e.orig)}')
-                    time.sleep(random.uniform(0.1, 0.5))
+                # we got some db error during the actual operation (not just while trying to get locks on the metadata
+                # records): we convert these into Errors, if asked to do so, and abort
+                # TODO: what other concurrency-related exceptions should we expect?
+
+                # we always convert UndefinedTable exceptions (they can't be retried)
+                if isinstance(e.orig, psycopg.errors.UndefinedTable):
+                    # the table got dropped in the middle of the table operation
+                    _logger.debug(f'Exception: undefined table ({tbl.tbl_name()}): Caught {type(e.orig)}: {e!r}')
+                    assert tbl is not None
+                    raise excs.Error(f'Table was dropped: {tbl.tbl_name()}') from None
+                elif isinstance(e.orig, psycopg.errors.SerializationFailure) and convert_db_excs:
+                    # we still got a serialization error, despite getting x-locks at the beginning
+                    msg = f'{tbl.tbl_name()} ({tbl.tbl_id})' if tbl is not None else ''
+                    _logger.debug(f'Exception: serialization failure: {msg} ({e})')
+                    raise excs.Error(
+                        'That Pixeltable operation could not be completed because it conflicted with another '
+                        'operation that was run on a different process.\n'
+                        'Please re-run the operation.'
+                    ) from None
                 else:
                     raise
+
             finally:
                 self._in_write_xact = False
-                self._x_locked_tbl_id = None
+                self._x_locked_tbl_ids = set()
+                self._column_dependents = None
 
                 # invalidate cached current TableVersion instances
                 for tv in self._tbl_versions.values():
@@ -240,20 +327,117 @@ class Catalog:
                         _logger.debug(f'invalidating table version {tv.id}:None (tv={id(tv):x})')
                         tv.is_validated = False
 
-                if _logger.isEnabledFor(logging.DEBUG):
-                    self.validate()
-
     @property
     def in_write_xact(self) -> bool:
         return self._in_write_xact
 
-    def _acquire_dir_xlock(self, parent_id: Optional[UUID], dir_id: Optional[UUID], dir_name: Optional[str]) -> None:
+    def _acquire_path_locks(
+        self, *, tbl: TableVersionPath, for_write: bool = False, lock_mutable_tree: bool = False
+    ) -> bool:
+        """
+        Path locking protocol:
+        - refresh cached TableVersions of ancestors (we need those even during inserts, for computed columns that
+          reference the base tables)
+        - refresh cached TableVersion of tbl or get X-lock, depending on for_write
+        - if lock_mutable_tree, also X-lock all mutable views of tbl
+
+        Returns False if trying to lock a pure snapshot with for_write == True
+        Raises Error if tbl doesn't exist.
+        """
+        start_idx = 1 if for_write else 0
+        for handle in tbl.get_tbl_versions()[start_idx::-1]:
+            _ = self.get_tbl_version(handle.id, handle.effective_version)
+        if not for_write:
+            return True  # nothing left to lock
+        return self._acquire_tbl_xlock(tbl_id=tbl.tbl_id, lock_mutable_tree=lock_mutable_tree, raise_if_not_exists=True)
+
+    def _acquire_tbl_xlock(
+        self,
+        *,
+        tbl_id: Optional[UUID] = None,
+        dir_id: Optional[UUID] = None,
+        tbl_name: Optional[str] = None,
+        lock_mutable_tree: bool = False,
+        raise_if_not_exists: bool = False,
+    ) -> bool:
+        """Force acquisition of an X-lock on a Table record via a blind update
+
+        Either tbl_id or dir_id/tbl_name need to be specified.
+        Returns True if the table was locked, False if it was a snapshot or not found.
+        If lock_mutable_tree, recursively locks all mutable views of the table.
+
+        Returns False if the table is a snapshot or not found and !raise_if_not_exists.
+        """
+        where_clause: sql.ColumnElement
+        if tbl_id is not None:
+            where_clause = schema.Table.id == tbl_id
+        else:
+            where_clause = sql.and_(schema.Table.dir_id == dir_id, schema.Table.md['name'].astext == tbl_name)
+            user = Env.get().user
+            if user is not None:
+                where_clause = sql.and_(where_clause, schema.Table.md['user'].astext == Env.get().user)
+
+        conn = Env.get().conn
+        row = conn.execute(sql.select(schema.Table).where(where_clause).with_for_update(nowait=True)).one_or_none()
+        if row is None:
+            if raise_if_not_exists:
+                raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
+            return False  # nothing to lock
+        if row.md['view_md'] is not None and row.md['view_md']['is_snapshot']:
+            return False  # nothing to lock
+        conn.execute(sql.update(schema.Table).values(lock_dummy=1).where(where_clause))
+
+        if not lock_mutable_tree:
+            return True
+        # also lock mutable views
+        tv = self.get_tbl_version(tbl_id, None)
+        for view in tv.mutable_views:
+            self._acquire_tbl_xlock(tbl_id=view.id, lock_mutable_tree=True, raise_if_not_exists=raise_if_not_exists)
+        return True
+
+    def _get_mutable_tree(self, tbl_id: UUID) -> set[UUID]:
+        """Returns ids of all tables that form the tree of mutable views starting at tbl_id; includes the root."""
+        tv = self.get_tbl_version(tbl_id, None)
+        result: set[UUID] = {tv.id}
+        for view in tv.mutable_views:
+            result.update(self._get_mutable_tree(view.id))
+        return result
+
+    def _compute_column_dependents(self, mutable_tree: set[UUID]) -> None:
+        """Populate self._column_dependents for all tables in mutable_tree"""
+        assert self._column_dependents is None
+        self._column_dependents = defaultdict(set)
+        for tbl_id in mutable_tree:
+            assert tbl_id in self._column_dependencies
+            for col, dependencies in self._column_dependencies[tbl_id].items():
+                for dependency in dependencies:
+                    if dependency.tbl_id not in mutable_tree:
+                        continue
+                    dependents = self._column_dependents[dependency]
+                    dependents.add(col)
+
+    def get_column_dependents(self, tbl_id: UUID, col_id: int) -> set[Column]:
+        """Return all Columns that transitively depend on the given column."""
+        assert self._column_dependents is not None
+        dependents = self._column_dependents[QColumnId(tbl_id, col_id)]
+        result: set[Column] = set()
+        for dependent in dependents:
+            tv = self.get_tbl_version(dependent.tbl_id, None)
+            col = tv.cols_by_id[dependent.col_id]
+            result.add(col)
+        return result
+
+    def _acquire_dir_xlock(
+        self, *, parent_id: Optional[UUID] = None, dir_id: Optional[UUID] = None, dir_name: Optional[str] = None
+    ) -> None:
         """Force acquisition of an X-lock on a Dir record via a blind update.
 
         If dir_id is present, then all other conditions are ignored.
         Note that (parent_id==None) is a valid where condition.
         If dir_id is not specified, the user from the environment is added to the directory filters.
         """
+        assert (dir_name is None) != (dir_id is None)
+        assert not (parent_id is not None and dir_name is None)
         user = Env.get().user
         assert self._in_write_xact
         q = sql.update(schema.Dir).values(lock_dummy=1)
@@ -367,7 +551,7 @@ class Catalog:
         add_dir: Optional[schema.Dir] = None
         drop_dir: Optional[schema.Dir] = None
         for p in sorted(dir_paths):
-            dir = self._get_dir(p, for_update=True)
+            dir = self._get_dir(p, lock_dir=True)
             if dir is None:
                 raise excs.Error(f'Directory {str(p)!r} does not exist.')
             if p == add_dir_path:
@@ -377,7 +561,7 @@ class Catalog:
 
         add_obj: Optional[SchemaObject] = None
         if add_dir is not None:
-            add_obj = self._get_dir_entry(add_dir.id, add_name, for_update=True)
+            add_obj = self._get_dir_entry(add_dir.id, add_name, lock_entry=True)
             if add_obj is not None and raise_if_exists:
                 add_path = add_dir_path.append(add_name)
                 raise excs.Error(f'Path {str(add_path)!r} already exists.')
@@ -385,7 +569,7 @@ class Catalog:
         drop_obj: Optional[SchemaObject] = None
         if drop_dir is not None:
             drop_path = drop_dir_path.append(drop_name)
-            drop_obj = self._get_dir_entry(drop_dir.id, drop_name, for_update=True)
+            drop_obj = self._get_dir_entry(drop_dir.id, drop_name, lock_entry=True)
             if drop_obj is None and raise_if_not_exists:
                 raise excs.Error(f'Path {str(drop_path)!r} does not exist.')
             if drop_obj is not None and drop_expected is not None and not isinstance(drop_obj, drop_expected):
@@ -397,13 +581,13 @@ class Catalog:
         add_dir_obj = Dir(add_dir.id, add_dir.parent_id, add_dir.md['name']) if add_dir is not None else None
         return add_obj, add_dir_obj, drop_obj
 
-    def _get_dir_entry(self, dir_id: UUID, name: str, for_update: bool = False) -> Optional[SchemaObject]:
+    def _get_dir_entry(self, dir_id: UUID, name: str, lock_entry: bool = False) -> Optional[SchemaObject]:
         user = Env.get().user
         conn = Env.get().conn
 
         # check for subdirectory
-        if for_update:
-            self._acquire_dir_xlock(dir_id, None, name)
+        if lock_entry:
+            self._acquire_dir_xlock(parent_id=dir_id, dir_id=None, dir_name=name)
         q = sql.select(schema.Dir).where(
             schema.Dir.parent_id == dir_id, schema.Dir.md['name'].astext == name, schema.Dir.md['user'].astext == user
         )
@@ -417,13 +601,13 @@ class Catalog:
             return Dir(dir_record.id, dir_record.parent_id, name)
 
         # check for table
+        if lock_entry:
+            self._acquire_tbl_xlock(dir_id=dir_id, tbl_name=name)
         q = sql.select(schema.Table.id).where(
             schema.Table.dir_id == dir_id,
             schema.Table.md['name'].astext == name,
             schema.Table.md['user'].astext == user,
         )
-        if for_update:
-            q = q.with_for_update()
         tbl_id = conn.execute(q).scalar_one_or_none()
         if tbl_id is not None:
             if tbl_id not in self._tbls:
@@ -438,7 +622,8 @@ class Catalog:
         expected: Optional[type[SchemaObject]] = None,
         raise_if_exists: bool = False,
         raise_if_not_exists: bool = False,
-        for_update: bool = False,
+        lock_parent: bool = False,
+        lock_obj: bool = False,
     ) -> Optional[SchemaObject]:
         """Return the schema object at the given path, or None if it doesn't exist.
 
@@ -454,16 +639,16 @@ class Catalog:
                 raise excs.Error(
                     f'{str(path)!r} needs to be a {expected._display_name()} but is a {Dir._display_name()}'
                 )
-            dir = self._get_dir(path, for_update=for_update)
+            dir = self._get_dir(path, lock_dir=lock_obj)
             if dir is None:
                 raise excs.Error(f'Unknown user: {Env.get().user}')
             return Dir(dir.id, dir.parent_id, dir.md['name'])
 
         parent_path = path.parent
-        parent_dir = self._get_dir(parent_path, for_update=False)
+        parent_dir = self._get_dir(parent_path, lock_dir=lock_parent)
         if parent_dir is None:
             raise excs.Error(f'Directory {str(parent_path)!r} does not exist.')
-        obj = self._get_dir_entry(parent_dir.id, path.name, for_update=for_update)
+        obj = self._get_dir_entry(parent_dir.id, path.name, lock_entry=lock_obj)
 
         if obj is None and raise_if_not_exists:
             raise excs.Error(f'Path {str(path)!r} does not exist.')
@@ -480,12 +665,12 @@ class Catalog:
             tbl = self._load_tbl(tbl_id)
             if tbl is None:
                 return None
-            # if this is a mutable table, we also need to have its mutable views loaded, in order to track column
-            # dependencies
-            tbl_version = tbl._tbl_version.get()
-            if tbl_version.is_mutable:
-                for v in tbl_version.mutable_views:
-                    _ = self.get_table_by_id(v.id)
+            # # if this is a mutable table, we also need to have its mutable views loaded, in order to track column
+            # # dependencies
+            # tbl_version = tbl._tbl_version.get()
+            # if tbl_version.is_mutable:
+            #     for v in tbl_version.mutable_views:
+            #         _ = self.get_table_by_id(v.id)
         return self._tbls[tbl_id]
 
     @_retry_loop(for_write=True)
@@ -539,6 +724,18 @@ class Catalog:
     ) -> Table:
         from pixeltable.utils.filecache import FileCache
 
+        if not is_snapshot and not base.is_snapshot():
+            # this is a mutable view of a mutable base; X-lock the base and advance its view_sn before adding the view
+            self._acquire_tbl_xlock(tbl_id=base.tbl_id)
+            base_tv = self.get_tbl_version(base.tbl_id, None)
+            base_tv.tbl_md.view_sn += 1
+            result = Env.get().conn.execute(
+                sql.update(schema.Table)
+                .values({schema.Table.md: dataclasses.asdict(base_tv.tbl_md)})
+                .where(schema.Table.id == base.tbl_id)
+            )
+            assert result.rowcount == 1, result.rowcount
+
         existing = self._handle_path_collision(path, View, is_snapshot, if_exists)
         if existing is not None:
             assert isinstance(existing, View)
@@ -718,38 +915,35 @@ class Catalog:
                     'This is likely due to data corruption in the replicated table.'
                 )
 
-        self.store_tbl_md(UUID(tbl_id), new_tbl_md, new_version_md, new_schema_version_md)
+        self.store_tbl_md(UUID(tbl_id), None, new_tbl_md, new_version_md, new_schema_version_md)
 
     @_retry_loop(for_write=False)
     def get_table(self, path: Path) -> Table:
-        obj = self._get_table(path)
-        return obj
-
-    def _get_table(self, path: Path) -> Table:
         obj = Catalog.get()._get_schema_object(path, expected=Table, raise_if_not_exists=True)
         assert isinstance(obj, Table)
-        tbl_version = obj._tbl_version.get()
-        # TODO: instead of calling this here, move the logic into TableVersion.init(), which is called after
-        # registering the instance in _tbl_versions
-        tbl_version.ensure_md_loaded()
-        # if this table has mutable views, we need to load those as well, in order to record column dependencies
-        for v in tbl_version.mutable_views:
-            self.get_table_by_id(v.id)
         return obj
 
     @_retry_loop(for_write=True)
     def drop_table(self, path: Path, if_not_exists: IfNotExistsParam, force: bool) -> None:
-        _, _, src_obj = self._prepare_dir_op(
-            drop_dir_path=path.parent,
-            drop_name=path.name,
-            drop_expected=Table,
+        tbl = self._get_schema_object(
+            path,
+            expected=Table,
             raise_if_not_exists=if_not_exists == IfNotExistsParam.ERROR and not force,
+            lock_parent=True,
+            lock_obj=False,
         )
-        if src_obj is None:
+        if tbl is None:
             _logger.info(f'Skipped table {str(path)!r} (does not exist).')
             return
-        assert isinstance(src_obj, Table)
-        self._drop_tbl(src_obj, force=force, is_replace=False)
+        assert isinstance(tbl, Table)
+
+        if isinstance(tbl, View) and tbl._tbl_version_path.is_mutable() and tbl._tbl_version_path.base.is_mutable():
+            # this is a mutable view of a mutable base;
+            # lock the base before the view, in order to avoid deadlocks with concurrent inserts/updates
+            base_id = tbl._tbl_version_path.base.tbl_id
+            self._acquire_tbl_xlock(tbl_id=base_id, lock_mutable_tree=False)
+
+        self._drop_tbl(tbl, force=force, is_replace=False)
 
     def _drop_tbl(self, tbl: Table, force: bool, is_replace: bool) -> None:
         """
@@ -759,8 +953,11 @@ class Catalog:
         - X-lock base before X-locking any view
         - deadlock-free wrt to TableVersion.insert() (insert propagation also proceeds top-down)
         - X-locks parent dir prior to calling TableVersion.drop(): prevent concurrent creation of another SchemaObject
-          in the same directory with the same name (which could lead to duplicate names if we get rolled back)
+          in the same directory with the same name (which could lead to duplicate names if we get aborted)
         """
+        self._acquire_dir_xlock(dir_id=tbl._dir_id)
+        self._acquire_tbl_xlock(tbl_id=tbl._id, lock_mutable_tree=False)
+
         view_ids = self.get_view_ids(tbl._id, for_update=True)
         if len(view_ids) > 0:
             if not force:
@@ -780,12 +977,34 @@ class Catalog:
                 view = self.get_table_by_id(view_id)
                 self._drop_tbl(view, force=force, is_replace=is_replace)
 
-        _ = self.get_dir(tbl._dir_id, for_update=True)  # X-lock the parent directory
-        tbl._drop()
+        # if this is a mutable view of a mutable base, advance the base's view_sn
+        if isinstance(tbl, View) and tbl._tbl_version_path.is_mutable() and tbl._tbl_version_path.base.is_mutable():
+            base_id = tbl._tbl_version_path.base.tbl_id
+            base_tv = self.get_tbl_version(base_id, None)
+            base_tv.tbl_md.view_sn += 1
+            result = Env.get().conn.execute(
+                sql.update(schema.Table.__table__)
+                .values({schema.Table.md: dataclasses.asdict(base_tv.tbl_md)})
+                .where(schema.Table.id == base_id)
+            )
+            assert result.rowcount == 1, result.rowcount
+
+        tv = tbl._tbl_version.get() if tbl._tbl_version is not None else None
+        if tv is not None:
+            tv = tbl._tbl_version.get()
+            # invalidate the TableVersion instance so that existing references to it can find out it has been dropped
+            tv.is_validated = False
+
+        self.delete_tbl_md(tbl._id)
         assert tbl._id in self._tbls
         del self._tbls[tbl._id]
         _logger.info(f'Dropped table `{tbl._path()}`.')
 
+        if tv is not None:
+            tv.drop()
+            assert (tv.id, tv.effective_version) in self._tbl_versions
+            del self._tbl_versions[tv.id, tv.effective_version]
+
     @_retry_loop(for_write=True)
     def create_dir(self, path: Path, if_exists: IfExistsParam, parents: bool) -> Dir:
         return self._create_dir(path, if_exists, parents)
@@ -846,7 +1065,7 @@ class Catalog:
                 raise excs.Error(f'Directory {str(dir_path)!r} is not empty.')
 
         # drop existing subdirs
-        self._acquire_dir_xlock(dir_id, None, None)
+        self._acquire_dir_xlock(dir_id=dir_id)
         dir_q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id)
         for row in conn.execute(dir_q).all():
             self._drop_dir(row.id, dir_path.append(row.md['name']), force=True)
@@ -866,6 +1085,11 @@ class Catalog:
     def get_view_ids(self, tbl_id: UUID, for_update: bool = False) -> list[UUID]:
         """Return the ids of views that directly reference the given table"""
         conn = Env.get().conn
+        # check whether this table still exists
+        q = sql.select(sql.func.count()).select_from(schema.Table).where(schema.Table.id == tbl_id)
+        tbl_count = conn.execute(q).scalar()
+        if tbl_count == 0:
+            raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
         q = sql.select(schema.Table.id).where(sql.text(f"md->'view_md'->'base_versions'->0->>0 = {tbl_id.hex!r}"))
         if for_update:
             q = q.with_for_update()
@@ -875,27 +1099,29 @@ class Catalog:
     def get_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
         # we need a transaction here, if we're not already in one; if this starts a new transaction,
         # the returned TableVersion instance will not be validated
-        with self.begin_xact(tbl_id=tbl_id, for_write=False) as conn:
+        with self.begin_xact(for_write=False) as conn:
             tv = self._tbl_versions.get((tbl_id, effective_version))
             if tv is None:
                 tv = self._load_tbl_version(tbl_id, effective_version)
             elif not tv.is_validated:
                 # only live instances are invalidated
                 assert effective_version is None
-                # we validate live instances by comparing our cached version number to the stored current version
+                # we validate live instances by comparing our cached TableMd.current_version/view_sn to what's stored
                 # _logger.debug(f'validating metadata for table {tbl_id}:{tv.version} ({id(tv):x})')
                 q = sql.select(schema.Table.md).where(schema.Table.id == tbl_id)
-                row = conn.execute(q).one()
-                current_version = row.md['current_version']
+                row = conn.execute(q).one_or_none()
+                if row is None:
+                    raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
+                current_version, view_sn = row.md['current_version'], row.md['view_sn']
 
                 # the stored version can be behind TableVersion.version, because we don't roll back the in-memory
                 # metadata changes after a failed update operation
-                if current_version != tv.version:
+                if current_version != tv.version or view_sn != tv.tbl_md.view_sn:
                     # the cached metadata is invalid
                     _logger.debug(
                         f'reloading metadata for table {tbl_id} '
-                        f'(cached version: {tv.version}, current version: {current_version}'
-                        # f', id: {id(tv):x})'
+                        f'(cached/current version: {tv.version}/{current_version}, '
+                        f'cached/current view_sn: {tv.tbl_md.view_sn}/{view_sn})'
                     )
                     tv = self._load_tbl_version(tbl_id, None)
                 else:
@@ -913,7 +1139,7 @@ class Catalog:
         """Return the Dir with the given id, or None if it doesn't exist"""
         conn = Env.get().conn
         if for_update:
-            self._acquire_dir_xlock(None, dir_id, None)
+            self._acquire_dir_xlock(dir_id=dir_id)
         q = sql.select(schema.Dir).where(schema.Dir.id == dir_id)
         row = conn.execute(q).one_or_none()
         if row is None:
@@ -921,24 +1147,24 @@ class Catalog:
         dir_record = schema.Dir(**row._mapping)
         return Dir(dir_record.id, dir_record.parent_id, dir_record.md['name'])
 
-    def _get_dir(self, path: Path, for_update: bool = False) -> Optional[schema.Dir]:
+    def _get_dir(self, path: Path, lock_dir: bool = False) -> Optional[schema.Dir]:
         """
-        Locking protocol: X locks on all ancestors
+        lock_dir: if True, X-locks target (but not the ancestors)
         """
         user = Env.get().user
         conn = Env.get().conn
         if path.is_root:
-            if for_update:
-                self._acquire_dir_xlock(parent_id=None, dir_id=None, dir_name='')
+            if lock_dir:
+                self._acquire_dir_xlock(dir_name='')
             q = sql.select(schema.Dir).where(schema.Dir.parent_id.is_(None), schema.Dir.md['user'].astext == user)
             row = conn.execute(q).one_or_none()
             return schema.Dir(**row._mapping) if row is not None else None
         else:
-            parent_dir = self._get_dir(path.parent, for_update=False)
+            parent_dir = self._get_dir(path.parent, lock_dir=False)
             if parent_dir is None:
                 return None
-            if for_update:
-                self._acquire_dir_xlock(parent_id=parent_dir.id, dir_id=None, dir_name=path.name)
+            if lock_dir:
+                self._acquire_dir_xlock(parent_id=parent_dir.id, dir_name=path.name)
             q = sql.select(schema.Dir).where(
                 schema.Dir.parent_id == parent_dir.id,
                 schema.Dir.md['name'].astext == path.name,
@@ -1006,6 +1232,43 @@ class Catalog:
         self._tbls[tbl_id] = view
         return view
 
+    @_retry_loop(for_write=False)
+    def collect_tbl_history(self, tbl_id: UUID, n: Optional[int]) -> list[schema.FullTableMd]:
+        """
+        Returns the history of up to n versions of the table with the given UUID.
+
+        Args:
+            tbl_id: the UUID of the table to collect history for.
+            n: Optional limit on the maximum number of versions returned.
+
+        Returns:
+            A sequence of rows, ordered by version number
+            Each row contains a TableVersion and a TableSchemaVersion object.
+        """
+        q = (
+            sql.select(schema.TableVersion, schema.TableSchemaVersion)
+            .select_from(schema.TableVersion)
+            .join(
+                schema.TableSchemaVersion,
+                sql.cast(schema.TableVersion.md['schema_version'], sql.Integer)
+                == schema.TableSchemaVersion.schema_version,
+            )
+            .where(schema.TableVersion.tbl_id == tbl_id)
+            .where(schema.TableSchemaVersion.tbl_id == tbl_id)
+            .order_by(schema.TableVersion.version.desc())
+        )
+        if n is not None:
+            q = q.limit(n)
+        src_rows = Env.get().session.execute(q).fetchall()
+        return [
+            schema.FullTableMd(
+                None,
+                schema.md_from_dict(schema.TableVersionMd, row.TableVersion.md),
+                schema.md_from_dict(schema.TableSchemaVersionMd, row.TableSchemaVersion.md),
+            )
+            for row in src_rows
+        ]
+
     def load_tbl_md(self, tbl_id: UUID, effective_version: Optional[int]) -> schema.FullTableMd:
         """
         Loads metadata from the store for a given table UUID and version.
@@ -1060,7 +1323,8 @@ class Catalog:
             )
 
         row = conn.execute(q).one_or_none()
-        assert row is not None, f'Table record not found: {tbl_id}:{effective_version}'
+        if row is None:
+            raise excs.Error(self._dropped_tbl_error_msg(tbl_id))
         tbl_record, version_record, schema_version_record = _unpack_row(
             row, [schema.Table, schema.TableVersion, schema.TableSchemaVersion]
         )
@@ -1074,19 +1338,27 @@ class Catalog:
     def store_tbl_md(
         self,
         tbl_id: UUID,
+        dir_id: Optional[UUID],
         tbl_md: Optional[schema.TableMd],
         version_md: Optional[schema.TableVersionMd],
         schema_version_md: Optional[schema.TableSchemaVersionMd],
     ) -> None:
         """
-        Stores metadata to the DB. If specified, `tbl_md` will be updated in place (only one such record can exist
-        per UUID); `version_md` and `schema_version_md` will be inserted as new records.
+        Stores metadata to the DB.
+
+        Args:
+            tbl_id: UUID of the table to store metadata for.
+            dir_id: If specified, the tbl_md will be added to the given directory; if None, the table must already exist
+            tbl_md: If specified, `tbl_md` will be inserted, or updated (only one such record can exist per UUID)
+            version_md: inserted as a new record if present
+            schema_version_md: will be inserted as a new record if present
 
         If inserting `version_md` or `schema_version_md` would be a primary key violation, an exception will be raised.
         """
-        conn = Env.get().conn
         assert self._in_write_xact
+        session = Env.get().session
 
+        # Construct and insert or update table record if requested.
         if tbl_md is not None:
             assert tbl_md.tbl_id == str(tbl_id)
             if version_md is not None:
@@ -1094,32 +1366,37 @@ class Catalog:
                 assert tbl_md.current_schema_version == version_md.schema_version
             if schema_version_md is not None:
                 assert tbl_md.current_schema_version == schema_version_md.schema_version
-            result = conn.execute(
-                sql.update(schema.Table.__table__)
-                .values({schema.Table.md: dataclasses.asdict(tbl_md)})
-                .where(schema.Table.id == tbl_id)
-            )
-            assert result.rowcount == 1, result.rowcount
+            if dir_id is not None:
+                # We are inserting a record while creating a new table.
+                tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(tbl_md))
+                session.add(tbl_record)
+            else:
+                # Update the existing table record.
+                result = session.execute(
+                    sql.update(schema.Table.__table__)
+                    .values({schema.Table.md: dataclasses.asdict(tbl_md)})
+                    .where(schema.Table.id == tbl_id)
+                )
+                assert result.rowcount == 1, result.rowcount
 
+        # Construct and insert new table version record if requested.
         if version_md is not None:
             assert version_md.tbl_id == str(tbl_id)
             if schema_version_md is not None:
                 assert version_md.schema_version == schema_version_md.schema_version
-            conn.execute(
-                sql.insert(schema.TableVersion.__table__).values(
-                    tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
-                )
+            tbl_version_record = schema.TableVersion(
+                tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
             )
+            session.add(tbl_version_record)
 
+        # Construct and insert a new schema version record if requested.
         if schema_version_md is not None:
             assert schema_version_md.tbl_id == str(tbl_id)
-            conn.execute(
-                sql.insert(schema.TableSchemaVersion.__table__).values(
-                    tbl_id=tbl_id,
-                    schema_version=schema_version_md.schema_version,
-                    md=dataclasses.asdict(schema_version_md),
-                )
+            schema_version_record = schema.TableSchemaVersion(
+                tbl_id=tbl_id, schema_version=schema_version_md.schema_version, md=dataclasses.asdict(schema_version_md)
             )
+            session.add(schema_version_record)
+        session.flush()  # Inform SQLAlchemy that we want to write these changes to the DB.
 
     def delete_tbl_md(self, tbl_id: UUID) -> None:
         """
@@ -1143,7 +1420,7 @@ class Catalog:
 
         # If `tbl` is a named pure snapshot, we're not quite done, since the snapshot metadata won't appear in the
         # TableVersionPath. We need to prepend it separately.
-        if tbl._id != tbl._tbl_version.id:
+        if isinstance(tbl, View) and tbl._snapshot_only:
             snapshot_md = self.load_tbl_md(tbl._id, 0)
             md = [snapshot_md, *md]
 
@@ -1212,9 +1489,20 @@ class Catalog:
 
         self._tbl_versions[tbl_id, effective_version] = tbl_version
         tbl_version.init()
-
         return tbl_version
 
+    def record_column_dependencies(self, tbl_version: TableVersion) -> None:
+        """Update self._column_dependencies. Only valid for non-snapshot versions."""
+        from pixeltable.exprs import Expr
+
+        assert not tbl_version.is_snapshot
+        dependencies: dict[QColumnId, set[QColumnId]] = {}
+        for col in tbl_version.cols_by_id.values():
+            if col.value_expr_dict is None:
+                continue
+            dependencies[QColumnId(tbl_version.id, col.id)] = Expr.get_refd_column_ids(col.value_expr_dict)
+        self._column_dependencies[tbl_version.id] = dependencies
+
     def _init_store(self) -> None:
         """One-time initialization of the stored catalog. Idempotent."""
         self.create_user(None)