pixeltable 0.3.10__py3-none-any.whl → 0.3.12__py3-none-any.whl

pixeltable/catalog/catalog.py

@@ -10,10 +10,10 @@ from uuid import UUID
 import psycopg
 import sqlalchemy as sql
 
-import pixeltable.exceptions as excs
-import pixeltable.metadata.schema as schema
+from pixeltable import exceptions as excs
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
+from pixeltable.metadata import schema
 
 from .dir import Dir
 from .globals import IfExistsParam, IfNotExistsParam, MediaValidation
@@ -33,16 +33,6 @@ if TYPE_CHECKING:
 _logger = logging.getLogger('pixeltable')
 
 
-def _lock_str(for_update: bool) -> str:
-    return 'X' if for_update else 'S'
-
-
-# TODO: remove once the concurrent update behavior has been debugged
-# def _debug_print(for_update: bool, msg: str) -> None:
-#     return
-#     print(f'{datetime.datetime.now()}: {_lock_str(for_update)}: {msg}')
-
-
 def _unpack_row(
     row: Optional[sql.engine.Row], entities: list[type[sql.orm.decl_api.DeclarativeBase]]
 ) -> Optional[list[Any]]:
@@ -79,14 +69,17 @@ def _retry_loop(op: Callable[..., T]) -> Callable[..., T]:
                 # in order for retry to work, we need to make sure that there aren't any prior db updates
                 # that are part of an ongoing transaction
                 assert not Env.get().in_xact()
-                with Env.get().begin_xact() as conn:
+                with Env.get().begin_xact():
                     return op(*args, **kwargs)
             except sql.exc.DBAPIError as e:
-                if isinstance(e.orig, psycopg.errors.SerializationFailure) and num_remaining_retries > 0:
-                    num_remaining_retries -= 1
-                    print(f'serialization failure:\n{e}')
-                    print('retrying ************************************************************')
-                    time.sleep(1)
+                if isinstance(e.orig, psycopg.errors.SerializationFailure):
+                    if num_remaining_retries > 0:
+                        num_remaining_retries -= 1
+                        # print(f'serialization failure:\n{e}')
+                        # print('retrying ************************************************************')
+                        time.sleep(1)
+                    else:
+                        raise excs.Error(f'Serialization retry limit ({_MAX_RETRIES}) exceeded') from e
                 else:
                     raise
 
@@ -123,20 +116,39 @@ class Catalog:
         self._tbls = {}  # don't use a defaultdict here, it doesn't cooperate with the debugger
         self._init_store()
 
+    @classmethod
+    def _lock_dir(cls, parent_id: Optional[UUID], dir_id: Optional[UUID], dir_name: Optional[str]) -> None:
+        """Update directory record(s) to sequentialize thread access. Lock is released when transaction commits.
+        If dir_id is present, then all other conditions are ignored.
+        Note that (parent_id==None) is a valid where condition.
+        If dir_id is not specified, the user from the environment is added to the directory filters.
+        """
+        user = Env.get().user
+        conn = Env.get().conn
+        q = sql.update(schema.Dir).values(lock_dummy=1)
+        if dir_id is not None:
+            q = q.where(schema.Dir.id == dir_id)
+        else:
+            q = q.where(schema.Dir.parent_id == parent_id)
+            if dir_name is not None:
+                q = q.where(schema.Dir.md['name'].astext == dir_name)
+            if user is not None:
+                q = q.where(schema.Dir.md['user'].astext == user)
+        conn.execute(q)
+
     def get_dir_path(self, dir_id: UUID) -> Path:
         """Return path for directory with given id"""
         conn = Env.get().conn
         names: list[str] = []
         while True:
             q = sql.select(schema.Dir).where(schema.Dir.id == dir_id)
-            # _debug_print(for_update=False, msg=f'dir id={dir_id}')
             row = conn.execute(q).one()
             dir = schema.Dir(**row._mapping)
             if dir.md['name'] == '':
                 break
             names.insert(0, dir.md['name'])
             dir_id = dir.parent_id
-        return Path('.'.join(names), empty_is_valid=True)
+        return Path('.'.join(names), empty_is_valid=True, allow_system_paths=True)
 
     @dataclasses.dataclass
     class DirEntry:
@@ -155,7 +167,6 @@ class Catalog:
         result: dict[str, Catalog.DirEntry] = {}
 
         q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id)
-        # _debug_print(for_update=False, msg=f'dirs parent_id={dir_id}')
         rows = conn.execute(q).all()
         for row in rows:
             dir = schema.Dir(**row._mapping)
@@ -165,7 +176,6 @@ class Catalog:
             result[dir.md['name']] = self.DirEntry(dir=dir, dir_entries=dir_contents, table=None)
 
         q = sql.select(schema.Table).where(schema.Table.dir_id == dir_id)
-        # _debug_print(for_update=False, msg=f'tbls parent_id={dir_id}')
         rows = conn.execute(q).all()
         for row in rows:
             tbl = schema.Table(**row._mapping)
@@ -175,6 +185,9 @@ class Catalog:
 
     @_retry_loop
     def move(self, path: Path, new_path: Path) -> None:
+        self._move(path, new_path)
+
+    def _move(self, path: Path, new_path: Path) -> None:
         _, dest_dir, src_obj = self._prepare_dir_op(
             add_dir_path=new_path.parent,
             add_name=new_path.name,
@@ -222,10 +235,10 @@ class Catalog:
 
         add_dir: Optional[schema.Dir] = None
         drop_dir: Optional[schema.Dir] = None
-        for p in sorted(list(dir_paths)):
+        for p in sorted(dir_paths):
             dir = self._get_dir(p, for_update=True)
             if dir is None:
-                raise excs.Error(f'Directory {str(p)!r} does not exist')
+                raise excs.Error(f'Directory {str(p)!r} does not exist.')
             if p == add_dir_path:
                 add_dir = dir
             if p == drop_dir_path:
@@ -236,14 +249,14 @@ class Catalog:
             add_obj = self._get_dir_entry(add_dir.id, add_name, for_update=True)
             if add_obj is not None and raise_if_exists:
                 add_path = add_dir_path.append(add_name)
-                raise excs.Error(f'Path {str(add_path)!r} already exists')
+                raise excs.Error(f'Path {str(add_path)!r} already exists.')
 
         drop_obj: Optional[SchemaObject] = None
         if drop_dir is not None:
             drop_path = drop_dir_path.append(drop_name)
             drop_obj = self._get_dir_entry(drop_dir.id, drop_name, for_update=True)
             if drop_obj is None and raise_if_not_exists:
-                raise excs.Error(f'Path {str(drop_path)!r} does not exist')
+                raise excs.Error(f'Path {str(drop_path)!r} does not exist.')
             if drop_obj is not None and drop_expected is not None and not isinstance(drop_obj, drop_expected):
                 raise excs.Error(
                     f'{str(drop_path)!r} needs to be a {drop_expected._display_name()} '
@@ -254,32 +267,35 @@ class Catalog:
         return add_obj, add_dir_obj, drop_obj
 
     def _get_dir_entry(self, dir_id: UUID, name: str, for_update: bool = False) -> Optional[SchemaObject]:
+        user = Env.get().user
         conn = Env.get().conn
 
         # check for subdirectory
-        q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id, schema.Dir.md['name'].astext == name)
         if for_update:
-            q = q.with_for_update()
-        # _debug_print(for_update, f'dir name={name!r} parent={dir_id}')
-        # row = conn.execute(q).one_or_none()
-        # if row is not None:
-        #     dir_record = schema.Dir(**row._mapping)
-        #     return Dir(dir_record.id, dir_record.parent_id, name)
+            self._lock_dir(dir_id, None, name)
+        q = sql.select(schema.Dir).where(
+            schema.Dir.parent_id == dir_id, schema.Dir.md['name'].astext == name, schema.Dir.md['user'].astext == user
+        )
         rows = conn.execute(q).all()
+        # The condition below can occur if there is a synchronization failure across multiple processes
+        # It indicates database inconsistency.
         if len(rows) > 1:
-            assert False, rows
+            raise AssertionError(rows)
         if len(rows) == 1:
             dir_record = schema.Dir(**rows[0]._mapping)
             return Dir(dir_record.id, dir_record.parent_id, name)
 
         # check for table
-        q = sql.select(schema.Table.id).where(schema.Table.dir_id == dir_id, schema.Table.md['name'].astext == name)
+        q = sql.select(schema.Table.id).where(
+            schema.Table.dir_id == dir_id,
+            schema.Table.md['name'].astext == name,
+            schema.Table.md['user'].astext == user,
+        )
         if for_update:
             q = q.with_for_update()
-        # _debug_print(for_update, f'table name={name!r} parent={dir_id}')
         tbl_id = conn.execute(q).scalar_one_or_none()
         if tbl_id is not None:
-            if not tbl_id in self._tbls:
+            if tbl_id not in self._tbls:
                 self._tbls[tbl_id] = self._load_tbl(tbl_id)
             return self._tbls[tbl_id]
 
@@ -304,26 +320,32 @@ class Catalog:
         if path.is_root:
             # the root dir
             if expected is not None and expected is not Dir:
-                raise excs.Error(f'{path!r} needs to be a {expected._display_name()} but is a {Dir._display_name()}')
+                raise excs.Error(
+                    f'{str(path)!r} needs to be a {expected._display_name()} but is a {Dir._display_name()}'
+                )
             dir = self._get_dir(path, for_update=for_update)
+            if dir is None:
+                raise excs.Error(f'Unknown user: {Env.get().user}')
             return Dir(dir.id, dir.parent_id, dir.md['name'])
 
         parent_path = path.parent
         parent_dir = self._get_dir(parent_path, for_update=False)
         if parent_dir is None:
-            raise excs.Error(f'Directory {parent_path!r} does not exist')
+            raise excs.Error(f'Directory {str(parent_path)!r} does not exist.')
         obj = self._get_dir_entry(parent_dir.id, path.name, for_update=for_update)
 
         if obj is None and raise_if_not_exists:
-            raise excs.Error(f'Path {path!r} does not exist')
+            raise excs.Error(f'Path {str(path)!r} does not exist.')
         elif obj is not None and raise_if_exists:
-            raise excs.Error(f'Path {path!r} is an existing {type(obj)._display_name()}')
+            raise excs.Error(f'Path {str(path)!r} is an existing {type(obj)._display_name()}.')
         elif obj is not None and expected is not None and not isinstance(obj, expected):
-            raise excs.Error(f'{path!r} needs to be a {expected._display_name()} but is a {type(obj)._display_name()}')
+            raise excs.Error(
+                f'{str(path)!r} needs to be a {expected._display_name()} but is a {type(obj)._display_name()}.'
+            )
         return obj
 
     def get_table_by_id(self, tbl_id: UUID) -> Optional[Table]:
-        if not tbl_id in self._tbls:
+        if tbl_id not in self._tbls:
             tbl = self._load_tbl(tbl_id)
             if tbl is None:
                 return None
@@ -409,6 +431,155 @@ class Catalog:
         self._tbls[view._id] = view
         return view
 
+    @_retry_loop
+    def create_replica(self, path: Path, md: list[schema.FullTableMd], if_exists: IfExistsParam) -> Table:
+        """
+        Creates table, table_version, and table_schema_version records for a replica with the given metadata.
+        The metadata should be presented in standard "ancestor order", with the table being replicated at
+        list position 0 and the (root) base table at list position -1.
+        """
+        tbl_id = UUID(md[0].tbl_md.tbl_id)
+
+        # First handle path collisions (if_exists='ignore' or 'replace' or etc).
+        existing = self._handle_path_collision(path, View, False, if_exists)
+        if existing is not None:
+            if existing._id != tbl_id:
+                raise excs.Error(
+                    f"An attempt was made to create a replica table at {path!r} with if_exists='ignore', "
+                    'but a different table already exists at that location.'
+                )
+            assert isinstance(existing, View)
+            return existing
+
+        # Ensure that the system directory exists.
+        self._create_dir(Path('_system', allow_system_paths=True), if_exists=IfExistsParam.IGNORE, parents=False)
+
+        # Now check to see if this table already exists in the catalog.
+        # TODO: Handle concurrency in create_replica()
+        existing = Catalog.get().get_table_by_id(tbl_id)
+        if existing is not None:
+            existing_path = Path(existing._path(), allow_system_paths=True)
+            # It does exist. If it's a non-system table, that's an error: it's already been replicated.
+            if not existing_path.is_system_path:
+                raise excs.Error(
+                    f'That table has already been replicated as {existing._path()!r}. \n'
+                    f'Drop the existing replica if you wish to re-create it.'
+                )
+            # If it's a system table, then this means it was created at some point as the ancestor of some other
+            # table (a snapshot-over-snapshot scenario). In that case, we simply move it to the new (named) location.
+            self._move(existing_path, path)
+
+        # Now store the metadata for this replica. In the case where the table already exists (and was just moved
+        # into a named location), this will be a no-op, but it still serves to validate that the newly received
+        # metadata is identical to what's in the catalog.
+        self.__store_replica_md(path, md[0])
+
+        # Now store the metadata for all of this table's proper ancestors. If one or more proper ancestors
+        # do not yet exist in the store, they will be created as anonymous system tables.
+        for ancestor_md in md[1:]:
+            ancestor_id = UUID(ancestor_md.tbl_md.tbl_id)
+            replica = Catalog.get().get_table_by_id(ancestor_id)
+            replica_path: Path
+            if replica is None:
+                # We've never seen this table before. Create a new anonymous system table for it.
+                replica_path = Path(f'_system.replica_{ancestor_id.hex}', allow_system_paths=True)
+            else:
+                # The table already exists in the catalog. The existing path might be a system path (if the table
+                # was created as an anonymous base table of some other table), or it might not (if it's a snapshot
+                # that was directly replicated by the user at some point). In either case, use the existing path.
+                replica_path = Path(replica._path(), allow_system_paths=True)
+
+            # Store the metadata; it could be a new version (in which case a new record will be created) or a
+            # known version (in which case the newly received metadata will be validated as identical).
+            self.__store_replica_md(replica_path, ancestor_md)
+
+        # Update the catalog (as a final step, after all DB operations completed successfully).
+        # Only the table being replicated is actually made visible in the catalog.
+        self._tbls[tbl_id] = self._load_tbl(tbl_id)
+        return self._tbls[tbl_id]
+
+    def __store_replica_md(self, path: Path, md: schema.FullTableMd) -> None:
+        _logger.info(f'Creating replica table at {path!r} with ID: {md.tbl_md.tbl_id}')
+        # TODO: Handle concurrency
+        dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
+        assert dir is not None
+
+        conn = Env.get().conn
+        tbl_id = md.tbl_md.tbl_id
+
+        new_tbl_md: Optional[schema.TableMd] = None
+        new_version_md: Optional[schema.TableVersionMd] = None
+        new_schema_version_md: Optional[schema.TableSchemaVersionMd] = None
+
+        # We need to ensure that the table metadata in the catalog always reflects the latest observed version of
+        # this table. (In particular, if this is a base table, then its table metadata need to be consistent
+        # with the latest version of this table having a replicated view somewhere in the catalog.)
+        q: sql.Executable = sql.select(schema.Table.md).where(schema.Table.id == tbl_id)
+        existing_md_row = conn.execute(q).one_or_none()
+
+        if existing_md_row is None:
+            # No existing table, so create a new record.
+            q = sql.insert(schema.Table.__table__).values(
+                id=tbl_id,
+                dir_id=dir._id,
+                md=dataclasses.asdict(
+                    dataclasses.replace(md.tbl_md, name=path.name, user=Env.get().user, is_replica=True)
+                ),
+            )
+            conn.execute(q)
+        else:
+            assert existing_md_row.md['is_replica']
+            if md.tbl_md.current_version > existing_md_row.md['current_version']:
+                # New metadata is more recent than the metadata currently stored in the DB; we'll update the record
+                # in place in the DB.
+                new_tbl_md = dataclasses.replace(md.tbl_md, name=path.name, user=Env.get().user, is_replica=True)
+
+        # Now see if a TableVersion record already exists in the DB for this table version. If not, insert it. If
+        # it already exists, check that the existing record is identical to the new one.
+        q = (
+            sql.select(schema.TableVersion.md)
+            .where(schema.TableVersion.tbl_id == tbl_id)
+            .where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {md.version_md.version}"))
+        )
+        existing_version_md_row = conn.execute(q).one_or_none()
+        if existing_version_md_row is None:
+            new_version_md = md.version_md
+        else:
+            existing_version_md = schema.md_from_dict(schema.TableVersionMd, existing_version_md_row.md)
+            if existing_version_md != md.version_md:
+                raise excs.Error(
+                    f'The version metadata for the replica {path!r}:{md.version_md.version} is inconsistent with '
+                    'the metadata recorded from a prior replica.\n'
+                    'This is likely due to data corruption in the replicated table.'
+                )
+
+        # Do the same thing for TableSchemaVersion.
+        q = (
+            sql.select(schema.TableSchemaVersion.md)
+            .where(schema.TableSchemaVersion.tbl_id == tbl_id)
+            .where(
+                sql.text(
+                    f"({schema.TableSchemaVersion.__table__}.md->>'schema_version')::int = "
+                    f'{md.schema_version_md.schema_version}'
+                )
+            )
+        )
+        existing_schema_version_md_row = conn.execute(q).one_or_none()
+        if existing_schema_version_md_row is None:
+            new_schema_version_md = md.schema_version_md
+        else:
+            existing_schema_version_md = schema.md_from_dict(
+                schema.TableSchemaVersionMd, existing_schema_version_md_row.md
+            )
+            if existing_schema_version_md != md.schema_version_md:
+                raise excs.Error(
+                    f'The schema version metadata for the replica {path!r}:{md.schema_version_md.schema_version} '
+                    'is inconsistent with the metadata recorded from a prior replica.\n'
+                    'This is likely due to data corruption in the replicated table.'
+                )
+
+        self.store_tbl_md(UUID(tbl_id), new_tbl_md, new_version_md, new_schema_version_md)
+
     @_retry_loop
     def get_table(self, path: Path) -> Table:
         obj = Catalog.get()._get_schema_object(path, expected=Table, raise_if_not_exists=True)
@@ -467,6 +638,9 @@ class Catalog:
 
     @_retry_loop
     def create_dir(self, path: Path, if_exists: IfExistsParam, parents: bool) -> Dir:
+        return self._create_dir(path, if_exists, parents)
+
+    def _create_dir(self, path: Path, if_exists: IfExistsParam, parents: bool) -> Dir:
         # existing = self._handle_path_collision(path, Dir, False, if_exists)
         # if existing is not None:
         #     assert isinstance(existing, Dir)
@@ -475,7 +649,7 @@ class Catalog:
         # parent = self._get_schema_object(path.parent)
         # assert parent is not None
         # dir = Dir._create(parent._id, path.name)
-        # Env.get().console_logger.info(f'Created directory {path!r}.')
+        # Env.get().console_logger.info(f'Created directory {str(path)!r}.')
         # return dir
 
         if parents:
@@ -519,10 +693,11 @@ class Catalog:
             q = sql.select(sql.func.count()).select_from(schema.Table).where(schema.Table.dir_id == dir_id)
             num_tbls = conn.execute(q).scalar()
             if num_subdirs + num_tbls > 0:
-                raise excs.Error(f'Directory {dir_path!r} is not empty.')
+                raise excs.Error(f'Directory {str(dir_path)!r} is not empty.')
 
         # drop existing subdirs
-        dir_q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id).with_for_update()
+        self._lock_dir(dir_id, None, None)
+        dir_q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id)
         for row in conn.execute(dir_q).all():
             self._drop_dir(row.id, dir_path.append(row.md['name']), force=True)
 
@@ -535,7 +710,6 @@ class Catalog:
                 self._drop_tbl(tbl, force=True, is_replace=False)
 
         # self.drop_dir(dir_id)
-        # _debug_print(for_update=True, msg=f'drop dir id={dir_id}')
         conn.execute(sql.delete(schema.Dir).where(schema.Dir.id == dir_id))
         _logger.info(f'Removed directory {str(dir_path)!r}.')
 
@@ -545,18 +719,17 @@ class Catalog:
         q = sql.select(schema.Table.id).where(sql.text(f"md->'view_md'->'base_versions'->0->>0 = {tbl_id.hex!r}"))
         if for_update:
             q = q.with_for_update()
-        # _debug_print(for_update=False, msg=f'views of tbl id={tbl_id}')
         result = [r[0] for r in conn.execute(q).all()]
         return result
 
     def get_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
         if (tbl_id, effective_version) not in self._tbl_versions:
-            self._tbl_versions[(tbl_id, effective_version)] = self._load_tbl_version(tbl_id, effective_version)
-        return self._tbl_versions[(tbl_id, effective_version)]
+            self._tbl_versions[tbl_id, effective_version] = self._load_tbl_version(tbl_id, effective_version)
+        return self._tbl_versions[tbl_id, effective_version]
 
     def add_tbl_version(self, tbl_version: TableVersion) -> None:
         """Explicitly add a TableVersion"""
-        self._tbl_versions[(tbl_version.id, tbl_version.effective_version)] = tbl_version
+        self._tbl_versions[tbl_version.id, tbl_version.effective_version] = tbl_version
         # if this is a mutable view, also record it in the base
         if tbl_version.is_view and tbl_version.effective_version is None:
             base = tbl_version.base.get()
@@ -564,15 +737,14 @@ class Catalog:
 
     def remove_tbl_version(self, tbl_version: TableVersion) -> None:
         assert (tbl_version.id, tbl_version.effective_version) in self._tbl_versions
-        del self._tbl_versions[(tbl_version.id, tbl_version.effective_version)]
+        del self._tbl_versions[tbl_version.id, tbl_version.effective_version]
 
     def get_dir(self, dir_id: UUID, for_update: bool = False) -> Optional[Dir]:
         """Return the Dir with the given id, or None if it doesn't exist"""
         conn = Env.get().conn
-        q = sql.select(schema.Dir).where(schema.Dir.id == dir_id)
         if for_update:
-            q = q.with_for_update()
-        # _debug_print(for_update=False, msg=f'dir id={dir_id!r}')
+            self._lock_dir(None, dir_id, None)
+        q = sql.select(schema.Dir).where(schema.Dir.id == dir_id)
         row = conn.execute(q).one_or_none()
         if row is None:
             return None
@@ -581,28 +753,27 @@ class Catalog:
 
     def _get_dir(self, path: Path, for_update: bool = False) -> Optional[schema.Dir]:
         """
-        Locking protocol:
-        - S locks on all ancestors
-        - X lock on dir if for_update == True, otherwise also an S lock
+        Locking protocol: X locks on all ancestors
         """
+        user = Env.get().user
         conn = Env.get().conn
         if path.is_root:
-            q = sql.select(schema.Dir).where(schema.Dir.parent_id.is_(None))
             if for_update:
-                q = q.with_for_update()
-            # _debug_print(for_update, 'root dir')
-            row = conn.execute(q).one()
-            return schema.Dir(**row._mapping)
+                self._lock_dir(parent_id=None, dir_id=None, dir_name='')
+            q = sql.select(schema.Dir).where(schema.Dir.parent_id.is_(None), schema.Dir.md['user'].astext == user)
+            row = conn.execute(q).one_or_none()
+            return schema.Dir(**row._mapping) if row is not None else None
         else:
             parent_dir = self._get_dir(path.parent, for_update=False)
             if parent_dir is None:
                 return None
+            if for_update:
+                self._lock_dir(parent_id=parent_dir.id, dir_id=None, dir_name=path.name)
             q = sql.select(schema.Dir).where(
-                schema.Dir.parent_id == parent_dir.id, schema.Dir.md['name'].astext == path.name
+                schema.Dir.parent_id == parent_dir.id,
+                schema.Dir.md['name'].astext == path.name,
+                schema.Dir.md['user'].astext == user,
             )
-            if for_update:
-                q = q.with_for_update()
-            # _debug_print(for_update, f'dir {str(path)}')
             row = conn.execute(q).one_or_none()
             return schema.Dir(**row._mapping) if row is not None else None
 
@@ -625,7 +796,6 @@ class Catalog:
             )
             .where(schema.Table.id == tbl_id)
         )
-        # _debug_print(for_update=False, msg=f'load table id={tbl_id!r}')
         row = conn.execute(q).one_or_none()
         if row is None:
             return None
@@ -636,7 +806,7 @@ class Catalog:
         if view_md is None:
             # this is a base table
             if (tbl_id, None) not in self._tbl_versions:
-                self._tbl_versions[(tbl_id, None)] = self._load_tbl_version(tbl_id, None)
+                self._tbl_versions[tbl_id, None] = self._load_tbl_version(tbl_id, None)
             tbl = InsertableTable(tbl_record.dir_id, TableVersionHandle(tbl_id, None))
             return tbl
 
@@ -657,20 +827,26 @@ class Catalog:
         view_path: Optional[TableVersionPath] = None
         for id, effective_version in tbl_version_path[::-1]:
             if (id, effective_version) not in self._tbl_versions:
-                self._tbl_versions[(id, effective_version)] = self._load_tbl_version(id, effective_version)
+                self._tbl_versions[id, effective_version] = self._load_tbl_version(id, effective_version)
             view_path = TableVersionPath(TableVersionHandle(id, effective_version), base=base_path)
             base_path = view_path
         view = View(tbl_id, tbl_record.dir_id, tbl_md.name, view_path, snapshot_only=pure_snapshot)
         # TODO: also load mutable views
         return view
 
-    def _load_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
-        _logger.info(f'Loading table version: {tbl_id}:{effective_version}')
+    def load_tbl_md(self, tbl_id: UUID, effective_version: Optional[int]) -> schema.FullTableMd:
+        """
+        Loads metadata from the store for a given table UUID and version.
+        """
+        _logger.info(f'Loading metadata for table version: {tbl_id}:{effective_version}')
         conn = Env.get().conn
+
         q = (
-            sql.select(schema.Table, schema.TableSchemaVersion)
+            sql.select(schema.Table, schema.TableVersion, schema.TableSchemaVersion)
             .select_from(schema.Table)
             .where(schema.Table.id == tbl_id)
+            .join(schema.TableVersion)
+            .where(schema.TableVersion.tbl_id == tbl_id)
             .join(schema.TableSchemaVersion)
             .where(schema.TableSchemaVersion.tbl_id == tbl_id)
         )
@@ -682,16 +858,11 @@ class Catalog:
             # JOIN TableVersion tv ON (tv.tbl_id = tbl_id AND tv.version = effective_version)
             # JOIN TableSchemaVersion tsv ON (tsv.tbl_id = tbl_id AND tv.md.schema_version = tsv.schema_version)
             # WHERE t.id = tbl_id
-            q = (
-                q.join(schema.TableVersion)
-                .where(schema.TableVersion.tbl_id == tbl_id)
-                .where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {effective_version}"))
-                .where(
-                    sql.text(
-                        (
-                            f"({schema.TableVersion.__table__}.md->>'schema_version')::int = "
-                            f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
-                        )
+            q = q.where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {effective_version}")).where(
+                sql.text(
+                    (
+                        f"({schema.TableVersion.__table__}.md->>'schema_version')::int = "
+                        f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
                     )
                 )
             )
@@ -699,9 +870,15 @@ class Catalog:
             # we are loading the current version
             # SELECT *
             # FROM Table t
+            # JOIN TableVersion tv ON (tv.tbl_id = tbl_id AND t.current_version = tv.version)
             # JOIN TableSchemaVersion tsv ON (tsv.tbl_id = tbl_id AND t.current_schema_version = tsv.schema_version)
             # WHERE t.id = tbl_id
             q = q.where(
+                sql.text(
+                    f"({schema.Table.__table__}.md->>'current_version')::int = "
+                    f'{schema.TableVersion.__table__}.{schema.TableVersion.version.name}'
+                )
+            ).where(
                 sql.text(
                     (
                         f"({schema.Table.__table__}.md->>'current_schema_version')::int = "
@@ -711,11 +888,100 @@ class Catalog:
             )
 
         row = conn.execute(q).one_or_none()
-        tbl_record, schema_version_record = _unpack_row(row, [schema.Table, schema.TableSchemaVersion])
+        assert row is not None, f'Table record not found: {tbl_id}:{effective_version}'
+        tbl_record, version_record, schema_version_record = _unpack_row(
+            row, [schema.Table, schema.TableVersion, schema.TableSchemaVersion]
+        )
+        assert tbl_record.id == tbl_id
         tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
+        version_md = schema.md_from_dict(schema.TableVersionMd, version_record.md)
         schema_version_md = schema.md_from_dict(schema.TableSchemaVersionMd, schema_version_record.md)
+
+        return schema.FullTableMd(tbl_md, version_md, schema_version_md)
+
+    def store_tbl_md(
+        self,
+        tbl_id: UUID,
+        tbl_md: Optional[schema.TableMd],
+        version_md: Optional[schema.TableVersionMd],
+        schema_version_md: Optional[schema.TableSchemaVersionMd],
+    ) -> None:
+        """
+        Stores metadata to the DB. If specified, `tbl_md` will be updated in place (only one such record can exist
+        per UUID); `version_md` and `schema_version_md` will be inserted as new records.
+
+        If inserting `version_md` or `schema_version_md` would be a primary key violation, an exception will be raised.
+        """
+        conn = Env.get().conn
+
+        if tbl_md is not None:
+            result = conn.execute(
+                sql.update(schema.Table.__table__)
+                .values({schema.Table.md: dataclasses.asdict(tbl_md)})
+                .where(schema.Table.id == tbl_id)
+            )
+            assert result.rowcount == 1, result.rowcount
+
+        if version_md is not None:
+            conn.execute(
+                sql.insert(schema.TableVersion.__table__).values(
+                    tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
+                )
+            )
+
+        if schema_version_md is not None:
+            conn.execute(
+                sql.insert(schema.TableSchemaVersion.__table__).values(
+                    tbl_id=tbl_id,
+                    schema_version=schema_version_md.schema_version,
+                    md=dataclasses.asdict(schema_version_md),
+                )
+            )
+
+    def delete_tbl_md(self, tbl_id: UUID) -> None:
+        """
+        Deletes all table metadata from the store for the given table UUID.
+        """
+        conn = Env.get().conn
+        conn.execute(sql.delete(schema.TableSchemaVersion.__table__).where(schema.TableSchemaVersion.tbl_id == tbl_id))
+        conn.execute(sql.delete(schema.TableVersion.__table__).where(schema.TableVersion.tbl_id == tbl_id))
+        conn.execute(sql.delete(schema.Table.__table__).where(schema.Table.id == tbl_id))
+
+    def load_replica_md(self, tbl: Table) -> list[schema.FullTableMd]:
+        """
+        Load metadata for the given table along with all its ancestors. The values of TableMd.current_version and
+        TableMd.current_schema_version will be adjusted to ensure that the metadata represent a valid (internally
+        consistent) table state.
+        """
+        # TODO: First acquire X-locks for all relevant metadata entries
+
+        # Load metadata for every table in the TableVersionPath for `tbl`.
+        md = [self.load_tbl_md(tv.id, tv.effective_version) for tv in tbl._tbl_version_path.get_tbl_versions()]
+
+        # If `tbl` is a named pure snapshot, we're not quite done, since the snapshot metadata won't appear in the
+        # TableVersionPath. We need to prepend it separately.
+        if tbl._id != tbl._tbl_version.id:
+            snapshot_md = self.load_tbl_md(tbl._id, 0)
+            md = [snapshot_md, *md]
+
+        for ancestor_md in md[1:]:
+            # For replica metadata, we guarantee that the current_version and current_schema_version of TableMd
+            # match the corresponding values in TableVersionMd and TableSchemaVersionMd. This is to ensure that,
+            # when the metadata is later stored in the catalog of a different Pixeltable instance, the values of
+            # current_version and current_schema_version will always point to versions that are known to the
+            # destination catalog.
+            ancestor_md.tbl_md.current_version = ancestor_md.version_md.version
+            ancestor_md.tbl_md.current_schema_version = ancestor_md.schema_version_md.schema_version
+
+        return md
+
+    def _load_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
+        tbl_md, _, schema_version_md = self.load_tbl_md(tbl_id, effective_version)
         view_md = tbl_md.view_md
 
+        _logger.info(f'Loading table version: {tbl_id}:{effective_version}')
+        conn = Env.get().conn
+
         # load mutable view ids
         q = sql.select(schema.Table.id).where(
             sql.text(
@@ -729,7 +995,7 @@ class Catalog:
         if view_md is None:
             # this is a base table
             tbl_version = TableVersion(
-                tbl_record.id, tbl_md, effective_version, schema_version_md, mutable_views=mutable_views
+                tbl_id, tbl_md, effective_version, schema_version_md, mutable_views=mutable_views
             )
             return tbl_version
 
@@ -746,7 +1012,7 @@ class Catalog:
             base = base_path.tbl_version
 
         tbl_version = TableVersion(
-            tbl_record.id,
+            tbl_id,
             tbl_md,
             effective_version,
             schema_version_md,
@@ -758,16 +1024,25 @@ class Catalog:
 
     def _init_store(self) -> None:
         """One-time initialization of the stored catalog. Idempotent."""
+        self.create_user(None)
+        _logger.info('Initialized catalog.')
+
+    def create_user(self, user: Optional[str]) -> None:
+        """
+        Creates a catalog record (root directory) for the specified user, if one does not already exist.
+        """
         with Env.get().begin_xact():
             session = Env.get().session
-            if session.query(sql.func.count(schema.Dir.id)).scalar() > 0:
+            # See if there are any directories in the catalog matching the specified user.
+            if session.query(schema.Dir).where(schema.Dir.md['user'].astext == user).count() > 0:
+                # At least one such directory exists; no need to create a new one.
                 return
-            # create a top-level directory, so that every schema object has a directory
-            dir_md = schema.DirMd(name='', user=None, additional_md={})
+
+            dir_md = schema.DirMd(name='', user=user, additional_md={})
             dir_record = schema.Dir(parent_id=None, md=dataclasses.asdict(dir_md))
             session.add(dir_record)
             session.flush()
-            _logger.info(f'Initialized catalog')
+            _logger.info(f'Added root directory record for user: {user!r}')
 
     def _handle_path_collision(
         self, path: Path, expected_obj_type: type[SchemaObject], expected_snapshot: bool, if_exists: IfExistsParam
@@ -775,13 +1050,14 @@ class Catalog:
         obj, _, _ = self._prepare_dir_op(add_dir_path=path.parent, add_name=path.name)
 
         if if_exists == IfExistsParam.ERROR and obj is not None:
-            raise excs.Error(f'Path {path!r} is an existing {type(obj)._display_name()}')
+            raise excs.Error(f'Path {str(path)!r} is an existing {type(obj)._display_name()}')
         else:
             is_snapshot = isinstance(obj, View) and obj._tbl_version_path.is_snapshot()
             if obj is not None and (not isinstance(obj, expected_obj_type) or (expected_snapshot and not is_snapshot)):
                 obj_type_str = 'snapshot' if expected_snapshot else expected_obj_type._display_name()
                 raise excs.Error(
-                    f'Path {path!r} already exists but is not a {obj_type_str}. Cannot {if_exists.name.lower()} it.'
+                    f'Path {str(path)!r} already exists but is not a {obj_type_str}. '
+                    f'Cannot {if_exists.name.lower()} it.'
                 )
 
         if obj is None:
@@ -794,7 +1070,8 @@ class Catalog:
             dir_contents = self._get_dir_contents(obj._id)
             if len(dir_contents) > 0 and if_exists == IfExistsParam.REPLACE:
                 raise excs.Error(
-                    f'Directory {path!r} already exists and is not empty. Use `if_exists="replace_force"` to replace it.'
+                    f'Directory {str(path)!r} already exists and is not empty. '
+                    'Use `if_exists="replace_force"` to replace it.'
                 )
             self._drop_dir(obj._id, path, force=True)
         else: