pixeltable 0.3.11__py3-none-any.whl → 0.3.12__py3-none-any.whl

pixeltable/__init__.py

@@ -4,7 +4,7 @@ from .__version__ import __version__, __version_tuple__
 from .catalog import Column, InsertableTable, Table, UpdateStatus, View
 from .dataframe import DataFrame
 from .exceptions import Error, ExprEvalError, PixeltableWarning
-from .func import Aggregator, Function, expr_udf, query, uda, udf
+from .func import Aggregator, Function, expr_udf, query, retrieval_udf, uda, udf
 from .globals import (
     array,
     configure_logging,

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = '0.3.11'
-__version_tuple__ = (0, 3, 11)
+__version__ = '0.3.12'
+__version_tuple__ = (0, 3, 12)

pixeltable/catalog/catalog.py

@@ -33,16 +33,6 @@ if TYPE_CHECKING:
 _logger = logging.getLogger('pixeltable')
 
 
-def _lock_str(for_update: bool) -> str:
-    return 'X' if for_update else 'S'
-
-
-# TODO: remove once the concurrent update behavior has been debugged
-# def _debug_print(for_update: bool, msg: str) -> None:
-#     return
-#     print(f'{datetime.datetime.now()}: {_lock_str(for_update)}: {msg}')
-
-
 def _unpack_row(
     row: Optional[sql.engine.Row], entities: list[type[sql.orm.decl_api.DeclarativeBase]]
 ) -> Optional[list[Any]]:
@@ -82,11 +72,14 @@ def _retry_loop(op: Callable[..., T]) -> Callable[..., T]:
                 with Env.get().begin_xact():
                     return op(*args, **kwargs)
             except sql.exc.DBAPIError as e:
-                if isinstance(e.orig, psycopg.errors.SerializationFailure) and num_remaining_retries > 0:
-                    num_remaining_retries -= 1
-                    print(f'serialization failure:\n{e}')
-                    print('retrying ************************************************************')
-                    time.sleep(1)
+                if isinstance(e.orig, psycopg.errors.SerializationFailure):
+                    if num_remaining_retries > 0:
+                        num_remaining_retries -= 1
+                        # print(f'serialization failure:\n{e}')
+                        # print('retrying ************************************************************')
+                        time.sleep(1)
+                    else:
+                        raise excs.Error(f'Serialization retry limit ({_MAX_RETRIES}) exceeded') from e
                 else:
                     raise
 
@@ -123,20 +116,39 @@ class Catalog:
         self._tbls = {}  # don't use a defaultdict here, it doesn't cooperate with the debugger
         self._init_store()
 
+    @classmethod
+    def _lock_dir(cls, parent_id: Optional[UUID], dir_id: Optional[UUID], dir_name: Optional[str]) -> None:
+        """Update directory record(s) to sequentialize thread access. Lock is released when transaction commits.
+        If dir_id is present, then all other conditions are ignored.
+        Note that (parent_id==None) is a valid where condition.
+        If dir_id is not specified, the user from the environment is added to the directory filters.
+        """
+        user = Env.get().user
+        conn = Env.get().conn
+        q = sql.update(schema.Dir).values(lock_dummy=1)
+        if dir_id is not None:
+            q = q.where(schema.Dir.id == dir_id)
+        else:
+            q = q.where(schema.Dir.parent_id == parent_id)
+            if dir_name is not None:
+                q = q.where(schema.Dir.md['name'].astext == dir_name)
+            if user is not None:
+                q = q.where(schema.Dir.md['user'].astext == user)
+        conn.execute(q)
+
     def get_dir_path(self, dir_id: UUID) -> Path:
         """Return path for directory with given id"""
         conn = Env.get().conn
         names: list[str] = []
         while True:
             q = sql.select(schema.Dir).where(schema.Dir.id == dir_id)
-            # _debug_print(for_update=False, msg=f'dir id={dir_id}')
             row = conn.execute(q).one()
             dir = schema.Dir(**row._mapping)
             if dir.md['name'] == '':
                 break
             names.insert(0, dir.md['name'])
             dir_id = dir.parent_id
-        return Path('.'.join(names), empty_is_valid=True)
+        return Path('.'.join(names), empty_is_valid=True, allow_system_paths=True)
 
     @dataclasses.dataclass
     class DirEntry:
@@ -155,7 +167,6 @@ class Catalog:
         result: dict[str, Catalog.DirEntry] = {}
 
         q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id)
-        # _debug_print(for_update=False, msg=f'dirs parent_id={dir_id}')
         rows = conn.execute(q).all()
         for row in rows:
             dir = schema.Dir(**row._mapping)
@@ -165,7 +176,6 @@ class Catalog:
             result[dir.md['name']] = self.DirEntry(dir=dir, dir_entries=dir_contents, table=None)
 
         q = sql.select(schema.Table).where(schema.Table.dir_id == dir_id)
-        # _debug_print(for_update=False, msg=f'tbls parent_id={dir_id}')
         rows = conn.execute(q).all()
         for row in rows:
             tbl = schema.Table(**row._mapping)
@@ -175,6 +185,9 @@ class Catalog:
 
     @_retry_loop
     def move(self, path: Path, new_path: Path) -> None:
+        self._move(path, new_path)
+
+    def _move(self, path: Path, new_path: Path) -> None:
         _, dest_dir, src_obj = self._prepare_dir_op(
             add_dir_path=new_path.parent,
             add_name=new_path.name,
@@ -258,17 +271,14 @@ class Catalog:
         conn = Env.get().conn
 
         # check for subdirectory
+        if for_update:
+            self._lock_dir(dir_id, None, name)
         q = sql.select(schema.Dir).where(
             schema.Dir.parent_id == dir_id, schema.Dir.md['name'].astext == name, schema.Dir.md['user'].astext == user
         )
-        if for_update:
-            q = q.with_for_update()
-        # _debug_print(for_update, f'dir name={name!r} parent={dir_id}')
-        # row = conn.execute(q).one_or_none()
-        # if row is not None:
-        #     dir_record = schema.Dir(**row._mapping)
-        #     return Dir(dir_record.id, dir_record.parent_id, name)
         rows = conn.execute(q).all()
+        # The condition below can occur if there is a synchronization failure across multiple processes
+        # It indicates database inconsistency.
         if len(rows) > 1:
             raise AssertionError(rows)
         if len(rows) == 1:
@@ -283,7 +293,6 @@ class Catalog:
         )
         if for_update:
             q = q.with_for_update()
-        # _debug_print(for_update, f'table name={name!r} parent={dir_id}')
         tbl_id = conn.execute(q).scalar_one_or_none()
         if tbl_id is not None:
             if tbl_id not in self._tbls:
@@ -422,6 +431,155 @@ class Catalog:
         self._tbls[view._id] = view
         return view
 
+    @_retry_loop
+    def create_replica(self, path: Path, md: list[schema.FullTableMd], if_exists: IfExistsParam) -> Table:
+        """
+        Creates table, table_version, and table_schema_version records for a replica with the given metadata.
+        The metadata should be presented in standard "ancestor order", with the table being replicated at
+        list position 0 and the (root) base table at list position -1.
+        """
+        tbl_id = UUID(md[0].tbl_md.tbl_id)
+
+        # First handle path collisions (if_exists='ignore' or 'replace' or etc).
+        existing = self._handle_path_collision(path, View, False, if_exists)
+        if existing is not None:
+            if existing._id != tbl_id:
+                raise excs.Error(
+                    f"An attempt was made to create a replica table at {path!r} with if_exists='ignore', "
+                    'but a different table already exists at that location.'
+                )
+            assert isinstance(existing, View)
+            return existing
+
+        # Ensure that the system directory exists.
+        self._create_dir(Path('_system', allow_system_paths=True), if_exists=IfExistsParam.IGNORE, parents=False)
+
+        # Now check to see if this table already exists in the catalog.
+        # TODO: Handle concurrency in create_replica()
+        existing = Catalog.get().get_table_by_id(tbl_id)
+        if existing is not None:
+            existing_path = Path(existing._path(), allow_system_paths=True)
+            # It does exist. If it's a non-system table, that's an error: it's already been replicated.
+            if not existing_path.is_system_path:
+                raise excs.Error(
+                    f'That table has already been replicated as {existing._path()!r}. \n'
+                    f'Drop the existing replica if you wish to re-create it.'
+                )
+            # If it's a system table, then this means it was created at some point as the ancestor of some other
+            # table (a snapshot-over-snapshot scenario). In that case, we simply move it to the new (named) location.
+            self._move(existing_path, path)
+
+        # Now store the metadata for this replica. In the case where the table already exists (and was just moved
+        # into a named location), this will be a no-op, but it still serves to validate that the newly received
+        # metadata is identical to what's in the catalog.
+        self.__store_replica_md(path, md[0])
+
+        # Now store the metadata for all of this table's proper ancestors. If one or more proper ancestors
+        # do not yet exist in the store, they will be created as anonymous system tables.
+        for ancestor_md in md[1:]:
+            ancestor_id = UUID(ancestor_md.tbl_md.tbl_id)
+            replica = Catalog.get().get_table_by_id(ancestor_id)
+            replica_path: Path
+            if replica is None:
+                # We've never seen this table before. Create a new anonymous system table for it.
+                replica_path = Path(f'_system.replica_{ancestor_id.hex}', allow_system_paths=True)
+            else:
+                # The table already exists in the catalog. The existing path might be a system path (if the table
+                # was created as an anonymous base table of some other table), or it might not (if it's a snapshot
+                # that was directly replicated by the user at some point). In either case, use the existing path.
+                replica_path = Path(replica._path(), allow_system_paths=True)
+
+            # Store the metadata; it could be a new version (in which case a new record will be created) or a
+            # known version (in which case the newly received metadata will be validated as identical).
+            self.__store_replica_md(replica_path, ancestor_md)
+
+        # Update the catalog (as a final step, after all DB operations completed successfully).
+        # Only the table being replicated is actually made visible in the catalog.
+        self._tbls[tbl_id] = self._load_tbl(tbl_id)
+        return self._tbls[tbl_id]
+
+    def __store_replica_md(self, path: Path, md: schema.FullTableMd) -> None:
+        _logger.info(f'Creating replica table at {path!r} with ID: {md.tbl_md.tbl_id}')
+        # TODO: Handle concurrency
+        dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
+        assert dir is not None
+
+        conn = Env.get().conn
+        tbl_id = md.tbl_md.tbl_id
+
+        new_tbl_md: Optional[schema.TableMd] = None
+        new_version_md: Optional[schema.TableVersionMd] = None
+        new_schema_version_md: Optional[schema.TableSchemaVersionMd] = None
+
+        # We need to ensure that the table metadata in the catalog always reflects the latest observed version of
+        # this table. (In particular, if this is a base table, then its table metadata need to be consistent
+        # with the latest version of this table having a replicated view somewhere in the catalog.)
+        q: sql.Executable = sql.select(schema.Table.md).where(schema.Table.id == tbl_id)
+        existing_md_row = conn.execute(q).one_or_none()
+
+        if existing_md_row is None:
+            # No existing table, so create a new record.
+            q = sql.insert(schema.Table.__table__).values(
+                id=tbl_id,
+                dir_id=dir._id,
+                md=dataclasses.asdict(
+                    dataclasses.replace(md.tbl_md, name=path.name, user=Env.get().user, is_replica=True)
+                ),
+            )
+            conn.execute(q)
+        else:
+            assert existing_md_row.md['is_replica']
+            if md.tbl_md.current_version > existing_md_row.md['current_version']:
+                # New metadata is more recent than the metadata currently stored in the DB; we'll update the record
+                # in place in the DB.
+                new_tbl_md = dataclasses.replace(md.tbl_md, name=path.name, user=Env.get().user, is_replica=True)
+
+        # Now see if a TableVersion record already exists in the DB for this table version. If not, insert it. If
+        # it already exists, check that the existing record is identical to the new one.
+        q = (
+            sql.select(schema.TableVersion.md)
+            .where(schema.TableVersion.tbl_id == tbl_id)
+            .where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {md.version_md.version}"))
+        )
+        existing_version_md_row = conn.execute(q).one_or_none()
+        if existing_version_md_row is None:
+            new_version_md = md.version_md
+        else:
+            existing_version_md = schema.md_from_dict(schema.TableVersionMd, existing_version_md_row.md)
+            if existing_version_md != md.version_md:
+                raise excs.Error(
+                    f'The version metadata for the replica {path!r}:{md.version_md.version} is inconsistent with '
+                    'the metadata recorded from a prior replica.\n'
+                    'This is likely due to data corruption in the replicated table.'
+                )
+
+        # Do the same thing for TableSchemaVersion.
+        q = (
+            sql.select(schema.TableSchemaVersion.md)
+            .where(schema.TableSchemaVersion.tbl_id == tbl_id)
+            .where(
+                sql.text(
+                    f"({schema.TableSchemaVersion.__table__}.md->>'schema_version')::int = "
+                    f'{md.schema_version_md.schema_version}'
+                )
+            )
+        )
+        existing_schema_version_md_row = conn.execute(q).one_or_none()
+        if existing_schema_version_md_row is None:
+            new_schema_version_md = md.schema_version_md
+        else:
+            existing_schema_version_md = schema.md_from_dict(
+                schema.TableSchemaVersionMd, existing_schema_version_md_row.md
+            )
+            if existing_schema_version_md != md.schema_version_md:
+                raise excs.Error(
+                    f'The schema version metadata for the replica {path!r}:{md.schema_version_md.schema_version} '
+                    'is inconsistent with the metadata recorded from a prior replica.\n'
+                    'This is likely due to data corruption in the replicated table.'
+                )
+
+        self.store_tbl_md(UUID(tbl_id), new_tbl_md, new_version_md, new_schema_version_md)
+
     @_retry_loop
     def get_table(self, path: Path) -> Table:
         obj = Catalog.get()._get_schema_object(path, expected=Table, raise_if_not_exists=True)
@@ -480,6 +638,9 @@ class Catalog:
 
     @_retry_loop
     def create_dir(self, path: Path, if_exists: IfExistsParam, parents: bool) -> Dir:
+        return self._create_dir(path, if_exists, parents)
+
+    def _create_dir(self, path: Path, if_exists: IfExistsParam, parents: bool) -> Dir:
         # existing = self._handle_path_collision(path, Dir, False, if_exists)
         # if existing is not None:
         #     assert isinstance(existing, Dir)
@@ -535,7 +696,8 @@ class Catalog:
                 raise excs.Error(f'Directory {str(dir_path)!r} is not empty.')
 
         # drop existing subdirs
-        dir_q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id).with_for_update()
+        self._lock_dir(dir_id, None, None)
+        dir_q = sql.select(schema.Dir).where(schema.Dir.parent_id == dir_id)
         for row in conn.execute(dir_q).all():
             self._drop_dir(row.id, dir_path.append(row.md['name']), force=True)
 
@@ -548,7 +710,6 @@ class Catalog:
                 self._drop_tbl(tbl, force=True, is_replace=False)
 
         # self.drop_dir(dir_id)
-        # _debug_print(for_update=True, msg=f'drop dir id={dir_id}')
         conn.execute(sql.delete(schema.Dir).where(schema.Dir.id == dir_id))
         _logger.info(f'Removed directory {str(dir_path)!r}.')
 
@@ -558,7 +719,6 @@ class Catalog:
         q = sql.select(schema.Table.id).where(sql.text(f"md->'view_md'->'base_versions'->0->>0 = {tbl_id.hex!r}"))
         if for_update:
             q = q.with_for_update()
-        # _debug_print(for_update=False, msg=f'views of tbl id={tbl_id}')
         result = [r[0] for r in conn.execute(q).all()]
         return result
 
@@ -582,10 +742,9 @@ class Catalog:
     def get_dir(self, dir_id: UUID, for_update: bool = False) -> Optional[Dir]:
         """Return the Dir with the given id, or None if it doesn't exist"""
         conn = Env.get().conn
-        q = sql.select(schema.Dir).where(schema.Dir.id == dir_id)
         if for_update:
-            q = q.with_for_update()
-        # _debug_print(for_update=False, msg=f'dir id={dir_id!r}')
+            self._lock_dir(None, dir_id, None)
+        q = sql.select(schema.Dir).where(schema.Dir.id == dir_id)
         row = conn.execute(q).one_or_none()
         if row is None:
             return None
@@ -594,31 +753,27 @@ class Catalog:
 
     def _get_dir(self, path: Path, for_update: bool = False) -> Optional[schema.Dir]:
         """
-        Locking protocol:
-        - S locks on all ancestors
-        - X lock on dir if for_update == True, otherwise also an S lock
+        Locking protocol: X locks on all ancestors
         """
         user = Env.get().user
         conn = Env.get().conn
         if path.is_root:
-            q = sql.select(schema.Dir).where(schema.Dir.parent_id.is_(None), schema.Dir.md['user'].astext == user)
             if for_update:
-                q = q.with_for_update()
-            # _debug_print(for_update, 'root dir')
+                self._lock_dir(parent_id=None, dir_id=None, dir_name='')
+            q = sql.select(schema.Dir).where(schema.Dir.parent_id.is_(None), schema.Dir.md['user'].astext == user)
             row = conn.execute(q).one_or_none()
             return schema.Dir(**row._mapping) if row is not None else None
         else:
             parent_dir = self._get_dir(path.parent, for_update=False)
             if parent_dir is None:
                 return None
+            if for_update:
+                self._lock_dir(parent_id=parent_dir.id, dir_id=None, dir_name=path.name)
             q = sql.select(schema.Dir).where(
                 schema.Dir.parent_id == parent_dir.id,
                 schema.Dir.md['name'].astext == path.name,
                 schema.Dir.md['user'].astext == user,
             )
-            if for_update:
-                q = q.with_for_update()
-            # _debug_print(for_update, f'dir {str(path)}')
             row = conn.execute(q).one_or_none()
             return schema.Dir(**row._mapping) if row is not None else None
 
@@ -641,7 +796,6 @@ class Catalog:
             )
             .where(schema.Table.id == tbl_id)
         )
-        # _debug_print(for_update=False, msg=f'load table id={tbl_id!r}')
         row = conn.execute(q).one_or_none()
         if row is None:
             return None
@@ -680,13 +834,19 @@ class Catalog:
         # TODO: also load mutable views
         return view
 
-    def _load_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
-        _logger.info(f'Loading table version: {tbl_id}:{effective_version}')
+    def load_tbl_md(self, tbl_id: UUID, effective_version: Optional[int]) -> schema.FullTableMd:
+        """
+        Loads metadata from the store for a given table UUID and version.
+        """
+        _logger.info(f'Loading metadata for table version: {tbl_id}:{effective_version}')
         conn = Env.get().conn
+
         q = (
-            sql.select(schema.Table, schema.TableSchemaVersion)
+            sql.select(schema.Table, schema.TableVersion, schema.TableSchemaVersion)
             .select_from(schema.Table)
             .where(schema.Table.id == tbl_id)
+            .join(schema.TableVersion)
+            .where(schema.TableVersion.tbl_id == tbl_id)
             .join(schema.TableSchemaVersion)
             .where(schema.TableSchemaVersion.tbl_id == tbl_id)
         )
@@ -698,16 +858,11 @@ class Catalog:
             # JOIN TableVersion tv ON (tv.tbl_id = tbl_id AND tv.version = effective_version)
             # JOIN TableSchemaVersion tsv ON (tsv.tbl_id = tbl_id AND tv.md.schema_version = tsv.schema_version)
             # WHERE t.id = tbl_id
-            q = (
-                q.join(schema.TableVersion)
-                .where(schema.TableVersion.tbl_id == tbl_id)
-                .where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {effective_version}"))
-                .where(
-                    sql.text(
-                        (
-                            f"({schema.TableVersion.__table__}.md->>'schema_version')::int = "
-                            f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
-                        )
+            q = q.where(sql.text(f"({schema.TableVersion.__table__}.md->>'version')::int = {effective_version}")).where(
+                sql.text(
+                    (
+                        f"({schema.TableVersion.__table__}.md->>'schema_version')::int = "
+                        f'{schema.TableSchemaVersion.__table__}.{schema.TableSchemaVersion.schema_version.name}'
                     )
                 )
             )
@@ -715,9 +870,15 @@ class Catalog:
             # we are loading the current version
             # SELECT *
             # FROM Table t
+            # JOIN TableVersion tv ON (tv.tbl_id = tbl_id AND t.current_version = tv.version)
             # JOIN TableSchemaVersion tsv ON (tsv.tbl_id = tbl_id AND t.current_schema_version = tsv.schema_version)
             # WHERE t.id = tbl_id
             q = q.where(
+                sql.text(
+                    f"({schema.Table.__table__}.md->>'current_version')::int = "
+                    f'{schema.TableVersion.__table__}.{schema.TableVersion.version.name}'
+                )
+            ).where(
                 sql.text(
                     (
                         f"({schema.Table.__table__}.md->>'current_schema_version')::int = "
@@ -727,11 +888,100 @@ class Catalog:
             )
 
         row = conn.execute(q).one_or_none()
-        tbl_record, schema_version_record = _unpack_row(row, [schema.Table, schema.TableSchemaVersion])
+        assert row is not None, f'Table record not found: {tbl_id}:{effective_version}'
+        tbl_record, version_record, schema_version_record = _unpack_row(
+            row, [schema.Table, schema.TableVersion, schema.TableSchemaVersion]
+        )
+        assert tbl_record.id == tbl_id
         tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
+        version_md = schema.md_from_dict(schema.TableVersionMd, version_record.md)
         schema_version_md = schema.md_from_dict(schema.TableSchemaVersionMd, schema_version_record.md)
+
+        return schema.FullTableMd(tbl_md, version_md, schema_version_md)
+
+    def store_tbl_md(
+        self,
+        tbl_id: UUID,
+        tbl_md: Optional[schema.TableMd],
+        version_md: Optional[schema.TableVersionMd],
+        schema_version_md: Optional[schema.TableSchemaVersionMd],
+    ) -> None:
+        """
+        Stores metadata to the DB. If specified, `tbl_md` will be updated in place (only one such record can exist
+        per UUID); `version_md` and `schema_version_md` will be inserted as new records.
+
+        If inserting `version_md` or `schema_version_md` would be a primary key violation, an exception will be raised.
+        """
+        conn = Env.get().conn
+
+        if tbl_md is not None:
+            result = conn.execute(
+                sql.update(schema.Table.__table__)
+                .values({schema.Table.md: dataclasses.asdict(tbl_md)})
+                .where(schema.Table.id == tbl_id)
+            )
+            assert result.rowcount == 1, result.rowcount
+
+        if version_md is not None:
+            conn.execute(
+                sql.insert(schema.TableVersion.__table__).values(
+                    tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
+                )
+            )
+
+        if schema_version_md is not None:
+            conn.execute(
+                sql.insert(schema.TableSchemaVersion.__table__).values(
+                    tbl_id=tbl_id,
+                    schema_version=schema_version_md.schema_version,
+                    md=dataclasses.asdict(schema_version_md),
+                )
+            )
+
+    def delete_tbl_md(self, tbl_id: UUID) -> None:
+        """
+        Deletes all table metadata from the store for the given table UUID.
+        """
+        conn = Env.get().conn
+        conn.execute(sql.delete(schema.TableSchemaVersion.__table__).where(schema.TableSchemaVersion.tbl_id == tbl_id))
+        conn.execute(sql.delete(schema.TableVersion.__table__).where(schema.TableVersion.tbl_id == tbl_id))
+        conn.execute(sql.delete(schema.Table.__table__).where(schema.Table.id == tbl_id))
+
+    def load_replica_md(self, tbl: Table) -> list[schema.FullTableMd]:
+        """
+        Load metadata for the given table along with all its ancestors. The values of TableMd.current_version and
+        TableMd.current_schema_version will be adjusted to ensure that the metadata represent a valid (internally
+        consistent) table state.
+        """
+        # TODO: First acquire X-locks for all relevant metadata entries
+
+        # Load metadata for every table in the TableVersionPath for `tbl`.
+        md = [self.load_tbl_md(tv.id, tv.effective_version) for tv in tbl._tbl_version_path.get_tbl_versions()]
+
+        # If `tbl` is a named pure snapshot, we're not quite done, since the snapshot metadata won't appear in the
+        # TableVersionPath. We need to prepend it separately.
+        if tbl._id != tbl._tbl_version.id:
+            snapshot_md = self.load_tbl_md(tbl._id, 0)
+            md = [snapshot_md, *md]
+
+        for ancestor_md in md[1:]:
+            # For replica metadata, we guarantee that the current_version and current_schema_version of TableMd
+            # match the corresponding values in TableVersionMd and TableSchemaVersionMd. This is to ensure that,
+            # when the metadata is later stored in the catalog of a different Pixeltable instance, the values of
+            # current_version and current_schema_version will always point to versions that are known to the
+            # destination catalog.
+            ancestor_md.tbl_md.current_version = ancestor_md.version_md.version
+            ancestor_md.tbl_md.current_schema_version = ancestor_md.schema_version_md.schema_version
+
+        return md
+
+    def _load_tbl_version(self, tbl_id: UUID, effective_version: Optional[int]) -> Optional[TableVersion]:
+        tbl_md, _, schema_version_md = self.load_tbl_md(tbl_id, effective_version)
         view_md = tbl_md.view_md
 
+        _logger.info(f'Loading table version: {tbl_id}:{effective_version}')
+        conn = Env.get().conn
+
         # load mutable view ids
         q = sql.select(schema.Table.id).where(
             sql.text(
@@ -745,7 +995,7 @@ class Catalog:
         if view_md is None:
             # this is a base table
             tbl_version = TableVersion(
-                tbl_record.id, tbl_md, effective_version, schema_version_md, mutable_views=mutable_views
+                tbl_id, tbl_md, effective_version, schema_version_md, mutable_views=mutable_views
             )
             return tbl_version
 
@@ -762,7 +1012,7 @@ class Catalog:
             base = base_path.tbl_version
 
         tbl_version = TableVersion(
-            tbl_record.id,
+            tbl_id,
             tbl_md,
             effective_version,
             schema_version_md,

pixeltable/catalog/globals.py

@@ -100,14 +100,14 @@ class IfNotExistsParam(enum.Enum):
             raise excs.Error(f'{param_name} must be one of: [{val_strs}]') from None
 
 
-def is_valid_identifier(name: str) -> bool:
-    return name.isidentifier() and not name.startswith('_')
+def is_valid_identifier(name: str, allow_system_identifiers: bool = False) -> bool:
+    return name.isidentifier() and (allow_system_identifiers or not name.startswith('_'))
 
 
-def is_valid_path(path: str, empty_is_valid: bool) -> bool:
-    if not path:
+def is_valid_path(path: str, empty_is_valid: bool, allow_system_paths: bool = False) -> bool:
+    if path == '':
         return empty_is_valid
-    return all(is_valid_identifier(part) for part in path.split('.'))
+    return all(is_valid_identifier(part, allow_system_paths) for part in path.split('.'))
 
 
 def is_system_column_name(name: str) -> bool:

pixeltable/catalog/insertable_table.py

@@ -17,6 +17,7 @@ from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
 
 if TYPE_CHECKING:
+    from pixeltable import exprs
     from pixeltable.globals import TableDataSource
     from pixeltable.io.table_data_conduit import TableDataConduit
 
@@ -210,7 +211,7 @@ class InsertableTable(Table):
                     msg = str(e)
                     raise excs.Error(f'Error in column {col.name}: {msg[0].lower() + msg[1:]}\nRow: {row}') from e
 
-    def delete(self, where: Optional['pxt.exprs.Expr'] = None) -> UpdateStatus:
+    def delete(self, where: Optional['exprs.Expr'] = None) -> UpdateStatus:
         """Delete rows in this table.
 
         Args:

pixeltable/catalog/path.py

@@ -11,8 +11,8 @@ _logger = logging.getLogger('pixeltable')
 
 
 class Path:
-    def __init__(self, path: str, empty_is_valid: bool = False):
-        if not is_valid_path(path, empty_is_valid):
+    def __init__(self, path: str, empty_is_valid: bool = False, allow_system_paths: bool = False):
+        if not is_valid_path(path, empty_is_valid, allow_system_paths):
             raise excs.Error(f"Invalid path format: '{path}'")
         self.components = path.split('.')
 
@@ -29,21 +29,25 @@ class Path:
     def is_root(self) -> bool:
         return not self.components[0]
 
+    @property
+    def is_system_path(self) -> bool:
+        return self.components[0].startswith('_')
+
     @property
     def parent(self) -> Path:
         if len(self.components) == 1:
             if self.is_root:
                 return self
             else:
-                return Path('', empty_is_valid=True)
+                return Path('', empty_is_valid=True, allow_system_paths=True)
         else:
-            return Path('.'.join(self.components[:-1]))
+            return Path('.'.join(self.components[:-1]), allow_system_paths=True)
 
     def append(self, name: str) -> Path:
         if self.is_root:
-            return Path(name)
+            return Path(name, allow_system_paths=True)
         else:
-            return Path(f'{self!s}.{name}')
+            return Path(f'{self}.{name}', allow_system_paths=True)
 
     def is_ancestor(self, other: Path, is_parent: bool = False) -> bool:
         """
@@ -67,6 +71,9 @@ class Path:
             for i in range(0, len(self.components)):
                 yield Path('.'.join(self.components[0:i]), empty_is_valid=True)
 
+    def __repr__(self) -> str:
+        return repr(str(self))
+
     def __str__(self) -> str:
         return '.'.join(self.components)