pixeltable 0.3.14__py3-none-any.whl → 0.4.0rc1__py3-none-any.whl

pixeltable/io/table_data_conduit.py

@@ -23,7 +23,6 @@ from .utils import normalize_schema_names
 
 _logger = logging.getLogger('pixeltable')
 
-# ---------------------------------------------------------------------------------------------------------
 
 if TYPE_CHECKING:
     import datasets  # type: ignore[import-untyped]
@@ -46,9 +45,6 @@ class TableDataConduitFormat(str, enum.Enum):
         return False
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 @dataclass
 class TableDataConduit:
     source: TableDataSource
@@ -129,9 +125,6 @@ class TableDataConduit:
             raise excs.Error(f'Missing required column(s) ({", ".join(missing_cols)})')
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class DFTableDataConduit(TableDataConduit):
     pxt_df: pxt.DataFrame = None
 
@@ -155,9 +148,6 @@ class DFTableDataConduit(TableDataConduit):
         self.check_source_columns_are_insertable(self.pxt_df.schema.keys())
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class RowDataTableDataConduit(TableDataConduit):
     raw_rows: Optional[RowData] = None
     disable_mapping: bool = True
@@ -235,9 +225,6 @@ class RowDataTableDataConduit(TableDataConduit):
             yield self.valid_rows
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class PandasTableDataConduit(TableDataConduit):
     pd_df: pd.DataFrame = None
     batch_count: int = 0
@@ -293,9 +280,6 @@ class PandasTableDataConduit(TableDataConduit):
             yield self.valid_rows
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class CSVTableDataConduit(TableDataConduit):
     @classmethod
     def from_tds(cls, tds: TableDataConduit) -> 'PandasTableDataConduit':
@@ -307,9 +291,6 @@ class CSVTableDataConduit(TableDataConduit):
         return PandasTableDataConduit.from_tds(t)
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class ExcelTableDataConduit(TableDataConduit):
     @classmethod
     def from_tds(cls, tds: TableDataConduit) -> 'PandasTableDataConduit':
@@ -321,9 +302,6 @@ class ExcelTableDataConduit(TableDataConduit):
         return PandasTableDataConduit.from_tds(t)
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class JsonTableDataConduit(TableDataConduit):
     @classmethod
     def from_tds(cls, tds: TableDataConduit) -> RowDataTableDataConduit:
@@ -346,9 +324,6 @@ class JsonTableDataConduit(TableDataConduit):
         return t2
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class HFTableDataConduit(TableDataConduit):
     hf_ds: Optional[Union[datasets.Dataset, datasets.DatasetDict]] = None
     column_name_for_split: Optional[str] = None
@@ -478,9 +453,6 @@ class HFTableDataConduit(TableDataConduit):
                 yield batch
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class ParquetTableDataConduit(TableDataConduit):
     pq_ds: Optional[ParquetDataset] = None
 
@@ -542,9 +514,6 @@ class ParquetTableDataConduit(TableDataConduit):
             raise e
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class UnkTableDataConduit(TableDataConduit):
     """Source type is not known at the time of creation"""
 

pixeltable/metadata/__init__.py

@@ -16,7 +16,7 @@ _console_logger = ConsoleLogger(logging.getLogger('pixeltable'))
 
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 35
+VERSION = 36
 
 
 def create_system_info(engine: sql.engine.Engine) -> None:

pixeltable/metadata/converters/convert_13.py

@@ -12,9 +12,9 @@ _logger = logging.getLogger('pixeltable')
 @register_converter(version=13)
 def _(engine: sql.engine.Engine) -> None:
     with engine.begin() as conn:
-        for row in conn.execute(sql.select(Table)):
+        for row in conn.execute(sql.select(Table.id, Table.md)):
             id = row[0]
-            md = row[2]
+            md = row[1]
             updated_md = __update_md(md)
             if updated_md != md:
                 _logger.info(f'Updating schema for table: {id}')

pixeltable/metadata/converters/convert_30.py

@@ -1,33 +1,28 @@
 import copy
+from uuid import UUID
 
 import sqlalchemy as sql
 
 from pixeltable.metadata import register_converter
 from pixeltable.metadata.converters.util import (
-    convert_table_record,
+    convert_table_md,
     convert_table_schema_version_record,
     convert_table_version_record,
 )
-from pixeltable.metadata.schema import Table, TableSchemaVersion, TableVersion
+from pixeltable.metadata.schema import TableSchemaVersion, TableVersion
 
 
 @register_converter(version=30)
 def _(engine: sql.engine.Engine) -> None:
-    convert_table_record(engine, table_record_updater=__update_table_record)
+    convert_table_md(engine, table_md_updater=__update_table_md)
     convert_table_version_record(engine, table_version_record_updater=__update_table_version_record)
     convert_table_schema_version_record(
         engine, table_schema_version_record_updater=__update_table_schema_version_record
     )
 
 
-def __update_table_record(record: Table) -> None:
-    """
-    Update TableMd with table_id
-    """
-    assert isinstance(record.md, dict)
-    md = copy.copy(record.md)
-    md['tbl_id'] = str(record.id)
-    record.md = md
+def __update_table_md(md: dict, tbl_id: UUID) -> None:
+    md['tbl_id'] = str(tbl_id)
 
 
 def __update_table_version_record(record: TableVersion) -> None:

pixeltable/metadata/converters/convert_35.py

@@ -0,0 +1,9 @@
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+
+
+@register_converter(version=35)
+def _(engine: sql.engine.Engine) -> None:
+    with engine.begin() as conn:
+        conn.execute(sql.text('ALTER TABLE tables ADD COLUMN lock_dummy int8'))

pixeltable/metadata/converters/util.py

@@ -33,9 +33,10 @@ def convert_table_md(
             the original entry will be replaced, and the traversal will continue with `v'`.
     """
     with engine.begin() as conn:
-        for row in conn.execute(sql.select(Table)):
+        # avoid a SELECT * here, which breaks when we add new columns to Table
+        for row in conn.execute(sql.select(Table.id, Table.md)):
             tbl_id = row[0]
-            table_md = row[2]
+            table_md = row[1]
             assert isinstance(table_md, dict)
             updated_table_md = copy.deepcopy(table_md)
             if table_md_updater is not None:
@@ -145,13 +146,6 @@ def __update_schema_column(table_schema_version_md: dict, schema_column_updater:
         schema_column_updater(schema_col)
 
 
-def convert_table_record(engine: sql.engine.Engine, table_record_updater: Optional[Callable[[Table], None]]) -> None:
-    with sql.orm.Session(engine, future=True) as session:
-        for record in session.query(Table).all():
-            table_record_updater(record)
-        session.commit()
-
-
 def convert_table_version_record(
     engine: sql.engine.Engine, table_version_record_updater: Optional[Callable[[TableVersion], None]]
 ) -> None:

pixeltable/metadata/notes.py

@@ -2,6 +2,7 @@
 # rather than as a comment, so that the existence of a description can be enforced by
 # the unit tests when new versions are added.
 VERSION_NOTES = {
+    36: 'Added Table.lock_dummy',
     35: 'Track reference_tbl in ColumnRef',
     34: 'Set default value for is_pk field in column metadata to False',
     33: 'Add is_replica field to table metadata',

pixeltable/metadata/schema.py

@@ -84,7 +84,8 @@ class Dir(Base):
     )
     parent_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), ForeignKey('dirs.id'), nullable=True)
     md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)  # DirMd
-    # This field is updated to synchronize database operations across multiple sessions
+
+    # used to force acquisition of an X-lock via an Update stmt
     lock_dummy: orm.Mapped[int] = orm.mapped_column(BigInteger, nullable=True)
 
 
@@ -200,6 +201,9 @@ class Table(Base):
     dir_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), ForeignKey('dirs.id'), nullable=False)
     md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)  # TableMd
 
+    # used to force acquisition of an X-lock via an Update stmt
+    lock_dummy: orm.Mapped[int] = orm.mapped_column(BigInteger, nullable=True)
+
 
 @dataclasses.dataclass
 class TableVersionMd:

pixeltable/plan.py

@@ -289,7 +289,7 @@ class Planner:
 
         # create InMemoryDataNode for 'rows'
         plan: exec.ExecNode = exec.InMemoryDataNode(
-            TableVersionHandle(tbl.id, tbl.effective_version), rows, row_builder, tbl.next_rowid
+            TableVersionHandle(tbl.id, tbl.effective_version), rows, row_builder, tbl.next_row_id
         )
 
         media_input_col_info = [
@@ -385,7 +385,7 @@ class Planner:
 
         cls.__check_valid_columns(tbl.tbl_version.get(), recomputed_cols, 'updated in')
 
-        recomputed_base_cols = {col for col in recomputed_cols if col.tbl == tbl.tbl_version}
+        recomputed_base_cols = {col for col in recomputed_cols if col.tbl.id == tbl.tbl_version.id}
         copied_cols = [
             col
             for col in target.cols_by_id.values()
@@ -409,7 +409,7 @@ class Planner:
         for i, col in enumerate(all_base_cols):
             plan.row_builder.add_table_column(col, select_list[i].slot_idx)
         recomputed_user_cols = [c for c in recomputed_cols if c.name is not None]
-        return plan, [f'{c.tbl.get().name}.{c.name}' for c in updated_cols + recomputed_user_cols], recomputed_user_cols
+        return plan, [f'{c.tbl.name}.{c.name}' for c in updated_cols + recomputed_user_cols], recomputed_user_cols
 
     @classmethod
     def __check_valid_columns(
@@ -465,7 +465,7 @@ class Planner:
         recomputed_cols.update(idx_val_cols)
         # we only need to recompute stored columns (unstored ones are substituted away)
         recomputed_cols = {c for c in recomputed_cols if c.is_stored}
-        recomputed_base_cols = {col for col in recomputed_cols if col.tbl == target}
+        recomputed_base_cols = {col for col in recomputed_cols if col.tbl.id == target.id}
         copied_cols = [
             col
             for col in target.cols_by_id.values()

pixeltable/share/packager.py

@@ -7,6 +7,7 @@ import urllib.request
 import uuid
 from pathlib import Path
 from typing import Any, Iterator, Optional
+from uuid import UUID
 
 import more_itertools
 import pyarrow as pa
@@ -17,6 +18,7 @@ import pixeltable as pxt
 from pixeltable import catalog, exceptions as excs, metadata
 from pixeltable.env import Env
 from pixeltable.metadata import schema
+from pixeltable.utils import sha256sum
 from pixeltable.utils.media_store import MediaStore
 
 _logger = logging.getLogger('pixeltable')
@@ -50,7 +52,7 @@ class TablePackager:
         self.media_files = {}
 
         # Load metadata
-        with Env.get().begin_xact():
+        with catalog.Catalog.get().begin_xact(for_write=False):
             tbl_md = catalog.Catalog.get().load_replica_md(table)
             self.md = {
                 'pxt_version': pxt.__version__,
@@ -65,15 +67,15 @@ class TablePackager:
         Export the table to a tarball containing Parquet tables and media files.
         """
         assert not self.tmp_dir.exists()  # Packaging can only be done once per TablePackager instance
-        _logger.info(f"Packaging table '{self.table._path}' and its ancestors in: {self.tmp_dir}")
+        _logger.info(f"Packaging table '{self.table._path()}' and its ancestors in: {self.tmp_dir}")
         self.tmp_dir.mkdir()
         with open(self.tmp_dir / 'metadata.json', 'w', encoding='utf8') as fp:
             json.dump(self.md, fp)
         self.tables_dir = self.tmp_dir / 'tables'
         self.tables_dir.mkdir()
-        with Env.get().begin_xact():
+        with catalog.Catalog.get().begin_xact(for_write=False):
             for tv in self.table._tbl_version_path.get_tbl_versions():
-                _logger.info(f"Exporting table '{tv.get().name}:{tv.get().version}'.")
+                _logger.info(f"Exporting table '{tv.get().versioned_name}'.")
                 self.__export_table(tv.get())
         _logger.info('Building archive.')
         bundle_path = self.__build_tarball()
@@ -88,7 +90,7 @@ class TablePackager:
         assert any(tv.id == base.id for base in self.table._tbl_version_path.get_tbl_versions())
         sql_types = {col.name: col.type for col in tv.store_tbl.sa_tbl.columns}
         media_cols: set[str] = set()
-        for col in tv.cols_by_name.values():
+        for col in tv.cols:
             if col.is_stored and col.col_type.is_media_type():
                 media_cols.add(col.store_name())
 
@@ -182,7 +184,12 @@ class TablePackager:
             path = Path(urllib.parse.unquote(urllib.request.url2pathname(parsed_url.path)))
             if path not in self.media_files:
                 # Create a new entry in the `media_files` dict so that we can copy the file into the tarball later.
-                dest_name = f'{uuid.uuid4().hex}{path.suffix}'
+                # We name the media files in the archive by their SHA256 hash. This ensures that we can properly
+                # deduplicate and validate them later.
+                # If we get a collision, it's not a problem; it just means we have two identical files (which will
+                # be conveniently deduplicated in the bundle).
+                sha = sha256sum(path)
+                dest_name = f'{sha}{path.suffix}'
                 self.media_files[path] = dest_name
             return f'pxtmedia://{self.media_files[path]}'
         # For any type of URL other than a local file, just return the URL as-is.
@@ -247,13 +254,26 @@ class TableRestorer:
         tbl_md = [schema.FullTableMd.from_dict(t) for t in self.md['md']['tables']]
 
         # Create the replica table
-        # TODO: This needs to be made concurrency-safe.
-        replica_tbl = catalog.Catalog.get().create_replica(catalog.Path(self.tbl_path), tbl_md)
-        assert replica_tbl._tbl_version.get().is_snapshot
+        # The logic here needs to be completely restructured in order to make it concurrency-safe.
+        # - Catalog.create_replica() needs to write the metadata and also create the physical store tables
+        #   and populate them, otherwise concurrent readers will see an inconsistent state (table metadata w/o
+        #   an actual table)
+        # - this could be done one replica at a time (instead of the entire hierarchy)
+        cat = catalog.Catalog.get()
+        cat.create_replica(catalog.Path(self.tbl_path), tbl_md)
+        # don't call get_table() until after the calls to create_replica() and __import_table() below;
+        # the TV instances created by get_table() would be replaced by create_replica(), which creates duplicate
+        # TV instances for the same replica version, which then leads to failures when constructing queries
 
         # Now we need to instantiate and load data for replica_tbl and its ancestors, except that we skip
         # replica_tbl itself if it's a pure snapshot.
-        if replica_tbl._id != replica_tbl._tbl_version.id:
+        target_md = tbl_md[0]
+        is_pure_snapshot = (
+            target_md.tbl_md.view_md is not None
+            and target_md.tbl_md.view_md.predicate is None
+            and len(target_md.schema_version_md.columns) == 0
+        )
+        if is_pure_snapshot:
             ancestor_md = tbl_md[1:]  # Pure snapshot; skip replica_tbl
         else:
             ancestor_md = tbl_md  # Not a pure snapshot; include replica_tbl
@@ -267,7 +287,8 @@ class TableRestorer:
                 _logger.info(f'Importing table {tv.name!r}.')
                 self.__import_table(self.tmp_dir, tv, md)
 
-        return replica_tbl
+        with cat.begin_xact(for_write=False):
+            return cat.get_table_by_id(UUID(tbl_md[0].tbl_md.tbl_id))
 
     def __import_table(self, bundle_path: Path, tv: catalog.TableVersion, tbl_md: schema.FullTableMd) -> None:
         """
@@ -276,11 +297,182 @@ class TableRestorer:
         tbl_id = uuid.UUID(tbl_md.tbl_md.tbl_id)
         parquet_dir = bundle_path / 'tables' / f'tbl_{tbl_id.hex}'
         parquet_table = pq.read_table(str(parquet_dir))
-
-        for batch in parquet_table.to_batches():
+        replica_version = tv.version
+
+        conn = Env.get().conn
+        store_sa_tbl = tv.store_tbl.sa_tbl
+        store_sa_tbl_name = tv.store_tbl._storage_name()
+
+        # Sometimes we are importing a table that has never been seen before. Other times, however, we are importing
+        # an existing replica table, and the table version and/or row selection differs from what was imported
+        # previously. Care must be taken to ensure that the new data is merged with existing data in a way that
+        # yields an internally consistent version history for each row.
+
+        # The overall strategy is this:
+        # 1. Import the parquet data into a temporary table;
+        # 2. "rectify" the v_max values in both the temporary table and the existing table (more on this below);
+        # 3. Delete any row instances from the temporary table that are already present in the existing table;
+        # 4. Copy the remaining rows from the temporary table into the existing table.
+
+        # Create a temporary table for the initial data load, containing columns for all columns present in the
+        # parquet table. The parquet columns have identical names to those in the store table, so we can use the
+        # store table schema to get their SQL types (which are not necessarily derivable from their Parquet types,
+        # e.g., pa.string() may hold either VARCHAR or serialized JSONB).
+        temp_cols: dict[str, sql.Column] = {}
+        for field in parquet_table.schema:
+            assert field.name in store_sa_tbl.columns
+            col_type = store_sa_tbl.columns[field.name].type
+            temp_cols[field.name] = sql.Column(field.name, col_type)
+        temp_sa_tbl_name = f'temp_{uuid.uuid4().hex}'
+        _logger.debug(f'Creating temporary table: {temp_sa_tbl_name}')
+        temp_md = sql.MetaData()
+        temp_sa_tbl = sql.Table(temp_sa_tbl_name, temp_md, *temp_cols.values(), prefixes=['TEMPORARY'])
+        temp_sa_tbl.create(conn)
+
+        # Populate the temporary table with data from the Parquet file.
+        _logger.debug(f'Loading {parquet_table.num_rows} row(s) into temporary table: {temp_sa_tbl_name}')
+        for batch in parquet_table.to_batches(max_chunksize=10_000):
             pydict = batch.to_pydict()
             rows = self.__from_pa_pydict(tv, pydict)
-            tv.store_tbl.load_rows(rows)
+            conn.execute(sql.insert(temp_sa_tbl), rows)
+
+        # Each row version is identified uniquely by its pk, a tuple (row_id, pos_0, pos_1, ..., pos_k, v_min).
+        # Conversely, v_max is not part of the primary key, but is simply a bookkeeping device.
+        # In an original table, v_max is always equal to the v_min of the succeeding row instance with the same
+        # row id, or MAX_VERSION if no such row instance exists. But in the replica, we need to be careful, since
+        # we might see only a subset of the original table's versions, and we might see them out of order.
+
+        # We'll adjust the v_max values according to the principle of "latest provable v_max":
+        # they will always correspond to the latest version for which we can prove the row instance was alive. This
+        # will enable us to maintain consistency of the v_max values if additional table versions are later imported,
+        # regardless of the order in which they are seen. It also means that replica tables (unlike original tables)
+        # may have gaps in their row version histories, but this is fine; the gaps simply correspond to table versions
+        # that have never been observed.
+
+        pk_predicates = [col == temp_cols[col.name] for col in tv.store_tbl.pk_columns()]
+        pk_clause = sql.and_(*pk_predicates)
+
+        # If the same pk exists in both the temporary table and the existing table, then the corresponding row data
+        # must be identical; the rows can differ only in their v_max value. As a sanity check, we go through the
+        # motion of verifying this; a failure implies data corruption in either the replica being imported or in a
+        # previously imported replica.
+
+        system_col_names = {col.name for col in tv.store_tbl.system_columns()}
+        media_col_names = {col.store_name() for col in tv.cols if col.col_type.is_media_type() and col.is_stored}
+        value_store_cols = [
+            store_sa_tbl.c[col_name]
+            for col_name in temp_cols
+            if col_name not in system_col_names and col_name not in media_col_names
+        ]
+        value_temp_cols = [
+            col
+            for col_name, col in temp_cols.items()
+            if col_name not in system_col_names and col_name not in media_col_names
+        ]
+        mismatch_predicates = [store_col != temp_col for store_col, temp_col in zip(value_store_cols, value_temp_cols)]
+        mismatch_clause = sql.or_(*mismatch_predicates)
+
+        # This query looks for rows that have matching primary keys (rowid + pos_k + v_min), but differ in at least
+        # one value column. Pseudo-SQL:
+        #
+        # SELECT store_tbl.col_0, ..., store_tbl.col_n, temp_tbl.col_0, ...,  temp_tbl.col_n
+        # FROM store_tbl, temp_tbl
+        # WHERE store_tbl.rowid = temp_tbl.rowid
+        #     AND store_tbl.pos_0 = temp_tbl.pos_0
+        #     AND ... AND store_tbl.pos_k = temp_tbl.pos_k
+        #     AND store_tbl.v_min = temp_tbl.v_min
+        #     AND (
+        #         store_tbl.col_0 != temp_tbl.col_0
+        #         OR store_tbl.col_1 != temp_tbl.col_1
+        #         OR ... OR store_tbl.col_n != temp_tbl.col_n
+        #     )
+        #
+        # The value column comparisons (store_tbl.col_0 != temp_tbl.col_0, etc.) will always be false for rows where
+        # either column is NULL; this is what we want, since it may indicate a column that is present in one version
+        # but not the other.
+        q = sql.select(*value_store_cols, *value_temp_cols).where(pk_clause).where(mismatch_clause)
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        if result.rowcount > 0:
+            _logger.debug(
+                f'Data corruption error between {temp_sa_tbl_name!r} and {store_sa_tbl_name!r}: '
+                f'{result.rowcount} inconsistent row(s).'
+            )
+            row = result.first()
+            _logger.debug('Example mismatch:')
+            _logger.debug(f'{store_sa_tbl_name}: {row[: len(value_store_cols)]}')
+            _logger.debug(f'{temp_sa_tbl_name}: {row[len(value_store_cols) :]}')
+            raise excs.Error(
+                'Data corruption error: the replica data are inconsistent with data retrieved from a previous replica.'
+            )
+        _logger.debug(f'Verified data integrity between {store_sa_tbl_name!r} and {temp_sa_tbl_name!r}.')
+
+        # Now rectify the v_max values in the temporary table.
+        # If a row instance has a concrete v_max value, then we know it's genuine: it's the unique and immutable
+        # version when the row was deleted. (This can only happen if later versions of the base table already
+        # existed at the time this replica was published.)
+        # But if a row instance has a v_max value of MAX_VERSION, then we don't know anything about its future.
+        # It might live indefinitely, or it might be deleted as early as version `n + 1`. Following the principle
+        # of "latest provable v_max", we simply set v_max equal to `n + 1`.
+        q = (
+            temp_sa_tbl.update()
+            .values(v_max=(replica_version + 1))
+            .where(temp_sa_tbl.c.v_max == schema.Table.MAX_VERSION)
+        )
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        _logger.debug(f'Rectified {result.rowcount} row(s) in {temp_sa_tbl_name!r}.')
+
+        # Now rectify the v_max values in the existing table. This is done by simply taking the later of the two v_max
+        # values (the existing one and the new one) for each row instance, following the "latest provable v_max"
+        # principle. Obviously we only need to do this for rows that exist in both tables (it's a simple join).
+        q = (
+            store_sa_tbl.update()
+            .values(v_max=sql.func.greatest(store_sa_tbl.c.v_max, temp_sa_tbl.c.v_max))
+            .where(pk_clause)
+        )
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        _logger.debug(f'Rectified {result.rowcount} row(s) in {store_sa_tbl_name!r}.')
+
+        # Now we need to update rows in the existing table that are also present in the temporary table. This is to
+        # account for the scenario where the temporary table has columns that are not present in the existing table.
+        # (We can't simply replace the rows with their versions in the temporary table, because the converse scenario
+        # might also occur; there may be columns in the existing table that are not present in the temporary table.)
+        value_update_clauses: dict[str, sql.ColumnElement] = {}
+        for temp_col in temp_cols.values():
+            if temp_col.name not in system_col_names:
+                store_col = store_sa_tbl.c[temp_col.name]
+                # Prefer the value from the existing table, substituting the value from the temporary table if it's
+                # NULL. This works in all cases (including media columns, where we prefer the existing media file).
+                clause = sql.case((store_col == None, temp_col), else_=store_col)
+                value_update_clauses[temp_col.name] = clause
+        if len(value_update_clauses) > 0:
+            q = store_sa_tbl.update().values(**value_update_clauses).where(pk_clause)
+            _logger.debug(q.compile())
+            result = conn.execute(q)
+            _logger.debug(
+                f'Merged values from {temp_sa_tbl_name!r} into {store_sa_tbl_name!r} for {result.rowcount} row(s).'
+            )
+
+        # Now drop any rows from the temporary table that are also present in the existing table.
+        # The v_max values have been rectified, data has been merged into NULL cells, and all other row values have
+        # been verified identical.
+        # TODO: Delete any media files that were orphaned by this operation (they're necessarily duplicates of media
+        #     files that are already present in the existing table).
+        q = temp_sa_tbl.delete().where(pk_clause)
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        _logger.debug(f'Deleted {result.rowcount} row(s) from {temp_sa_tbl_name!r}.')
+
+        # Finally, copy the remaining data (consisting entirely of new row instances) from the temporary table into
+        # the actual table.
+        q = store_sa_tbl.insert().from_select(
+            [store_sa_tbl.c[col_name] for col_name in temp_cols], sql.select(*temp_cols.values())
+        )
+        _logger.debug(q.compile())
+        result = conn.execute(q)
+        _logger.debug(f'Inserted {result.rowcount} row(s) from {temp_sa_tbl_name!r} into {store_sa_tbl_name!r}.')
 
     def __from_pa_pydict(self, tv: catalog.TableVersion, pydict: dict[str, Any]) -> list[dict[str, Any]]:
         # Data conversions from pyarrow to Pixeltable
@@ -289,7 +481,7 @@ class TableRestorer:
             assert col_name in tv.store_tbl.sa_tbl.columns
             sql_types[col_name] = tv.store_tbl.sa_tbl.columns[col_name].type
         media_col_ids: dict[str, int] = {}
-        for col in tv.cols_by_name.values():
+        for col in tv.cols:
             if col.is_stored and col.col_type.is_media_type():
                 media_col_ids[col.store_name()] = col.id
 

pixeltable/share/publish.py

@@ -35,7 +35,7 @@ def push_replica(dest_tbl_uri: str, src_tbl: pxt.Table) -> str:
     upload_id = response_json['upload_id']
     destination_uri = response_json['destination_uri']
 
-    Env.get().console_logger.info(f"Creating a snapshot of '{src_tbl._path}' at: {dest_tbl_uri}")
+    Env.get().console_logger.info(f"Creating a snapshot of '{src_tbl._path()}' at: {dest_tbl_uri}")
 
     bundle = packager.package()
 
@@ -117,7 +117,7 @@ def pull_replica(dest_path: str, src_tbl_uri: str) -> pxt.Table:
 
     restorer = TableRestorer(dest_path, response_json)
     tbl = restorer.restore(bundle_path)
-    Env.get().console_logger.info(f'Created local replica {tbl._path!r} from URI: {src_tbl_uri}')
+    Env.get().console_logger.info(f'Created local replica {tbl._path()!r} from URI: {src_tbl_uri}')
     return tbl