pixeltable 0.4.19__py3-none-any.whl → 0.4.21__py3-none-any.whl

pixeltable/_version.py

@@ -1 +1 @@
-__version__: str = '0.4.19'
+__version__: str = '0.4.21'

pixeltable/catalog/catalog.py

@@ -27,7 +27,7 @@ from .insertable_table import InsertableTable
 from .path import Path
 from .schema_object import SchemaObject
 from .table import Table
-from .table_version import TableVersion
+from .table_version import TableVersion, TableVersionCompleteMd
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
 from .tbl_ops import TableOp
@@ -103,7 +103,16 @@ def retry_loop(
                     Catalog.get()._finalize_pending_ops(e.tbl_id)
                 except (sql_exc.DBAPIError, sql_exc.OperationalError) as e:
                     # TODO: what other exceptions should we be looking for?
-                    if isinstance(e.orig, (psycopg.errors.SerializationFailure, psycopg.errors.LockNotAvailable)):
+                    if isinstance(
+                        # TODO: Investigate whether DeadlockDetected points to a bug in our locking protocol,
+                        #     which is supposed to be deadlock-free.
+                        e.orig,
+                        (
+                            psycopg.errors.SerializationFailure,
+                            psycopg.errors.LockNotAvailable,
+                            psycopg.errors.DeadlockDetected,
+                        ),
+                    ):
                         if num_retries < _MAX_RETRIES or _MAX_RETRIES == -1:
                             num_retries += 1
                             _logger.debug(f'Retrying ({num_retries}) after {type(e.orig)}')
@@ -452,12 +461,15 @@ class Catalog:
             _logger.debug(f'Exception: undefined table {tbl_name!r}: Caught {type(e.orig)}: {e!r}')
             raise excs.Error(f'Table was dropped: {tbl_name}') from None
         elif (
+            # TODO: Investigate whether DeadlockDetected points to a bug in our locking protocol,
+            #     which is supposed to be deadlock-free.
             isinstance(
                 e.orig,
                 (
                     psycopg.errors.SerializationFailure,  # serialization error despite getting x-locks
                     psycopg.errors.InFailedSqlTransaction,  # can happen after tx fails for another reason
                     psycopg.errors.DuplicateColumn,  # if a different process added a column concurrently
+                    psycopg.errors.DeadlockDetected,  # locking protocol contention
                 ),
             )
             and convert_db_excs
@@ -1102,7 +1114,7 @@ class Catalog:
             tv.is_validated = False
             del self._tbl_versions[tbl_id, effective_version]
 
-    def create_replica(self, path: Path, md: list[schema.FullTableMd]) -> None:
+    def create_replica(self, path: Path, md: list[TableVersionCompleteMd]) -> None:
         """
         Creates table, table_version, and table_schema_version records for a replica with the given metadata.
         The metadata should be presented in standard "ancestor order", with the table being replicated at
@@ -1184,7 +1196,7 @@ class Catalog:
         system_path = Path.parse('_system', allow_system_path=True)
         return self._create_dir(system_path, if_exists=IfExistsParam.IGNORE, parents=False)
 
-    def __store_replica_md(self, path: Path, md: schema.FullTableMd) -> None:
+    def __store_replica_md(self, path: Path, md: TableVersionCompleteMd) -> None:
         _logger.info(f'Creating replica table at {path!r} with ID: {md.tbl_md.tbl_id}')
         dir = self._get_schema_object(path.parent, expected=Dir, raise_if_not_exists=True)
         assert dir is not None
@@ -1220,7 +1232,6 @@ class Catalog:
                 # New metadata is more recent than the metadata currently stored in the DB; we'll update the record
                 # in place in the DB.
                 new_tbl_md = dataclasses.replace(md.tbl_md, name=path.name, user=Env.get().user, is_replica=True)
-
         # Now see if a TableVersion record already exists in the DB for this table version. If not, insert it. If
         # it already exists, check that the existing record is identical to the new one.
         q = (
@@ -1234,9 +1245,16 @@ class Catalog:
             is_new_tbl_version = True
         else:
             existing_version_md = schema.md_from_dict(schema.TableVersionMd, existing_version_md_row.md)
-            # Validate that the existing metadata are identical to the new metadata, except that their is_fragment
-            # flags may differ.
-            if dataclasses.replace(existing_version_md, is_fragment=md.version_md.is_fragment) != md.version_md:
+            # Validate that the existing metadata are identical to the new metadata, except is_fragment
+            # and additional_md which may differ.
+            if (
+                dataclasses.replace(
+                    existing_version_md,
+                    is_fragment=md.version_md.is_fragment,
+                    additional_md=md.version_md.additional_md,
+                )
+                != md.version_md
+            ):
                 raise excs.Error(
                     f'The version metadata for the replica {path!r}:{md.version_md.version} is inconsistent with '
                     'the metadata recorded from a prior replica.\n'
@@ -1716,35 +1734,42 @@ class Catalog:
         tbl_record, version_record = _unpack_row(row, [schema.Table, schema.TableVersion])
         tbl_md = schema.md_from_dict(schema.TableMd, tbl_record.md)
         version_md = schema.md_from_dict(schema.TableVersionMd, version_record.md)
+        tvp = self.construct_tvp(tbl_id, version, tbl_md.ancestor_ids, version_md.created_at)
 
-        # Reconstruct the TableVersionPath for the specified TableVersion. We do this by examining the created_at
+        view = View(tbl_id, tbl_record.dir_id, tbl_md.name, tvp, snapshot_only=True)
+        self._tbls[tbl_id, version] = view
+        return view
+
+    def construct_tvp(self, tbl_id: UUID, version: int, ancestor_ids: list[str], created_at: float) -> TableVersionPath:
+        # Construct the TableVersionPath for the specified TableVersion. We do this by examining the created_at
         # timestamps of this table and all its ancestors.
         # TODO: Store the relevant TableVersionPaths in the database, so that we don't need to rely on timestamps
         #     (which might be nondeterministic in the future).
 
+        assert Env.get().conn is not None
+
         # Build the list of ancestor versions, starting with the given table and traversing back to the base table.
         # For each proper ancestor, we use the version whose created_at timestamp equals or most nearly precedes the
         # given TableVersion's created_at timestamp.
-        ancestors: list[tuple[UUID, int | None]] = [(tbl_id, version)]
-        if tbl_md.view_md is not None:
-            for ancestor_id, _ in tbl_md.view_md.base_versions:
-                q = (
-                    sql.select(schema.TableVersion)
-                    .where(schema.TableVersion.tbl_id == ancestor_id)
-                    .where(schema.TableVersion.md['created_at'].cast(sql.Float) <= version_md.created_at)
-                    .order_by(schema.TableVersion.md['created_at'].cast(sql.Float).desc())
-                    .limit(1)
-                )
-                row = conn.execute(q).one_or_none()
-                if row is None:
-                    # This can happen if an ancestor version is garbage collected; it can also happen in
-                    # rare circumstances involving table versions created specifically with Pixeltable 0.4.3.
-                    _logger.info(f'Ancestor {ancestor_id} not found for table {tbl_id}:{version}')
-                    raise excs.Error('The specified table version is no longer valid and cannot be retrieved.')
-                ancestor_version_record = _unpack_row(row, [schema.TableVersion])[0]
-                ancestor_version_md = schema.md_from_dict(schema.TableVersionMd, ancestor_version_record.md)
-                assert ancestor_version_md.created_at <= version_md.created_at
-                ancestors.append((UUID(ancestor_id), ancestor_version_md.version))
+        ancestors: list[tuple[UUID, int]] = [(tbl_id, version)]
+        for ancestor_id in ancestor_ids:
+            q = (
+                sql.select(schema.TableVersion)
+                .where(schema.TableVersion.tbl_id == ancestor_id)
+                .where(schema.TableVersion.md['created_at'].cast(sql.Float) <= created_at)
+                .order_by(schema.TableVersion.md['created_at'].cast(sql.Float).desc())
+                .limit(1)
+            )
+            row = Env.get().conn.execute(q).one_or_none()
+            if row is None:
+                # This can happen if an ancestor version is garbage collected; it can also happen in
+                # rare circumstances involving table versions created specifically with Pixeltable 0.4.3.
+                _logger.info(f'Ancestor {ancestor_id} not found for table {tbl_id}:{version}')
+                raise excs.Error('The specified table version is no longer valid and cannot be retrieved.')
+            ancestor_version_record = _unpack_row(row, [schema.TableVersion])[0]
+            ancestor_version_md = schema.md_from_dict(schema.TableVersionMd, ancestor_version_record.md)
+            assert ancestor_version_md.created_at <= created_at
+            ancestors.append((UUID(ancestor_id), ancestor_version_md.version))
 
         # Force any ancestors to be loaded (base table first).
         for anc_id, anc_version in ancestors[::-1]:
@@ -1756,15 +1781,13 @@ class Catalog:
         for anc_id, anc_version in ancestors[::-1]:
             tvp = TableVersionPath(TableVersionHandle(anc_id, anc_version), base=tvp)
 
-        view = View(tbl_id, tbl_record.dir_id, tbl_md.name, tvp, snapshot_only=True)
-        self._tbls[tbl_id, version] = view
-        return view
+        return tvp
 
     @retry_loop(for_write=False)
-    def collect_tbl_history(self, tbl_id: UUID, n: int | None) -> list[schema.FullTableMd]:
+    def collect_tbl_history(self, tbl_id: UUID, n: int | None) -> list[TableVersionCompleteMd]:
         return self._collect_tbl_history(tbl_id, n)
 
-    def _collect_tbl_history(self, tbl_id: UUID, n: int | None) -> list[schema.FullTableMd]:
+    def _collect_tbl_history(self, tbl_id: UUID, n: int | None) -> list[TableVersionCompleteMd]:
         """
         Returns the history of up to n versions of the table with the given UUID.
 
@@ -1792,15 +1815,15 @@ class Catalog:
             q = q.limit(n)
         src_rows = Env.get().session.execute(q).fetchall()
         return [
-            schema.FullTableMd(
-                schema.md_from_dict(schema.TableMd, row.Table.md),
-                schema.md_from_dict(schema.TableVersionMd, row.TableVersion.md),
-                schema.md_from_dict(schema.TableSchemaVersionMd, row.TableSchemaVersion.md),
+            TableVersionCompleteMd(
+                tbl_md=schema.md_from_dict(schema.TableMd, row.Table.md),
+                version_md=schema.md_from_dict(schema.TableVersionMd, row.TableVersion.md),
+                schema_version_md=schema.md_from_dict(schema.TableSchemaVersionMd, row.TableSchemaVersion.md),
             )
             for row in src_rows
         ]
 
-    def load_tbl_md(self, tbl_id: UUID, effective_version: int | None) -> schema.FullTableMd:
+    def load_tbl_md(self, tbl_id: UUID, effective_version: int | None) -> TableVersionCompleteMd:
         """
         Loads metadata from the store for a given table UUID and version.
         """
@@ -1851,7 +1874,7 @@ class Catalog:
         version_md = schema.md_from_dict(schema.TableVersionMd, version_record.md)
         schema_version_md = schema.md_from_dict(schema.TableSchemaVersionMd, schema_version_record.md)
 
-        return schema.FullTableMd(tbl_md, version_md, schema_version_md)
+        return TableVersionCompleteMd(tbl_md, version_md, schema_version_md)
 
     def store_tbl_md(
         self,
@@ -1925,8 +1948,12 @@ class Catalog:
                 # This table version already exists; update it.
                 assert len(tv_rows) == 1  # must be unique
                 tv = tv_rows[0]
-                # Validate that the only field that can change is 'is_fragment'.
-                assert tv.md == dataclasses.asdict(dataclasses.replace(version_md, is_fragment=tv.md['is_fragment']))
+                # Validate that the only fields that can change are 'is_fragment' and 'additional_md'.
+                assert tv.md == dataclasses.asdict(
+                    dataclasses.replace(
+                        version_md, is_fragment=tv.md['is_fragment'], additional_md=tv.md['additional_md']
+                    )
+                )
                 result = session.execute(
                     sql.update(schema.TableVersion.__table__)
                     .values({schema.TableVersion.md: dataclasses.asdict(version_md)})
@@ -1977,7 +2004,7 @@ class Catalog:
         conn.execute(sql.delete(schema.PendingTableOp.__table__).where(schema.PendingTableOp.tbl_id == tbl_id))
         conn.execute(sql.delete(schema.Table.__table__).where(schema.Table.id == tbl_id))
 
-    def load_replica_md(self, tbl: Table) -> list[schema.FullTableMd]:
+    def load_replica_md(self, tbl: Table) -> list[TableVersionCompleteMd]:
         """
         Load metadata for the given table along with all its ancestors. The values of TableMd.current_version and
         TableMd.current_schema_version will be adjusted to ensure that the metadata represent a valid (internally
@@ -1998,7 +2025,7 @@ class Catalog:
             # Set the `is_replica` flag on every ancestor's TableMd.
             ancestor_md.tbl_md.is_replica = True
             # For replica metadata, we guarantee that the current_version and current_schema_version of TableMd
-            # match the corresponding values in TableVersionMd and TableSchemaVersionMd. This is to ensure that,
+            # match the corresponding values in TableVersionCompleteMd and TableSchemaVersionMd. This is to ensure that,
             # when the metadata is later stored in the catalog of a different Pixeltable instance, the values of
             # current_version and current_schema_version will always point to versions that are known to the
             # destination catalog.
@@ -2017,7 +2044,10 @@ class Catalog:
         self, tbl_id: UUID, effective_version: int | None, check_pending_ops: bool = True
     ) -> TableVersion | None:
         """Creates TableVersion instance from stored metadata and registers it in _tbl_versions."""
-        tbl_md, version_md, schema_version_md = self.load_tbl_md(tbl_id, effective_version)
+        table_version_md = self.load_tbl_md(tbl_id, effective_version)
+        tbl_md = table_version_md.tbl_md
+        version_md = table_version_md.version_md
+        schema_version_md = table_version_md.schema_version_md
         view_md = tbl_md.view_md
 
         conn = Env.get().conn
@@ -2041,10 +2071,6 @@ class Catalog:
 
         # load mutable view ids for mutable TableVersions
         mutable_view_ids: list[UUID] = []
-        # If this is a replica, effective_version should not be None. We see this today, because
-        # the replica's TV instance's Column instances contain value_expr_dicts that reference the live version.
-        # This is presumably a source of bugs, because it ignores schema version changes (eg, column renames).
-        # TODO: retarget the value_expr_dict when instantiating Columns for a particular TV instance.
         if effective_version is None and not tbl_md.is_replica:
             q = (
                 sql.select(schema.Table.id)
@@ -2061,7 +2087,7 @@ class Catalog:
             tbl_version = TableVersion(tbl_id, tbl_md, version_md, effective_version, schema_version_md, mutable_views)
         else:
             assert len(view_md.base_versions) > 0  # a view needs to have a base
-            # TODO: add TableVersionMd.is_pure_snapshot() and use that
+            # TODO: add TableVersionCompleteMd.is_pure_snapshot() and use that
             pure_snapshot = (
                 view_md.is_snapshot
                 and view_md.predicate is None

pixeltable/catalog/column.py

@@ -18,6 +18,7 @@ from .globals import MediaValidation, QColumnId, is_valid_identifier
 if TYPE_CHECKING:
     from .table_version import TableVersion
     from .table_version_handle import ColumnHandle, TableVersionHandle
+    from .table_version_path import TableVersionPath
 
 _logger = logging.getLogger('pixeltable')
 
@@ -162,26 +163,38 @@ class Column:
         )
         return col_md, sch_md
 
-    def init_value_expr(self) -> None:
+    def init_value_expr(self, tvp: 'TableVersionPath' | None) -> None:
+        """
+        Initialize the value_expr from its dict representation, if necessary.
+
+        If `tvp` is not None, retarget the value_expr to the given TableVersionPath.
+        """
         from pixeltable import exprs
 
-        if self._value_expr is not None or self.value_expr_dict is None:
+        if self._value_expr is None and self.value_expr_dict is None:
             return
-        self._value_expr = exprs.Expr.from_dict(self.value_expr_dict)
-        self._value_expr.bind_rel_paths()
-        if not self._value_expr.is_valid:
-            message = (
-                dedent(
-                    f"""
-                    The computed column {self.name!r} in table {self.get_tbl().name!r} is no longer valid.
-                    {{validation_error}}
-                    You can continue to query existing data from this column, but evaluating it on new data will raise an error.
-                    """  # noqa: E501
+
+        if self._value_expr is None:
+            # Instantiate the Expr from its dict
+            self._value_expr = exprs.Expr.from_dict(self.value_expr_dict)
+            self._value_expr.bind_rel_paths()
+            if not self._value_expr.is_valid:
+                message = (
+                    dedent(
+                        f"""
+                        The computed column {self.name!r} in table {self.get_tbl().name!r} is no longer valid.
+                        {{validation_error}}
+                        You can continue to query existing data from this column, but evaluating it on new data will raise an error.
+                        """  # noqa: E501
+                    )
+                    .strip()
+                    .format(validation_error=self._value_expr.validation_error)
                 )
-                .strip()
-                .format(validation_error=self._value_expr.validation_error)
-            )
-            warnings.warn(message, category=excs.PixeltableWarning, stacklevel=2)
+                warnings.warn(message, category=excs.PixeltableWarning, stacklevel=2)
+
+        if tvp is not None:
+            # Retarget the Expr
+            self._value_expr = self._value_expr.retarget(tvp)
 
     def get_tbl(self) -> TableVersion:
         tv = self.tbl_handle.get()

pixeltable/catalog/insertable_table.py

@@ -16,7 +16,7 @@ from pixeltable.utils.pydantic import is_json_convertible
 
 from .globals import MediaValidation
 from .table import Table
-from .table_version import TableVersion, TableVersionMd
+from .table_version import TableVersion, TableVersionCompleteMd
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
 from .tbl_ops import CreateStoreTableOp, TableOp
@@ -73,7 +73,7 @@ class InsertableTable(Table):
         comment: str,
         media_validation: MediaValidation,
         create_default_idxs: bool,
-    ) -> tuple[TableVersionMd, list[TableOp]]:
+    ) -> tuple[TableVersionCompleteMd, list[TableOp]]:
         columns = cls._create_columns(schema)
         cls._verify_schema(columns)
         column_names = [col.name for col in columns]

pixeltable/catalog/path.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+from typing import NamedTuple
 
 from pixeltable import exceptions as excs
 
@@ -9,14 +10,9 @@ from .globals import is_valid_identifier
 _logger = logging.getLogger('pixeltable')
 
 
-class Path:
+class Path(NamedTuple):
     components: list[str]
-    version: int | None
-
-    def __init__(self, components: list[str], version: int | None = None) -> None:
-        assert len(components) > 0
-        self.components = components
-        self.version = version
+    version: int | None = None
 
     @classmethod
     def parse(
@@ -50,6 +46,7 @@ class Path:
         if version is not None and not allow_versioned_path:
             raise excs.Error(f'Versioned path not allowed here: {path}')
 
+        assert len(components) > 0
         return Path(components, version)
 
     @property
@@ -118,8 +115,5 @@ class Path:
     def __hash__(self) -> int:
         return hash(str(self))
 
-    def __lt__(self, other: Path) -> bool:
-        return str(self) < str(other)
-
 
 ROOT_PATH = Path([''])

pixeltable/catalog/table.py

@@ -1624,6 +1624,57 @@ class Table(SchemaObject):
             # remove cached md in order to force a reload on the next operation
             self._tbl_version_path.clear_cached_md()
 
+    def push(self, *, version: int | None = None) -> None:
+        from pixeltable.share import push_replica
+        from pixeltable.share.protocol import PxtUri
+
+        tbl_version = self._tbl_version.get()
+        pxt_uri = tbl_version.pxt_uri
+
+        if tbl_version.is_replica:
+            raise excs.Error(f'push(): Cannot push replica table {self._name!r}. (Did you mean `pull()`?)')
+        if pxt_uri is None:
+            raise excs.Error(
+                f'push(): Table {self._name!r} has not yet been published to Pixeltable Cloud. '
+                'To publish it, use `pxt.publish()` instead.'
+            )
+
+        # Parse the pxt URI to extract org/db and create a UUID-based URI for pushing
+        parsed_uri = PxtUri(uri=pxt_uri)
+        uuid_uri_obj = PxtUri.from_components(org=parsed_uri.org, id=self._id, db=parsed_uri.db)
+        uuid_uri = str(uuid_uri_obj)
+
+        if version is None:
+            # Push this version
+            push_replica(uuid_uri, self)
+        else:
+            versioned_path = catalog.Path.parse(self._path())._replace(version=version)
+            versioned_tbl = catalog.Catalog.get().get_table(versioned_path, IfNotExistsParam.IGNORE)
+            if versioned_tbl is None:
+                raise excs.Error(f'Table {self._name!r} has no known version {version}')
+            assert versioned_tbl._id == self._id
+            push_replica(uuid_uri, versioned_tbl)
+
+    def pull(self, *, version: int | None = None) -> None:
+        from pixeltable.share import pull_replica
+        from pixeltable.share.protocol import PxtUri
+
+        tbl_version = self._tbl_version_path.tbl_version.get()
+        pxt_uri = tbl_version.pxt_uri
+
+        if not tbl_version.is_replica:
+            raise excs.Error(
+                f'pull(): Table {self._name!r} is not a replica of a Pixeltable Cloud table (nothing to `pull()`).'
+            )
+        assert pxt_uri is not None
+
+        # Parse the pxt URI to extract org/db and create a UUID-based URI for pulling
+        parsed_uri = PxtUri(uri=pxt_uri)
+        uuid_uri_obj = PxtUri.from_components(org=parsed_uri.org, id=self._id, db=parsed_uri.db, version=version)
+        uuid_uri = str(uuid_uri_obj)
+
+        pull_replica(self._path(), uuid_uri)
+
     def external_stores(self) -> list[str]:
         return list(self._tbl_version.get().external_stores.keys())
 

pixeltable/catalog/table_version.py

@@ -43,17 +43,25 @@ _logger = logging.getLogger('pixeltable')
 
 
 @dataclasses.dataclass(frozen=True)
-class TableVersionMd:
+class TableVersionCompleteMd:
     """
     Complete set of md records for a specific TableVersion instance.
-
-    TODO: subsume schema.FullTableMd
     """
 
     tbl_md: schema.TableMd
     version_md: schema.TableVersionMd
     schema_version_md: schema.TableSchemaVersionMd
 
+    @property
+    def is_pure_snapshot(self) -> bool:
+        return (
+            self.tbl_md is not None
+            and self.tbl_md.view_md is not None
+            and self.tbl_md.view_md.is_snapshot
+            and self.tbl_md.view_md.predicate is None
+            and len(self.schema_version_md.columns) == 0
+        )
+
 
 class TableVersion:
     """
@@ -240,7 +248,7 @@ class TableVersion:
         media_validation: MediaValidation,
         create_default_idxs: bool,
         view_md: schema.ViewMd | None = None,
-    ) -> TableVersionMd:
+    ) -> TableVersionCompleteMd:
         from .table_version_handle import TableVersionHandle
 
         user = Env.get().user
@@ -337,7 +345,7 @@ class TableVersion:
             media_validation=media_validation.name.lower(),
             additional_md={},
         )
-        return TableVersionMd(tbl_md, table_version_md, schema_version_md)
+        return TableVersionCompleteMd(tbl_md, table_version_md, schema_version_md)
 
     def exec_op(self, op: TableOp) -> None:
         if op.create_store_table_op is not None:
@@ -365,7 +373,7 @@ class TableVersion:
             _logger.debug(f'Loaded view {self.name} with {row_counts.num_rows} rows')
 
     @classmethod
-    def create_replica(cls, md: schema.FullTableMd) -> TableVersion:
+    def create_replica(cls, md: TableVersionCompleteMd) -> TableVersion:
         from .catalog import Catalog, TableVersionPath
 
         assert Env.get().in_xact
@@ -437,6 +445,8 @@ class TableVersion:
     def _init_schema(self) -> None:
         from pixeltable.store import StoreComponentView, StoreTable, StoreView
 
+        from .catalog import Catalog
+
         # initialize IndexBase instances and collect sa_col_types
         idxs: dict[int, index.IndexBase] = {}
         val_col_idxs: dict[int, index.IndexBase] = {}  # key: id of value column
@@ -529,8 +539,16 @@ class TableVersion:
                 self.idxs_by_col.setdefault(indexed_col_id, []).append(info)
 
         # create value exprs, now that we have all lookup structures in place
+        tvp: TableVersionPath | None = None
+        if self.effective_version is not None:
+            # for snapshot TableVersion instances, we need to retarget the column value_exprs to the snapshot;
+            # otherwise they'll incorrectly refer to the live table. So, construct a full TableVersionPath to
+            # use for retargeting.
+            tvp = Catalog.get().construct_tvp(
+                self.id, self.effective_version, self.tbl_md.ancestor_ids, self.version_md.created_at
+            )
         for col in self.cols_by_id.values():
-            col.init_value_expr()
+            col.init_value_expr(tvp)
 
         # create the sqlalchemy schema, after instantiating all Columns
         if self.is_component_view:
@@ -1413,6 +1431,21 @@ class TableVersion:
         self._tbl_md.external_stores.pop(idx)
         self._write_md(new_version=True, new_schema_version=True)
 
+    @property
+    def pxt_uri(self) -> str | None:
+        return self._tbl_md.additional_md.get('pxt_uri')
+
+    def update_pxt_uri(self, pxt_uri: str | None) -> None:
+        if self._tbl_md.additional_md.get('pxt_uri') == pxt_uri:
+            return  # Nothing to do
+
+        if pxt_uri is None:
+            del self._tbl_md.additional_md['pxt_uri']  # must be present due to preceding check
+        else:
+            self._tbl_md.additional_md['pxt_uri'] = pxt_uri
+
+        self._write_md(new_version=False, new_schema_version=False)
+
     @property
     def tbl_md(self) -> schema.TableMd:
         return self._tbl_md

pixeltable/catalog/view.py

@@ -14,7 +14,7 @@ from pixeltable.iterators import ComponentIterator
 from .column import Column
 from .globals import _POS_COLUMN_NAME, MediaValidation
 from .table import Table
-from .table_version import TableVersion, TableVersionMd
+from .table_version import TableVersion, TableVersionCompleteMd
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
 from .tbl_ops import CreateStoreTableOp, LoadViewOp, TableOp
@@ -84,7 +84,7 @@ class View(Table):
         media_validation: MediaValidation,
         iterator_cls: type[ComponentIterator] | None,
         iterator_args: dict | None,
-    ) -> tuple[TableVersionMd, list[TableOp] | None]:
+    ) -> tuple[TableVersionCompleteMd, list[TableOp] | None]:
         from pixeltable.plan import SampleClause
 
         # Convert select_list to more additional_columns if present

pixeltable/config.py

@@ -176,6 +176,7 @@ KNOWN_CONFIG_OPTIONS = {
     'gemini': {'api_key': 'Gemini API key', 'rate_limits': 'Per-model rate limits for Gemini API requests'},
     'hf': {'auth_token': 'Hugging Face access token'},
     'imagen': {'rate_limits': 'Per-model rate limits for Imagen API requests'},
+    'reve': {'api_key': 'Reve API key', 'rate_limit': 'Rate limit for Reve API requests (requests per minute)'},
     'groq': {'api_key': 'Groq API key', 'rate_limit': 'Rate limit for Groq API requests'},
     'label_studio': {'api_key': 'Label Studio API key', 'url': 'Label Studio server URL'},
     'mistral': {'api_key': 'Mistral API key', 'rate_limit': 'Rate limit for Mistral API requests'},

pixeltable/env.py

@@ -792,6 +792,7 @@ class Env:
         self.__register_package('pyarrow')
         self.__register_package('pydantic')
         self.__register_package('replicate')
+        self.__register_package('reve')
         self.__register_package('sentencepiece')
         self.__register_package('sentence_transformers', library_name='sentence-transformers')
         self.__register_package('soundfile')
@@ -806,6 +807,7 @@ class Env:
         self.__register_package('whisperx')
         self.__register_package('yolox', library_name='pixeltable-yolox')
         self.__register_package('lancedb')
+        self.__register_package('scenedetect')
 
     def __register_package(self, package_name: str, library_name: str | None = None) -> None:
         is_installed: bool

pixeltable/exprs/column_ref.py

@@ -221,7 +221,8 @@ class ColumnRef(Expr):
         return self._descriptors().to_html()
 
     def _descriptors(self) -> DescriptionHelper:
-        tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl_handle.id)
+        with catalog.Catalog.get().begin_xact():
+            tbl = catalog.Catalog.get().get_table_by_id(self.col.tbl_handle.id)
         helper = DescriptionHelper()
         helper.append(f'Column\n{self.col.name!r}\n(of table {tbl._path()!r})')
         helper.append(tbl._col_descriptor([self.col.name]))

pixeltable/functions/__init__.py

@@ -21,6 +21,7 @@ from . import (
     openai,
     openrouter,
     replicate,
+    reve,
     string,
     timestamp,
     together,

pixeltable/functions/image.py

@@ -10,14 +10,13 @@ t.select(t.img_col.convert('L')).collect()
 ```
 """
 
-import base64
-
 import PIL.Image
 
 import pixeltable as pxt
 import pixeltable.type_system as ts
 from pixeltable.exprs import Expr
 from pixeltable.utils.code import local_public_names
+from pixeltable.utils.image import to_base64
 
 
 @pxt.udf(is_method=True)
@@ -29,12 +28,7 @@ def b64_encode(img: PIL.Image.Image, image_format: str = 'png') -> str:
         img: image
         image_format: image format [supported by PIL](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#fully-supported-formats)
     """
-    import io
-
-    bytes_arr = io.BytesIO()
-    img.save(bytes_arr, format=image_format)
-    b64_bytes = base64.b64encode(bytes_arr.getvalue())
-    return b64_bytes.decode('utf-8')
+    return to_base64(img, format=image_format)
 
 
 @pxt.udf(substitute_fn=PIL.Image.alpha_composite, is_method=True)