pixeltable 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

pixeltable/__init__.py

@@ -20,6 +20,7 @@ from .globals import (
     list_dirs,
     list_functions,
     list_tables,
+    ls,
     move,
     tool,
     tools,

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = '0.4.1'
-__version_tuple__ = (0, 4, 1)
+__version__ = '0.4.3'
+__version_tuple__ = (0, 4, 3)

pixeltable/catalog/__init__.py

@@ -3,21 +3,14 @@
 from .catalog import Catalog
 from .column import Column
 from .dir import Dir
-from .globals import (
-    IfExistsParam,
-    IfNotExistsParam,
-    MediaValidation,
-    QColumnId,
-    UpdateStatus,
-    is_valid_identifier,
-    is_valid_path,
-)
+from .globals import IfExistsParam, IfNotExistsParam, MediaValidation, QColumnId, is_valid_identifier, is_valid_path
 from .insertable_table import InsertableTable
 from .named_function import NamedFunction
 from .path import Path
 from .schema_object import SchemaObject
 from .table import Table
 from .table_version import TableVersion
-from .table_version_handle import TableVersionHandle
+from .table_version_handle import ColumnHandle, TableVersionHandle
 from .table_version_path import TableVersionPath
+from .update_status import RowCountStats, UpdateStatus
 from .view import View

pixeltable/catalog/catalog.py

@@ -308,7 +308,11 @@ class Catalog:
                     # we still got a serialization error, despite getting x-locks at the beginning
                     msg = f'{tbl.tbl_name()} ({tbl.tbl_id})' if tbl is not None else ''
                     _logger.debug(f'Exception: serialization failure: {msg} ({e})')
-                    raise excs.Error('Serialization failure. Please re-run the operation.') from None
+                    raise excs.Error(
+                        'That Pixeltable operation could not be completed because it conflicted with another '
+                        'operation that was run on a different process.\n'
+                        'Please re-run the operation.'
+                    ) from None
                 else:
                     raise
 
@@ -762,56 +766,47 @@ class Catalog:
         self._tbls[view._id] = view
         return view
 
-    @_retry_loop(for_write=True)
-    def create_replica(
-        self, path: Path, md: list[schema.FullTableMd], if_exists: IfExistsParam = IfExistsParam.ERROR
-    ) -> None:
+    def create_replica(self, path: Path, md: list[schema.FullTableMd]) -> None:
         """
         Creates table, table_version, and table_schema_version records for a replica with the given metadata.
         The metadata should be presented in standard "ancestor order", with the table being replicated at
         list position 0 and the (root) base table at list position -1.
-
-        TODO: create_replica() also needs to create the store tables and populate them in order to make
-        replica creation atomic.
         """
+        assert Env.get().in_xact
+
         tbl_id = UUID(md[0].tbl_md.tbl_id)
 
-        # First handle path collisions (if_exists='ignore' or 'replace' or etc).
-        existing = self._handle_path_collision(path, View, False, if_exists)
-        if existing is not None:
-            if existing._id != tbl_id:
-                raise excs.Error(
-                    f"An attempt was made to create a replica table at {path!r} with if_exists='ignore', "
-                    'but a different table already exists at that location.'
-                )
-            assert isinstance(existing, View)
-            return
+        existing = self._handle_path_collision(path, Table, False, if_exists=IfExistsParam.IGNORE)  # type: ignore[type-abstract]
+        if existing is not None and existing._id != tbl_id:
+            raise excs.Error(
+                f'An attempt was made to create a replica table at {path!r}, '
+                'but a different table already exists at that location.'
+            )
 
         # Ensure that the system directory exists.
         self._create_dir(Path('_system', allow_system_paths=True), if_exists=IfExistsParam.IGNORE, parents=False)
 
-        # Now check to see if this table already exists in the catalog.
+        # Now check to see if this table UUID already exists in the catalog.
         existing = Catalog.get().get_table_by_id(tbl_id)
         if existing is not None:
             existing_path = Path(existing._path(), allow_system_paths=True)
-            # It does exist. If it's a non-system table, that's an error: it's already been replicated.
-            if not existing_path.is_system_path:
-                raise excs.Error(
-                    f'That table has already been replicated as {existing._path()!r}. \n'
-                    f'Drop the existing replica if you wish to re-create it.'
-                )
-            # If it's a system table, then this means it was created at some point as the ancestor of some other
-            # table (a snapshot-over-snapshot scenario). In that case, we simply move it to the new (named) location.
-            self._move(existing_path, path)
-
-        # Now store the metadata for this replica. In the case where the table already exists (and was just moved
-        # into a named location), this will be a no-op, but it still serves to validate that the newly received
-        # metadata is identical to what's in the catalog.
-        self.__store_replica_md(path, md[0])
+            if existing_path != path:
+                # It does exist, under a different path from the specified one.
+                if not existing_path.is_system_path:
+                    raise excs.Error(
+                        f'That table has already been replicated as {existing_path!r}.\n'
+                        f'Drop the existing replica if you wish to re-create it.'
+                    )
+                # If it's a system table, then this means it was created at some point as the ancestor of some other
+                # table (a snapshot-over-snapshot scenario). In that case, we simply move it to the new (named)
+                # location.
+                self._move(existing_path, path)
 
-        # Now store the metadata for all of this table's proper ancestors. If one or more proper ancestors
+        # Now store the metadata for this replica's proper ancestors. If one or more proper ancestors
         # do not yet exist in the store, they will be created as anonymous system tables.
-        for ancestor_md in md[1:]:
+        # We instantiate the ancestors starting with the base table and ending with the immediate parent of the
+        # table being replicated.
+        for ancestor_md in md[:0:-1]:
             ancestor_id = UUID(ancestor_md.tbl_md.tbl_id)
             replica = Catalog.get().get_table_by_id(ancestor_id)
             replica_path: Path
@@ -824,12 +819,22 @@ class Catalog:
                 # that was directly replicated by the user at some point). In either case, use the existing path.
                 replica_path = Path(replica._path(), allow_system_paths=True)
 
-            # Store the metadata; it could be a new version (in which case a new record will be created) or a
-            # known version (in which case the newly received metadata will be validated as identical).
+            # Store the metadata; it could be a new version (in which case a new record will be created), or a known
+            # version (in which case the newly received metadata will be validated as identical).
+            # If it's a new version, this will result in a new TableVersion record being created.
             self.__store_replica_md(replica_path, ancestor_md)
 
-        # don't create TableVersion instances at this point, they would be superseded by calls to TV.create_replica()
-        # in TableRestorer.restore()
+            # Now we must clear cached metadata for the ancestor table, to force the next table operation to pick up
+            # the new TableVersion instance. This is necessary because computed columns of descendant tables might
+            # reference columns of the ancestor table that only exist in the new version.
+            replica = Catalog.get().get_table_by_id(ancestor_id)
+            assert replica is not None  # If it didn't exist before, it must have been created by now.
+            replica._tbl_version_path.clear_cached_md()
+
+        # Finally, store the metadata for the table being replicated; as before, it could be a new version or a known
+        # version. If it's a new version, then a TableVersion record will be created, unless the table being replicated
+        # is a pure snapshot.
+        self.__store_replica_md(path, md[0])
 
     def __store_replica_md(self, path: Path, md: schema.FullTableMd) -> None:
         _logger.info(f'Creating replica table at {path!r} with ID: {md.tbl_md.tbl_id}')
@@ -911,12 +916,19 @@ class Catalog:
                     'This is likely due to data corruption in the replicated table.'
                 )
 
-        self.store_tbl_md(UUID(tbl_id), new_tbl_md, new_version_md, new_schema_version_md)
+        self.store_tbl_md(UUID(tbl_id), None, new_tbl_md, new_version_md, new_schema_version_md)
+
+        if new_version_md is not None and not md.is_pure_snapshot:
+            # It's a new version of a table that has a physical store, so we need to create a TableVersion instance.
+            TableVersion.create_replica(md)
 
     @_retry_loop(for_write=False)
     def get_table(self, path: Path) -> Table:
         obj = Catalog.get()._get_schema_object(path, expected=Table, raise_if_not_exists=True)
         assert isinstance(obj, Table)
+        # We need to clear cached metadata from tbl_version_path, in case the schema has been changed
+        # by another process.
+        obj._tbl_version_path.clear_cached_md()
         return obj
 
     @_retry_loop(for_write=True)
@@ -1228,6 +1240,43 @@ class Catalog:
         self._tbls[tbl_id] = view
         return view
 
+    @_retry_loop(for_write=False)
+    def collect_tbl_history(self, tbl_id: UUID, n: Optional[int]) -> list[schema.FullTableMd]:
+        """
+        Returns the history of up to n versions of the table with the given UUID.
+
+        Args:
+            tbl_id: the UUID of the table to collect history for.
+            n: Optional limit on the maximum number of versions returned.
+
+        Returns:
+            A sequence of rows, ordered by version number
+            Each row contains a TableVersion and a TableSchemaVersion object.
+        """
+        q = (
+            sql.select(schema.TableVersion, schema.TableSchemaVersion)
+            .select_from(schema.TableVersion)
+            .join(
+                schema.TableSchemaVersion,
+                sql.cast(schema.TableVersion.md['schema_version'], sql.Integer)
+                == schema.TableSchemaVersion.schema_version,
+            )
+            .where(schema.TableVersion.tbl_id == tbl_id)
+            .where(schema.TableSchemaVersion.tbl_id == tbl_id)
+            .order_by(schema.TableVersion.version.desc())
+        )
+        if n is not None:
+            q = q.limit(n)
+        src_rows = Env.get().session.execute(q).fetchall()
+        return [
+            schema.FullTableMd(
+                None,
+                schema.md_from_dict(schema.TableVersionMd, row.TableVersion.md),
+                schema.md_from_dict(schema.TableSchemaVersionMd, row.TableSchemaVersion.md),
+            )
+            for row in src_rows
+        ]
+
     def load_tbl_md(self, tbl_id: UUID, effective_version: Optional[int]) -> schema.FullTableMd:
         """
         Loads metadata from the store for a given table UUID and version.
@@ -1297,19 +1346,27 @@ class Catalog:
     def store_tbl_md(
         self,
         tbl_id: UUID,
+        dir_id: Optional[UUID],
         tbl_md: Optional[schema.TableMd],
         version_md: Optional[schema.TableVersionMd],
         schema_version_md: Optional[schema.TableSchemaVersionMd],
     ) -> None:
         """
-        Stores metadata to the DB. If specified, `tbl_md` will be updated in place (only one such record can exist
-        per UUID); `version_md` and `schema_version_md` will be inserted as new records.
+        Stores metadata to the DB.
+
+        Args:
+            tbl_id: UUID of the table to store metadata for.
+            dir_id: If specified, the tbl_md will be added to the given directory; if None, the table must already exist
+            tbl_md: If specified, `tbl_md` will be inserted, or updated (only one such record can exist per UUID)
+            version_md: inserted as a new record if present
+            schema_version_md: will be inserted as a new record if present
 
         If inserting `version_md` or `schema_version_md` would be a primary key violation, an exception will be raised.
         """
-        conn = Env.get().conn
         assert self._in_write_xact
+        session = Env.get().session
 
+        # Construct and insert or update table record if requested.
         if tbl_md is not None:
             assert tbl_md.tbl_id == str(tbl_id)
             if version_md is not None:
@@ -1317,32 +1374,55 @@ class Catalog:
                 assert tbl_md.current_schema_version == version_md.schema_version
             if schema_version_md is not None:
                 assert tbl_md.current_schema_version == schema_version_md.schema_version
-            result = conn.execute(
-                sql.update(schema.Table.__table__)
-                .values({schema.Table.md: dataclasses.asdict(tbl_md)})
-                .where(schema.Table.id == tbl_id)
-            )
-            assert result.rowcount == 1, result.rowcount
+            if dir_id is not None:
+                # We are inserting a record while creating a new table.
+                tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(tbl_md))
+                session.add(tbl_record)
+            else:
+                # Update the existing table record.
+                result = session.execute(
+                    sql.update(schema.Table.__table__)
+                    .values({schema.Table.md: dataclasses.asdict(tbl_md)})
+                    .where(schema.Table.id == tbl_id)
+                )
+                assert result.rowcount == 1, result.rowcount
 
+        # Construct and insert new table version record if requested.
         if version_md is not None:
             assert version_md.tbl_id == str(tbl_id)
             if schema_version_md is not None:
                 assert version_md.schema_version == schema_version_md.schema_version
-            conn.execute(
-                sql.insert(schema.TableVersion.__table__).values(
-                    tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
-                )
+            tbl_version_record = schema.TableVersion(
+                tbl_id=tbl_id, version=version_md.version, md=dataclasses.asdict(version_md)
             )
+            session.add(tbl_version_record)
 
+        # Construct and insert a new schema version record if requested.
         if schema_version_md is not None:
             assert schema_version_md.tbl_id == str(tbl_id)
-            conn.execute(
-                sql.insert(schema.TableSchemaVersion.__table__).values(
-                    tbl_id=tbl_id,
-                    schema_version=schema_version_md.schema_version,
-                    md=dataclasses.asdict(schema_version_md),
-                )
+            schema_version_record = schema.TableSchemaVersion(
+                tbl_id=tbl_id, schema_version=schema_version_md.schema_version, md=dataclasses.asdict(schema_version_md)
             )
+            session.add(schema_version_record)
+        session.flush()  # Inform SQLAlchemy that we want to write these changes to the DB.
+
+    def update_tbl_version_md(self, version_md: Optional[schema.TableVersionMd]) -> None:
+        """
+        Update the TableVersion.md field in the DB. Typically used to update the cascade row count status.
+
+        Args:
+            version_md: TableVersionMd
+        """
+        assert self._in_write_xact
+        session = Env.get().session
+
+        session.execute(
+            sql.update(schema.TableVersion.__table__)
+            .values({schema.TableVersion.md: dataclasses.asdict(version_md)})
+            .where(schema.TableVersion.tbl_id == version_md.tbl_id, schema.TableVersion.version == version_md.version)
+        )
+
+        session.flush()  # Inform SQLAlchemy that we want to write these changes to the DB.
 
     def delete_tbl_md(self, tbl_id: UUID) -> None:
         """

pixeltable/catalog/column.py

@@ -15,6 +15,7 @@ from .globals import MediaValidation, is_valid_identifier
 
 if TYPE_CHECKING:
     from .table_version import TableVersion
+    from .table_version_handle import ColumnHandle
     from .table_version_path import TableVersionPath
 
 _logger = logging.getLogger('pixeltable')
@@ -35,11 +36,10 @@ class Column:
     _media_validation: Optional[MediaValidation]  # if not set, TableVersion.media_validation applies
     schema_version_add: Optional[int]
     schema_version_drop: Optional[int]
-    _records_errors: Optional[bool]
+    _stores_cellmd: Optional[bool]
     sa_col: Optional[sql.schema.Column]
     sa_col_type: Optional[sql.sqltypes.TypeEngine]
-    sa_errormsg_col: Optional[sql.schema.Column]
-    sa_errortype_col: Optional[sql.schema.Column]
+    sa_cellmd_col: Optional[sql.schema.Column]  # JSON metadata for the cell, e.g. errortype, errormsg for media columns
     _value_expr: Optional[exprs.Expr]
     value_expr_dict: Optional[dict[str, Any]]
     # we store a TableVersion here, not a TableVersionHandle, because this column is owned by that TableVersion instance
@@ -58,7 +58,7 @@ class Column:
         schema_version_add: Optional[int] = None,
         schema_version_drop: Optional[int] = None,
         sa_col_type: Optional[sql.sqltypes.TypeEngine] = None,
-        records_errors: Optional[bool] = None,
+        stores_cellmd: Optional[bool] = None,
         value_expr_dict: Optional[dict[str, Any]] = None,
         tbl: Optional[TableVersion] = None,
     ):
@@ -117,15 +117,14 @@ class Column:
         self.schema_version_add = schema_version_add
         self.schema_version_drop = schema_version_drop
 
-        self._records_errors = records_errors
+        self._stores_cellmd = stores_cellmd
 
         # column in the stored table for the values of this Column
         self.sa_col = None
         self.sa_col_type = sa_col_type
 
         # computed cols also have storage columns for the exception string and type
-        self.sa_errormsg_col = None
-        self.sa_errortype_col = None
+        self.sa_cellmd_col = None
 
     def init_value_expr(self) -> None:
         from pixeltable import exprs
@@ -148,6 +147,15 @@ class Column:
             )
             warnings.warn(message, category=excs.PixeltableWarning, stacklevel=2)
 
+    @property
+    def handle(self) -> 'ColumnHandle':
+        """Returns a ColumnHandle for this Column."""
+        from .table_version_handle import ColumnHandle
+
+        assert self.tbl is not None
+        assert self.id is not None
+        return ColumnHandle(self.tbl.handle, self.id)
+
     @property
     def value_expr(self) -> Optional[exprs.Expr]:
         assert self.value_expr_dict is None or self._value_expr is not None
@@ -193,11 +201,11 @@ class Column:
         return self.stored
 
     @property
-    def records_errors(self) -> bool:
+    def stores_cellmd(self) -> bool:
         """True if this column also stores error information."""
         # default: record errors for computed and media columns
-        if self._records_errors is not None:
-            return self._records_errors
+        if self._stores_cellmd is not None:
+            return self._stores_cellmd
         return self.is_stored and (self.is_computed or self.col_type.is_media_type())
 
     @property
@@ -233,28 +241,29 @@ class Column:
         """
         assert self.is_stored
         # all storage columns are nullable (we deal with null errors in Pixeltable directly)
-        self.sa_col = sql.Column(
-            self.store_name(),
-            self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type,
-            nullable=True,
-        )
-        if self.is_computed or self.col_type.is_media_type():
-            self.sa_errormsg_col = sql.Column(self.errormsg_store_name(), ts.StringType().to_sa_type(), nullable=True)
-            self.sa_errortype_col = sql.Column(self.errortype_store_name(), ts.StringType().to_sa_type(), nullable=True)
+        self.sa_col = sql.Column(self.store_name(), self.get_sa_col_type(), nullable=True)
+        if self.stores_cellmd:
+            # JSON metadata for the cell, e.g. errortype, errormsg for media columns
+            self.sa_cellmd_col = sql.Column(self.cellmd_store_name(), self.sa_cellmd_type(), nullable=True)
 
     def get_sa_col_type(self) -> sql.sqltypes.TypeEngine:
         return self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type
 
+    @classmethod
+    def cellmd_type(cls) -> ts.ColumnType:
+        return ts.JsonType(nullable=True)
+
+    @classmethod
+    def sa_cellmd_type(cls) -> sql.sqltypes.TypeEngine:
+        return cls.cellmd_type().to_sa_type()
+
     def store_name(self) -> str:
         assert self.id is not None
         assert self.is_stored
         return f'col_{self.id}'
 
-    def errormsg_store_name(self) -> str:
-        return f'{self.store_name()}_errormsg'
-
-    def errortype_store_name(self) -> str:
-        return f'{self.store_name()}_errortype'
+    def cellmd_store_name(self) -> str:
+        return f'{self.store_name()}_cellmd'
 
     def __str__(self) -> str:
         return f'{self.name}: {self.col_type}'

pixeltable/catalog/globals.py

@@ -1,14 +1,12 @@
 from __future__ import annotations
 
-import dataclasses
 import enum
 import itertools
 import logging
+from dataclasses import dataclass
 from typing import Optional
 from uuid import UUID
 
-from typing_extensions import Self
-
 import pixeltable.exceptions as excs
 
 _logger = logging.getLogger('pixeltable')
@@ -22,54 +20,13 @@ _ROWID_COLUMN_NAME = '_rowid'
 _PREDEF_SYMBOLS: Optional[set[str]] = None
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclass(frozen=True)
 class QColumnId:
     """Qualified column id"""
 
     tbl_id: UUID
     col_id: int
 
-    # def __hash__(self) -> int:
-    #     return hash((self.tbl_id, self.col_id))
-
-
-@dataclasses.dataclass
-class UpdateStatus:
-    """
-    Information about updates that resulted from a table operation.
-    """
-
-    num_rows: int = 0
-    # TODO: disambiguate what this means: # of slots computed or # of columns computed?
-    num_computed_values: int = 0
-    num_excs: int = 0
-    updated_cols: list[str] = dataclasses.field(default_factory=list)
-    cols_with_excs: list[str] = dataclasses.field(default_factory=list)
-
-    def __iadd__(self, other: 'UpdateStatus') -> Self:
-        self.num_rows += other.num_rows
-        self.num_computed_values += other.num_computed_values
-        self.num_excs += other.num_excs
-        self.updated_cols = list(dict.fromkeys(self.updated_cols + other.updated_cols))
-        self.cols_with_excs = list(dict.fromkeys(self.cols_with_excs + other.cols_with_excs))
-        return self
-
-    @property
-    def insert_msg(self) -> str:
-        """Return a message describing the results of an insert operation."""
-        if self.num_excs == 0:
-            cols_with_excs_str = ''
-        else:
-            cols_with_excs_str = (
-                f' across {len(self.cols_with_excs)} column{"" if len(self.cols_with_excs) == 1 else "s"}'
-            )
-            cols_with_excs_str += f' ({", ".join(self.cols_with_excs)})'
-        msg = (
-            f'Inserted {self.num_rows} row{"" if self.num_rows == 1 else "s"} '
-            f'with {self.num_excs} error{"" if self.num_excs == 1 else "s"}{cols_with_excs_str}.'
-        )
-        return msg
-
 
 class MediaValidation(enum.Enum):
     ON_READ = 0

pixeltable/catalog/insertable_table.py

@@ -10,11 +10,12 @@ from pixeltable import exceptions as excs, type_system as ts
 from pixeltable.env import Env
 from pixeltable.utils.filecache import FileCache
 
-from .globals import MediaValidation, UpdateStatus
+from .globals import MediaValidation
 from .table import Table
 from .table_version import TableVersion
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
+from .update_status import UpdateStatus
 
 if TYPE_CHECKING:
     from pixeltable import exprs
@@ -106,6 +107,7 @@ class InsertableTable(Table):
 
     def _get_metadata(self) -> dict[str, Any]:
         md = super()._get_metadata()
+        md['base'] = None
         md['is_view'] = False
         md['is_snapshot'] = False
         return md
@@ -171,13 +173,14 @@ class InsertableTable(Table):
         from pixeltable.catalog import Catalog
         from pixeltable.io.table_data_conduit import DFTableDataConduit
 
-        status = pxt.UpdateStatus()
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
             if isinstance(data_source, DFTableDataConduit):
+                status = pxt.UpdateStatus()
                 status += self._tbl_version.get().insert(
                     rows=None, df=data_source.pxt_df, print_stats=print_stats, fail_on_exception=fail_on_exception
                 )
             else:
+                status = pxt.UpdateStatus()
                 for row_batch in data_source.valid_row_batch():
                     status += self._tbl_version.get().insert(
                         rows=row_batch, df=None, print_stats=print_stats, fail_on_exception=fail_on_exception

pixeltable/catalog/path.py

@@ -77,5 +77,11 @@ class Path:
     def __str__(self) -> str:
         return '.'.join(self.components)
 
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, Path) and str(self) == str(other)
+
+    def __hash__(self) -> int:
+        return hash(str(self))
+
     def __lt__(self, other: Path) -> bool:
         return str(self) < str(other)