pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.20__py3-none-any.whl

pixeltable/catalog/table_version_handle.py

@@ -1,13 +1,16 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Optional
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
 from uuid import UUID
 
+from pixeltable import exceptions as excs
+
 from .table_version import TableVersion
 
 if TYPE_CHECKING:
-    pass
+    from pixeltable.catalog import Column
 
 _logger = logging.getLogger('pixeltable')
 
@@ -18,10 +21,10 @@ class TableVersionHandle:
     """
 
     id: UUID
-    effective_version: Optional[int]
-    _tbl_version: Optional[TableVersion]
+    effective_version: int | None
+    _tbl_version: TableVersion | None
 
-    def __init__(self, tbl_id: UUID, effective_version: Optional[int], tbl_version: Optional[TableVersion] = None):
+    def __init__(self, tbl_id: UUID, effective_version: int | None, tbl_version: TableVersion | None = None):
         self.id = tbl_id
         self.effective_version = effective_version
         self._tbl_version = tbl_version
@@ -34,6 +37,13 @@ class TableVersionHandle:
     def __hash__(self) -> int:
         return hash((self.id, self.effective_version))
 
+    def __repr__(self) -> str:
+        return f'TableVersionHandle(id={self.id!r}, effective_version={self.effective_version})'
+
+    @property
+    def is_snapshot(self) -> bool:
+        return self.effective_version is not None
+
     @classmethod
     def create(cls, tbl_version: TableVersion) -> TableVersionHandle:
         return cls(tbl_version.id, tbl_version.effective_version, tbl_version)
@@ -53,7 +63,6 @@ class TableVersionHandle:
             else:
                 self._tbl_version = Catalog.get().get_tbl_version(self.id, self.effective_version)
         if self.effective_version is None:
-            # make sure we don't see a discarded instance of a live TableVersion
             tvs = list(Catalog.get()._tbl_versions.values())
             assert self._tbl_version in tvs
         return self._tbl_version
@@ -64,3 +73,25 @@ class TableVersionHandle:
     @classmethod
     def from_dict(cls, d: dict) -> TableVersionHandle:
         return cls(UUID(d['id']), d['effective_version'])
+
+
+@dataclass(frozen=True)
+class ColumnHandle:
+    tbl_version: TableVersionHandle
+    col_id: int
+
+    def get(self) -> 'Column':
+        if self.col_id not in self.tbl_version.get().cols_by_id:
+            schema_version_drop = self.tbl_version.get()._tbl_md.column_md[self.col_id].schema_version_drop
+            raise excs.Error(
+                f'Column was dropped (no record for column ID {self.col_id} in table '
+                f'{self.tbl_version.get().versioned_name!r}; it was dropped in table version {schema_version_drop})'
+            )
+        return self.tbl_version.get().cols_by_id[self.col_id]
+
+    def as_dict(self) -> dict:
+        return {'tbl_version': self.tbl_version.as_dict(), 'col_id': self.col_id}
+
+    @classmethod
+    def from_dict(cls, d: dict) -> ColumnHandle:
+        return cls(tbl_version=TableVersionHandle.from_dict(d['tbl_version']), col_id=d['col_id'])

pixeltable/catalog/table_version_path.py

@@ -1,13 +1,13 @@
 from __future__ import annotations
 
 import logging
-from typing import Optional
 from uuid import UUID
 
 from pixeltable.env import Env
 from pixeltable.metadata import schema
 
 from .column import Column
+from .globals import MediaValidation
 from .table_version import TableVersion
 from .table_version_handle import TableVersionHandle
 
@@ -38,10 +38,10 @@ class TableVersionPath:
     """
 
     tbl_version: TableVersionHandle
-    base: Optional[TableVersionPath]
-    _cached_tbl_version: Optional[TableVersion]
+    base: TableVersionPath | None
+    _cached_tbl_version: TableVersion | None
 
-    def __init__(self, tbl_version: TableVersionHandle, base: Optional[TableVersionPath] = None):
+    def __init__(self, tbl_version: TableVersionHandle, base: TableVersionPath | None = None):
         assert tbl_version is not None
         self.tbl_version = tbl_version
         self.base = base
@@ -50,7 +50,7 @@ class TableVersionPath:
     @classmethod
     def from_md(cls, path: schema.TableVersionPath) -> TableVersionPath:
         assert len(path) > 0
-        result: Optional[TableVersionPath] = None
+        result: TableVersionPath | None = None
         for tbl_id_str, effective_version in path[::-1]:
             tbl_id = UUID(tbl_id_str)
             result = TableVersionPath(TableVersionHandle(tbl_id, effective_version), base=result)
@@ -75,7 +75,7 @@ class TableVersionPath:
         elif self._cached_tbl_version is not None:
             return
 
-        with Catalog.get().begin_xact(for_write=False):
+        with Catalog.get().begin_xact(tbl_id=self.tbl_version.id, for_write=False):
             self._cached_tbl_version = self.tbl_version.get()
 
     def clear_cached_md(self) -> None:
@@ -83,6 +83,7 @@ class TableVersionPath:
         if self.base is not None:
             self.base.clear_cached_md()
 
+    @property
     def tbl_id(self) -> UUID:
         """Return the id of the table/view that this path represents"""
         return self.tbl_version.id
@@ -92,6 +93,11 @@ class TableVersionPath:
         self.refresh_cached_md()
         return self._cached_tbl_version.version
 
+    def schema_version(self) -> int:
+        """Return the version of the table/view that this path represents"""
+        self.refresh_cached_md()
+        return self._cached_tbl_version.schema_version
+
     def tbl_name(self) -> str:
         """Return the name of the table/view that this path represents"""
         self.refresh_cached_md()
@@ -103,10 +109,7 @@ class TableVersionPath:
 
     def is_snapshot(self) -> bool:
         """Return True if this is a path of snapshot versions"""
-        self.refresh_cached_md()
-        if not self._cached_tbl_version.is_snapshot:
-            return False
-        return self.base.is_snapshot() if self.base is not None else True
+        return self.tbl_version.is_snapshot
 
     def is_view(self) -> bool:
         self.refresh_cached_md()
@@ -116,10 +119,30 @@ class TableVersionPath:
         self.refresh_cached_md()
         return self._cached_tbl_version.is_component_view
 
+    def is_replica(self) -> bool:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_replica
+
+    def is_mutable(self) -> bool:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_mutable
+
     def is_insertable(self) -> bool:
         self.refresh_cached_md()
         return self._cached_tbl_version.is_insertable
 
+    def comment(self) -> str:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.comment
+
+    def num_retained_versions(self) -> int:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.num_retained_versions
+
+    def media_validation(self) -> MediaValidation:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.media_validation
+
     def get_tbl_versions(self) -> list[TableVersionHandle]:
         """Return all tbl versions"""
         if self.base is None:
@@ -132,7 +155,7 @@ class TableVersionPath:
             return []
         return self.base.get_tbl_versions()
 
-    def find_tbl_version(self, id: UUID) -> Optional[TableVersionHandle]:
+    def find_tbl_version(self, id: UUID) -> TableVersionHandle | None:
         """Return the matching TableVersion in the chain of TableVersions, starting with this one"""
         if self.tbl_version.id == id:
             return self.tbl_version
@@ -160,40 +183,30 @@ class TableVersionPath:
         cols = self.columns()
         return {col.id: col for col in cols}
 
-    def get_column(self, name: str, include_bases: Optional[bool] = None) -> Optional[Column]:
+    def get_column(self, name: str) -> Column | None:
         """Return the column with the given name, or None if not found"""
         self.refresh_cached_md()
         col = self._cached_tbl_version.cols_by_name.get(name)
         if col is not None:
             return col
-        elif self.base is not None and (include_bases or self._cached_tbl_version.include_base_columns):
+        elif self.base is not None and self._cached_tbl_version.include_base_columns:
             return self.base.get_column(name)
         else:
             return None
 
-    def get_column_by_id(self, tbl_id: UUID, col_id: int) -> Optional[Column]:
-        """Return the column for the given tbl/col id"""
-        self.refresh_cached_md()
-        if self.tbl_version.id == tbl_id:
-            assert col_id in self._cached_tbl_version.cols_by_id
-            return self._cached_tbl_version.cols_by_id[col_id]
-        elif self.base is not None:
-            return self.base.get_column_by_id(tbl_id, col_id)
-        else:
-            return None
-
-    def has_column(self, col: Column, include_bases: bool = True) -> bool:
+    def has_column(self, col: Column) -> bool:
         """Return True if this table has the given column."""
+        assert col.get_tbl() is not None
         self.refresh_cached_md()
-        assert col.tbl is not None
+
         if (
-            col.tbl.id == self.tbl_version.id
-            and col.tbl.effective_version == self.tbl_version.effective_version
+            col.get_tbl().id == self.tbl_version.id
+            and col.get_tbl().effective_version == self.tbl_version.effective_version
             and col.id in self._cached_tbl_version.cols_by_id
         ):
             # the column is visible in this table version
             return True
-        elif self.base is not None and include_bases:
+        elif self.base is not None:
             return self.base.has_column(col)
         else:
             return False

pixeltable/catalog/tbl_ops.py

@@ -0,0 +1,50 @@
+# This file contains all dataclasses related to schema.PendingTableOp:
+# - TableOp: the container for each log entry
+# - <>Op: the actual operation, which is performed by TableVersion.exec_op(); each <>Op class contains
+#   enough information for exec_op() to perform the operation without having to reference data outside of
+#   TableVersion
+
+import dataclasses
+from typing import Any
+
+
+@dataclasses.dataclass
+class CreateStoreTableOp:
+    pass
+
+
+@dataclasses.dataclass
+class CreateIndexOp:
+    idx_id: int
+
+
+@dataclasses.dataclass
+class LoadViewOp:
+    view_path: dict[str, Any]  # needed to create the view load plan
+
+
+@dataclasses.dataclass
+class DeleteTableMdOp:
+    pass
+
+
+@dataclasses.dataclass
+class DeleteTableMediaFilesOp:
+    pass
+
+
+@dataclasses.dataclass
+class DropStoreTableOp:
+    pass
+
+
+@dataclasses.dataclass
+class TableOp:
+    tbl_id: str  # uuid.UUID
+    op_sn: int  # sequence number within the update operation; [0, num_ops)
+    num_ops: int  # total number of ops forming the update operation
+    needs_xact: bool  # if True, op must be run as part of a transaction
+
+    create_store_table_op: CreateStoreTableOp | None = None
+    create_index_op: CreateIndexOp | None = None
+    load_view_op: LoadViewOp | None = None

pixeltable/catalog/update_status.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from IPython.lib.pretty import RepresentationPrinter
+
+
+@dataclass(frozen=True)
+class RowCountStats:
+    """
+    Statistics about the counts of rows affected by a table operation.
+    """
+
+    ins_rows: int = 0  # rows inserted
+    del_rows: int = 0  # rows deleted
+    upd_rows: int = 0  # rows updated
+    num_excs: int = 0  # total number of exceptions
+    # TODO: disambiguate what this means: # of slots computed or # of columns computed?
+    computed_values: int = 0  # number of computed values (e.g., computed columns) affected by the operation
+
+    @property
+    def num_rows(self) -> int:
+        return self.ins_rows + self.del_rows + self.upd_rows
+
+    def insert_to_update(self) -> 'RowCountStats':
+        """
+        Convert insert row count stats to update row count stats.
+        This is used when an insert operation is treated as an update.
+        """
+        return RowCountStats(
+            ins_rows=0,
+            del_rows=self.del_rows,
+            upd_rows=self.upd_rows + self.ins_rows,
+            num_excs=self.num_excs,
+            computed_values=self.computed_values,
+        )
+
+    def __add__(self, other: 'RowCountStats') -> 'RowCountStats':
+        """
+        Add the stats from two RowCountStats objects together.
+        """
+        return RowCountStats(
+            ins_rows=self.ins_rows + other.ins_rows,
+            del_rows=self.del_rows + other.del_rows,
+            upd_rows=self.upd_rows + other.upd_rows,
+            num_excs=self.num_excs + other.num_excs,
+            computed_values=self.computed_values + other.computed_values,
+        )
+
+
+@dataclass(frozen=True)
+class UpdateStatus:
+    """
+    Information about changes to table data or table schema
+    """
+
+    updated_cols: list[str] = field(default_factory=list)
+    """Columns that were updated."""
+    cols_with_excs: list[str] = field(default_factory=list)
+    """Columns that encountered exceptions."""
+
+    # stats for the rows affected by the operation
+    row_count_stats: RowCountStats = field(default_factory=RowCountStats)
+    """Row count statistics for rows affected by this operation."""
+
+    # stats for changes cascaded to other tables
+    cascade_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
+    """Row count statistics for changes cascaded to other tables."""
+
+    # stats for the rows affected by the operation in an external store
+    ext_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
+    """Row count statistics for rows affected in an external store."""
+
+    @property
+    def num_rows(self) -> int:
+        """Total number of rows affected (including cascaded changes)."""
+        return self.row_count_stats.num_rows + self.cascade_row_count_stats.num_rows
+
+    @property
+    def num_excs(self) -> int:
+        """Total number of exceptions encountered (including cascaded changes)."""
+        return self.row_count_stats.num_excs + self.cascade_row_count_stats.num_excs
+
+    @property
+    def num_computed_values(self) -> int:
+        """Total number of computed values affected (including cascaded changes)."""
+        return self.row_count_stats.computed_values + self.cascade_row_count_stats.computed_values
+
+    def insert_to_update(self) -> 'UpdateStatus':
+        """
+        Convert the update status from an insert operation to an update operation.
+        This is used when an insert operation is treated as an update.
+        """
+        return UpdateStatus(
+            updated_cols=self.updated_cols,
+            cols_with_excs=self.cols_with_excs,
+            row_count_stats=self.row_count_stats.insert_to_update(),
+            cascade_row_count_stats=self.cascade_row_count_stats.insert_to_update(),
+            ext_row_count_stats=self.ext_row_count_stats,
+        )
+
+    def to_cascade(self) -> 'UpdateStatus':
+        """
+        Convert the update status to a cascade update status.
+        This is used when an operation cascades changes to other tables.
+        """
+        return UpdateStatus(
+            updated_cols=self.updated_cols,
+            cols_with_excs=self.cols_with_excs,
+            row_count_stats=RowCountStats(),
+            cascade_row_count_stats=self.cascade_row_count_stats + self.row_count_stats,
+            ext_row_count_stats=self.ext_row_count_stats,
+        )
+
+    def __add__(self, other: 'UpdateStatus') -> UpdateStatus:
+        """
+        Add the update status from two UpdateStatus objects together.
+        """
+        return UpdateStatus(
+            updated_cols=list(dict.fromkeys(self.updated_cols + other.updated_cols)),
+            cols_with_excs=list(dict.fromkeys(self.cols_with_excs + other.cols_with_excs)),
+            row_count_stats=self.row_count_stats + other.row_count_stats,
+            cascade_row_count_stats=self.cascade_row_count_stats + other.cascade_row_count_stats,
+            ext_row_count_stats=self.ext_row_count_stats + other.ext_row_count_stats,
+        )
+
+    @property
+    def insert_msg(self) -> str:
+        """Return a message describing the results of an insert operation."""
+        if self.num_excs == 0:
+            cols_with_excs_str = ''
+        else:
+            cols_with_excs_str = (
+                f' across {len(self.cols_with_excs)} column{"" if len(self.cols_with_excs) == 1 else "s"}'
+            )
+            cols_with_excs_str += f' ({", ".join(self.cols_with_excs)})'
+        msg = (
+            f'Inserted {self.num_rows} row{"" if self.num_rows == 1 else "s"} '
+            f'with {self.num_excs} error{"" if self.num_excs == 1 else "s"}{cols_with_excs_str}.'
+        )
+        return msg
+
+    @classmethod
+    def __cnt_str(cls, cnt: int, item: str) -> str:
+        assert cnt > 0
+        return f'{cnt} {item}{"" if cnt == 1 else "s"}'
+
+    def _repr_pretty_(self, p: 'RepresentationPrinter', cycle: bool) -> None:
+        messages = []
+        # Combine row count stats and cascade row count stats
+        stats = self.row_count_stats + self.cascade_row_count_stats
+        if stats.ins_rows > 0:
+            messages.append(f'{self.__cnt_str(stats.ins_rows, "row")} inserted')
+        if stats.del_rows > 0:
+            messages.append(f'{self.__cnt_str(stats.del_rows, "row")} deleted')
+        if stats.upd_rows > 0:
+            messages.append(f'{self.__cnt_str(stats.upd_rows, "row")} updated')
+        if stats.computed_values > 0:
+            messages.append(f'{self.__cnt_str(stats.computed_values, "value")} computed')
+        if stats.num_excs > 0:
+            messages.append(self.__cnt_str(stats.num_excs, 'exception'))
+        p.text(', '.join(messages) + '.' if len(messages) > 0 else 'No rows affected.')
+
+    @property
+    def pxt_rows_updated(self) -> int:
+        """
+        Returns the number of Pixeltable rows that were updated as a result of the operation.
+        """
+        return (self.row_count_stats + self.cascade_row_count_stats).upd_rows
+
+    @property
+    def external_rows_updated(self) -> int:
+        """Number of rows updated in an external store."""
+        return self.ext_row_count_stats.upd_rows
+
+    @property
+    def external_rows_created(self) -> int:
+        """Number of rows created in an external store."""
+        return self.ext_row_count_stats.ins_rows
+
+    @property
+    def external_rows_deleted(self) -> int:
+        """Number of rows deleted from an external store."""
+        return self.ext_row_count_stats.del_rows
+
+    @property
+    def ext_num_rows(self) -> int:
+        """Total number of rows affected in an external store."""
+        return self.ext_row_count_stats.num_rows