pixeltable 0.3.14__py3-none-any.whl → 0.5.7__py3-none-any.whl

pixeltable/catalog/table_version_handle.py

@@ -1,13 +1,16 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Optional
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
 from uuid import UUID
 
-from .table_version import TableVersion
+from pixeltable import exceptions as excs
+
+from .table_version import TableVersion, TableVersionKey
 
 if TYPE_CHECKING:
-    pass
+    from pixeltable.catalog import Column
 
 _logger = logging.getLogger('pixeltable')
 
@@ -15,15 +18,15 @@ _logger = logging.getLogger('pixeltable')
 class TableVersionHandle:
     """
     Indirection mechanism for TableVersion instances, which get resolved against the catalog at runtime.
+
+    See the TableVersion docstring for details on the semantics of `effective_version` and `anchor_tbl_id`.
     """
 
-    id: UUID
-    effective_version: Optional[int]
-    _tbl_version: Optional[TableVersion]
+    key: TableVersionKey
+    _tbl_version: TableVersion | None
 
-    def __init__(self, tbl_id: UUID, effective_version: Optional[int], tbl_version: Optional[TableVersion] = None):
-        self.id = tbl_id
-        self.effective_version = effective_version
+    def __init__(self, key: TableVersionKey, *, tbl_version: TableVersion | None = None):
+        self.key = key
         self._tbl_version = tbl_version
 
     def __eq__(self, other: object) -> bool:
@@ -34,20 +37,73 @@ class TableVersionHandle:
     def __hash__(self) -> int:
         return hash((self.id, self.effective_version))
 
-    @classmethod
-    def create(cls, tbl_version: TableVersion) -> TableVersionHandle:
-        return cls(tbl_version.id, tbl_version.effective_version, tbl_version)
+    def __repr__(self) -> str:
+        return (
+            f'TableVersionHandle(id={self.id!r}, effective_version={self.effective_version}, '
+            f'anchor_tbl_id={self.anchor_tbl_id})'
+        )
+
+    @property
+    def id(self) -> UUID:
+        return self.key.tbl_id
+
+    @property
+    def effective_version(self) -> int | None:
+        return self.key.effective_version
+
+    @property
+    def anchor_tbl_id(self) -> UUID | None:
+        return self.key.anchor_tbl_id
+
+    @property
+    def is_snapshot(self) -> bool:
+        return self.effective_version is not None
 
     def get(self) -> TableVersion:
         from .catalog import Catalog
 
-        if self._tbl_version is None:
-            self._tbl_version = Catalog.get().get_tbl_version(self.id, self.effective_version)
+        cat = Catalog.get()
+        if self._tbl_version is None or not self._tbl_version.is_validated:
+            if self.effective_version is not None and self._tbl_version is not None:
+                # this is a snapshot version; we need to make sure we refer to the instance cached
+                # in Catalog, in order to avoid mixing sa_tbl instances in the same transaction
+                # (which will lead to duplicates in the From clause generated in SqlNode.create_from_clause())
+                assert self.key in cat._tbl_versions
+                self._tbl_version = cat._tbl_versions[self.key]
+                self._tbl_version.is_validated = True
+            else:
+                self._tbl_version = Catalog.get().get_tbl_version(self.key)
+                assert self._tbl_version.key == self.key
+        if self.effective_version is None:
+            tvs = list(Catalog.get()._tbl_versions.values())
+            assert self._tbl_version in tvs, self._tbl_version
         return self._tbl_version
 
     def as_dict(self) -> dict:
-        return {'id': str(self.id), 'effective_version': self.effective_version}
+        return self.key.as_dict()
 
     @classmethod
     def from_dict(cls, d: dict) -> TableVersionHandle:
-        return cls(UUID(d['id']), d['effective_version'])
+        return cls(TableVersionKey.from_dict(d))
+
+
+@dataclass(frozen=True)
+class ColumnHandle:
+    tbl_version: TableVersionHandle
+    col_id: int
+
+    def get(self) -> 'Column':
+        if self.col_id not in self.tbl_version.get().cols_by_id:
+            schema_version_drop = self.tbl_version.get()._tbl_md.column_md[self.col_id].schema_version_drop
+            raise excs.Error(
+                f'Column was dropped (no record for column ID {self.col_id} in table '
+                f'{self.tbl_version.get().versioned_name!r}; it was dropped in table version {schema_version_drop})'
+            )
+        return self.tbl_version.get().cols_by_id[self.col_id]
+
+    def as_dict(self) -> dict:
+        return {'tbl_version': self.tbl_version.as_dict(), 'col_id': self.col_id}
+
+    @classmethod
+    def from_dict(cls, d: dict) -> ColumnHandle:
+        return cls(tbl_version=TableVersionHandle.from_dict(d['tbl_version']), col_id=d['col_id'])

pixeltable/catalog/table_version_path.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import logging
-from typing import Optional
 from uuid import UUID
 
+from pixeltable.env import Env
 from pixeltable.metadata import schema
 
 from .column import Column
+from .globals import MediaValidation
+from .table_version import TableVersion, TableVersionKey
 from .table_version_handle import TableVersionHandle
 
 _logger = logging.getLogger('pixeltable')
@@ -22,23 +24,40 @@ class TableVersionPath:
 
     TableVersionPath contains all metadata needed to execute queries and updates against a particular version of a
     table/view.
+
+    TableVersionPath supplies metadata needed for query construction (eg, column names), for which it uses a
+    cached TableVersion instance.
+    - when running inside a transaction, this instance is guaranteed to be validated
+    - when running outside a transaction, we use an unvalidated instance in order to avoid repeated validation
+      on every metadata-related method call (the instance won't stay validated, because TableVersionHandle.get()
+      runs a local transaction, at the end of which the instance is again invalidated)
+    - supplying metadata from an unvalidated instance is okay, because it needs to get revalidated anyway when a
+      query actually runs (at which point there is a transaction context) - there is no guarantee that in between
+      constructing a Query and executing it, the underlying table schema hasn't changed (eg, a concurrent process
+      could have dropped a column referenced in the query).
     """
 
     tbl_version: TableVersionHandle
-    base: Optional[TableVersionPath]
+    base: TableVersionPath | None
+    _cached_tbl_version: TableVersion | None
 
-    def __init__(self, tbl_version: TableVersionHandle, base: Optional[TableVersionPath] = None):
+    def __init__(self, tbl_version: TableVersionHandle, base: TableVersionPath | None = None):
         assert tbl_version is not None
         self.tbl_version = tbl_version
         self.base = base
+        self._cached_tbl_version = None
+
+        if self.base is not None and tbl_version.anchor_tbl_id is not None:
+            self.base = self.base.anchor_to(tbl_version.anchor_tbl_id)
 
     @classmethod
     def from_md(cls, path: schema.TableVersionPath) -> TableVersionPath:
         assert len(path) > 0
-        result: Optional[TableVersionPath] = None
+        result: TableVersionPath | None = None
         for tbl_id_str, effective_version in path[::-1]:
             tbl_id = UUID(tbl_id_str)
-            result = TableVersionPath(TableVersionHandle(tbl_id, effective_version), base=result)
+            key = TableVersionKey(tbl_id, effective_version, None)
+            result = TableVersionPath(TableVersionHandle(key), base=result)
         return result
 
     def as_md(self) -> schema.TableVersionPath:
@@ -47,17 +66,59 @@ class TableVersionPath:
             result.extend(self.base.as_md())
         return result
 
+    def refresh_cached_md(self) -> None:
+        from pixeltable.catalog import Catalog
+
+        if Env.get().in_xact:
+            # when we're running inside a transaction, we need to make sure to supply current metadata;
+            # mixing stale metadata with current metadata leads to query construction failures
+            # (multiple sqlalchemy Table instances for the same underlying table create corrupted From clauses)
+            if self._cached_tbl_version is not None and self._cached_tbl_version.is_validated:
+                # nothing to refresh
+                return
+        elif self._cached_tbl_version is not None:
+            return
+
+        with Catalog.get().begin_xact(tbl_id=self.tbl_version.id, for_write=False):
+            self._cached_tbl_version = self.tbl_version.get()
+
+    def anchor_to(self, anchor_tbl_id: UUID | None) -> TableVersionPath:
+        """
+        Return a new TableVersionPath with all of its non-snapshot TableVersions pointing to the given anchor_tbl_id.
+        (This will clear the existing anchor_tbl_id in the case anchor_tbl_id=None.)
+        """
+        if self.tbl_version.effective_version is not None:
+            return self
+
+        return TableVersionPath(
+            TableVersionHandle(TableVersionKey(self.tbl_version.id, None, anchor_tbl_id)),
+            base=self.base.anchor_to(anchor_tbl_id) if self.base is not None else None,
+        )
+
+    def clear_cached_md(self) -> None:
+        self._cached_tbl_version = None
+        if self.base is not None:
+            self.base.clear_cached_md()
+
+    @property
     def tbl_id(self) -> UUID:
         """Return the id of the table/view that this path represents"""
         return self.tbl_version.id
 
     def version(self) -> int:
         """Return the version of the table/view that this path represents"""
-        return self.tbl_version.get().version
+        self.refresh_cached_md()
+        return self._cached_tbl_version.version
+
+    def schema_version(self) -> int:
+        """Return the version of the table/view that this path represents"""
+        self.refresh_cached_md()
+        return self._cached_tbl_version.schema_version
 
     def tbl_name(self) -> str:
         """Return the name of the table/view that this path represents"""
-        return self.tbl_version.get().name
+        self.refresh_cached_md()
+        return self._cached_tbl_version.name
 
     def path_len(self) -> int:
         """Return the length of the path"""
@@ -65,18 +126,39 @@ class TableVersionPath:
 
     def is_snapshot(self) -> bool:
         """Return True if this is a path of snapshot versions"""
-        if not self.tbl_version.get().is_snapshot:
-            return False
-        return self.base.is_snapshot() if self.base is not None else True
+        return self.tbl_version.is_snapshot
 
     def is_view(self) -> bool:
-        return self.tbl_version.get().is_view
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_view
 
     def is_component_view(self) -> bool:
-        return self.tbl_version.get().is_component_view
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_component_view
+
+    def is_replica(self) -> bool:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_replica
+
+    def is_mutable(self) -> bool:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_mutable
 
     def is_insertable(self) -> bool:
-        return self.tbl_version.get().is_insertable()
+        self.refresh_cached_md()
+        return self._cached_tbl_version.is_insertable
+
+    def comment(self) -> str:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.comment
+
+    def num_retained_versions(self) -> int:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.num_retained_versions
+
+    def media_validation(self) -> MediaValidation:
+        self.refresh_cached_md()
+        return self._cached_tbl_version.media_validation
 
     def get_tbl_versions(self) -> list[TableVersionHandle]:
         """Return all tbl versions"""
@@ -90,7 +172,7 @@ class TableVersionPath:
             return []
         return self.base.get_tbl_versions()
 
-    def find_tbl_version(self, id: UUID) -> Optional[TableVersionHandle]:
+    def find_tbl_version(self, id: UUID) -> TableVersionHandle | None:
         """Return the matching TableVersion in the chain of TableVersions, starting with this one"""
         if self.tbl_version.id == id:
             return self.tbl_version
@@ -98,20 +180,14 @@ class TableVersionPath:
             return None
         return self.base.find_tbl_version(id)
 
-    @property
-    def ancestor_paths(self) -> list[TableVersionPath]:
-        if self.base is None:
-            return [self]
-        else:
-            return [self, *self.base.ancestor_paths]
-
     def columns(self) -> list[Column]:
         """Return all user columns visible in this tbl version path, including columns from bases"""
-        result = list(self.tbl_version.get().cols_by_name.values())
-        if self.base is not None and self.tbl_version.get().include_base_columns:
+        self.refresh_cached_md()
+        result = list(self._cached_tbl_version.cols_by_name.values())
+        if self.base is not None and self._cached_tbl_version.include_base_columns:
             base_cols = self.base.columns()
             # we only include base columns that don't conflict with one of our column names
-            result.extend(c for c in base_cols if c.name not in self.tbl_version.get().cols_by_name)
+            result.extend(c for c in base_cols if c.name not in self._cached_tbl_version.cols_by_name)
         return result
 
     def cols_by_name(self) -> dict[str, Column]:
@@ -124,37 +200,30 @@ class TableVersionPath:
         cols = self.columns()
         return {col.id: col for col in cols}
 
-    def get_column(self, name: str, include_bases: Optional[bool] = None) -> Optional[Column]:
+    def get_column(self, name: str) -> Column | None:
         """Return the column with the given name, or None if not found"""
-        col = self.tbl_version.get().cols_by_name.get(name)
+        self.refresh_cached_md()
+        col = self._cached_tbl_version.cols_by_name.get(name)
         if col is not None:
             return col
-        elif self.base is not None and (include_bases or self.tbl_version.get().include_base_columns):
+        elif self.base is not None and self._cached_tbl_version.include_base_columns:
             return self.base.get_column(name)
         else:
             return None
 
-    def get_column_by_id(self, tbl_id: UUID, col_id: int) -> Optional[Column]:
-        """Return the column for the given tbl/col id"""
-        if self.tbl_version.id == tbl_id:
-            assert col_id in self.tbl_version.get().cols_by_id
-            return self.tbl_version.get().cols_by_id[col_id]
-        elif self.base is not None:
-            return self.base.get_column_by_id(tbl_id, col_id)
-        else:
-            return None
-
-    def has_column(self, col: Column, include_bases: bool = True) -> bool:
+    def has_column(self, col: Column) -> bool:
         """Return True if this table has the given column."""
-        assert col.tbl is not None
+        assert col.get_tbl() is not None
+        self.refresh_cached_md()
+
         if (
-            col.tbl.id == self.tbl_version.id
-            and col.tbl.effective_version == self.tbl_version.effective_version
-            and col.id in self.tbl_version.get().cols_by_id
+            col.get_tbl().id == self.tbl_version.id
+            and col.get_tbl().effective_version == self.tbl_version.effective_version
+            and col.id in self._cached_tbl_version.cols_by_id
         ):
             # the column is visible in this table version
             return True
-        elif self.base is not None and include_bases:
+        elif self.base is not None:
             return self.base.has_column(col)
         else:
             return False

pixeltable/catalog/tbl_ops.py

@@ -0,0 +1,53 @@
+# This file contains all dataclasses related to schema.PendingTableOp:
+# - TableOp: the container for each log entry
+# - <>Op: the actual operation, which is performed by TableVersion.exec_op(); each <>Op class contains
+#   enough information for exec_op() to perform the operation without having to reference data outside of
+#   TableVersion
+
+import dataclasses
+from typing import Any
+
+
+@dataclasses.dataclass
+class CreateStoreTableOp:
+    pass
+
+
+@dataclasses.dataclass
+class CreateIndexOp:
+    idx_id: int
+
+
+@dataclasses.dataclass
+class LoadViewOp:
+    view_path: dict[str, Any]  # needed to create the view load plan
+
+
+@dataclasses.dataclass
+class DeleteTableMdOp:
+    pass
+
+
+@dataclasses.dataclass
+class DeleteTableMediaFilesOp:
+    pass
+
+
+@dataclasses.dataclass
+class DropStoreTableOp:
+    pass
+
+
+@dataclasses.dataclass
+class TableOp:
+    tbl_id: str  # uuid.UUID
+    op_sn: int  # sequence number within the update operation; [0, num_ops)
+    num_ops: int  # total number of ops forming the update operation
+    needs_xact: bool  # if True, op must be run as part of a transaction
+
+    create_store_table_op: CreateStoreTableOp | None = None
+    create_index_op: CreateIndexOp | None = None
+    load_view_op: LoadViewOp | None = None
+    delete_table_md_op: DeleteTableMdOp | None = None
+    delete_table_media_files_op: DeleteTableMediaFilesOp | None = None
+    drop_store_table_op: DropStoreTableOp | None = None

pixeltable/catalog/update_status.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from IPython.lib.pretty import RepresentationPrinter
+
+
+@dataclass(frozen=True)
+class RowCountStats:
+    """
+    Statistics about the counts of rows affected by a table operation.
+    """
+
+    ins_rows: int = 0  # rows inserted
+    del_rows: int = 0  # rows deleted
+    upd_rows: int = 0  # rows updated
+    num_excs: int = 0  # total number of exceptions
+    # TODO: disambiguate what this means: # of slots computed or # of columns computed?
+    computed_values: int = 0  # number of computed values (e.g., computed columns) affected by the operation
+
+    @property
+    def num_rows(self) -> int:
+        return self.ins_rows + self.del_rows + self.upd_rows
+
+    def insert_to_update(self) -> 'RowCountStats':
+        """
+        Convert insert row count stats to update row count stats.
+        This is used when an insert operation is treated as an update.
+        """
+        return RowCountStats(
+            ins_rows=0,
+            del_rows=self.del_rows,
+            upd_rows=self.upd_rows + self.ins_rows,
+            num_excs=self.num_excs,
+            computed_values=self.computed_values,
+        )
+
+    def __add__(self, other: 'RowCountStats') -> 'RowCountStats':
+        """
+        Add the stats from two RowCountStats objects together.
+        """
+        return RowCountStats(
+            ins_rows=self.ins_rows + other.ins_rows,
+            del_rows=self.del_rows + other.del_rows,
+            upd_rows=self.upd_rows + other.upd_rows,
+            num_excs=self.num_excs + other.num_excs,
+            computed_values=self.computed_values + other.computed_values,
+        )
+
+
+@dataclass(frozen=True)
+class UpdateStatus:
+    """
+    Information about changes to table data or table schema
+    """
+
+    updated_cols: list[str] = field(default_factory=list)
+    """Columns that were updated."""
+    cols_with_excs: list[str] = field(default_factory=list)
+    """Columns that encountered exceptions."""
+
+    # stats for the rows affected by the operation
+    row_count_stats: RowCountStats = field(default_factory=RowCountStats)
+    """Row count statistics for rows affected by this operation."""
+
+    # stats for changes cascaded to other tables
+    cascade_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
+    """Row count statistics for changes cascaded to other tables."""
+
+    # stats for the rows affected by the operation in an external store
+    ext_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
+    """Row count statistics for rows affected in an external store."""
+
+    @property
+    def num_rows(self) -> int:
+        """Total number of rows affected (including cascaded changes)."""
+        return self.row_count_stats.num_rows + self.cascade_row_count_stats.num_rows
+
+    @property
+    def num_excs(self) -> int:
+        """Total number of exceptions encountered (including cascaded changes)."""
+        return self.row_count_stats.num_excs + self.cascade_row_count_stats.num_excs
+
+    @property
+    def num_computed_values(self) -> int:
+        """Total number of computed values affected (including cascaded changes)."""
+        return self.row_count_stats.computed_values + self.cascade_row_count_stats.computed_values
+
+    def insert_to_update(self) -> 'UpdateStatus':
+        """
+        Convert the update status from an insert operation to an update operation.
+        This is used when an insert operation is treated as an update.
+        """
+        return UpdateStatus(
+            updated_cols=self.updated_cols,
+            cols_with_excs=self.cols_with_excs,
+            row_count_stats=self.row_count_stats.insert_to_update(),
+            cascade_row_count_stats=self.cascade_row_count_stats.insert_to_update(),
+            ext_row_count_stats=self.ext_row_count_stats,
+        )
+
+    def to_cascade(self) -> 'UpdateStatus':
+        """
+        Convert the update status to a cascade update status.
+        This is used when an operation cascades changes to other tables.
+        """
+        return UpdateStatus(
+            updated_cols=self.updated_cols,
+            cols_with_excs=self.cols_with_excs,
+            row_count_stats=RowCountStats(),
+            cascade_row_count_stats=self.cascade_row_count_stats + self.row_count_stats,
+            ext_row_count_stats=self.ext_row_count_stats,
+        )
+
+    def __add__(self, other: 'UpdateStatus') -> UpdateStatus:
+        """
+        Add the update status from two UpdateStatus objects together.
+        """
+        return UpdateStatus(
+            updated_cols=list(dict.fromkeys(self.updated_cols + other.updated_cols)),
+            cols_with_excs=list(dict.fromkeys(self.cols_with_excs + other.cols_with_excs)),
+            row_count_stats=self.row_count_stats + other.row_count_stats,
+            cascade_row_count_stats=self.cascade_row_count_stats + other.cascade_row_count_stats,
+            ext_row_count_stats=self.ext_row_count_stats + other.ext_row_count_stats,
+        )
+
+    @property
+    def insert_msg(self) -> str:
+        """A message describing the results of an insert operation."""
+        if self.num_excs == 0:
+            cols_with_excs_str = ''
+        else:
+            cols_with_excs_str = (
+                f' across {len(self.cols_with_excs)} column{"" if len(self.cols_with_excs) == 1 else "s"}'
+            )
+            cols_with_excs_str += f' ({", ".join(self.cols_with_excs)})'
+        msg = (
+            f'Inserted {self.num_rows} row{"" if self.num_rows == 1 else "s"} '
+            f'with {self.num_excs} error{"" if self.num_excs == 1 else "s"}{cols_with_excs_str}.'
+        )
+        return msg
+
+    @classmethod
+    def __cnt_str(cls, cnt: int, item: str) -> str:
+        assert cnt > 0
+        return f'{cnt} {item}{"" if cnt == 1 else "s"}'
+
+    def _repr_pretty_(self, p: 'RepresentationPrinter', cycle: bool) -> None:
+        messages = []
+        # Combine row count stats and cascade row count stats
+        stats = self.row_count_stats + self.cascade_row_count_stats
+        if stats.ins_rows > 0:
+            messages.append(f'{self.__cnt_str(stats.ins_rows, "row")} inserted')
+        if stats.del_rows > 0:
+            messages.append(f'{self.__cnt_str(stats.del_rows, "row")} deleted')
+        if stats.upd_rows > 0:
+            messages.append(f'{self.__cnt_str(stats.upd_rows, "row")} updated')
+        if stats.computed_values > 0:
+            messages.append(f'{self.__cnt_str(stats.computed_values, "value")} computed')
+        if stats.num_excs > 0:
+            messages.append(self.__cnt_str(stats.num_excs, 'exception'))
+        p.text(', '.join(messages) + '.' if len(messages) > 0 else 'No rows affected.')
+
+    @property
+    def pxt_rows_updated(self) -> int:
+        """
+        Returns the number of Pixeltable rows that were updated as a result of the operation.
+        """
+        return (self.row_count_stats + self.cascade_row_count_stats).upd_rows
+
+    @property
+    def external_rows_updated(self) -> int:
+        """Number of rows updated in an external store."""
+        return self.ext_row_count_stats.upd_rows
+
+    @property
+    def external_rows_created(self) -> int:
+        """Number of rows created in an external store."""
+        return self.ext_row_count_stats.ins_rows
+
+    @property
+    def external_rows_deleted(self) -> int:
+        """Number of rows deleted from an external store."""
+        return self.ext_row_count_stats.del_rows
+
+    @property
+    def ext_num_rows(self) -> int:
+        """Total number of rows affected in an external store."""
+        return self.ext_row_count_stats.num_rows