pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.2__py3-none-any.whl

pixeltable/catalog/column.py

@@ -15,6 +15,7 @@ from .globals import MediaValidation, is_valid_identifier
 
 if TYPE_CHECKING:
     from .table_version import TableVersion
+    from .table_version_handle import ColumnHandle
     from .table_version_path import TableVersionPath
 
 _logger = logging.getLogger('pixeltable')
@@ -42,11 +43,9 @@ class Column:
     sa_errortype_col: Optional[sql.schema.Column]
     _value_expr: Optional[exprs.Expr]
     value_expr_dict: Optional[dict[str, Any]]
-    dependent_cols: set[Column]
     # we store a TableVersion here, not a TableVersionHandle, because this column is owned by that TableVersion instance
     # (re-resolving it later to a different instance doesn't make sense)
     tbl: Optional[TableVersion]
-    # tbl: Optional[TableVersionHandle]
 
     def __init__(
         self,
@@ -62,6 +61,7 @@ class Column:
         sa_col_type: Optional[sql.sqltypes.TypeEngine] = None,
         records_errors: Optional[bool] = None,
         value_expr_dict: Optional[dict[str, Any]] = None,
+        tbl: Optional[TableVersion] = None,
     ):
         """Column constructor.
 
@@ -86,6 +86,7 @@ class Column:
         if name is not None and not is_valid_identifier(name):
             raise excs.Error(f"Invalid column name: '{name}'")
         self.name = name
+        self.tbl = tbl
         if col_type is None and computed_with is None:
             raise excs.Error(f'Column `{name}`: col_type is required if computed_with is not specified')
 
@@ -94,6 +95,7 @@ class Column:
         if computed_with is not None:
             value_expr = exprs.Expr.from_object(computed_with)
             if value_expr is None:
+                # TODO: this shouldn't be a user-facing error
                 raise excs.Error(
                     f'Column {name}: computed_with needs to be a valid Pixeltable expression, '
                     f'but it is a {type(computed_with)}'
@@ -101,13 +103,15 @@ class Column:
             else:
                 self._value_expr = value_expr.copy()
                 self.col_type = self._value_expr.col_type
+        if self._value_expr is not None and self.value_expr_dict is None:
+            self.value_expr_dict = self._value_expr.as_dict()
 
         if col_type is not None:
             self.col_type = col_type
         assert self.col_type is not None
 
         self.stored = stored
-        self.dependent_cols = set()  # cols with value_exprs that reference us; set by TableVersion
+        # self.dependent_cols = set()  # cols with value_exprs that reference us; set by TableVersion
         self.id = col_id
         self.is_pk = is_pk
         self._media_validation = media_validation
@@ -124,36 +128,44 @@ class Column:
         self.sa_errormsg_col = None
         self.sa_errortype_col = None
 
-        self.tbl = None  # set by owning TableVersion
+    def init_value_expr(self) -> None:
+        from pixeltable import exprs
+
+        if self._value_expr is not None or self.value_expr_dict is None:
+            return
+        self._value_expr = exprs.Expr.from_dict(self.value_expr_dict)
+        self._value_expr.bind_rel_paths()
+        if not self._value_expr.is_valid:
+            message = (
+                dedent(
+                    f"""
+                    The computed column {self.name!r} in table {self.tbl.name!r} is no longer valid.
+                    {{validation_error}}
+                    You can continue to query existing data from this column, but evaluating it on new data will raise an error.
+                    """  # noqa: E501
+                )
+                .strip()
+                .format(validation_error=self._value_expr.validation_error)
+            )
+            warnings.warn(message, category=excs.PixeltableWarning, stacklevel=2)
+
+    @property
+    def handle(self) -> 'ColumnHandle':
+        """Returns a ColumnHandle for this Column."""
+        from .table_version_handle import ColumnHandle
+
+        assert self.tbl is not None
+        assert self.id is not None
+        return ColumnHandle(self.tbl.handle, self.id)
 
     @property
     def value_expr(self) -> Optional[exprs.Expr]:
-        """Instantiate value_expr on-demand"""
-        # TODO: instantiate expr in the c'tor and add an Expr.prepare() that can create additional state after the
-        # catalog has been fully loaded; that way, we encounter bugs in the serialization/deserialization logic earlier
-        if self.value_expr_dict is not None and self._value_expr is None:
-            from pixeltable import exprs
-
-            self._value_expr = exprs.Expr.from_dict(self.value_expr_dict)
-            self._value_expr.bind_rel_paths()
-            if not self._value_expr.is_valid:
-                message = (
-                    dedent(
-                        f"""
-                        The computed column {self.name!r} in table {self.tbl.name!r} is no longer valid.
-                        {{validation_error}}
-                        You can continue to query existing data from this column, but evaluating it on new data will raise an error.
-                        """  # noqa: E501
-                    )
-                    .strip()
-                    .format(validation_error=self._value_expr.validation_error)
-                )
-                warnings.warn(message, category=excs.PixeltableWarning, stacklevel=2)
+        assert self.value_expr_dict is None or self._value_expr is not None
         return self._value_expr
 
     def set_value_expr(self, value_expr: exprs.Expr) -> None:
         self._value_expr = value_expr
-        self.value_expr_dict = None
+        self.value_expr_dict = self._value_expr.as_dict()
 
     def check_value_expr(self) -> None:
         assert self._value_expr is not None

pixeltable/catalog/globals.py

@@ -4,12 +4,16 @@ import dataclasses
 import enum
 import itertools
 import logging
-from typing import Optional
-
-from typing_extensions import Self
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Optional
+from uuid import UUID
 
 import pixeltable.exceptions as excs
 
+if TYPE_CHECKING:
+    from IPython.lib.pretty import RepresentationPrinter
+
+
 _logger = logging.getLogger('pixeltable')
 
 # name of the position column in a component view
@@ -21,26 +25,118 @@ _ROWID_COLUMN_NAME = '_rowid'
 _PREDEF_SYMBOLS: Optional[set[str]] = None
 
 
-@dataclasses.dataclass
+@dataclass(frozen=True)
+class QColumnId:
+    """Qualified column id"""
+
+    tbl_id: UUID
+    col_id: int
+
+
+@dataclass(frozen=True)
+class RowCountStats:
+    """
+    Statistics about the counts of rows affected by a table operation.
+    """
+
+    ins_rows: int = 0  # rows inserted
+    del_rows: int = 0  # rows deleted
+    upd_rows: int = 0  # rows updated
+    num_excs: int = 0  # total number of exceptions
+    # TODO: disambiguate what this means: # of slots computed or # of columns computed?
+    computed_values: int = 0  # number of computed values (e.g., computed columns) affected by the operation
+
+    @property
+    def num_rows(self) -> int:
+        return self.ins_rows + self.del_rows + self.upd_rows
+
+    def insert_to_update(self) -> 'RowCountStats':
+        """
+        Convert insert row count stats to update row count stats.
+        This is used when an insert operation is treated as an update.
+        """
+        return RowCountStats(
+            ins_rows=0,
+            del_rows=self.del_rows,
+            upd_rows=self.upd_rows + self.ins_rows,
+            num_excs=self.num_excs,
+            computed_values=self.computed_values,
+        )
+
+    def __add__(self, other: 'RowCountStats') -> 'RowCountStats':
+        """
+        Add the stats from two RowCountStats objects together.
+        """
+        return RowCountStats(
+            ins_rows=self.ins_rows + other.ins_rows,
+            del_rows=self.del_rows + other.del_rows,
+            upd_rows=self.upd_rows + other.upd_rows,
+            num_excs=self.num_excs + other.num_excs,
+            computed_values=self.computed_values + other.computed_values,
+        )
+
+
+@dataclass(frozen=True)
 class UpdateStatus:
     """
     Information about updates that resulted from a table operation.
     """
 
-    num_rows: int = 0
-    # TODO: disambiguate what this means: # of slots computed or # of columns computed?
-    num_computed_values: int = 0
-    num_excs: int = 0
     updated_cols: list[str] = dataclasses.field(default_factory=list)
     cols_with_excs: list[str] = dataclasses.field(default_factory=list)
 
-    def __iadd__(self, other: 'UpdateStatus') -> Self:
-        self.num_rows += other.num_rows
-        self.num_computed_values += other.num_computed_values
-        self.num_excs += other.num_excs
-        self.updated_cols = list(dict.fromkeys(self.updated_cols + other.updated_cols))
-        self.cols_with_excs = list(dict.fromkeys(self.cols_with_excs + other.cols_with_excs))
-        return self
+    # stats for the rows affected by the operation
+    row_count_stats: RowCountStats = field(default_factory=RowCountStats)
+
+    # stats for changes cascaded to other tables
+    cascade_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
+
+    @property
+    def num_rows(self) -> int:
+        return self.row_count_stats.num_rows + self.cascade_row_count_stats.num_rows
+
+    @property
+    def num_excs(self) -> int:
+        return self.row_count_stats.num_excs + self.cascade_row_count_stats.num_excs
+
+    @property
+    def num_computed_values(self) -> int:
+        return self.row_count_stats.computed_values + self.cascade_row_count_stats.computed_values
+
+    def insert_to_update(self) -> 'UpdateStatus':
+        """
+        Convert the update status from an insert operation to an update operation.
+        This is used when an insert operation is treated as an update.
+        """
+        return UpdateStatus(
+            updated_cols=self.updated_cols,
+            cols_with_excs=self.cols_with_excs,
+            row_count_stats=self.row_count_stats.insert_to_update(),
+            cascade_row_count_stats=self.cascade_row_count_stats.insert_to_update(),
+        )
+
+    def to_cascade(self) -> 'UpdateStatus':
+        """
+        Convert the update status to a cascade update status.
+        This is used when an operation cascades changes to other tables.
+        """
+        return UpdateStatus(
+            updated_cols=self.updated_cols,
+            cols_with_excs=self.cols_with_excs,
+            row_count_stats=RowCountStats(),
+            cascade_row_count_stats=self.cascade_row_count_stats + self.row_count_stats,
+        )
+
+    def __add__(self, other: 'UpdateStatus') -> UpdateStatus:
+        """
+        Add the update status from two UpdateStatus objects together.
+        """
+        return UpdateStatus(
+            updated_cols=list(dict.fromkeys(self.updated_cols + other.updated_cols)),
+            cols_with_excs=list(dict.fromkeys(self.cols_with_excs + other.cols_with_excs)),
+            row_count_stats=self.row_count_stats + other.row_count_stats,
+            cascade_row_count_stats=self.cascade_row_count_stats + other.cascade_row_count_stats,
+        )
 
     @property
     def insert_msg(self) -> str:
@@ -58,6 +154,25 @@ class UpdateStatus:
         )
         return msg
 
+    @classmethod
+    def __cnt_str(cls, cnt: int, item: str) -> str:
+        assert cnt > 0
+        return f'{cnt} {item}{"" if cnt == 1 else "s"}'
+
+    def _repr_pretty_(self, p: 'RepresentationPrinter', cycle: bool) -> None:
+        messages = []
+        if self.row_count_stats.ins_rows > 0:
+            messages.append(f'{self.__cnt_str(self.row_count_stats.ins_rows, "row")} inserted')
+        if self.row_count_stats.del_rows > 0:
+            messages.append(f'{self.__cnt_str(self.row_count_stats.del_rows, "row")} deleted')
+        if self.row_count_stats.upd_rows > 0:
+            messages.append(f'{self.__cnt_str(self.row_count_stats.upd_rows, "row")} updated')
+        if self.num_computed_values > 0:
+            messages.append(f'{self.__cnt_str(self.num_computed_values, "value")} computed')
+        if self.row_count_stats.num_excs > 0:
+            messages.append(self.__cnt_str(self.row_count_stats.num_excs, 'exception'))
+        p.text(', '.join(messages) + '.' if len(messages) > 0 else 'No rows affected.')
+
 
 class MediaValidation(enum.Enum):
     ON_READ = 0

pixeltable/catalog/insertable_table.py

@@ -51,6 +51,7 @@ class InsertableTable(Table):
     def __init__(self, dir_id: UUID, tbl_version: TableVersionHandle):
         tbl_version_path = TableVersionPath(tbl_version)
         super().__init__(tbl_version.id, dir_id, tbl_version.get().name, tbl_version_path)
+        self._tbl_version = tbl_version
 
     @classmethod
     def _display_name(cls) -> str:
@@ -103,8 +104,8 @@ class InsertableTable(Table):
         Env.get().console_logger.info(f'Created table `{name}`.')
         return tbl
 
-    def get_metadata(self) -> dict[str, Any]:
-        md = super().get_metadata()
+    def _get_metadata(self) -> dict[str, Any]:
+        md = super()._get_metadata()
         md['is_view'] = False
         md['is_snapshot'] = False
         return md
@@ -141,7 +142,7 @@ class InsertableTable(Table):
         from pixeltable.catalog import Catalog
         from pixeltable.io.table_data_conduit import UnkTableDataConduit
 
-        with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
             table = self
             if source is None:
                 source = [kwargs]
@@ -171,12 +172,13 @@ class InsertableTable(Table):
         from pixeltable.io.table_data_conduit import DFTableDataConduit
 
         status = pxt.UpdateStatus()
-        with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
             if isinstance(data_source, DFTableDataConduit):
-                status += self._tbl_version.get().insert(
+                status = self._tbl_version.get().insert(
                     rows=None, df=data_source.pxt_df, print_stats=print_stats, fail_on_exception=fail_on_exception
                 )
             else:
+                status = UpdateStatus()
                 for row_batch in data_source.valid_row_batch():
                     status += self._tbl_version.get().insert(
                         rows=row_batch, df=None, print_stats=print_stats, fail_on_exception=fail_on_exception
@@ -189,7 +191,7 @@ class InsertableTable(Table):
 
     def _validate_input_rows(self, rows: list[dict[str, Any]]) -> None:
         """Verify that the input rows match the table schema"""
-        valid_col_names = set(self._schema.keys())
+        valid_col_names = set(self._get_schema().keys())
         reqd_col_names = set(self._tbl_version_path.tbl_version.get().get_required_col_names())
         computed_col_names = set(self._tbl_version_path.tbl_version.get().get_computed_col_names())
         for row in rows:
@@ -231,11 +233,10 @@ class InsertableTable(Table):
         """
         from pixeltable.catalog import Catalog
 
-        with Catalog.get().begin_xact(tbl_id=self._id, for_write=True):
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
             return self._tbl_version.get().delete(where=where)
 
-    @property
-    def _base_table(self) -> Optional['Table']:
+    def _get_base_table(self) -> Optional['Table']:
         return None
 
     @property

pixeltable/catalog/schema_object.py

@@ -42,6 +42,12 @@ class SchemaObject:
 
     def get_metadata(self) -> dict[str, Any]:
         """Returns metadata associated with this schema object."""
+        from pixeltable.catalog import Catalog
+
+        with Catalog.get().begin_xact(for_write=False):
+            return self._get_metadata()
+
+    def _get_metadata(self) -> dict[str, Any]:
         return {'name': self._name, 'path': self._path()}
 
     @classmethod