pixeltable 0.4.2__py3-none-any.whl → 0.4.4__py3-none-any.whl

pixeltable/catalog/column.py

@@ -36,11 +36,10 @@ class Column:
     _media_validation: Optional[MediaValidation]  # if not set, TableVersion.media_validation applies
     schema_version_add: Optional[int]
     schema_version_drop: Optional[int]
-    _records_errors: Optional[bool]
+    _stores_cellmd: Optional[bool]
     sa_col: Optional[sql.schema.Column]
     sa_col_type: Optional[sql.sqltypes.TypeEngine]
-    sa_errormsg_col: Optional[sql.schema.Column]
-    sa_errortype_col: Optional[sql.schema.Column]
+    sa_cellmd_col: Optional[sql.schema.Column]  # JSON metadata for the cell, e.g. errortype, errormsg for media columns
     _value_expr: Optional[exprs.Expr]
     value_expr_dict: Optional[dict[str, Any]]
     # we store a TableVersion here, not a TableVersionHandle, because this column is owned by that TableVersion instance
@@ -59,7 +58,7 @@ class Column:
         schema_version_add: Optional[int] = None,
         schema_version_drop: Optional[int] = None,
         sa_col_type: Optional[sql.sqltypes.TypeEngine] = None,
-        records_errors: Optional[bool] = None,
+        stores_cellmd: Optional[bool] = None,
         value_expr_dict: Optional[dict[str, Any]] = None,
         tbl: Optional[TableVersion] = None,
     ):
@@ -118,15 +117,14 @@ class Column:
         self.schema_version_add = schema_version_add
         self.schema_version_drop = schema_version_drop
 
-        self._records_errors = records_errors
+        self._stores_cellmd = stores_cellmd
 
         # column in the stored table for the values of this Column
         self.sa_col = None
         self.sa_col_type = sa_col_type
 
         # computed cols also have storage columns for the exception string and type
-        self.sa_errormsg_col = None
-        self.sa_errortype_col = None
+        self.sa_cellmd_col = None
 
     def init_value_expr(self) -> None:
         from pixeltable import exprs
@@ -203,11 +201,11 @@ class Column:
         return self.stored
 
     @property
-    def records_errors(self) -> bool:
+    def stores_cellmd(self) -> bool:
         """True if this column also stores error information."""
         # default: record errors for computed and media columns
-        if self._records_errors is not None:
-            return self._records_errors
+        if self._stores_cellmd is not None:
+            return self._stores_cellmd
         return self.is_stored and (self.is_computed or self.col_type.is_media_type())
 
     @property
@@ -243,28 +241,29 @@ class Column:
         """
         assert self.is_stored
         # all storage columns are nullable (we deal with null errors in Pixeltable directly)
-        self.sa_col = sql.Column(
-            self.store_name(),
-            self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type,
-            nullable=True,
-        )
-        if self.is_computed or self.col_type.is_media_type():
-            self.sa_errormsg_col = sql.Column(self.errormsg_store_name(), ts.StringType().to_sa_type(), nullable=True)
-            self.sa_errortype_col = sql.Column(self.errortype_store_name(), ts.StringType().to_sa_type(), nullable=True)
+        self.sa_col = sql.Column(self.store_name(), self.get_sa_col_type(), nullable=True)
+        if self.stores_cellmd:
+            # JSON metadata for the cell, e.g. errortype, errormsg for media columns
+            self.sa_cellmd_col = sql.Column(self.cellmd_store_name(), self.sa_cellmd_type(), nullable=True)
 
     def get_sa_col_type(self) -> sql.sqltypes.TypeEngine:
         return self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type
 
+    @classmethod
+    def cellmd_type(cls) -> ts.ColumnType:
+        return ts.JsonType(nullable=True)
+
+    @classmethod
+    def sa_cellmd_type(cls) -> sql.sqltypes.TypeEngine:
+        return cls.cellmd_type().to_sa_type()
+
     def store_name(self) -> str:
         assert self.id is not None
         assert self.is_stored
         return f'col_{self.id}'
 
-    def errormsg_store_name(self) -> str:
-        return f'{self.store_name()}_errormsg'
-
-    def errortype_store_name(self) -> str:
-        return f'{self.store_name()}_errortype'
+    def cellmd_store_name(self) -> str:
+        return f'{self.store_name()}_cellmd'
 
     def __str__(self) -> str:
         return f'{self.name}: {self.col_type}'

pixeltable/catalog/dir.py

@@ -34,8 +34,7 @@ class Dir(SchemaObject):
         dir = cls(dir_record.id, parent_id, name)
         return dir
 
-    @classmethod
-    def _display_name(cls) -> str:
+    def _display_name(self) -> str:
         return 'directory'
 
     def _path(self) -> str:

pixeltable/catalog/globals.py

@@ -1,19 +1,14 @@
 from __future__ import annotations
 
-import dataclasses
 import enum
 import itertools
 import logging
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Optional
+from dataclasses import dataclass
+from typing import Optional
 from uuid import UUID
 
 import pixeltable.exceptions as excs
 
-if TYPE_CHECKING:
-    from IPython.lib.pretty import RepresentationPrinter
-
-
 _logger = logging.getLogger('pixeltable')
 
 # name of the position column in a component view
@@ -33,147 +28,6 @@ class QColumnId:
     col_id: int
 
 
-@dataclass(frozen=True)
-class RowCountStats:
-    """
-    Statistics about the counts of rows affected by a table operation.
-    """
-
-    ins_rows: int = 0  # rows inserted
-    del_rows: int = 0  # rows deleted
-    upd_rows: int = 0  # rows updated
-    num_excs: int = 0  # total number of exceptions
-    # TODO: disambiguate what this means: # of slots computed or # of columns computed?
-    computed_values: int = 0  # number of computed values (e.g., computed columns) affected by the operation
-
-    @property
-    def num_rows(self) -> int:
-        return self.ins_rows + self.del_rows + self.upd_rows
-
-    def insert_to_update(self) -> 'RowCountStats':
-        """
-        Convert insert row count stats to update row count stats.
-        This is used when an insert operation is treated as an update.
-        """
-        return RowCountStats(
-            ins_rows=0,
-            del_rows=self.del_rows,
-            upd_rows=self.upd_rows + self.ins_rows,
-            num_excs=self.num_excs,
-            computed_values=self.computed_values,
-        )
-
-    def __add__(self, other: 'RowCountStats') -> 'RowCountStats':
-        """
-        Add the stats from two RowCountStats objects together.
-        """
-        return RowCountStats(
-            ins_rows=self.ins_rows + other.ins_rows,
-            del_rows=self.del_rows + other.del_rows,
-            upd_rows=self.upd_rows + other.upd_rows,
-            num_excs=self.num_excs + other.num_excs,
-            computed_values=self.computed_values + other.computed_values,
-        )
-
-
-@dataclass(frozen=True)
-class UpdateStatus:
-    """
-    Information about updates that resulted from a table operation.
-    """
-
-    updated_cols: list[str] = dataclasses.field(default_factory=list)
-    cols_with_excs: list[str] = dataclasses.field(default_factory=list)
-
-    # stats for the rows affected by the operation
-    row_count_stats: RowCountStats = field(default_factory=RowCountStats)
-
-    # stats for changes cascaded to other tables
-    cascade_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
-
-    @property
-    def num_rows(self) -> int:
-        return self.row_count_stats.num_rows + self.cascade_row_count_stats.num_rows
-
-    @property
-    def num_excs(self) -> int:
-        return self.row_count_stats.num_excs + self.cascade_row_count_stats.num_excs
-
-    @property
-    def num_computed_values(self) -> int:
-        return self.row_count_stats.computed_values + self.cascade_row_count_stats.computed_values
-
-    def insert_to_update(self) -> 'UpdateStatus':
-        """
-        Convert the update status from an insert operation to an update operation.
-        This is used when an insert operation is treated as an update.
-        """
-        return UpdateStatus(
-            updated_cols=self.updated_cols,
-            cols_with_excs=self.cols_with_excs,
-            row_count_stats=self.row_count_stats.insert_to_update(),
-            cascade_row_count_stats=self.cascade_row_count_stats.insert_to_update(),
-        )
-
-    def to_cascade(self) -> 'UpdateStatus':
-        """
-        Convert the update status to a cascade update status.
-        This is used when an operation cascades changes to other tables.
-        """
-        return UpdateStatus(
-            updated_cols=self.updated_cols,
-            cols_with_excs=self.cols_with_excs,
-            row_count_stats=RowCountStats(),
-            cascade_row_count_stats=self.cascade_row_count_stats + self.row_count_stats,
-        )
-
-    def __add__(self, other: 'UpdateStatus') -> UpdateStatus:
-        """
-        Add the update status from two UpdateStatus objects together.
-        """
-        return UpdateStatus(
-            updated_cols=list(dict.fromkeys(self.updated_cols + other.updated_cols)),
-            cols_with_excs=list(dict.fromkeys(self.cols_with_excs + other.cols_with_excs)),
-            row_count_stats=self.row_count_stats + other.row_count_stats,
-            cascade_row_count_stats=self.cascade_row_count_stats + other.cascade_row_count_stats,
-        )
-
-    @property
-    def insert_msg(self) -> str:
-        """Return a message describing the results of an insert operation."""
-        if self.num_excs == 0:
-            cols_with_excs_str = ''
-        else:
-            cols_with_excs_str = (
-                f' across {len(self.cols_with_excs)} column{"" if len(self.cols_with_excs) == 1 else "s"}'
-            )
-            cols_with_excs_str += f' ({", ".join(self.cols_with_excs)})'
-        msg = (
-            f'Inserted {self.num_rows} row{"" if self.num_rows == 1 else "s"} '
-            f'with {self.num_excs} error{"" if self.num_excs == 1 else "s"}{cols_with_excs_str}.'
-        )
-        return msg
-
-    @classmethod
-    def __cnt_str(cls, cnt: int, item: str) -> str:
-        assert cnt > 0
-        return f'{cnt} {item}{"" if cnt == 1 else "s"}'
-
-    def _repr_pretty_(self, p: 'RepresentationPrinter', cycle: bool) -> None:
-        messages = []
-        if self.row_count_stats.ins_rows > 0:
-            messages.append(f'{self.__cnt_str(self.row_count_stats.ins_rows, "row")} inserted')
-        if self.row_count_stats.del_rows > 0:
-            messages.append(f'{self.__cnt_str(self.row_count_stats.del_rows, "row")} deleted')
-        if self.row_count_stats.upd_rows > 0:
-            messages.append(f'{self.__cnt_str(self.row_count_stats.upd_rows, "row")} updated')
-        if self.num_computed_values > 0:
-            messages.append(f'{self.__cnt_str(self.num_computed_values, "value")} computed')
-        if self.row_count_stats.num_excs > 0:
-            messages.append(self.__cnt_str(self.row_count_stats.num_excs, 'exception'))
-        p.text(', '.join(messages) + '.' if len(messages) > 0 else 'No rows affected.')
-
-
 class MediaValidation(enum.Enum):
     ON_READ = 0
     ON_WRITE = 1

pixeltable/catalog/insertable_table.py

@@ -10,11 +10,12 @@ from pixeltable import exceptions as excs, type_system as ts
 from pixeltable.env import Env
 from pixeltable.utils.filecache import FileCache
 
-from .globals import MediaValidation, UpdateStatus
+from .globals import MediaValidation
 from .table import Table
 from .table_version import TableVersion
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
+from .update_status import UpdateStatus
 
 if TYPE_CHECKING:
     from pixeltable import exprs
@@ -53,8 +54,8 @@ class InsertableTable(Table):
         super().__init__(tbl_version.id, dir_id, tbl_version.get().name, tbl_version_path)
         self._tbl_version = tbl_version
 
-    @classmethod
-    def _display_name(cls) -> str:
+    def _display_name(self) -> str:
+        assert not self._tbl_version_path.is_replica()
         return 'table'
 
     @classmethod
@@ -74,10 +75,10 @@ class InsertableTable(Table):
         column_names = [col.name for col in columns]
         for pk_col in primary_key:
             if pk_col not in column_names:
-                raise excs.Error(f'Primary key column {pk_col} not found in table schema')
+                raise excs.Error(f'Primary key column {pk_col!r} not found in table schema.')
             col = columns[column_names.index(pk_col)]
             if col.col_type.nullable:
-                raise excs.Error(f'Primary key column {pk_col} cannot be nullable')
+                raise excs.Error(f'Primary key column {pk_col!r} cannot be nullable.')
             col.is_pk = True
 
         _, tbl_version = TableVersion.create(
@@ -100,12 +101,13 @@ class InsertableTable(Table):
             tbl_version.insert(None, df, fail_on_exception=True)
         session.commit()
 
-        _logger.info(f'Created table `{name}`, id={tbl_version.id}')
-        Env.get().console_logger.info(f'Created table `{name}`.')
+        _logger.info(f'Created table {name!r}, id={tbl_version.id}')
+        Env.get().console_logger.info(f'Created table {name!r}.')
         return tbl
 
     def _get_metadata(self) -> dict[str, Any]:
         md = super()._get_metadata()
+        md['base'] = None
         md['is_view'] = False
         md['is_snapshot'] = False
         return md
@@ -171,14 +173,14 @@ class InsertableTable(Table):
         from pixeltable.catalog import Catalog
         from pixeltable.io.table_data_conduit import DFTableDataConduit
 
-        status = pxt.UpdateStatus()
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
             if isinstance(data_source, DFTableDataConduit):
-                status = self._tbl_version.get().insert(
+                status = pxt.UpdateStatus()
+                status += self._tbl_version.get().insert(
                     rows=None, df=data_source.pxt_df, print_stats=print_stats, fail_on_exception=fail_on_exception
                 )
             else:
-                status = UpdateStatus()
+                status = pxt.UpdateStatus()
                 for row_batch in data_source.valid_row_batch():
                     status += self._tbl_version.get().insert(
                         rows=row_batch, df=None, print_stats=print_stats, fail_on_exception=fail_on_exception
@@ -202,9 +204,9 @@ class InsertableTable(Table):
 
             for col_name, val in row.items():
                 if col_name not in valid_col_names:
-                    raise excs.Error(f'Unknown column name {col_name} in row {row}')
+                    raise excs.Error(f'Unknown column name {col_name!r} in row {row}')
                 if col_name in computed_col_names:
-                    raise excs.Error(f'Value for computed column {col_name} in row {row}')
+                    raise excs.Error(f'Value for computed column {col_name!r} in row {row}')
 
                 # validate data
                 col = self._tbl_version_path.get_column(col_name)
@@ -244,4 +246,4 @@ class InsertableTable(Table):
         return []
 
     def _table_descriptor(self) -> str:
-        return f'Table {self._path()!r}'
+        return self._display_str()

pixeltable/catalog/path.py

@@ -77,5 +77,11 @@ class Path:
     def __str__(self) -> str:
         return '.'.join(self.components)
 
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, Path) and str(self) == str(other)
+
+    def __hash__(self) -> int:
+        return hash(str(self))
+
     def __lt__(self, other: Path) -> bool:
         return str(self) < str(other)

pixeltable/catalog/schema_object.py

@@ -42,22 +42,27 @@ class SchemaObject:
 
     def get_metadata(self) -> dict[str, Any]:
         """Returns metadata associated with this schema object."""
-        from pixeltable.catalog import Catalog
+        from pixeltable.catalog import retry_loop
 
-        with Catalog.get().begin_xact(for_write=False):
+        @retry_loop(for_write=False)
+        def op() -> dict[str, Any]:
             return self._get_metadata()
 
+        return op()
+
     def _get_metadata(self) -> dict[str, Any]:
         return {'name': self._name, 'path': self._path()}
 
-    @classmethod
     @abstractmethod
-    def _display_name(cls) -> str:
+    def _display_name(self) -> str:
         """
         Return name displayed in error messages.
         """
         pass
 
+    def _display_str(self) -> str:
+        return f'{self._display_name()} {self._path()!r}'
+
     def _move(self, new_name: str, new_dir_id: UUID) -> None:
         """Subclasses need to override this to make the change persistent"""
         self._name = new_name