pixeltable 0.3.15__py3-none-any.whl → 0.4.0__py3-none-any.whl

pixeltable/catalog/column.py

@@ -15,7 +15,6 @@ from .globals import MediaValidation, is_valid_identifier
 
 if TYPE_CHECKING:
     from .table_version import TableVersion
-    from .table_version_handle import TableVersionHandle
     from .table_version_path import TableVersionPath
 
 _logger = logging.getLogger('pixeltable')
@@ -43,8 +42,9 @@ class Column:
     sa_errortype_col: Optional[sql.schema.Column]
     _value_expr: Optional[exprs.Expr]
     value_expr_dict: Optional[dict[str, Any]]
-    dependent_cols: set[Column]
-    tbl: Optional[TableVersionHandle]
+    # we store a TableVersion here, not a TableVersionHandle, because this column is owned by that TableVersion instance
+    # (re-resolving it later to a different instance doesn't make sense)
+    tbl: Optional[TableVersion]
 
     def __init__(
         self,
@@ -60,6 +60,7 @@ class Column:
         sa_col_type: Optional[sql.sqltypes.TypeEngine] = None,
         records_errors: Optional[bool] = None,
         value_expr_dict: Optional[dict[str, Any]] = None,
+        tbl: Optional[TableVersion] = None,
     ):
         """Column constructor.
 
@@ -84,6 +85,7 @@ class Column:
         if name is not None and not is_valid_identifier(name):
             raise excs.Error(f"Invalid column name: '{name}'")
         self.name = name
+        self.tbl = tbl
         if col_type is None and computed_with is None:
             raise excs.Error(f'Column `{name}`: col_type is required if computed_with is not specified')
 
@@ -92,6 +94,7 @@ class Column:
         if computed_with is not None:
             value_expr = exprs.Expr.from_object(computed_with)
             if value_expr is None:
+                # TODO: this shouldn't be a user-facing error
                 raise excs.Error(
                     f'Column {name}: computed_with needs to be a valid Pixeltable expression, '
                     f'but it is a {type(computed_with)}'
@@ -99,13 +102,15 @@ class Column:
             else:
                 self._value_expr = value_expr.copy()
                 self.col_type = self._value_expr.col_type
+        if self._value_expr is not None and self.value_expr_dict is None:
+            self.value_expr_dict = self._value_expr.as_dict()
 
         if col_type is not None:
             self.col_type = col_type
         assert self.col_type is not None
 
         self.stored = stored
-        self.dependent_cols = set()  # cols with value_exprs that reference us; set by TableVersion
+        # self.dependent_cols = set()  # cols with value_exprs that reference us; set by TableVersion
         self.id = col_id
         self.is_pk = is_pk
         self._media_validation = media_validation
@@ -122,36 +127,35 @@ class Column:
         self.sa_errormsg_col = None
         self.sa_errortype_col = None
 
-        self.tbl = None  # set by owning TableVersion
+    def init_value_expr(self) -> None:
+        from pixeltable import exprs
+
+        if self._value_expr is not None or self.value_expr_dict is None:
+            return
+        self._value_expr = exprs.Expr.from_dict(self.value_expr_dict)
+        self._value_expr.bind_rel_paths()
+        if not self._value_expr.is_valid:
+            message = (
+                dedent(
+                    f"""
+                    The computed column {self.name!r} in table {self.tbl.name!r} is no longer valid.
+                    {{validation_error}}
+                    You can continue to query existing data from this column, but evaluating it on new data will raise an error.
+                    """  # noqa: E501
+                )
+                .strip()
+                .format(validation_error=self._value_expr.validation_error)
+            )
+            warnings.warn(message, category=excs.PixeltableWarning, stacklevel=2)
 
     @property
     def value_expr(self) -> Optional[exprs.Expr]:
-        """Instantiate value_expr on-demand"""
-        # TODO: instantiate expr in the c'tor and add an Expr.prepare() that can create additional state after the
-        # catalog has been fully loaded; that way, we encounter bugs in the serialization/deserialization logic earlier
-        if self.value_expr_dict is not None and self._value_expr is None:
-            from pixeltable import exprs
-
-            self._value_expr = exprs.Expr.from_dict(self.value_expr_dict)
-            self._value_expr.bind_rel_paths()
-            if not self._value_expr.is_valid:
-                message = (
-                    dedent(
-                        f"""
-                        The computed column {self.name!r} in table {self.tbl.get().name!r} is no longer valid.
-                        {{validation_error}}
-                        You can continue to query existing data from this column, but evaluating it on new data will raise an error.
-                        """  # noqa: E501
-                    )
-                    .strip()
-                    .format(validation_error=self._value_expr.validation_error)
-                )
-                warnings.warn(message, category=excs.PixeltableWarning, stacklevel=2)
+        assert self.value_expr_dict is None or self._value_expr is not None
         return self._value_expr
 
     def set_value_expr(self, value_expr: exprs.Expr) -> None:
         self._value_expr = value_expr
-        self.value_expr_dict = None
+        self.value_expr_dict = self._value_expr.as_dict()
 
     def check_value_expr(self) -> None:
         assert self._value_expr is not None
@@ -175,8 +179,8 @@ class Column:
     #     multiple dependents)
     def get_idx_info(self, reference_tbl: Optional['TableVersionPath'] = None) -> dict[str, 'TableVersion.IndexInfo']:
         assert self.tbl is not None
-        tbl = reference_tbl.tbl_version if reference_tbl is not None else self.tbl
-        return {name: info for name, info in tbl.get().idxs_by_name.items() if info.col == self}
+        tbl = reference_tbl.tbl_version.get() if reference_tbl is not None else self.tbl
+        return {name: info for name, info in tbl.idxs_by_name.items() if info.col == self}
 
     @property
     def is_computed(self) -> bool:
@@ -199,14 +203,14 @@ class Column:
     @property
     def qualified_name(self) -> str:
         assert self.tbl is not None
-        return f'{self.tbl.get().name}.{self.name}'
+        return f'{self.tbl.name}.{self.name}'
 
     @property
     def media_validation(self) -> MediaValidation:
         if self._media_validation is not None:
             return self._media_validation
         assert self.tbl is not None
-        return self.tbl.get().media_validation
+        return self.tbl.media_validation
 
     @property
     def is_required_for_insert(self) -> bool:
@@ -256,7 +260,7 @@ class Column:
         return f'{self.name}: {self.col_type}'
 
     def __repr__(self) -> str:
-        return f'Column({self.id!r}, {self.name!r}, tbl={self.tbl.get().name!r})'
+        return f'Column({self.id!r}, {self.name!r}, tbl={self.tbl.name!r})'
 
     def __hash__(self) -> int:
         # TODO(aaron-siegel): This and __eq__ do not capture the table version. We need to rethink the Column

pixeltable/catalog/dir.py

@@ -38,13 +38,12 @@ class Dir(SchemaObject):
     def _display_name(cls) -> str:
         return 'directory'
 
-    @property
     def _path(self) -> str:
         """Returns the path to this schema object."""
         if self._dir_id is None:
             # we're the root dir
             return ''
-        return super()._path
+        return super()._path()
 
     def _move(self, new_name: str, new_dir_id: UUID) -> None:
         # print(

pixeltable/catalog/globals.py

@@ -5,6 +5,7 @@ import enum
 import itertools
 import logging
 from typing import Optional
+from uuid import UUID
 
 from typing_extensions import Self
 
@@ -21,6 +22,17 @@ _ROWID_COLUMN_NAME = '_rowid'
 _PREDEF_SYMBOLS: Optional[set[str]] = None
 
 
+@dataclasses.dataclass(frozen=True)
+class QColumnId:
+    """Qualified column id"""
+
+    tbl_id: UUID
+    col_id: int
+
+    # def __hash__(self) -> int:
+    #     return hash((self.tbl_id, self.col_id))
+
+
 @dataclasses.dataclass
 class UpdateStatus:
     """

pixeltable/catalog/insertable_table.py

@@ -51,6 +51,7 @@ class InsertableTable(Table):
     def __init__(self, dir_id: UUID, tbl_version: TableVersionHandle):
         tbl_version_path = TableVersionPath(tbl_version)
         super().__init__(tbl_version.id, dir_id, tbl_version.get().name, tbl_version_path)
+        self._tbl_version = tbl_version
 
     @classmethod
     def _display_name(cls) -> str:
@@ -103,8 +104,8 @@ class InsertableTable(Table):
         Env.get().console_logger.info(f'Created table `{name}`.')
         return tbl
 
-    def get_metadata(self) -> dict[str, Any]:
-        md = super().get_metadata()
+    def _get_metadata(self) -> dict[str, Any]:
+        md = super()._get_metadata()
         md['is_view'] = False
         md['is_snapshot'] = False
         return md
@@ -138,37 +139,40 @@ class InsertableTable(Table):
         print_stats: bool = False,
         **kwargs: Any,
     ) -> UpdateStatus:
+        from pixeltable.catalog import Catalog
         from pixeltable.io.table_data_conduit import UnkTableDataConduit
 
-        table = self
-        if source is None:
-            source = [kwargs]
-            kwargs = None
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
+            table = self
+            if source is None:
+                source = [kwargs]
+                kwargs = None
 
-        tds = UnkTableDataConduit(
-            source, source_format=source_format, src_schema_overrides=schema_overrides, extra_fields=kwargs
-        )
-        data_source = tds.specialize()
-        if data_source.source_column_map is None:
-            data_source.src_pk = []
+            tds = UnkTableDataConduit(
+                source, source_format=source_format, src_schema_overrides=schema_overrides, extra_fields=kwargs
+            )
+            data_source = tds.specialize()
+            if data_source.source_column_map is None:
+                data_source.src_pk = []
 
-        assert isinstance(table, Table)
-        data_source.add_table_info(table)
-        data_source.prepare_for_insert_into_table()
+            assert isinstance(table, Table)
+            data_source.add_table_info(table)
+            data_source.prepare_for_insert_into_table()
 
-        fail_on_exception = OnErrorParameter.fail_on_exception(on_error)
-        return table.insert_table_data_source(
-            data_source=data_source, fail_on_exception=fail_on_exception, print_stats=print_stats
-        )
+            fail_on_exception = OnErrorParameter.fail_on_exception(on_error)
+            return table.insert_table_data_source(
+                data_source=data_source, fail_on_exception=fail_on_exception, print_stats=print_stats
+            )
 
     def insert_table_data_source(
         self, data_source: TableDataConduit, fail_on_exception: bool, print_stats: bool = False
     ) -> pxt.UpdateStatus:
         """Insert row batches into this table from a `TableDataConduit`."""
+        from pixeltable.catalog import Catalog
         from pixeltable.io.table_data_conduit import DFTableDataConduit
 
         status = pxt.UpdateStatus()
-        with Env.get().begin_xact():
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
             if isinstance(data_source, DFTableDataConduit):
                 status += self._tbl_version.get().insert(
                     rows=None, df=data_source.pxt_df, print_stats=print_stats, fail_on_exception=fail_on_exception
@@ -186,7 +190,7 @@ class InsertableTable(Table):
 
     def _validate_input_rows(self, rows: list[dict[str, Any]]) -> None:
         """Verify that the input rows match the table schema"""
-        valid_col_names = set(self._schema.keys())
+        valid_col_names = set(self._get_schema().keys())
         reqd_col_names = set(self._tbl_version_path.tbl_version.get().get_required_col_names())
         computed_col_names = set(self._tbl_version_path.tbl_version.get().get_computed_col_names())
         for row in rows:
@@ -226,11 +230,12 @@ class InsertableTable(Table):
 
             >>> tbl.delete(tbl.a > 5)
         """
-        with Env.get().begin_xact():
+        from pixeltable.catalog import Catalog
+
+        with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
             return self._tbl_version.get().delete(where=where)
 
-    @property
-    def _base_table(self) -> Optional['Table']:
+    def _get_base_table(self) -> Optional['Table']:
         return None
 
     @property
@@ -238,4 +243,4 @@ class InsertableTable(Table):
         return []
 
     def _table_descriptor(self) -> str:
-        return f'Table {self._path!r}'
+        return f'Table {self._path()!r}'

pixeltable/catalog/schema_object.py

@@ -2,8 +2,6 @@ from abc import abstractmethod
 from typing import TYPE_CHECKING, Any, Optional
 from uuid import UUID
 
-from pixeltable.env import Env
-
 if TYPE_CHECKING:
     from pixeltable import catalog
 
@@ -28,24 +26,29 @@ class SchemaObject:
         """Returns the parent directory of this schema object."""
         from .catalog import Catalog
 
-        with Env.get().begin_xact():
+        with Catalog.get().begin_xact(for_write=False):
             if self._dir_id is None:
                 return None
             return Catalog.get().get_dir(self._dir_id)
 
-    @property
     def _path(self) -> str:
         """Returns the path to this schema object."""
         from .catalog import Catalog
 
         assert self._dir_id is not None
-        with Env.get().begin_xact():
+        with Catalog.get().begin_xact(for_write=False):
             path = Catalog.get().get_dir_path(self._dir_id)
             return str(path.append(self._name))
 
     def get_metadata(self) -> dict[str, Any]:
         """Returns metadata associated with this schema object."""
-        return {'name': self._name, 'path': self._path}
+        from pixeltable.catalog import Catalog
+
+        with Catalog.get().begin_xact(for_write=False):
+            return self._get_metadata()
+
+    def _get_metadata(self) -> dict[str, Any]:
+        return {'name': self._name, 'path': self._path()}
 
     @classmethod
     @abstractmethod