pixeltable 0.3.9__py3-none-any.whl → 0.3.11__py3-none-any.whl

pixeltable/catalog/table_version.py

@@ -13,9 +13,8 @@ import sqlalchemy as sql
 
 import pixeltable as pxt
 import pixeltable.exceptions as excs
-import pixeltable.exprs as exprs
-import pixeltable.index as index
 import pixeltable.type_system as ts
+from pixeltable import exprs, index
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 from pixeltable.metadata import schema
@@ -54,6 +53,7 @@ class TableVersion:
 
     id: UUID
     name: str
+    user: Optional[str]
     effective_version: Optional[int]
     version: int
     comment: str
@@ -108,6 +108,7 @@ class TableVersion:
     ):
         self.id = id
         self.name = tbl_md.name
+        self.user = tbl_md.user
         self.effective_version = effective_version
         self.version = tbl_md.current_version if effective_version is None else effective_version
         self.comment = schema_version_md.comment
@@ -211,6 +212,7 @@ class TableVersion:
         view_md: Optional[schema.ViewMd] = None,
     ) -> tuple[UUID, Optional[TableVersion]]:
         session = Env.get().session
+        user = Env.get().user
 
         # assign ids
         cols_by_name: dict[str, Column] = {}
@@ -225,9 +227,11 @@ class TableVersion:
         # create schema.Table
         # Column.dependent_cols for existing cols is wrong at this point, but init() will set it correctly
         column_md = cls._create_column_md(cols)
+        tbl_id = uuid.uuid4()
         table_md = schema.TableMd(
+            tbl_id=str(tbl_id),
             name=name,
-            user=None,
+            user=user,
             current_version=0,
             current_schema_version=0,
             next_col_id=len(cols),
@@ -241,11 +245,12 @@ class TableVersion:
         )
         # create a schema.Table here, we need it to call our c'tor;
         # don't add it to the session yet, we might add index metadata
-        tbl_id = uuid.uuid4()
         tbl_record = schema.Table(id=tbl_id, dir_id=dir_id, md=dataclasses.asdict(table_md))
 
         # create schema.TableVersion
-        table_version_md = schema.TableVersionMd(created_at=timestamp, version=0, schema_version=0, additional_md={})
+        table_version_md = schema.TableVersionMd(
+            tbl_id=str(tbl_record.id), created_at=timestamp, version=0, schema_version=0, additional_md={}
+        )
         tbl_version_record = schema.TableVersion(
             tbl_id=tbl_record.id, version=0, md=dataclasses.asdict(table_version_md)
         )
@@ -261,6 +266,7 @@ class TableVersion:
             schema_col_md[col.id] = md
 
         schema_version_md = schema.TableSchemaVersionMd(
+            tbl_id=str(tbl_record.id),
             schema_version=0,
             preceding_schema_version=None,
             columns=schema_col_md,
@@ -337,8 +343,11 @@ class TableVersion:
         self.cols = []
         self.cols_by_name = {}
         self.cols_by_id = {}
-        for col_md in tbl_md.column_md.values():
-            schema_col_md = schema_version_md.columns[col_md.id] if col_md.id in schema_version_md.columns else None
+        # Sort columns in column_md by the position specified in col_md.id to guarantee that all references
+        # point backward.
+        sorted_column_md = sorted(tbl_md.column_md.values(), key=lambda item: item.id)
+        for col_md in sorted_column_md:
+            schema_col_md = schema_version_md.columns.get(col_md.id)
             col_name = schema_col_md.name if schema_col_md is not None else None
             media_val = (
                 MediaValidation[schema_col_md.media_validation.upper()]
@@ -381,10 +390,8 @@ class TableVersion:
         import pixeltable.index as index_module
 
         for md in tbl_md.index_md.values():
-            if (
-                md.schema_version_add > self.schema_version
-                or md.schema_version_drop is not None
-                and md.schema_version_drop <= self.schema_version
+            if md.schema_version_add > self.schema_version or (
+                md.schema_version_drop is not None and md.schema_version_drop <= self.schema_version
             ):
                 # index not visible in this schema version
                 continue
@@ -615,11 +622,10 @@ class TableVersion:
         cols = list(cols)
         row_count = self.store_tbl.count()
         for col in cols:
-            if not col.col_type.nullable and not col.is_computed:
-                if row_count > 0:
-                    raise excs.Error(
-                        f'Cannot add non-nullable column "{col.name}" to table {self.name} with existing rows'
-                    )
+            if not col.col_type.nullable and not col.is_computed and row_count > 0:
+                raise excs.Error(
+                    f'Cannot add non-nullable column {col.name!r} to table {self.name!r} with existing rows'
+                )
 
         num_excs = 0
         cols_with_excs: list[Column] = []
@@ -658,13 +664,13 @@ class TableVersion:
                     cols_with_excs.append(col)
             except excs.Error as exc:
                 self.cols.pop()
-                for col in cols:
+                for c in cols:
                     # remove columns that we already added
-                    if col.id not in self.cols_by_id:
+                    if c.id not in self.cols_by_id:
                         continue
-                    if col.name is not None:
-                        del self.cols_by_name[col.name]
-                    del self.cols_by_id[col.id]
+                    if c.name is not None:
+                        del self.cols_by_name[c.name]
+                    del self.cols_by_id[c.id]
                 # we need to re-initialize the sqlalchemy schema
                 self.store_tbl.create_sa_tbl()
                 raise exc
@@ -752,19 +758,20 @@ class TableVersion:
         self._update_md(time.time(), preceding_schema_version=preceding_schema_version)
         _logger.info(f'Renamed column {old_name} to {new_name} in table {self.name}, new version: {self.version}')
 
-    def set_comment(self, new_comment: Optional[str]):
+    def set_comment(self, new_comment: Optional[str]) -> None:
         _logger.info(f'[{self.name}] Updating comment: {new_comment}')
         self.comment = new_comment
         self._create_schema_version()
 
-    def set_num_retained_versions(self, new_num_retained_versions: int):
+    def set_num_retained_versions(self, new_num_retained_versions: int) -> None:
         _logger.info(
-            f'[{self.name}] Updating num_retained_versions: {new_num_retained_versions} (was {self.num_retained_versions})'
+            f'[{self.name}] Updating num_retained_versions: {new_num_retained_versions} '
+            f'(was {self.num_retained_versions})'
         )
         self.num_retained_versions = new_num_retained_versions
         self._create_schema_version()
 
-    def _create_schema_version(self):
+    def _create_schema_version(self) -> None:
         # we're creating a new schema version
         self.version += 1
         preceding_schema_version = self.schema_version
@@ -854,7 +861,7 @@ class TableVersion:
 
         from pixeltable.plan import Planner
 
-        update_spec = self._validate_update_spec(value_spec, allow_pk=False, allow_exprs=True)
+        update_spec = self._validate_update_spec(value_spec, allow_pk=False, allow_exprs=True, allow_media=True)
         if where is not None:
             if not isinstance(where, exprs.Expr):
                 raise excs.Error(f"'where' argument must be a predicate, got {type(where)}")
@@ -893,7 +900,6 @@ class TableVersion:
         """
         # if we do lookups of rowids, we must have one for each row in the batch
         assert len(rowids) == 0 or len(rowids) == len(batch)
-        cols_with_excs: set[str] = set()
 
         from pixeltable.plan import Planner
 
@@ -915,7 +921,7 @@ class TableVersion:
         return result
 
     def _validate_update_spec(
-        self, value_spec: dict[str, Any], allow_pk: bool, allow_exprs: bool
+        self, value_spec: dict[str, Any], allow_pk: bool, allow_exprs: bool, allow_media: bool
     ) -> dict[Column, exprs.Expr]:
         update_targets: dict[Column, exprs.Expr] = {}
         for col_name, val in value_spec.items():
@@ -935,27 +941,31 @@ class TableVersion:
                 raise excs.Error(f'Column {col_name} is computed and cannot be updated')
             if col.is_pk and not allow_pk:
                 raise excs.Error(f'Column {col_name} is a primary key column and cannot be updated')
+            if col.col_type.is_media_type() and not allow_media:
+                raise excs.Error(f'Column {col_name} is a media column and cannot be updated')
 
             # make sure that the value is compatible with the column type
             value_expr: exprs.Expr
             try:
                 # check if this is a literal
                 value_expr = exprs.Literal(val, col_type=col.col_type)
-            except (TypeError, jsonschema.exceptions.ValidationError):
+            except (TypeError, jsonschema.exceptions.ValidationError) as exc:
                 if not allow_exprs:
                     raise excs.Error(
                         f'Column {col_name}: value {val!r} is not a valid literal for this column '
                         f'(expected {col.col_type})'
-                    )
+                    ) from exc
                 # it's not a literal, let's try to create an expr from it
                 value_expr = exprs.Expr.from_object(val)
                 if value_expr is None:
-                    raise excs.Error(f'Column {col_name}: value {val!r} is not a recognized literal or expression')
+                    raise excs.Error(
+                        f'Column {col_name}: value {val!r} is not a recognized literal or expression'
+                    ) from exc
                 if not col.col_type.is_supertype_of(value_expr.col_type, ignore_nullable=True):
                     raise excs.Error(
                         f'Type of value {val!r} ({value_expr.col_type}) is not compatible with the type of column '
                         f'{col_name} ({col.col_type})'
-                    )
+                    ) from exc
             update_targets[col] = value_expr
 
         return update_targets
@@ -984,7 +994,7 @@ class TableVersion:
             self._update_md(timestamp)
 
         if cascade:
-            base_versions = [None if plan is None else self.version] + base_versions  # don't update in place
+            base_versions = [None if plan is None else self.version, *base_versions]  # don't update in place
             # propagate to views
             for view in self.mutable_views:
                 recomputed_cols = [col for col in recomputed_view_cols if col.tbl == view]
@@ -1044,11 +1054,9 @@ class TableVersion:
             # we're creating a new version
             self.version += 1
             self._update_md(timestamp)
-        else:
-            pass
         for view in self.mutable_views:
             num_rows += view.get().propagate_delete(
-                where=None, base_versions=[self.version] + base_versions, timestamp=timestamp
+                where=None, base_versions=[self.version, *base_versions], timestamp=timestamp
             )
         return num_rows
 
@@ -1228,9 +1236,7 @@ class TableVersion:
 
     def is_system_column(self, col: Column) -> bool:
         """Return True if column was created by Pixeltable"""
-        if col.name == _POS_COLUMN_NAME and self.is_component_view:
-            return True
-        return False
+        return col.name == _POS_COLUMN_NAME and self.is_component_view
 
     def user_columns(self) -> list[Column]:
         """Return all non-system columns"""
@@ -1240,6 +1246,11 @@ class TableVersion:
         """Return all non-system columns"""
         return [c for c in self.cols if c.is_pk]
 
+    @property
+    def primary_key(self) -> list[str]:
+        """Return the names of the primary key columns"""
+        return [c.name for c in self.cols if c.is_pk]
+
     def get_required_col_names(self) -> list[str]:
         """Return the names of all columns for which values must be specified in insert()"""
         assert not self.is_view
@@ -1253,7 +1264,7 @@ class TableVersion:
 
     def _record_refd_columns(self, col: Column) -> None:
         """Update Column.dependent_cols for all cols referenced in col.value_expr."""
-        import pixeltable.exprs as exprs
+        from pixeltable import exprs
 
         if col.value_expr_dict is not None:
             # if we have a value_expr_dict, use that instead of instantiating the value_expr
@@ -1306,8 +1317,9 @@ class TableVersion:
 
     def _create_tbl_md(self) -> schema.TableMd:
         return schema.TableMd(
+            tbl_id=str(self.id),
             name=self.name,
-            user=None,
+            user=self.user,
             current_version=self.version,
             current_schema_version=self.schema_version,
             next_col_id=self.next_col_id,
@@ -1322,7 +1334,11 @@ class TableVersion:
 
     def _create_version_md(self, timestamp: float) -> schema.TableVersionMd:
         return schema.TableVersionMd(
-            created_at=timestamp, version=self.version, schema_version=self.schema_version, additional_md={}
+            tbl_id=str(self.id),
+            created_at=timestamp,
+            version=self.version,
+            schema_version=self.schema_version,
+            additional_md={},
         )
 
     def _create_schema_version_md(self, preceding_schema_version: int) -> schema.TableSchemaVersionMd:
@@ -1335,6 +1351,7 @@ class TableVersion:
             )
         # preceding_schema_version to be set by the caller
         return schema.TableSchemaVersionMd(
+            tbl_id=str(self.id),
             schema_version=self.schema_version,
             preceding_schema_version=preceding_schema_version,
             columns=column_md,
@@ -1349,7 +1366,7 @@ class TableVersion:
 
     @classmethod
     def from_dict(cls, d: dict) -> TableVersion:
-        import pixeltable.catalog as catalog
+        from pixeltable import catalog
 
         id = UUID(d['id'])
         effective_version = d['effective_version']

pixeltable/catalog/table_version_handle.py

@@ -31,6 +31,9 @@ class TableVersionHandle:
             return False
         return self.id == other.id and self.effective_version == other.effective_version
 
+    def __hash__(self) -> int:
+        return hash((self.id, self.effective_version))
+
     @classmethod
     def create(cls, tbl_version: TableVersion) -> TableVersionHandle:
         return cls(tbl_version.id, tbl_version.effective_version, tbl_version)

pixeltable/catalog/table_version_path.py

@@ -82,7 +82,7 @@ class TableVersionPath:
         """Return all tbl versions"""
         if self.base is None:
             return [self.tbl_version]
-        return [self.tbl_version] + self.base.get_tbl_versions()
+        return [self.tbl_version, *self.base.get_tbl_versions()]
 
     def get_bases(self) -> list[TableVersionHandle]:
         """Return all tbl versions"""

pixeltable/catalog/view.py

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 import inspect
 import logging
-from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional
+from typing import TYPE_CHECKING, Any, List, Literal, Optional
 from uuid import UUID
 
 import pixeltable.exceptions as excs
 import pixeltable.metadata.schema as md_schema
 import pixeltable.type_system as ts
-from pixeltable import catalog, exprs, func
+from pixeltable import exprs, func
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 
@@ -98,7 +98,8 @@ class View(Table):
             # make sure that the value can be computed in the context of the base
             if col.value_expr is not None and not col.value_expr.is_bound_by([base]):
                 raise excs.Error(
-                    f'Column {col.name}: value expression cannot be computed in the context of the base {base.tbl_name()}'
+                    f'Column {col.name}: value expression cannot be computed in the context of the '
+                    f'base {base.tbl_name()}'
                 )
 
         if iterator_cls is not None:
@@ -111,8 +112,8 @@ class View(Table):
             bound_args: dict[str, Any]
             try:
                 bound_args = py_signature.bind(None, **iterator_args).arguments  # None: arg for self
-            except TypeError as e:
-                raise excs.Error(f'Invalid iterator arguments: {e}')
+            except TypeError as exc:
+                raise excs.Error(f'Invalid iterator arguments: {exc}') from exc
             # we ignore 'self'
             first_param_name = next(iter(py_signature.parameters))  # can't guarantee it's actually 'self'
             del bound_args[first_param_name]
@@ -203,8 +204,8 @@ class View(Table):
 
             from pixeltable.plan import Planner
 
-            plan, num_values_per_row = Planner.create_view_load_plan(view._tbl_version_path)
-            num_rows, num_excs, cols_with_excs = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
+            plan, _ = Planner.create_view_load_plan(view._tbl_version_path)
+            num_rows, num_excs, _ = tbl_version.store_tbl.insert_rows(plan, v_min=tbl_version.version)
             Env.get().console_logger.info(f'Created view `{name}` with {num_rows} rows, {num_excs} exceptions.')
 
         session.commit()
@@ -251,13 +252,20 @@ class View(Table):
         md['is_snapshot'] = self._tbl_version_path.is_snapshot()
         return md
 
+    if TYPE_CHECKING:
+        import datasets  # type: ignore[import-untyped]
+
+        from pixeltable.globals import RowData, TableDataSource
+
     def insert(
         self,
-        rows: Optional[Iterable[dict[str, Any]]] = None,
+        source: Optional[TableDataSource] = None,
         /,
         *,
-        print_stats: bool = False,
+        source_format: Optional[Literal['csv', 'excel', 'parquet', 'json']] = None,
+        schema_overrides: Optional[dict[str, ts.ColumnType]] = None,
         on_error: Literal['abort', 'ignore'] = 'abort',
+        print_stats: bool = False,
         **kwargs: Any,
     ) -> UpdateStatus:
         raise excs.Error(f'{self._display_name()} {self._name!r}: cannot insert into view')

pixeltable/dataframe.py

@@ -88,12 +88,12 @@ class DataFrameResultSet:
     def __iter__(self) -> Iterator[dict[str, Any]]:
         return (self._row_to_dict(i) for i in range(len(self)))
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
         if not isinstance(other, DataFrameResultSet):
             return False
         return self.to_pandas().equals(other.to_pandas())
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash(self.to_pandas())
 
 
@@ -571,7 +571,7 @@ class DataFrame:
             expr = exprs.Expr.from_object(raw_expr)
             if expr is None:
                 raise excs.Error(f'Invalid expression: {raw_expr}')
-            if expr.col_type.is_invalid_type():
+            if expr.col_type.is_invalid_type() and not (isinstance(expr, exprs.Literal) and expr.val is None):
                 raise excs.Error(f'Invalid type: {raw_expr}')
             if not expr.is_bound_by(self._from_clause.tbls):
                 raise excs.Error(
@@ -624,6 +624,8 @@ class DataFrame:
 
             >>> df = person.where(t.age > 30)
         """
+        if self.where_clause is not None:
+            raise excs.Error('Where clause already specified')
         if not isinstance(pred, exprs.Expr):
             raise excs.Error(f'Where() requires a Pixeltable expression, but instead got {type(pred)}')
         if not pred.col_type.is_bool_type():