pixeltable 0.2.21__py3-none-any.whl → 0.2.23__py3-none-any.whl

pixeltable/__init__.py

@@ -3,8 +3,8 @@ from .dataframe import DataFrame
 from .exceptions import Error
 from .exprs import RELATIVE_PATH_ROOT
 from .func import Aggregator, Function, expr_udf, uda, udf
-from .globals import (array, configure_logging, create_dir, create_table, create_view, drop_dir, drop_table, get_table,
-                      init, list_dirs, list_functions, list_tables, move)
+from .globals import (array, configure_logging, create_dir, create_snapshot, create_table, create_view, drop_dir,
+                      drop_table, get_table, init, list_dirs, list_functions, list_tables, move)
 from .type_system import (Array, ArrayType, Audio, AudioType, Bool, BoolType, ColumnType, Document, DocumentType, Float,
                           FloatType, Image, ImageType, Int, IntType, Json, JsonType, Required, String, StringType,
                           Timestamp, TimestampType, Video, VideoType)

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = "0.2.21"
-__version_tuple__ = (0, 2, 21)
+__version__ = "0.2.23"
+__version_tuple__ = (0, 2, 23)

pixeltable/catalog/__init__.py

@@ -1,7 +1,7 @@
 from .catalog import Catalog
 from .column import Column
 from .dir import Dir
-from .globals import UpdateStatus, is_valid_identifier, is_valid_path
+from .globals import UpdateStatus, is_valid_identifier, is_valid_path, MediaValidation
 from .insertable_table import InsertableTable
 from .named_function import NamedFunction
 from .path import Path

pixeltable/catalog/column.py

@@ -1,31 +1,49 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Callable, Optional, Union
+from typing import TYPE_CHECKING, Any, Optional
 
 import sqlalchemy as sql
 
 import pixeltable.exceptions as excs
 import pixeltable.type_system as ts
 from pixeltable import exprs
-
-from .globals import is_valid_identifier
+from .globals import is_valid_identifier, MediaValidation
 
 if TYPE_CHECKING:
     from .table_version import TableVersion
 
 _logger = logging.getLogger('pixeltable')
 
+
 class Column:
     """Representation of a column in the schema of a Table/DataFrame.
 
     A Column contains all the metadata necessary for executing queries and updates against a particular version of a
     table/view.
     """
+    name: str
+    id: Optional[int]
+    col_type: ts.ColumnType
+    stored: bool
+    is_pk: bool
+    _media_validation: Optional[MediaValidation]  # if not set, TableVersion.media_validation applies
+    schema_version_add: Optional[int]
+    schema_version_drop: Optional[int]
+    _records_errors: Optional[bool]
+    sa_col: Optional[sql.schema.Column]
+    sa_col_type: Optional[sql.sqltypes.TypeEngine]
+    sa_errormsg_col: Optional[sql.schema.Column]
+    sa_errortype_col: Optional[sql.schema.Column]
+    _value_expr: Optional[exprs.Expr]
+    value_expr_dict: Optional[dict[str, Any]]
+    dependent_cols: set[Column]
+    tbl: Optional[TableVersion]
+
     def __init__(
             self, name: Optional[str], col_type: Optional[ts.ColumnType] = None,
-            computed_with: Optional[Union[exprs.Expr, Callable]] = None,
-            is_pk: bool = False, stored: bool = True,
+            computed_with: Optional[exprs.Expr] = None,
+            is_pk: bool = False, stored: bool = True, media_validation: Optional[MediaValidation] = None,
             col_id: Optional[int] = None, schema_version_add: Optional[int] = None,
             schema_version_drop: Optional[int] = None, sa_col_type: Optional[sql.sqltypes.TypeEngine] = None,
             records_errors: Optional[bool] = None, value_expr_dict: Optional[dict[str, Any]] = None,
@@ -35,7 +53,7 @@ class Column:
         Args:
             name: column name; None for system columns (eg, index columns)
             col_type: column type; can be None if the type can be derived from ``computed_with``
-            computed_with: a callable or an Expr object that computes the column value
+            computed_with: an Expr that computes the column value
             is_pk: if True, this column is part of the primary key
             stored: determines whether a computed column is present in the stored table or recomputed on demand
             col_id: column ID (only used internally)
@@ -45,11 +63,6 @@ class Column:
           col_type is None
         - when loaded from md store: ``computed_with`` is set and col_type is set
 
-        ``computed_with`` is a Callable:
-        - the callable's parameter names must correspond to existing columns in the table for which this Column
-          is being used
-        - ``col_type`` needs to be set to the callable's return type
-
         ``stored`` (only valid for computed image columns):
         - if True: the column is present in the stored table
         - if False: the column is not present in the stored table and recomputed during a query
@@ -62,21 +75,13 @@ class Column:
             raise excs.Error(f'Column `{name}`: col_type is required if computed_with is not specified')
 
         self._value_expr: Optional[exprs.Expr] = None
-        self.compute_func: Optional[Callable] = None
         self.value_expr_dict = value_expr_dict
         if computed_with is not None:
             value_expr = exprs.Expr.from_object(computed_with)
             if value_expr is None:
-                # computed_with needs to be a Callable
-                if not callable(computed_with):
-                    raise excs.Error(
-                        f'Column {name}: computed_with needs to be either a Pixeltable expression or a Callable, '
-                        f'but it is a {type(computed_with)}')
-                if col_type is None:
-                    raise excs.Error(f'Column {name}: col_type is required if computed_with is a Callable')
-                # we need to turn the computed_with function into an Expr, but this requires resolving
-                # column name references and for that we need to wait until we're assigned to a Table
-                self.compute_func = computed_with
+                raise excs.Error(
+                    f'Column {name}: computed_with needs to be a valid Pixeltable expression, '
+                    f'but it is a {type(computed_with)}')
             else:
                 self._value_expr = value_expr.copy()
                 self.col_type = self._value_expr.col_type
@@ -86,24 +91,24 @@ class Column:
         assert self.col_type is not None
 
         self.stored = stored
-        self.dependent_cols: set[Column] = set()  # cols with value_exprs that reference us; set by TableVersion
+        self.dependent_cols = set()  # cols with value_exprs that reference us; set by TableVersion
         self.id = col_id
         self.is_pk = is_pk
+        self._media_validation = media_validation
         self.schema_version_add = schema_version_add
         self.schema_version_drop = schema_version_drop
 
         self._records_errors = records_errors
 
         # column in the stored table for the values of this Column
-        self.sa_col: Optional[sql.schema.Column] = None
+        self.sa_col = None
         self.sa_col_type = sa_col_type
 
         # computed cols also have storage columns for the exception string and type
-        self.sa_errormsg_col: Optional[sql.schema.Column] = None
-        self.sa_errortype_col: Optional[sql.schema.Column] = None
+        self.sa_errormsg_col = None
+        self.sa_errortype_col = None
 
-        from .table_version import TableVersion
-        self.tbl: Optional[TableVersion] = None  # set by owning TableVersion
+        self.tbl = None  # set by owning TableVersion
 
     @property
     def value_expr(self) -> Optional[exprs.Expr]:
@@ -139,7 +144,7 @@ class Column:
 
     @property
     def is_computed(self) -> bool:
-        return self.compute_func is not None or self._value_expr is not None or self.value_expr_dict is not None
+        return self._value_expr is not None or self.value_expr_dict is not None
 
     @property
     def is_stored(self) -> bool:
@@ -160,6 +165,13 @@ class Column:
         assert self.tbl is not None
         return f'{self.tbl.name}.{self.name}'
 
+    @property
+    def media_validation(self) -> MediaValidation:
+        if self._media_validation is not None:
+            return self._media_validation
+        assert self.tbl is not None
+        return self.tbl.media_validation
+
     def source(self) -> None:
         """
         If this is a computed col and the top-level expr is a function call, print the source, if possible.

pixeltable/catalog/globals.py

@@ -1,8 +1,12 @@
+from __future__ import annotations
 import dataclasses
+import enum
 import itertools
 import logging
 from typing import Optional
 
+import pixeltable.exceptions as excs
+
 _logger = logging.getLogger('pixeltable')
 
 # name of the position column in a component view
@@ -34,6 +38,20 @@ class UpdateStatus:
         self.cols_with_excs = list(dict.fromkeys(self.cols_with_excs + other.cols_with_excs))
         return self
 
+
+class MediaValidation(enum.Enum):
+    ON_READ = 0
+    ON_WRITE = 1
+
+    @classmethod
+    def validated(cls, name: str, error_prefix: str) -> MediaValidation:
+        try:
+            return cls[name.upper()]
+        except KeyError:
+            val_strs = ', '.join(f'{s.lower()!r}' for s in cls.__members__.keys())
+            raise excs.Error(f'{error_prefix} must be one of: [{val_strs}]')
+
+
 def is_valid_identifier(name: str) -> bool:
     return name.isidentifier() and not name.startswith('_')
 

pixeltable/catalog/insertable_table.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import logging
-from typing import Any, Dict, Iterable, List, Optional, overload
+from typing import Any, Iterable, Literal, Optional, overload
 from uuid import UUID
 
 import sqlalchemy.orm as orm
@@ -13,7 +13,7 @@ from pixeltable.env import Env
 from pixeltable.utils.filecache import FileCache
 
 from .catalog import Catalog
-from .globals import UpdateStatus
+from .globals import MediaValidation, UpdateStatus
 from .table import Table
 from .table_version import TableVersion
 from .table_version_path import TableVersionPath
@@ -35,8 +35,8 @@ class InsertableTable(Table):
     # MODULE-LOCAL, NOT PUBLIC
     @classmethod
     def _create(
-            cls, dir_id: UUID, name: str, schema: dict[str, ts.ColumnType], df: Optional[pxt.DataFrame], primary_key: List[str],
-            num_retained_versions: int, comment: str
+        cls, dir_id: UUID, name: str, schema: dict[str, ts.ColumnType], df: Optional[pxt.DataFrame],
+        primary_key: list[str], num_retained_versions: int, comment: str, media_validation: MediaValidation
     ) -> InsertableTable:
         columns = cls._create_columns(schema)
         cls._verify_schema(columns)
@@ -50,7 +50,9 @@ class InsertableTable(Table):
             col.is_pk = True
 
         with orm.Session(Env.get().engine, future=True) as session:
-            _, tbl_version = TableVersion.create(session, dir_id, name, columns, num_retained_versions, comment)
+            _, tbl_version = TableVersion.create(
+                session, dir_id, name, columns, num_retained_versions=num_retained_versions, comment=comment,
+                media_validation=media_validation)
             tbl = cls(dir_id, tbl_version)
             # TODO We need to commit before doing the insertion, in order to avoid a primary key (version) collision
             #   when the table metadata gets updated. Once we have a notion of user-defined transactions in
@@ -77,15 +79,31 @@ class InsertableTable(Table):
 
     @overload
     def insert(
-            self, rows: Iterable[Dict[str, Any]], /, *, print_stats: bool = False, fail_on_exception: bool = True
+        self,
+        rows: Iterable[dict[str, Any]],
+        /,
+        *,
+        print_stats: bool = False,
+        on_error: Literal['abort', 'ignore'] = 'abort'
     ) -> UpdateStatus: ...
 
     @overload
-    def insert(self, *, print_stats: bool = False, fail_on_exception: bool = True, **kwargs: Any) -> UpdateStatus: ...
+    def insert(
+        self,
+        *,
+        print_stats: bool = False,
+        on_error: Literal['abort', 'ignore'] = 'abort',
+        **kwargs: Any
+    ) -> UpdateStatus: ...
 
     def insert(  # type: ignore[misc]
-            self, rows: Optional[Iterable[dict[str, Any]]] = None, /, *, print_stats: bool = False,
-            fail_on_exception: bool = True, **kwargs: Any
+        self,
+        rows: Optional[Iterable[dict[str, Any]]] = None,
+        /,
+        *,
+        print_stats: bool = False,
+        on_error: Literal['abort', 'ignore'] = 'abort',
+        **kwargs: Any
     ) -> UpdateStatus:
         if rows is None:
             rows = [kwargs]
@@ -94,6 +112,8 @@ class InsertableTable(Table):
             if len(kwargs) > 0:
                 raise excs.Error('`kwargs` cannot be specified unless `rows is None`.')
 
+        fail_on_exception = on_error == 'abort'
+
         if not isinstance(rows, list):
             raise excs.Error('rows must be a list of dictionaries')
         if len(rows) == 0:
@@ -119,7 +139,7 @@ class InsertableTable(Table):
         FileCache.get().emit_eviction_warnings()
         return status
 
-    def _validate_input_rows(self, rows: List[Dict[str, Any]]) -> None:
+    def _validate_input_rows(self, rows: list[dict[str, Any]]) -> None:
         """Verify that the input rows match the table schema"""
         valid_col_names = set(self._schema.keys())
         reqd_col_names = set(self._tbl_version_path.tbl_version.get_required_col_names())