pixeltable 0.2.21__py3-none-any.whl → 0.2.23__py3-none-any.whl

pixeltable/catalog/table_version.py

@@ -26,7 +26,7 @@ from pixeltable.utils.media_store import MediaStore
 
 from ..func.globals import resolve_symbol
 from .column import Column
-from .globals import _POS_COLUMN_NAME, _ROWID_COLUMN_NAME, UpdateStatus, is_valid_identifier
+from .globals import _POS_COLUMN_NAME, _ROWID_COLUMN_NAME, UpdateStatus, is_valid_identifier, MediaValidation
 
 if TYPE_CHECKING:
     from pixeltable import exec, store
@@ -53,6 +53,7 @@ class TableVersion:
     name: str
     version: int
     comment: str
+    media_validation: MediaValidation
     num_retained_versions: int
     schema_version: int
     view_md: Optional[schema.ViewMd]
@@ -109,6 +110,7 @@ class TableVersion:
         self.view_md = tbl_md.view_md  # save this as-is, it's needed for _create_md()
         is_view = tbl_md.view_md is not None
         self.is_snapshot = (is_view and tbl_md.view_md.is_snapshot) or bool(is_snapshot)
+        self.media_validation = MediaValidation[schema_version_md.media_validation.upper()]
         # a mutable TableVersion doesn't have a static version
         self.effective_version = self.version if self.is_snapshot else None
 
@@ -182,7 +184,7 @@ class TableVersion:
     @classmethod
     def create(
             cls, session: orm.Session, dir_id: UUID, name: str, cols: list[Column], num_retained_versions: int,
-            comment: str, base_path: Optional[pxt.catalog.TableVersionPath] = None,
+            comment: str, media_validation: MediaValidation, base_path: Optional[pxt.catalog.TableVersionPath] = None,
             view_md: Optional[schema.ViewMd] = None
     ) -> tuple[UUID, Optional[TableVersion]]:
         # assign ids
@@ -191,8 +193,6 @@ class TableVersion:
             col.id = pos
             col.schema_version_add = 0
             cols_by_name[col.name] = col
-            if col.value_expr is None and col.compute_func is not None:
-                cls._create_value_expr(col, base_path)
             if col.is_computed:
                 col.check_value_expr()
 
@@ -214,11 +214,17 @@ class TableVersion:
             tbl_id=tbl_record.id, version=0, md=dataclasses.asdict(table_version_md))
 
         # create schema.TableSchemaVersion
-        schema_col_md = {col.id: schema.SchemaColumn(pos=pos, name=col.name) for pos, col in enumerate(cols)}
+        schema_col_md: dict[int, schema.SchemaColumn] = {}
+        for pos, col in enumerate(cols):
+            md = schema.SchemaColumn(
+                pos=pos, name=col.name,
+                media_validation=col._media_validation.name.lower() if col._media_validation is not None else None)
+            schema_col_md[col.id] = md
 
         schema_version_md = schema.TableSchemaVersionMd(
             schema_version=0, preceding_schema_version=None, columns=schema_col_md,
-            num_retained_versions=num_retained_versions, comment=comment)
+            num_retained_versions=num_retained_versions, comment=comment,
+            media_validation=media_validation.name.lower())
         schema_version_record = schema.TableSchemaVersion(
             tbl_id=tbl_record.id, schema_version=0, md=dataclasses.asdict(schema_version_md))
 
@@ -285,10 +291,15 @@ class TableVersion:
         self.cols_by_name = {}
         self.cols_by_id = {}
         for col_md in tbl_md.column_md.values():
-            col_name = schema_version_md.columns[col_md.id].name if col_md.id in schema_version_md.columns else None
+            schema_col_md = schema_version_md.columns[col_md.id] if col_md.id in schema_version_md.columns else None
+            col_name = schema_col_md.name if schema_col_md is not None else None
+            media_val = (
+                MediaValidation[schema_col_md.media_validation.upper()]
+                if schema_col_md is not None and schema_col_md.media_validation is not None else None
+            )
             col = Column(
                 col_id=col_md.id, name=col_name, col_type=ts.ColumnType.from_dict(col_md.col_type),
-                is_pk=col_md.is_pk, stored=col_md.stored,
+                is_pk=col_md.is_pk, stored=col_md.stored, media_validation=media_val,
                 schema_version_add=col_md.schema_version_add, schema_version_drop=col_md.schema_version_drop,
                 value_expr_dict=col_md.value_expr)
             col.tbl = self
@@ -349,7 +360,8 @@ class TableVersion:
             self.store_tbl = StoreTable(self)
 
     def _update_md(
-            self, timestamp: float, conn: sql.engine.Connection, update_tbl_version: bool = True, preceding_schema_version: Optional[int] = None
+        self, timestamp: float, conn: sql.engine.Connection, update_tbl_version: bool = True,
+        preceding_schema_version: Optional[int] = None
     ) -> None:
         """Writes table metadata to the database.
 
@@ -480,37 +492,35 @@ class TableVersion:
             self._update_md(time.time(), conn, preceding_schema_version=preceding_schema_version)
             _logger.info(f'Dropped index {idx_md.name} on table {self.name}')
 
-    def add_column(self, col: Column, print_stats: bool, on_error: Literal['abort', 'ignore']) -> UpdateStatus:
+    def add_columns(self, cols: Iterable[Column], print_stats: bool, on_error: Literal['abort', 'ignore']) -> UpdateStatus:
         """Adds a column to the table.
         """
         assert not self.is_snapshot
-        assert is_valid_identifier(col.name)
-        assert col.stored is not None
-        assert col.name not in self.cols_by_name
-        col.tbl = self
-        col.id = self.next_col_id
-        self.next_col_id += 1
-
-        if col.compute_func is not None:
-            # create value_expr from compute_func
-            self._create_value_expr(col, self.path)
+        assert all(is_valid_identifier(col.name) for col in cols)
+        assert all(col.stored is not None for col in cols)
+        assert all(col.name not in self.cols_by_name for col in cols)
+        for col in cols:
+            col.tbl = self
+            col.id = self.next_col_id
+            self.next_col_id += 1
 
         # we're creating a new schema version
         self.version += 1
         preceding_schema_version = self.schema_version
         self.schema_version = self.version
         with Env.get().engine.begin() as conn:
-            status = self._add_columns([col], conn, print_stats=print_stats, on_error=on_error)
-            _ = self._add_default_index(col, conn)
+            status = self._add_columns(cols, conn, print_stats=print_stats, on_error=on_error)
+            for col in cols:
+                _ = self._add_default_index(col, conn)
             self._update_md(time.time(), conn, preceding_schema_version=preceding_schema_version)
-        _logger.info(f'Added column {col.name} to table {self.name}, new version: {self.version}')
+        _logger.info(f'Added columns {[col.name for col in cols]} to table {self.name}, new version: {self.version}')
 
         msg = (
             f'Added {status.num_rows} column value{"" if status.num_rows == 1 else "s"} '
             f'with {status.num_excs} error{"" if status.num_excs == 1 else "s"}.'
         )
         print(msg)
-        _logger.info(f'Column {col.name}: {msg}')
+        _logger.info(f'Columns {[col.name for col in cols]}: {msg}')
         return status
 
     def _add_columns(
@@ -710,20 +720,22 @@ class TableVersion:
 
         if conn is None:
             with Env.get().engine.begin() as conn:
-                return self._insert(plan, conn, time.time(), print_stats=print_stats, rowids=rowids())
+                return self._insert(
+                    plan, conn, time.time(), print_stats=print_stats, rowids=rowids(), abort_on_exc=fail_on_exception)
         else:
-            return self._insert(plan, conn, time.time(), print_stats=print_stats, rowids=rowids())
+            return self._insert(
+                plan, conn, time.time(), print_stats=print_stats, rowids=rowids(), abort_on_exc=fail_on_exception)
 
     def _insert(
         self, exec_plan: 'exec.ExecNode', conn: sql.engine.Connection, timestamp: float, *,
-        rowids: Optional[Iterator[int]] = None, print_stats: bool = False,
+        rowids: Optional[Iterator[int]] = None, print_stats: bool = False, abort_on_exc: bool = False
     ) -> UpdateStatus:
         """Insert rows produced by exec_plan and propagate to views"""
         # we're creating a new version
         self.version += 1
         result = UpdateStatus()
         num_rows, num_excs, cols_with_excs = self.store_tbl.insert_rows(
-            exec_plan, conn, v_min=self.version, rowids=rowids)
+            exec_plan, conn, v_min=self.version, rowids=rowids, abort_on_exc=abort_on_exc)
         result.num_rows = num_rows
         result.num_excs = num_excs
         result.num_computed_values += exec_plan.ctx.num_computed_exprs * num_rows
@@ -1124,28 +1136,6 @@ class TableVersion:
         names = [c.name for c in self.cols_by_name.values() if c.is_computed]
         return names
 
-    @classmethod
-    def _create_value_expr(cls, col: Column, path: pxt.catalog.TableVersionPath) -> None:
-        """
-        Create col.value_expr, given col.compute_func.
-        Interprets compute_func's parameters to be references to columns and construct ColumnRefs as args.
-        Does not update Column.dependent_cols.
-        """
-        assert col.value_expr is None
-        assert col.compute_func is not None
-        from pixeltable import exprs
-        params = inspect.signature(col.compute_func).parameters
-        args: list[exprs.ColumnRef] = []
-        for param_name in params:
-            param = path.get_column(param_name)
-            if param is None:
-                raise excs.Error(
-                    f'Column {col.name}: Callable parameter refers to an unknown column: {param_name}')
-            args.append(exprs.ColumnRef(param))
-        fn = func.make_function(
-            col.compute_func, return_type=col.col_type, param_types=[arg.col_type for arg in args])
-        col.set_value_expr(fn(*args))
-
     def _record_refd_columns(self, col: Column) -> None:
         """Update Column.dependent_cols for all cols referenced in col.value_expr.
         """
@@ -1203,7 +1193,8 @@ class TableVersion:
             name=self.name, current_version=self.version, current_schema_version=self.schema_version,
             next_col_id=self.next_col_id, next_idx_id=self.next_idx_id, next_row_id=self.next_rowid,
             column_md=self._create_column_md(self.cols), index_md=self.idx_md,
-            external_stores=self._create_stores_md(self.external_stores.values()), view_md=self.view_md)
+            external_stores=self._create_stores_md(self.external_stores.values()), view_md=self.view_md,
+        )
 
     def _create_version_md(self, timestamp: float) -> schema.TableVersionMd:
         return schema.TableVersionMd(created_at=timestamp, version=self.version, schema_version=self.schema_version)
@@ -1211,11 +1202,14 @@ class TableVersion:
     def _create_schema_version_md(self, preceding_schema_version: int) -> schema.TableSchemaVersionMd:
         column_md: dict[int, schema.SchemaColumn] = {}
         for pos, col in enumerate(self.cols_by_name.values()):
-            column_md[col.id] = schema.SchemaColumn(pos=pos, name=col.name)
+            column_md[col.id] = schema.SchemaColumn(
+                pos=pos, name=col.name,
+                media_validation=col._media_validation.name.lower() if col._media_validation is not None else None)
         # preceding_schema_version to be set by the caller
         return schema.TableSchemaVersionMd(
             schema_version=self.schema_version, preceding_schema_version=preceding_schema_version,
-            columns=column_md, num_retained_versions=self.num_retained_versions, comment=self.comment)
+            columns=column_md, num_retained_versions=self.num_retained_versions, comment=self.comment,
+            media_validation=self.media_validation.name.lower())
 
     def as_dict(self) -> dict:
         return {'id': str(self.id), 'effective_version': self.effective_version}

pixeltable/catalog/table_version_path.py

@@ -81,13 +81,13 @@ class TableVersionPath:
             return None
         return self.base.find_tbl_version(id)
 
-    def __getattr__(self, col_name: str) -> exprs.ColumnRef:
+    def get_column_ref(self, col_name: str) -> exprs.ColumnRef:
         """Return a ColumnRef for the given column name."""
         from pixeltable.exprs import ColumnRef
         if col_name not in self.tbl_version.cols_by_name:
             if self.base is None:
                 raise AttributeError(f'Column {col_name} unknown')
-            return getattr(self.base, col_name)
+            return self.base.get_column_ref(col_name)
         col = self.tbl_version.cols_by_name[col_name]
         return ColumnRef(col)
 

pixeltable/catalog/view.py

@@ -2,24 +2,21 @@ from __future__ import annotations
 
 import inspect
 import logging
-from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Set, Type
+from typing import TYPE_CHECKING, Any, Iterable, Literal, Optional
 from uuid import UUID
 
 import sqlalchemy.orm as orm
 
-import pixeltable.catalog as catalog
 import pixeltable.exceptions as excs
-import pixeltable.exprs as exprs
-import pixeltable.func as func
 import pixeltable.metadata.schema as md_schema
+import pixeltable.type_system as ts
+from pixeltable import catalog, exprs, func
 from pixeltable.env import Env
-from pixeltable.exceptions import Error
 from pixeltable.iterators import ComponentIterator
-from pixeltable.type_system import IntType, InvalidType
 
 from .catalog import Catalog
 from .column import Column
-from .globals import _POS_COLUMN_NAME, UpdateStatus
+from .globals import _POS_COLUMN_NAME, UpdateStatus, MediaValidation
 from .table import Table
 from .table_version import TableVersion
 from .table_version_path import TableVersionPath
@@ -52,9 +49,10 @@ class View(Table):
 
     @classmethod
     def _create(
-            cls, dir_id: UUID, name: str, base: TableVersionPath, additional_columns: Dict[str, Any],
+            cls, dir_id: UUID, name: str, base: TableVersionPath, additional_columns: dict[str, Any],
             predicate: Optional['pxt.exprs.Expr'], is_snapshot: bool, num_retained_versions: int, comment: str,
-            iterator_cls: Optional[Type[ComponentIterator]], iterator_args: Optional[Dict]
+            media_validation: MediaValidation,
+            iterator_cls: Optional[type[ComponentIterator]], iterator_args: Optional[dict]
     ) -> View:
         columns = cls._create_columns(additional_columns)
         cls._verify_schema(columns)
@@ -92,17 +90,17 @@ class View(Table):
                     func.Parameter(param_name, param_type, kind=inspect.Parameter.POSITIONAL_OR_KEYWORD)
                     for param_name, param_type in iterator_cls.input_schema().items()
                 ]
-                sig = func.Signature(InvalidType(), params)
+                sig = func.Signature(ts.InvalidType(), params)
                 from pixeltable.exprs import FunctionCall
                 FunctionCall.normalize_args(iterator_cls.__name__, sig, bound_args)
             except TypeError as e:
-                raise Error(f'Cannot instantiate iterator with given arguments: {e}')
+                raise excs.Error(f'Cannot instantiate iterator with given arguments: {e}')
 
             # prepend pos and output_schema columns to cols:
             # a component view exposes the pos column of its rowid;
             # we create that column here, so it gets assigned a column id;
             # stored=False: it is not stored separately (it's already stored as part of the rowid)
-            iterator_cols = [Column(_POS_COLUMN_NAME, IntType(), stored=False)]
+            iterator_cols = [Column(_POS_COLUMN_NAME, ts.IntType(), stored=False)]
             output_dict, unstored_cols = iterator_cls.output_schema(**bound_args)
             iterator_cols.extend([
                 Column(col_name, col_type, stored=col_name not in unstored_cols)
@@ -112,12 +110,12 @@ class View(Table):
             iterator_col_names = {col.name for col in iterator_cols}
             for col in columns:
                 if col.name in iterator_col_names:
-                    raise Error(f'Duplicate name: column {col.name} is already present in the iterator output schema')
+                    raise excs.Error(f'Duplicate name: column {col.name} is already present in the iterator output schema')
             columns = iterator_cols + columns
 
         with orm.Session(Env.get().engine, future=True) as session:
             from pixeltable.exprs import InlineDict
-            iterator_args_expr = InlineDict(iterator_args) if iterator_args is not None else None
+            iterator_args_expr: exprs.Expr = InlineDict(iterator_args) if iterator_args is not None else None
             iterator_class_fqn = f'{iterator_cls.__module__}.{iterator_cls.__name__}' if iterator_cls is not None \
                 else None
             base_version_path = cls._get_snapshot_path(base) if is_snapshot else base
@@ -142,7 +140,8 @@ class View(Table):
                 iterator_args=iterator_args_expr.as_dict() if iterator_args_expr is not None else None)
 
             id, tbl_version = TableVersion.create(
-                session, dir_id, name, columns, num_retained_versions, comment, base_path=base_version_path, view_md=view_md)
+                session, dir_id, name, columns, num_retained_versions, comment, media_validation=media_validation,
+                base_path=base_version_path, view_md=view_md)
             if tbl_version is None:
                 # this is purely a snapshot: we use the base's tbl version path
                 view = cls(id, dir_id, name, base_version_path, base.tbl_id(), snapshot_only=True)
@@ -168,11 +167,11 @@ class View(Table):
 
     @classmethod
     def _verify_column(
-            cls, col: Column, existing_column_names: Set[str], existing_query_names: Optional[Set[str]] = None
+            cls, col: Column, existing_column_names: set[str], existing_query_names: Optional[set[str]] = None
     ) -> None:
         # make sure that columns are nullable or have a default
         if not col.col_type.nullable and not col.is_computed:
-            raise Error(f'Column {col.name}: non-computed columns in views must be nullable')
+            raise excs.Error(f'Column {col.name}: non-computed columns in views must be nullable')
         super()._verify_column(col, existing_column_names, existing_query_names)
 
     @classmethod
@@ -217,7 +216,7 @@ class View(Table):
 
     def insert(
             self, rows: Optional[Iterable[dict[str, Any]]] = None, /, *, print_stats: bool = False,
-            fail_on_exception: bool = True, **kwargs: Any
+            on_error: Literal['abort', 'ignore'] = 'abort', **kwargs: Any
     ) -> UpdateStatus:
         raise excs.Error(f'{self._display_name()} {self._name!r}: cannot insert into view')
 

pixeltable/dataframe.py

@@ -8,7 +8,7 @@ import logging
 import mimetypes
 import traceback
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, Hashable, Iterator, List, Optional, Sequence, Set, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Hashable, Iterator, Optional, Sequence, Union
 
 import pandas as pd
 import pandas.io.formats.style
@@ -34,14 +34,6 @@ __all__ = ['DataFrame']
 _logger = logging.getLogger('pixeltable')
 
 
-def _create_source_tag(file_path: str) -> str:
-    src_url = get_file_uri(Env.get().http_address, file_path)
-    mime = mimetypes.guess_type(src_url)[0]
-    # if mime is None, the attribute string would not be valid html.
-    mime_attr = f'type="{mime}"' if mime is not None else ''
-    return f'<source src="{src_url}" {mime_attr} />'
-
-
 class DataFrameResultSet:
     def __init__(self, rows: list[list[Any]], schema: dict[str, ColumnType]):
         self._rows = rows
@@ -77,7 +69,7 @@ class DataFrameResultSet:
     def to_pandas(self) -> pd.DataFrame:
         return pd.DataFrame.from_records(self._rows, columns=self._col_names)
 
-    def _row_to_dict(self, row_idx: int) -> Dict[str, Any]:
+    def _row_to_dict(self, row_idx: int) -> dict[str, Any]:
         return {self._col_names[i]: self._rows[row_idx][i] for i in range(len(self._col_names))}
 
     def __getitem__(self, index: Any) -> Any:
@@ -111,22 +103,22 @@ class DataFrameResultSet:
 #     def __init__(self, tbl: catalog.TableVersion):
 #         self.tbl = tbl
 #         # output of the SQL scan stage
-#         self.sql_scan_output_exprs: List[exprs.Expr] = []
+#         self.sql_scan_output_exprs: list[exprs.Expr] = []
 #         # output of the agg stage
-#         self.agg_output_exprs: List[exprs.Expr] = []
+#         self.agg_output_exprs: list[exprs.Expr] = []
 #         # Where clause of the Select stmt of the SQL scan stage
 #         self.sql_where_clause: Optional[sql.ClauseElement] = None
 #         # filter predicate applied to input rows of the SQL scan stage
 #         self.filter: Optional[exprs.Predicate] = None
 #         self.similarity_clause: Optional[exprs.ImageSimilarityPredicate] = None
-#         self.agg_fn_calls: List[exprs.FunctionCall] = []  # derived from unique_exprs
+#         self.agg_fn_calls: list[exprs.FunctionCall] = []  # derived from unique_exprs
 #         self.has_frame_col: bool = False  # True if we're referencing the frame col
 #
 #         self.evaluator: Optional[exprs.Evaluator] = None
-#         self.sql_scan_eval_ctx: List[exprs.Expr] = []  # needed to materialize output of SQL scan stage
-#         self.agg_eval_ctx: List[exprs.Expr] = []  # needed to materialize output of agg stage
-#         self.filter_eval_ctx: List[exprs.Expr] = []
-#         self.group_by_eval_ctx: List[exprs.Expr] = []
+#         self.sql_scan_eval_ctx: list[exprs.Expr] = []  # needed to materialize output of SQL scan stage
+#         self.agg_eval_ctx: list[exprs.Expr] = []  # needed to materialize output of agg stage
+#         self.filter_eval_ctx: list[exprs.Expr] = []
+#         self.group_by_eval_ctx: list[exprs.Expr] = []
 #
 #     def finalize_exec(self) -> None:
 #         """
@@ -142,11 +134,11 @@ class DataFrame:
     def __init__(
         self,
         tbl: catalog.TableVersionPath,
-        select_list: Optional[List[Tuple[exprs.Expr, Optional[str]]]] = None,
+        select_list: Optional[list[tuple[exprs.Expr, Optional[str]]]] = None,
         where_clause: Optional[exprs.Expr] = None,
-        group_by_clause: Optional[List[exprs.Expr]] = None,
+        group_by_clause: Optional[list[exprs.Expr]] = None,
         grouping_tbl: Optional[catalog.TableVersion] = None,
-        order_by_clause: Optional[List[Tuple[exprs.Expr, bool]]] = None,  # List[(expr, asc)]
+        order_by_clause: Optional[list[tuple[exprs.Expr, bool]]] = None,  # list[(expr, asc)]
         limit: Optional[int] = None,
     ):
         self.tbl = tbl
@@ -174,7 +166,7 @@ class DataFrame:
     @classmethod
     def _select_list_check_rep(
         cls,
-        select_list: Optional[List[Tuple[exprs.Expr, Optional[str]]]],
+        select_list: Optional[list[tuple[exprs.Expr, Optional[str]]]],
     ) -> None:
         """Validate basic select list types."""
         if select_list is None:  # basic check for valid select list
@@ -371,15 +363,10 @@ class DataFrame:
             group_by_clause=group_by_clause, grouping_tbl=self.grouping_tbl,
             order_by_clause=order_by_clause, limit=self.limit_val)
 
-    def collect(self) -> DataFrameResultSet:
-        return self._collect()
-
-    def _collect(self, conn: Optional[sql.engine.Connection] = None) -> DataFrameResultSet:
+    def _output_row_iterator(self, conn: Optional[sql.engine.Connection] = None) -> Iterator[list]:
         try:
-            result_rows = []
             for data_row in self._exec(conn):
-                result_row = [data_row[e.slot_idx] for e in self._select_list_exprs]
-                result_rows.append(result_row)
+                yield [data_row[e.slot_idx] for e in self._select_list_exprs]
         except excs.ExprEvalError as e:
             msg = f'In row {e.row_num} the {e.expr_msg} encountered exception ' f'{type(e.exc).__name__}:\n{str(e.exc)}'
             if len(e.input_vals) > 0:
@@ -399,7 +386,11 @@ class DataFrame:
         except sql.exc.DBAPIError as e:
             raise excs.Error(f'Error during SQL execution:\n{e}')
 
-        return DataFrameResultSet(result_rows, self.schema)
+    def collect(self) -> DataFrameResultSet:
+        return self._collect()
+
+    def _collect(self, conn: Optional[sql.engine.Connection] = None) -> DataFrameResultSet:
+        return DataFrameResultSet(list(self._output_row_iterator(conn)), self.schema)
 
     def count(self) -> int:
         from pixeltable.plan import Planner
@@ -412,8 +403,8 @@ class DataFrame:
 
     def _description(self) -> pd.DataFrame:
         """see DataFrame.describe()"""
-        heading_vals: List[str] = []
-        info_vals: List[str] = []
+        heading_vals: list[str] = []
+        info_vals: list[str] = []
         if self.select_list is not None:
             assert len(self.select_list) > 0
             heading_vals.append('Select')
@@ -498,7 +489,7 @@ class DataFrame:
 
         # check user provided names do not conflict among themselves
         # or with auto-generated ones
-        seen: Set[str] = set()
+        seen: set[str] = set()
         _, names = DataFrame._normalize_select_list(self.tbl, select_list)
         for name in names:
             if name in seen:
@@ -541,7 +532,7 @@ class DataFrame:
         if self.group_by_clause is not None:
             raise excs.Error(f'Group-by already specified')
         grouping_tbl: Optional[catalog.TableVersion] = None
-        group_by_clause: Optional[List[exprs.Expr]] = None
+        group_by_clause: Optional[list[exprs.Expr]] = None
         for item in grouping_items:
             if isinstance(item, catalog.Table):
                 if len(grouping_items) > 1:
@@ -619,7 +610,7 @@ class DataFrame:
     def __getitem__(self, index: Union[exprs.Expr, Sequence[exprs.Expr]]) -> DataFrame:
         """
         Allowed:
-        - [List[Expr]]/[Tuple[Expr]]: setting the select list
+        - [list[Expr]]/[tuple[Expr]]: setting the select list
         - [Expr]: setting a single-col select list
         """
         if isinstance(index, exprs.Expr):
@@ -628,7 +619,7 @@ class DataFrame:
             return self.select(*index)
         raise TypeError(f'Invalid index type: {type(index)}')
 
-    def as_dict(self) -> Dict[str, Any]:
+    def as_dict(self) -> dict[str, Any]:
         """
         Returns:
             Dictionary representing this dataframe.
@@ -650,7 +641,7 @@ class DataFrame:
         return d
 
     @classmethod
-    def from_dict(cls, d: Dict[str, Any]) -> 'DataFrame':
+    def from_dict(cls, d: dict[str, Any]) -> 'DataFrame':
         tbl = catalog.TableVersionPath.from_dict(d['tbl'])
         select_list = [(exprs.Expr.from_dict(e), name) for e, name in d['select_list']] \
             if d['select_list'] is not None else None

pixeltable/env.py

@@ -494,18 +494,25 @@ class Env:
         self.__register_package('anthropic')
         self.__register_package('boto3')
         self.__register_package('datasets')
+        self.__register_package('fiftyone')
         self.__register_package('fireworks', library_name='fireworks-ai')
+        self.__register_package('huggingface_hub', library_name='huggingface-hub')
         self.__register_package('label_studio_sdk', library_name='label-studio-sdk')
+        self.__register_package('llama_cpp', library_name='llama-cpp-python')
         self.__register_package('mistralai')
         self.__register_package('mistune')
+        self.__register_package('ollama')
         self.__register_package('openai')
         self.__register_package('openpyxl')
         self.__register_package('pyarrow')
+        self.__register_package('replicate')
+        self.__register_package('sentencepiece')
         self.__register_package('sentence_transformers', library_name='sentence-transformers')
         self.__register_package('spacy')
         self.__register_package('tiktoken')
         self.__register_package('together')
         self.__register_package('torch')
+        self.__register_package('torchaudio')
         self.__register_package('torchvision')
         self.__register_package('transformers')
         self.__register_package('whisper', library_name='openai-whisper')

pixeltable/exec/__init__.py

@@ -6,6 +6,5 @@ from .exec_context import ExecContext
 from .exec_node import ExecNode
 from .expr_eval_node import ExprEvalNode
 from .in_memory_data_node import InMemoryDataNode
-from .media_validation_node import MediaValidationNode
 from .row_update_node import RowUpdateNode
 from .sql_node import SqlLookupNode, SqlScanNode, SqlAggregationNode, SqlNode

pixeltable/exec/aggregation_node.py

@@ -2,11 +2,12 @@ from __future__ import annotations
 
 import logging
 import sys
-from typing import Iterable, Optional, Any, Iterator
+from typing import Any, Iterable, Iterator, Optional, cast
 
 import pixeltable.catalog as catalog
 import pixeltable.exceptions as excs
 import pixeltable.exprs as exprs
+
 from .data_row_batch import DataRowBatch
 from .exec_node import ExecNode
 
@@ -28,13 +29,15 @@ class AggregationNode(ExecNode):
             self, tbl: catalog.TableVersion, row_builder: exprs.RowBuilder, group_by: Optional[list[exprs.Expr]],
             agg_fn_calls: list[exprs.FunctionCall], input_exprs: Iterable[exprs.Expr], input: ExecNode
     ):
-        super().__init__(row_builder, group_by + agg_fn_calls, input_exprs, input)
+        output_exprs: list[exprs.Expr] = [] if group_by is None else list(group_by)
+        output_exprs.extend(agg_fn_calls)
+        super().__init__(row_builder, output_exprs, input_exprs, input)
         self.input = input
         self.group_by = group_by
         self.input_exprs = list(input_exprs)
         self.agg_fn_eval_ctx = row_builder.create_eval_ctx(agg_fn_calls, exclude=self.input_exprs)
         # we need to make sure to refer to the same exprs that RowBuilder.eval() will use
-        self.agg_fn_calls = self.agg_fn_eval_ctx.target_exprs
+        self.agg_fn_calls = [cast(exprs.FunctionCall, e) for e in self.agg_fn_eval_ctx.target_exprs]
         # create output_batch here, rather than in __iter__(), so we don't need to remember tbl and row_builder
         self.output_batch = DataRowBatch(tbl, row_builder, 0)