pixeltable 0.4.11__py3-none-any.whl → 0.4.13__py3-none-any.whl

pixeltable/__init__.py

@@ -1,7 +1,17 @@
 # ruff: noqa: F401
 
 from .__version__ import __version__, __version_tuple__
-from .catalog import Column, ColumnMetadata, IndexMetadata, InsertableTable, Table, TableMetadata, UpdateStatus, View
+from .catalog import (
+    Column,
+    ColumnMetadata,
+    IndexMetadata,
+    InsertableTable,
+    Table,
+    TableMetadata,
+    UpdateStatus,
+    VersionMetadata,
+    View,
+)
 from .dataframe import DataFrame
 from .exceptions import Error, ExprEvalError, PixeltableWarning
 from .func import Aggregator, Function, Tool, ToolChoice, Tools, expr_udf, mcp_udfs, query, retrieval_udf, uda, udf

pixeltable/catalog/__init__.py

@@ -8,7 +8,8 @@ from .insertable_table import InsertableTable
 from .named_function import NamedFunction
 from .path import Path
 from .schema_object import SchemaObject
-from .table import ColumnMetadata, IndexMetadata, Table, TableMetadata
+from .table import Table
+from .table_metadata import ColumnMetadata, IndexMetadata, TableMetadata, VersionMetadata
 from .table_version import TableVersion
 from .table_version_handle import ColumnHandle, TableVersionHandle
 from .table_version_path import TableVersionPath

pixeltable/catalog/table.py

@@ -7,9 +7,7 @@ import json
 import logging
 from keyword import iskeyword as is_python_keyword
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar, Iterable, Literal, Optional, TypedDict, overload
-
-from typing import _GenericAlias  # type: ignore[attr-defined]  # isort: skip
+from typing import TYPE_CHECKING, Any, Iterable, Literal, Optional, overload
 from uuid import UUID
 
 import pandas as pd
@@ -17,6 +15,13 @@ import sqlalchemy as sql
 
 import pixeltable as pxt
 from pixeltable import catalog, env, exceptions as excs, exprs, index, type_system as ts
+from pixeltable.catalog.table_metadata import (
+    ColumnMetadata,
+    EmbeddingIndexParams,
+    IndexMetadata,
+    TableMetadata,
+    VersionMetadata,
+)
 from pixeltable.metadata import schema
 from pixeltable.metadata.utils import MetadataUtils
 
@@ -37,6 +42,9 @@ from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
 from .update_status import UpdateStatus
 
+from typing import _GenericAlias  # type: ignore[attr-defined]  # isort: skip
+
+
 if TYPE_CHECKING:
     import torch.utils.data
 
@@ -95,7 +103,7 @@ class Table(SchemaObject):
 
         return op()
 
-    def _get_metadata(self) -> 'TableMetadata':
+    def _get_metadata(self) -> TableMetadata:
         columns = self._tbl_version_path.columns()
         column_info: dict[str, ColumnMetadata] = {}
         for col in columns:
@@ -1478,12 +1486,17 @@ class Table(SchemaObject):
             return result
 
     def recompute_columns(
-        self, *columns: str | ColumnRef, errors_only: bool = False, cascade: bool = True
+        self,
+        *columns: str | ColumnRef,
+        where: 'exprs.Expr' | None = None,
+        errors_only: bool = False,
+        cascade: bool = True,
     ) -> UpdateStatus:
         """Recompute the values in one or more computed columns of this table.
 
         Args:
             columns: The names or references of the computed columns to recompute.
+            where: A predicate to filter rows to recompute.
             errors_only: If True, only run the recomputation for rows that have errors in the column (ie, the column's
                 `errortype` property indicates that an error occurred). Only allowed for recomputing a single column.
             cascade: if True, also update all computed columns that transitively depend on the recomputed columns.
@@ -1499,6 +1512,10 @@ class Table(SchemaObject):
 
             >>> tbl.recompute_columns(tbl.c1, tbl.c2, cascade=False)
 
+            Recompute column `c1` and its dependents, but only for rows with `c2` == 0:
+
+            >>> tbl.recompute_columns('c1', where=tbl.c2 == 0)
+
             Recompute column `c1` and its dependents, but only for rows that have errors in it:
 
             >>> tbl.recompute_columns('c1', errors_only=True)
@@ -1535,7 +1552,12 @@ class Table(SchemaObject):
                     raise excs.Error(f'Cannot recompute column of a base: {col_name!r}')
                 col_names.append(col_name)
 
-            result = self._tbl_version.get().recompute_columns(col_names, errors_only=errors_only, cascade=cascade)
+            if where is not None and not where.is_bound_by([self._tbl_version_path]):
+                raise excs.Error(f"'where' ({where}) not bound by {self._display_str()}")
+
+            result = self._tbl_version.get().recompute_columns(
+                col_names, where=where, errors_only=errors_only, cascade=cascade
+            )
             FileCache.get().emit_eviction_warnings()
             return result
 
@@ -1676,43 +1698,35 @@ class Table(SchemaObject):
     def _ipython_key_completions_(self) -> list[str]:
         return list(self._get_schema().keys())
 
-    _REPORT_SCHEMA: ClassVar[dict[str, ts.ColumnType]] = {
-        'version': ts.IntType(),
-        'created_at': ts.TimestampType(),
-        'user': ts.StringType(nullable=True),
-        'note': ts.StringType(),
-        'inserts': ts.IntType(nullable=True),
-        'updates': ts.IntType(nullable=True),
-        'deletes': ts.IntType(nullable=True),
-        'errors': ts.IntType(nullable=True),
-        'computed': ts.IntType(),
-        'schema_change': ts.StringType(),
-    }
-
-    def history(self, n: Optional[int] = None) -> pixeltable.dataframe.DataFrameResultSet:
-        """Returns rows of information about the versions of this table, most recent first.
+    def get_versions(self, n: Optional[int] = None) -> list[VersionMetadata]:
+        """
+        Returns information about versions of this table, most recent first.
+
+        `get_versions()` is intended for programmatic access to version metadata; for human-readable
+        output, use [`history()`][pixeltable.Table.history] instead.
 
         Args:
-            n: a limit to the number of versions listed
+            n: if specified, will return at most `n` versions
 
-        Examples:
-            Report history:
+        Returns:
+            A list of [VersionMetadata][pixeltable.VersionMetadata] dictionaries, one per version retrieved, most
+            recent first.
 
-            >>> tbl.history()
+        Examples:
+            Retrieve metadata about all versions of the table `tbl`:
 
-            Report only the most recent 5 changes to the table:
+            >>> tbl.get_versions()
 
-            >>> tbl.history(n=5)
+            Retrieve metadata about the most recent 5 versions of the table `tbl`:
 
-        Returns:
-            A list of information about each version, ordered from most recent to oldest version.
+            >>> tbl.get_versions(n=5)
         """
         from pixeltable.catalog import Catalog
 
         if n is None:
             n = 1_000_000_000
         if not isinstance(n, int) or n < 1:
-            raise excs.Error(f'Invalid value for n: {n}')
+            raise excs.Error(f'Invalid value for `n`: {n}')
 
         # Retrieve the table history components from the catalog
         tbl_id = self._id
@@ -1730,104 +1744,60 @@ class Table(SchemaObject):
         else:
             over_count = 0
 
-        report_lines: list[list[Any]] = []
+        metadata_dicts: list[VersionMetadata] = []
         for vers_md in vers_list[0 : len(vers_list) - over_count]:
             version = vers_md.version_md.version
-            schema_change = md_dict.get(version, '')
+            schema_change = md_dict.get(version, None)
             update_status = vers_md.version_md.update_status
             if update_status is None:
                 update_status = UpdateStatus()
-            change_type = 'schema' if schema_change != '' else ''
-            if change_type == '':
-                change_type = 'data'
+            change_type: Literal['schema', 'data'] = 'schema' if schema_change is not None else 'data'
             rcs = update_status.row_count_stats + update_status.cascade_row_count_stats
-            report_line = [
-                version,
-                datetime.datetime.fromtimestamp(vers_md.version_md.created_at),
-                vers_md.version_md.user,
-                change_type,
-                rcs.ins_rows,
-                rcs.upd_rows,
-                rcs.del_rows,
-                rcs.num_excs,
-                rcs.computed_values,
-                schema_change,
-            ]
-            report_lines.append(report_line)
+            metadata_dicts.append(
+                VersionMetadata(
+                    version=version,
+                    created_at=datetime.datetime.fromtimestamp(vers_md.version_md.created_at, tz=datetime.timezone.utc),
+                    user=vers_md.version_md.user,
+                    change_type=change_type,
+                    inserts=rcs.ins_rows,
+                    updates=rcs.upd_rows,
+                    deletes=rcs.del_rows,
+                    errors=rcs.num_excs,
+                    computed=rcs.computed_values,
+                    schema_change=schema_change,
+                )
+            )
+
+        return metadata_dicts
+
+    def history(self, n: Optional[int] = None) -> pd.DataFrame:
+        """
+        Returns a human-readable report about versions of this table.
 
-        return pxt.dataframe.DataFrameResultSet(report_lines, self._REPORT_SCHEMA)
+        `history()` is intended for human-readable output of version metadata; for programmatic access,
+        use [`get_versions()`][pixeltable.Table.get_versions] instead.
+
+        Args:
+            n: if specified, will return at most `n` versions
+
+        Returns:
+            A report with information about each version, one per row, most recent first.
+
+        Examples:
+            Report all versions of the table:
+
+            >>> tbl.history()
+
+            Report only the most recent 5 changes to the table:
+
+            >>> tbl.history(n=5)
+        """
+        versions = self.get_versions(n)
+        assert len(versions) > 0
+        return pd.DataFrame([list(v.values()) for v in versions], columns=list(versions[0].keys()))
 
     def __check_mutable(self, op_descr: str) -> None:
         if self._tbl_version_path.is_snapshot():
             raise excs.Error(f'{self._display_str()}: Cannot {op_descr} a snapshot.')
         if self._tbl_version_path.is_replica():
             raise excs.Error(f'{self._display_str()}: Cannot {op_descr} a {self._display_name()}.')
-
-
-class ColumnMetadata(TypedDict):
-    """Metadata for a column of a Pixeltable table."""
-
-    name: str
-    """The name of the column."""
-    type_: str
-    """The type specifier of the column."""
-    version_added: int
-    """The table version when this column was added."""
-    is_stored: bool
-    """`True` if this is a stored column; `False` if it is dynamically computed."""
-    is_primary_key: bool
-    """`True` if this column is part of the table's primary key."""
-    media_validation: Optional[Literal['on_read', 'on_write']]
-    """The media validation policy for this column."""
-    computed_with: Optional[str]
-    """Expression used to compute this column; `None` if this is not a computed column."""
-
-
-class IndexMetadata(TypedDict):
-    """Metadata for a column of a Pixeltable table."""
-
-    name: str
-    """The name of the index."""
-    columns: list[str]
-    """The table columns that are indexed."""
-    index_type: Literal['embedding']
-    """The type of index (currently only `'embedding'` is supported, but others will be added in the future)."""
-    parameters: EmbeddingIndexParams
-
-
-class EmbeddingIndexParams(TypedDict):
-    metric: Literal['cosine', 'ip', 'l2']
-    """Index metric."""
-    embeddings: list[str]
-    """List of embeddings defined for this index."""
-
-
-class TableMetadata(TypedDict):
-    """Metadata for a Pixeltable table."""
-
-    name: str
-    """The name of the table (ex: `'my_table'`)."""
-    path: str
-    """The full path of the table (ex: `'my_dir.my_subdir.my_table'`)."""
-    columns: dict[str, ColumnMetadata]
-    """Column metadata for all of the visible columns of the table."""
-    indices: dict[str, IndexMetadata]
-    """Index metadata for all of the indices of the table."""
-    is_replica: bool
-    """`True` if this table is a replica of another (shared) table."""
-    is_view: bool
-    """`True` if this table is a view."""
-    is_snapshot: bool
-    """`True` if this table is a snapshot."""
-    version: int
-    """The current version of the table."""
-    version_created: datetime.datetime
-    """The timestamp when this table version was created."""
-    schema_version: int
-    """The current schema version of the table."""
-    comment: Optional[str]
-    """User-provided table comment, if one exists."""
-    media_validation: Literal['on_read', 'on_write']
-    """The media validation policy for this table."""
-    base: Optional[str]
-    """If this table is a view or snapshot, the full path of its base table; otherwise `None`."""

pixeltable/catalog/table_metadata.py

@@ -0,0 +1,96 @@
+import datetime
+from typing import Literal, Optional, TypedDict
+
+
+class ColumnMetadata(TypedDict):
+    """Metadata for a column of a Pixeltable table."""
+
+    name: str
+    """The name of the column."""
+    type_: str
+    """The type specifier of the column."""
+    version_added: int
+    """The table version when this column was added."""
+    is_stored: bool
+    """`True` if this is a stored column; `False` if it is dynamically computed."""
+    is_primary_key: bool
+    """`True` if this column is part of the table's primary key."""
+    media_validation: Optional[Literal['on_read', 'on_write']]
+    """The media validation policy for this column."""
+    computed_with: Optional[str]
+    """Expression used to compute this column; `None` if this is not a computed column."""
+
+
+class EmbeddingIndexParams(TypedDict):
+    metric: Literal['cosine', 'ip', 'l2']
+    """Index metric."""
+    embeddings: list[str]
+    """List of embeddings defined for this index."""
+
+
+class IndexMetadata(TypedDict):
+    """Metadata for a column of a Pixeltable table."""
+
+    name: str
+    """The name of the index."""
+    columns: list[str]
+    """The table columns that are indexed."""
+    index_type: Literal['embedding']
+    """The type of index (currently only `'embedding'` is supported, but others will be added in the future)."""
+    parameters: EmbeddingIndexParams
+
+
+class TableMetadata(TypedDict):
+    """Metadata for a Pixeltable table."""
+
+    name: str
+    """The name of the table (ex: `'my_table'`)."""
+    path: str
+    """The full path of the table (ex: `'my_dir.my_subdir.my_table'`)."""
+    columns: dict[str, ColumnMetadata]
+    """Column metadata for all of the visible columns of the table."""
+    indices: dict[str, IndexMetadata]
+    """Index metadata for all of the indices of the table."""
+    is_replica: bool
+    """`True` if this table is a replica of another (shared) table."""
+    is_view: bool
+    """`True` if this table is a view."""
+    is_snapshot: bool
+    """`True` if this table is a snapshot."""
+    version: int
+    """The current version of the table."""
+    version_created: datetime.datetime
+    """The timestamp when this table version was created."""
+    schema_version: int
+    """The current schema version of the table."""
+    comment: Optional[str]
+    """User-provided table comment, if one exists."""
+    media_validation: Literal['on_read', 'on_write']
+    """The media validation policy for this table."""
+    base: Optional[str]
+    """If this table is a view or snapshot, the full path of its base table; otherwise `None`."""
+
+
+class VersionMetadata(TypedDict):
+    """Metadata for a specific version of a Pixeltable table."""
+
+    """The version number."""
+    version: int
+    """The timestamp when this version was created."""
+    created_at: datetime.datetime
+    """The user who created this version, if defined."""
+    user: str | None
+    """The type of table transformation that this version represents (`'data'` or `'schema'`)."""
+    change_type: Literal['data', 'schema']
+    """The number of rows inserted in this version."""
+    inserts: int
+    """The number of rows updated in this version."""
+    updates: int
+    """The number of rows deleted in this version."""
+    deletes: int
+    """The number of errors encountered during this version."""
+    errors: int
+    """The number of computed values calculated in this version."""
+    computed: int
+    """A description of the schema change that occurred in this version, if any."""
+    schema_change: str | None

pixeltable/catalog/table_version.py

@@ -1065,21 +1065,28 @@ class TableVersion:
 
         return update_targets
 
-    def recompute_columns(self, col_names: list[str], errors_only: bool = False, cascade: bool = True) -> UpdateStatus:
+    def recompute_columns(
+        self, col_names: list[str], where: exprs.Expr | None = None, errors_only: bool = False, cascade: bool = True
+    ) -> UpdateStatus:
         assert self.is_mutable
         assert all(name in self.cols_by_name for name in col_names)
         assert len(col_names) > 0
         assert len(col_names) == 1 or not errors_only
 
+        from pixeltable.exprs import CompoundPredicate
         from pixeltable.plan import Planner
 
         target_columns = [self.cols_by_name[name] for name in col_names]
         where_clause: Optional[exprs.Expr] = None
+        if where is not None:
+            self._validate_where_clause(where, error_prefix="'where' argument")
+            where_clause = where
         if errors_only:
-            where_clause = (
+            errortype_pred = (
                 exprs.ColumnPropertyRef(exprs.ColumnRef(target_columns[0]), exprs.ColumnPropertyRef.Property.ERRORTYPE)
                 != None
             )
+            where_clause = CompoundPredicate.make_conjunction([where_clause, errortype_pred])
         plan, updated_cols, recomputed_cols = Planner.create_update_plan(
             self.path, update_targets={}, recompute_targets=target_columns, where_clause=where_clause, cascade=cascade
         )
@@ -1142,40 +1149,30 @@ class TableVersion:
             self._write_md(new_version=True, new_schema_version=False)
         return result
 
-    def delete(self, where: Optional[exprs.Expr] = None) -> UpdateStatus:
-        """Delete rows in this table.
-        Args:
-            where: a predicate to filter rows to delete.
-
-        Returns:
-            UpdateStatus: an object containing the number of deleted rows and other statistics.
-        """
+    def _validate_where_clause(self, pred: exprs.Expr, error_prefix: str) -> None:
+        """Validates that pred can be expressed as a SQL Where clause"""
         assert self.is_insertable
         from pixeltable.exprs import Expr
         from pixeltable.plan import Planner
 
-        sql_where_clause: Optional[Expr] = None
-        if where is not None:
-            if not isinstance(where, Expr):
-                raise excs.Error(f"'where' argument must be a predicate, got {type(where)}")
-            analysis_info = Planner.analyze(self.path, where)
-            # for now we require that the updated rows can be identified via SQL, rather than via a Python filter
-            if analysis_info.filter is not None:
-                raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')
-            sql_where_clause = analysis_info.sql_where_clause
+        if not isinstance(pred, Expr):
+            raise excs.Error(f'{error_prefix} must be a predicate, got {type(pred)}')
+        analysis_info = Planner.analyze(self.path, pred)
+        # for now we require that the updated rows can be identified via SQL, rather than via a Python filter
+        if analysis_info.filter is not None:
+            raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')
 
-        status = self.propagate_delete(sql_where_clause, base_versions=[], timestamp=time.time())
+    def delete(self, where: exprs.Expr | None = None) -> UpdateStatus:
+        assert self.is_insertable
+        if where is not None:
+            self._validate_where_clause(where, error_prefix="'where' argument")
+        status = self.propagate_delete(where, base_versions=[], timestamp=time.time())
         return status
 
     def propagate_delete(
         self, where: Optional[exprs.Expr], base_versions: list[Optional[int]], timestamp: float
     ) -> UpdateStatus:
-        """Delete rows in this table and propagate to views.
-        Args:
-            where: a predicate to filter rows to delete.
-        Returns:
-            number of deleted rows
-        """
+        """Delete rows in this table and propagate to views"""
         # print(f'calling sql_expr()')
         sql_where_clause = where.sql_expr(exprs.SqlElementCache()) if where is not None else None
         # #print(f'sql_where_clause={str(sql_where_clause) if sql_where_clause is not None else None}')

pixeltable/catalog/view.py

@@ -323,7 +323,9 @@ class View(Table):
             else:
                 base_descr = f'{base._path()}:{effective_version}'
                 bases_descrs.append(f'{base_descr!r}')
-        result.append(f' (of {", ".join(bases_descrs)})')
+        if len(bases_descrs) > 0:
+            # bases_descrs can be empty in the case of a table-replica
+            result.append(f' (of {", ".join(bases_descrs)})')
 
         if self._tbl_version_path.tbl_version.get().predicate is not None:
             result.append(f'\nWhere: {self._tbl_version_path.tbl_version.get().predicate!s}')

pixeltable/dataframe.py

@@ -1210,17 +1210,42 @@ class DataFrame:
             Via the above DataFrame person, update the column 'city' to 'Oakland'
             and 'state' to 'CA' in the table t:
 
-            >>> df = person.update({'city': 'Oakland', 'state': 'CA'})
+            >>> person.update({'city': 'Oakland', 'state': 'CA'})
 
             Via the above DataFrame person, update the column 'age' to 30 for any
             rows where 'year' is 2014 in the table t:
 
-            >>> df = person.where(t.year == 2014).update({'age': 30})
+            >>> person.where(t.year == 2014).update({'age': 30})
         """
         self._validate_mutable('update', False)
         with Catalog.get().begin_xact(tbl=self._first_tbl, for_write=True, lock_mutable_tree=True):
             return self._first_tbl.tbl_version.get().update(value_spec, where=self.where_clause, cascade=cascade)
 
+    def recompute_columns(
+        self, *columns: str | exprs.ColumnRef, errors_only: bool = False, cascade: bool = True
+    ) -> UpdateStatus:
+        """Recompute one or more computed columns of the underlying table of the DataFrame.
+
+        Args:
+            columns: The names or references of the computed columns to recompute.
+            errors_only: If True, only run the recomputation for rows that have errors in the column (ie, the column's
+                `errortype` property indicates that an error occurred). Only allowed for recomputing a single column.
+            cascade: if True, also update all computed columns that transitively depend on the recomputed columns.
+
+        Returns:
+            UpdateStatus: the status of the operation.
+
+        Example:
+            For table `person` with column `age` and computed column `height`, recompute the value of `height` for all
+            rows where `age` is less than 18:
+
+            >>> df = person.where(t.age < 18).recompute_columns(person.height)
+        """
+        self._validate_mutable('recompute_columns', False)
+        with Catalog.get().begin_xact(tbl=self._first_tbl, for_write=True, lock_mutable_tree=True):
+            tbl = Catalog.get().get_table_by_id(self._first_tbl.tbl_id)
+            return tbl.recompute_columns(*columns, where=self.where_clause, errors_only=errors_only, cascade=cascade)
+
     def delete(self) -> UpdateStatus:
         """Delete rows form the underlying table of the DataFrame.
 
@@ -1230,13 +1255,9 @@ class DataFrame:
             UpdateStatus: the status of the delete operation.
 
         Example:
-            Given the DataFrame person from a table t with all its columns and rows:
-
-            >>> person = t.select()
-
-            Via the above DataFrame person, delete all rows from the table t where the column 'age' is less than 18:
+            For a table `person` with column `age`, delete all rows where 'age' is less than 18:
 
-            >>> df = person.where(t.age < 18).delete()
+            >>> person.where(t.age < 18).delete()
         """
         self._validate_mutable('delete', False)
         if not self._first_tbl.is_insertable():

pixeltable/env.py

@@ -605,12 +605,7 @@ class Env:
         metadata.upgrade_md(self._sa_engine)
 
     @property
-    def pxt_api_key(self) -> str:
-        if self._pxt_api_key is None:
-            raise excs.Error(
-                'No API key is configured. Set the PIXELTABLE_API_KEY environment variable, or add an entry to '
-                'config.toml as described here:\nhttps://pixeltable.github.io/pixeltable/config/'
-            )
+    def pxt_api_key(self) -> Optional[str]:
         return self._pxt_api_key
 
     def get_client(self, name: str) -> Any:
@@ -748,6 +743,7 @@ class Env:
         self.__register_package('whisper', library_name='openai-whisper')
         self.__register_package('whisperx')
         self.__register_package('yolox', library_name='pixeltable-yolox')
+        self.__register_package('lancedb')
 
     def __register_package(self, package_name: str, library_name: Optional[str] = None) -> None:
         is_installed: bool

pixeltable/exprs/compound_predicate.py

@@ -36,7 +36,8 @@ class CompoundPredicate(Expr):
         return f' {self.operator} '.join([f'({e})' for e in self.components])
 
     @classmethod
-    def make_conjunction(cls, operands: list[Expr]) -> Optional[Expr]:
+    def make_conjunction(cls, operands: list[Expr | None]) -> Expr | None:
+        operands = [e for e in operands if e is not None]
         if len(operands) == 0:
             return None
         if len(operands) == 1: