pixeltable 0.2.20__py3-none-any.whl → 0.2.22__py3-none-any.whl

pixeltable/functions/whisper.py

@@ -14,27 +14,9 @@ from pixeltable.env import Env
 if TYPE_CHECKING:
     from whisper import Whisper  # type: ignore[import-untyped]
 
-
-@pxt.udf(
-    param_types=[
-        pxt.AudioType(),
-        pxt.StringType(),
-        pxt.JsonType(nullable=True),
-        pxt.FloatType(nullable=True),
-        pxt.FloatType(nullable=True),
-        pxt.FloatType(nullable=True),
-        pxt.BoolType(),
-        pxt.StringType(nullable=True),
-        pxt.BoolType(),
-        pxt.StringType(),
-        pxt.StringType(),
-        pxt.StringType(),
-        pxt.FloatType(nullable=True),
-        pxt.JsonType(nullable=True),
-    ]
-)
+@pxt.udf
 def transcribe(
-    audio: str,
+    audio: pxt.Audio,
     *,
     model: str,
     temperature: Optional[list[float]] = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0],

pixeltable/globals.py

@@ -1,6 +1,6 @@
 import dataclasses
 import logging
-from typing import Any, Iterable, Optional, Union
+from typing import Any, Iterable, Optional, Union, Literal
 from uuid import UUID
 
 import pandas as pd
@@ -33,6 +33,7 @@ def create_table(
     primary_key: Optional[Union[str, list[str]]] = None,
     num_retained_versions: int = 10,
     comment: str = '',
+    media_validation: Literal['on_read', 'on_write'] = 'on_write'
 ) -> catalog.Table:
     """Create a new base table.
 
@@ -44,6 +45,9 @@ def create_table(
             table.
         num_retained_versions: Number of versions of the table to retain.
         comment: An optional comment; its meaning is user-defined.
+        media_validation: Media validation policy for the table.
+            - `'on_read'`: validate media files at query time
+            - `'on_write'`: validate media files during insert/update operations
 
     Returns:
         A handle to the newly created [`Table`][pixeltable.Table].
@@ -54,11 +58,13 @@ def create_table(
     Examples:
         Create a table with an int and a string column:
 
-        >>> table = pxt.create_table('my_table', schema={'col1': IntType(), 'col2': StringType()})
+        >>> tbl = pxt.create_table('my_table', schema={'col1': pxt.Int, 'col2': pxt.String})
 
-        Create a table from a select statement over an existing table `tbl`:
+        Create a table from a select statement over an existing table `orig_table` (this will create a new table
+        containing the exact contents of the query):
 
-        >>> table = pxt.create_table('my_table', tbl.where(tbl.col1 < 10).select(tbl.col2))
+        >>> tbl1 = pxt.get_table('orig_table')
+        ... tbl2 = pxt.create_table('new_table', tbl1.where(tbl1.col1 < 10).select(tbl1.col2))
     """
     path = catalog.Path(path_str)
     Catalog.get().paths.check_is_valid(path, expected=None)
@@ -87,14 +93,8 @@ def create_table(
             raise excs.Error('primary_key must be a single column name or a list of column names')
 
     tbl = catalog.InsertableTable._create(
-        dir._id,
-        path.name,
-        schema,
-        df,
-        primary_key=primary_key,
-        num_retained_versions=num_retained_versions,
-        comment=comment,
-    )
+        dir._id, path.name, schema, df, primary_key=primary_key, num_retained_versions=num_retained_versions,
+        comment=comment, media_validation=catalog.MediaValidation.validated(media_validation, 'media_validation'))
     Catalog.get().paths[path] = tbl
 
     _logger.info(f'Created table `{path_str}`.')
@@ -105,22 +105,24 @@ def create_view(
     path_str: str,
     base: Union[catalog.Table, DataFrame],
     *,
-    schema: Optional[dict[str, Any]] = None,
-    filter: Optional[exprs.Expr] = None,
+    additional_columns: Optional[dict[str, Any]] = None,
     is_snapshot: bool = False,
     iterator: Optional[tuple[type[ComponentIterator], dict[str, Any]]] = None,
     num_retained_versions: int = 10,
     comment: str = '',
+    media_validation: Literal['on_read', 'on_write'] = 'on_write',
     ignore_errors: bool = False,
 ) -> Optional[catalog.Table]:
     """Create a view of an existing table object (which itself can be a view or a snapshot or a base table).
 
     Args:
-        path_str: Path to the view.
+        path_str: A name for the view; can be either a simple name such as `my_view`, or a pathname such as
+            `dir1.my_view`.
         base: [`Table`][pixeltable.Table] (i.e., table or view or snapshot) or [`DataFrame`][pixeltable.DataFrame] to
             base the view on.
-        schema: dictionary mapping column names to column types, value expressions, or to column specifications.
-        filter: predicate to filter rows of the base table.
+        additional_columns: If specified, will add these columns to the view once it is created. The format
+            of the `additional_columns` parameter is identical to the format of the `schema_or_df` parameter in
+            [`create_table`][pixeltable.create_table].
         is_snapshot: Whether the view is a snapshot.
         iterator: The iterator to use for this view. If specified, then this view will be a one-to-many view of
             the base table.
@@ -130,36 +132,29 @@ def create_view(
 
     Returns:
         A handle to the [`Table`][pixeltable.Table] representing the newly created view. If the path already
-        exists or is invalid and `ignore_errors=True`, returns `None`.
+            exists or is invalid and `ignore_errors=True`, returns `None`.
 
     Raises:
         Error: if the path already exists or is invalid and `ignore_errors=False`.
 
     Examples:
-        Create a view with an additional int and a string column and a filter:
-
-        >>> view = cl.create_view(
-            'my_view', base, schema={'col3': IntType(), 'col4': StringType()}, filter=base.col1 > 10)
+        Create a view `my_view` of an existing table `my_table`, filtering on rows where `col1` is greater than 10:
 
-        Create a table snapshot:
+        >>> tbl = pxt.get_table('my_table')
+        ... view = pxt.create_view('my_view', tbl.where(tbl.col1 > 10))
 
-        >>> snapshot_view = cl.create_view('my_snapshot_view', base, is_snapshot=True)
+        Create a snapshot of `my_table`:
 
-        Create an immutable view with additional computed columns and a filter:
-
-        >>> snapshot_view = cl.create_view(
-            'my_snapshot', base, schema={'col3': base.col2 + 1}, filter=base.col1 > 10, is_snapshot=True)
+        >>> tbl = pxt.get_table('my_table')
+        ... snapshot_view = pxt.create_view('my_snapshot_view', tbl, is_snapshot=True)
     """
+    where: Optional[exprs.Expr] = None
     if isinstance(base, catalog.Table):
         tbl_version_path = base._tbl_version_path
     elif isinstance(base, DataFrame):
         base._validate_mutable('create_view')
         tbl_version_path = base.tbl
-        if base.where_clause is not None and filter is not None:
-            raise excs.Error(
-                'Cannot specify a `filter` directly if one is already declared in a `DataFrame.where` clause'
-            )
-        filter = base.where_clause
+        where = base.where_clause
     else:
         raise excs.Error('`base` must be an instance of `Table` or `DataFrame`')
     assert isinstance(base, catalog.Table) or isinstance(base, DataFrame)
@@ -173,25 +168,18 @@ def create_view(
             raise e
     dir = Catalog.get().paths[path.parent]
 
-    if schema is None:
-        schema = {}
+    if additional_columns is None:
+        additional_columns = {}
     if iterator is None:
         iterator_class, iterator_args = None, None
     else:
         iterator_class, iterator_args = iterator
 
     view = catalog.View._create(
-        dir._id,
-        path.name,
-        base=tbl_version_path,
-        schema=schema,
-        predicate=filter,
-        is_snapshot=is_snapshot,
-        iterator_cls=iterator_class,
-        iterator_args=iterator_args,
-        num_retained_versions=num_retained_versions,
-        comment=comment,
-    )
+        dir._id, path.name, base=tbl_version_path, additional_columns=additional_columns, predicate=where,
+        is_snapshot=is_snapshot, iterator_cls=iterator_class, iterator_args=iterator_args,
+        num_retained_versions=num_retained_versions, comment=comment,
+        media_validation=catalog.MediaValidation.validated(media_validation, 'media_validation'))
     Catalog.get().paths[path] = view
     _logger.info(f'Created view `{path_str}`.')
     FileCache.get().emit_eviction_warnings()
@@ -199,7 +187,7 @@ def create_view(
 
 
 def get_table(path: str) -> catalog.Table:
-    """Get a handle to an existing table or view or snapshot.
+    """Get a handle to an existing table, view, or snapshot.
 
     Args:
         path: Path to the table.
@@ -213,15 +201,15 @@ def get_table(path: str) -> catalog.Table:
     Examples:
         Get handle for a table in the top-level directory:
 
-        >>> table = cl.get_table('my_table')
+        >>> tbl = pxt.get_table('my_table')
 
         For a table in a subdirectory:
 
-        >>> table = cl.get_table('subdir.my_table')
+        >>> tbl = pxt.get_table('subdir.my_table')
 
-        For a snapshot in the top-level directory:
+        Handles to views and snapshots are retrieved in the same way:
 
-        >>> table = cl.get_table('my_snapshot')
+        >>> tbl = pxt.get_table('my_snapshot')
     """
     p = catalog.Path(path)
     Catalog.get().paths.check_is_valid(p, expected=catalog.Table)
@@ -243,11 +231,11 @@ def move(path: str, new_path: str) -> None:
     Examples:
         Move a table to a different directory:
 
-        >>>> cl.move('dir1.my_table', 'dir2.my_table')
+        >>>> pxt.move('dir1.my_table', 'dir2.my_table')
 
         Rename a table:
 
-        >>>> cl.move('dir1.my_table', 'dir1.new_name')
+        >>>> pxt.move('dir1.my_table', 'dir1.new_name')
     """
     p = catalog.Path(path)
     Catalog.get().paths.check_is_valid(p, expected=catalog.SchemaObject)
@@ -260,18 +248,18 @@ def move(path: str, new_path: str) -> None:
 
 
 def drop_table(path: str, force: bool = False, ignore_errors: bool = False) -> None:
-    """Drop a table or view or snapshot.
+    """Drop a table, view, or snapshot.
 
     Args:
         path: Path to the [`Table`][pixeltable.Table].
-        force: If `True`, will also drop all views or sub-views of this table.
-        ignore_errors: Whether to ignore errors if the table does not exist.
+        force: If `True`, will also drop all views and sub-views of this table.
+        ignore_errors: If `True`, return silently if the table does not exist (without throwing an exception).
 
     Raises:
-        Error: If the path does not exist or does not designate a table object and ignore_errors is False.
+        Error: If the path does not exist or does not designate a table object, and `ignore_errors=False`.
 
     Examples:
-        >>> cl.drop_table('my_table')
+        >>> pxt.drop_table('my_table')
     """
     cat = Catalog.get()
     path_obj = catalog.Path(path)
@@ -302,7 +290,8 @@ def list_tables(dir_path: str = '', recursive: bool = True) -> list[str]:
 
     Args:
         dir_path: Path to the directory. Defaults to the root directory.
-        recursive: Whether to list tables in subdirectories as well.
+        recursive: If `False`, returns only those tables that are directly contained in specified directory; if
+            `True`, returns all tables that are descendants of the specified directory, recursively.
 
     Returns:
         A list of [`Table`][pixeltable.Table] paths.
@@ -313,13 +302,11 @@ def list_tables(dir_path: str = '', recursive: bool = True) -> list[str]:
     Examples:
         List tables in top-level directory:
 
-        >>> cl.list_tables()
-        ['my_table', ...]
+        >>> pxt.list_tables()
 
         List tables in 'dir1':
 
-        >>> cl.list_tables('dir1')
-        [...]
+        >>> pxt.list_tables('dir1')
     """
     assert dir_path is not None
     path = catalog.Path(dir_path, empty_is_valid=True)
@@ -332,17 +319,17 @@ def create_dir(path_str: str, ignore_errors: bool = False) -> Optional[catalog.D
 
     Args:
         path_str: Path to the directory.
-        ignore_errors: if True, silently returns on error
+        ignore_errors: if `True`, will return silently instead of throwing an exception if an error occurs.
 
     Raises:
-        Error: If the path already exists or the parent is not a directory.
+        Error: If the path already exists or the parent is not a directory, and `ignore_errors=False`.
 
     Examples:
-        >>> cl.create_dir('my_dir')
+        >>> pxt.create_dir('my_dir')
 
         Create a subdirectory:
 
-        >>> cl.create_dir('my_dir.sub_dir')
+        >>> pxt.create_dir('my_dir.sub_dir')
     """
     try:
         path = catalog.Path(path_str)
@@ -373,17 +360,21 @@ def drop_dir(path_str: str, force: bool = False, ignore_errors: bool = False) ->
     """Remove a directory.
 
     Args:
-        path_str: Path to the directory.
+        path_str: Name or path of the directory.
+        force: If `True`, will also drop all tables and subdirectories of this directory, recursively, along
+            with any views or snapshots that depend on any of the dropped tables.
+        ignore_errors: if `True`, will return silently instead of throwing an exception if the directory
+            does not exist.
 
     Raises:
-        Error: If the path does not exist or does not designate a directory or if the directory is not empty.
+        Error: If the path does not exist or does not designate a directory, or if the directory is not empty.
 
     Examples:
-        >>> cl.drop_dir('my_dir')
+        >>> pxt.drop_dir('my_dir')
 
         Remove a subdirectory:
 
-        >>> cl.drop_dir('my_dir.sub_dir')
+        >>> pxt.drop_dir('my_dir.sub_dir')
     """
     cat = Catalog.get()
     path = catalog.Path(path_str)
@@ -428,14 +419,14 @@ def list_dirs(path_str: str = '', recursive: bool = True) -> list[str]:
     """List the directories in a directory.
 
     Args:
-        path_str: Path to the directory.
-        recursive: Whether to list subdirectories recursively.
+        path_str: Name or path of the directory.
+        recursive: If `True`, lists all descendants of this directory recursively.
 
     Returns:
         List of directory paths.
 
     Raises:
-        Error: If the path does not exist or does not designate a directory.
+        Error: If `path_str` does not exist or does not designate a directory.
 
     Examples:
         >>> cl.list_dirs('my_dir', recursive=True)

pixeltable/index/base.py

@@ -5,7 +5,7 @@ from typing import Any
 
 import sqlalchemy as sql
 
-import pixeltable.catalog as catalog
+from pixeltable import catalog, exprs
 
 
 class IndexBase(abc.ABC):
@@ -22,7 +22,7 @@ class IndexBase(abc.ABC):
         pass
 
     @abc.abstractmethod
-    def index_value_expr(self) -> 'pixeltable.exprs.Expr':
+    def index_value_expr(self) -> exprs.Expr:
         """Return expression that computes the value that goes into the index"""
         pass
 

pixeltable/index/btree.py

@@ -1,13 +1,16 @@
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 
 import sqlalchemy as sql
 
 # TODO: why does this import result in a circular import, but the one im embedding_index.py doesn't?
-#import pixeltable.catalog as catalog
+# import pixeltable.catalog as catalog
 import pixeltable.exceptions as excs
-import pixeltable.func as func
+from pixeltable import catalog, exprs
+from pixeltable.func.udf import udf
 from .base import IndexBase
 
+if TYPE_CHECKING:
+    import pixeltable.exprs
 
 class BtreeIndex(IndexBase):
     """
@@ -15,7 +18,10 @@ class BtreeIndex(IndexBase):
     """
     MAX_STRING_LEN = 256
 
-    @func.udf
+    value_expr: 'pixeltable.exprs.Expr'
+
+    @staticmethod
+    @udf
     def str_filter(s: Optional[str]) -> Optional[str]:
         if s is None:
             return None
@@ -24,10 +30,16 @@ class BtreeIndex(IndexBase):
     def __init__(self, c: 'catalog.Column'):
         if not c.col_type.is_scalar_type() and not c.col_type.is_media_type():
             raise excs.Error(f'Index on column {c.name}: B-tree index requires scalar or media type, got {c.col_type}')
-        from pixeltable.exprs import ColumnRef
-        self.value_expr = self.str_filter(ColumnRef(c)) if c.col_type.is_string_type() else ColumnRef(c)
+        if c.col_type.is_media_type():
+            # an index on a media column is an index on the file url
+            # no validation for media columns: we're only interested in the string value
+            self.value_expr = exprs.ColumnRef(c, perform_validation=False)
+        else:
+            self.value_expr = (
+                BtreeIndex.str_filter(exprs.ColumnRef(c)) if c.col_type.is_string_type() else exprs.ColumnRef(c)
+            )
 
-    def index_value_expr(self) -> 'pixeltable.exprs.Expr':
+    def index_value_expr(self) -> 'exprs.Expr':
         return self.value_expr
 
     def records_value_errors(self) -> bool:
@@ -52,3 +64,4 @@ class BtreeIndex(IndexBase):
     @classmethod
     def from_dict(cls, c: 'catalog.Column', d: dict) -> 'BtreeIndex':
         return cls(c)
+

pixeltable/index/embedding_index.py

@@ -1,18 +1,17 @@
 from __future__ import annotations
 
-from typing import Optional, Any
 import enum
+from typing import Any, Optional
 
-import PIL.Image
 import numpy as np
-import pgvector.sqlalchemy
+import pgvector.sqlalchemy  # type: ignore[import-untyped]
 import PIL.Image
 import sqlalchemy as sql
 
-import pixeltable.catalog as catalog
 import pixeltable.exceptions as excs
-import pixeltable.func as func
 import pixeltable.type_system as ts
+from pixeltable import catalog, exprs, func
+
 from .base import IndexBase
 
 
@@ -58,16 +57,15 @@ class EmbeddingIndex(IndexBase):
             self._validate_embedding_fn(image_embed, 'image_embed', ts.ColumnType.Type.IMAGE)
 
         self.metric = self.Metric[metric.upper()]
-        from pixeltable.exprs import ColumnRef
-        self.value_expr = string_embed(ColumnRef(c)) if c.col_type.is_string_type() else image_embed(ColumnRef(c))
-        assert self.value_expr.col_type.is_array_type()
+        self.value_expr = string_embed(exprs.ColumnRef(c)) if c.col_type.is_string_type() else image_embed(exprs.ColumnRef(c))
+        assert isinstance(self.value_expr.col_type, ts.ArrayType)
         self.string_embed = string_embed
         self.image_embed = image_embed
         vector_size = self.value_expr.col_type.shape[0]
         assert vector_size is not None
         self.index_col_type = pgvector.sqlalchemy.Vector(vector_size)
 
-    def index_value_expr(self) -> 'pixeltable.exprs.Expr':
+    def index_value_expr(self) -> exprs.Expr:
         """Return expression that computes the value that goes into the index"""
         return self.value_expr
 
@@ -88,8 +86,8 @@ class EmbeddingIndex(IndexBase):
         )
         idx.create(bind=conn)
 
-    def similarity_clause(self, val_column: catalog.Column, item: Any) -> sql.ClauseElement:
-        """Create a ClauseElement that represents '<val_column> <op> <item>'"""
+    def similarity_clause(self, val_column: catalog.Column, item: Any) -> sql.ColumnElement:
+        """Create a ColumnElement that represents '<val_column> <op> <item>'"""
         assert isinstance(item, (str, PIL.Image.Image))
         if isinstance(item, str):
             assert self.string_embed is not None
@@ -106,8 +104,8 @@ class EmbeddingIndex(IndexBase):
             assert self.metric == self.Metric.L2
             return val_column.sa_col.l2_distance(embedding)
 
-    def order_by_clause(self, val_column: catalog.Column, item: Any, is_asc: bool) -> sql.ClauseElement:
-        """Create a ClauseElement that is used in an ORDER BY clause"""
+    def order_by_clause(self, val_column: catalog.Column, item: Any, is_asc: bool) -> sql.ColumnElement:
+        """Create a ColumnElement that is used in an ORDER BY clause"""
         assert isinstance(item, (str, PIL.Image.Image))
         embedding: Optional[np.ndarray] = None
         if isinstance(item, str):
@@ -151,7 +149,7 @@ class EmbeddingIndex(IndexBase):
             img = PIL.Image.new('RGB', (512, 512))
             return_type = embed_fn.call_return_type({param_name: img})
         assert return_type is not None
-        if not return_type.is_array_type():
+        if not isinstance(return_type, ts.ArrayType):
             raise excs.Error(f'{name} must return an array, but returns {return_type}')
         else:
             shape = return_type.shape

pixeltable/io/__init__.py

@@ -1,10 +1,9 @@
 from .external_store import ExternalStore, SyncStatus
-from .globals import create_label_studio_project, import_rows, import_json
+from .globals import create_label_studio_project, export_images_as_fo_dataset, import_json, import_rows
 from .hf_datasets import import_huggingface_dataset
 from .pandas import import_csv, import_excel, import_pandas
 from .parquet import import_parquet
 
-
 __default_dir = set(symbol for symbol in dir() if not symbol.startswith('_'))
 __removed_symbols = {'globals', 'hf_datasets', 'pandas', 'parquet'}
 __all__ = sorted(list(__default_dir - __removed_symbols))

pixeltable/io/external_store.py

@@ -69,6 +69,9 @@ class Project(ExternalStore, abc.ABC):
     An `ExternalStore` that represents a labeling project. Extends `ExternalStore` with a few
     additional capabilities specific to such projects.
     """
+
+    stored_proxies: dict[Column, Column]
+
     def __init__(self, name: str, col_mapping: dict[Column, str], stored_proxies: Optional[dict[Column, Column]]):
         super().__init__(name)
         self._col_mapping = col_mapping
@@ -116,7 +119,7 @@ class Project(ExternalStore, abc.ABC):
             tbl_version.schema_version = tbl_version.version
             proxy_cols = [self.create_stored_proxy(tbl_version, col) for col in stored_proxies_needed]
             # Add the columns; this will also update table metadata.
-            tbl_version._add_columns(proxy_cols, conn)
+            tbl_version._add_columns(proxy_cols, conn, print_stats=False, on_error='ignore')
             # We don't need to retain `UpdateStatus` since the stored proxies are intended to be
             # invisible to the user.
             tbl_version._update_md(time.time(), conn, preceding_schema_version=preceding_schema_version)
@@ -126,7 +129,7 @@ class Project(ExternalStore, abc.ABC):
         # any *other* external store for this table.)
         deletions_needed: set[Column] = set(self.stored_proxies.values())
         for name, store in tbl_version.external_stores.items():
-            if name != self.name:
+            if isinstance(store, Project) and name != self.name:
                 deletions_needed = deletions_needed.difference(set(store.stored_proxies.values()))
         if len(deletions_needed) > 0:
             _logger.info(f'Removing stored proxies for columns: {[col.name for col in deletions_needed]}')
@@ -210,6 +213,8 @@ class Project(ExternalStore, abc.ABC):
         If validation fails, an exception will be raised. If validation succeeds, a new mapping will be returned
         in which the Pixeltable column names are resolved to the corresponding `Column` objects.
         """
+        from pixeltable import exprs
+
         is_user_specified_col_mapping = col_mapping is not None
         if col_mapping is None:
             col_mapping = {col: col for col in itertools.chain(export_cols.keys(), import_cols.keys())}
@@ -235,8 +240,9 @@ class Project(ExternalStore, abc.ABC):
                     f'Column name `{ext_col}` appears as a value in `col_mapping`, but the external store '
                     f'configuration has no column `{ext_col}`.'
                 )
-            col = table[t_col].col
-            resolved_col_mapping[col] = ext_col
+            col_ref = table[t_col]
+            assert isinstance(col_ref, exprs.ColumnRef)
+            resolved_col_mapping[col_ref.col] = ext_col
         # Validate column specs
         t_col_types = table._schema
         for t_col, ext_col in col_mapping.items():
@@ -329,7 +335,7 @@ class MockProject(Project):
     def get_import_columns(self) -> dict[str, ts.ColumnType]:
         return self.import_cols
 
-    def sync(self, t: Table, export_data: bool, import_data: bool) -> NotImplemented:
+    def sync(self, t: Table, export_data: bool, import_data: bool) -> SyncStatus:
         raise NotImplementedError()
 
     def delete(self) -> None: