pixeltable 0.2.30__py3-none-any.whl → 0.3.1__py3-none-any.whl

pixeltable/__init__.py

@@ -2,7 +2,7 @@ from .catalog import Column, InsertableTable, Table, UpdateStatus, View
 from .dataframe import DataFrame
 from .exceptions import Error
 from .exprs import RELATIVE_PATH_ROOT
-from .func import Aggregator, Function, expr_udf, uda, udf
+from .func import Aggregator, Function, expr_udf, query, uda, udf
 from .globals import (array, configure_logging, create_dir, create_snapshot, create_table, create_view, drop_dir,
                       drop_table, get_table, init, list_dirs, list_functions, list_tables, move, tool, tools)
 from .type_system import (Array, ArrayType, Audio, AudioType, Bool, BoolType, ColumnType, Document, DocumentType, Float,

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = "0.2.30"
-__version_tuple__ = (0, 2, 30)
+__version__ = "0.3.1"
+__version_tuple__ = (0, 3, 1)

pixeltable/catalog/table.py

@@ -25,13 +25,15 @@ from ..exprs import ColumnRef
 from ..utils.description_helper import DescriptionHelper
 from ..utils.filecache import FileCache
 from .column import Column
-from .globals import _ROWID_COLUMN_NAME, MediaValidation, UpdateStatus, is_system_column_name, is_valid_identifier, IfNotExistsParam
+from .globals import (_ROWID_COLUMN_NAME, IfExistsParam, IfNotExistsParam, MediaValidation, UpdateStatus,
+                      is_system_column_name, is_valid_identifier)
 from .schema_object import SchemaObject
 from .table_version import TableVersion
 from .table_version_path import TableVersionPath
 
 if TYPE_CHECKING:
     import torch.utils.data
+
     import pixeltable.plan
 
 _logger = logging.getLogger('pixeltable')
@@ -48,20 +50,6 @@ class Table(SchemaObject):
         super().__init__(id, name, dir_id)
         self._is_dropped = False
         self.__tbl_version_path = tbl_version_path
-        self.__query_scope = self.QueryScope(self)
-
-    class QueryScope:
-        __table: 'Table'
-        _queries: dict[str, pxt.func.QueryTemplateFunction]
-
-        def __init__(self, table: 'Table') -> None:
-            self.__table = table
-            self._queries = {}
-
-        def __getattr__(self, name: str) -> pxt.func.QueryTemplateFunction:
-            if name in self._queries:
-                return self._queries[name]
-            raise AttributeError(f'Table {self.__table._name!r} has no query with that name: {name!r}')
 
     @property
     def _has_dependents(self) -> bool:
@@ -138,23 +126,12 @@ class Table(SchemaObject):
             raise excs.Error(f'{self._display_name()} {self._name} has been dropped')
 
     def __getattr__(self, name: str) -> 'pxt.exprs.ColumnRef':
-        """Return a ColumnRef for the given name.
-        """
+        """Return a ColumnRef for the given name."""
         return self._tbl_version_path.get_column_ref(name)
 
-    @overload
-    def __getitem__(self, name: str) -> 'pxt.exprs.ColumnRef': ...
-
-    @overload
-    def __getitem__(self, index: Union[exprs.Expr, Sequence[exprs.Expr]]) -> 'pxt.DataFrame': ...
-
-    def __getitem__(self, index):
-        """Return a ColumnRef or QueryTemplateFunction for the given name, or a DataFrame for the given slice.
-        """
-        if isinstance(index, str):
-            return getattr(self, index)
-        else:
-            return self._df()[index]
+    def __getitem__(self, name: str) -> 'pxt.exprs.ColumnRef':
+        """Return a ColumnRef for the given name."""
+        return getattr(self, name)
 
     def list_views(self, *, recursive: bool = True) -> list[str]:
         """
@@ -184,10 +161,6 @@ class Table(SchemaObject):
         from pixeltable.plan import FromClause
         return pxt.DataFrame(FromClause(tbls=[self._tbl_version_path]))
 
-    @property
-    def queries(self) -> 'Table.QueryScope':
-        return self.__query_scope
-
     def select(self, *items: Any, **named_items: Any) -> 'pxt.DataFrame':
         """ Select columns or expressions from this table.
 
@@ -264,11 +237,6 @@ class Table(SchemaObject):
         """Return the schema (column names and column types) of this table."""
         return {c.name: c.col_type for c in self._tbl_version_path.columns()}
 
-    @property
-    def _query_names(self) -> list[str]:
-        """Return the names of the registered queries for this table."""
-        return list(self.__query_scope._queries.keys())
-
     @property
     def _base(self) -> Optional['Table']:
         """
@@ -422,28 +390,54 @@ class Table(SchemaObject):
         """
         return self._df().to_coco_dataset()
 
-    def __setitem__(self, col_name: str, spec: Union[ts.ColumnType, exprs.Expr]) -> None:
-        """
-        Adds a column to the table. This is an alternate syntax for `add_column()`; the meaning of
-
-        >>> tbl['new_col'] = pxt.Int
-
-        is exactly equivalent to
+    def _column_has_dependents(self, col: Column) -> bool:
+        """Returns True if the column has dependents, False otherwise."""
+        assert col is not None
+        assert col.name in self._schema.keys()
+        if any(c.name is not None for c in col.dependent_cols):
+            return True
+        return any(
+            col in store.get_local_columns()
+            for view in [self] + self._get_views(recursive=True)
+            for store in view._tbl_version.external_stores.values())
 
-        >>> tbl.add_column(new_col=pxt.Int)
+    def _ignore_or_drop_existing_columns(self, new_col_names: list[str], if_exists: IfExistsParam) -> list[str]:
+        """ Check and handle existing columns in the new column specification based on the if_exists parameter.
 
-        For details, see the documentation for [`add_column()`][pixeltable.catalog.Table.add_column].
+        If `if_exists='ignore'`, returns a list of existing columns, if any, in `new_col_names`.
         """
-        self._check_is_dropped()
-        if not isinstance(col_name, str):
-            raise excs.Error(f'Column name must be a string, got {type(col_name)}')
-        if not isinstance(spec, (ts.ColumnType, exprs.Expr, type, _GenericAlias)):
-            raise excs.Error(f'Column spec must be a ColumnType, Expr, or type, got {type(spec)}')
-        self.add_column(stored=None, print_stats=False, on_error='abort', **{col_name: spec})
+        assert not self.get_metadata()['is_snapshot']
+        existing_col_names = set(self._schema.keys())
+        cols_to_ignore = []
+        for new_col_name in new_col_names:
+            if new_col_name in existing_col_names:
+                if if_exists == IfExistsParam.ERROR:
+                    raise excs.Error(f'Duplicate column name: {new_col_name!r}')
+                elif if_exists == IfExistsParam.IGNORE:
+                    cols_to_ignore.append(new_col_name)
+                elif if_exists == IfExistsParam.REPLACE or if_exists == IfExistsParam.REPLACE_FORCE:
+                    if new_col_name not in self._tbl_version.cols_by_name:
+                        # for views, it is possible that the existing column
+                        # is a base table column; in that case, we should not
+                        # drop/replace that column. Continue to raise error.
+                        raise excs.Error(
+                            f'Column {new_col_name!r} is a base table column. Cannot replace it.'
+                        )
+                    col = self._tbl_version.cols_by_name[new_col_name]
+                    # cannot drop a column with dependents; so reject
+                    # replace directive if column has dependents.
+                    if self._column_has_dependents(col):
+                        raise excs.Error(
+                            f'Column {new_col_name!r} already exists and has dependents. Cannot {if_exists.name.lower()} it.'
+                        )
+                    self.drop_column(new_col_name)
+                    assert new_col_name not in self._tbl_version.cols_by_name
+        return cols_to_ignore
 
     def add_columns(
         self,
-        schema: dict[str, Union[ts.ColumnType, builtins.type, _GenericAlias]]
+        schema: dict[str, Union[ts.ColumnType, builtins.type, _GenericAlias]],
+        if_exists: Literal['error', 'ignore', 'replace', 'replace_force'] = 'error'
     ) -> UpdateStatus:
         """
         Adds multiple columns to the table. The columns must be concrete (non-computed) columns; to add computed columns,
@@ -454,12 +448,21 @@ class Table(SchemaObject):
 
         Args:
             schema: A dictionary mapping column names to types.
+            if_exists: Determines the behavior if a column already exists. Must be one of the following:
+
+                - `'error'`: an exception will be raised.
+                - `'ignore'`: do nothing and return.
+                - `'replace' or 'replace_force'`: drop the existing column and add the new column, if it has no dependents.
+
+                Note that the `if_exists` parameter is applied to all columns in the schema.
+                To apply different behaviors to different columns, please use [`add_column()`][pixeltable.Table.add_column] for each column.
 
         Returns:
             Information about the execution status of the operation.
 
         Raises:
-            Error: If any column name is invalid or already exists.
+            Error: If any column name is invalid, or already exists and `if_exists='error'`,
+                or `if_exists='replace*'` but the column has dependents or is a basetable column.
 
         Examples:
             Add multiple columns to the table `my_table`:
@@ -472,49 +475,51 @@ class Table(SchemaObject):
             ... tbl.add_columns(schema)
         """
         self._check_is_dropped()
+        if self.get_metadata()['is_snapshot']:
+            raise excs.Error('Cannot add column to a snapshot.')
         col_schema = {
             col_name: {'type': ts.ColumnType.normalize_type(spec, nullable_default=True, allow_builtin_types=False)}
             for col_name, spec in schema.items()
         }
+        # handle existing columns based on if_exists parameter
+        cols_to_ignore = self._ignore_or_drop_existing_columns(list(col_schema.keys()), IfExistsParam.validated(if_exists, 'if_exists'))
+        # if all columns to be added already exist and user asked to ignore
+        # existing columns, there's nothing to do.
+        for cname in cols_to_ignore:
+            assert cname in col_schema
+            del col_schema[cname]
+        if len(col_schema) == 0:
+            return UpdateStatus()
         new_cols = self._create_columns(col_schema)
         for new_col in new_cols:
-            self._verify_column(new_col, set(self._schema.keys()), set(self._query_names))
+            self._verify_column(new_col)
         status = self._tbl_version.add_columns(new_cols, print_stats=False, on_error='abort')
         FileCache.get().emit_eviction_warnings()
         return status
 
-    # TODO: add_column() still supports computed columns for backward-compatibility. In the future, computed columns
-    #     will be supported only through add_computed_column(). At that point, we can remove the `stored`,
-    #     `print_stats`, and `on_error` parameters, and change the method body to simply call self.add_columns(kwargs),
-    #     simplifying the code. For the time being, there's some obvious code duplication.
     def add_column(
         self,
         *,
-        stored: Optional[bool] = None,
-        print_stats: bool = False,
-        on_error: Literal['abort', 'ignore'] = 'abort',
+        if_exists: Literal['error', 'ignore', 'replace', 'replace_force'] = 'error',
         **kwargs: Union[ts.ColumnType, builtins.type, _GenericAlias, exprs.Expr]
     ) -> UpdateStatus:
         """
-        Adds a column to the table.
+        Adds an ordinary (non-computed) column to the table.
 
         Args:
             kwargs: Exactly one keyword argument of the form `col_name=col_type`.
-            stored: Whether the column is materialized and stored or computed on demand. Only valid for image columns.
-            print_stats: If `True`, print execution metrics during evaluation.
-            on_error: Determines the behavior if an error occurs while evaluating the column expression for at least one
-                row.
+            if_exists: Determines the behavior if the column already exists. Must be one of the following:
 
-                - `'abort'`: an exception will be raised and the column will not be added.
-                - `'ignore'`: execution will continue and the column will be added. Any rows
-                  with errors will have a `None` value for the column, with information about the error stored in the
-                  corresponding `tbl.col_name.errortype` and `tbl.col_name.errormsg` fields.
+                - `'error'`: an exception will be raised.
+                - `'ignore'`: do nothing and return.
+                - `'replace' or 'replace_force'`: drop the existing column and add the new column, if it has no dependents.
 
         Returns:
             Information about the execution status of the operation.
 
         Raises:
-            Error: If the column name is invalid or already exists.
+            Error: If the column name is invalid, or already exists and `if_exists='erorr'`,
+                or `if_exists='replace*'` but the column has dependents or is a basetable column.
 
         Examples:
             Add an int column:
@@ -526,29 +531,22 @@ class Table(SchemaObject):
             >>> tbl['new_col'] = pxt.Int
         """
         self._check_is_dropped()
+        # verify kwargs
+        if self._tbl_version.is_snapshot:
+            raise excs.Error('Cannot add column to a snapshot.')
         # verify kwargs and construct column schema dict
         if len(kwargs) != 1:
             raise excs.Error(
                 f'add_column() requires exactly one keyword argument of the form "col_name=col_type"; '
-                f'got {len(kwargs)} instead ({", ".join(list(kwargs.keys()))})'
+                f'got {len(kwargs)} instead ({", ".join(kwargs.keys())})'
             )
-        col_name, spec = next(iter(kwargs.items()))
-        if not is_valid_identifier(col_name):
-            raise excs.Error(f'Invalid column name: {col_name!r}')
-
-        col_schema: dict[str, Any] = {}
-        if isinstance(spec, (ts.ColumnType, builtins.type, _GenericAlias)):
-            col_schema['type'] = ts.ColumnType.normalize_type(spec, nullable_default=True, allow_builtin_types=False)
-        else:
-            col_schema['value'] = spec
-        if stored is not None:
-            col_schema['stored'] = stored
+        col_type = next(iter(kwargs.values()))
+        if not isinstance(col_type, (ts.ColumnType, type, _GenericAlias)):
+            raise excs.Error(
+                f'The argument to add_column() must be a type; did you intend to use add_computed_column() instead?'
+            )
+        return self.add_columns(kwargs, if_exists=if_exists)
 
-        new_col = self._create_columns({col_name: col_schema})[0]
-        self._verify_column(new_col, set(self._schema.keys()), set(self._query_names))
-        status = self._tbl_version.add_columns([new_col], print_stats=print_stats, on_error=on_error)
-        FileCache.get().emit_eviction_warnings()
-        return status
 
     def add_computed_column(
         self,
@@ -556,6 +554,7 @@ class Table(SchemaObject):
         stored: Optional[bool] = None,
         print_stats: bool = False,
         on_error: Literal['abort', 'ignore'] = 'abort',
+        if_exists: Literal['error', 'ignore', 'replace'] = 'error',
         **kwargs: exprs.Expr
     ) -> UpdateStatus:
         """
@@ -563,12 +562,27 @@ class Table(SchemaObject):
 
         Args:
             kwargs: Exactly one keyword argument of the form `col_name=expression`.
+            stored: Whether the column is materialized and stored or computed on demand. Only valid for image columns.
+            print_stats: If `True`, print execution metrics during evaluation.
+            on_error: Determines the behavior if an error occurs while evaluating the column expression for at least one
+                row.
+
+                - `'abort'`: an exception will be raised and the column will not be added.
+                - `'ignore'`: execution will continue and the column will be added. Any rows
+                    with errors will have a `None` value for the column, with information about the error stored in the
+                    corresponding `tbl.col_name.errortype` and `tbl.col_name.errormsg` fields.
+            if_exists: Determines the behavior if the column already exists. Must be one of the following:
+
+                - `'error'`: an exception will be raised.
+                - `'ignore'`: do nothing and return.
+                - `'replace' or 'replace_force'`: drop the existing column and add the new column, iff it has no dependents.
 
         Returns:
             Information about the execution status of the operation.
 
         Raises:
-            Error: If the column name is invalid or already exists.
+            Error: If the column name is invalid or already exists and `if_exists='error'`,
+                or `if_exists='replace*'` but the column has dependents or is a basetable column.
 
         Examples:
             For a table with an image column `frame`, add an image column `rotated` that rotates the image by
@@ -581,6 +595,8 @@ class Table(SchemaObject):
             >>> tbl.add_computed_column(rotated=tbl.frame.rotate(90), stored=False)
         """
         self._check_is_dropped()
+        if self.get_metadata()['is_snapshot']:
+            raise excs.Error('Cannot add column to a snapshot.')
         if len(kwargs) != 1:
             raise excs.Error(
                 f'add_computed_column() requires exactly one keyword argument of the form "column-name=type|value-expression"; '
@@ -594,8 +610,16 @@ class Table(SchemaObject):
         if stored is not None:
             col_schema['stored'] = stored
 
+        # handle existing columns based on if_exists parameter
+        cols_to_ignore = self._ignore_or_drop_existing_columns([col_name], IfExistsParam.validated(if_exists, 'if_exists'))
+        # if the column to add already exists and user asked to ignore
+        # exiting column, there's nothing to do.
+        if len(cols_to_ignore) != 0:
+            assert cols_to_ignore[0] == col_name
+            return UpdateStatus()
+
         new_col = self._create_columns({col_name: col_schema})[0]
-        self._verify_column(new_col, set(self._schema.keys()), set(self._query_names))
+        self._verify_column(new_col)
         status = self._tbl_version.add_columns([new_col], print_stats=print_stats, on_error=on_error)
         FileCache.get().emit_eviction_warnings()
         return status
@@ -675,18 +699,12 @@ class Table(SchemaObject):
         return columns
 
     @classmethod
-    def _verify_column(
-            cls, col: Column, existing_column_names: set[str], existing_query_names: Optional[set[str]] = None
-    ) -> None:
+    def _verify_column(cls, col: Column) -> None:
         """Check integrity of user-supplied Column and supply defaults"""
         if is_system_column_name(col.name):
             raise excs.Error(f'{col.name!r} is a reserved name in Pixeltable; please choose a different column name.')
         if not is_valid_identifier(col.name):
             raise excs.Error(f"Invalid column name: {col.name!r}")
-        if col.name in existing_column_names:
-            raise excs.Error(f'Duplicate column name: {col.name!r}')
-        if existing_query_names is not None and col.name in existing_query_names:
-            raise excs.Error(f'Column name conflicts with a registered query: {col.name!r}')
         if col.stored is False and not (col.is_computed and col.col_type.is_image_type()):
             raise excs.Error(f'Column {col.name!r}: stored={col.stored} only applies to computed image columns')
         if col.stored is False and col.has_window_fn_call():
@@ -699,7 +717,7 @@ class Table(SchemaObject):
         """Check integrity of user-supplied schema and set defaults"""
         column_names: set[str] = set()
         for col in schema:
-            cls._verify_column(col, column_names)
+            cls._verify_column(col)
             column_names.add(col.name)
 
     def __check_column_name_exists(self, column_name: str, include_bases: bool = False) -> None:
@@ -809,61 +827,108 @@ class Table(SchemaObject):
         """
         self._tbl_version.rename_column(old_name, new_name)
 
+    def _list_index_info_for_test(self) -> list[dict[str, Any]]:
+        """
+        Returns list of all the indexes on this table. Used for testing.
+
+        Returns:
+            A list of index information, each containing the index's
+            id, name, and the name of the column it indexes.
+        """
+        assert not self._is_dropped
+        index_info = []
+        for idx_name, idx in self._tbl_version.idxs_by_name.items():
+            index_info.append({
+                '_id': idx.id,
+                '_name': idx_name,
+                '_column': idx.col.name
+            })
+        return index_info
+
     def add_embedding_index(
             self, column: Union[str, ColumnRef], *, idx_name: Optional[str] = None,
+            embedding: Optional[pxt.Function] = None,
             string_embed: Optional[pxt.Function] = None, image_embed: Optional[pxt.Function] = None,
-            metric: str = 'cosine'
+            metric: str = 'cosine',
+            if_exists: Literal['error', 'ignore', 'replace', 'replace_force'] = 'error'
     ) -> None:
         """
-        Add an embedding index to the table. Once the index is added, it will be automatically kept up to data as new
+        Add an embedding index to the table. Once the index is created, it will be automatically kept up-to-date as new
         rows are inserted into the table.
 
-        Indices are currently supported only for `String` and `Image` columns. The index must specify, at
-        minimum, an embedding of the appropriate type (string or image). It may optionally specify _both_ a string
-        and image embedding (into the same vector space); in particular, this can be used to provide similarity search
-        of text over an image column.
+        To add an embedding index, one must specify, at minimum, the column to be indexed and an embedding UDF.
+        Only `String` and `Image` columns are currently supported. Here's an example that uses a
+        [CLIP embedding][pixeltable.functions.huggingface.clip] to index an image column:
+
+        >>> from pixeltable.functions.huggingface import clip
+        ... embedding_fn = clip.using(model_id='openai/clip-vit-base-patch32')
+        ... tbl.add_embedding_index(tbl.img, embedding=embedding_fn)
+
+        Once the index is created, similiarity lookups can be performed using the `similarity` pseudo-function.
+
+        >>> reference_img = PIL.Image.open('my_image.jpg')
+        ... sim = tbl.img.similarity(reference_img)
+        ... tbl.select(tbl.img, sim).order_by(sim, asc=False).limit(5)
+
+        If the embedding UDF is a multimodal embedding (supporting more than one data type), then lookups may be
+        performed using any of its supported types. In our example, CLIP supports both text and images, so we can
+        also search for images using a text description:
+
+        >>> sim = tbl.img.similarity('a picture of a train')
+        ... tbl.select(tbl.img, sim).order_by(sim, asc=False).limit(5)
 
         Args:
-            column: The name of, or reference to, the column to index; must be a `String` or `Image` column.
-            idx_name: The name of index. If not specified, a name such as `'idx0'` will be generated automatically.
-                If specified, the name must be unique for this table.
-            string_embed: A function to embed text; required if the column is a `String` column.
-            image_embed: A function to embed images; required if the column is an `Image` column.
-            metric: Distance metric to use for the index; one of `'cosine'`, `'ip'`, or `'l2'`;
-                the default is `'cosine'`.
+            column: The name of, or reference to, the column to be indexed; must be a `String` or `Image` column.
+            idx_name: An optional name for the index. If not specified, a name such as `'idx0'` will be generated
+                automatically. If specified, the name must be unique for this table.
+            embedding: The UDF to use for the embedding. Must be a UDF that accepts a single argument of type `String`
+                or `Image` (as appropriate for the column being indexed) and returns a fixed-size 1-dimensional
+                array of floats.
+            string_embed: An optional UDF to use for the string embedding component of this index.
+                Can be used in conjunction with `image_embed` to construct multimodal embeddings manually, by
+                specifying different embedding functions for different data types.
+            image_embed: An optional UDF to use for the image embedding component of this index.
+                Can be used in conjunction with `string_embed` to construct multimodal embeddings manually, by
+                specifying different embedding functions for different data types.
+            metric: Distance metric to use for the index; one of `'cosine'`, `'ip'`, or `'l2'`.
+                The default is `'cosine'`.
+            if_exists: Directive for handling an existing index with the same name. Must be one of the following:
+
+                - `'error'`: raise an error if an index with the same name already exists.
+                - `'ignore'`: do nothing if an index with the same name already exists.
+                - `'replace'` or `'replace_force'`: replace the existing index with the new one.
 
         Raises:
-            Error: If an index with that name already exists for the table, or if the specified column does not exist.
+            Error: If an index with the specified name already exists for the table and `if_exists='error'`, or if the specified column does not exist.
 
         Examples:
-            Add an index to the `img` column of the table `my_table` by column name:
+            Add an index to the `img` column of the table `my_table`:
 
-            >>> tbl = pxt.get_table('my_table')
-            ... tbl.add_embedding_index('img', image_embed=my_image_func)
+            >>> from pixeltable.functions.huggingface import clip
+            ... tbl = pxt.get_table('my_table')
+            ... embedding_fn = clip.using(model_id='openai/clip-vit-base-patch32')
+            ... tbl.add_embedding_index(tbl.img, embedding=embedding_fn)
 
-            Add an index to the `img` column of the table `my_table` by column reference:
-            >>> tbl = pxt.get_table('my_table')
-            ... tbl.add_embedding_index(tbl.img, image_embed=my_image_func)
+            Alternatively, the `img` column may be specified by name:
+
+            >>> tbl.add_embedding_index('img', embedding=embedding_fn)
 
-            Add another index to the `img` column, using the inner product as the distance metric,
-            and with a specific name; `string_embed` is also specified in order to search with text:
+            Add a second index to the `img` column, using the inner product as the distance metric,
+            and with a specific name:
 
             >>> tbl.add_embedding_index(
-            ...     'img',
-            ...     idx_name='clip_idx',
-            ...     image_embed=my_image_func,
-            ...     string_embed=my_string_func,
+            ...     tbl.img,
+            ...     idx_name='ip_idx',
+            ...     embedding=embedding_fn,
             ...     metric='ip'
             ... )
 
-            Alternatively:
+            Add an index using separately specified string and image embeddings:
 
             >>> tbl.add_embedding_index(
             ...     tbl.img,
-            ...     idx_name='clip_idx',
-            ...     image_embed=my_image_func,
-            ...     string_embed=my_string_func,
-            ...     metric='ip'
+            ...     string_embed=string_embedding_fn,
+            ...     image_embed=image_embedding_fn
             ... )
         """
         if self._tbl_version_path.is_snapshot():
@@ -877,11 +942,22 @@ class Table(SchemaObject):
             col = column.col
 
         if idx_name is not None and idx_name in self._tbl_version.idxs_by_name:
-            raise excs.Error(f'Duplicate index name: {idx_name}')
+            _if_exists = IfExistsParam.validated(if_exists, 'if_exists')
+            # An index with the same name already exists.
+            # Handle it according to if_exists.
+            if _if_exists == IfExistsParam.ERROR:
+                raise excs.Error(f'Duplicate index name: {idx_name}')
+            if not isinstance(self._tbl_version.idxs_by_name[idx_name].idx, index.EmbeddingIndex):
+                raise excs.Error(f'Index `{idx_name}` is not an embedding index. Cannot {_if_exists.name.lower()} it.')
+            if _if_exists == IfExistsParam.IGNORE:
+                return
+            assert _if_exists == IfExistsParam.REPLACE or _if_exists == IfExistsParam.REPLACE_FORCE
+            self.drop_index(idx_name=idx_name)
+            assert idx_name not in self._tbl_version.idxs_by_name
         from pixeltable.index import EmbeddingIndex
 
         # create the EmbeddingIndex instance to verify args
-        idx = EmbeddingIndex(col, metric=metric, string_embed=string_embed, image_embed=image_embed)
+        idx = EmbeddingIndex(col, metric=metric, embed=embedding, string_embed=string_embed, image_embed=image_embed)
         status = self._tbl_version.add_index(col, idx_name=idx_name, idx=idx)
         # TODO: how to deal with exceptions here? drop the index and raise?
         FileCache.get().emit_eviction_warnings()
@@ -1255,43 +1331,6 @@ class Table(SchemaObject):
             raise excs.Error('Cannot revert a snapshot')
         self._tbl_version.revert()
 
-    @overload
-    def query(self, py_fn: Callable) -> 'pxt.func.QueryTemplateFunction': ...
-
-    @overload
-    def query(
-            self, *, param_types: Optional[list[ts.ColumnType]] = None
-    ) -> Callable[[Callable], 'pxt.func.QueryTemplateFunction']: ...
-
-    def query(self, *args: Any, **kwargs: Any) -> Any:
-        def make_query_template(
-                py_fn: Callable, param_types: Optional[list[ts.ColumnType]]
-        ) -> 'pxt.func.QueryTemplateFunction':
-            if py_fn.__module__ != '__main__' and py_fn.__name__.isidentifier():
-                # this is a named function in a module
-                function_path = f'{py_fn.__module__}.{py_fn.__qualname__}'
-            else:
-                function_path = None
-            query_name = py_fn.__name__
-            if query_name in self._schema.keys():
-                raise excs.Error(f'Query name {query_name!r} conflicts with existing column')
-            if query_name in self.__query_scope._queries and function_path is not None:
-                raise excs.Error(f'Duplicate query name: {query_name!r}')
-            query_fn = pxt.func.QueryTemplateFunction.create(
-                py_fn, param_types=param_types, path=function_path, name=query_name)
-            self.__query_scope._queries[query_name] = query_fn
-            return query_fn
-
-            # TODO: verify that the inferred return type matches that of the template
-            # TODO: verify that the signature doesn't contain batched parameters
-
-        if len(args) == 1:
-            assert len(kwargs) == 0 and callable(args[0])
-            return make_query_template(args[0], None)
-        else:
-            assert len(args) == 0 and len(kwargs) == 1 and 'param_types' in kwargs
-            return lambda py_fn: make_query_template(py_fn, kwargs['param_types'])
-
     @property
     def external_stores(self) -> list[str]:
         return list(self._tbl_version.external_stores.keys())
@@ -1381,7 +1420,7 @@ class Table(SchemaObject):
         return sync_status
 
     def __dir__(self) -> list[str]:
-        return list(super().__dir__()) + list(self._schema.keys()) + self._query_names
+        return list(super().__dir__()) + list(self._schema.keys())
 
     def _ipython_key_completions_(self) -> list[str]:
-        return list(self._schema.keys()) + self._query_names
+        return list(self._schema.keys())

pixeltable/catalog/table_version.py

@@ -734,7 +734,8 @@ class TableVersion:
         if conn is None:
             with Env.get().engine.begin() as conn:
                 return self._insert(
-                    plan, conn, time.time(), print_stats=print_stats, rowids=rowids(), abort_on_exc=fail_on_exception)
+                    plan, conn, time.time(), print_stats=print_stats, rowids=rowids(),
+                    abort_on_exc=fail_on_exception)
         else:
             return self._insert(
                 plan, conn, time.time(), print_stats=print_stats, rowids=rowids(), abort_on_exc=fail_on_exception)

pixeltable/catalog/view.py

@@ -16,7 +16,7 @@ from pixeltable.iterators import ComponentIterator
 
 from .catalog import Catalog
 from .column import Column
-from .globals import _POS_COLUMN_NAME, UpdateStatus, MediaValidation
+from .globals import _POS_COLUMN_NAME, MediaValidation, UpdateStatus
 from .table import Table
 from .table_version import TableVersion
 from .table_version_path import TableVersionPath
@@ -166,13 +166,11 @@ class View(Table):
             return view
 
     @classmethod
-    def _verify_column(
-            cls, col: Column, existing_column_names: set[str], existing_query_names: Optional[set[str]] = None
-    ) -> None:
+    def _verify_column(cls, col: Column) -> None:
         # make sure that columns are nullable or have a default
         if not col.col_type.nullable and not col.is_computed:
             raise excs.Error(f'Column {col.name}: non-computed columns in views must be nullable')
-        super()._verify_column(col, existing_column_names, existing_query_names)
+        super()._verify_column(col)
 
     @classmethod
     def _get_snapshot_path(cls, tbl_version_path: TableVersionPath) -> TableVersionPath: