pixeltable 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

pixeltable/__init__.py

@@ -1,18 +1,32 @@
 from .catalog import Column, Table, InsertableTable, View
-from .client import Client
 from .dataframe import DataFrame
+from .datatransfer import Remote
+from .catalog import Column, Table, InsertableTable, View
 from .exceptions import Error, Error
 from .exprs import RELATIVE_PATH_ROOT
 from .func import Function, udf, uda, Aggregator, expr_udf
-from .type_system import \
-    ColumnType, StringType, IntType, FloatType, BoolType, TimestampType, JsonType, ArrayType, ImageType, VideoType, \
-    AudioType, DocumentType
+from .globals import *
+from .type_system import (
+    ColumnType,
+    StringType,
+    IntType,
+    FloatType,
+    BoolType,
+    TimestampType,
+    JsonType,
+    ArrayType,
+    ImageType,
+    VideoType,
+    AudioType,
+    DocumentType,
+)
 from .utils.help import help
+
 # noinspection PyUnresolvedReferences
-from . import functions
+from . import functions, io, iterators
+from .__version__ import __version__, __version_tuple__
 
 __all__ = [
-    'Client',
     'DataFrame',
     'Column',
     'Table',
@@ -39,6 +53,3 @@ __all__ = [
     'uda',
     'expr_udf',
 ]
-
-
-

pixeltable/__version__.py

@@ -0,0 +1,3 @@
+# These version placeholders will be replaced during build.
+__version__ = "0.2.7"
+__version_tuple__ = (0, 2, 7)

pixeltable/catalog/column.py

@@ -5,8 +5,8 @@ from typing import Optional, Union, Callable, Set
 
 import sqlalchemy as sql
 
-from pixeltable import exceptions as excs
-from pixeltable.type_system import ColumnType, StringType
+import pixeltable.exceptions as excs
+import pixeltable.type_system as ts
 from .globals import is_valid_identifier
 
 _logger = logging.getLogger('pixeltable')
@@ -18,11 +18,12 @@ class Column:
     table/view.
     """
     def __init__(
-            self, name: Optional[str], col_type: Optional[ColumnType] = None,
+            self, name: Optional[str], col_type: Optional[ts.ColumnType] = None,
             computed_with: Optional[Union['Expr', Callable]] = None,
             is_pk: bool = False, stored: Optional[bool] = None,
             col_id: Optional[int] = None, schema_version_add: Optional[int] = None,
-            schema_version_drop: Optional[int] = None, sa_col_type: Optional[sql.sqltypes.TypeEngine] = None
+            schema_version_drop: Optional[int] = None, sa_col_type: Optional[sql.sqltypes.TypeEngine] = None,
+            records_errors: Optional[bool] = None
     ):
         """Column constructor.
 
@@ -80,12 +81,19 @@ class Column:
         assert self.col_type is not None
 
         self.stored = stored
-        self.dependent_cols: Set[Column] = set()  # cols with value_exprs that reference us; set by TableVersion
+        self.dependent_cols: set[Column] = set()  # cols with value_exprs that reference us; set by TableVersion
         self.id = col_id
         self.is_pk = is_pk
         self.schema_version_add = schema_version_add
         self.schema_version_drop = schema_version_drop
 
+        # stored_proxy may be set later if this is a non-stored column.
+        # if col1.stored_proxy == col2, then also col1 == col2.proxy_base.
+        self.stored_proxy: Optional[Column] = None
+        self.proxy_base: Optional[Column] = None
+
+        self._records_errors = records_errors
+
         # column in the stored table for the values of this Column
         self.sa_col: Optional[sql.schema.Column] = None
         self.sa_col_type = sa_col_type
@@ -93,6 +101,7 @@ class Column:
         # computed cols also have storage columns for the exception string and type
         self.sa_errormsg_col: Optional[sql.schema.Column] = None
         self.sa_errortype_col: Optional[sql.schema.Column] = None
+
         from .table_version import TableVersion
         self.tbl: Optional[TableVersion] = None  # set by owning TableVersion
 
@@ -114,6 +123,10 @@ class Column:
         l = list(self.value_expr.subexprs(filter=lambda e: isinstance(e, exprs.FunctionCall) and e.is_window_fn_call))
         return len(l) > 0
 
+    def get_idx_info(self) -> dict[str, 'pixeltable.catalog.TableVersion.IndexInfo']:
+        assert self.tbl is not None
+        return {name: info for name, info in self.tbl.idxs_by_name.items() if info.col == self}
+
     @property
     def is_computed(self) -> bool:
         return self.compute_func is not None or self.value_expr is not None
@@ -127,6 +140,9 @@ class Column:
     @property
     def records_errors(self) -> bool:
         """True if this column also stores error information."""
+        # default: record errors for computed and media columns
+        if self._records_errors is not None:
+            return self._records_errors
         return self.is_stored and (self.is_computed or self.col_type.is_media_type())
 
     def source(self) -> None:
@@ -148,8 +164,8 @@ class Column:
             self.store_name(), self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type,
             nullable=True)
         if self.is_computed or self.col_type.is_media_type():
-            self.sa_errormsg_col = sql.Column(self.errormsg_store_name(), StringType().to_sa_type(), nullable=True)
-            self.sa_errortype_col = sql.Column(self.errortype_store_name(), StringType().to_sa_type(), nullable=True)
+            self.sa_errormsg_col = sql.Column(self.errormsg_store_name(), ts.StringType().to_sa_type(), nullable=True)
+            self.sa_errortype_col = sql.Column(self.errortype_store_name(), ts.StringType().to_sa_type(), nullable=True)
 
     def get_sa_col_type(self) -> sql.sqltypes.TypeEngine:
         return self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type

pixeltable/catalog/insertable_table.py

@@ -60,25 +60,29 @@ class InsertableTable(Table):
             return tbl
 
     @overload
-    def insert(self, rows: Iterable[Dict[str, Any]], /, print_stats: bool = False, fail_on_exception: bool = True): ...
+    def insert(
+            self, rows: Iterable[Dict[str, Any]], /, *, print_stats: bool = False, fail_on_exception: bool = True
+    ) -> UpdateStatus: ...
 
     @overload
-    def insert(self, print_stats: bool = False, fail_on_exception: bool = True, **kwargs: Any): ...
+    def insert(self, *, print_stats: bool = False, fail_on_exception: bool = True, **kwargs: Any) -> UpdateStatus: ...
 
-    def insert(self, *args, **kwargs) -> UpdateStatus:
-        """Insert rows into table.
+    def insert(
+            self, rows: Optional[Iterable[dict[str, Any]]] = None, /, *, print_stats: bool = False,
+            fail_on_exception: bool = True, **kwargs: Any
+    ) -> UpdateStatus:
+        """Inserts rows into this table. There are two mutually exclusive call patterns:
 
         To insert multiple rows at a time:
-
-        ``insert(rows: List[Dict[str, Any]], print_stats: bool = False, fail_on_exception: bool = True)``
+        ``insert(rows: Iterable[dict[str, Any]], /, *, print_stats: bool = False, fail_on_exception: bool = True)``
 
         To insert just a single row, you can use the more convenient syntax:
-        ``insert(print_stats: bool = False, fail_on_exception: bool = True, **kwargs: Any)``
+        ``insert(*, print_stats: bool = False, fail_on_exception: bool = True, **kwargs: Any)``
 
         Args:
             rows: (if inserting multiple rows) A list of rows to insert, each of which is a dictionary mapping column
                 names to values.
-            kwargs: (if inserting a single row) keyword-argument pairs representing column names and values.
+            kwargs: (if inserting a single row) Keyword-argument pairs representing column names and values.
             print_stats: If ``True``, print statistics about the cost of computed columns.
             fail_on_exception:
                 Determines how exceptions in computed columns and invalid media files (e.g., corrupt images)
@@ -102,16 +106,27 @@ class InsertableTable(Table):
 
             >>> tbl.insert(a=1, b=1, c=1)
         """
-        print_stats = kwargs.pop('print_stats', False)
-        fail_on_exception = kwargs.pop('fail_on_exception', True)
-        if len(args) > 0:
-            # There's a positional argument; this means `rows` is expressed as a
-            # list of dicts (multi-insert)
-            rows = list(args[0])
-        else:
-            # No positional argument; this means we're inserting a single row
-            # using kwargs syntax
+        # The commented code is the intended implementation, with signature (*args, **kwargs).
+        # That signature cannot be used currently, due to a present limitation in mkdocs.
+        # See: https://github.com/mkdocstrings/mkdocstrings/issues/669
+
+        # print_stats = kwargs.pop('print_stats', False)
+        # fail_on_exception = kwargs.pop('fail_on_exception', True)
+        # if len(args) > 0:
+        #     # There's a positional argument; this means `rows` is expressed as a
+        #     # list of dicts (multi-insert)
+        #     rows = list(args[0])
+        # else:
+        #     # No positional argument; this means we're inserting a single row
+        #     # using kwargs syntax
+        #     rows = [kwargs]
+
+        if rows is None:
             rows = [kwargs]
+        else:
+            rows = list(rows)
+            if len(kwargs) > 0:
+                raise excs.Error('`kwargs` cannot be specified unless `rows is None`.')
 
         if not isinstance(rows, list):
             raise excs.Error('rows must be a list of dictionaries')
@@ -185,8 +200,6 @@ class InsertableTable(Table):
             if not isinstance(where, Predicate):
                 raise excs.Error(f"'where' argument must be a Predicate, got {type(where)}")
             analysis_info = Planner.analyze(self.tbl_version_path, where)
-            if analysis_info.similarity_clause is not None:
-                raise excs.Error('nearest() cannot be used with delete()')
             # for now we require that the updated rows can be identified via SQL, rather than via a Python filter
             if analysis_info.filter is not None:
                 raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')

pixeltable/catalog/table.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
+import itertools
 import json
 import logging
 from pathlib import Path
-from typing import Union, Any, List, Dict, Optional, Callable, Set, Tuple, Iterable
+from typing import Union, Any, List, Dict, Optional, Callable, Set, Tuple, Iterable, Type
 from uuid import UUID
 
 import pandas as pd
@@ -16,6 +17,7 @@ import pixeltable.exceptions as excs
 import pixeltable.exprs as exprs
 import pixeltable.metadata.schema as schema
 import pixeltable.type_system as ts
+import pixeltable.index as index
 from .column import Column
 from .globals import is_valid_identifier, is_system_column_name, UpdateStatus
 from .schema_object import SchemaObject
@@ -97,27 +99,31 @@ class Table(SchemaObject):
         from pixeltable.dataframe import DataFrame
         return DataFrame(self.tbl_version_path).order_by(*items, asc=asc)
 
-    def collect(self) -> 'pixeltable.dataframe.DataFrameResultSet':  # type: ignore[name-defined, no-untyped-def]
-        """Return rows from this table.
-        """
+    def group_by(self, *items: 'exprs.Expr') -> 'pixeltable.dataframe.DataFrame':
+        """Return a DataFrame for this table."""
+        from pixeltable.dataframe import DataFrame
+        return DataFrame(self.tbl_version_path).group_by(*items)
+
+    def collect(self) -> 'pixeltable.dataframe.DataFrameResultSet':
+        """Return rows from this table."""
         return self.df().collect()
 
     def show(
             self, *args, **kwargs
-    ) -> 'pixeltable.dataframe.DataFrameResultSet':  # type: ignore[name-defined, no-untyped-def]
+    ) -> 'pixeltable.dataframe.DataFrameResultSet':
         """Return rows from this table.
         """
         return self.df().show(*args, **kwargs)
 
     def head(
             self, *args, **kwargs
-    ) -> 'pixeltable.dataframe.DataFrameResultSet':  # type: ignore[name-defined, no-untyped-def]
+    ) -> 'pixeltable.dataframe.DataFrameResultSet':
         """Return the first n rows inserted into this table."""
         return self.df().head(*args, **kwargs)
 
     def tail(
             self, *args, **kwargs
-    ) -> 'pixeltable.dataframe.DataFrameResultSet':  # type: ignore[name-defined, no-untyped-def]
+    ) -> 'pixeltable.dataframe.DataFrameResultSet':
         """Return the last n rows inserted into this table."""
         return self.df().tail(*args, **kwargs)
 
@@ -470,13 +476,16 @@ class Table(SchemaObject):
 
     def add_embedding_index(
             self, col_name: str, *, idx_name: Optional[str] = None,
-            text_embed: Optional[pixeltable.Function] = None, img_embed: Optional[pixeltable.Function] = None
+            text_embed: Optional[pixeltable.Function] = None, img_embed: Optional[pixeltable.Function] = None,
+            metric: str = 'cosine'
     ) -> None:
         """Add an index to the table.
         Args:
             col_name: name of column to index
             idx_name: name of index, which needs to be unique for the table; if not provided, a name will be generated
-            idx_type: type of index (one of 'embedding')
+            text_embed: function to embed text; required if the column is a text column
+            img_embed: function to embed images; required if the column is an image column
+            metric: distance metric to use for the index; one of 'cosine', 'ip', 'l2'; default is 'cosine'
 
         Raises:
             Error: If an index with that name already exists for the table or if the column does not exist.
@@ -484,11 +493,13 @@ class Table(SchemaObject):
         Examples:
             Add an index to the ``img`` column:
 
-            >>> tbl.add_embedding_index('img', text_embed=...)
+            >>> tbl.add_embedding_index('img', img_embed=...)
 
-            Add another index to the ``img`` column, with a specific name:
+            Add another index to the ``img`` column, using the inner product as the distance metric,
+            and with a specific name; ``text_embed`` is also specified in order to search with text:
 
-            >>> tbl.add_embedding_index('img', idx_name='clip_idx', text_embed=...)
+            >>> tbl.add_embedding_index(
+                'img', idx_name='clip_idx', img_embed=..., text_embed=...text_embed..., metric='ip')
         """
         if self.tbl_version_path.is_snapshot():
             raise excs.Error('Cannot add an index to a snapshot')
@@ -500,10 +511,28 @@ class Table(SchemaObject):
             raise excs.Error(f'Duplicate index name: {idx_name}')
         from pixeltable.index import EmbeddingIndex
         # create the EmbeddingIndex instance to verify args
-        idx = EmbeddingIndex(col, text_embed=text_embed, img_embed=img_embed)
+        idx = EmbeddingIndex(col, metric=metric, text_embed=text_embed, img_embed=img_embed)
         status = self.tbl_version_path.tbl_version.add_index(col, idx_name=idx_name, idx=idx)
         # TODO: how to deal with exceptions here? drop the index and raise?
 
+    def drop_embedding_index(self, *, column_name: Optional[str] = None, idx_name: Optional[str] = None) -> None:
+        """Drop an embedding index from the table.
+
+        Args:
+            column_name: The name of the column whose embedding index to drop. Invalid if the column has multiple
+              embedding indices.
+            idx_name: The name of the index to drop.
+
+        Raises:
+            Error: If the index does not exist.
+
+        Examples:
+            Drop embedding index on the ``img`` column:
+
+            >>> tbl.drop_embedding_index(column_name='img')
+        """
+        self._drop_index(column_name=column_name, idx_name=idx_name, _idx_class=index.EmbeddingIndex)
+
     def drop_index(self, *, column_name: Optional[str] = None, idx_name: Optional[str] = None) -> None:
         """Drop an index from the table.
 
@@ -519,6 +548,12 @@ class Table(SchemaObject):
 
             >>> tbl.drop_index(column_name='img')
         """
+        self._drop_index(column_name=column_name, idx_name=idx_name)
+
+    def _drop_index(
+            self, *, column_name: Optional[str] = None, idx_name: Optional[str] = None,
+            _idx_class: Optional[Type[index.IndexBase]] = None
+    ) -> None:
         if self.tbl_version_path.is_snapshot():
             raise excs.Error('Cannot drop an index from a snapshot')
         self._check_is_dropped()
@@ -537,12 +572,14 @@ class Table(SchemaObject):
             if col.tbl.id != tbl_version.id:
                 raise excs.Error(
                     f'Column {column_name}: cannot drop index from column that belongs to base ({col.tbl.name})')
-            idx_ids = [info.id for info in tbl_version.idxs_by_name.values() if info.col.id == col.id]
-            if len(idx_ids) == 0:
+            idx_info = [info for info in tbl_version.idxs_by_name.values() if info.col.id == col.id]
+            if _idx_class is not None:
+                idx_info = [info for info in idx_info if isinstance(info.idx, _idx_class)]
+            if len(idx_info) == 0:
                 raise excs.Error(f'Column {column_name} does not have an index')
-            if len(idx_ids) > 1:
+            if len(idx_info) > 1:
                 raise excs.Error(f'Column {column_name} has multiple indices; specify idx_name instead')
-            idx_id = idx_ids[0]
+            idx_id = idx_info[0].id
         self.tbl_version_path.tbl_version.drop_index(idx_id)
 
     def update(
@@ -582,8 +619,6 @@ class Table(SchemaObject):
             if not isinstance(where, exprs.Predicate):
                 raise excs.Error(f"'where' argument must be a Predicate, got {type(where)}")
             analysis_info = Planner.analyze(self.tbl_version_path, where)
-            if analysis_info.similarity_clause is not None:
-                raise excs.Error('nearest() cannot be used with update()')
             # for now we require that the updated rows can be identified via SQL, rather than via a Python filter
             if analysis_info.filter is not None:
                 raise excs.Error(f'Filter {analysis_info.filter} not expressible in SQL')
@@ -674,7 +709,6 @@ class Table(SchemaObject):
 
         return update_targets
 
-
     def revert(self) -> None:
         """Reverts the table to the previous version.
 
@@ -685,3 +719,159 @@ class Table(SchemaObject):
             raise excs.Error('Cannot revert a snapshot')
         self._check_is_dropped()
         self.tbl_version_path.tbl_version.revert()
+
+    def _link(
+            self,
+            remote: 'pixeltable.datatransfer.Remote',
+            col_mapping: Optional[dict[str, str]] = None
+    ) -> None:
+        """
+        Links the specified `Remote` to this table. Once a remote is linked, it can be synchronized with
+        this `Table` by calling [`Table.sync()`]. A record of the link
+        is stored in table metadata and will persist across sessions.
+
+        Args:
+            remote (pixeltable.datatransfer.Remote): The `Remote` to link to this table.
+            col_mapping: An optional mapping of columns from this `Table` to columns in the `Remote`.
+        """
+        # TODO(aaron-siegel): Refactor `col_mapping`
+        self._check_is_dropped()
+        if remote in self._get_remotes():
+            raise excs.Error(f'That remote is already linked to table `{self.get_name()}`: {remote}')
+        push_cols = remote.get_export_columns()
+        pull_cols = remote.get_import_columns()
+        is_col_mapping_user_specified = col_mapping is not None
+        if col_mapping is None:
+            # Use the identity mapping by default if `col_mapping` is not specified
+            col_mapping = {col: col for col in itertools.chain(push_cols.keys(), pull_cols.keys())}
+        self._validate_remote(push_cols, pull_cols, col_mapping, is_col_mapping_user_specified)
+        _logger.info(f'Linking remote {remote} to table `{self.get_name()}`.')
+        self.tbl_version_path.tbl_version.link(remote, col_mapping)
+        print(f'Linked remote {remote} to table `{self.get_name()}`.')
+
+    def unlink(
+            self,
+            remotes: Optional['pixeltable.datatransfer.Remote' | list['pixeltable.datatransfer.Remote']] = None,
+            *,
+            delete_remote_data: bool = False,
+            ignore_errors: bool = False
+    ) -> None:
+        """
+        Unlinks this table's `Remote`s.
+
+        Args:
+            remotes: If specified, will unlink only the specified `Remote` or list of `Remote`s. If not specified,
+                will unlink all of this table's `Remote`s.
+            ignore_errors (bool): If `True`, no exception will be thrown if the specified `Remote` is not linked
+                to this table.
+            delete_remote_data (bool): If `True`, then the remote data source will also be deleted. WARNING: This
+                is a destructive operation that will delete data outside Pixeltable, and cannot be undone.
+
+        """
+        self._check_is_dropped()
+        all_remotes = self._get_remotes()
+
+        if remotes is None:
+            remotes = list(all_remotes.keys())
+        elif isinstance(remotes, pixeltable.datatransfer.Remote):
+            remotes = [remotes]
+
+        # Validation
+        if not ignore_errors:
+            for remote in remotes:
+                if remote not in all_remotes:
+                    raise excs.Error(f'Remote {remote} is not linked to table `{self.get_name()}`')
+
+        for remote in remotes:
+            self.tbl_version_path.tbl_version.unlink(remote)
+            print(f'Unlinked remote {remote} from table `{self.get_name()}`.')
+            if delete_remote_data:
+                remote.delete()
+
+    def _validate_remote(
+            self,
+            export_cols: dict[str, ts.ColumnType],
+            import_cols: dict[str, ts.ColumnType],
+            col_mapping: Optional[dict[str, str]],
+            is_col_mapping_user_specified: bool
+    ):
+        # Validate names
+        t_cols = self.column_names()
+        for t_col, r_col in col_mapping.items():
+            if t_col not in t_cols:
+                if is_col_mapping_user_specified:
+                    raise excs.Error(
+                        f'Column name `{t_col}` appears as a key in `col_mapping`, but Table `{self.get_name()}` '
+                        'contains no such column.'
+                    )
+                else:
+                    raise excs.Error(
+                        f'Column `{t_col}` does not exist in Table `{self.get_name()}`. Either add a column `{t_col}`, '
+                        f'or specify a `col_mapping` to associate a different column with the remote field `{r_col}`.'
+                    )
+            if r_col not in export_cols and r_col not in import_cols:
+                raise excs.Error(
+                    f'Column name `{r_col}` appears as a value in `col_mapping`, but the remote '
+                    f'configuration has no column `{r_col}`.'
+                )
+        # Validate column specs
+        t_col_types = self.column_types()
+        for t_col, r_col in col_mapping.items():
+            t_col_type = t_col_types[t_col]
+            if r_col in export_cols:
+                # Validate that the table column can be assigned to the remote column
+                r_col_type = export_cols[r_col]
+                if not r_col_type.is_supertype_of(t_col_type):
+                    raise excs.Error(
+                        f'Column `{t_col}` cannot be exported to remote column `{r_col}` (incompatible types; expecting `{r_col_type}`)'
+                    )
+            if r_col in import_cols:
+                # Validate that the remote column can be assigned to the table column
+                if self.tbl_version_path.get_column(t_col).is_computed:
+                    raise excs.Error(
+                        f'Column `{t_col}` is a computed column, which cannot be populated from a remote column'
+                    )
+                r_col_type = import_cols[r_col]
+                if not t_col_type.is_supertype_of(r_col_type):
+                    raise excs.Error(
+                        f'Column `{t_col}` cannot be imported from remote column `{r_col}` (incompatible types; expecting `{r_col_type}`)'
+                    )
+
+    def _get_remotes(self) -> dict[pixeltable.datatransfer.Remote, dict[str, str]]:
+        """
+        Gets a `dict` of all `Remote`s linked to this table.
+        """
+        return self.tbl_version_path.tbl_version.get_remotes()
+
+    def sync(
+            self,
+            *,
+            export_data: bool = True,
+            import_data: bool = True
+    ):
+        """
+        Synchronizes this table with its linked `Remote`s.
+
+        Args:
+            export_data: If `True`, data from this table will be exported to the external store during synchronization.
+            import_data: If `True`, data from the external store will be imported to this table during synchronization.
+        """
+        remotes = self._get_remotes()
+        assert len(remotes) <= 1
+
+        # Validation
+        for remote in remotes:
+            col_mapping = remotes[remote]
+            r_cols = set(col_mapping.values())
+            # Validate export/import
+            if export_data and not any(col in r_cols for col in remote.get_export_columns()):
+                raise excs.Error(
+                    f'Attempted to sync with export_data=True, but there are no columns to export: {remote}'
+                )
+            if import_data and not any(col in r_cols for col in remote.get_import_columns()):
+                raise excs.Error(
+                    f'Attempted to sync with import_data=True, but there are no columns to import: {remote}'
+                )
+
+        for remote in remotes:
+            remote.sync(self, remotes[remote], export_data=export_data, import_data=import_data)