pixeltable 0.2.7__tar.gz → 0.2.8__tar.gz

{pixeltable-0.2.7 → pixeltable-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.7
+Version: 0.2.8
 Summary: Pixeltable: The Multimodal AI Data Plane
 Author: Marcel Kornacker
 Author-email: marcelk@gmail.com
@@ -21,7 +21,7 @@ Requires-Dist: more-itertools (>=10.2,<11.0)
 Requires-Dist: numpy (>=1.25)
 Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
 Requires-Dist: pandas (>=2.0,<3.0)
-Requires-Dist: pgserver (==0.1.3)
+Requires-Dist: pgserver (==0.1.4)
 Requires-Dist: pgvector (>=0.2.1,<0.3.0)
 Requires-Dist: pillow (>=9.3.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)

{pixeltable-0.2.7 → pixeltable-0.2.8}/pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = "0.2.7"
-__version_tuple__ = (0, 2, 7)
+__version__ = "0.2.8"
+__version_tuple__ = (0, 2, 8)

{pixeltable-0.2.7 → pixeltable-0.2.8}/pixeltable/catalog/column.py

@@ -87,11 +87,6 @@ class Column:
         self.schema_version_add = schema_version_add
         self.schema_version_drop = schema_version_drop
 
-        # stored_proxy may be set later if this is a non-stored column.
-        # if col1.stored_proxy == col2, then also col1 == col2.proxy_base.
-        self.stored_proxy: Optional[Column] = None
-        self.proxy_base: Optional[Column] = None
-
         self._records_errors = records_errors
 
         # column in the stored table for the values of this Column
@@ -101,7 +96,6 @@ class Column:
         # computed cols also have storage columns for the exception string and type
         self.sa_errormsg_col: Optional[sql.schema.Column] = None
         self.sa_errortype_col: Optional[sql.schema.Column] = None
-
         from .table_version import TableVersion
         self.tbl: Optional[TableVersion] = None  # set by owning TableVersion
 

{pixeltable-0.2.7 → pixeltable-0.2.8}/pixeltable/catalog/table.py

@@ -735,58 +735,31 @@ class Table(SchemaObject):
             col_mapping: An optional mapping of columns from this `Table` to columns in the `Remote`.
         """
         # TODO(aaron-siegel): Refactor `col_mapping`
+        if len(self._get_remotes()) > 0:
+            raise excs.Error('Linking more than one `Remote` to a table is not currently supported.')
         self._check_is_dropped()
-        if remote in self._get_remotes():
-            raise excs.Error(f'That remote is already linked to table `{self.get_name()}`: {remote}')
-        push_cols = remote.get_export_columns()
-        pull_cols = remote.get_import_columns()
+        export_cols = remote.get_export_columns()
+        import_cols = remote.get_import_columns()
         is_col_mapping_user_specified = col_mapping is not None
         if col_mapping is None:
             # Use the identity mapping by default if `col_mapping` is not specified
-            col_mapping = {col: col for col in itertools.chain(push_cols.keys(), pull_cols.keys())}
-        self._validate_remote(push_cols, pull_cols, col_mapping, is_col_mapping_user_specified)
-        _logger.info(f'Linking remote {remote} to table `{self.get_name()}`.')
+            col_mapping = {col: col for col in itertools.chain(export_cols.keys(), import_cols.keys())}
+        self._validate_remote(export_cols, import_cols, col_mapping, is_col_mapping_user_specified)
         self.tbl_version_path.tbl_version.link(remote, col_mapping)
         print(f'Linked remote {remote} to table `{self.get_name()}`.')
 
-    def unlink(
-            self,
-            remotes: Optional['pixeltable.datatransfer.Remote' | list['pixeltable.datatransfer.Remote']] = None,
-            *,
-            delete_remote_data: bool = False,
-            ignore_errors: bool = False
-    ) -> None:
+    def unlink(self) -> None:
         """
         Unlinks this table's `Remote`s.
-
-        Args:
-            remotes: If specified, will unlink only the specified `Remote` or list of `Remote`s. If not specified,
-                will unlink all of this table's `Remote`s.
-            ignore_errors (bool): If `True`, no exception will be thrown if the specified `Remote` is not linked
-                to this table.
-            delete_remote_data (bool): If `True`, then the remote data source will also be deleted. WARNING: This
-                is a destructive operation that will delete data outside Pixeltable, and cannot be undone.
-
         """
         self._check_is_dropped()
-        all_remotes = self._get_remotes()
-
-        if remotes is None:
-            remotes = list(all_remotes.keys())
-        elif isinstance(remotes, pixeltable.datatransfer.Remote):
-            remotes = [remotes]
-
-        # Validation
-        if not ignore_errors:
-            for remote in remotes:
-                if remote not in all_remotes:
-                    raise excs.Error(f'Remote {remote} is not linked to table `{self.get_name()}`')
+        remotes = self._get_remotes()
+        assert len(remotes) <= 1
 
-        for remote in remotes:
-            self.tbl_version_path.tbl_version.unlink(remote)
-            print(f'Unlinked remote {remote} from table `{self.get_name()}`.')
-            if delete_remote_data:
-                remote.delete()
+        remote = next(iter(remotes.keys()))
+        self.tbl_version_path.tbl_version.unlink(remote)
+        # TODO: Provide an option to auto-delete the project
+        print(f'Unlinked remote {remote} from table `{self.get_name()}`.')
 
     def _validate_remote(
             self,
@@ -823,7 +796,7 @@ class Table(SchemaObject):
                 r_col_type = export_cols[r_col]
                 if not r_col_type.is_supertype_of(t_col_type):
                     raise excs.Error(
-                        f'Column `{t_col}` cannot be exported to remote column `{r_col}` (incompatible types; expecting `{r_col_type}`)'
+                        f'Column `{t_col}` cannot be exported to remote column `{r_col}` (incompatible types)'
                     )
             if r_col in import_cols:
                 # Validate that the remote column can be assigned to the table column
@@ -834,7 +807,7 @@ class Table(SchemaObject):
                 r_col_type = import_cols[r_col]
                 if not t_col_type.is_supertype_of(r_col_type):
                     raise excs.Error(
-                        f'Column `{t_col}` cannot be imported from remote column `{r_col}` (incompatible types; expecting `{r_col_type}`)'
+                        f'Column `{t_col}` cannot be imported from remote column `{r_col}` (incompatible types)'
                     )
 
     def _get_remotes(self) -> dict[pixeltable.datatransfer.Remote, dict[str, str]]:

{pixeltable-0.2.7 → pixeltable-0.2.8}/pixeltable/catalog/table_version.py

@@ -24,7 +24,6 @@ from pixeltable.utils.filecache import FileCache
 from pixeltable.utils.media_store import MediaStore
 from .column import Column
 from .globals import UpdateStatus, POS_COLUMN_NAME, is_valid_identifier
-from ..func.globals import resolve_symbol
 
 _logger = logging.getLogger('pixeltable')
 
@@ -121,7 +120,7 @@ class TableVersion:
         # init schema after we determined whether we're a component view, and before we create the store table
         self.cols: list[Column] = []  # contains complete history of columns, incl dropped ones
         self.cols_by_name: dict[str, Column] = {}  # contains only user-facing (named) columns visible in this version
-        self.cols_by_id: dict[int, Column] = {}  # contains only columns visible in this version, both system and user
+        self.cols_by_id: dict[int, Column] = {}  # contains only columns visible in this version
         self.idx_md = tbl_md.index_md  # needed for _create_tbl_md()
         self.idxs_by_name: dict[str, TableVersion.IndexInfo] = {}  # contains only actively maintained indices
         self._init_schema(tbl_md, schema_version_md)
@@ -269,16 +268,6 @@ class TableVersion:
                 col.value_expr = exprs.Expr.from_dict(col_md.value_expr)
                 self._record_value_expr(col)
 
-            # if this is a stored proxy column, resolve the relationships with its proxy base.
-            if col_md.proxy_base is not None:
-                # proxy_base must have a strictly smaller id, so we must already have encountered it
-                # in traversal order; and if the proxy column is active at this version, then the
-                # proxy base must necessarily be active as well. This motivates the following assertion.
-                assert col_md.proxy_base in self.cols_by_id
-                base_col = self.cols_by_id[col_md.proxy_base]
-                base_col.stored_proxy = col
-                col.proxy_base = base_col
-
     def _init_idxs(self, tbl_md: schema.TableMd) -> None:
         self.idx_md = tbl_md.index_md
         self.idxs_by_name = {}
@@ -544,16 +533,8 @@ class TableVersion:
         dependent_user_cols = [c for c in col.dependent_cols if c.name is not None]
         if len(dependent_user_cols) > 0:
             raise excs.Error(
-                f'Cannot drop column `{name}` because the following columns depend on it:\n'
-                f'{", ".join(c.name for c in dependent_user_cols)}'
-            )
-        dependent_remotes = [remote for remote, col_mapping in self.remotes.items() if name in col_mapping]
-        if len(dependent_remotes) > 0:
-            raise excs.Error(
-                f'Cannot drop column `{name}` because the following remotes depend on it:\n'
-                f'{", ".join(str(r) for r in dependent_remotes)}'
-            )
-        assert col.stored_proxy is None  # since there are no dependent remotes
+                f'Cannot drop column {name} because the following columns depend on it:\n',
+                f'{", ".join([c.name for c in dependent_user_cols])}')
 
         # we're creating a new schema version
         self.version += 1
@@ -964,88 +945,26 @@ class TableVersion:
 
     @classmethod
     def _init_remote(cls, remote_md: dict[str, Any]) -> Tuple[pixeltable.datatransfer.Remote, dict[str, str]]:
-        remote_cls = resolve_symbol(remote_md['class'])
-        assert isinstance(remote_cls, type) and issubclass(remote_cls, pixeltable.datatransfer.Remote)
+        module = importlib.import_module(remote_md['module'])
+        remote_cls = getattr(module, remote_md['class'])
         remote = remote_cls.from_dict(remote_md['remote_md'])
         col_mapping = remote_md['col_mapping']
         return remote, col_mapping
 
     def link(self, remote: pixeltable.datatransfer.Remote, col_mapping: dict[str, str]) -> None:
-        # All of the media columns being linked need to either be stored, computed columns or have stored proxies.
-        # This ensures that the media in those columns resides in the media cache, where it can be served.
-        # First determine which columns (if any) need stored proxies, but don't have one yet.
-        cols_by_name = self.path.cols_by_name()  # Includes base columns
-        stored_proxies_needed = []
-        for col_name in col_mapping.keys():
-            col = cols_by_name[col_name]
-            if col.col_type.is_media_type() and not (col.is_stored and col.compute_func) and not col.stored_proxy:
-                stored_proxies_needed.append(col)
+        timestamp = time.time()
+        self.version += 1
+        self.remotes[remote] = col_mapping
         with Env.get().engine.begin() as conn:
-            self.version += 1
-            self.remotes[remote] = col_mapping
-            preceding_schema_version = None
-            if len(stored_proxies_needed) > 0:
-                _logger.info(f'Creating stored proxies for columns: {[col.name for col in stored_proxies_needed]}')
-                # Create stored proxies for columns that need one. Increment the schema version
-                # accordingly.
-                preceding_schema_version = self.schema_version
-                self.schema_version = self.version
-                proxy_cols = [self.create_stored_proxy(col) for col in stored_proxies_needed]
-                # Add the columns; this will also update table metadata.
-                # TODO Add to base tables
-                self._add_columns(proxy_cols, conn)
-                # We don't need to retain `UpdateStatus` since the stored proxies are intended to be
-                # invisible to the user.
-            self._update_md(time.time(), preceding_schema_version, conn)
-
-    def create_stored_proxy(self, col: Column) -> Column:
-        from pixeltable import exprs
-
-        assert col.col_type.is_media_type() and not (col.is_stored and col.compute_func) and not col.stored_proxy
-        proxy_col = Column(
-            name=None,
-            computed_with=exprs.ColumnRef(col).apply(lambda x: x, col_type=col.col_type),
-            stored=True,
-            col_id=self.next_col_id,
-            sa_col_type=col.col_type.to_sa_type(),
-            schema_version_add=self.schema_version
-        )
-        proxy_col.tbl = self
-        self.next_col_id += 1
-        col.stored_proxy = proxy_col
-        proxy_col.proxy_base = col
-        return proxy_col
+            self._update_md(timestamp, None, conn)
 
     def unlink(self, remote: pixeltable.datatransfer.Remote) -> None:
         assert remote in self.remotes
         timestamp = time.time()
-        this_remote_col_names = list(self.remotes[remote].keys())
-        other_remote_col_names = {
-            col_name
-            for other_remote, col_mapping in self.remotes.items() if other_remote != remote
-            for col_name in col_mapping.keys()
-        }
-        cols_by_name = self.path.cols_by_name()  # Includes base columns
-        stored_proxy_deletions_needed = [
-            cols_by_name[col_name]
-            for col_name in this_remote_col_names
-            if col_name not in other_remote_col_names and cols_by_name[col_name].stored_proxy
-        ]
+        self.version += 1
+        del self.remotes[remote]
         with Env.get().engine.begin() as conn:
-            self.version += 1
-            del self.remotes[remote]
-            preceding_schema_version = None
-            if len(stored_proxy_deletions_needed) > 0:
-                preceding_schema_version = self.schema_version
-                self.schema_version = self.version
-                proxy_cols = [col.stored_proxy for col in stored_proxy_deletions_needed]
-                for col in stored_proxy_deletions_needed:
-                    assert col.stored_proxy is not None and col.stored_proxy.proxy_base == col
-                    col.stored_proxy.proxy_base = None
-                    col.stored_proxy = None
-                # TODO Drop from base tables
-                self._drop_columns(proxy_cols)
-            self._update_md(timestamp, preceding_schema_version, conn)
+            self._update_md(timestamp, None, conn)
 
     def get_remotes(self) -> dict[pixeltable.datatransfer.Remote, dict[str, str]]:
         return self.remotes
@@ -1151,15 +1070,15 @@ class TableVersion:
             column_md[col.id] = schema.ColumnMd(
                 id=col.id, col_type=col.col_type.as_dict(), is_pk=col.is_pk,
                 schema_version_add=col.schema_version_add, schema_version_drop=col.schema_version_drop,
-                value_expr=value_expr_dict, stored=col.stored,
-                proxy_base=col.proxy_base.id if col.proxy_base else None)
+                value_expr=value_expr_dict, stored=col.stored)
         return column_md
 
     @classmethod
     def _create_remotes_md(cls, remotes: dict['pixeltable.datatransfer.Remote', dict[str, str]]) -> list[dict[str, Any]]:
         return [
             {
-                'class': f'{type(remote).__module__}.{type(remote).__qualname__}',
+                'module': type(remote).__module__,
+                'class': type(remote).__qualname__,
                 'remote_md': remote.to_dict(),
                 'col_mapping': col_mapping
             }

{pixeltable-0.2.7 → pixeltable-0.2.8}/pixeltable/catalog/table_version_path.py

@@ -106,14 +106,9 @@ class TableVersionPath:
         if self.base is not None:
             base_cols = self.base.columns()
             # we only include base columns that don't conflict with one of our column names
-            result.extend(c for c in base_cols if c.name not in self.tbl_version.cols_by_name)
+            result.extend([c for c in base_cols if c.name not in self.tbl_version.cols_by_name])
         return result
 
-    def cols_by_name(self) -> dict[str, Column]:
-        """Return a dict of all user columns visible in this tbl version path, including columns from bases"""
-        cols = self.columns()
-        return {col.name: col for col in cols}
-
     def get_column(self, name: str, include_bases: bool = True) -> Optional[Column]:
         """Return the column with the given name, or None if not found"""
         col = self.tbl_version.cols_by_name.get(name)