pixeltable 0.2.7__py3-none-any.whl → 0.2.8__py3-none-any.whl

pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = "0.2.7"
-__version_tuple__ = (0, 2, 7)
+__version__ = "0.2.8"
+__version_tuple__ = (0, 2, 8)

pixeltable/catalog/column.py

@@ -87,11 +87,6 @@ class Column:
         self.schema_version_add = schema_version_add
         self.schema_version_drop = schema_version_drop
 
-        # stored_proxy may be set later if this is a non-stored column.
-        # if col1.stored_proxy == col2, then also col1 == col2.proxy_base.
-        self.stored_proxy: Optional[Column] = None
-        self.proxy_base: Optional[Column] = None
-
         self._records_errors = records_errors
 
         # column in the stored table for the values of this Column
@@ -101,7 +96,6 @@ class Column:
         # computed cols also have storage columns for the exception string and type
         self.sa_errormsg_col: Optional[sql.schema.Column] = None
         self.sa_errortype_col: Optional[sql.schema.Column] = None
-
         from .table_version import TableVersion
         self.tbl: Optional[TableVersion] = None  # set by owning TableVersion
 

pixeltable/catalog/table.py

@@ -735,58 +735,31 @@ class Table(SchemaObject):
             col_mapping: An optional mapping of columns from this `Table` to columns in the `Remote`.
         """
         # TODO(aaron-siegel): Refactor `col_mapping`
+        if len(self._get_remotes()) > 0:
+            raise excs.Error('Linking more than one `Remote` to a table is not currently supported.')
         self._check_is_dropped()
-        if remote in self._get_remotes():
-            raise excs.Error(f'That remote is already linked to table `{self.get_name()}`: {remote}')
-        push_cols = remote.get_export_columns()
-        pull_cols = remote.get_import_columns()
+        export_cols = remote.get_export_columns()
+        import_cols = remote.get_import_columns()
         is_col_mapping_user_specified = col_mapping is not None
         if col_mapping is None:
             # Use the identity mapping by default if `col_mapping` is not specified
-            col_mapping = {col: col for col in itertools.chain(push_cols.keys(), pull_cols.keys())}
-        self._validate_remote(push_cols, pull_cols, col_mapping, is_col_mapping_user_specified)
-        _logger.info(f'Linking remote {remote} to table `{self.get_name()}`.')
+            col_mapping = {col: col for col in itertools.chain(export_cols.keys(), import_cols.keys())}
+        self._validate_remote(export_cols, import_cols, col_mapping, is_col_mapping_user_specified)
         self.tbl_version_path.tbl_version.link(remote, col_mapping)
         print(f'Linked remote {remote} to table `{self.get_name()}`.')
 
-    def unlink(
-            self,
-            remotes: Optional['pixeltable.datatransfer.Remote' | list['pixeltable.datatransfer.Remote']] = None,
-            *,
-            delete_remote_data: bool = False,
-            ignore_errors: bool = False
-    ) -> None:
+    def unlink(self) -> None:
         """
         Unlinks this table's `Remote`s.
-
-        Args:
-            remotes: If specified, will unlink only the specified `Remote` or list of `Remote`s. If not specified,
-                will unlink all of this table's `Remote`s.
-            ignore_errors (bool): If `True`, no exception will be thrown if the specified `Remote` is not linked
-                to this table.
-            delete_remote_data (bool): If `True`, then the remote data source will also be deleted. WARNING: This
-                is a destructive operation that will delete data outside Pixeltable, and cannot be undone.
-
         """
         self._check_is_dropped()
-        all_remotes = self._get_remotes()
-
-        if remotes is None:
-            remotes = list(all_remotes.keys())
-        elif isinstance(remotes, pixeltable.datatransfer.Remote):
-            remotes = [remotes]
-
-        # Validation
-        if not ignore_errors:
-            for remote in remotes:
-                if remote not in all_remotes:
-                    raise excs.Error(f'Remote {remote} is not linked to table `{self.get_name()}`')
+        remotes = self._get_remotes()
+        assert len(remotes) <= 1
 
-        for remote in remotes:
-            self.tbl_version_path.tbl_version.unlink(remote)
-            print(f'Unlinked remote {remote} from table `{self.get_name()}`.')
-            if delete_remote_data:
-                remote.delete()
+        remote = next(iter(remotes.keys()))
+        self.tbl_version_path.tbl_version.unlink(remote)
+        # TODO: Provide an option to auto-delete the project
+        print(f'Unlinked remote {remote} from table `{self.get_name()}`.')
 
     def _validate_remote(
             self,
@@ -823,7 +796,7 @@ class Table(SchemaObject):
                 r_col_type = export_cols[r_col]
                 if not r_col_type.is_supertype_of(t_col_type):
                     raise excs.Error(
-                        f'Column `{t_col}` cannot be exported to remote column `{r_col}` (incompatible types; expecting `{r_col_type}`)'
+                        f'Column `{t_col}` cannot be exported to remote column `{r_col}` (incompatible types)'
                     )
             if r_col in import_cols:
                 # Validate that the remote column can be assigned to the table column
@@ -834,7 +807,7 @@ class Table(SchemaObject):
                 r_col_type = import_cols[r_col]
                 if not t_col_type.is_supertype_of(r_col_type):
                     raise excs.Error(
-                        f'Column `{t_col}` cannot be imported from remote column `{r_col}` (incompatible types; expecting `{r_col_type}`)'
+                        f'Column `{t_col}` cannot be imported from remote column `{r_col}` (incompatible types)'
                     )
 
     def _get_remotes(self) -> dict[pixeltable.datatransfer.Remote, dict[str, str]]:

pixeltable/catalog/table_version.py

@@ -24,7 +24,6 @@ from pixeltable.utils.filecache import FileCache
 from pixeltable.utils.media_store import MediaStore
 from .column import Column
 from .globals import UpdateStatus, POS_COLUMN_NAME, is_valid_identifier
-from ..func.globals import resolve_symbol
 
 _logger = logging.getLogger('pixeltable')
 
@@ -121,7 +120,7 @@ class TableVersion:
         # init schema after we determined whether we're a component view, and before we create the store table
         self.cols: list[Column] = []  # contains complete history of columns, incl dropped ones
         self.cols_by_name: dict[str, Column] = {}  # contains only user-facing (named) columns visible in this version
-        self.cols_by_id: dict[int, Column] = {}  # contains only columns visible in this version, both system and user
+        self.cols_by_id: dict[int, Column] = {}  # contains only columns visible in this version
         self.idx_md = tbl_md.index_md  # needed for _create_tbl_md()
         self.idxs_by_name: dict[str, TableVersion.IndexInfo] = {}  # contains only actively maintained indices
         self._init_schema(tbl_md, schema_version_md)
@@ -269,16 +268,6 @@ class TableVersion:
                 col.value_expr = exprs.Expr.from_dict(col_md.value_expr)
                 self._record_value_expr(col)
 
-            # if this is a stored proxy column, resolve the relationships with its proxy base.
-            if col_md.proxy_base is not None:
-                # proxy_base must have a strictly smaller id, so we must already have encountered it
-                # in traversal order; and if the proxy column is active at this version, then the
-                # proxy base must necessarily be active as well. This motivates the following assertion.
-                assert col_md.proxy_base in self.cols_by_id
-                base_col = self.cols_by_id[col_md.proxy_base]
-                base_col.stored_proxy = col
-                col.proxy_base = base_col
-
     def _init_idxs(self, tbl_md: schema.TableMd) -> None:
         self.idx_md = tbl_md.index_md
         self.idxs_by_name = {}
@@ -544,16 +533,8 @@ class TableVersion:
         dependent_user_cols = [c for c in col.dependent_cols if c.name is not None]
         if len(dependent_user_cols) > 0:
             raise excs.Error(
-                f'Cannot drop column `{name}` because the following columns depend on it:\n'
-                f'{", ".join(c.name for c in dependent_user_cols)}'
-            )
-        dependent_remotes = [remote for remote, col_mapping in self.remotes.items() if name in col_mapping]
-        if len(dependent_remotes) > 0:
-            raise excs.Error(
-                f'Cannot drop column `{name}` because the following remotes depend on it:\n'
-                f'{", ".join(str(r) for r in dependent_remotes)}'
-            )
-        assert col.stored_proxy is None  # since there are no dependent remotes
+                f'Cannot drop column {name} because the following columns depend on it:\n',
+                f'{", ".join([c.name for c in dependent_user_cols])}')
 
         # we're creating a new schema version
         self.version += 1
@@ -964,88 +945,26 @@ class TableVersion:
 
     @classmethod
     def _init_remote(cls, remote_md: dict[str, Any]) -> Tuple[pixeltable.datatransfer.Remote, dict[str, str]]:
-        remote_cls = resolve_symbol(remote_md['class'])
-        assert isinstance(remote_cls, type) and issubclass(remote_cls, pixeltable.datatransfer.Remote)
+        module = importlib.import_module(remote_md['module'])
+        remote_cls = getattr(module, remote_md['class'])
         remote = remote_cls.from_dict(remote_md['remote_md'])
         col_mapping = remote_md['col_mapping']
         return remote, col_mapping
 
     def link(self, remote: pixeltable.datatransfer.Remote, col_mapping: dict[str, str]) -> None:
-        # All of the media columns being linked need to either be stored, computed columns or have stored proxies.
-        # This ensures that the media in those columns resides in the media cache, where it can be served.
-        # First determine which columns (if any) need stored proxies, but don't have one yet.
-        cols_by_name = self.path.cols_by_name()  # Includes base columns
-        stored_proxies_needed = []
-        for col_name in col_mapping.keys():
-            col = cols_by_name[col_name]
-            if col.col_type.is_media_type() and not (col.is_stored and col.compute_func) and not col.stored_proxy:
-                stored_proxies_needed.append(col)
+        timestamp = time.time()
+        self.version += 1
+        self.remotes[remote] = col_mapping
         with Env.get().engine.begin() as conn:
-            self.version += 1
-            self.remotes[remote] = col_mapping
-            preceding_schema_version = None
-            if len(stored_proxies_needed) > 0:
-                _logger.info(f'Creating stored proxies for columns: {[col.name for col in stored_proxies_needed]}')
-                # Create stored proxies for columns that need one. Increment the schema version
-                # accordingly.
-                preceding_schema_version = self.schema_version
-                self.schema_version = self.version
-                proxy_cols = [self.create_stored_proxy(col) for col in stored_proxies_needed]
-                # Add the columns; this will also update table metadata.
-                # TODO Add to base tables
-                self._add_columns(proxy_cols, conn)
-                # We don't need to retain `UpdateStatus` since the stored proxies are intended to be
-                # invisible to the user.
-            self._update_md(time.time(), preceding_schema_version, conn)
-
-    def create_stored_proxy(self, col: Column) -> Column:
-        from pixeltable import exprs
-
-        assert col.col_type.is_media_type() and not (col.is_stored and col.compute_func) and not col.stored_proxy
-        proxy_col = Column(
-            name=None,
-            computed_with=exprs.ColumnRef(col).apply(lambda x: x, col_type=col.col_type),
-            stored=True,
-            col_id=self.next_col_id,
-            sa_col_type=col.col_type.to_sa_type(),
-            schema_version_add=self.schema_version
-        )
-        proxy_col.tbl = self
-        self.next_col_id += 1
-        col.stored_proxy = proxy_col
-        proxy_col.proxy_base = col
-        return proxy_col
+            self._update_md(timestamp, None, conn)
 
     def unlink(self, remote: pixeltable.datatransfer.Remote) -> None:
         assert remote in self.remotes
         timestamp = time.time()
-        this_remote_col_names = list(self.remotes[remote].keys())
-        other_remote_col_names = {
-            col_name
-            for other_remote, col_mapping in self.remotes.items() if other_remote != remote
-            for col_name in col_mapping.keys()
-        }
-        cols_by_name = self.path.cols_by_name()  # Includes base columns
-        stored_proxy_deletions_needed = [
-            cols_by_name[col_name]
-            for col_name in this_remote_col_names
-            if col_name not in other_remote_col_names and cols_by_name[col_name].stored_proxy
-        ]
+        self.version += 1
+        del self.remotes[remote]
         with Env.get().engine.begin() as conn:
-            self.version += 1
-            del self.remotes[remote]
-            preceding_schema_version = None
-            if len(stored_proxy_deletions_needed) > 0:
-                preceding_schema_version = self.schema_version
-                self.schema_version = self.version
-                proxy_cols = [col.stored_proxy for col in stored_proxy_deletions_needed]
-                for col in stored_proxy_deletions_needed:
-                    assert col.stored_proxy is not None and col.stored_proxy.proxy_base == col
-                    col.stored_proxy.proxy_base = None
-                    col.stored_proxy = None
-                # TODO Drop from base tables
-                self._drop_columns(proxy_cols)
-            self._update_md(timestamp, preceding_schema_version, conn)
+            self._update_md(timestamp, None, conn)
 
     def get_remotes(self) -> dict[pixeltable.datatransfer.Remote, dict[str, str]]:
         return self.remotes
@@ -1151,15 +1070,15 @@ class TableVersion:
             column_md[col.id] = schema.ColumnMd(
                 id=col.id, col_type=col.col_type.as_dict(), is_pk=col.is_pk,
                 schema_version_add=col.schema_version_add, schema_version_drop=col.schema_version_drop,
-                value_expr=value_expr_dict, stored=col.stored,
-                proxy_base=col.proxy_base.id if col.proxy_base else None)
+                value_expr=value_expr_dict, stored=col.stored)
         return column_md
 
     @classmethod
     def _create_remotes_md(cls, remotes: dict['pixeltable.datatransfer.Remote', dict[str, str]]) -> list[dict[str, Any]]:
         return [
             {
-                'class': f'{type(remote).__module__}.{type(remote).__qualname__}',
+                'module': type(remote).__module__,
+                'class': type(remote).__qualname__,
                 'remote_md': remote.to_dict(),
                 'col_mapping': col_mapping
             }

pixeltable/catalog/table_version_path.py

@@ -106,14 +106,9 @@ class TableVersionPath:
         if self.base is not None:
             base_cols = self.base.columns()
             # we only include base columns that don't conflict with one of our column names
-            result.extend(c for c in base_cols if c.name not in self.tbl_version.cols_by_name)
+            result.extend([c for c in base_cols if c.name not in self.tbl_version.cols_by_name])
         return result
 
-    def cols_by_name(self) -> dict[str, Column]:
-        """Return a dict of all user columns visible in this tbl version path, including columns from bases"""
-        cols = self.columns()
-        return {col.name: col for col in cols}
-
     def get_column(self, name: str, include_bases: bool = True) -> Optional[Column]:
         """Return the column with the given name, or None if not found"""
         col = self.tbl_version.cols_by_name.get(name)

pixeltable/datatransfer/label_studio.py

@@ -1,9 +1,8 @@
-import json
 import logging
 import os
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Iterator, Optional, Literal
+from typing import Any, Iterator, Optional
 from xml.etree import ElementTree
 
 import PIL.Image
@@ -16,7 +15,6 @@ import pixeltable.env as env
 import pixeltable.exceptions as excs
 from pixeltable import Table
 from pixeltable.datatransfer.remote import Remote
-from pixeltable.exprs import ColumnRef, DataRow
 from pixeltable.utils import coco
 
 _logger = logging.getLogger('pixeltable')
@@ -43,55 +41,26 @@ class LabelStudioProject(Remote):
     """
     # TODO(aaron-siegel): Add link in docstring to a Label Studio howto
 
-    def __init__(self, project_id: int, media_import_method: Literal['post', 'file']):
+    def __init__(self, project_id: int):
         self.project_id = project_id
-        self.media_import_method = media_import_method
         self._project: Optional[label_studio_sdk.project.Project] = None
 
     @classmethod
-    def create(cls, title: str, label_config: str, media_import_method: Literal['post', 'file'] = 'file', **kwargs: Any) -> 'LabelStudioProject':
+    def create(cls, title: str, label_config: str, **kwargs: Any) -> 'LabelStudioProject':
         """
         Creates a new Label Studio project, using the Label Studio client configured in Pixeltable.
 
         Args:
             title: The title of the project.
             label_config: The Label Studio project configuration, in XML format.
-            media_import_method: The method to use when importing media columns to Label Studio:
-                - `file`: Media will be sent to Label Studio as a file on the local filesystem. This method can be
-                    used if Pixeltable and Label Studio are running on the same host.
-                - `post`: Media will be sent to Label Studio via HTTP post. This should generally only be used for
-                    prototyping; due to restrictions in Label Studio, it can only be used with projects that have
-                    just one data field.
             **kwargs: Additional keyword arguments for the new project; these will be passed to `start_project`
                 in the Label Studio SDK.
         """
-        # TODO(aaron-siegel): Add media_import_method = 'url' as an option
         # Check that the config is valid before creating the project
-        config = cls.__parse_project_config(label_config)
-        if media_import_method == 'post' and len(config.data_keys) > 1:
-            raise excs.Error('`media_import_method` cannot be `post` if there is more than one data key')
-
+        cls._parse_project_config(label_config)
         project = _label_studio_client().start_project(title=title, label_config=label_config, **kwargs)
-
-        if media_import_method == 'file':
-            # We need to set up a local storage connection to receive media files
-            os.environ['LABEL_STUDIO_LOCAL_FILES_DOCUMENT_ROOT'] = str(env.Env.get().home)
-            try:
-                project.connect_local_import_storage(local_store_path=str(env.Env.get().media_dir))
-            except HTTPError as exc:
-                if exc.errno == 400:
-                    response: dict = json.loads(exc.response.text)
-                    if 'validation_errors' in response and 'non_field_errors' in response['validation_errors'] \
-                            and 'LOCAL_FILES_SERVING_ENABLED' in response['validation_errors']['non_field_errors'][0]:
-                        raise excs.Error(
-                            '`media_import_method` is set to `file`, but your Label Studio server is not configured '
-                            'for local file storage.\nPlease set the `LABEL_STUDIO_LOCAL_FILES_SERVING_ENABLED` '
-                            'environment variable to `true` in the environment where your Label Studio server is running.'
-                        ) from exc
-                raise  # Handle any other exception type normally
-
         project_id = project.get_params()['id']
-        return LabelStudioProject(project_id, media_import_method)
+        return LabelStudioProject(project_id)
 
     @property
     def project(self) -> label_studio_sdk.project.Project:
@@ -115,14 +84,14 @@ class LabelStudioProject(Remote):
         return self.project_params['title']
 
     @property
-    def __project_config(self) -> '_LabelStudioConfig':
-        return self.__parse_project_config(self.project_params['label_config'])
+    def _project_config(self) -> '_LabelStudioConfig':
+        return self._parse_project_config(self.project_params['label_config'])
 
     def get_export_columns(self) -> dict[str, pxt.ColumnType]:
         """
         The data keys and preannotation fields specified in this Label Studio project.
         """
-        return self.__project_config.export_columns
+        return self._project_config.export_columns
 
     def get_import_columns(self) -> dict[str, pxt.ColumnType]:
         """
@@ -138,13 +107,13 @@ class LabelStudioProject(Remote):
         _logger.info(f'Syncing Label Studio project "{self.project_title}" with table `{t.get_name()}`'
                      f' (export: {export_data}, import: {import_data}).')
         # Collect all existing tasks into a dict with entries `rowid: task`
-        tasks = {tuple(task['meta']['rowid']): task for task in self.__fetch_all_tasks()}
+        tasks = {tuple(task['meta']['rowid']): task for task in self._fetch_all_tasks()}
         if export_data:
-            self.__update_tasks(t, col_mapping, tasks)
+            self._create_tasks_from_table(t, col_mapping, tasks)
         if import_data:
-            self.__update_table_from_tasks(t, col_mapping, tasks)
+            self._update_table_from_tasks(t, col_mapping, tasks)
 
-    def __fetch_all_tasks(self) -> Iterator[dict]:
+    def _fetch_all_tasks(self) -> Iterator[dict]:
         page = 1
         unknown_task_count = 0
         while True:
@@ -163,10 +132,32 @@ class LabelStudioProject(Remote):
                 f'Skipped {unknown_task_count} unrecognized task(s) when syncing Label Studio project "{self.project_title}".'
             )
 
-    def __update_tasks(self, t: Table, col_mapping: dict[str, str], existing_tasks: dict[tuple, dict]) -> None:
+    def _update_table_from_tasks(self, t: Table, col_mapping: dict[str, str], tasks: dict[tuple, dict]) -> None:
+        # `col_mapping` is guaranteed to be a one-to-one dict whose values are a superset
+        # of `get_import_columns`
+        assert ANNOTATIONS_COLUMN in col_mapping.values()
+        annotations_column = next(k for k, v in col_mapping.items() if v == ANNOTATIONS_COLUMN)
+        updates = [
+            {
+                '_rowid': task['meta']['rowid'],
+                # Replace [] by None to indicate no annotations. We do want to sync rows with no annotations,
+                # in order to properly handle the scenario where existing annotations have been deleted in
+                # Label Studio.
+                annotations_column: task[ANNOTATIONS_COLUMN] if len(task[ANNOTATIONS_COLUMN]) > 0 else None
+            }
+            for task in tasks.values()
+        ]
+        if len(updates) > 0:
+            _logger.info(
+                f'Updating table `{t.get_name()}`, column `{annotations_column}` with {len(updates)} total annotations.'
+            )
+            t.batch_update(updates)
+            annotations_count = sum(len(task[ANNOTATIONS_COLUMN]) for task in tasks.values())
+            print(f'Synced {annotations_count} annotation(s) from {len(updates)} existing task(s) in {self}.')
 
+    def _create_tasks_from_table(self, t: Table, col_mapping: dict[str, str], existing_tasks: dict[tuple, dict]) -> None:
         t_col_types = t.column_types()
-        config = self.__project_config
+        config = self._project_config
 
         # Columns in `t` that map to Label Studio data keys
         t_data_cols = [
@@ -187,23 +178,24 @@ class LabelStudioProject(Remote):
         _logger.debug('`t_rl_cols`: %s', t_rl_cols)
         _logger.debug('`rl_info`: %s', rl_info)
 
-        if self.media_import_method == 'post':
-            # Send media to Label Studio by HTTP post.
-            self.__update_tasks_by_post(t, col_mapping, existing_tasks, t_data_cols[0], t_rl_cols, rl_info)
-        elif self.media_import_method == 'file':
-            # Send media to Label Studio by local file transfer.
-            self.__update_tasks_by_files(t, col_mapping, existing_tasks, t_data_cols, t_rl_cols, rl_info)
+        if len(t_data_cols) == 1 and t_col_types[t_data_cols[0]].is_media_type():
+            # With a single media column, we can post local files to Label Studio using
+            # the file transfer API.
+            self._create_tasks_by_post(t, col_mapping, existing_tasks, t_rl_cols, rl_info, t_data_cols[0])
         else:
-            assert False
+            # Either a single non-media column or multiple columns. Either way, we can't
+            # use the file upload API and need to rely on externally accessible URLs for
+            # media columns.
+            self._create_tasks_by_urls(t, col_mapping, existing_tasks, t_data_cols, t_col_types, t_rl_cols, rl_info)
 
-    def __update_tasks_by_post(
+    def _create_tasks_by_post(
             self,
             t: Table,
             col_mapping: dict[str, str],
             existing_tasks: dict[tuple, dict],
-            media_col_name: str,
             t_rl_cols: list[str],
-            rl_info: list['_RectangleLabel']
+            rl_info: list['_RectangleLabel'],
+            media_col_name: str
     ) -> None:
         is_stored = t[media_col_name].col.is_stored
         # If it's a stored column, we can use `localpath`
@@ -233,13 +225,13 @@ class LabelStudioProject(Remote):
                     os.remove(file)
 
                 # Update the task with `rowid` metadata
-                self.project.update_task(task_id, meta={'rowid': row.rowid, 'v_min': row.v_min})
+                self.project.update_task(task_id, meta={'rowid': row.rowid})
 
                 # Convert coco annotations to predictions
                 coco_annotations = [row.vals[i] for i in rl_col_idxs]
                 _logger.debug('`coco_annotations`: %s', coco_annotations)
                 predictions = [
-                    self.__coco_to_predictions(
+                    self._coco_to_predictions(
                         coco_annotations[i], col_mapping[t_rl_cols[i]], rl_info[i], task_id=task_id
                     )
                     for i in range(len(coco_annotations))
@@ -250,95 +242,67 @@ class LabelStudioProject(Remote):
 
         print(f'Created {tasks_created} new task(s) in {self}.')
 
-        self.__delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
+        self._delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
 
-    def __update_tasks_by_files(
+    def _create_tasks_by_urls(
             self,
             t: Table,
             col_mapping: dict[str, str],
             existing_tasks: dict[tuple, dict],
             t_data_cols: list[str],
+            t_col_types: dict[str, pxt.ColumnType],
             t_rl_cols: list[str],
             rl_info: list['_RectangleLabel']
-    ) -> None:
+    ):
+        # TODO(aaron-siegel): This is just a placeholder (implementation is not complete or tested!)
+        selection = [
+            t[col_name].fileurl if t_col_types[col_name].is_media_type() else t[col_name]
+            for col_name in t_data_cols
+        ]
         r_data_cols = [col_mapping[col_name] for col_name in t_data_cols]
-        col_refs = {}
-        for col_name in t_data_cols:
-            if not t[col_name].col_type.is_media_type():
-                # Not a media column; query the data directly
-                col_refs[col_name] = t[col_name]
-            elif t[col_name].col.stored_proxy:
-                # Media column that has a stored proxy; use it. We have to give it a name,
-                # since it's an anonymous column
-                col_refs[f'{col_name}_proxy'] = ColumnRef(t[col_name].col.stored_proxy).localpath
-            else:
-                # Media column without a stored proxy; this means it's a stored computed column,
-                # and we can just use the localpath
-                col_refs[col_name] = t[col_name].localpath
-
-        df = t.select(*[t[col] for col in t_rl_cols], **col_refs)
-        rl_col_idxs: Optional[list[int]] = None  # We have to wait until we begin iterating to populate these
-        data_col_idxs: Optional[list[int]] = None
-
-        row_ids_in_pxt: set[tuple] = set()
+        rows = t.select(*selection, *[t[col] for col in t_rl_cols])
+        new_rows = filter(lambda row: row.rowid not in existing_tasks, rows._exec())
         tasks_created = 0
-        tasks_updated = 0
-        page = []
-
-        def create_task_info(row: DataRow) -> dict:
-            data_vals = [row.vals[idx] for idx in data_col_idxs]
-            coco_annotations = [row.vals[idx] for idx in rl_col_idxs]
-            # For media columns, we need to transform the paths into Label Studio's bespoke path format
-            for i in range(len(t_data_cols)):
-                if t[t_data_cols[i]].col_type.is_media_type():
-                    data_vals[i] = self.__localpath_to_lspath(data_vals[i])
-            predictions = [
-                self.__coco_to_predictions(coco_annotations[i], col_mapping[t_rl_cols[i]], rl_info[i])
-                for i in range(len(coco_annotations))
-            ]
-            return {
-                'data': dict(zip(r_data_cols, data_vals)),
-                'meta': {'rowid': row.rowid, 'v_min': row.v_min},
-                'predictions': predictions
-            }
+        row_ids_in_pxt: set[tuple] = set()
 
-        for row in df._exec():
-            if rl_col_idxs is None:
-                rl_col_idxs = [expr.slot_idx for expr in df._select_list_exprs[:len(t_rl_cols)]]
-                data_col_idxs = [expr.slot_idx for expr in df._select_list_exprs[len(t_rl_cols):]]
-            row_ids_in_pxt.add(row.rowid)
-            if row.rowid in existing_tasks:
-                # A task for this row already exists; see if it needs an update.
-                # Get the v_min record from task metadata. Default to 0 if no v_min record is found
-                old_v_min = int(existing_tasks[row.rowid]['meta'].get('v_min', 0))
-                print(f'{old_v_min}  {row.v_min}')
-                if row.v_min > old_v_min:
-                    _logger.debug(f'Updating task for rowid {row.rowid} ({row.v_min} > {old_v_min}).')
-                    task_info = create_task_info(row)
-                    self.project.update_task(existing_tasks[row.rowid]['id'], **task_info)
-                    tasks_updated += 1
-            else:
-                # No task exists for this row; we need to create one.
-                page.append(create_task_info(row))
-                tasks_created += 1
-                if len(page) == _PAGE_SIZE:
-                    self.project.import_tasks(page)
-                    page.clear()
+        for page in more_itertools.batched(new_rows, n=_PAGE_SIZE):
+            data_col_idxs = [expr.slot_idx for expr in rows._select_list_exprs[:len(t_data_cols)]]
+            rl_col_idxs = [expr.slot_idx for expr in rows._select_list_exprs[len(t_data_cols):]]
+            tasks = []
 
-        if len(page) > 0:
-            self.project.import_tasks(page)
+            for row in page:
+                row_ids_in_pxt.add(row.rowid)
+                data_vals = [row.vals[i] for i in data_col_idxs]
+                coco_annotations = [row.vals[i] for i in rl_col_idxs]
+                predictions = [
+                    self._coco_to_predictions(coco_annotations[i], col_mapping[t_rl_cols[i]], rl_info[i])
+                    for i in range(len(coco_annotations))
+                ]
 
-        print(f'Created {tasks_created} new task(s) and updated {tasks_updated} existing task(s) in {self}.')
+                # Validate media columns
+                # TODO Support this if label studio is running on localhost?
+                for i in range(len(data_vals)):
+                    if t[t_data_cols[i]].col_type.is_media_type() and data_vals[i].startswith("file://"):
+                        raise excs.Error(
+                            'Cannot use locally stored media files in a `LabelStudioProject` with more than one '
+                            'data key. (This is a limitation of Label Studio; see warning here: '
+                            'https://labelstud.io/guide/tasks.html)'
+                        )
 
-        self.__delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
+                tasks.append({
+                    'data': zip(r_data_cols, data_vals),
+                    'meta': {'rowid': row.rowid},
+                    'predictions': predictions
+                })
 
-    @classmethod
-    def __localpath_to_lspath(self, localpath: str) -> str:
-        assert isinstance(localpath, str)
-        relpath = Path(localpath).relative_to(env.Env.get().home)
-        return f'/data/local-files/?d={str(relpath)}'
+            self.project.import_tasks(tasks)
+            tasks_created += len(tasks)
+
+        print(f'Created {tasks_created} new task(s) in {self}.')
+
+        self._delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
 
-    def __delete_stale_tasks(self, existing_tasks: dict[tuple, dict], row_ids_in_pxt: set[tuple], tasks_created: int):
+    def _delete_stale_tasks(self, existing_tasks: dict[tuple, dict], row_ids_in_pxt: set[tuple], tasks_created: int):
         tasks_to_delete = [
             task['id'] for rowid, task in existing_tasks.items()
             if rowid not in row_ids_in_pxt
@@ -350,42 +314,19 @@ class LabelStudioProject(Remote):
             self.project.delete_tasks(tasks_to_delete)
             print(f'Deleted {len(tasks_to_delete)} tasks(s) in {self} that are no longer present in Pixeltable.')
 
-    def __update_table_from_tasks(self, t: Table, col_mapping: dict[str, str], tasks: dict[tuple, dict]) -> None:
-        # `col_mapping` is guaranteed to be a one-to-one dict whose values are a superset
-        # of `get_pull_columns`
-        assert ANNOTATIONS_COLUMN in col_mapping.values()
-        annotations_column = next(k for k, v in col_mapping.items() if v == ANNOTATIONS_COLUMN)
-        updates = [
-            {
-                '_rowid': task['meta']['rowid'],
-                # Replace [] by None to indicate no annotations. We do want to sync rows with no annotations,
-                # in order to properly handle the scenario where existing annotations have been deleted in
-                # Label Studio.
-                annotations_column: task[ANNOTATIONS_COLUMN] if len(task[ANNOTATIONS_COLUMN]) > 0 else None
-            }
-            for task in tasks.values()
-        ]
-        if len(updates) > 0:
-            _logger.info(
-                f'Updating table `{t.get_name()}`, column `{annotations_column}` with {len(updates)} total annotations.'
-            )
-            t.batch_update(updates)
-            annotations_count = sum(len(task[ANNOTATIONS_COLUMN]) for task in tasks.values())
-            print(f'Synced {annotations_count} annotation(s) from {len(updates)} existing task(s) in {self}.')
-
     def to_dict(self) -> dict[str, Any]:
-        return {'project_id': self.project_id, 'media_import_method': self.media_import_method}
+        return {'project_id': self.project_id}
 
     @classmethod
     def from_dict(cls, md: dict[str, Any]) -> 'LabelStudioProject':
-        return LabelStudioProject(md['project_id'], md['media_import_method'])
+        return LabelStudioProject(md['project_id'])
 
     def __repr__(self) -> str:
         name = self.project.get_params()['title']
         return f'LabelStudioProject `{name}`'
 
     @classmethod
-    def __parse_project_config(cls, xml_config: str) -> '_LabelStudioConfig':
+    def _parse_project_config(cls, xml_config: str) -> '_LabelStudioConfig':
         """
         Parses a Label Studio XML config, extracting the names and Pixeltable types of
         all input variables.
@@ -394,27 +335,28 @@ class LabelStudioProject(Remote):
         if root.tag.lower() != 'view':
             raise excs.Error('Root of Label Studio config must be a `View`')
         config = _LabelStudioConfig(
-            data_keys=dict(cls.__parse_data_keys_config(root)),
-            rectangle_labels=dict(cls.__parse_rectangle_labels_config(root))
+            data_keys=dict(cls._parse_data_keys_config(root)),
+            rectangle_labels=dict(cls._parse_rectangle_labels_config(root))
         )
         config.validate()
         return config
 
     @classmethod
-    def __parse_data_keys_config(cls, root: ElementTree.Element) -> Iterator[tuple[str, '_DataKey']]:
+    def _parse_data_keys_config(cls, root: ElementTree.Element) -> Iterator[tuple[str, '_DataKey']]:
         for element in root:
             if 'value' in element.attrib and element.attrib['value'][0] == '$':
                 remote_col_name = element.attrib['value'][1:]
-                data_key_name = element.attrib.get('name')
+                if 'name' not in element.attrib:
+                    raise excs.Error(f'Data key is missing `name` attribute: `{remote_col_name}`')
                 element_type = _LS_TAG_MAP.get(element.tag.lower())
                 if element_type is None:
                     raise excs.Error(
                         f'Unsupported Label Studio data type: `{element.tag}` (in data key `{remote_col_name}`)'
                     )
-                yield remote_col_name, _DataKey(data_key_name, element_type)
+                yield remote_col_name, _DataKey(element.attrib['name'], element_type)
 
     @classmethod
-    def __parse_rectangle_labels_config(cls, root: ElementTree.Element) -> Iterator[tuple[str, '_RectangleLabel']]:
+    def _parse_rectangle_labels_config(cls, root: ElementTree.Element) -> Iterator[tuple[str, '_RectangleLabel']]:
         for element in root:
             if element.tag.lower() == 'rectanglelabels':
                 name = element.attrib['name']
@@ -429,7 +371,7 @@ class LabelStudioProject(Remote):
                 yield name, _RectangleLabel(to_name=to_name, labels=labels)
 
     @classmethod
-    def __coco_to_predictions(
+    def _coco_to_predictions(
             cls,
             coco_annotations: dict[str, Any],
             from_name: str,
@@ -466,25 +408,10 @@ class LabelStudioProject(Remote):
         else:
             return {'result': result}
 
-    def delete(self) -> None:
-        """
-        Deletes this Label Studio project. This will remove all data and annotations
-        associated with this project in Label Studio.
-        """
-        title = self.project_title
-        _label_studio_client().delete_project(self.project_id)
-        print(f'Deleted Label Studio project: {title}')
-
-    def __eq__(self, other) -> bool:
-        return isinstance(other, LabelStudioProject) and self.project_id == other.project_id
-
-    def __hash__(self) -> int:
-        return hash(self.project_id)
-
 
 @dataclass(frozen=True)
 class _DataKey:
-    name: Optional[str]  # The 'name' attribute of the data key; may differ from the field name
+    name: str  # The 'name' attribute of the data key; may differ from the field name
     column_type: pxt.ColumnType
 
 
@@ -500,7 +427,7 @@ class _LabelStudioConfig:
     rectangle_labels: dict[str, _RectangleLabel]
 
     def validate(self) -> None:
-        data_key_names = set(key.name for key in self.data_keys.values() if key is not None)
+        data_key_names = set(key.name for key in self.data_keys.values())
         for name, rl in self.rectangle_labels.items():
             if rl.to_name not in data_key_names:
                 raise excs.Error(
@@ -510,7 +437,7 @@ class _LabelStudioConfig:
 
     @property
     def export_columns(self) -> dict[str, pxt.ColumnType]:
-        data_key_cols = {key_id: key_info.column_type for key_id, key_info in self.data_keys.items()}
+        data_key_cols = {key_name: key_info.column_type for key_name, key_info in self.data_keys.items()}
         rl_cols = {name: pxt.JsonType() for name in self.rectangle_labels.keys()}
         return {**data_key_cols, **rl_cols}
 
@@ -518,7 +445,6 @@ class _LabelStudioConfig:
 ANNOTATIONS_COLUMN = 'annotations'
 _PAGE_SIZE = 100  # This is the default used in the LS SDK
 _LS_TAG_MAP = {
-    'header': pxt.StringType(),
     'text': pxt.StringType(),
     'image': pxt.ImageType(),
     'video': pxt.VideoType(),

pixeltable/datatransfer/remote.py

@@ -45,12 +45,6 @@ class Remote(abc.ABC):
             import_data: If `True`, data from this table will be imported from the remote during synchronization.
         """
 
-    @abc.abstractmethod
-    def delete(self) -> None:
-        """
-        Deletes this `Remote`.
-        """
-
     @abc.abstractmethod
     def to_dict(self) -> dict[str, Any]: ...
 
@@ -62,11 +56,9 @@ class Remote(abc.ABC):
 # A remote that cannot be synced, used mainly for testing.
 class MockRemote(Remote):
 
-    def __init__(self, name: str, export_cols: dict[str, ts.ColumnType], import_cols: dict[str, ts.ColumnType]):
-        self.name = name
+    def __init__(self, export_cols: dict[str, ts.ColumnType], import_cols: dict[str, ts.ColumnType]):
         self.export_cols = export_cols
         self.import_cols = import_cols
-        self.__is_deleted = False
 
     def get_export_columns(self) -> dict[str, ts.ColumnType]:
         return self.export_cols
@@ -77,17 +69,9 @@ class MockRemote(Remote):
     def sync(self, t: Table, col_mapping: dict[str, str], export_data: bool, import_data: bool) -> NotImplemented:
         raise NotImplementedError()
 
-    def delete(self) -> None:
-        self.__is_deleted = True
-
-    @property
-    def is_deleted(self) -> bool:
-        return self.__is_deleted
-
     def to_dict(self) -> dict[str, Any]:
         return {
             # TODO Change in next schema version
-            'name': self.name,
             'push_cols': {k: v.as_dict() for k, v in self.export_cols.items()},
             'pull_cols': {k: v.as_dict() for k, v in self.import_cols.items()}
         }
@@ -95,19 +79,7 @@ class MockRemote(Remote):
     @classmethod
     def from_dict(cls, md: dict[str, Any]) -> Remote:
         return cls(
-            name=md['name'],
             # TODO Change in next schema version
-            export_cols={k: ts.ColumnType.from_dict(v) for k, v in md['push_cols'].items()},
-            import_cols={k: ts.ColumnType.from_dict(v) for k, v in md['pull_cols'].items()}
+            {k: ts.ColumnType.from_dict(v) for k, v in md['push_cols'].items()},
+            {k: ts.ColumnType.from_dict(v) for k, v in md['pull_cols'].items()}
         )
-
-    def __eq__(self, other: Any) -> bool:
-        if not isinstance(other, MockRemote):
-            return False
-        return self.name == other.name
-
-    def __hash__(self) -> int:
-        return hash(self.name)
-
-    def __repr__(self) -> str:
-        return f'MockRemote `{self.name}`'

pixeltable/env.py

@@ -37,16 +37,12 @@ class Env:
     @classmethod
     def get(cls) -> Env:
         if cls._instance is None:
-            cls._init_env()
+            env = Env()
+            env._set_up()
+            env._upgrade_metadata()
+            cls._instance = env
         return cls._instance
 
-    @classmethod
-    def _init_env(cls, reinit_db: bool = False) -> None:
-        env = Env()
-        env._set_up(reinit_db=reinit_db)
-        env._upgrade_metadata()
-        cls._instance = env
-
     def __init__(self):
         self._home: Optional[Path] = None
         self._media_dir: Optional[Path] = None  # computed media files

pixeltable/io/globals.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional, Literal
+from typing import Any, Optional
 
 import pixeltable as pxt
 from pixeltable import Table
@@ -9,7 +9,6 @@ def create_label_studio_project(
         label_config: str,
         col_mapping: Optional[dict[str, str]] = None,
         title: Optional[str] = None,
-        media_import_method: Literal['post', 'file'] = 'file',
         sync_immediately: bool = True,
         **kwargs: Any
 ) -> None:
@@ -43,7 +42,7 @@ def create_label_studio_project(
     """
     from pixeltable.datatransfer.label_studio import LabelStudioProject, ANNOTATIONS_COLUMN
 
-    ls_project = LabelStudioProject.create(title or t.get_name(), label_config, media_import_method, **kwargs)
+    ls_project = LabelStudioProject.create(title or t.get_name(), label_config, **kwargs)
 
     # Create a column to hold the annotations, if one does not yet exist.
     if col_mapping is not None and ANNOTATIONS_COLUMN in col_mapping.values():

pixeltable/metadata/__init__.py

@@ -10,7 +10,7 @@ import sqlalchemy.orm as orm
 from .schema import SystemInfo, SystemInfoMd
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 16
+VERSION = 15
 
 
 def create_system_info(engine: sql.engine.Engine) -> None:

pixeltable/metadata/schema.py

@@ -92,9 +92,6 @@ class ColumnMd:
     # if True, the column is present in the stored table
     stored: Optional[bool]
 
-    # if specified, the column is a stored proxy of another column
-    proxy_base: Optional[int]
-
 
 @dataclasses.dataclass
 class IndexMd:

pixeltable/tool/create_test_db_dump.py

@@ -30,8 +30,6 @@ class Dumper:
         os.environ['PIXELTABLE_DB'] = db_name
         os.environ['PIXELTABLE_PGDATA'] = str(shared_home / 'pgdata')
 
-        Env._init_env(reinit_db=True)
-
         Env.get().configure_logging(level=logging.DEBUG, to_stdout=True)
 
     def dump_db(self) -> None:
@@ -166,15 +164,15 @@ class Dumper:
 
         # Add remotes
         from pixeltable.datatransfer.remote import MockRemote
-        v.link(
-            MockRemote('remote', {'int_field': pxt.IntType()}, {'str_field': pxt.StringType()}),
+        v.link_remote(
+            MockRemote({'int_field': pxt.IntType()}, {'str_field': pxt.StringType()}),
             col_mapping={'test_udf': 'int_field', 'c1': 'str_field'}
         )
         # We're just trying to test metadata here, so reach "under the covers" and link a fake
         # Label Studio project without validation (so we don't need a real Label Studio server)
         from pixeltable.datatransfer.label_studio import LabelStudioProject
         v.tbl_version_path.tbl_version.link(
-            LabelStudioProject(4171780, media_import_method='file'),
+            LabelStudioProject(4171780),
             col_mapping={'str_format': 'str_format'}
         )
 

{pixeltable-0.2.7.dist-info → pixeltable-0.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.7
+Version: 0.2.8
 Summary: Pixeltable: The Multimodal AI Data Plane
 Author: Marcel Kornacker
 Author-email: marcelk@gmail.com
@@ -21,7 +21,7 @@ Requires-Dist: more-itertools (>=10.2,<11.0)
 Requires-Dist: numpy (>=1.25)
 Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
 Requires-Dist: pandas (>=2.0,<3.0)
-Requires-Dist: pgserver (==0.1.3)
+Requires-Dist: pgserver (==0.1.4)
 Requires-Dist: pgvector (>=0.2.1,<0.3.0)
 Requires-Dist: pillow (>=9.3.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)

{pixeltable-0.2.7.dist-info → pixeltable-0.2.8.dist-info}/RECORD

@@ -1,8 +1,8 @@
 pixeltable/__init__.py,sha256=DzVevwic1g8Tp4QYrcCIzPFFwPZ66KGWeYprlRC9Uwc,1142
-pixeltable/__version__.py,sha256=QpFd23iUxkd6iv60M7Xgh5GNlwHpgZV9y3kI5aNgy58,112
+pixeltable/__version__.py,sha256=AyN0bhxu_oExUztSHc2d8uAemad8-aDrT7QgYAM_JCs,112
 pixeltable/catalog/__init__.py,sha256=E41bxaPeQIcgRYzTWc2vkDOboQhRymrJf4IcHQO7o_8,453
 pixeltable/catalog/catalog.py,sha256=0TYWB1R6YBp9qCkWF7kCcX2Yw70UuburKKIemv5L1Js,7908
-pixeltable/catalog/column.py,sha256=EW2nDHDcHiCSYN6Gboh5D39c0_NqheWsxSl62IrFWgg,8584
+pixeltable/catalog/column.py,sha256=J8irt6PfT1ofC3wVKi-hDGjNUZ1Ceq2qzbmZyEw-ddA,8335
 pixeltable/catalog/dir.py,sha256=pG1nMpG123POo6WMSHhAmnwXOQ26uUJfUcbzL-Jb4ws,919
 pixeltable/catalog/globals.py,sha256=yLEGNbsSnLzjWNHVJacfjA9hbw13Q6QXLOSCRmdTlq0,943
 pixeltable/catalog/insertable_table.py,sha256=vMa_XUWT3DG3ZlxkScDZ9-mYNw31G8XB4ODUlxXt7NU,8927
@@ -10,15 +10,15 @@ pixeltable/catalog/named_function.py,sha256=a96gnKtx-nz5_MzDIiD4t4Hxqdjkg9ZtijRQ
 pixeltable/catalog/path.py,sha256=QgccEi_QOfaKt8YsR2zLtd_z7z7QQkU_1kprJFi2SPQ,1677
 pixeltable/catalog/path_dict.py,sha256=xfvxg1Ze5jZCARUGASF2DRbQPh7pRVTYhuJ_u82gYUo,5941
 pixeltable/catalog/schema_object.py,sha256=-UxmPLbuEBqJiJi_GGRbFdr7arAFxTqs4bt6TFmSt3M,1059
-pixeltable/catalog/table.py,sha256=FbLDJGFuvHtLT7rFTbFaayNKhmK1kzcr4lzdqyRjGBg,39507
-pixeltable/catalog/table_version.py,sha256=U-GWWSAvVvNinEDSAvN-m2dOO8SviuN1py442OO0hlo,57459
-pixeltable/catalog/table_version_path.py,sha256=P-HL9H1UIiJrKfa0wIRsESGrG3f815nGvHOEpVLJ3no,5718
+pixeltable/catalog/table.py,sha256=50UOSt7zltvthygiiXbgP-XMOiYUpcIaEXiM1uJFcaA,38220
+pixeltable/catalog/table_version.py,sha256=BeP-4Io6euT6hOQXVJwpZNjZ6ZNehqOH6S98zvQsU9E,52751
+pixeltable/catalog/table_version_path.py,sha256=2Ofzd0n36flcNm86KWwIWDBAfgnV5Z-FxAHdMSPgMLc,5482
 pixeltable/catalog/view.py,sha256=BIL3s4DV3tWbOcqtqnhn46B2UvLaBhppfJUlNEt5nec,9734
 pixeltable/dataframe.py,sha256=lzSzR7mi9C4BO39fNXYo64k3KxILyG_Z7eET6DXTgKY,31922
 pixeltable/datatransfer/__init__.py,sha256=cRWdQ_LUNkJgmionI1RrYD71A1CSI92P4o8_XXOnFmU,27
-pixeltable/datatransfer/label_studio.py,sha256=7E70MdgbAKW4AkZcR0uBuYXjAlUzruYgsOIR7FEbZY4,23802
-pixeltable/datatransfer/remote.py,sha256=PaxdVggZMbHyG9FROfKXQxTmGAa8PiVhETl7lp5CNYg,3801
-pixeltable/env.py,sha256=6IEbaG7a-Jz_hJ7nqA32wPmejF9Njl1UIzadubq0SjI,21487
+pixeltable/datatransfer/label_studio.py,sha256=3DLsqfIUNVG9xVRVUU4NayLuC-xUTIM1Sz92kGvrTUc,19643
+pixeltable/datatransfer/remote.py,sha256=t-VeDIq62mX67xBoHLi8voa4V5XqMkr-8UZ-8DhIgk0,3100
+pixeltable/env.py,sha256=OEZv6NS8Z41rdCx73Md5j78ImnKaZf3YhdAexFJR7gw,21381
 pixeltable/exceptions.py,sha256=MSP9zeL0AmXT93XqjdvgGN4rzno1_KRrGriq6hpemnw,376
 pixeltable/exec/__init__.py,sha256=RK7SKvrQ7Ky3G_LXDP4Bf7lHmMM_uYZl8dJaZYs0FjY,454
 pixeltable/exec/aggregation_node.py,sha256=cf6rVAgrGh_uaMrCIgXJIwQTmbcboJlnrH_MmPIQSd0,3321
@@ -87,7 +87,7 @@ pixeltable/index/base.py,sha256=YAQ5Dz1mfI0dfu9rxWHWroE8TjB90yKfPtXAzoADq38,1568
 pixeltable/index/btree.py,sha256=NE4GYhcJWYJhdKyeHI0sQBlFvUaIgGOF9KLyCZOfFjE,1822
 pixeltable/index/embedding_index.py,sha256=AYphEggN-0B4GNrm4nMmi46CEtrQw5tguyk67BK2sWo,7627
 pixeltable/io/__init__.py,sha256=Io5ZLrcvRPeqRQwnU2iODvWMqkYroWErkbp7dLxE4Kk,197
-pixeltable/io/globals.py,sha256=5EdMVoRVvk1AeLaq-P4_LNdNlEDVfsl9bRTFuoTgexQ,2735
+pixeltable/io/globals.py,sha256=ArnuWVhdKHT9ds84PZBl0-fszmEu-W62P4Su21c9oN4,2642
 pixeltable/io/hf_datasets.py,sha256=h5M1NkXOvEU8kaeT3AON1A18Vmhnc1lVo5a3TZ5AAic,8004
 pixeltable/io/pandas.py,sha256=cDHUDW2CGiBbsEJB9zE5vkXopTKxDdI-CZxNcp0OnIk,6478
 pixeltable/io/parquet.py,sha256=Z1b92gsPeCBf4P9_jgWWHAEHtu51nhuC8nSJgoKiywQ,8150
@@ -95,17 +95,15 @@ pixeltable/iterators/__init__.py,sha256=kokLguXBY_nxBTqUiXZVvCxTv-vGsX4cK8tgIbsW
 pixeltable/iterators/base.py,sha256=cnEh1tNN2JAxRzrLTg3dhun3N1oNQ8vifCm6ts3_UiE,1687
 pixeltable/iterators/document.py,sha256=netSCJatG8NcgbHZ69BvQVICdAorQlYi8OlcpqwLQD4,19436
 pixeltable/iterators/video.py,sha256=xtxODL1AfZwTfHVzWekhTCLA8gwTJIvJFdxC0KecD9Q,3836
-pixeltable/metadata/__init__.py,sha256=k3aptna__92VIJ4C6bIHLQrfYE1MOpBCKKMJ_b03nxA,2228
+pixeltable/metadata/__init__.py,sha256=beGPpClpNaN7seM_AeKli5R33TSIkb7_mIBWoExT_5M,2228
 pixeltable/metadata/converters/convert_10.py,sha256=0mSGCn7vqtef63riPi9msUaaUvsSQIj-NFj9QFDYPdA,733
 pixeltable/metadata/converters/convert_12.py,sha256=g9rHTcKlDQZbM3_k4eBv0FBdWmQXHWCnMwx1_l6KpMI,107
 pixeltable/metadata/converters/convert_13.py,sha256=FEgOH5PKf05xVoCaioDDDHOSuoWPyBzodojmsSMMZ5U,1366
 pixeltable/metadata/converters/convert_14.py,sha256=UAWHEipZ-NrQtI5zZN1u9C5AD24ZpYXsdpC3Te0t-qE,402
-pixeltable/metadata/converters/convert_15.py,sha256=WFpIOMRZDgb-_bh1V07EbLv5Hshh1aArdavSnCsUwoQ,1004
-pixeltable/metadata/converters/util.py,sha256=97efM1Hx1qKMIOaEI4bjmX93POie7XvBEF_HJhFhzE0,2400
-pixeltable/metadata/schema.py,sha256=IdvV_UIyQqQL25cf36Rz6dMhHRWXvsApKt_uFSHW5kk,8497
+pixeltable/metadata/schema.py,sha256=ZYBbt_jESRrX7BAx35xKY1CpIgRuJnd2LJYo4MrPnn0,8399
 pixeltable/plan.py,sha256=A_kPsX3bjLyfYbeQ6eCgDbrb_Oldk4w8HdFRqRSDpPY,32653
 pixeltable/store.py,sha256=foQe9y8rRbl35f3naL7rbYVrD8LO00cmD53vWP2A9XI,18850
-pixeltable/tool/create_test_db_dump.py,sha256=W5XNaKpi_-YnOCDVYjNs3g_p2CnRQp0xMtIwyu2rP14,7607
+pixeltable/tool/create_test_db_dump.py,sha256=17MdBqsSNj7j61w0Re9pS4aDIEdML_4hnE-uZJcEW4I,7537
 pixeltable/tool/create_test_video.py,sha256=OLfccymYReIpzE8osZn4rQvLXxxiPC_l0vc06U74hVM,2899
 pixeltable/type_system.py,sha256=nljZs4O_dsVFMs4aB3z7Szc9LgFtl5eOuloxJkk7tPE,29503
 pixeltable/utils/__init__.py,sha256=UYlrf6TIWJT0g-Hac0b34-dEk478B5Qx8dGco34YlIk,439
@@ -120,7 +118,7 @@ pixeltable/utils/pytorch.py,sha256=BR4tgfUWw-2rwWTOgzXj5qdMBpe1Arpp5SK4ax6jjpk,3
 pixeltable/utils/s3.py,sha256=rkanuhk9DWvSfmbOLQW1j1Iov4sl2KhxGGKN-AJ8LSE,432
 pixeltable/utils/sql.py,sha256=5n5_OmXAGtqFdL6z5XvgnU-vlx6Ba6f1WJrO1ZwUle8,765
 pixeltable/utils/transactional_directory.py,sha256=UGzCrGtLR3hEEf8sYGuWBzLVFAEQml3vdIavigWeTBM,1349
-pixeltable-0.2.7.dist-info/LICENSE,sha256=0UNMmwuqWPC0xDY1NWMm4uNJ2_MyA1pnTNRgQTvuBiQ,746
-pixeltable-0.2.7.dist-info/METADATA,sha256=umL9Z-G7tgcumv96Wu89_5i2S_PXLEb9wuvGLoSo2SQ,9806
-pixeltable-0.2.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pixeltable-0.2.7.dist-info/RECORD,,
+pixeltable-0.2.8.dist-info/LICENSE,sha256=0UNMmwuqWPC0xDY1NWMm4uNJ2_MyA1pnTNRgQTvuBiQ,746
+pixeltable-0.2.8.dist-info/METADATA,sha256=VKFyaYflvsFwUgiC1Y2iukgPHGI4W9sQpnLmUCRdMiQ,9806
+pixeltable-0.2.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pixeltable-0.2.8.dist-info/RECORD,,

pixeltable/metadata/converters/convert_15.py

@@ -1,29 +0,0 @@
-import uuid
-
-import sqlalchemy as sql
-
-from pixeltable.metadata import register_converter
-from pixeltable.metadata.converters.util import convert_table_md
-
-
-def convert_15(engine: sql.engine.Engine) -> None:
-    convert_table_md(engine, column_md_updater=update_column_md, remote_md_updater=update_remote_md)
-
-
-def update_column_md(column_md: dict) -> None:
-    column_md['proxy_base'] = None
-
-
-def update_remote_md(remote_md: dict) -> None:
-    remote_md['class'] = f'{remote_md["module"]}.{remote_md["class"]}'
-    del remote_md['module']
-    if remote_md['class'] == 'pixeltable.datatransfer.remote.MockRemote':
-        remote_md['remote_md']['name'] = f'remote_{uuid.uuid4()}'
-    elif remote_md['class'] == 'pixeltable.datatransfer.label_studio.LabelStudioProject':
-        # 'post' is the media_import_method for legacy LabelStudioProject remotes
-        remote_md['remote_md']['media_import_method'] = 'post'
-    else:
-        assert False, remote_md['class']
-
-
-register_converter(15, convert_15)

pixeltable/metadata/converters/util.py

@@ -1,63 +0,0 @@
-import copy
-import logging
-from typing import Any, Callable, Optional
-
-import sqlalchemy as sql
-
-from pixeltable.metadata.schema import Table
-
-__logger = logging.getLogger('pixeltable')
-
-
-def convert_table_md(
-    engine: sql.engine.Engine,
-    column_md_updater: Optional[Callable[[dict], None]] = None,
-    remote_md_updater: Optional[Callable[[dict], None]] = None,
-    substitution_fn: Optional[Callable[[Any, Any], Optional[tuple[Any, Any]]]] = None
-) -> None:
-    with engine.begin() as conn:
-        for row in conn.execute(sql.select(Table)):
-            id = row[0]
-            table_md = row[2]
-            assert isinstance(table_md, dict)
-            updated_table_md = copy.deepcopy(table_md)
-            if column_md_updater is not None:
-                __update_column_md(updated_table_md, column_md_updater)
-            if remote_md_updater is not None:
-                __update_remote_md(updated_table_md, remote_md_updater)
-            if substitution_fn is not None:
-                updated_table_md = __substitute_md_rec(updated_table_md, substitution_fn)
-            if updated_table_md != table_md:
-                __logger.info(f'Updating schema for table: {id}')
-                conn.execute(sql.update(Table).where(Table.id == id).values(md=updated_table_md))
-
-
-def __update_column_md(table_md: dict, column_md_updater: Callable[[dict], None]) -> None:
-    columns_md = table_md['column_md']
-    assert isinstance(columns_md, dict)
-    for column_md in columns_md.values():
-        column_md_updater(column_md)
-
-
-def __update_remote_md(table_md: dict, remote_md_updater: Callable[[dict], None]) -> None:
-    remotes_md = table_md['remotes']
-    assert isinstance(remotes_md, list)
-    for remote_md in remotes_md:
-        remote_md_updater(remote_md)
-
-
-def __substitute_md_rec(md: Any, substitution_fn: Callable[[Any, Any], Optional[tuple[Any, Any]]]) -> Any:
-    if isinstance(md, dict):
-        updated_md = {}
-        for k, v in md.items():
-            substitute = substitution_fn(k, v)
-            if substitute is not None:
-                updated_k, updated_v = substitute
-                updated_md[updated_k] = updated_v
-            else:
-                updated_md[k] = __substitute_md_rec(v, substitution_fn)
-        return updated_md
-    elif isinstance(md, list):
-        return [__substitute_md_rec(v, substitution_fn) for v in md]
-    else:
-        return md