pixeltable 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

pixeltable/io/external_store.py

@@ -3,14 +3,13 @@ from __future__ import annotations
 import abc
 import itertools
 import logging
-from dataclasses import dataclass
 from typing import Any, Optional
-from uuid import UUID
 
 import pixeltable.exceptions as excs
 import pixeltable.type_system as ts
 from pixeltable import Column, Table
-from pixeltable.catalog import TableVersion
+from pixeltable.catalog import ColumnHandle, TableVersion
+from pixeltable.catalog.update_status import UpdateStatus
 
 _logger = logging.getLogger('pixeltable')
 
@@ -22,6 +21,8 @@ class ExternalStore(abc.ABC):
     and stateful external stores.
     """
 
+    __name: str
+
     def __init__(self, name: str) -> None:
         self.__name = name
 
@@ -38,13 +39,13 @@ class ExternalStore(abc.ABC):
         """Removes store-specific metadata created in link()."""
 
     @abc.abstractmethod
-    def get_local_columns(self) -> list[Column]:
+    def get_local_columns(self) -> list[ColumnHandle]:
         """
         Gets a list of all local (Pixeltable) columns that are associated with this external store.
         """
 
     @abc.abstractmethod
-    def sync(self, t: Table, export_data: bool, import_data: bool) -> SyncStatus:
+    def sync(self, t: Table, export_data: bool, import_data: bool) -> UpdateStatus:
         """
         Called by `Table.sync()` to implement store-specific synchronization logic.
         """
@@ -63,9 +64,15 @@ class Project(ExternalStore, abc.ABC):
     additional capabilities specific to such projects.
     """
 
-    stored_proxies: dict[Column, Column]
+    _col_mapping: dict[ColumnHandle, str]  # col -> external col name
+    stored_proxies: dict[ColumnHandle, ColumnHandle]  # original col -> proxy col
 
-    def __init__(self, name: str, col_mapping: dict[Column, str], stored_proxies: Optional[dict[Column, Column]]):
+    def __init__(
+        self,
+        name: str,
+        col_mapping: dict[ColumnHandle, str],
+        stored_proxies: Optional[dict[ColumnHandle, ColumnHandle]],
+    ):
         super().__init__(name)
         self._col_mapping = col_mapping
 
@@ -80,11 +87,11 @@ class Project(ExternalStore, abc.ABC):
         # Note from aaron-siegel: This methodology is inefficient in the case where a table has many views with a high
         # proportion of overlapping rows, all proxying the same base column.
         if stored_proxies is None:
-            self.stored_proxies: dict[Column, Column] = {}
+            self.stored_proxies: dict[ColumnHandle, ColumnHandle] = {}
         else:
             self.stored_proxies = stored_proxies
 
-    def get_local_columns(self) -> list[Column]:
+    def get_local_columns(self) -> list[ColumnHandle]:
         return list(self.col_mapping.keys())
 
     def link(self, tbl_version: TableVersion) -> None:
@@ -92,15 +99,16 @@ class Project(ExternalStore, abc.ABC):
         # This ensures that the media in those columns resides in the media store.
         # First determine which columns (if any) need stored proxies, but don't have one yet.
         stored_proxies_needed: list[Column] = []
-        for col in self.col_mapping:
+        for col_handle in self.col_mapping:
+            col = col_handle.get()
             if col.col_type.is_media_type() and not (col.is_stored and col.is_computed):
                 # If this column is already proxied in some other Project, use the existing proxy to avoid
                 # duplication. Otherwise, we'll create a new one.
                 for store in tbl_version.external_stores.values():
-                    if isinstance(store, Project) and col in store.stored_proxies:
-                        self.stored_proxies[col] = store.stored_proxies[col]
+                    if isinstance(store, Project) and col_handle in store.stored_proxies:
+                        self.stored_proxies[col_handle] = store.stored_proxies[col_handle]
                         break
-                if col not in self.stored_proxies:
+                if col_handle not in self.stored_proxies:
                     # We didn't find it in an existing Project
                     stored_proxies_needed.append(col)
 
@@ -110,17 +118,20 @@ class Project(ExternalStore, abc.ABC):
             proxy_cols = [self.create_stored_proxy(col) for col in stored_proxies_needed]
             # Add the columns; this will also update table metadata.
             tbl_version.add_columns(proxy_cols, print_stats=False, on_error='ignore')
+            self.stored_proxies.update(
+                {col.handle: proxy_col.handle for col, proxy_col in zip(stored_proxies_needed, proxy_cols)}
+            )
 
     def unlink(self, tbl_version: TableVersion) -> None:
         # Determine which stored proxies can be deleted. (A stored proxy can be deleted if it is not referenced by
         # any *other* external store for this table.)
-        deletions_needed: set[Column] = set(self.stored_proxies.values())
+        deletions_needed: set[ColumnHandle] = set(self.stored_proxies.values())
         for name, store in tbl_version.external_stores.items():
             if isinstance(store, Project) and name != self.name:
                 deletions_needed = deletions_needed.difference(set(store.stored_proxies.values()))
         if len(deletions_needed) > 0:
-            _logger.info(f'Removing stored proxies for columns: {[col.name for col in deletions_needed]}')
-            tbl_version._drop_columns(deletions_needed)
+            _logger.info(f'Removing stored proxies for columns: {[col.get().name for col in deletions_needed]}')
+            tbl_version._drop_columns(col.get() for col in deletions_needed)
             self.stored_proxies.clear()
 
     def create_stored_proxy(self, col: Column) -> Column:
@@ -142,11 +153,10 @@ class Project(ExternalStore, abc.ABC):
             computed_with=exprs.ColumnRef(col).apply(lambda x: x, col_type=col.col_type),
             stored=True,
         )
-        self.stored_proxies[col] = proxy_col
         return proxy_col
 
     @property
-    def col_mapping(self) -> dict[Column, str]:
+    def col_mapping(self) -> dict[ColumnHandle, str]:
         return self._col_mapping
 
     @abc.abstractmethod
@@ -181,7 +191,7 @@ class Project(ExternalStore, abc.ABC):
         export_cols: dict[str, ts.ColumnType],
         import_cols: dict[str, ts.ColumnType],
         col_mapping: Optional[dict[str, str]],
-    ) -> dict[Column, str]:
+    ) -> dict[ColumnHandle, str]:
         """
         Verifies that the specified `col_mapping` is valid. In particular, checks that:
         (i) the keys of `col_mapping` are valid columns of the specified `Table`;
@@ -199,7 +209,7 @@ class Project(ExternalStore, abc.ABC):
         if col_mapping is None:
             col_mapping = {col: col for col in itertools.chain(export_cols.keys(), import_cols.keys())}
 
-        resolved_col_mapping: dict[Column, str] = {}
+        resolved_col_mapping: dict[ColumnHandle, str] = {}
 
         # Validate names
         t_cols = set(table._get_schema().keys())
@@ -223,7 +233,8 @@ class Project(ExternalStore, abc.ABC):
                 )
             col_ref = table[t_col]
             assert isinstance(col_ref, exprs.ColumnRef)
-            resolved_col_mapping[col_ref.col] = ext_col
+            resolved_col_mapping[col_ref.col.handle] = ext_col
+
         # Validate column specs
         t_col_types = table._get_schema()
         for t_col, ext_col in col_mapping.items():
@@ -250,40 +261,6 @@ class Project(ExternalStore, abc.ABC):
                     )
         return resolved_col_mapping
 
-    @classmethod
-    def _column_as_dict(cls, col: Column) -> dict[str, Any]:
-        return {'tbl_id': str(col.tbl.id), 'col_id': col.id}
-
-    @classmethod
-    def _column_from_dict(cls, d: dict[str, Any]) -> Column:
-        from pixeltable.catalog import Catalog
-
-        tbl_id = UUID(d['tbl_id'])
-        col_id = d['col_id']
-        return Catalog.get().get_tbl_version(tbl_id, None).cols_by_id[col_id]
-
-
-@dataclass(frozen=True)
-class SyncStatus:
-    external_rows_created: int = 0
-    external_rows_deleted: int = 0
-    external_rows_updated: int = 0
-    pxt_rows_updated: int = 0
-    num_excs: int = 0
-
-    def combine(self, other: 'SyncStatus') -> 'SyncStatus':
-        return SyncStatus(
-            external_rows_created=self.external_rows_created + other.external_rows_created,
-            external_rows_deleted=self.external_rows_deleted + other.external_rows_deleted,
-            external_rows_updated=self.external_rows_updated + other.external_rows_updated,
-            pxt_rows_updated=self.pxt_rows_updated + other.pxt_rows_updated,
-            num_excs=self.num_excs + other.num_excs,
-        )
-
-    @classmethod
-    def empty(cls) -> 'SyncStatus':
-        return SyncStatus(0, 0, 0, 0, 0)
-
 
 class MockProject(Project):
     """A project that cannot be synced, used mainly for testing."""
@@ -293,8 +270,8 @@ class MockProject(Project):
         name: str,
         export_cols: dict[str, ts.ColumnType],
         import_cols: dict[str, ts.ColumnType],
-        col_mapping: dict[Column, str],
-        stored_proxies: Optional[dict[Column, Column]] = None,
+        col_mapping: dict[ColumnHandle, str],
+        stored_proxies: Optional[dict[ColumnHandle, ColumnHandle]] = None,
     ):
         super().__init__(name, col_mapping, stored_proxies)
         self.export_cols = export_cols
@@ -319,7 +296,7 @@ class MockProject(Project):
     def get_import_columns(self) -> dict[str, ts.ColumnType]:
         return self.import_cols
 
-    def sync(self, t: Table, export_data: bool, import_data: bool) -> SyncStatus:
+    def sync(self, t: Table, export_data: bool, import_data: bool) -> UpdateStatus:
         raise NotImplementedError()
 
     def delete(self) -> None:
@@ -334,10 +311,8 @@ class MockProject(Project):
             'name': self.name,
             'export_cols': {k: v.as_dict() for k, v in self.export_cols.items()},
             'import_cols': {k: v.as_dict() for k, v in self.import_cols.items()},
-            'col_mapping': [[self._column_as_dict(k), v] for k, v in self.col_mapping.items()],
-            'stored_proxies': [
-                [self._column_as_dict(k), self._column_as_dict(v)] for k, v in self.stored_proxies.items()
-            ],
+            'col_mapping': [[k.as_dict(), v] for k, v in self.col_mapping.items()],
+            'stored_proxies': [[k.as_dict(), v.as_dict()] for k, v in self.stored_proxies.items()],
         }
 
     @classmethod
@@ -346,8 +321,8 @@ class MockProject(Project):
             md['name'],
             {k: ts.ColumnType.from_dict(v) for k, v in md['export_cols'].items()},
             {k: ts.ColumnType.from_dict(v) for k, v in md['import_cols'].items()},
-            {cls._column_from_dict(entry[0]): entry[1] for entry in md['col_mapping']},
-            {cls._column_from_dict(entry[0]): cls._column_from_dict(entry[1]) for entry in md['stored_proxies']},
+            {ColumnHandle.from_dict(entry[0]): entry[1] for entry in md['col_mapping']},
+            {ColumnHandle.from_dict(entry[0]): ColumnHandle.from_dict(entry[1]) for entry in md['stored_proxies']},
         )
 
     def __eq__(self, other: object) -> bool:

pixeltable/io/globals.py

@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union
 import pixeltable as pxt
 import pixeltable.exceptions as excs
 from pixeltable import Table, exprs
+from pixeltable.catalog.update_status import UpdateStatus
 from pixeltable.env import Env
-from pixeltable.io.external_store import SyncStatus
 
 if TYPE_CHECKING:
     import fiftyone as fo  # type: ignore[import-untyped]
@@ -22,7 +22,7 @@ def create_label_studio_project(
     sync_immediately: bool = True,
     s3_configuration: Optional[dict[str, Any]] = None,
     **kwargs: Any,
-) -> SyncStatus:
+) -> UpdateStatus:
     """
     Create a new Label Studio project and link it to the specified [`Table`][pixeltable.Table].
 
@@ -96,7 +96,7 @@ def create_label_studio_project(
             [Label Studio start_project docs](https://labelstud.io/sdk/project.html#label_studio_sdk.project.Project.start_project).
 
     Returns:
-        A `SyncStatus` representing the status of any synchronization operations that occurred.
+        An `UpdateStatus` representing the status of any synchronization operations that occurred.
 
     Examples:
         Create a Label Studio project whose tasks correspond to videos stored in the `video_col`
@@ -136,7 +136,7 @@ def create_label_studio_project(
     if sync_immediately:
         return t.sync()
     else:
-        return SyncStatus.empty()
+        return UpdateStatus()
 
 
 def export_images_as_fo_dataset(

pixeltable/io/hf_datasets.py

@@ -50,10 +50,18 @@ def _to_pixeltable_type(feature_type: Any, nullable: bool) -> Optional[ts.Column
     elif isinstance(feature_type, datasets.Sequence):
         # example: cohere wiki. Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None)
         dtype = _to_pixeltable_type(feature_type.feature, nullable)
-        length = feature_type.length if feature_type.length != -1 else None
-        return ts.ArrayType(shape=(length,), dtype=dtype)
+        if dtype is None:
+            return None
+        if dtype.is_int_type() or dtype.is_float_type() or dtype.is_bool_type() or dtype.is_string_type():
+            length = feature_type.length if feature_type.length != -1 else None
+            return ts.ArrayType(shape=(length,), dtype=dtype, nullable=nullable)
+        else:
+            # Sequence of dicts must be cast as Json
+            return ts.JsonType(nullable=nullable)
     elif isinstance(feature_type, datasets.Image):
         return ts.ImageType(nullable=nullable)
+    elif isinstance(feature_type, dict):
+        return ts.JsonType(nullable=nullable)
     else:
         return None
 

pixeltable/io/label_studio.py

@@ -4,18 +4,20 @@ import logging
 import os
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Iterator, Literal, Optional, cast
+from typing import Any, Iterator, Literal, Optional
 from xml.etree import ElementTree as ET
 
-import label_studio_sdk  # type: ignore[import-untyped]
+import label_studio_sdk
 import PIL.Image
 from requests.exceptions import HTTPError
 
 import pixeltable.type_system as ts
 from pixeltable import Column, Table, env, exceptions as excs
+from pixeltable.catalog import ColumnHandle
+from pixeltable.catalog.update_status import RowCountStats, UpdateStatus
 from pixeltable.config import Config
 from pixeltable.exprs import ColumnRef, DataRow, Expr
-from pixeltable.io.external_store import Project, SyncStatus
+from pixeltable.io.external_store import Project
 from pixeltable.utils import coco
 
 # label_studio_sdk>=1 and label_studio_sdk<1 are not compatible, so we need to try
@@ -25,7 +27,7 @@ try:
     import label_studio_sdk.project as ls_project  # type: ignore
 except ImportError:
     # label_studio_sdk>=1 compatibility
-    import label_studio_sdk._legacy.project as ls_project  # type: ignore
+    import label_studio_sdk._legacy.project as ls_project
 
 _logger = logging.getLogger('pixeltable')
 
@@ -45,13 +47,17 @@ class LabelStudioProject(Project):
     for synchronizing between a Pixeltable table and a Label Studio project.
     """
 
+    project_id: int  # Label Studio project ID
+    media_import_method: Literal['post', 'file', 'url']
+    _project: Optional[ls_project.Project]
+
     def __init__(
         self,
         name: str,
         project_id: int,
         media_import_method: Literal['post', 'file', 'url'],
-        col_mapping: dict[Column, str],
-        stored_proxies: Optional[dict[Column, Column]] = None,
+        col_mapping: dict[ColumnHandle, str],
+        stored_proxies: Optional[dict[ColumnHandle, ColumnHandle]] = None,
     ):
         """
         The constructor will NOT create a new Label Studio project; it is also used when loading
@@ -59,7 +65,7 @@ class LabelStudioProject(Project):
         """
         self.project_id = project_id
         self.media_import_method = media_import_method
-        self._project: Optional[ls_project.Project] = None
+        self._project = None
         super().__init__(name, col_mapping, stored_proxies)
 
     @property
@@ -105,20 +111,20 @@ class LabelStudioProject(Project):
         """
         return {ANNOTATIONS_COLUMN: ts.JsonType(nullable=True)}
 
-    def sync(self, t: Table, export_data: bool, import_data: bool) -> SyncStatus:
+    def sync(self, t: Table, export_data: bool, import_data: bool) -> UpdateStatus:
         _logger.info(
             f'Syncing Label Studio project "{self.project_title}" with table `{t._name}`'
             f' (export: {export_data}, import: {import_data}).'
         )
         # Collect all existing tasks into a dict with entries `rowid: task`
         tasks = {tuple(task['meta']['rowid']): task for task in self.__fetch_all_tasks()}
-        sync_status = SyncStatus.empty()
+        sync_status = UpdateStatus()
         if export_data:
             export_sync_status = self.__update_tasks(t, tasks)
-            sync_status = sync_status.combine(export_sync_status)
+            sync_status += export_sync_status
         if import_data:
             import_sync_status = self.__update_table_from_tasks(t, tasks)
-            sync_status = sync_status.combine(import_sync_status)
+            sync_status += import_sync_status
         return sync_status
 
     def __fetch_all_tasks(self) -> Iterator[dict[str, Any]]:
@@ -142,7 +148,7 @@ class LabelStudioProject(Project):
                 f'Label Studio project {self.project_title!r}.'
             )
 
-    def __update_tasks(self, t: Table, existing_tasks: dict[tuple, dict]) -> SyncStatus:
+    def __update_tasks(self, t: Table, existing_tasks: dict[tuple, dict]) -> UpdateStatus:
         """
         Updates all tasks in this Label Studio project based on the Pixeltable data:
         - Creates new tasks for rows that don't map to any existing task;
@@ -155,7 +161,7 @@ class LabelStudioProject(Project):
         t_data_cols = [t_col for t_col, ext_col_name in self.col_mapping.items() if ext_col_name in config.data_keys]
 
         if len(t_data_cols) == 0:
-            return SyncStatus.empty()
+            return UpdateStatus()
 
         # Columns in `t` that map to `rectanglelabels` preannotations
         t_rl_cols = [
@@ -183,15 +189,15 @@ class LabelStudioProject(Project):
         self,
         t: Table,
         existing_tasks: dict[tuple, dict],
-        media_col: Column,
-        t_rl_cols: list[Column],
+        media_col: ColumnHandle,
+        t_rl_cols: list[ColumnHandle],
         rl_info: list['_RectangleLabel'],
-    ) -> SyncStatus:
-        is_stored = media_col.is_stored
+    ) -> UpdateStatus:
+        is_stored = media_col.get().is_stored
         # If it's a stored column, we can use `localpath`
-        localpath_col_opt = [t[media_col.name].localpath] if is_stored else []
+        localpath_col_opt = [t[media_col.get().name].localpath] if is_stored else []
         # Select the media column, rectanglelabels columns, and localpath (if appropriate)
-        rows = t.select(t[media_col.name], *[t[col.name] for col in t_rl_cols], *localpath_col_opt)
+        rows = t.select(t[media_col.get().name], *[t[col.get().name] for col in t_rl_cols], *localpath_col_opt)
         tasks_created = 0
         row_ids_in_pxt: set[tuple] = set()
 
@@ -232,42 +238,42 @@ class LabelStudioProject(Project):
 
         env.Env.get().console_logger.info(f'Created {tasks_created} new task(s) in {self}.')
 
-        sync_status = SyncStatus(external_rows_created=tasks_created)
+        sync_status = UpdateStatus(ext_row_count_stats=RowCountStats(ins_rows=tasks_created))
 
         deletion_sync_status = self.__delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
-
-        return sync_status.combine(deletion_sync_status)
+        sync_status += deletion_sync_status
+        return sync_status
 
     def __update_tasks_by_files(
         self,
         t: Table,
         existing_tasks: dict[tuple, dict],
-        t_data_cols: list[Column],
-        t_rl_cols: list[Column],
+        t_data_cols: list[ColumnHandle],
+        t_rl_cols: list[ColumnHandle],
         rl_info: list['_RectangleLabel'],
-    ) -> SyncStatus:
+    ) -> UpdateStatus:
         ext_data_cols = [self.col_mapping[col] for col in t_data_cols]
         expr_refs: dict[str, Expr] = {}  # kwargs for the select statement
         for col in t_data_cols:
-            col_name = col.name
+            col_name = col.get().name
             if self.media_import_method == 'url':
                 expr_refs[col_name] = t[col_name].fileurl
             else:
                 assert self.media_import_method == 'file'
-                if not col.col_type.is_media_type():
+                if not col.get().col_type.is_media_type():
                     # Not a media column; query the data directly
-                    expr_refs[col_name] = cast(ColumnRef, t[col_name])
+                    expr_refs[col_name] = t[col_name]
                 elif col in self.stored_proxies:
                     # Media column that has a stored proxy; use it. We have to give it a name,
                     # since it's an anonymous column
-                    stored_proxy_col = self.stored_proxies[col]
+                    stored_proxy_col = self.stored_proxies[col].get()
                     expr_refs[f'{col_name}_proxy'] = ColumnRef(stored_proxy_col).localpath
                 else:
                     # Media column without a stored proxy; this means it's a stored computed column,
                     # and we can just use the localpath
                     expr_refs[col_name] = t[col_name].localpath
 
-        df = t.select(*[t[col.name] for col in t_rl_cols], **expr_refs)
+        df = t.select(*[t[col.get().name] for col in t_rl_cols], **expr_refs)
         # The following buffers will hold `DataRow` indices that correspond to each of the selected
         # columns. `rl_col_idxs` holds the indices for the columns that map to RectangleLabels
         # preannotations; `data_col_idxs` holds the indices for the columns that map to data fields.
@@ -286,11 +292,11 @@ class LabelStudioProject(Project):
             data_vals = [row[idx] for idx in data_col_idxs]
             coco_annotations = [row[idx] for idx in rl_col_idxs]
             for i in range(len(t_data_cols)):
-                if t_data_cols[i].col_type.is_media_type():
+                if t_data_cols[i].get().col_type.is_media_type():
                     # Special handling for media columns
                     assert isinstance(data_vals[i], str)
                     if self.media_import_method == 'url':
-                        data_vals[i] = self.__validate_fileurl(t_data_cols[i], data_vals[i])
+                        data_vals[i] = self.__validate_fileurl(t_data_cols[i].get(), data_vals[i])
                     else:
                         assert self.media_import_method == 'file'
                         data_vals[i] = self.__localpath_to_lspath(data_vals[i])
@@ -336,11 +342,11 @@ class LabelStudioProject(Project):
             f'Created {tasks_created} new task(s) and updated {tasks_updated} existing task(s) in {self}.'
         )
 
-        sync_status = SyncStatus(external_rows_created=tasks_created, external_rows_updated=tasks_updated)
+        sync_status = UpdateStatus(ext_row_count_stats=RowCountStats(ins_rows=tasks_created, upd_rows=tasks_updated))
 
         deletion_sync_status = self.__delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
-
-        return sync_status.combine(deletion_sync_status)
+        sync_status += deletion_sync_status
+        return sync_status
 
     @classmethod
     def __validate_fileurl(cls, col: Column, url: str) -> Optional[str]:
@@ -361,7 +367,7 @@ class LabelStudioProject(Project):
 
     def __delete_stale_tasks(
         self, existing_tasks: dict[tuple, dict], row_ids_in_pxt: set[tuple], tasks_created: int
-    ) -> SyncStatus:
+    ) -> UpdateStatus:
         deleted_rowids = set(existing_tasks.keys()) - row_ids_in_pxt
         # Sanity check the math
         assert len(deleted_rowids) == len(existing_tasks) + tasks_created - len(row_ids_in_pxt)
@@ -377,11 +383,11 @@ class LabelStudioProject(Project):
         for rowid in deleted_rowids:
             del existing_tasks[rowid]
 
-        return SyncStatus(external_rows_deleted=len(deleted_rowids))
+        return UpdateStatus(ext_row_count_stats=RowCountStats(del_rows=len(deleted_rowids)))
 
-    def __update_table_from_tasks(self, t: Table, tasks: dict[tuple, dict]) -> SyncStatus:
+    def __update_table_from_tasks(self, t: Table, tasks: dict[tuple, dict]) -> UpdateStatus:
         if ANNOTATIONS_COLUMN not in self.col_mapping.values():
-            return SyncStatus.empty()
+            return UpdateStatus()
 
         annotations = {
             # Replace [] by None to indicate no annotations. We do want to sync rows with no annotations,
@@ -391,7 +397,7 @@ class LabelStudioProject(Project):
             for task in tasks.values()
         }
 
-        local_annotations_col = next(k for k, v in self.col_mapping.items() if v == ANNOTATIONS_COLUMN)
+        local_annotations_col = next(k for k, v in self.col_mapping.items() if v == ANNOTATIONS_COLUMN).get()
 
         # Prune the annotations down to just the ones that have actually changed.
         rows = t.select(t[local_annotations_col.name])
@@ -416,19 +422,17 @@ class LabelStudioProject(Project):
                 ancestor = ancestor._get_base_table()
             update_status = ancestor.batch_update(updates)
             env.Env.get().console_logger.info(f'Updated annotation(s) from {len(updates)} task(s) in {self}.')
-            return SyncStatus(pxt_rows_updated=update_status.num_rows, num_excs=update_status.num_excs)
+            return update_status
         else:
-            return SyncStatus.empty()
+            return UpdateStatus()
 
     def as_dict(self) -> dict[str, Any]:
         return {
             'name': self.name,
             'project_id': self.project_id,
             'media_import_method': self.media_import_method,
-            'col_mapping': [[self._column_as_dict(k), v] for k, v in self.col_mapping.items()],
-            'stored_proxies': [
-                [self._column_as_dict(k), self._column_as_dict(v)] for k, v in self.stored_proxies.items()
-            ],
+            'col_mapping': [[k.as_dict(), v] for k, v in self.col_mapping.items()],
+            'stored_proxies': [[k.as_dict(), v.as_dict()] for k, v in self.stored_proxies.items()],
         }
 
     @classmethod
@@ -437,8 +441,8 @@ class LabelStudioProject(Project):
             md['name'],
             md['project_id'],
             md['media_import_method'],
-            {cls._column_from_dict(entry[0]): entry[1] for entry in md['col_mapping']},
-            {cls._column_from_dict(entry[0]): cls._column_from_dict(entry[1]) for entry in md['stored_proxies']},
+            {ColumnHandle.from_dict(entry[0]): entry[1] for entry in md['col_mapping']},
+            {ColumnHandle.from_dict(entry[0]): ColumnHandle.from_dict(entry[1]) for entry in md['stored_proxies']},
         )
 
     def __repr__(self) -> str:

pixeltable/metadata/__init__.py

@@ -18,7 +18,7 @@ _console_logger = ConsoleLogger(logging.getLogger('pixeltable'))
 _logger = logging.getLogger('pixeltable')
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 38
+VERSION = 40
 
 
 def create_system_info(engine: sql.engine.Engine) -> None:

pixeltable/metadata/converters/convert_38.py

@@ -0,0 +1,39 @@
+from typing import Any, Optional
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=38)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, substitution_fn=__substitute_md)
+
+
+def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+    if k == 'col_mapping':
+        assert isinstance(v, list)
+        return k, [__col_mapping_entry(e) for e in v]
+    if k == 'stored_proxies':
+        assert isinstance(v, list)
+        return k, [__stored_proxies_entry(e) for e in v]
+    return None
+
+
+def __col_mapping_entry(e: list) -> list:
+    assert isinstance(e, list)
+    assert isinstance(e[0], dict)
+    assert isinstance(e[1], str)
+    return [__col_handle(e[0]), e[1]]
+
+
+def __stored_proxies_entry(e: list) -> list:
+    assert isinstance(e, list)
+    assert isinstance(e[0], dict)
+    assert isinstance(e[1], dict)
+    return [__col_handle(e[0]), __col_handle(e[1])]
+
+
+def __col_handle(e: dict) -> dict:
+    return {'tbl_version': {'id': e['tbl_id'], 'effective_version': None}, 'col_id': e['col_id']}