pixeltable 0.4.1__py3-none-any.whl → 0.4.2__py3-none-any.whl

pixeltable/exprs/row_builder.py

@@ -63,6 +63,7 @@ class RowBuilder:
 
     input_exprs: ExprSet
 
+    tbl: Optional[catalog.TableVersion]  # reference table of the RowBuilder; used to identify pk columns for writes
     table_columns: list[ColumnSlotIdx]
     default_eval_ctx: EvalCtx
     unstored_iter_args: dict[UUID, Expr]
@@ -93,7 +94,13 @@ class RowBuilder:
         target_slot_idxs: list[int]  # slot idxs of target exprs; might contain duplicates
         target_exprs: list[Expr]  # exprs corresponding to target_slot_idxs
 
-    def __init__(self, output_exprs: Sequence[Expr], columns: Sequence[catalog.Column], input_exprs: Iterable[Expr]):
+    def __init__(
+        self,
+        output_exprs: Sequence[Expr],
+        columns: Sequence[catalog.Column],
+        input_exprs: Iterable[Expr],
+        tbl: Optional[catalog.TableVersion] = None,
+    ):
         """
         Args:
             output_exprs: list of Exprs to be evaluated
@@ -125,6 +132,7 @@ class RowBuilder:
         #   * further references to that column (eg, computed cols) need to resolve to the validating ColumnRef
         from .column_ref import ColumnRef
 
+        self.tbl = tbl
         self.table_columns: list[ColumnSlotIdx] = []
         self.input_exprs = ExprSet()
         validating_colrefs: dict[Expr, Expr] = {}  # key: non-validating colref, value: corresp. validating colref
@@ -229,6 +237,7 @@ class RowBuilder:
 
     def add_table_column(self, col: catalog.Column, slot_idx: int) -> None:
         """Record a column that is part of the table row"""
+        assert self.tbl is not None
         self.table_columns.append(ColumnSlotIdx(col, slot_idx))
 
     def output_slot_idxs(self) -> list[ColumnSlotIdx]:
@@ -427,33 +436,55 @@ class RowBuilder:
                         expr, f'expression {expr}', data_row.get_exc(expr.slot_idx), exc_tb, input_vals, 0
                     ) from exc
 
-    def create_table_row(self, data_row: DataRow, exc_col_ids: set[int]) -> tuple[dict[str, Any], int]:
+    def create_table_row(
+        self, data_row: DataRow, cols_with_excs: Optional[set[int]], pk: tuple[int, ...]
+    ) -> tuple[list[Any], int]:
         """Create a table row from the slots that have an output column assigned
 
-        Return tuple[dict that represents a stored row (can be passed to sql.insert()), # of exceptions]
+        Return tuple[list of row values in `self.table_columns` order, # of exceptions]
             This excludes system columns.
         """
         num_excs = 0
-        table_row: dict[str, Any] = {}
+        table_row: list[Any] = list(pk)
         for info in self.table_columns:
             col, slot_idx = info.col, info.slot_idx
             if data_row.has_exc(slot_idx):
-                # exceptions get stored in the errortype/-msg columns
                 exc = data_row.get_exc(slot_idx)
                 num_excs += 1
-                exc_col_ids.add(col.id)
-                table_row[col.store_name()] = None
-                table_row[col.errortype_store_name()] = type(exc).__name__
-                table_row[col.errormsg_store_name()] = str(exc)
+                if cols_with_excs is not None:
+                    cols_with_excs.add(col.id)
+                table_row.append(None)
+                if col.records_errors:
+                    # exceptions get stored in the errortype/-msg columns
+                    table_row.extend((type(exc).__name__, str(exc)))
             else:
                 if col.col_type.is_image_type() and data_row.file_urls[slot_idx] is None:
                     # we have yet to store this image
                     filepath = str(MediaStore.prepare_media_path(col.tbl.id, col.id, col.tbl.version))
                     data_row.flush_img(slot_idx, filepath)
                 val = data_row.get_stored_val(slot_idx, col.get_sa_col_type())
-                table_row[col.store_name()] = val
-                # we unfortunately need to set these, even if there are no errors
-                table_row[col.errortype_store_name()] = None
-                table_row[col.errormsg_store_name()] = None
+                table_row.append(val)
+                if col.records_errors:
+                    table_row.extend((None, None))
 
         return table_row, num_excs
+
+    def store_column_names(self) -> tuple[list[str], dict[int, catalog.Column]]:
+        """
+        Returns the list of store column names corresponding to the table_columns of this RowBuilder.
+        The second tuple element of the return value is a dictionary containing all media columns in the
+        table; it's the mapping {list_index: column}.
+        """
+        assert self.tbl is not None, self.table_columns
+        store_col_names: list[str] = [pk_col.name for pk_col in self.tbl.store_tbl.pk_columns()]
+        media_cols: dict[int, catalog.Column] = {}
+
+        for col in self.table_columns:
+            if col.col.col_type.is_media_type():
+                media_cols[len(store_col_names)] = col.col
+            store_col_names.append(col.col.store_name())
+            if col.col.records_errors:
+                store_col_names.append(col.col.errortype_store_name())
+                store_col_names.append(col.col.errormsg_store_name())
+
+        return store_col_names, media_cols

pixeltable/io/external_store.py

@@ -3,14 +3,14 @@ from __future__ import annotations
 import abc
 import itertools
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any, Optional
-from uuid import UUID
 
 import pixeltable.exceptions as excs
 import pixeltable.type_system as ts
 from pixeltable import Column, Table
-from pixeltable.catalog import TableVersion
+from pixeltable.catalog import ColumnHandle, TableVersion
+from pixeltable.catalog.globals import RowCountStats, UpdateStatus
 
 _logger = logging.getLogger('pixeltable')
 
@@ -22,6 +22,8 @@ class ExternalStore(abc.ABC):
     and stateful external stores.
     """
 
+    __name: str
+
     def __init__(self, name: str) -> None:
         self.__name = name
 
@@ -38,7 +40,7 @@ class ExternalStore(abc.ABC):
         """Removes store-specific metadata created in link()."""
 
     @abc.abstractmethod
-    def get_local_columns(self) -> list[Column]:
+    def get_local_columns(self) -> list[ColumnHandle]:
         """
         Gets a list of all local (Pixeltable) columns that are associated with this external store.
         """
@@ -63,9 +65,15 @@ class Project(ExternalStore, abc.ABC):
     additional capabilities specific to such projects.
     """
 
-    stored_proxies: dict[Column, Column]
+    _col_mapping: dict[ColumnHandle, str]  # col -> external col name
+    stored_proxies: dict[ColumnHandle, ColumnHandle]  # original col -> proxy col
 
-    def __init__(self, name: str, col_mapping: dict[Column, str], stored_proxies: Optional[dict[Column, Column]]):
+    def __init__(
+        self,
+        name: str,
+        col_mapping: dict[ColumnHandle, str],
+        stored_proxies: Optional[dict[ColumnHandle, ColumnHandle]],
+    ):
         super().__init__(name)
         self._col_mapping = col_mapping
 
@@ -80,11 +88,11 @@ class Project(ExternalStore, abc.ABC):
         # Note from aaron-siegel: This methodology is inefficient in the case where a table has many views with a high
         # proportion of overlapping rows, all proxying the same base column.
         if stored_proxies is None:
-            self.stored_proxies: dict[Column, Column] = {}
+            self.stored_proxies: dict[ColumnHandle, ColumnHandle] = {}
         else:
             self.stored_proxies = stored_proxies
 
-    def get_local_columns(self) -> list[Column]:
+    def get_local_columns(self) -> list[ColumnHandle]:
         return list(self.col_mapping.keys())
 
     def link(self, tbl_version: TableVersion) -> None:
@@ -92,15 +100,16 @@ class Project(ExternalStore, abc.ABC):
         # This ensures that the media in those columns resides in the media store.
         # First determine which columns (if any) need stored proxies, but don't have one yet.
         stored_proxies_needed: list[Column] = []
-        for col in self.col_mapping:
+        for col_handle in self.col_mapping:
+            col = col_handle.get()
             if col.col_type.is_media_type() and not (col.is_stored and col.is_computed):
                 # If this column is already proxied in some other Project, use the existing proxy to avoid
                 # duplication. Otherwise, we'll create a new one.
                 for store in tbl_version.external_stores.values():
-                    if isinstance(store, Project) and col in store.stored_proxies:
-                        self.stored_proxies[col] = store.stored_proxies[col]
+                    if isinstance(store, Project) and col_handle in store.stored_proxies:
+                        self.stored_proxies[col_handle] = store.stored_proxies[col_handle]
                         break
-                if col not in self.stored_proxies:
+                if col_handle not in self.stored_proxies:
                     # We didn't find it in an existing Project
                     stored_proxies_needed.append(col)
 
@@ -110,17 +119,20 @@ class Project(ExternalStore, abc.ABC):
             proxy_cols = [self.create_stored_proxy(col) for col in stored_proxies_needed]
             # Add the columns; this will also update table metadata.
             tbl_version.add_columns(proxy_cols, print_stats=False, on_error='ignore')
+            self.stored_proxies.update(
+                {col.handle: proxy_col.handle for col, proxy_col in zip(stored_proxies_needed, proxy_cols)}
+            )
 
     def unlink(self, tbl_version: TableVersion) -> None:
         # Determine which stored proxies can be deleted. (A stored proxy can be deleted if it is not referenced by
         # any *other* external store for this table.)
-        deletions_needed: set[Column] = set(self.stored_proxies.values())
+        deletions_needed: set[ColumnHandle] = set(self.stored_proxies.values())
         for name, store in tbl_version.external_stores.items():
             if isinstance(store, Project) and name != self.name:
                 deletions_needed = deletions_needed.difference(set(store.stored_proxies.values()))
         if len(deletions_needed) > 0:
-            _logger.info(f'Removing stored proxies for columns: {[col.name for col in deletions_needed]}')
-            tbl_version._drop_columns(deletions_needed)
+            _logger.info(f'Removing stored proxies for columns: {[col.get().name for col in deletions_needed]}')
+            tbl_version._drop_columns(col.get() for col in deletions_needed)
             self.stored_proxies.clear()
 
     def create_stored_proxy(self, col: Column) -> Column:
@@ -142,11 +154,10 @@ class Project(ExternalStore, abc.ABC):
             computed_with=exprs.ColumnRef(col).apply(lambda x: x, col_type=col.col_type),
             stored=True,
         )
-        self.stored_proxies[col] = proxy_col
         return proxy_col
 
     @property
-    def col_mapping(self) -> dict[Column, str]:
+    def col_mapping(self) -> dict[ColumnHandle, str]:
         return self._col_mapping
 
     @abc.abstractmethod
@@ -181,7 +192,7 @@ class Project(ExternalStore, abc.ABC):
         export_cols: dict[str, ts.ColumnType],
         import_cols: dict[str, ts.ColumnType],
         col_mapping: Optional[dict[str, str]],
-    ) -> dict[Column, str]:
+    ) -> dict[ColumnHandle, str]:
         """
         Verifies that the specified `col_mapping` is valid. In particular, checks that:
         (i) the keys of `col_mapping` are valid columns of the specified `Table`;
@@ -199,7 +210,7 @@ class Project(ExternalStore, abc.ABC):
         if col_mapping is None:
             col_mapping = {col: col for col in itertools.chain(export_cols.keys(), import_cols.keys())}
 
-        resolved_col_mapping: dict[Column, str] = {}
+        resolved_col_mapping: dict[ColumnHandle, str] = {}
 
         # Validate names
         t_cols = set(table._get_schema().keys())
@@ -223,7 +234,8 @@ class Project(ExternalStore, abc.ABC):
                 )
             col_ref = table[t_col]
             assert isinstance(col_ref, exprs.ColumnRef)
-            resolved_col_mapping[col_ref.col] = ext_col
+            resolved_col_mapping[col_ref.col.handle] = ext_col
+
         # Validate column specs
         t_col_types = table._get_schema()
         for t_col, ext_col in col_mapping.items():
@@ -250,39 +262,56 @@ class Project(ExternalStore, abc.ABC):
                     )
         return resolved_col_mapping
 
-    @classmethod
-    def _column_as_dict(cls, col: Column) -> dict[str, Any]:
-        return {'tbl_id': str(col.tbl.id), 'col_id': col.id}
 
-    @classmethod
-    def _column_from_dict(cls, d: dict[str, Any]) -> Column:
-        from pixeltable.catalog import Catalog
+@dataclass(frozen=True)
+class SyncStatus:
+    # stats for the rows affected by the operation in the external store
+    ext_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
 
-        tbl_id = UUID(d['tbl_id'])
-        col_id = d['col_id']
-        return Catalog.get().get_tbl_version(tbl_id, None).cols_by_id[col_id]
+    # stats for the rows affected by the operation
+    row_count_stats: RowCountStats = field(default_factory=RowCountStats)
 
+    @property
+    def num_excs(self) -> int:
+        """
+        Returns the total number of Pixeltable exceptions that occurred during the operation.
+        """
+        return self.row_count_stats.num_excs
 
-@dataclass(frozen=True)
-class SyncStatus:
-    external_rows_created: int = 0
-    external_rows_deleted: int = 0
-    external_rows_updated: int = 0
-    pxt_rows_updated: int = 0
-    num_excs: int = 0
+    @property
+    def pxt_rows_updated(self) -> int:
+        """
+        Returns the number of Pixeltable rows that were updated as a result of the operation.
+        """
+        return self.row_count_stats.upd_rows
+
+    @property
+    def external_rows_updated(self) -> int:
+        return self.ext_row_count_stats.upd_rows
+
+    @property
+    def external_rows_created(self) -> int:
+        return self.ext_row_count_stats.ins_rows
+
+    @property
+    def external_rows_deleted(self) -> int:
+        return self.ext_row_count_stats.del_rows
 
-    def combine(self, other: 'SyncStatus') -> 'SyncStatus':
+    def __add__(self, other: 'SyncStatus') -> 'SyncStatus':
+        """
+        Add the sync status from two SyncStatus objects together.
+        """
         return SyncStatus(
-            external_rows_created=self.external_rows_created + other.external_rows_created,
-            external_rows_deleted=self.external_rows_deleted + other.external_rows_deleted,
-            external_rows_updated=self.external_rows_updated + other.external_rows_updated,
-            pxt_rows_updated=self.pxt_rows_updated + other.pxt_rows_updated,
-            num_excs=self.num_excs + other.num_excs,
+            ext_row_count_stats=self.ext_row_count_stats + other.ext_row_count_stats,
+            row_count_stats=self.row_count_stats + other.row_count_stats,
         )
 
     @classmethod
-    def empty(cls) -> 'SyncStatus':
-        return SyncStatus(0, 0, 0, 0, 0)
+    def from_update_status(cls, us: UpdateStatus) -> 'SyncStatus':
+        """
+        Copy information from an UpdateStatus to a SyncStatus.
+        """
+        return SyncStatus(row_count_stats=us.row_count_stats + us.cascade_row_count_stats)
 
 
 class MockProject(Project):
@@ -293,8 +322,8 @@ class MockProject(Project):
         name: str,
         export_cols: dict[str, ts.ColumnType],
         import_cols: dict[str, ts.ColumnType],
-        col_mapping: dict[Column, str],
-        stored_proxies: Optional[dict[Column, Column]] = None,
+        col_mapping: dict[ColumnHandle, str],
+        stored_proxies: Optional[dict[ColumnHandle, ColumnHandle]] = None,
     ):
         super().__init__(name, col_mapping, stored_proxies)
         self.export_cols = export_cols
@@ -334,10 +363,8 @@ class MockProject(Project):
             'name': self.name,
             'export_cols': {k: v.as_dict() for k, v in self.export_cols.items()},
             'import_cols': {k: v.as_dict() for k, v in self.import_cols.items()},
-            'col_mapping': [[self._column_as_dict(k), v] for k, v in self.col_mapping.items()],
-            'stored_proxies': [
-                [self._column_as_dict(k), self._column_as_dict(v)] for k, v in self.stored_proxies.items()
-            ],
+            'col_mapping': [[k.as_dict(), v] for k, v in self.col_mapping.items()],
+            'stored_proxies': [[k.as_dict(), v.as_dict()] for k, v in self.stored_proxies.items()],
         }
 
     @classmethod
@@ -346,8 +373,8 @@ class MockProject(Project):
             md['name'],
             {k: ts.ColumnType.from_dict(v) for k, v in md['export_cols'].items()},
             {k: ts.ColumnType.from_dict(v) for k, v in md['import_cols'].items()},
-            {cls._column_from_dict(entry[0]): entry[1] for entry in md['col_mapping']},
-            {cls._column_from_dict(entry[0]): cls._column_from_dict(entry[1]) for entry in md['stored_proxies']},
+            {ColumnHandle.from_dict(entry[0]): entry[1] for entry in md['col_mapping']},
+            {ColumnHandle.from_dict(entry[0]): ColumnHandle.from_dict(entry[1]) for entry in md['stored_proxies']},
         )
 
     def __eq__(self, other: object) -> bool:

pixeltable/io/globals.py

@@ -136,7 +136,7 @@ def create_label_studio_project(
     if sync_immediately:
         return t.sync()
     else:
-        return SyncStatus.empty()
+        return SyncStatus()
 
 
 def export_images_as_fo_dataset(

pixeltable/io/label_studio.py

@@ -4,15 +4,17 @@ import logging
 import os
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Iterator, Literal, Optional, cast
+from typing import Any, Iterator, Literal, Optional
 from xml.etree import ElementTree as ET
 
-import label_studio_sdk  # type: ignore[import-untyped]
+import label_studio_sdk
 import PIL.Image
 from requests.exceptions import HTTPError
 
 import pixeltable.type_system as ts
 from pixeltable import Column, Table, env, exceptions as excs
+from pixeltable.catalog import ColumnHandle
+from pixeltable.catalog.globals import RowCountStats
 from pixeltable.config import Config
 from pixeltable.exprs import ColumnRef, DataRow, Expr
 from pixeltable.io.external_store import Project, SyncStatus
@@ -25,7 +27,7 @@ try:
     import label_studio_sdk.project as ls_project  # type: ignore
 except ImportError:
     # label_studio_sdk>=1 compatibility
-    import label_studio_sdk._legacy.project as ls_project  # type: ignore
+    import label_studio_sdk._legacy.project as ls_project
 
 _logger = logging.getLogger('pixeltable')
 
@@ -45,13 +47,17 @@ class LabelStudioProject(Project):
     for synchronizing between a Pixeltable table and a Label Studio project.
     """
 
+    project_id: int  # Label Studio project ID
+    media_import_method: Literal['post', 'file', 'url']
+    _project: Optional[ls_project.Project]
+
     def __init__(
         self,
         name: str,
         project_id: int,
         media_import_method: Literal['post', 'file', 'url'],
-        col_mapping: dict[Column, str],
-        stored_proxies: Optional[dict[Column, Column]] = None,
+        col_mapping: dict[ColumnHandle, str],
+        stored_proxies: Optional[dict[ColumnHandle, ColumnHandle]] = None,
     ):
         """
         The constructor will NOT create a new Label Studio project; it is also used when loading
@@ -59,7 +65,7 @@ class LabelStudioProject(Project):
         """
         self.project_id = project_id
         self.media_import_method = media_import_method
-        self._project: Optional[ls_project.Project] = None
+        self._project = None
         super().__init__(name, col_mapping, stored_proxies)
 
     @property
@@ -112,13 +118,13 @@ class LabelStudioProject(Project):
         )
         # Collect all existing tasks into a dict with entries `rowid: task`
         tasks = {tuple(task['meta']['rowid']): task for task in self.__fetch_all_tasks()}
-        sync_status = SyncStatus.empty()
+        sync_status = SyncStatus()
         if export_data:
             export_sync_status = self.__update_tasks(t, tasks)
-            sync_status = sync_status.combine(export_sync_status)
+            sync_status += export_sync_status
         if import_data:
             import_sync_status = self.__update_table_from_tasks(t, tasks)
-            sync_status = sync_status.combine(import_sync_status)
+            sync_status += import_sync_status
         return sync_status
 
     def __fetch_all_tasks(self) -> Iterator[dict[str, Any]]:
@@ -155,7 +161,7 @@ class LabelStudioProject(Project):
         t_data_cols = [t_col for t_col, ext_col_name in self.col_mapping.items() if ext_col_name in config.data_keys]
 
         if len(t_data_cols) == 0:
-            return SyncStatus.empty()
+            return SyncStatus()
 
         # Columns in `t` that map to `rectanglelabels` preannotations
         t_rl_cols = [
@@ -183,15 +189,15 @@ class LabelStudioProject(Project):
         self,
         t: Table,
         existing_tasks: dict[tuple, dict],
-        media_col: Column,
-        t_rl_cols: list[Column],
+        media_col: ColumnHandle,
+        t_rl_cols: list[ColumnHandle],
         rl_info: list['_RectangleLabel'],
     ) -> SyncStatus:
-        is_stored = media_col.is_stored
+        is_stored = media_col.get().is_stored
         # If it's a stored column, we can use `localpath`
-        localpath_col_opt = [t[media_col.name].localpath] if is_stored else []
+        localpath_col_opt = [t[media_col.get().name].localpath] if is_stored else []
         # Select the media column, rectanglelabels columns, and localpath (if appropriate)
-        rows = t.select(t[media_col.name], *[t[col.name] for col in t_rl_cols], *localpath_col_opt)
+        rows = t.select(t[media_col.get().name], *[t[col.get().name] for col in t_rl_cols], *localpath_col_opt)
         tasks_created = 0
         row_ids_in_pxt: set[tuple] = set()
 
@@ -232,42 +238,42 @@ class LabelStudioProject(Project):
 
         env.Env.get().console_logger.info(f'Created {tasks_created} new task(s) in {self}.')
 
-        sync_status = SyncStatus(external_rows_created=tasks_created)
+        sync_status = SyncStatus(ext_row_count_stats=RowCountStats(ins_rows=tasks_created))
 
         deletion_sync_status = self.__delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
-
-        return sync_status.combine(deletion_sync_status)
+        sync_status += deletion_sync_status
+        return sync_status
 
     def __update_tasks_by_files(
         self,
         t: Table,
         existing_tasks: dict[tuple, dict],
-        t_data_cols: list[Column],
-        t_rl_cols: list[Column],
+        t_data_cols: list[ColumnHandle],
+        t_rl_cols: list[ColumnHandle],
         rl_info: list['_RectangleLabel'],
     ) -> SyncStatus:
         ext_data_cols = [self.col_mapping[col] for col in t_data_cols]
         expr_refs: dict[str, Expr] = {}  # kwargs for the select statement
         for col in t_data_cols:
-            col_name = col.name
+            col_name = col.get().name
             if self.media_import_method == 'url':
                 expr_refs[col_name] = t[col_name].fileurl
             else:
                 assert self.media_import_method == 'file'
-                if not col.col_type.is_media_type():
+                if not col.get().col_type.is_media_type():
                     # Not a media column; query the data directly
-                    expr_refs[col_name] = cast(ColumnRef, t[col_name])
+                    expr_refs[col_name] = t[col_name]
                 elif col in self.stored_proxies:
                     # Media column that has a stored proxy; use it. We have to give it a name,
                     # since it's an anonymous column
-                    stored_proxy_col = self.stored_proxies[col]
+                    stored_proxy_col = self.stored_proxies[col].get()
                     expr_refs[f'{col_name}_proxy'] = ColumnRef(stored_proxy_col).localpath
                 else:
                     # Media column without a stored proxy; this means it's a stored computed column,
                     # and we can just use the localpath
                     expr_refs[col_name] = t[col_name].localpath
 
-        df = t.select(*[t[col.name] for col in t_rl_cols], **expr_refs)
+        df = t.select(*[t[col.get().name] for col in t_rl_cols], **expr_refs)
         # The following buffers will hold `DataRow` indices that correspond to each of the selected
         # columns. `rl_col_idxs` holds the indices for the columns that map to RectangleLabels
         # preannotations; `data_col_idxs` holds the indices for the columns that map to data fields.
@@ -286,11 +292,11 @@ class LabelStudioProject(Project):
             data_vals = [row[idx] for idx in data_col_idxs]
             coco_annotations = [row[idx] for idx in rl_col_idxs]
             for i in range(len(t_data_cols)):
-                if t_data_cols[i].col_type.is_media_type():
+                if t_data_cols[i].get().col_type.is_media_type():
                     # Special handling for media columns
                     assert isinstance(data_vals[i], str)
                     if self.media_import_method == 'url':
-                        data_vals[i] = self.__validate_fileurl(t_data_cols[i], data_vals[i])
+                        data_vals[i] = self.__validate_fileurl(t_data_cols[i].get(), data_vals[i])
                     else:
                         assert self.media_import_method == 'file'
                         data_vals[i] = self.__localpath_to_lspath(data_vals[i])
@@ -336,11 +342,11 @@ class LabelStudioProject(Project):
             f'Created {tasks_created} new task(s) and updated {tasks_updated} existing task(s) in {self}.'
         )
 
-        sync_status = SyncStatus(external_rows_created=tasks_created, external_rows_updated=tasks_updated)
+        sync_status = SyncStatus(ext_row_count_stats=RowCountStats(ins_rows=tasks_created, upd_rows=tasks_updated))
 
         deletion_sync_status = self.__delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
-
-        return sync_status.combine(deletion_sync_status)
+        sync_status += deletion_sync_status
+        return sync_status
 
     @classmethod
     def __validate_fileurl(cls, col: Column, url: str) -> Optional[str]:
@@ -377,11 +383,11 @@ class LabelStudioProject(Project):
         for rowid in deleted_rowids:
             del existing_tasks[rowid]
 
-        return SyncStatus(external_rows_deleted=len(deleted_rowids))
+        return SyncStatus(ext_row_count_stats=RowCountStats(del_rows=len(deleted_rowids)))
 
     def __update_table_from_tasks(self, t: Table, tasks: dict[tuple, dict]) -> SyncStatus:
         if ANNOTATIONS_COLUMN not in self.col_mapping.values():
-            return SyncStatus.empty()
+            return SyncStatus()
 
         annotations = {
             # Replace [] by None to indicate no annotations. We do want to sync rows with no annotations,
@@ -391,7 +397,7 @@ class LabelStudioProject(Project):
             for task in tasks.values()
         }
 
-        local_annotations_col = next(k for k, v in self.col_mapping.items() if v == ANNOTATIONS_COLUMN)
+        local_annotations_col = next(k for k, v in self.col_mapping.items() if v == ANNOTATIONS_COLUMN).get()
 
         # Prune the annotations down to just the ones that have actually changed.
         rows = t.select(t[local_annotations_col.name])
@@ -416,19 +422,17 @@ class LabelStudioProject(Project):
                 ancestor = ancestor._get_base_table()
             update_status = ancestor.batch_update(updates)
             env.Env.get().console_logger.info(f'Updated annotation(s) from {len(updates)} task(s) in {self}.')
-            return SyncStatus(pxt_rows_updated=update_status.num_rows, num_excs=update_status.num_excs)
+            return SyncStatus.from_update_status(update_status)
         else:
-            return SyncStatus.empty()
+            return SyncStatus()
 
     def as_dict(self) -> dict[str, Any]:
         return {
             'name': self.name,
             'project_id': self.project_id,
             'media_import_method': self.media_import_method,
-            'col_mapping': [[self._column_as_dict(k), v] for k, v in self.col_mapping.items()],
-            'stored_proxies': [
-                [self._column_as_dict(k), self._column_as_dict(v)] for k, v in self.stored_proxies.items()
-            ],
+            'col_mapping': [[k.as_dict(), v] for k, v in self.col_mapping.items()],
+            'stored_proxies': [[k.as_dict(), v.as_dict()] for k, v in self.stored_proxies.items()],
         }
 
     @classmethod
@@ -437,8 +441,8 @@ class LabelStudioProject(Project):
             md['name'],
             md['project_id'],
             md['media_import_method'],
-            {cls._column_from_dict(entry[0]): entry[1] for entry in md['col_mapping']},
-            {cls._column_from_dict(entry[0]): cls._column_from_dict(entry[1]) for entry in md['stored_proxies']},
+            {ColumnHandle.from_dict(entry[0]): entry[1] for entry in md['col_mapping']},
+            {ColumnHandle.from_dict(entry[0]): ColumnHandle.from_dict(entry[1]) for entry in md['stored_proxies']},
         )
 
     def __repr__(self) -> str:

pixeltable/metadata/__init__.py

@@ -18,7 +18,7 @@ _console_logger = ConsoleLogger(logging.getLogger('pixeltable'))
 _logger = logging.getLogger('pixeltable')
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 38
+VERSION = 39
 
 
 def create_system_info(engine: sql.engine.Engine) -> None: