pixeltable 0.4.2__py3-none-any.whl → 0.4.3__py3-none-any.whl

pixeltable/functions/audio.py

@@ -1,14 +1,5 @@
 """
 Pixeltable [UDFs](https://pixeltable.readme.io/docs/user-defined-functions-udfs) for `AudioType`.
-
-Example:
-```python
-import pixeltable as pxt
-import pixeltable.functions as pxtf
-
-t = pxt.get_table(...)
-t.select(pxtf.audio.get_metadata()).collect()
-```
 """
 
 import pixeltable as pxt
@@ -19,6 +10,42 @@ from pixeltable.utils.code import local_public_names
 def get_metadata(audio: pxt.Audio) -> dict:
     """
     Gets various metadata associated with an audio file and returns it as a dictionary.
+
+    Args:
+        audio: The audio to get metadata for.
+
+    Returns:
+        A `dict` such as the following:
+
+            ```json
+            {
+                'size': 2568827,
+                'streams': [
+                    {
+                        'type': 'audio',
+                        'frames': 0,
+                        'duration': 2646000,
+                        'metadata': {},
+                        'time_base': 2.2675736961451248e-05,
+                        'codec_context': {
+                            'name': 'flac',
+                            'profile': None,
+                            'channels': 1,
+                            'codec_tag': '\\x00\\x00\\x00\\x00',
+                        },
+                        'duration_seconds': 60.0,
+                    }
+                ],
+                'bit_rate': 342510,
+                'metadata': {'encoder': 'Lavf61.1.100'},
+                'bit_exact': False,
+            }
+            ```
+
+    Examples:
+        Extract metadata for files in the `audio_col` column of the table `tbl`:
+
+        >>> tbl.select(tbl.audio_col.get_metadata()).collect()
     """
     return pxt.functions.video._get_metadata(audio)
 

pixeltable/functions/video.py

@@ -1,14 +1,5 @@
 """
 Pixeltable [UDFs](https://pixeltable.readme.io/docs/user-defined-functions-udfs) for `VideoType`.
-
-Example:
-```python
-import pixeltable as pxt
-import pixeltable.functions as pxtf
-
-t = pxt.get_table(...)
-t.select(pxtf.video.extract_audio(t.video_col)).collect()
-```
 """
 
 import tempfile
@@ -92,12 +83,22 @@ def extract_audio(
     video_path: pxt.Video, stream_idx: int = 0, format: str = 'wav', codec: Optional[str] = None
 ) -> pxt.Audio:
     """
-    Extract an audio stream from a video file, save it as a media file and return its path.
+    Extract an audio stream from a video.
 
     Args:
         stream_idx: Index of the audio stream to extract.
         format: The target audio format. (`'wav'`, `'mp3'`, `'flac'`).
         codec: The codec to use for the audio stream. If not provided, a default codec will be used.
+
+    Returns:
+        The extracted audio.
+
+    Examples:
+        Add a computed column to a table `tbl` that extracts audio from an existing column `video_col`:
+
+        >>> tbl.add_computed_column(
+        ...     extracted_audio=tbl.video_col.extract_audio(format='flac')
+        ... )
     """
     if format not in _format_defaults:
         raise ValueError(f'extract_audio(): unsupported audio format: {format}')
@@ -124,6 +125,52 @@ def extract_audio(
 def get_metadata(video: pxt.Video) -> dict:
     """
     Gets various metadata associated with a video file and returns it as a dictionary.
+
+    Args:
+        video: The video to get metadata for.
+
+    Returns:
+        A `dict` such as the following:
+
+            ```json
+            {
+                'bit_exact': False,
+                'bit_rate': 967260,
+                'size': 2234371,
+                'metadata': {
+                    'encoder': 'Lavf60.16.100',
+                    'major_brand': 'isom',
+                    'minor_version': '512',
+                    'compatible_brands': 'isomiso2avc1mp41',
+                },
+                'streams': [
+                    {
+                        'type': 'video',
+                        'width': 640,
+                        'height': 360,
+                        'frames': 462,
+                        'time_base': 1.0 / 12800,
+                        'duration': 236544,
+                        'duration_seconds': 236544.0 / 12800,
+                        'average_rate': 25.0,
+                        'base_rate': 25.0,
+                        'guessed_rate': 25.0,
+                        'metadata': {
+                            'language': 'und',
+                            'handler_name': 'L-SMASH Video Handler',
+                            'vendor_id': '[0][0][0][0]',
+                            'encoder': 'Lavc60.31.102 libx264',
+                        },
+                        'codec_context': {'name': 'h264', 'codec_tag': 'avc1', 'profile': 'High', 'pix_fmt': 'yuv420p'},
+                    }
+                ],
+            }
+            ```
+
+    Examples:
+        Extract metadata for files in the `video_col` column of the table `tbl`:
+
+        >>> tbl.select(tbl.video_col.get_metadata()).collect()
     """
     return _get_metadata(video)
 

pixeltable/globals.py

@@ -11,6 +11,7 @@ from pandas.io.formats.style import Styler
 from pixeltable import DataFrame, catalog, exceptions as excs, exprs, func, share
 from pixeltable.catalog import Catalog, TableVersionPath
 from pixeltable.catalog.insertable_table import OnErrorParameter
+from pixeltable.config import Config
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
 
@@ -34,8 +35,11 @@ if TYPE_CHECKING:
 _logger = logging.getLogger('pixeltable')
 
 
-def init() -> None:
+def init(config_overrides: Optional[dict[str, Any]] = None) -> None:
     """Initializes the Pixeltable environment."""
+    if config_overrides is None:
+        config_overrides = {}
+    Config.init(config_overrides)
     _ = Catalog.get()
 
 
@@ -633,6 +637,62 @@ def drop_dir(path: str, force: bool = False, if_not_exists: Literal['error', 'ig
     Catalog.get().drop_dir(path_obj, if_not_exists=if_not_exists_, force=force)
 
 
+def ls(path: str = '') -> pd.DataFrame:
+    """
+    List the contents of a Pixeltable directory.
+
+    This function returns a Pandas DataFrame representing a human-readable listing of the specified directory,
+    including various attributes such as version and base table, as appropriate.
+
+    To get a programmatic list of tables and/or directories, use [list_tables()][pixeltable.list_tables] and/or
+    [list_dirs()][pixeltable.list_dirs] instead.
+    """
+    from pixeltable.metadata import schema
+
+    cat = Catalog.get()
+    path_obj = catalog.Path(path, empty_is_valid=True)
+    dir_entries = cat.get_dir_contents(path_obj)
+    rows: list[list[str]] = []
+    with Catalog.get().begin_xact():
+        for name, entry in dir_entries.items():
+            if name.startswith('_'):
+                continue
+            if entry.dir is not None:
+                kind = 'dir'
+                version = ''
+                base = ''
+            else:
+                assert entry.table is not None
+                assert isinstance(entry.table, schema.Table)
+                tbl = cat.get_table_by_id(entry.table.id)
+                md = tbl.get_metadata()
+                base = md['base'] or ''
+                if base.startswith('_'):
+                    base = '<anonymous base table>'
+                if md['is_snapshot']:
+                    kind = 'snapshot'
+                elif md['is_view']:
+                    kind = 'view'
+                else:
+                    kind = 'table'
+                version = '' if kind == 'snapshot' else md['version']
+                if md['is_replica']:
+                    kind = f'{kind}-replica'
+            rows.append([name, kind, version, base])
+
+    rows = sorted(rows, key=lambda x: x[0])
+    df = pd.DataFrame(
+        {
+            'Name': [row[0] for row in rows],
+            'Kind': [row[1] for row in rows],
+            'Version': [row[2] for row in rows],
+            'Base': [row[3] for row in rows],
+        },
+        index=([''] * len(rows)),
+    )
+    return df
+
+
 def _extract_paths(
     dir_entries: dict[str, Catalog.DirEntry],
     parent: catalog.Path,

pixeltable/io/__init__.py

@@ -1,7 +1,7 @@
 # ruff: noqa: F401
 
 from .datarows import import_json, import_rows
-from .external_store import ExternalStore, SyncStatus
+from .external_store import ExternalStore
 from .globals import create_label_studio_project, export_images_as_fo_dataset
 from .hf_datasets import import_huggingface_dataset
 from .pandas import import_csv, import_excel, import_pandas

pixeltable/io/external_store.py

@@ -3,14 +3,13 @@ from __future__ import annotations
 import abc
 import itertools
 import logging
-from dataclasses import dataclass, field
 from typing import Any, Optional
 
 import pixeltable.exceptions as excs
 import pixeltable.type_system as ts
 from pixeltable import Column, Table
 from pixeltable.catalog import ColumnHandle, TableVersion
-from pixeltable.catalog.globals import RowCountStats, UpdateStatus
+from pixeltable.catalog.update_status import UpdateStatus
 
 _logger = logging.getLogger('pixeltable')
 
@@ -46,7 +45,7 @@ class ExternalStore(abc.ABC):
         """
 
     @abc.abstractmethod
-    def sync(self, t: Table, export_data: bool, import_data: bool) -> SyncStatus:
+    def sync(self, t: Table, export_data: bool, import_data: bool) -> UpdateStatus:
         """
         Called by `Table.sync()` to implement store-specific synchronization logic.
         """
@@ -263,57 +262,6 @@ class Project(ExternalStore, abc.ABC):
         return resolved_col_mapping
 
 
-@dataclass(frozen=True)
-class SyncStatus:
-    # stats for the rows affected by the operation in the external store
-    ext_row_count_stats: RowCountStats = field(default_factory=RowCountStats)
-
-    # stats for the rows affected by the operation
-    row_count_stats: RowCountStats = field(default_factory=RowCountStats)
-
-    @property
-    def num_excs(self) -> int:
-        """
-        Returns the total number of Pixeltable exceptions that occurred during the operation.
-        """
-        return self.row_count_stats.num_excs
-
-    @property
-    def pxt_rows_updated(self) -> int:
-        """
-        Returns the number of Pixeltable rows that were updated as a result of the operation.
-        """
-        return self.row_count_stats.upd_rows
-
-    @property
-    def external_rows_updated(self) -> int:
-        return self.ext_row_count_stats.upd_rows
-
-    @property
-    def external_rows_created(self) -> int:
-        return self.ext_row_count_stats.ins_rows
-
-    @property
-    def external_rows_deleted(self) -> int:
-        return self.ext_row_count_stats.del_rows
-
-    def __add__(self, other: 'SyncStatus') -> 'SyncStatus':
-        """
-        Add the sync status from two SyncStatus objects together.
-        """
-        return SyncStatus(
-            ext_row_count_stats=self.ext_row_count_stats + other.ext_row_count_stats,
-            row_count_stats=self.row_count_stats + other.row_count_stats,
-        )
-
-    @classmethod
-    def from_update_status(cls, us: UpdateStatus) -> 'SyncStatus':
-        """
-        Copy information from an UpdateStatus to a SyncStatus.
-        """
-        return SyncStatus(row_count_stats=us.row_count_stats + us.cascade_row_count_stats)
-
-
 class MockProject(Project):
     """A project that cannot be synced, used mainly for testing."""
 
@@ -348,7 +296,7 @@ class MockProject(Project):
     def get_import_columns(self) -> dict[str, ts.ColumnType]:
         return self.import_cols
 
-    def sync(self, t: Table, export_data: bool, import_data: bool) -> SyncStatus:
+    def sync(self, t: Table, export_data: bool, import_data: bool) -> UpdateStatus:
         raise NotImplementedError()
 
     def delete(self) -> None:

pixeltable/io/globals.py

@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union
 import pixeltable as pxt
 import pixeltable.exceptions as excs
 from pixeltable import Table, exprs
+from pixeltable.catalog.update_status import UpdateStatus
 from pixeltable.env import Env
-from pixeltable.io.external_store import SyncStatus
 
 if TYPE_CHECKING:
     import fiftyone as fo  # type: ignore[import-untyped]
@@ -22,7 +22,7 @@ def create_label_studio_project(
     sync_immediately: bool = True,
     s3_configuration: Optional[dict[str, Any]] = None,
     **kwargs: Any,
-) -> SyncStatus:
+) -> UpdateStatus:
     """
     Create a new Label Studio project and link it to the specified [`Table`][pixeltable.Table].
 
@@ -96,7 +96,7 @@ def create_label_studio_project(
             [Label Studio start_project docs](https://labelstud.io/sdk/project.html#label_studio_sdk.project.Project.start_project).
 
     Returns:
-        A `SyncStatus` representing the status of any synchronization operations that occurred.
+        An `UpdateStatus` representing the status of any synchronization operations that occurred.
 
     Examples:
         Create a Label Studio project whose tasks correspond to videos stored in the `video_col`
@@ -136,7 +136,7 @@ def create_label_studio_project(
     if sync_immediately:
         return t.sync()
     else:
-        return SyncStatus()
+        return UpdateStatus()
 
 
 def export_images_as_fo_dataset(

pixeltable/io/hf_datasets.py

@@ -50,10 +50,18 @@ def _to_pixeltable_type(feature_type: Any, nullable: bool) -> Optional[ts.Column
     elif isinstance(feature_type, datasets.Sequence):
         # example: cohere wiki. Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None)
         dtype = _to_pixeltable_type(feature_type.feature, nullable)
-        length = feature_type.length if feature_type.length != -1 else None
-        return ts.ArrayType(shape=(length,), dtype=dtype)
+        if dtype is None:
+            return None
+        if dtype.is_int_type() or dtype.is_float_type() or dtype.is_bool_type() or dtype.is_string_type():
+            length = feature_type.length if feature_type.length != -1 else None
+            return ts.ArrayType(shape=(length,), dtype=dtype, nullable=nullable)
+        else:
+            # Sequence of dicts must be cast as Json
+            return ts.JsonType(nullable=nullable)
     elif isinstance(feature_type, datasets.Image):
         return ts.ImageType(nullable=nullable)
+    elif isinstance(feature_type, dict):
+        return ts.JsonType(nullable=nullable)
     else:
         return None
 

pixeltable/io/label_studio.py

@@ -14,10 +14,10 @@ from requests.exceptions import HTTPError
 import pixeltable.type_system as ts
 from pixeltable import Column, Table, env, exceptions as excs
 from pixeltable.catalog import ColumnHandle
-from pixeltable.catalog.globals import RowCountStats
+from pixeltable.catalog.update_status import RowCountStats, UpdateStatus
 from pixeltable.config import Config
 from pixeltable.exprs import ColumnRef, DataRow, Expr
-from pixeltable.io.external_store import Project, SyncStatus
+from pixeltable.io.external_store import Project
 from pixeltable.utils import coco
 
 # label_studio_sdk>=1 and label_studio_sdk<1 are not compatible, so we need to try
@@ -111,14 +111,14 @@ class LabelStudioProject(Project):
         """
         return {ANNOTATIONS_COLUMN: ts.JsonType(nullable=True)}
 
-    def sync(self, t: Table, export_data: bool, import_data: bool) -> SyncStatus:
+    def sync(self, t: Table, export_data: bool, import_data: bool) -> UpdateStatus:
         _logger.info(
             f'Syncing Label Studio project "{self.project_title}" with table `{t._name}`'
             f' (export: {export_data}, import: {import_data}).'
         )
         # Collect all existing tasks into a dict with entries `rowid: task`
         tasks = {tuple(task['meta']['rowid']): task for task in self.__fetch_all_tasks()}
-        sync_status = SyncStatus()
+        sync_status = UpdateStatus()
         if export_data:
             export_sync_status = self.__update_tasks(t, tasks)
             sync_status += export_sync_status
@@ -148,7 +148,7 @@ class LabelStudioProject(Project):
                 f'Label Studio project {self.project_title!r}.'
             )
 
-    def __update_tasks(self, t: Table, existing_tasks: dict[tuple, dict]) -> SyncStatus:
+    def __update_tasks(self, t: Table, existing_tasks: dict[tuple, dict]) -> UpdateStatus:
         """
         Updates all tasks in this Label Studio project based on the Pixeltable data:
         - Creates new tasks for rows that don't map to any existing task;
@@ -161,7 +161,7 @@ class LabelStudioProject(Project):
         t_data_cols = [t_col for t_col, ext_col_name in self.col_mapping.items() if ext_col_name in config.data_keys]
 
         if len(t_data_cols) == 0:
-            return SyncStatus()
+            return UpdateStatus()
 
         # Columns in `t` that map to `rectanglelabels` preannotations
         t_rl_cols = [
@@ -192,7 +192,7 @@ class LabelStudioProject(Project):
         media_col: ColumnHandle,
         t_rl_cols: list[ColumnHandle],
         rl_info: list['_RectangleLabel'],
-    ) -> SyncStatus:
+    ) -> UpdateStatus:
         is_stored = media_col.get().is_stored
         # If it's a stored column, we can use `localpath`
         localpath_col_opt = [t[media_col.get().name].localpath] if is_stored else []
@@ -238,7 +238,7 @@ class LabelStudioProject(Project):
 
         env.Env.get().console_logger.info(f'Created {tasks_created} new task(s) in {self}.')
 
-        sync_status = SyncStatus(ext_row_count_stats=RowCountStats(ins_rows=tasks_created))
+        sync_status = UpdateStatus(ext_row_count_stats=RowCountStats(ins_rows=tasks_created))
 
         deletion_sync_status = self.__delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
         sync_status += deletion_sync_status
@@ -251,7 +251,7 @@ class LabelStudioProject(Project):
         t_data_cols: list[ColumnHandle],
         t_rl_cols: list[ColumnHandle],
         rl_info: list['_RectangleLabel'],
-    ) -> SyncStatus:
+    ) -> UpdateStatus:
         ext_data_cols = [self.col_mapping[col] for col in t_data_cols]
         expr_refs: dict[str, Expr] = {}  # kwargs for the select statement
         for col in t_data_cols:
@@ -342,7 +342,7 @@ class LabelStudioProject(Project):
             f'Created {tasks_created} new task(s) and updated {tasks_updated} existing task(s) in {self}.'
         )
 
-        sync_status = SyncStatus(ext_row_count_stats=RowCountStats(ins_rows=tasks_created, upd_rows=tasks_updated))
+        sync_status = UpdateStatus(ext_row_count_stats=RowCountStats(ins_rows=tasks_created, upd_rows=tasks_updated))
 
         deletion_sync_status = self.__delete_stale_tasks(existing_tasks, row_ids_in_pxt, tasks_created)
         sync_status += deletion_sync_status
@@ -367,7 +367,7 @@ class LabelStudioProject(Project):
 
     def __delete_stale_tasks(
         self, existing_tasks: dict[tuple, dict], row_ids_in_pxt: set[tuple], tasks_created: int
-    ) -> SyncStatus:
+    ) -> UpdateStatus:
         deleted_rowids = set(existing_tasks.keys()) - row_ids_in_pxt
         # Sanity check the math
         assert len(deleted_rowids) == len(existing_tasks) + tasks_created - len(row_ids_in_pxt)
@@ -383,11 +383,11 @@ class LabelStudioProject(Project):
         for rowid in deleted_rowids:
             del existing_tasks[rowid]
 
-        return SyncStatus(ext_row_count_stats=RowCountStats(del_rows=len(deleted_rowids)))
+        return UpdateStatus(ext_row_count_stats=RowCountStats(del_rows=len(deleted_rowids)))
 
-    def __update_table_from_tasks(self, t: Table, tasks: dict[tuple, dict]) -> SyncStatus:
+    def __update_table_from_tasks(self, t: Table, tasks: dict[tuple, dict]) -> UpdateStatus:
         if ANNOTATIONS_COLUMN not in self.col_mapping.values():
-            return SyncStatus()
+            return UpdateStatus()
 
         annotations = {
             # Replace [] by None to indicate no annotations. We do want to sync rows with no annotations,
@@ -422,9 +422,9 @@ class LabelStudioProject(Project):
                 ancestor = ancestor._get_base_table()
             update_status = ancestor.batch_update(updates)
             env.Env.get().console_logger.info(f'Updated annotation(s) from {len(updates)} task(s) in {self}.')
-            return SyncStatus.from_update_status(update_status)
+            return update_status
         else:
-            return SyncStatus()
+            return UpdateStatus()
 
     def as_dict(self) -> dict[str, Any]:
         return {

pixeltable/metadata/__init__.py

@@ -18,7 +18,7 @@ _console_logger = ConsoleLogger(logging.getLogger('pixeltable'))
 _logger = logging.getLogger('pixeltable')
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 39
+VERSION = 40
 
 
 def create_system_info(engine: sql.engine.Engine) -> None:

pixeltable/metadata/converters/convert_39.py

@@ -0,0 +1,125 @@
+import logging
+from typing import Optional
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+_logger = logging.getLogger('pixeltable')
+
+
+@register_converter(version=39)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_modifier=__table_modifier)
+
+
+def __table_modifier(conn: sql.Connection, tbl_id: UUID, orig_table_md: dict, updated_table_md: dict) -> None:
+    store_prefix = 'view' if orig_table_md['view_md'] is not None else 'tbl'
+    store_name = f'{store_prefix}_{tbl_id.hex}'
+
+    # Get the list of column names that need to be migrated
+    col_names = find_error_columns(conn=conn, store_name=store_name)
+    if len(col_names) == 0:
+        _logger.info(f'No error columns found in table {store_name}. Skipping migration.')
+        return
+
+    # Check if the table exists, outside of the metadata we were given
+    # There seem to be cases where the metadata is present in the catalog,
+    # but the table itself is not in the database.
+    check_table_sql = sql.text(f"""
+        SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.tables
+            WHERE table_name = '{store_name}'
+        )
+    """)
+    table_exists = conn.execute(check_table_sql).scalar()
+    if not table_exists:
+        _logger.warning(f'Table {store_name} does not exist. Skipping migration.')
+        return
+
+    return migrate_error_to_cellmd_columns(conn, store_name, col_names)
+
+
+def find_error_columns(conn: sql.Connection, store_name: str) -> list[str]:
+    """
+    Return and errormsg or errortype columns in the given table
+
+    Args:
+        conn: SQLAlchemy connection
+        store_name: Name of the table to check
+
+    Returns:
+        List of column name roots (root_errormsg, root_errortype)
+    """
+    check_columns_sql = sql.text(f"""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_name = '{store_name}'
+    """)
+    found_columns = [
+        row[0]
+        for row in conn.execute(check_columns_sql)
+        if row[0].endswith('_errormsg') or row[0].endswith('_errortype')
+    ]
+    column_roots = {s.removesuffix('_errormsg').removesuffix('_errortype') for s in found_columns}
+    return [*column_roots]
+
+
+def migrate_error_to_cellmd_columns(
+    conn: sql.Connection, store_name: str, col_names: list[str], backup_table: Optional[str] = None
+) -> None:
+    """
+    Safe version with error handling and optional backup.
+
+    Args:
+        engine: SQLAlchemy engine
+        store_name: Name of the table to modify
+        col_names: List of column name prefixes
+        backup_table: Optional name for backup table
+
+    Usage:
+        migrate_error_to_cellmd_columns(engine, 'my_table', ['columnname'], 'my_table_backup')
+    """
+
+    try:
+        # Optional: Create backup
+        if backup_table:
+            backup_sql = sql.text(f"""
+                CREATE TABLE {backup_table} AS SELECT * FROM {store_name}
+            """)
+            conn.execute(backup_sql)
+            _logger.info(f'Backup created: {backup_table}')
+
+        # Step 1: Add new columns
+        add_column_str = ', '.join(f'ADD COLUMN {col}_cellmd JSONB DEFAULT NULL' for col in col_names)
+        add_column_sql = sql.text(f'ALTER TABLE {store_name} {add_column_str}')
+        conn.execute(add_column_sql)
+        _logger.info(f'Added columns: {", ".join(f"{col}_cellmd" for col in col_names)}')
+
+        # Step 2: Populate new columns
+        set_column_str = ', '.join(
+            [
+                f'{col}_cellmd = CASE WHEN {col}_errormsg IS NULL OR {col}_errortype IS NULL '
+                f"THEN NULL ELSE jsonb_build_object('errormsg', {col}_errormsg, 'errortype', {col}_errortype) END"
+                for col in col_names
+            ]
+        )
+        populate_sql = sql.text(f'UPDATE {store_name} SET {set_column_str}')
+        result = conn.execute(populate_sql)
+        _logger.info(f'Updated {result.rowcount} rows')
+
+        # Step 3: Drop old columns
+        drop_columns_str = ', '.join(
+            [f'DROP COLUMN IF EXISTS {col}_errormsg, DROP COLUMN IF EXISTS {col}_errortype' for col in col_names]
+        )
+        drop_columns_sql = sql.text(f'ALTER TABLE {store_name} {drop_columns_str}')
+        conn.execute(drop_columns_sql)
+        _logger.info(f'Dropped columns: {", ".join(f"{col}_errormsg, {col}_errortype" for col in col_names)}')
+        _logger.info(f'Migration completed successfully for table: {store_name}')
+
+    except sql.exc.SQLAlchemyError as e:
+        _logger.error(f'Migration for table {store_name} failed: {e}')
+        raise

pixeltable/metadata/converters/util.py

@@ -16,6 +16,7 @@ def convert_table_md(
     column_md_updater: Optional[Callable[[dict], None]] = None,
     external_store_md_updater: Optional[Callable[[dict], None]] = None,
     substitution_fn: Optional[Callable[[Optional[str], Any], Optional[tuple[Optional[str], Any]]]] = None,
+    table_modifier: Optional[Callable[[sql.Connection, UUID, dict, dict], None]] = None,
 ) -> None:
     """
     Converts schema.TableMd dicts based on the specified conversion functions.
@@ -50,6 +51,8 @@ def convert_table_md(
             if updated_table_md != table_md:
                 __logger.info(f'Updating schema for table: {tbl_id}')
                 conn.execute(sql.update(Table).where(Table.id == tbl_id).values(md=updated_table_md))
+            if table_modifier is not None:
+                table_modifier(conn, tbl_id, table_md, updated_table_md)
 
         for row in conn.execute(sql.select(Function)):
             fn_id = row[0]

pixeltable/metadata/notes.py

@@ -2,6 +2,7 @@
 # rather than as a comment, so that the existence of a description can be enforced by
 # the unit tests when new versions are added.
 VERSION_NOTES = {
+    40: 'Convert error property columns to cellmd columns',
     39: 'ColumnHandles in external stores',
     38: 'Added TableMd.view_sn',
     37: 'Add support for the sample() method on DataFrames',