pixeltable 0.2.26__py3-none-any.whl → 0.5.7__py3-none-any.whl

pixeltable/metadata/converters/convert_30.py

@@ -0,0 +1,63 @@
+import copy
+import uuid
+from typing import Any
+
+import sqlalchemy as sql
+from sqlalchemy import orm
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_sql_table_record, convert_table_md
+
+
+@register_converter(version=30)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_md_updater=__update_table_md)
+    convert_sql_table_record(TableVersionAtV30, engine, record_updater=__update_table_version_record)
+    convert_sql_table_record(TableSchemaVersionAtV30, engine, record_updater=__update_table_schema_version_record)
+
+
+def __update_table_md(md: dict, tbl_id: uuid.UUID) -> None:
+    md['tbl_id'] = str(tbl_id)
+
+
+# We can't use the ORM declarations from pixeltable.metadata.schema, because those might have changed since the
+# version being converted. So we include static declarations here of the tables as they existed at version 30.
+
+Base: type = orm.declarative_base()
+
+
+class TableVersionAtV30(Base):
+    __tablename__ = 'tableversions'
+
+    tbl_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), primary_key=True, nullable=False)
+    version: orm.Mapped[int] = orm.mapped_column(sql.BigInteger, primary_key=True, nullable=False)
+    md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)
+
+
+class TableSchemaVersionAtV30(Base):
+    __tablename__ = 'tableschemaversions'
+
+    tbl_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), primary_key=True, nullable=False)
+    schema_version: orm.Mapped[int] = orm.mapped_column(sql.BigInteger, primary_key=True, nullable=False)
+    md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)  # TableSchemaVersionMd
+
+
+def __update_table_version_record(record: TableVersionAtV30) -> None:
+    """
+    Update TableVersion with table_id.
+    """
+    assert isinstance(record.md, dict)
+    md = copy.copy(record.md)
+    md['tbl_id'] = str(record.tbl_id)
+    record.md = md
+
+
+def __update_table_schema_version_record(record: TableSchemaVersionAtV30) -> None:
+    """
+    Update TableSchemaVersion with table_id.
+    """
+    assert isinstance(record.md, dict)
+    md = copy.copy(record.md)
+    md['tbl_id'] = str(record.tbl_id)
+    record.md = md

pixeltable/metadata/converters/convert_31.py

@@ -0,0 +1,11 @@
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+
+
+@register_converter(version=31)
+def _(engine: sql.engine.Engine) -> None:
+    # Add a column "lock_dummy: int8" to the dirs table in the store
+    # This column is the target of an UPDATE operation to synchronize directory operations
+    with engine.begin() as conn:
+        conn.execute(sql.text('ALTER TABLE dirs ADD COLUMN lock_dummy int8'))

pixeltable/metadata/converters/convert_32.py

@@ -0,0 +1,15 @@
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=32)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_md_updater=__update_table_md)
+
+
+def __update_table_md(table_md: dict, table_id: UUID) -> None:
+    table_md['is_replica'] = False

pixeltable/metadata/converters/convert_33.py

@@ -0,0 +1,17 @@
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=33)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_md_updater=__update_table_md)
+
+
+def __update_table_md(table_md: dict, table_id: UUID) -> None:
+    """Set default value of 'is_pk' field in column metadata to False"""
+    for col_md in table_md['column_md'].values():
+        col_md['is_pk'] = False if col_md['is_pk'] is None else col_md['is_pk']

pixeltable/metadata/converters/convert_34.py

@@ -0,0 +1,21 @@
+from typing import Any
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=34)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, substitution_fn=__substitute_md)
+
+
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
+    if isinstance(v, dict) and '_classname' in v and v['_classname'] == 'ColumnRef':
+        # Add reference_tbl to ColumnRef; for historical metadata it is always equal to tbl
+        assert 'reference_tbl' not in v
+        v['reference_tbl'] = None
+        return k, v
+
+    return None

pixeltable/metadata/converters/convert_35.py

@@ -0,0 +1,9 @@
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+
+
+@register_converter(version=35)
+def _(engine: sql.engine.Engine) -> None:
+    with engine.begin() as conn:
+        conn.execute(sql.text('ALTER TABLE tables ADD COLUMN lock_dummy int8'))

pixeltable/metadata/converters/convert_36.py

@@ -0,0 +1,38 @@
+import logging
+from typing import Any
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+_logger = logging.getLogger('pixeltable')
+
+
+@register_converter(version=36)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_md_updater=__update_table_md, substitution_fn=__substitute_md)
+
+
+def __update_table_md(table_md: dict, table_id: UUID) -> None:
+    """Update the view metadata to add the sample_clause field if it is missing
+
+    Args:
+        table_md (dict): copy of the original table metadata. this gets updated in place.
+        table_id (UUID): the table id
+
+    """
+    if table_md['view_md'] is None:
+        return
+    if 'sample_clause' not in table_md['view_md']:
+        table_md['view_md']['sample_clause'] = None
+        _logger.info(f'Updating view metadata for table: {table_id}')
+
+
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
+    if isinstance(v, dict) and (v.get('_classname') == 'DataFrame'):
+        if 'sample_clause' not in v:
+            v['sample_clause'] = None
+        return k, v
+    return None

pixeltable/metadata/converters/convert_37.py

@@ -0,0 +1,15 @@
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=37)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_md_updater=__update_table_md)
+
+
+def __update_table_md(table_md: dict, _: UUID) -> None:
+    table_md['view_sn'] = 0

pixeltable/metadata/converters/convert_38.py

@@ -0,0 +1,39 @@
+from typing import Any
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=38)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, substitution_fn=__substitute_md)
+
+
+def __substitute_md(k: str | None, v: Any) -> tuple[str | None, Any] | None:
+    if k == 'col_mapping':
+        assert isinstance(v, list)
+        return k, [__col_mapping_entry(e) for e in v]
+    if k == 'stored_proxies':
+        assert isinstance(v, list)
+        return k, [__stored_proxies_entry(e) for e in v]
+    return None
+
+
+def __col_mapping_entry(e: list) -> list:
+    assert isinstance(e, list)
+    assert isinstance(e[0], dict)
+    assert isinstance(e[1], str)
+    return [__col_handle(e[0]), e[1]]
+
+
+def __stored_proxies_entry(e: list) -> list:
+    assert isinstance(e, list)
+    assert isinstance(e[0], dict)
+    assert isinstance(e[1], dict)
+    return [__col_handle(e[0]), __col_handle(e[1])]
+
+
+def __col_handle(e: dict) -> dict:
+    return {'tbl_version': {'id': e['tbl_id'], 'effective_version': None}, 'col_id': e['col_id']}

pixeltable/metadata/converters/convert_39.py

@@ -0,0 +1,124 @@
+import logging
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+_logger = logging.getLogger('pixeltable')
+
+
+@register_converter(version=39)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_modifier=__table_modifier)
+
+
+def __table_modifier(conn: sql.Connection, tbl_id: UUID, orig_table_md: dict, updated_table_md: dict) -> None:
+    store_prefix = 'view' if orig_table_md['view_md'] is not None else 'tbl'
+    store_name = f'{store_prefix}_{tbl_id.hex}'
+
+    # Get the list of column names that need to be migrated
+    col_names = find_error_columns(conn=conn, store_name=store_name)
+    if len(col_names) == 0:
+        _logger.info(f'No error columns found in table {store_name}. Skipping migration.')
+        return
+
+    # Check if the table exists, outside of the metadata we were given
+    # There seem to be cases where the metadata is present in the catalog,
+    # but the table itself is not in the database.
+    check_table_sql = sql.text(f"""
+        SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.tables
+            WHERE table_name = '{store_name}'
+        )
+    """)
+    table_exists = conn.execute(check_table_sql).scalar()
+    if not table_exists:
+        _logger.warning(f'Table {store_name} does not exist. Skipping migration.')
+        return
+
+    return migrate_error_to_cellmd_columns(conn, store_name, col_names)
+
+
+def find_error_columns(conn: sql.Connection, store_name: str) -> list[str]:
+    """
+    Return and errormsg or errortype columns in the given table
+
+    Args:
+        conn: SQLAlchemy connection
+        store_name: Name of the table to check
+
+    Returns:
+        List of column name roots (root_errormsg, root_errortype)
+    """
+    check_columns_sql = sql.text(f"""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_name = '{store_name}'
+    """)
+    found_columns = [
+        row[0]
+        for row in conn.execute(check_columns_sql)
+        if row[0].endswith('_errormsg') or row[0].endswith('_errortype')
+    ]
+    column_roots = {s.removesuffix('_errormsg').removesuffix('_errortype') for s in found_columns}
+    return [*column_roots]
+
+
+def migrate_error_to_cellmd_columns(
+    conn: sql.Connection, store_name: str, col_names: list[str], backup_table: str | None = None
+) -> None:
+    """
+    Safe version with error handling and optional backup.
+
+    Args:
+        engine: SQLAlchemy engine
+        store_name: Name of the table to modify
+        col_names: List of column name prefixes
+        backup_table: Optional name for backup table
+
+    Usage:
+        migrate_error_to_cellmd_columns(engine, 'my_table', ['columnname'], 'my_table_backup')
+    """
+
+    try:
+        # Optional: Create backup
+        if backup_table:
+            backup_sql = sql.text(f"""
+                CREATE TABLE {backup_table} AS SELECT * FROM {store_name}
+            """)
+            conn.execute(backup_sql)
+            _logger.info(f'Backup created: {backup_table}')
+
+        # Step 1: Add new columns
+        add_column_str = ', '.join(f'ADD COLUMN {col}_cellmd JSONB DEFAULT NULL' for col in col_names)
+        add_column_sql = sql.text(f'ALTER TABLE {store_name} {add_column_str}')
+        conn.execute(add_column_sql)
+        _logger.info(f'Added columns: {", ".join(f"{col}_cellmd" for col in col_names)}')
+
+        # Step 2: Populate new columns
+        set_column_str = ', '.join(
+            [
+                f'{col}_cellmd = CASE WHEN {col}_errormsg IS NULL OR {col}_errortype IS NULL '
+                f"THEN NULL ELSE jsonb_build_object('errormsg', {col}_errormsg, 'errortype', {col}_errortype) END"
+                for col in col_names
+            ]
+        )
+        populate_sql = sql.text(f'UPDATE {store_name} SET {set_column_str}')
+        result = conn.execute(populate_sql)
+        _logger.info(f'Updated {result.rowcount} rows')
+
+        # Step 3: Drop old columns
+        drop_columns_str = ', '.join(
+            [f'DROP COLUMN IF EXISTS {col}_errormsg, DROP COLUMN IF EXISTS {col}_errortype' for col in col_names]
+        )
+        drop_columns_sql = sql.text(f'ALTER TABLE {store_name} {drop_columns_str}')
+        conn.execute(drop_columns_sql)
+        _logger.info(f'Dropped columns: {", ".join(f"{col}_errormsg, {col}_errortype" for col in col_names)}')
+        _logger.info(f'Migration completed successfully for table: {store_name}')
+
+    except sql.exc.SQLAlchemyError as e:
+        _logger.error(f'Migration for table {store_name} failed: {e}')
+        raise

pixeltable/metadata/converters/convert_40.py

@@ -0,0 +1,73 @@
+import logging
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+_logger = logging.getLogger('pixeltable')
+
+
+@register_converter(version=40)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_modifier=__table_modifier)
+
+
+def __table_modifier(conn: sql.Connection, tbl_id: UUID, orig_table_md: dict, updated_table_md: dict) -> None:
+    store_prefix = 'view' if orig_table_md['view_md'] is not None else 'tbl'
+    store_name = f'{store_prefix}_{tbl_id.hex}'
+
+    # Get the list of column names that need _cellmd columns
+    _logger.info(f'Checking table {orig_table_md["name"]} ({store_name})')
+    col_ids = find_target_columns(orig_table_md)
+    if len(col_ids) == 0:
+        _logger.info(f'No Array or Json columns found in table {orig_table_md["name"]}. Skipping migration.')
+        return
+
+    # Check which columns already exist in the table
+    check_columns_sql = sql.text(f"""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_name = '{store_name}'
+    """)
+    existing_columns = {row[0] for row in conn.execute(check_columns_sql)}
+
+    # Filter out columns that already have _cellmd
+    col_ids_to_add: list[int] = []
+    for col_id in col_ids:
+        cellmd_col = f'col_{col_id}_cellmd'
+        if cellmd_col not in existing_columns:
+            col_ids_to_add.append(col_id)
+        else:
+            _logger.info(f'Column {cellmd_col} already exists in table {orig_table_md["name"]}. Skipping.')
+
+    if len(col_ids_to_add) == 0:
+        _logger.info(f'All _cellmd columns already exist in table {orig_table_md["name"]}. Skipping migration.')
+        return
+
+    return add_cellmd_columns(conn, store_name, col_ids_to_add)
+
+
+def find_target_columns(table_md: dict) -> list[int]:
+    """Returns ids of stored array and json columns"""
+    result: list[int] = []
+    for col_id, col_md in table_md['column_md'].items():
+        col_type = col_md['col_type']
+        classname = col_type.get('_classname')
+        if classname in ['ArrayType', 'JsonType'] and col_md.get('stored', False):
+            result.append(col_id)
+            _logger.info(f'Found {classname} column: {col_id}')
+    return result
+
+
+def add_cellmd_columns(conn: sql.Connection, store_name: str, col_ids: list[int]) -> None:
+    try:
+        # Add new columns
+        add_column_str = ', '.join(f'ADD COLUMN col_{col_id}_cellmd JSONB DEFAULT NULL' for col_id in col_ids)
+        add_column_sql = sql.text(f'ALTER TABLE {store_name} {add_column_str}')
+        conn.execute(add_column_sql)
+        _logger.info(f'Added columns to {store_name}: {", ".join(f"col_{col_id}_cellmd" for col_id in col_ids)}')
+    except sql.exc.SQLAlchemyError as e:
+        _logger.error(f'Migration for table {store_name} failed: {e}')
+        raise

pixeltable/metadata/converters/convert_41.py

@@ -0,0 +1,12 @@
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+
+
+@register_converter(version=41)
+def _(engine: sql.engine.Engine) -> None:
+    with engine.begin() as conn:
+        conn.execute(sql.text("ALTER TABLE dirs ADD COLUMN additional_md JSONB DEFAULT '{}'::JSONB"))
+        conn.execute(sql.text("ALTER TABLE tables ADD COLUMN additional_md JSONB DEFAULT '{}'::JSONB"))
+        conn.execute(sql.text("ALTER TABLE tableversions ADD COLUMN additional_md JSONB DEFAULT '{}'::JSONB"))
+        conn.execute(sql.text("ALTER TABLE tableschemaversions ADD COLUMN additional_md JSONB DEFAULT '{}'::JSONB"))

pixeltable/metadata/converters/convert_42.py

@@ -0,0 +1,9 @@
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+
+
+@register_converter(version=42)
+def _(engine: sql.engine.Engine) -> None:
+    with engine.begin() as conn:
+        conn.execute(sql.text('ALTER TABLE tables ALTER COLUMN dir_id DROP NOT NULL'))

pixeltable/metadata/converters/convert_43.py

@@ -0,0 +1,44 @@
+from typing import Any
+
+import numpy as np
+import sqlalchemy as sql
+
+from pixeltable import type_system as ts
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=43)
+def _(engine: sql.engine.Engine) -> None:
+    """Converts ArrayTypes by replacing legacy dtype (which was a pxt Type ID) to numpy dtype."""
+    convert_table_md(engine, substitution_fn=_substitution_fn)
+
+
+def _substitution_fn(key: str | None, value: Any) -> tuple[str | None, Any] | None:
+    if not isinstance(value, dict):
+        return None
+    if value.get('_classname', None) != 'ArrayType':
+        return None
+    if 'numpy_dtype' in value:
+        return None
+    assert 'dtype' in value
+
+    legacy_dtype_val = value['dtype']
+    new_dtype: np.dtype | None
+    if legacy_dtype_val is None:
+        new_dtype = None
+    else:
+        legacy_dtype = ts.ColumnType.Type(legacy_dtype_val)
+        new_dtype = ts.ArrayType.pxt_dtype_to_numpy_dtype.get(legacy_dtype, None)
+        if new_dtype is None:
+            raise ValueError(f'Unrecognized dtype: {legacy_dtype_val} ({legacy_dtype}) in {key}, {value}')
+
+    del value['dtype']
+    if new_dtype is None:
+        value['numpy_dtype'] = None
+    elif new_dtype == np.str_:
+        # str(np.str_) would be something like '<U'
+        value['numpy_dtype'] = 'str'
+    else:
+        value['numpy_dtype'] = str(new_dtype)
+    return key, value

pixeltable/metadata/converters/util.py

@@ -1,20 +1,22 @@
 import copy
 import logging
-from typing import Any, Callable, Optional
+from typing import Any, Callable, TypeVar
+from uuid import UUID
 
 import sqlalchemy as sql
 
-from pixeltable.metadata.schema import Table, TableSchemaVersion
+from pixeltable.metadata.schema import Function, Table, TableSchemaVersion
 
 __logger = logging.getLogger('pixeltable')
 
 
 def convert_table_md(
     engine: sql.engine.Engine,
-    table_md_updater: Optional[Callable[[dict], None]] = None,
-    column_md_updater: Optional[Callable[[dict], None]] = None,
-    external_store_md_updater: Optional[Callable[[dict], None]] = None,
-    substitution_fn: Optional[Callable[[Optional[str], Any], Optional[tuple[Optional[str], Any]]]] = None
+    table_md_updater: Callable[[dict, UUID], None] | None = None,
+    column_md_updater: Callable[[dict], None] | None = None,
+    external_store_md_updater: Callable[[dict], None] | None = None,
+    substitution_fn: Callable[[str | None, Any], tuple[str | None, Any] | None] | None = None,
+    table_modifier: Callable[[sql.Connection, UUID, dict, dict], None] | None = None,
 ) -> None:
     """
     Converts schema.TableMd dicts based on the specified conversion functions.
@@ -22,6 +24,7 @@ def convert_table_md(
     Args:
         engine: The SQLAlchemy engine.
         table_md_updater: A function that updates schema.TableMd dicts in place.
+            It takes two arguments: the metadata dict (new values) and the table id.
         column_md_updater: A function that updates schema.ColumnMd dicts in place.
         external_store_md_updater: A function that updates the external store metadata in place.
         substitution_fn: A function that substitutes metadata values. If specified, all metadata will be traversed
@@ -31,13 +34,14 @@ def convert_table_md(
             the original entry will be replaced, and the traversal will continue with `v'`.
     """
     with engine.begin() as conn:
-        for row in conn.execute(sql.select(Table)):
-            id = row[0]
-            table_md = row[2]
+        # avoid a SELECT * here, which breaks when we add new columns to Table
+        for row in conn.execute(sql.select(Table.id, Table.md)):
+            tbl_id = row[0]
+            table_md = row[1]
             assert isinstance(table_md, dict)
             updated_table_md = copy.deepcopy(table_md)
             if table_md_updater is not None:
-                table_md_updater(updated_table_md)
+                table_md_updater(updated_table_md, tbl_id)
             if column_md_updater is not None:
                 __update_column_md(updated_table_md, column_md_updater)
             if external_store_md_updater is not None:
@@ -45,8 +49,21 @@ def convert_table_md(
             if substitution_fn is not None:
                 updated_table_md = __substitute_md_rec(updated_table_md, substitution_fn)
             if updated_table_md != table_md:
-                __logger.info(f'Updating schema for table: {id}')
-                conn.execute(sql.update(Table).where(Table.id == id).values(md=updated_table_md))
+                __logger.info(f'Updating schema for table: {tbl_id}')
+                conn.execute(sql.update(Table).where(Table.id == tbl_id).values(md=updated_table_md))
+            if table_modifier is not None:
+                table_modifier(conn, tbl_id, table_md, updated_table_md)
+
+        for row in conn.execute(sql.select(Function)):
+            fn_id = row[0]
+            function_md = row[2]
+            assert isinstance(function_md, dict)
+            updated_function_md = copy.deepcopy(function_md)
+            if substitution_fn is not None:
+                updated_function_md = __substitute_md_rec(updated_function_md, substitution_fn)
+            if updated_function_md != function_md:
+                __logger.info(f'Updating function: {fn_id}')
+                conn.execute(sql.update(Function).where(Function.id == fn_id).values(md=updated_function_md))
 
 
 def __update_column_md(table_md: dict, column_md_updater: Callable[[dict], None]) -> None:
@@ -63,10 +80,7 @@ def __update_external_store_md(table_md: dict, external_store_md_updater: Callab
         external_store_md_updater(store_md)
 
 
-def __substitute_md_rec(
-    md: Any,
-    substitution_fn: Callable[[Optional[str], Any], Optional[tuple[Optional[str], Any]]]
-) -> Any:
+def __substitute_md_rec(md: Any, substitution_fn: Callable[[str | None, Any], tuple[str | None, Any] | None]) -> Any:
     if isinstance(md, dict):
         updated_dict: dict[str, Any] = {}
         for k, v in md.items():
@@ -94,8 +108,8 @@ def __substitute_md_rec(
 
 def convert_table_schema_version_md(
     engine: sql.engine.Engine,
-    table_schema_version_md_updater: Optional[Callable[[dict], None]] = None,
-    schema_column_updater: Optional[Callable[[dict], None]] = None
+    table_schema_version_md_updater: Callable[[dict], None] | None = None,
+    schema_column_updater: Callable[[dict], None] | None = None,
 ) -> None:
     """
     Converts schema.TableSchemaVersionMd dicts based on the specified conversion functions.
@@ -131,3 +145,15 @@ def __update_schema_column(table_schema_version_md: dict, schema_column_updater:
     assert isinstance(cols, dict)
     for schema_col in cols.values():
         schema_column_updater(schema_col)
+
+
+T = TypeVar('T')
+
+
+def convert_sql_table_record(
+    schema: type[T], engine: sql.engine.Engine, record_updater: Callable[[T], None] | None
+) -> None:
+    with sql.orm.Session(engine, future=True) as session:
+        for record in session.query(schema).all():
+            record_updater(record)
+        session.commit()

pixeltable/metadata/notes.py

@@ -2,6 +2,27 @@
 # rather than as a comment, so that the existence of a description can be enforced by
 # the unit tests when new versions are added.
 VERSION_NOTES = {
+    44: 'ArrayType dtype migration from pxt types to numpy dtypes',
+    43: 'Changing tables.dir_id to nullable',
+    42: 'Add additional_md columns to metadata tables',
+    41: 'Cellmd columns for array and json columns',
+    40: 'Convert error property columns to cellmd columns',
+    39: 'ColumnHandles in external stores',
+    38: 'Added TableMd.view_sn',
+    37: 'Add support for the sample() method on DataFrames',
+    36: 'Added Table.lock_dummy',
+    35: 'Track reference_tbl in ColumnRef',
+    34: 'Set default value for is_pk field in column metadata to False',
+    33: 'Add is_replica field to table metadata',
+    32: 'Add the lock_dummy BIGINT column to the dirs table',
+    31: 'Add table ids to metadata structs',
+    30: 'Store default values and constant arguments as literals',
+    29: 'Add user and additional_md fields to metadata structs',
+    28: 'Enable view creation from DataFrame with select clause',
+    27: 'Enable pxt.query parameterization of limit clauses',
+    26: 'Rename clip_text and clip_image to clip',
+    25: 'Functions with multiple signatures',
+    24: 'Added TableMd/IndexMd.indexed_col_tbl_id',
     23: 'DataFrame.from_clause',
     22: 'TableMd/ColumnMd.media_validation',
     21: 'Separate InlineArray and InlineList',