pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.20__py3-none-any.whl

pixeltable/metadata/converters/convert_39.py

@@ -0,0 +1,124 @@
+import logging
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+_logger = logging.getLogger('pixeltable')
+
+
+@register_converter(version=39)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_modifier=__table_modifier)
+
+
+def __table_modifier(conn: sql.Connection, tbl_id: UUID, orig_table_md: dict, updated_table_md: dict) -> None:
+    store_prefix = 'view' if orig_table_md['view_md'] is not None else 'tbl'
+    store_name = f'{store_prefix}_{tbl_id.hex}'
+
+    # Get the list of column names that need to be migrated
+    col_names = find_error_columns(conn=conn, store_name=store_name)
+    if len(col_names) == 0:
+        _logger.info(f'No error columns found in table {store_name}. Skipping migration.')
+        return
+
+    # Check if the table exists, outside of the metadata we were given
+    # There seem to be cases where the metadata is present in the catalog,
+    # but the table itself is not in the database.
+    check_table_sql = sql.text(f"""
+        SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.tables
+            WHERE table_name = '{store_name}'
+        )
+    """)
+    table_exists = conn.execute(check_table_sql).scalar()
+    if not table_exists:
+        _logger.warning(f'Table {store_name} does not exist. Skipping migration.')
+        return
+
+    return migrate_error_to_cellmd_columns(conn, store_name, col_names)
+
+
+def find_error_columns(conn: sql.Connection, store_name: str) -> list[str]:
+    """
+    Return and errormsg or errortype columns in the given table
+
+    Args:
+        conn: SQLAlchemy connection
+        store_name: Name of the table to check
+
+    Returns:
+        List of column name roots (root_errormsg, root_errortype)
+    """
+    check_columns_sql = sql.text(f"""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_name = '{store_name}'
+    """)
+    found_columns = [
+        row[0]
+        for row in conn.execute(check_columns_sql)
+        if row[0].endswith('_errormsg') or row[0].endswith('_errortype')
+    ]
+    column_roots = {s.removesuffix('_errormsg').removesuffix('_errortype') for s in found_columns}
+    return [*column_roots]
+
+
+def migrate_error_to_cellmd_columns(
+    conn: sql.Connection, store_name: str, col_names: list[str], backup_table: str | None = None
+) -> None:
+    """
+    Safe version with error handling and optional backup.
+
+    Args:
+        engine: SQLAlchemy engine
+        store_name: Name of the table to modify
+        col_names: List of column name prefixes
+        backup_table: Optional name for backup table
+
+    Usage:
+        migrate_error_to_cellmd_columns(engine, 'my_table', ['columnname'], 'my_table_backup')
+    """
+
+    try:
+        # Optional: Create backup
+        if backup_table:
+            backup_sql = sql.text(f"""
+                CREATE TABLE {backup_table} AS SELECT * FROM {store_name}
+            """)
+            conn.execute(backup_sql)
+            _logger.info(f'Backup created: {backup_table}')
+
+        # Step 1: Add new columns
+        add_column_str = ', '.join(f'ADD COLUMN {col}_cellmd JSONB DEFAULT NULL' for col in col_names)
+        add_column_sql = sql.text(f'ALTER TABLE {store_name} {add_column_str}')
+        conn.execute(add_column_sql)
+        _logger.info(f'Added columns: {", ".join(f"{col}_cellmd" for col in col_names)}')
+
+        # Step 2: Populate new columns
+        set_column_str = ', '.join(
+            [
+                f'{col}_cellmd = CASE WHEN {col}_errormsg IS NULL OR {col}_errortype IS NULL '
+                f"THEN NULL ELSE jsonb_build_object('errormsg', {col}_errormsg, 'errortype', {col}_errortype) END"
+                for col in col_names
+            ]
+        )
+        populate_sql = sql.text(f'UPDATE {store_name} SET {set_column_str}')
+        result = conn.execute(populate_sql)
+        _logger.info(f'Updated {result.rowcount} rows')
+
+        # Step 3: Drop old columns
+        drop_columns_str = ', '.join(
+            [f'DROP COLUMN IF EXISTS {col}_errormsg, DROP COLUMN IF EXISTS {col}_errortype' for col in col_names]
+        )
+        drop_columns_sql = sql.text(f'ALTER TABLE {store_name} {drop_columns_str}')
+        conn.execute(drop_columns_sql)
+        _logger.info(f'Dropped columns: {", ".join(f"{col}_errormsg, {col}_errortype" for col in col_names)}')
+        _logger.info(f'Migration completed successfully for table: {store_name}')
+
+    except sql.exc.SQLAlchemyError as e:
+        _logger.error(f'Migration for table {store_name} failed: {e}')
+        raise

pixeltable/metadata/converters/convert_40.py

@@ -0,0 +1,73 @@
+import logging
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+_logger = logging.getLogger('pixeltable')
+
+
+@register_converter(version=40)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_modifier=__table_modifier)
+
+
+def __table_modifier(conn: sql.Connection, tbl_id: UUID, orig_table_md: dict, updated_table_md: dict) -> None:
+    store_prefix = 'view' if orig_table_md['view_md'] is not None else 'tbl'
+    store_name = f'{store_prefix}_{tbl_id.hex}'
+
+    # Get the list of column names that need _cellmd columns
+    _logger.info(f'Checking table {orig_table_md["name"]} ({store_name})')
+    col_ids = find_target_columns(orig_table_md)
+    if len(col_ids) == 0:
+        _logger.info(f'No Array or Json columns found in table {orig_table_md["name"]}. Skipping migration.')
+        return
+
+    # Check which columns already exist in the table
+    check_columns_sql = sql.text(f"""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_name = '{store_name}'
+    """)
+    existing_columns = {row[0] for row in conn.execute(check_columns_sql)}
+
+    # Filter out columns that already have _cellmd
+    col_ids_to_add: list[int] = []
+    for col_id in col_ids:
+        cellmd_col = f'col_{col_id}_cellmd'
+        if cellmd_col not in existing_columns:
+            col_ids_to_add.append(col_id)
+        else:
+            _logger.info(f'Column {cellmd_col} already exists in table {orig_table_md["name"]}. Skipping.')
+
+    if len(col_ids_to_add) == 0:
+        _logger.info(f'All _cellmd columns already exist in table {orig_table_md["name"]}. Skipping migration.')
+        return
+
+    return add_cellmd_columns(conn, store_name, col_ids_to_add)
+
+
+def find_target_columns(table_md: dict) -> list[int]:
+    """Returns ids of stored array and json columns"""
+    result: list[int] = []
+    for col_id, col_md in table_md['column_md'].items():
+        col_type = col_md['col_type']
+        classname = col_type.get('_classname')
+        if classname in ['ArrayType', 'JsonType'] and col_md.get('stored', False):
+            result.append(col_id)
+            _logger.info(f'Found {classname} column: {col_id}')
+    return result
+
+
+def add_cellmd_columns(conn: sql.Connection, store_name: str, col_ids: list[int]) -> None:
+    try:
+        # Add new columns
+        add_column_str = ', '.join(f'ADD COLUMN col_{col_id}_cellmd JSONB DEFAULT NULL' for col_id in col_ids)
+        add_column_sql = sql.text(f'ALTER TABLE {store_name} {add_column_str}')
+        conn.execute(add_column_sql)
+        _logger.info(f'Added columns to {store_name}: {", ".join(f"col_{col_id}_cellmd" for col_id in col_ids)}')
+    except sql.exc.SQLAlchemyError as e:
+        _logger.error(f'Migration for table {store_name} failed: {e}')
+        raise

pixeltable/metadata/converters/util.py

@@ -1,6 +1,6 @@
 import copy
 import logging
-from typing import Any, Callable, Optional
+from typing import Any, Callable
 from uuid import UUID
 
 import sqlalchemy as sql
@@ -12,10 +12,11 @@ __logger = logging.getLogger('pixeltable')
 
 def convert_table_md(
     engine: sql.engine.Engine,
-    table_md_updater: Optional[Callable[[dict, UUID], None]] = None,
-    column_md_updater: Optional[Callable[[dict], None]] = None,
-    external_store_md_updater: Optional[Callable[[dict], None]] = None,
-    substitution_fn: Optional[Callable[[Optional[str], Any], Optional[tuple[Optional[str], Any]]]] = None,
+    table_md_updater: Callable[[dict, UUID], None] | None = None,
+    column_md_updater: Callable[[dict], None] | None = None,
+    external_store_md_updater: Callable[[dict], None] | None = None,
+    substitution_fn: Callable[[str | None, Any], tuple[str | None, Any] | None] | None = None,
+    table_modifier: Callable[[sql.Connection, UUID, dict, dict], None] | None = None,
 ) -> None:
     """
     Converts schema.TableMd dicts based on the specified conversion functions.
@@ -50,6 +51,8 @@ def convert_table_md(
             if updated_table_md != table_md:
                 __logger.info(f'Updating schema for table: {tbl_id}')
                 conn.execute(sql.update(Table).where(Table.id == tbl_id).values(md=updated_table_md))
+            if table_modifier is not None:
+                table_modifier(conn, tbl_id, table_md, updated_table_md)
 
         for row in conn.execute(sql.select(Function)):
             fn_id = row[0]
@@ -77,9 +80,7 @@ def __update_external_store_md(table_md: dict, external_store_md_updater: Callab
         external_store_md_updater(store_md)
 
 
-def __substitute_md_rec(
-    md: Any, substitution_fn: Callable[[Optional[str], Any], Optional[tuple[Optional[str], Any]]]
-) -> Any:
+def __substitute_md_rec(md: Any, substitution_fn: Callable[[str | None, Any], tuple[str | None, Any] | None]) -> Any:
     if isinstance(md, dict):
         updated_dict: dict[str, Any] = {}
         for k, v in md.items():
@@ -107,8 +108,8 @@ def __substitute_md_rec(
 
 def convert_table_schema_version_md(
     engine: sql.engine.Engine,
-    table_schema_version_md_updater: Optional[Callable[[dict], None]] = None,
-    schema_column_updater: Optional[Callable[[dict], None]] = None,
+    table_schema_version_md_updater: Callable[[dict], None] | None = None,
+    schema_column_updater: Callable[[dict], None] | None = None,
 ) -> None:
     """
     Converts schema.TableSchemaVersionMd dicts based on the specified conversion functions.
@@ -147,7 +148,7 @@ def __update_schema_column(table_schema_version_md: dict, schema_column_updater:
 
 
 def convert_table_version_record(
-    engine: sql.engine.Engine, table_version_record_updater: Optional[Callable[[TableVersion], None]]
+    engine: sql.engine.Engine, table_version_record_updater: Callable[[TableVersion], None] | None
 ) -> None:
     with sql.orm.Session(engine, future=True) as session:
         for record in session.query(TableVersion).all():
@@ -156,7 +157,7 @@ def convert_table_version_record(
 
 
 def convert_table_schema_version_record(
-    engine: sql.engine.Engine, table_schema_version_record_updater: Optional[Callable[[TableSchemaVersion], None]]
+    engine: sql.engine.Engine, table_schema_version_record_updater: Callable[[TableSchemaVersion], None] | None
 ) -> None:
     with sql.orm.Session(engine, future=True) as session:
         for record in session.query(TableSchemaVersion).all():

pixeltable/metadata/notes.py

@@ -2,6 +2,10 @@
 # rather than as a comment, so that the existence of a description can be enforced by
 # the unit tests when new versions are added.
 VERSION_NOTES = {
+    41: 'Cellmd columns for array and json columns',
+    40: 'Convert error property columns to cellmd columns',
+    39: 'ColumnHandles in external stores',
+    38: 'Added TableMd.view_sn',
     37: 'Add support for the sample() method on DataFrames',
     36: 'Added Table.lock_dummy',
     35: 'Track reference_tbl in ColumnRef',

pixeltable/metadata/schema.py

@@ -1,13 +1,16 @@
 import dataclasses
+import types
 import typing
 import uuid
-from typing import Any, NamedTuple, Optional, TypeVar, Union, get_type_hints
+from typing import Any, TypeVar, Union, get_type_hints
 
 import sqlalchemy as sql
 from sqlalchemy import BigInteger, ForeignKey, Integer, LargeBinary, orm
 from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.orm.decl_api import DeclarativeMeta
 
+from ..catalog.update_status import UpdateStatus
+
 # Base has to be marked explicitly as a type, in order to be used elsewhere as a type hint. But in addition to being
 # a type, it's also a `DeclarativeMeta`. The following pattern enables us to expose both `Base` and `Base.metadata`
 # outside of the module in a typesafe way.
@@ -22,13 +25,13 @@ def md_from_dict(data_class_type: type[T], data: Any) -> T:
     """Re-instantiate a dataclass instance that contains nested dataclasses from a dict."""
     if dataclasses.is_dataclass(data_class_type):
         fieldtypes = get_type_hints(data_class_type)
-        return data_class_type(**{f: md_from_dict(fieldtypes[f], data[f]) for f in data})  # type: ignore[return-value]
+        return data_class_type(**{f: md_from_dict(fieldtypes[f], data[f]) for f in data})
 
     origin = typing.get_origin(data_class_type)
     if origin is not None:
         type_args = typing.get_args(data_class_type)
-        if origin is Union and type(None) in type_args:
-            # Handling Optional types
+        if (origin is Union or origin is types.UnionType) and type(None) in type_args:
+            # handling T | None, T | None
             non_none_args = [arg for arg in type_args if arg is not type(None)]
             assert len(non_none_args) == 1
             return md_from_dict(non_none_args[0], data) if data is not None else None
@@ -72,7 +75,7 @@ class SystemInfo(Base):
 @dataclasses.dataclass
 class DirMd:
     name: str
-    user: Optional[str]
+    user: str | None
     additional_md: dict[str, Any]
 
 
@@ -101,17 +104,20 @@ class ColumnMd:
 
     id: int
     schema_version_add: int
-    schema_version_drop: Optional[int]
+    schema_version_drop: int | None
     col_type: dict
 
     # if True, is part of the primary key
     is_pk: bool
 
     # if set, this is a computed column
-    value_expr: Optional[dict]
+    value_expr: dict | None
 
     # if True, the column is present in the stored table
-    stored: Optional[bool]
+    stored: bool | None
+
+    # If present, the URI for the destination for column values
+    destination: str | None = None
 
 
 @dataclasses.dataclass
@@ -127,13 +133,13 @@ class IndexMd:
     index_val_col_id: int  # column holding the values to be indexed
     index_val_undo_col_id: int  # column holding index values for deleted rows
     schema_version_add: int
-    schema_version_drop: Optional[int]
+    schema_version_drop: int | None
     class_fqn: str
     init_args: dict[str, Any]
 
 
 # a stored table version path is a list of (table id as str, effective table version)
-TableVersionPath = list[tuple[str, Optional[int]]]
+TableVersionPath = list[tuple[str, int | None]]
 
 
 @dataclasses.dataclass
@@ -145,16 +151,16 @@ class ViewMd:
     base_versions: TableVersionPath
 
     # filter predicate applied to the base table; view-only
-    predicate: Optional[dict[str, Any]]
+    predicate: dict[str, Any] | None
 
     # sampling predicate applied to the base table; view-only
-    sample_clause: Optional[dict[str, Any]]
+    sample_clause: dict[str, Any] | None
 
     # ComponentIterator subclass; only for component views
-    iterator_class_fqn: Optional[str]
+    iterator_class_fqn: str | None
 
     # args to pass to the iterator class constructor; only for component views
-    iterator_args: Optional[dict[str, Any]]
+    iterator_args: dict[str, Any] | None
 
 
 @dataclasses.dataclass
@@ -163,7 +169,7 @@ class TableMd:
     name: str
     is_replica: bool
 
-    user: Optional[str]
+    user: str | None
 
     # monotonically increasing w/in Table for both data and schema changes, starting at 0
     current_version: int
@@ -177,15 +183,47 @@ class TableMd:
     # - every row is assigned a unique and immutable rowid on insertion
     next_row_id: int
 
+    # sequence number to track changes in the set of mutable views of this table (ie, this table = the view base)
+    # - incremented for each add/drop of a mutable view
+    # - only maintained for mutable tables
+    # TODO: replace with mutable_views: list[UUID] to help with debugging
+    view_sn: int
+
     # Metadata format for external stores:
     # {'class': 'pixeltable.io.label_studio.LabelStudioProject', 'md': {'project_id': 3}}
     external_stores: list[dict[str, Any]]
 
     column_md: dict[int, ColumnMd]  # col_id -> ColumnMd
     index_md: dict[int, IndexMd]  # index_id -> IndexMd
-    view_md: Optional[ViewMd]
+    view_md: ViewMd | None
     additional_md: dict[str, Any]
 
+    has_pending_ops: bool = False
+
+    @property
+    def is_snapshot(self) -> bool:
+        return self.view_md is not None and self.view_md.is_snapshot
+
+    @property
+    def is_mutable(self) -> bool:
+        return not self.is_snapshot and not self.is_replica
+
+    @property
+    def is_pure_snapshot(self) -> bool:
+        return (
+            self.view_md is not None
+            and self.view_md.is_snapshot
+            and self.view_md.sample_clause is None
+            and self.view_md.predicate is None
+            and len(self.column_md) == 0
+        )
+
+    @property
+    def ancestor_ids(self) -> list[str]:
+        if self.view_md is None:
+            return []
+        return [id for id, _ in self.view_md.base_versions]
+
 
 class Table(Base):
     """
@@ -214,7 +252,12 @@ class TableVersionMd:
     created_at: float  # time.time()
     version: int
     schema_version: int
-    additional_md: dict[str, Any]
+    user: str | None = None  # User that created this version
+    update_status: UpdateStatus | None = None  # UpdateStatus of the change that created this version
+    # A version fragment cannot be queried or instantiated via get_table(). A fragment represents a version of a
+    # replica table that has incomplete data, and exists only to provide base table support for a dependent view.
+    is_fragment: bool = False
+    additional_md: dict[str, Any] = dataclasses.field(default_factory=dict)
 
 
 class TableVersion(Base):
@@ -237,7 +280,7 @@ class SchemaColumn:
 
     # media validation strategy of this particular media column; if not set, TableMd.media_validation applies
     # stores column.MediaValiation.name.lower()
-    media_validation: Optional[str]
+    media_validation: str | None
 
 
 @dataclasses.dataclass
@@ -248,7 +291,7 @@ class TableSchemaVersionMd:
 
     tbl_id: str  # uuid.UUID
     schema_version: int
-    preceding_schema_version: Optional[int]
+    preceding_schema_version: int | None
     columns: dict[int, SchemaColumn]  # col_id -> SchemaColumn
     num_retained_versions: int
     comment: str
@@ -270,6 +313,22 @@ class TableSchemaVersion(Base):
     md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)  # TableSchemaVersionMd
 
 
+class PendingTableOp(Base):
+    """
+    Table operation that needs to be completed before the table can be used.
+
+    Operations need to be completed in order of increasing seq_num.
+    """
+
+    __tablename__ = 'pendingtableops'
+
+    tbl_id: orm.Mapped[uuid.UUID] = orm.mapped_column(
+        UUID(as_uuid=True), ForeignKey('tables.id'), primary_key=True, nullable=False
+    )
+    op_sn: orm.Mapped[int] = orm.mapped_column(Integer, primary_key=True, nullable=False)  # catalog.TableOp.op_sn
+    op: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)  # catalog.TableOp
+
+
 @dataclasses.dataclass
 class FunctionMd:
     name: str
@@ -295,26 +354,4 @@ class Function(Base):
     )
     dir_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), ForeignKey('dirs.id'), nullable=True)
     md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)  # FunctionMd
-    binary_obj: orm.Mapped[Optional[bytes]] = orm.mapped_column(LargeBinary, nullable=True)
-
-
-class FullTableMd(NamedTuple):
-    tbl_md: TableMd
-    version_md: TableVersionMd
-    schema_version_md: TableSchemaVersionMd
-
-    def as_dict(self) -> dict[str, Any]:
-        return {
-            'table_id': self.tbl_md.tbl_id,
-            'table_md': dataclasses.asdict(self.tbl_md),
-            'table_version_md': dataclasses.asdict(self.version_md),
-            'table_schema_version_md': dataclasses.asdict(self.schema_version_md),
-        }
-
-    @classmethod
-    def from_dict(cls, data_dict: dict[str, Any]) -> 'FullTableMd':
-        return FullTableMd(
-            tbl_md=md_from_dict(TableMd, data_dict['table_md']),
-            version_md=md_from_dict(TableVersionMd, data_dict['table_version_md']),
-            schema_version_md=md_from_dict(TableSchemaVersionMd, data_dict['table_schema_version_md']),
-        )
+    binary_obj: orm.Mapped[bytes | None] = orm.mapped_column(LargeBinary, nullable=True)

pixeltable/metadata/utils.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from pixeltable.metadata import schema
+
+
+class MetadataUtils:
+    @classmethod
+    def _diff_md(
+        cls, old_md: dict[int, schema.SchemaColumn] | None, new_md: dict[int, schema.SchemaColumn] | None
+    ) -> str:
+        """Return a string reporting the differences in a specific entry in two dictionaries
+
+        Results are formatted as follows:
+        - If `old_md` is `None`, returns 'Initial Version'.
+        - If `old_md` and `new_md` are the same, returns an empty string.
+        - If there are additions, changes, or deletions, returns a string summarizing the changes.
+        """
+        assert new_md is not None
+        if old_md is None:
+            return 'Initial Version'
+        if old_md == new_md:
+            return ''
+        added = {k: v.name for k, v in new_md.items() if k not in old_md}
+        changed = {
+            k: f'{old_md[k].name!r} to {v.name!r}'
+            for k, v in new_md.items()
+            if k in old_md and old_md[k].name != v.name
+        }
+        deleted = {k: v.name for k, v in old_md.items() if k not in new_md}
+        if len(added) == 0 and len(changed) == 0 and len(deleted) == 0:
+            return ''
+        # Format the result
+        t = []
+        if len(added) > 0:
+            t.append('Added: ' + ', '.join(added.values()))
+        if len(changed) > 0:
+            t.append('Renamed: ' + ', '.join(changed.values()))
+        if len(deleted) > 0:
+            t.append('Deleted: ' + ', '.join(deleted.values()))
+        r = ', '.join(t)
+        return r
+
+    @classmethod
+    def _create_md_change_dict(cls, md_list: list[tuple[int, dict[int, schema.SchemaColumn]]] | None) -> dict[int, str]:
+        """Return a dictionary of schema changes by version
+        Args:
+            md_list: a list of tuples, each containing a version number and a metadata dictionary.
+        """
+        r: dict[int, str] = {}
+        if md_list is None or len(md_list) == 0:
+            return r
+
+        # Sort the list in place by version number
+        md_list.sort()
+
+        first_retrieved_version = md_list[0][0]
+        if first_retrieved_version == 0:
+            prev_md = None
+            prev_ver = -1
+            start = 0
+        else:
+            prev_md = md_list[0][1]
+            prev_ver = first_retrieved_version
+            start = 1
+
+        for ver, curr_md in md_list[start:]:
+            if ver == prev_ver:
+                continue
+            assert ver > prev_ver
+            tf = cls._diff_md(prev_md, curr_md)
+            if tf != '':
+                r[ver] = tf
+            prev_md = curr_md
+        return r

pixeltable/mypy/__init__.py

@@ -0,0 +1,3 @@
+from .mypy_plugin import plugin
+
+__all__ = ['plugin']