pixeltable 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

pixeltable/metadata/converters/convert_39.py

@@ -0,0 +1,125 @@
+import logging
+from typing import Optional
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+_logger = logging.getLogger('pixeltable')
+
+
+@register_converter(version=39)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_modifier=__table_modifier)
+
+
+def __table_modifier(conn: sql.Connection, tbl_id: UUID, orig_table_md: dict, updated_table_md: dict) -> None:
+    store_prefix = 'view' if orig_table_md['view_md'] is not None else 'tbl'
+    store_name = f'{store_prefix}_{tbl_id.hex}'
+
+    # Get the list of column names that need to be migrated
+    col_names = find_error_columns(conn=conn, store_name=store_name)
+    if len(col_names) == 0:
+        _logger.info(f'No error columns found in table {store_name}. Skipping migration.')
+        return
+
+    # Check if the table exists, outside of the metadata we were given
+    # There seem to be cases where the metadata is present in the catalog,
+    # but the table itself is not in the database.
+    check_table_sql = sql.text(f"""
+        SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.tables
+            WHERE table_name = '{store_name}'
+        )
+    """)
+    table_exists = conn.execute(check_table_sql).scalar()
+    if not table_exists:
+        _logger.warning(f'Table {store_name} does not exist. Skipping migration.')
+        return
+
+    return migrate_error_to_cellmd_columns(conn, store_name, col_names)
+
+
+def find_error_columns(conn: sql.Connection, store_name: str) -> list[str]:
+    """
+    Return and errormsg or errortype columns in the given table
+
+    Args:
+        conn: SQLAlchemy connection
+        store_name: Name of the table to check
+
+    Returns:
+        List of column name roots (root_errormsg, root_errortype)
+    """
+    check_columns_sql = sql.text(f"""
+        SELECT column_name
+        FROM information_schema.columns
+        WHERE table_name = '{store_name}'
+    """)
+    found_columns = [
+        row[0]
+        for row in conn.execute(check_columns_sql)
+        if row[0].endswith('_errormsg') or row[0].endswith('_errortype')
+    ]
+    column_roots = {s.removesuffix('_errormsg').removesuffix('_errortype') for s in found_columns}
+    return [*column_roots]
+
+
+def migrate_error_to_cellmd_columns(
+    conn: sql.Connection, store_name: str, col_names: list[str], backup_table: Optional[str] = None
+) -> None:
+    """
+    Safe version with error handling and optional backup.
+
+    Args:
+        engine: SQLAlchemy engine
+        store_name: Name of the table to modify
+        col_names: List of column name prefixes
+        backup_table: Optional name for backup table
+
+    Usage:
+        migrate_error_to_cellmd_columns(engine, 'my_table', ['columnname'], 'my_table_backup')
+    """
+
+    try:
+        # Optional: Create backup
+        if backup_table:
+            backup_sql = sql.text(f"""
+                CREATE TABLE {backup_table} AS SELECT * FROM {store_name}
+            """)
+            conn.execute(backup_sql)
+            _logger.info(f'Backup created: {backup_table}')
+
+        # Step 1: Add new columns
+        add_column_str = ', '.join(f'ADD COLUMN {col}_cellmd JSONB DEFAULT NULL' for col in col_names)
+        add_column_sql = sql.text(f'ALTER TABLE {store_name} {add_column_str}')
+        conn.execute(add_column_sql)
+        _logger.info(f'Added columns: {", ".join(f"{col}_cellmd" for col in col_names)}')
+
+        # Step 2: Populate new columns
+        set_column_str = ', '.join(
+            [
+                f'{col}_cellmd = CASE WHEN {col}_errormsg IS NULL OR {col}_errortype IS NULL '
+                f"THEN NULL ELSE jsonb_build_object('errormsg', {col}_errormsg, 'errortype', {col}_errortype) END"
+                for col in col_names
+            ]
+        )
+        populate_sql = sql.text(f'UPDATE {store_name} SET {set_column_str}')
+        result = conn.execute(populate_sql)
+        _logger.info(f'Updated {result.rowcount} rows')
+
+        # Step 3: Drop old columns
+        drop_columns_str = ', '.join(
+            [f'DROP COLUMN IF EXISTS {col}_errormsg, DROP COLUMN IF EXISTS {col}_errortype' for col in col_names]
+        )
+        drop_columns_sql = sql.text(f'ALTER TABLE {store_name} {drop_columns_str}')
+        conn.execute(drop_columns_sql)
+        _logger.info(f'Dropped columns: {", ".join(f"{col}_errormsg, {col}_errortype" for col in col_names)}')
+        _logger.info(f'Migration completed successfully for table: {store_name}')
+
+    except sql.exc.SQLAlchemyError as e:
+        _logger.error(f'Migration for table {store_name} failed: {e}')
+        raise

pixeltable/metadata/converters/util.py

@@ -16,6 +16,7 @@ def convert_table_md(
     column_md_updater: Optional[Callable[[dict], None]] = None,
     external_store_md_updater: Optional[Callable[[dict], None]] = None,
     substitution_fn: Optional[Callable[[Optional[str], Any], Optional[tuple[Optional[str], Any]]]] = None,
+    table_modifier: Optional[Callable[[sql.Connection, UUID, dict, dict], None]] = None,
 ) -> None:
     """
     Converts schema.TableMd dicts based on the specified conversion functions.
@@ -50,6 +51,8 @@ def convert_table_md(
             if updated_table_md != table_md:
                 __logger.info(f'Updating schema for table: {tbl_id}')
                 conn.execute(sql.update(Table).where(Table.id == tbl_id).values(md=updated_table_md))
+            if table_modifier is not None:
+                table_modifier(conn, tbl_id, table_md, updated_table_md)
 
         for row in conn.execute(sql.select(Function)):
             fn_id = row[0]

pixeltable/metadata/notes.py

@@ -2,6 +2,8 @@
 # rather than as a comment, so that the existence of a description can be enforced by
 # the unit tests when new versions are added.
 VERSION_NOTES = {
+    40: 'Convert error property columns to cellmd columns',
+    39: 'ColumnHandles in external stores',
     38: 'Added TableMd.view_sn',
     37: 'Add support for the sample() method on DataFrames',
     36: 'Added Table.lock_dummy',

pixeltable/metadata/schema.py

@@ -8,6 +8,8 @@ from sqlalchemy import BigInteger, ForeignKey, Integer, LargeBinary, orm
 from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.orm.decl_api import DeclarativeMeta
 
+from ..catalog.update_status import UpdateStatus
+
 # Base has to be marked explicitly as a type, in order to be used elsewhere as a type hint. But in addition to being
 # a type, it's also a `DeclarativeMeta`. The following pattern enables us to expose both `Base` and `Base.metadata`
 # outside of the module in a typesafe way.
@@ -213,13 +215,15 @@ class Table(Base):
     lock_dummy: orm.Mapped[int] = orm.mapped_column(BigInteger, nullable=True)
 
 
-@dataclasses.dataclass
+@dataclasses.dataclass(frozen=True)
 class TableVersionMd:
     tbl_id: str  # uuid.UUID
     created_at: float  # time.time()
     version: int
     schema_version: int
-    additional_md: dict[str, Any]
+    user: Optional[str] = None  # User that created this version
+    update_status: Optional[UpdateStatus] = None  # UpdateStatus of the change that created this version
+    additional_md: dict[str, Any] = dataclasses.field(default_factory=dict)
 
 
 class TableVersion(Base):
@@ -308,6 +312,14 @@ class FullTableMd(NamedTuple):
     version_md: TableVersionMd
     schema_version_md: TableSchemaVersionMd
 
+    @property
+    def is_pure_snapshot(self) -> bool:
+        return (
+            self.tbl_md.view_md is not None
+            and self.tbl_md.view_md.predicate is None
+            and len(self.schema_version_md.columns) == 0
+        )
+
     def as_dict(self) -> dict[str, Any]:
         return {
             'table_id': self.tbl_md.tbl_id,

pixeltable/metadata/utils.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from pixeltable.metadata import schema
+
+
+class MetadataUtils:
+    @classmethod
+    def _diff_md(
+        cls, old_md: Optional[dict[int, schema.SchemaColumn]], new_md: Optional[dict[int, schema.SchemaColumn]]
+    ) -> str:
+        """Return a string reporting the differences in a specific entry in two dictionaries
+
+        Results are formatted as follows:
+        - If `old_md` is `None`, returns 'Initial Version'.
+        - If `old_md` and `new_md` are the same, returns an empty string.
+        - If there are additions, changes, or deletions, returns a string summarizing the changes.
+        """
+        assert new_md is not None
+        if old_md is None:
+            return 'Initial Version'
+        if old_md == new_md:
+            return ''
+        added = {k: v.name for k, v in new_md.items() if k not in old_md}
+        changed = {
+            k: f'{old_md[k].name!r} to {v.name!r}'
+            for k, v in new_md.items()
+            if k in old_md and old_md[k].name != v.name
+        }
+        deleted = {k: v.name for k, v in old_md.items() if k not in new_md}
+        if len(added) == 0 and len(changed) == 0 and len(deleted) == 0:
+            return ''
+        # Format the result
+        t = []
+        if len(added) > 0:
+            t.append('Added: ' + ', '.join(added.values()))
+        if len(changed) > 0:
+            t.append('Renamed: ' + ', '.join(changed.values()))
+        if len(deleted) > 0:
+            t.append('Deleted: ' + ', '.join(deleted.values()))
+        r = ', '.join(t)
+        return r
+
+    @classmethod
+    def _create_md_change_dict(
+        cls, md_list: Optional[list[tuple[int, dict[int, schema.SchemaColumn]]]]
+    ) -> dict[int, str]:
+        """Return a dictionary of schema changes by version
+        Args:
+            md_list: a list of tuples, each containing a version number and a metadata dictionary.
+        """
+        r: dict[int, str] = {}
+        if md_list is None or len(md_list) == 0:
+            return r
+
+        # Sort the list in place by version number
+        md_list.sort()
+
+        first_retrieved_version = md_list[0][0]
+        if first_retrieved_version == 0:
+            prev_md = None
+            prev_ver = -1
+            start = 0
+        else:
+            prev_md = md_list[0][1]
+            prev_ver = first_retrieved_version
+            start = 1
+
+        for ver, curr_md in md_list[start:]:
+            if ver == prev_ver:
+                continue
+            assert ver > prev_ver
+            tf = cls._diff_md(prev_md, curr_md)
+            if tf != '':
+                r[ver] = tf
+            prev_md = curr_md
+        return r

pixeltable/plan.py

@@ -378,7 +378,7 @@ class Planner:
 
         cls.__check_valid_columns(tbl, stored_cols, 'inserted into')
 
-        row_builder = exprs.RowBuilder([], stored_cols, [])
+        row_builder = exprs.RowBuilder([], stored_cols, [], tbl)
 
         # create InMemoryDataNode for 'rows'
         plan: exec.ExecNode = exec.InMemoryDataNode(
@@ -473,15 +473,19 @@ class Planner:
         assert isinstance(tbl, catalog.TableVersionPath)
         target = tbl.tbl_version.get()  # the one we need to update
         updated_cols = list(update_targets.keys())
+        recomputed_cols: set[Column]
         if len(recompute_targets) > 0:
-            recomputed_cols = set(recompute_targets)
+            assert len(update_targets) == 0
+            recomputed_cols = {*recompute_targets}
+            if cascade:
+                recomputed_cols |= target.get_dependent_columns(recomputed_cols)
         else:
             recomputed_cols = target.get_dependent_columns(updated_cols) if cascade else set()
-            # regardless of cascade, we need to update all indices on any updated column
-            idx_val_cols = target.get_idx_val_columns(updated_cols)
-            recomputed_cols.update(idx_val_cols)
-            # we only need to recompute stored columns (unstored ones are substituted away)
-            recomputed_cols = {c for c in recomputed_cols if c.is_stored}
+        # regardless of cascade, we need to update all indices on any updated/recomputed column
+        idx_val_cols = target.get_idx_val_columns(set(updated_cols) | recomputed_cols)
+        recomputed_cols.update(idx_val_cols)
+        # we only need to recompute stored columns (unstored ones are substituted away)
+        recomputed_cols = {c for c in recomputed_cols if c.is_stored}
 
         cls.__check_valid_columns(tbl.tbl_version.get(), recomputed_cols, 'updated in')
 
@@ -508,6 +512,7 @@ class Planner:
         # update row builder with column information
         for i, col in enumerate(all_base_cols):
             plan.row_builder.add_table_column(col, select_list[i].slot_idx)
+        plan.ctx.num_computed_exprs = len(recomputed_exprs)
         recomputed_user_cols = [c for c in recomputed_cols if c.name is not None]
         return plan, [f'{c.tbl.name}.{c.name}' for c in updated_cols + recomputed_user_cols], recomputed_user_cols
 
@@ -588,7 +593,7 @@ class Planner:
         sql_exprs = list(
             exprs.Expr.list_subexprs(analyzer.all_exprs, filter=analyzer.sql_elements.contains, traverse_matches=False)
         )
-        row_builder = exprs.RowBuilder(analyzer.all_exprs, [], sql_exprs)
+        row_builder = exprs.RowBuilder(analyzer.all_exprs, [], sql_exprs, target)
         analyzer.finalize(row_builder)
         sql_lookup_node = exec.SqlLookupNode(tbl, row_builder, sql_exprs, sa_key_cols, key_vals)
         col_vals = [{col: row[col].val for col in updated_cols} for row in batch]
@@ -602,8 +607,7 @@ class Planner:
         row_builder.set_slot_idxs(select_list, remove_duplicates=False)
         for i, col in enumerate(all_base_cols):
             plan.row_builder.add_table_column(col, select_list[i].slot_idx)
-
-        ctx = exec.ExecContext(row_builder)
+        ctx = exec.ExecContext(row_builder, num_computed_exprs=len(recomputed_exprs))
         # we're returning everything to the user, so we might as well do it in a single batch
         ctx.batch_size = 0
         plan.set_ctx(ctx)
@@ -656,6 +660,7 @@ class Planner:
             ignore_errors=True,
             exact_version_only=view.get_bases(),
         )
+        plan.ctx.num_computed_exprs = len(recomputed_exprs)
         for i, col in enumerate(copied_cols + list(recomputed_cols)):  # same order as select_list
             plan.row_builder.add_table_column(col, select_list[i].slot_idx)
         # TODO: avoid duplication with view_load_plan() logic (where does this belong?)
@@ -695,7 +700,7 @@ class Planner:
         base_analyzer = Analyzer(
             from_clause, iterator_args, where_clause=target.predicate, sample_clause=target.sample_clause
         )
-        row_builder = exprs.RowBuilder(base_analyzer.all_exprs, stored_cols, [])
+        row_builder = exprs.RowBuilder(base_analyzer.all_exprs, stored_cols, [], target)
 
         # if we're propagating an insert, we only want to see those base rows that were created for the current version
         # execution plan:
@@ -832,7 +837,11 @@ class Planner:
             order_by_clause=order_by_clause,
             sample_clause=sample_clause,
         )
-        row_builder = exprs.RowBuilder(analyzer.all_exprs, [], [])
+        # If the from_clause has a single table, we can use it as the context table for the RowBuilder.
+        # Otherwise there is no context table, but that's ok, because the context table is only needed for
+        # table mutations, which can't happen during a join.
+        context_tbl = from_clause.tbls[0].tbl_version.get() if len(from_clause.tbls) == 1 else None
+        row_builder = exprs.RowBuilder(analyzer.all_exprs, [], [], context_tbl)
 
         analyzer.finalize(row_builder)
         # select_list: we need to materialize everything that's been collected
@@ -1035,16 +1044,14 @@ class Planner:
         return Analyzer(FromClause(tbls=[tbl]), [], where_clause=where_clause)
 
     @classmethod
-    def create_add_column_plan(
-        cls, tbl: catalog.TableVersionPath, col: catalog.Column
-    ) -> tuple[exec.ExecNode, Optional[int]]:
+    def create_add_column_plan(cls, tbl: catalog.TableVersionPath, col: catalog.Column) -> exec.ExecNode:
         """Creates a plan for InsertableTable.add_column()
         Returns:
             plan: the plan to execute
             value_expr slot idx for the plan output (for computed cols)
         """
         assert isinstance(tbl, catalog.TableVersionPath)
-        row_builder = exprs.RowBuilder(output_exprs=[], columns=[col], input_exprs=[])
+        row_builder = exprs.RowBuilder(output_exprs=[], columns=[col], input_exprs=[], tbl=tbl.tbl_version.get())
         analyzer = Analyzer(FromClause(tbls=[tbl]), row_builder.default_eval_ctx.target_exprs)
         plan = cls._create_query_plan(
             row_builder=row_builder, analyzer=analyzer, eval_ctx=row_builder.default_eval_ctx, with_pk=True
@@ -1052,9 +1059,10 @@ class Planner:
         plan.ctx.batch_size = 16
         plan.ctx.show_pbar = True
         plan.ctx.ignore_errors = True
+        computed_exprs = row_builder.output_exprs - row_builder.input_exprs
+        plan.ctx.num_computed_exprs = len(computed_exprs)  # we are adding a computed column, so we need to evaluate it
 
         # we want to flush images
         if col.is_computed and col.is_stored and col.col_type.is_image_type():
             plan.set_stored_img_cols(row_builder.output_slot_idxs())
-        value_expr_slot_idx = row_builder.output_slot_idxs()[0].slot_idx if col.is_computed else None
-        return plan, value_expr_slot_idx
+        return plan

pixeltable/share/packager.py

@@ -361,49 +361,32 @@ class TableRestorer:
             )
 
         tbl_md = [schema.FullTableMd.from_dict(t) for t in self.md['md']['tables']]
+        for md in tbl_md:
+            md.tbl_md.is_replica = True
 
-        # Create the replica table
-        # The logic here needs to be completely restructured in order to make it concurrency-safe.
-        # - Catalog.create_replica() needs to write the metadata and also create the physical store tables
-        #   and populate them, otherwise concurrent readers will see an inconsistent state (table metadata w/o
-        #   an actual table)
-        # - this could be done one replica at a time (instead of the entire hierarchy)
         cat = catalog.Catalog.get()
-        cat.create_replica(catalog.Path(self.tbl_path), tbl_md)
-        # don't call get_table() until after the calls to create_replica() and __import_table() below;
-        # the TV instances created by get_table() would be replaced by create_replica(), which creates duplicate
-        # TV instances for the same replica version, which then leads to failures when constructing queries
-
-        # Now we need to instantiate and load data for replica_tbl and its ancestors, except that we skip
-        # replica_tbl itself if it's a pure snapshot.
-        target_md = tbl_md[0]
-        is_pure_snapshot = (
-            target_md.tbl_md.view_md is not None
-            and target_md.tbl_md.view_md.predicate is None
-            and len(target_md.schema_version_md.columns) == 0
-        )
-        if is_pure_snapshot:
-            ancestor_md = tbl_md[1:]  # Pure snapshot; skip replica_tbl
-        else:
-            ancestor_md = tbl_md  # Not a pure snapshot; include replica_tbl
-
-        # Instantiate data from the Parquet tables.
-        with Env.get().begin_xact():
-            for md in ancestor_md[::-1]:  # Base table first
-                # Create a TableVersion instance (and a store table) for this ancestor.
-                tv = catalog.TableVersion.create_replica(md)
-                # Now import data from Parquet.
-                _logger.info(f'Importing table {tv.name!r}.')
-                self.__import_table(self.tmp_dir, tv, md)
-
-        with cat.begin_xact(for_write=False):
+
+        with cat.begin_xact(for_write=True):
+            # Create (or update) the replica table and its ancestors, along with TableVersion instances for any
+            # versions that have not been seen before.
+            cat.create_replica(catalog.Path(self.tbl_path), tbl_md)
+
+            # Now we need to load data for replica_tbl and its ancestors, except that we skip
+            # replica_tbl itself if it's a pure snapshot.
+            for md in tbl_md[::-1]:  # Base table first
+                if not md.is_pure_snapshot:
+                    tv = cat.get_tbl_version(UUID(md.tbl_md.tbl_id), md.version_md.version)
+                    # Import data from Parquet.
+                    _logger.info(f'Importing table {tv.name!r}.')
+                    self.__import_table(self.tmp_dir, tv, md)
+
             return cat.get_table_by_id(UUID(tbl_md[0].tbl_md.tbl_id))
 
     def __import_table(self, bundle_path: Path, tv: catalog.TableVersion, tbl_md: schema.FullTableMd) -> None:
         """
         Import the Parquet table into the Pixeltable catalog.
         """
-        tbl_id = uuid.UUID(tbl_md.tbl_md.tbl_id)
+        tbl_id = UUID(tbl_md.tbl_md.tbl_id)
         parquet_dir = bundle_path / 'tables' / f'tbl_{tbl_id.hex}'
         parquet_table = pq.read_table(str(parquet_dir))
         replica_version = tv.version
@@ -626,9 +609,8 @@ class TableRestorer:
                 # First time seeing this pxtmedia:// URL. Relocate the file to the media store and record the mapping
                 # in self.media_files.
                 src_path = self.tmp_dir / 'media' / parsed_url.netloc
-                dest_path = MediaStore.prepare_media_path(tv.id, media_col_id, tv.version, ext=src_path.suffix)
-                src_path.rename(dest_path)
-                self.media_files[url] = urllib.parse.urljoin('file:', urllib.request.pathname2url(str(dest_path)))
+                # Move the file to the media store and update the URL.
+                self.media_files[url] = MediaStore.relocate_local_media_file(src_path, tv.id, media_col_id, tv.version)
             return self.media_files[url]
         # For any type of URL other than a local file, just return the URL as-is.
         return url