pixeltable 0.4.0rc3__py3-none-any.whl → 0.4.2__py3-none-any.whl

pixeltable/metadata/utils.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from pixeltable.metadata import schema
+
+
+class MetadataUtils:
+    @classmethod
+    def _diff_md(
+        cls, old_md: Optional[dict[int, schema.SchemaColumn]], new_md: Optional[dict[int, schema.SchemaColumn]]
+    ) -> str:
+        """Return a string reporting the differences in a specific entry in two dictionaries
+
+        Results are formatted as follows:
+        - If `old_md` is `None`, returns 'Initial Version'.
+        - If `old_md` and `new_md` are the same, returns an empty string.
+        - If there are additions, changes, or deletions, returns a string summarizing the changes.
+        """
+        assert new_md is not None
+        if old_md is None:
+            return 'Initial Version'
+        if old_md == new_md:
+            return ''
+        added = {k: v.name for k, v in new_md.items() if k not in old_md}
+        changed = {
+            k: f'{old_md[k].name!r} to {v.name!r}'
+            for k, v in new_md.items()
+            if k in old_md and old_md[k].name != v.name
+        }
+        deleted = {k: v.name for k, v in old_md.items() if k not in new_md}
+        if len(added) == 0 and len(changed) == 0 and len(deleted) == 0:
+            return ''
+        # Format the result
+        t = []
+        if len(added) > 0:
+            t.append('Added: ' + ', '.join(added.values()))
+        if len(changed) > 0:
+            t.append('Renamed: ' + ', '.join(changed.values()))
+        if len(deleted) > 0:
+            t.append('Deleted: ' + ', '.join(deleted.values()))
+        r = ', '.join(t)
+        return r
+
+    @classmethod
+    def _create_md_change_dict(
+        cls, md_list: Optional[list[tuple[int, dict[int, schema.SchemaColumn]]]]
+    ) -> dict[int, str]:
+        """Return a dictionary of schema changes by version
+        Args:
+            md_list: a list of tuples, each containing a version number and a metadata dictionary.
+        """
+        r: dict[int, str] = {}
+        if md_list is None or len(md_list) == 0:
+            return r
+
+        # Sort the list in place by version number
+        md_list.sort()
+
+        first_retrieved_version = md_list[0][0]
+        if first_retrieved_version == 0:
+            prev_md = None
+            prev_ver = -1
+            start = 0
+        else:
+            prev_md = md_list[0][1]
+            prev_ver = first_retrieved_version
+            start = 1
+
+        for ver, curr_md in md_list[start:]:
+            if ver == prev_ver:
+                continue
+            assert ver > prev_ver
+            tf = cls._diff_md(prev_md, curr_md)
+            if tf != '':
+                r[ver] = tf
+            prev_md = curr_md
+        return r

pixeltable/plan.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import dataclasses
 import enum
 from textwrap import dedent
-from typing import Any, Iterable, Literal, NamedTuple, Optional, Sequence
+from typing import Any, Iterable, Literal, Optional, Sequence
 from uuid import UUID
 
 import sqlalchemy as sql
@@ -12,7 +12,6 @@ import pixeltable as pxt
 from pixeltable import catalog, exceptions as excs, exec, exprs
 from pixeltable.catalog import Column, TableVersionHandle
 from pixeltable.exec.sql_node import OrderByClause, OrderByItem, combine_order_by_clauses, print_order_by_clause
-from pixeltable.utils.sample import sample_key
 
 
 def _is_agg_fn_call(e: exprs.Expr) -> bool:
@@ -159,16 +158,6 @@ class SampleClause:
         return format(threshold_int, '08x') + 'ffffffffffffffffffffffff'
 
 
-class SamplingClauses(NamedTuple):
-    """Clauses provided when rewriting a SampleClause"""
-
-    where: exprs.Expr
-    group_by_clause: Optional[list[exprs.Expr]]
-    order_by_clause: Optional[list[tuple[exprs.Expr, bool]]]
-    limit: Optional[exprs.Expr]
-    sample_clause: Optional[SampleClause]
-
-
 class Analyzer:
     """
     Performs semantic analysis of a query and stores the analysis state.
@@ -180,6 +169,8 @@ class Analyzer:
     group_by_clause: Optional[list[exprs.Expr]]  # None for non-aggregate queries; [] for agg query w/o grouping
     grouping_exprs: list[exprs.Expr]  # [] for non-aggregate queries or agg query w/o grouping
     order_by_clause: OrderByClause
+    stratify_exprs: list[exprs.Expr]  # [] if no stratiifcation is required
+    sample_clause: Optional[SampleClause]  # None if no sampling clause is present
 
     sql_elements: exprs.SqlElementCache
 
@@ -200,6 +191,7 @@ class Analyzer:
         where_clause: Optional[exprs.Expr] = None,
         group_by_clause: Optional[list[exprs.Expr]] = None,
         order_by_clause: Optional[list[tuple[exprs.Expr, bool]]] = None,
+        sample_clause: Optional[SampleClause] = None,
     ):
         if order_by_clause is None:
             order_by_clause = []
@@ -213,6 +205,11 @@ class Analyzer:
         self.group_by_clause = (
             [e.resolve_computed_cols() for e in group_by_clause] if group_by_clause is not None else None
         )
+        self.sample_clause = sample_clause
+        if self.sample_clause is not None and self.sample_clause.is_stratified:
+            self.stratify_exprs = [e.resolve_computed_cols() for e in sample_clause.stratify_exprs]
+        else:
+            self.stratify_exprs = []
         self.order_by_clause = [OrderByItem(e.resolve_computed_cols(), asc) for e, asc in order_by_clause]
 
         self.sql_where_clause = None
@@ -228,8 +225,11 @@ class Analyzer:
                 self.all_exprs.append(join_clause.join_predicate)
         if self.group_by_clause is not None:
             self.all_exprs.extend(self.group_by_clause)
+        self.all_exprs.extend(self.stratify_exprs)
         self.all_exprs.extend(e for e, _ in self.order_by_clause)
         if self.filter is not None:
+            if sample_clause is not None:
+                raise excs.Error(f'Filter {self.filter} not expressible in SQL')
             self.all_exprs.append(self.filter)
 
         self.agg_order_by = []
@@ -378,7 +378,7 @@ class Planner:
 
         cls.__check_valid_columns(tbl, stored_cols, 'inserted into')
 
-        row_builder = exprs.RowBuilder([], stored_cols, [])
+        row_builder = exprs.RowBuilder([], stored_cols, [], tbl)
 
         # create InMemoryDataNode for 'rows'
         plan: exec.ExecNode = exec.InMemoryDataNode(
@@ -473,15 +473,19 @@ class Planner:
         assert isinstance(tbl, catalog.TableVersionPath)
         target = tbl.tbl_version.get()  # the one we need to update
         updated_cols = list(update_targets.keys())
+        recomputed_cols: set[Column]
         if len(recompute_targets) > 0:
-            recomputed_cols = set(recompute_targets)
+            assert len(update_targets) == 0
+            recomputed_cols = {*recompute_targets}
+            if cascade:
+                recomputed_cols |= target.get_dependent_columns(recomputed_cols)
         else:
             recomputed_cols = target.get_dependent_columns(updated_cols) if cascade else set()
-            # regardless of cascade, we need to update all indices on any updated column
-            idx_val_cols = target.get_idx_val_columns(updated_cols)
-            recomputed_cols.update(idx_val_cols)
-            # we only need to recompute stored columns (unstored ones are substituted away)
-            recomputed_cols = {c for c in recomputed_cols if c.is_stored}
+        # regardless of cascade, we need to update all indices on any updated/recomputed column
+        idx_val_cols = target.get_idx_val_columns(set(updated_cols) | recomputed_cols)
+        recomputed_cols.update(idx_val_cols)
+        # we only need to recompute stored columns (unstored ones are substituted away)
+        recomputed_cols = {c for c in recomputed_cols if c.is_stored}
 
         cls.__check_valid_columns(tbl.tbl_version.get(), recomputed_cols, 'updated in')
 
@@ -588,7 +592,7 @@ class Planner:
         sql_exprs = list(
             exprs.Expr.list_subexprs(analyzer.all_exprs, filter=analyzer.sql_elements.contains, traverse_matches=False)
         )
-        row_builder = exprs.RowBuilder(analyzer.all_exprs, [], sql_exprs)
+        row_builder = exprs.RowBuilder(analyzer.all_exprs, [], sql_exprs, target)
         analyzer.finalize(row_builder)
         sql_lookup_node = exec.SqlLookupNode(tbl, row_builder, sql_exprs, sa_key_cols, key_vals)
         col_vals = [{col: row[col].val for col in updated_cols} for row in batch]
@@ -602,8 +606,7 @@ class Planner:
         row_builder.set_slot_idxs(select_list, remove_duplicates=False)
         for i, col in enumerate(all_base_cols):
             plan.row_builder.add_table_column(col, select_list[i].slot_idx)
-
-        ctx = exec.ExecContext(row_builder)
+        ctx = exec.ExecContext(row_builder, num_computed_exprs=len(recomputed_exprs))
         # we're returning everything to the user, so we might as well do it in a single batch
         ctx.batch_size = 0
         plan.set_ctx(ctx)
@@ -691,25 +694,13 @@ class Planner:
         # 2. for component views: iterator args
         iterator_args = [target.iterator_args] if target.iterator_args is not None else []
 
-        # If this contains a sample specification, modify / create where, group_by, order_by, and limit clauses
         from_clause = FromClause(tbls=[view.base])
-        where, group_by_clause, order_by_clause, limit, sample_clause = cls.create_sample_clauses(
-            from_clause, target.sample_clause, target.predicate, None, [], None
-        )
-
-        # if we're propagating an insert, we only want to see those base rows that were created for the current version
         base_analyzer = Analyzer(
-            from_clause,
-            iterator_args,
-            where_clause=where,
-            group_by_clause=group_by_clause,
-            order_by_clause=order_by_clause,
+            from_clause, iterator_args, where_clause=target.predicate, sample_clause=target.sample_clause
         )
-        row_builder = exprs.RowBuilder(base_analyzer.all_exprs, stored_cols, [])
-
-        if target.sample_clause is not None and base_analyzer.filter is not None:
-            raise excs.Error(f'Filter {base_analyzer.filter} not expressible in SQL')
+        row_builder = exprs.RowBuilder(base_analyzer.all_exprs, stored_cols, [], target)
 
+        # if we're propagating an insert, we only want to see those base rows that were created for the current version
         # execution plan:
         # 1. materialize exprs computed from the base that are needed for stored view columns
         # 2. if it's an iterator view, expand the base rows into component rows
@@ -723,19 +714,13 @@ class Planner:
 
         # Create a new analyzer reflecting exactly what is required from the base table
         base_analyzer = Analyzer(
-            from_clause,
-            base_output_exprs,
-            where_clause=where,
-            group_by_clause=group_by_clause,
-            order_by_clause=order_by_clause,
+            from_clause, base_output_exprs, where_clause=target.predicate, sample_clause=target.sample_clause
         )
         base_eval_ctx = row_builder.create_eval_ctx(base_analyzer.all_exprs)
         plan = cls._create_query_plan(
             row_builder=row_builder,
             analyzer=base_analyzer,
             eval_ctx=base_eval_ctx,
-            limit=limit,
-            sample_clause=sample_clause,
             with_pk=True,
             exact_version_only=view.get_bases() if propagates_insert else [],
         )
@@ -818,62 +803,6 @@ class Planner:
         prefetch_node = exec.CachePrefetchNode(tbl_id, file_col_info, input_node)
         return prefetch_node
 
-    @classmethod
-    def create_sample_clauses(
-        cls,
-        from_clause: FromClause,
-        sample_clause: SampleClause,
-        where_clause: Optional[exprs.Expr],
-        group_by_clause: Optional[list[exprs.Expr]],
-        order_by_clause: Optional[list[tuple[exprs.Expr, bool]]],
-        limit: Optional[exprs.Expr],
-    ) -> SamplingClauses:
-        """tuple[
-            exprs.Expr,
-            Optional[list[exprs.Expr]],
-            Optional[list[tuple[exprs.Expr, bool]]],
-            Optional[exprs.Expr],
-            Optional[SampleClause],
-        ]:"""
-        """Construct clauses required for sampling under various conditions.
-        If there is no sampling, then return the original clauses.
-        If the sample is stratified, then return only the group by clause. The rest of the
-        mechanism for stratified sampling is provided by the SampleSqlNode.
-        If the sample is non-stratified, then rewrite the query to accommodate the supplied where clause,
-        and provide the other clauses required for sampling
-        """
-
-        # If no sample clause, return the original clauses
-        if sample_clause is None:
-            return SamplingClauses(where_clause, group_by_clause, order_by_clause, limit, None)
-
-        # If the sample clause is stratified, create a group by clause
-        if sample_clause.is_stratified:
-            group_by = sample_clause.stratify_exprs
-            # Note that limit is not possible here
-            return SamplingClauses(where_clause, group_by, order_by_clause, None, sample_clause)
-
-        else:
-            # If non-stratified sampling, construct a where clause, order_by, and limit clauses
-            # Construct an expression for sorting rows and limiting row counts
-            s_key = sample_key(
-                exprs.Literal(sample_clause.seed), *cls.rowid_columns(from_clause._first_tbl.tbl_version)
-            )
-
-            # Construct a suitable where clause
-            where = where_clause
-            if sample_clause.fraction is not None:
-                fraction_md5_hex = exprs.Expr.from_object(
-                    sample_clause.fraction_to_md5_hex(float(sample_clause.fraction))
-                )
-                f_where = s_key < fraction_md5_hex
-                where = where & f_where if where is not None else f_where
-
-            order_by: list[tuple[exprs.Expr, bool]] = [(s_key, True)]
-            limit = exprs.Literal(sample_clause.n)
-            # Note that group_by is not possible here
-            return SamplingClauses(where, None, order_by, limit, None)
-
     @classmethod
     def create_query_plan(
         cls,
@@ -898,21 +827,19 @@ class Planner:
         if exact_version_only is None:
             exact_version_only = []
 
-        # Modify clauses to include sample clause
-        where, group_by_clause, order_by_clause, limit, sample = cls.create_sample_clauses(
-            from_clause, sample_clause, where_clause, group_by_clause, order_by_clause, limit
-        )
-
         analyzer = Analyzer(
             from_clause,
             select_list,
-            where_clause=where,
+            where_clause=where_clause,
             group_by_clause=group_by_clause,
             order_by_clause=order_by_clause,
+            sample_clause=sample_clause,
         )
-        row_builder = exprs.RowBuilder(analyzer.all_exprs, [], [])
-        if sample_clause is not None and analyzer.filter is not None:
-            raise excs.Error(f'Filter {analyzer.filter} not expressible in SQL')
+        # If the from_clause has a single table, we can use it as the context table for the RowBuilder.
+        # Otherwise there is no context table, but that's ok, because the context table is only needed for
+        # table mutations, which can't happen during a join.
+        context_tbl = from_clause.tbls[0].tbl_version.get() if len(from_clause.tbls) == 1 else None
+        row_builder = exprs.RowBuilder(analyzer.all_exprs, [], [], context_tbl)
 
         analyzer.finalize(row_builder)
         # select_list: we need to materialize everything that's been collected
@@ -923,7 +850,6 @@ class Planner:
             analyzer=analyzer,
             eval_ctx=eval_ctx,
             limit=limit,
-            sample_clause=sample,
             with_pk=True,
             exact_version_only=exact_version_only,
         )
@@ -939,7 +865,6 @@ class Planner:
         analyzer: Analyzer,
         eval_ctx: exprs.RowBuilder.EvalCtx,
         limit: Optional[exprs.Expr] = None,
-        sample_clause: Optional[SampleClause] = None,
         with_pk: bool = False,
         exact_version_only: Optional[list[catalog.TableVersionHandle]] = None,
     ) -> exec.ExecNode:
@@ -966,6 +891,7 @@ class Planner:
         # - join clause subexprs
         # - subexprs of Where clause conjuncts that can't be run in SQL
         # - all grouping exprs
+        # - all stratify exprs
         candidates = list(
             exprs.Expr.list_subexprs(
                 analyzer.select_list,
@@ -980,10 +906,12 @@ class Planner:
             candidates.extend(
                 exprs.Expr.subexprs(analyzer.filter, filter=sql_elements.contains, traverse_matches=False)
             )
-        if analyzer.group_by_clause is not None:
-            candidates.extend(
-                exprs.Expr.list_subexprs(analyzer.group_by_clause, filter=sql_elements.contains, traverse_matches=False)
-            )
+        candidates.extend(
+            exprs.Expr.list_subexprs(analyzer.grouping_exprs, filter=sql_elements.contains, traverse_matches=False)
+        )
+        candidates.extend(
+            exprs.Expr.list_subexprs(analyzer.stratify_exprs, filter=sql_elements.contains, traverse_matches=False)
+        )
         # not isinstance(...): we don't want to materialize Literals via a Select
         sql_exprs = exprs.ExprSet(e for e in candidates if not isinstance(e, exprs.Literal))
 
@@ -1028,6 +956,15 @@ class Planner:
             # we need to order the input for window functions
             plan.set_order_by(analyzer.get_window_fn_ob_clause())
 
+        if analyzer.sample_clause is not None:
+            plan = exec.SqlSampleNode(
+                row_builder,
+                input=plan,
+                select_list=tbl_scan_exprs,
+                sample_clause=analyzer.sample_clause,
+                stratify_exprs=analyzer.stratify_exprs,
+            )
+
         plan = cls._insert_prefetch_node(tbl.tbl_version.id, row_builder, plan)
 
         if analyzer.group_by_clause is not None:
@@ -1050,26 +987,12 @@ class Planner:
                 sql_elements.contains_all(analyzer.select_list)
                 and sql_elements.contains_all(analyzer.grouping_exprs)
                 and isinstance(plan, exec.SqlNode)
-                and plan.to_cte(keep_pk=(sample_clause is not None)) is not None
+                and plan.to_cte() is not None
             ):
-                if sample_clause is not None:
-                    plan = exec.SqlSampleNode(
-                        row_builder,
-                        input=plan,
-                        select_list=analyzer.select_list,
-                        stratify_exprs=analyzer.group_by_clause,
-                        sample_clause=sample_clause,
-                    )
-                else:
-                    plan = exec.SqlAggregationNode(
-                        row_builder,
-                        input=plan,
-                        select_list=analyzer.select_list,
-                        group_by_items=analyzer.group_by_clause,
-                    )
+                plan = exec.SqlAggregationNode(
+                    row_builder, input=plan, select_list=analyzer.select_list, group_by_items=analyzer.group_by_clause
+                )
             else:
-                if sample_clause is not None:
-                    raise excs.Error('Sample clause not supported with Python aggregation')
                 input_sql_node = plan.get_node(exec.SqlNode)
                 assert combined_ordering is not None
                 input_sql_node.set_order_by(combined_ordering)
@@ -1119,16 +1042,14 @@ class Planner:
         return Analyzer(FromClause(tbls=[tbl]), [], where_clause=where_clause)
 
     @classmethod
-    def create_add_column_plan(
-        cls, tbl: catalog.TableVersionPath, col: catalog.Column
-    ) -> tuple[exec.ExecNode, Optional[int]]:
+    def create_add_column_plan(cls, tbl: catalog.TableVersionPath, col: catalog.Column) -> exec.ExecNode:
         """Creates a plan for InsertableTable.add_column()
         Returns:
             plan: the plan to execute
             value_expr slot idx for the plan output (for computed cols)
         """
         assert isinstance(tbl, catalog.TableVersionPath)
-        row_builder = exprs.RowBuilder(output_exprs=[], columns=[col], input_exprs=[])
+        row_builder = exprs.RowBuilder(output_exprs=[], columns=[col], input_exprs=[], tbl=tbl.tbl_version.get())
         analyzer = Analyzer(FromClause(tbls=[tbl]), row_builder.default_eval_ctx.target_exprs)
         plan = cls._create_query_plan(
             row_builder=row_builder, analyzer=analyzer, eval_ctx=row_builder.default_eval_ctx, with_pk=True
@@ -1140,5 +1061,4 @@ class Planner:
         # we want to flush images
         if col.is_computed and col.is_stored and col.col_type.is_image_type():
             plan.set_stored_img_cols(row_builder.output_slot_idxs())
-        value_expr_slot_idx = row_builder.output_slot_idxs()[0].slot_idx if col.is_computed else None
-        return plan, value_expr_slot_idx
+        return plan

pixeltable/share/packager.py

@@ -127,7 +127,7 @@ class TablePackager:
         # We use snappy compression for the Parquet tables; the entire bundle will be bzip2-compressed later, so
         # faster compression should provide good performance while still reducing temporary storage utilization.
         parquet_writer = pq.ParquetWriter(parquet_file, parquet_schema, compression='SNAPPY')
-        filter_tv = self.table._tbl_version.get()
+        filter_tv = self.table._tbl_version_path.tbl_version.get()
         row_iter = tv.store_tbl.dump_rows(tv.version, filter_tv.store_tbl, filter_tv.version)
         for pa_table in self.__to_pa_tables(row_iter, sql_types, media_cols, parquet_schema):
             parquet_writer.write_table(pa_table)
@@ -238,7 +238,7 @@ class TablePackager:
         - Documents are replaced by a thumbnail as a base64-encoded webp
         """
         # First 8 columns
-        preview_cols = dict(itertools.islice(self.table._schema.items(), 0, 8))
+        preview_cols = dict(itertools.islice(self.table._get_schema().items(), 0, 8))
         select_list = [self.table[col_name] for col_name in preview_cols]
         # First 5 rows
         rows = list(self.table.select(*select_list).head(n=5))