pixeltable 0.4.0rc2__py3-none-any.whl → 0.4.1__py3-none-any.whl

pixeltable/globals.py

@@ -428,8 +428,6 @@ def get_table(path: str) -> catalog.Table:
     """
     path_obj = catalog.Path(path)
     tbl = Catalog.get().get_table(path_obj)
-    tv = tbl._tbl_version.get()
-    _logger.debug(f'get_table(): tbl={tv.id}:{tv.effective_version} sa_tbl={id(tv.store_tbl.sa_tbl):x} tv={id(tv):x}')
     return tbl
 
 

pixeltable/io/external_store.py

@@ -202,7 +202,7 @@ class Project(ExternalStore, abc.ABC):
         resolved_col_mapping: dict[Column, str] = {}
 
         # Validate names
-        t_cols = set(table._schema.keys())
+        t_cols = set(table._get_schema().keys())
         for t_col, ext_col in col_mapping.items():
             if t_col not in t_cols:
                 if is_user_specified_col_mapping:
@@ -225,7 +225,7 @@ class Project(ExternalStore, abc.ABC):
             assert isinstance(col_ref, exprs.ColumnRef)
             resolved_col_mapping[col_ref.col] = ext_col
         # Validate column specs
-        t_col_types = table._schema
+        t_col_types = table._get_schema()
         for t_col, ext_col in col_mapping.items():
             t_col_type = t_col_types[t_col]
             if ext_col in export_cols:

pixeltable/io/label_studio.py

@@ -412,8 +412,8 @@ class LabelStudioProject(Project):
             # TODO(aaron-siegel): Simplify this once propagation is properly implemented in batch_update
             ancestor = t
             while local_annotations_col not in ancestor._tbl_version.get().cols:
-                assert ancestor._base_table is not None
-                ancestor = ancestor._base_table
+                assert ancestor._get_base_table is not None
+                ancestor = ancestor._get_base_table()
             update_status = ancestor.batch_update(updates)
             env.Env.get().console_logger.info(f'Updated annotation(s) from {len(updates)} task(s) in {self}.')
             return SyncStatus(pxt_rows_updated=update_status.num_rows, num_excs=update_status.num_excs)
@@ -560,7 +560,7 @@ class LabelStudioProject(Project):
 
         if name is None:
             # Create a default name that's unique to the table
-            all_stores = t.external_stores
+            all_stores = t.external_stores()
             n = 0
             while f'ls_project_{n}' in all_stores:
                 n += 1
@@ -576,7 +576,7 @@ class LabelStudioProject(Project):
                 local_annotations_column = ANNOTATIONS_COLUMN
             else:
                 local_annotations_column = next(k for k, v in col_mapping.items() if v == ANNOTATIONS_COLUMN)
-            if local_annotations_column not in t._schema:
+            if local_annotations_column not in t._get_schema():
                 t.add_columns({local_annotations_column: ts.Json})
 
         resolved_col_mapping = cls.validate_columns(

pixeltable/io/table_data_conduit.py

@@ -101,7 +101,7 @@ class TableDataConduit:
     def add_table_info(self, table: pxt.Table) -> None:
         """Add information about the table into which we are inserting data"""
         assert isinstance(table, pxt.Table)
-        self.pxt_schema = table._schema
+        self.pxt_schema = table._get_schema()
         self.pxt_pk = table._tbl_version.get().primary_key
         for col in table._tbl_version_path.columns():
             if col.is_required_for_insert:

pixeltable/metadata/__init__.py

@@ -18,7 +18,7 @@ _console_logger = ConsoleLogger(logging.getLogger('pixeltable'))
 _logger = logging.getLogger('pixeltable')
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 37
+VERSION = 38
 
 
 def create_system_info(engine: sql.engine.Engine) -> None:

pixeltable/metadata/converters/convert_37.py

@@ -0,0 +1,15 @@
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=37)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_md_updater=__update_table_md)
+
+
+def __update_table_md(table_md: dict, _: UUID) -> None:
+    table_md['view_sn'] = 0

pixeltable/metadata/notes.py

@@ -2,6 +2,7 @@
 # rather than as a comment, so that the existence of a description can be enforced by
 # the unit tests when new versions are added.
 VERSION_NOTES = {
+    38: 'Added TableMd.view_sn',
     37: 'Add support for the sample() method on DataFrames',
     36: 'Added Table.lock_dummy',
     35: 'Track reference_tbl in ColumnRef',

pixeltable/metadata/schema.py

@@ -177,6 +177,11 @@ class TableMd:
     # - every row is assigned a unique and immutable rowid on insertion
     next_row_id: int
 
+    # sequence number to track changes in the set of mutable views of this table (ie, this table = the view base)
+    # - incremented for each add/drop of a mutable view
+    # - only maintained for mutable tables
+    view_sn: int
+
     # Metadata format for external stores:
     # {'class': 'pixeltable.io.label_studio.LabelStudioProject', 'md': {'project_id': 3}}
     external_stores: list[dict[str, Any]]

pixeltable/plan.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import dataclasses
 import enum
 from textwrap import dedent
-from typing import Any, Iterable, Literal, NamedTuple, Optional, Sequence
+from typing import Any, Iterable, Literal, Optional, Sequence
 from uuid import UUID
 
 import sqlalchemy as sql
@@ -12,7 +12,6 @@ import pixeltable as pxt
 from pixeltable import catalog, exceptions as excs, exec, exprs
 from pixeltable.catalog import Column, TableVersionHandle
 from pixeltable.exec.sql_node import OrderByClause, OrderByItem, combine_order_by_clauses, print_order_by_clause
-from pixeltable.utils.sample import sample_key
 
 
 def _is_agg_fn_call(e: exprs.Expr) -> bool:
@@ -159,16 +158,6 @@ class SampleClause:
         return format(threshold_int, '08x') + 'ffffffffffffffffffffffff'
 
 
-class SamplingClauses(NamedTuple):
-    """Clauses provided when rewriting a SampleClause"""
-
-    where: exprs.Expr
-    group_by_clause: Optional[list[exprs.Expr]]
-    order_by_clause: Optional[list[tuple[exprs.Expr, bool]]]
-    limit: Optional[exprs.Expr]
-    sample_clause: Optional[SampleClause]
-
-
 class Analyzer:
     """
     Performs semantic analysis of a query and stores the analysis state.
@@ -180,6 +169,8 @@ class Analyzer:
     group_by_clause: Optional[list[exprs.Expr]]  # None for non-aggregate queries; [] for agg query w/o grouping
     grouping_exprs: list[exprs.Expr]  # [] for non-aggregate queries or agg query w/o grouping
     order_by_clause: OrderByClause
+    stratify_exprs: list[exprs.Expr]  # [] if no stratiifcation is required
+    sample_clause: Optional[SampleClause]  # None if no sampling clause is present
 
     sql_elements: exprs.SqlElementCache
 
@@ -200,6 +191,7 @@ class Analyzer:
         where_clause: Optional[exprs.Expr] = None,
         group_by_clause: Optional[list[exprs.Expr]] = None,
         order_by_clause: Optional[list[tuple[exprs.Expr, bool]]] = None,
+        sample_clause: Optional[SampleClause] = None,
     ):
         if order_by_clause is None:
             order_by_clause = []
@@ -213,6 +205,11 @@ class Analyzer:
         self.group_by_clause = (
             [e.resolve_computed_cols() for e in group_by_clause] if group_by_clause is not None else None
         )
+        self.sample_clause = sample_clause
+        if self.sample_clause is not None and self.sample_clause.is_stratified:
+            self.stratify_exprs = [e.resolve_computed_cols() for e in sample_clause.stratify_exprs]
+        else:
+            self.stratify_exprs = []
         self.order_by_clause = [OrderByItem(e.resolve_computed_cols(), asc) for e, asc in order_by_clause]
 
         self.sql_where_clause = None
@@ -228,8 +225,11 @@ class Analyzer:
                 self.all_exprs.append(join_clause.join_predicate)
         if self.group_by_clause is not None:
             self.all_exprs.extend(self.group_by_clause)
+        self.all_exprs.extend(self.stratify_exprs)
         self.all_exprs.extend(e for e, _ in self.order_by_clause)
         if self.filter is not None:
+            if sample_clause is not None:
+                raise excs.Error(f'Filter {self.filter} not expressible in SQL')
             self.all_exprs.append(self.filter)
 
         self.agg_order_by = []
@@ -691,25 +691,13 @@ class Planner:
         # 2. for component views: iterator args
         iterator_args = [target.iterator_args] if target.iterator_args is not None else []
 
-        # If this contains a sample specification, modify / create where, group_by, order_by, and limit clauses
         from_clause = FromClause(tbls=[view.base])
-        where, group_by_clause, order_by_clause, limit, sample_clause = cls.create_sample_clauses(
-            from_clause, target.sample_clause, target.predicate, None, [], None
-        )
-
-        # if we're propagating an insert, we only want to see those base rows that were created for the current version
         base_analyzer = Analyzer(
-            from_clause,
-            iterator_args,
-            where_clause=where,
-            group_by_clause=group_by_clause,
-            order_by_clause=order_by_clause,
+            from_clause, iterator_args, where_clause=target.predicate, sample_clause=target.sample_clause
         )
         row_builder = exprs.RowBuilder(base_analyzer.all_exprs, stored_cols, [])
 
-        if target.sample_clause is not None and base_analyzer.filter is not None:
-            raise excs.Error(f'Filter {base_analyzer.filter} not expressible in SQL')
-
+        # if we're propagating an insert, we only want to see those base rows that were created for the current version
         # execution plan:
         # 1. materialize exprs computed from the base that are needed for stored view columns
         # 2. if it's an iterator view, expand the base rows into component rows
@@ -723,19 +711,13 @@ class Planner:
 
         # Create a new analyzer reflecting exactly what is required from the base table
         base_analyzer = Analyzer(
-            from_clause,
-            base_output_exprs,
-            where_clause=where,
-            group_by_clause=group_by_clause,
-            order_by_clause=order_by_clause,
+            from_clause, base_output_exprs, where_clause=target.predicate, sample_clause=target.sample_clause
         )
         base_eval_ctx = row_builder.create_eval_ctx(base_analyzer.all_exprs)
         plan = cls._create_query_plan(
             row_builder=row_builder,
             analyzer=base_analyzer,
             eval_ctx=base_eval_ctx,
-            limit=limit,
-            sample_clause=sample_clause,
             with_pk=True,
             exact_version_only=view.get_bases() if propagates_insert else [],
         )
@@ -818,62 +800,6 @@ class Planner:
         prefetch_node = exec.CachePrefetchNode(tbl_id, file_col_info, input_node)
         return prefetch_node
 
-    @classmethod
-    def create_sample_clauses(
-        cls,
-        from_clause: FromClause,
-        sample_clause: SampleClause,
-        where_clause: Optional[exprs.Expr],
-        group_by_clause: Optional[list[exprs.Expr]],
-        order_by_clause: Optional[list[tuple[exprs.Expr, bool]]],
-        limit: Optional[exprs.Expr],
-    ) -> SamplingClauses:
-        """tuple[
-            exprs.Expr,
-            Optional[list[exprs.Expr]],
-            Optional[list[tuple[exprs.Expr, bool]]],
-            Optional[exprs.Expr],
-            Optional[SampleClause],
-        ]:"""
-        """Construct clauses required for sampling under various conditions.
-        If there is no sampling, then return the original clauses.
-        If the sample is stratified, then return only the group by clause. The rest of the
-        mechanism for stratified sampling is provided by the SampleSqlNode.
-        If the sample is non-stratified, then rewrite the query to accommodate the supplied where clause,
-        and provide the other clauses required for sampling
-        """
-
-        # If no sample clause, return the original clauses
-        if sample_clause is None:
-            return SamplingClauses(where_clause, group_by_clause, order_by_clause, limit, None)
-
-        # If the sample clause is stratified, create a group by clause
-        if sample_clause.is_stratified:
-            group_by = sample_clause.stratify_exprs
-            # Note that limit is not possible here
-            return SamplingClauses(where_clause, group_by, order_by_clause, None, sample_clause)
-
-        else:
-            # If non-stratified sampling, construct a where clause, order_by, and limit clauses
-            # Construct an expression for sorting rows and limiting row counts
-            s_key = sample_key(
-                exprs.Literal(sample_clause.seed), *cls.rowid_columns(from_clause._first_tbl.tbl_version)
-            )
-
-            # Construct a suitable where clause
-            where = where_clause
-            if sample_clause.fraction is not None:
-                fraction_md5_hex = exprs.Expr.from_object(
-                    sample_clause.fraction_to_md5_hex(float(sample_clause.fraction))
-                )
-                f_where = s_key < fraction_md5_hex
-                where = where & f_where if where is not None else f_where
-
-            order_by: list[tuple[exprs.Expr, bool]] = [(s_key, True)]
-            limit = exprs.Literal(sample_clause.n)
-            # Note that group_by is not possible here
-            return SamplingClauses(where, None, order_by, limit, None)
-
     @classmethod
     def create_query_plan(
         cls,
@@ -898,21 +824,15 @@ class Planner:
         if exact_version_only is None:
             exact_version_only = []
 
-        # Modify clauses to include sample clause
-        where, group_by_clause, order_by_clause, limit, sample = cls.create_sample_clauses(
-            from_clause, sample_clause, where_clause, group_by_clause, order_by_clause, limit
-        )
-
         analyzer = Analyzer(
             from_clause,
             select_list,
-            where_clause=where,
+            where_clause=where_clause,
             group_by_clause=group_by_clause,
             order_by_clause=order_by_clause,
+            sample_clause=sample_clause,
         )
         row_builder = exprs.RowBuilder(analyzer.all_exprs, [], [])
-        if sample_clause is not None and analyzer.filter is not None:
-            raise excs.Error(f'Filter {analyzer.filter} not expressible in SQL')
 
         analyzer.finalize(row_builder)
         # select_list: we need to materialize everything that's been collected
@@ -923,7 +843,6 @@ class Planner:
             analyzer=analyzer,
             eval_ctx=eval_ctx,
             limit=limit,
-            sample_clause=sample,
             with_pk=True,
             exact_version_only=exact_version_only,
         )
@@ -939,7 +858,6 @@ class Planner:
         analyzer: Analyzer,
         eval_ctx: exprs.RowBuilder.EvalCtx,
         limit: Optional[exprs.Expr] = None,
-        sample_clause: Optional[SampleClause] = None,
         with_pk: bool = False,
         exact_version_only: Optional[list[catalog.TableVersionHandle]] = None,
     ) -> exec.ExecNode:
@@ -966,6 +884,7 @@ class Planner:
         # - join clause subexprs
         # - subexprs of Where clause conjuncts that can't be run in SQL
         # - all grouping exprs
+        # - all stratify exprs
         candidates = list(
             exprs.Expr.list_subexprs(
                 analyzer.select_list,
@@ -980,10 +899,12 @@ class Planner:
             candidates.extend(
                 exprs.Expr.subexprs(analyzer.filter, filter=sql_elements.contains, traverse_matches=False)
             )
-        if analyzer.group_by_clause is not None:
-            candidates.extend(
-                exprs.Expr.list_subexprs(analyzer.group_by_clause, filter=sql_elements.contains, traverse_matches=False)
-            )
+        candidates.extend(
+            exprs.Expr.list_subexprs(analyzer.grouping_exprs, filter=sql_elements.contains, traverse_matches=False)
+        )
+        candidates.extend(
+            exprs.Expr.list_subexprs(analyzer.stratify_exprs, filter=sql_elements.contains, traverse_matches=False)
+        )
         # not isinstance(...): we don't want to materialize Literals via a Select
         sql_exprs = exprs.ExprSet(e for e in candidates if not isinstance(e, exprs.Literal))
 
@@ -1028,6 +949,15 @@ class Planner:
             # we need to order the input for window functions
             plan.set_order_by(analyzer.get_window_fn_ob_clause())
 
+        if analyzer.sample_clause is not None:
+            plan = exec.SqlSampleNode(
+                row_builder,
+                input=plan,
+                select_list=tbl_scan_exprs,
+                sample_clause=analyzer.sample_clause,
+                stratify_exprs=analyzer.stratify_exprs,
+            )
+
         plan = cls._insert_prefetch_node(tbl.tbl_version.id, row_builder, plan)
 
         if analyzer.group_by_clause is not None:
@@ -1050,26 +980,12 @@ class Planner:
                 sql_elements.contains_all(analyzer.select_list)
                 and sql_elements.contains_all(analyzer.grouping_exprs)
                 and isinstance(plan, exec.SqlNode)
-                and plan.to_cte(keep_pk=(sample_clause is not None)) is not None
+                and plan.to_cte() is not None
             ):
-                if sample_clause is not None:
-                    plan = exec.SqlSampleNode(
-                        row_builder,
-                        input=plan,
-                        select_list=analyzer.select_list,
-                        stratify_exprs=analyzer.group_by_clause,
-                        sample_clause=sample_clause,
-                    )
-                else:
-                    plan = exec.SqlAggregationNode(
-                        row_builder,
-                        input=plan,
-                        select_list=analyzer.select_list,
-                        group_by_items=analyzer.group_by_clause,
-                    )
+                plan = exec.SqlAggregationNode(
+                    row_builder, input=plan, select_list=analyzer.select_list, group_by_items=analyzer.group_by_clause
+                )
             else:
-                if sample_clause is not None:
-                    raise excs.Error('Sample clause not supported with Python aggregation')
                 input_sql_node = plan.get_node(exec.SqlNode)
                 assert combined_ordering is not None
                 input_sql_node.set_order_by(combined_ordering)

pixeltable/share/packager.py

@@ -127,7 +127,7 @@ class TablePackager:
         # We use snappy compression for the Parquet tables; the entire bundle will be bzip2-compressed later, so
         # faster compression should provide good performance while still reducing temporary storage utilization.
         parquet_writer = pq.ParquetWriter(parquet_file, parquet_schema, compression='SNAPPY')
-        filter_tv = self.table._tbl_version.get()
+        filter_tv = self.table._tbl_version_path.tbl_version.get()
         row_iter = tv.store_tbl.dump_rows(tv.version, filter_tv.store_tbl, filter_tv.version)
         for pa_table in self.__to_pa_tables(row_iter, sql_types, media_cols, parquet_schema):
             parquet_writer.write_table(pa_table)
@@ -238,7 +238,7 @@ class TablePackager:
         - Documents are replaced by a thumbnail as a base64-encoded webp
         """
         # First 8 columns
-        preview_cols = dict(itertools.islice(self.table._schema.items(), 0, 8))
+        preview_cols = dict(itertools.islice(self.table._get_schema().items(), 0, 8))
         select_list = [self.table[col_name] for col_name in preview_cols]
         # First 5 rows
         rows = list(self.table.select(*select_list).head(n=5))

pixeltable/type_system.py

@@ -395,6 +395,36 @@ class ColumnType:
                 raise excs.Error(f'Standard Python type `{name}` cannot be used here; use `{suggestion}` instead')
         raise excs.Error(f'Unknown type: {t}')
 
+    @classmethod
+    def from_json_schema(cls, schema: dict[str, Any]) -> Optional[ColumnType]:
+        # We first express the JSON schema as a Python type, and then convert it to a Pixeltable type.
+        # TODO: Is there a meaningful fallback if one of these operations fails? (Maybe another use case for a pxt Any
+        #     type?)
+        py_type = cls.__json_schema_to_py_type(schema)
+        return cls.from_python_type(py_type) if py_type is not None else None
+
+    @classmethod
+    def __json_schema_to_py_type(cls, schema: dict[str, Any]) -> Union[type, _GenericAlias, None]:
+        if 'type' in schema:
+            if schema['type'] == 'null':
+                return type(None)
+            if schema['type'] == 'string':
+                return str
+            if schema['type'] == 'integer':
+                return int
+            if schema['type'] == 'number':
+                return float
+            if schema['type'] == 'boolean':
+                return bool
+            if schema['type'] in ('array', 'object'):
+                return list
+        elif 'anyOf' in schema:
+            subscripts = tuple(cls.__json_schema_to_py_type(subschema) for subschema in schema['anyOf'])
+            if all(subscript is not None for subscript in subscripts):
+                return Union[subscripts]
+
+        return None
+
     def validate_literal(self, val: Any) -> None:
         """Raise TypeError if val is not a valid literal for this type"""
         if val is None:

{pixeltable-0.4.0rc2.dist-info → pixeltable-0.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pixeltable
-Version: 0.4.0rc2
+Version: 0.4.1
 Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
 License: Apache-2.0
 Keywords: data-science,machine-learning,database,ai,computer-vision,chatbot,ml,artificial-intelligence,feature-engineering,multimodal,mlops,feature-store,vector-database,llm,genai