pixeltable 0.3.14__py3-none-any.whl → 0.4.0__py3-none-any.whl

pixeltable/index/btree.py

@@ -59,6 +59,11 @@ class BtreeIndex(IndexBase):
         conn = Env.get().conn
         idx.create(bind=conn)
 
+    def drop_index(self, index_name: str, index_value_col: 'catalog.Column') -> None:
+        """Drop the index on the index value column"""
+        # TODO: implement
+        raise NotImplementedError()
+
     @classmethod
     def display_name(cls) -> str:
         return 'btree'

pixeltable/index/embedding_index.py

@@ -148,6 +148,11 @@ class EmbeddingIndex(IndexBase):
         conn = Env.get().conn
         idx.create(bind=conn)
 
+    def drop_index(self, index_name: str, index_value_col: catalog.Column) -> None:
+        """Drop the index on the index value column"""
+        # TODO: implement
+        raise NotImplementedError()
+
     def similarity_clause(self, val_column: catalog.Column, item: Any) -> sql.ColumnElement:
         """Create a ColumnElement that represents '<val_column> <op> <item>'"""
         assert isinstance(item, (str, PIL.Image.Image))

pixeltable/io/external_store.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import abc
 import itertools
 import logging
-import time
 from dataclasses import dataclass
 from typing import Any, Optional
 from uuid import UUID
@@ -11,7 +10,7 @@ from uuid import UUID
 import pixeltable.exceptions as excs
 import pixeltable.type_system as ts
 from pixeltable import Column, Table
-from pixeltable.catalog import TableVersion, TableVersionHandle
+from pixeltable.catalog import TableVersion
 
 _logger = logging.getLogger('pixeltable')
 
@@ -32,15 +31,11 @@ class ExternalStore(abc.ABC):
 
     @abc.abstractmethod
     def link(self, tbl_version: TableVersion) -> None:
-        """
-        Called by `TableVersion.link()` to implement store-specific logic.
-        """
+        """Creates store-specific metadata needed to implement sync()."""
 
     @abc.abstractmethod
     def unlink(self, tbl_version: TableVersion) -> None:
-        """
-        Called by `TableVersion.unlink()` to implement store-specific logic.
-        """
+        """Removes store-specific metadata created in link()."""
 
     @abc.abstractmethod
     def get_local_columns(self) -> list[Column]:
@@ -111,17 +106,10 @@ class Project(ExternalStore, abc.ABC):
 
         if len(stored_proxies_needed) > 0:
             _logger.info(f'Creating stored proxies for columns: {[col.name for col in stored_proxies_needed]}')
-            # Create stored proxies for columns that need one. Increment the schema version
-            # accordingly.
-            tbl_version.version += 1
-            preceding_schema_version = tbl_version.schema_version
-            tbl_version.schema_version = tbl_version.version
-            proxy_cols = [self.create_stored_proxy(tbl_version, col) for col in stored_proxies_needed]
+            # Create stored proxies for columns that need one
+            proxy_cols = [self.create_stored_proxy(col) for col in stored_proxies_needed]
             # Add the columns; this will also update table metadata.
-            tbl_version._add_columns(proxy_cols, print_stats=False, on_error='ignore')
-            # We don't need to retain `UpdateStatus` since the stored proxies are intended to be
-            # invisible to the user.
-            tbl_version._update_md(time.time(), preceding_schema_version=preceding_schema_version)
+            tbl_version.add_columns(proxy_cols, print_stats=False, on_error='ignore')
 
     def unlink(self, tbl_version: TableVersion) -> None:
         # Determine which stored proxies can be deleted. (A stored proxy can be deleted if it is not referenced by
@@ -132,15 +120,10 @@ class Project(ExternalStore, abc.ABC):
                 deletions_needed = deletions_needed.difference(set(store.stored_proxies.values()))
         if len(deletions_needed) > 0:
             _logger.info(f'Removing stored proxies for columns: {[col.name for col in deletions_needed]}')
-            # Delete stored proxies that are no longer needed.
-            tbl_version.version += 1
-            preceding_schema_version = tbl_version.schema_version
-            tbl_version.schema_version = tbl_version.version
             tbl_version._drop_columns(deletions_needed)
             self.stored_proxies.clear()
-            tbl_version._update_md(time.time(), preceding_schema_version=preceding_schema_version)
 
-    def create_stored_proxy(self, tbl_version: TableVersion, col: Column) -> Column:
+    def create_stored_proxy(self, col: Column) -> Column:
         """
         Creates a proxy column for the specified column. The proxy column will be created in the specified
         `TableVersion`.
@@ -158,12 +141,7 @@ class Project(ExternalStore, abc.ABC):
             #   Once `destination` is implemented, it can be replaced with a simple `ColumnRef`.
             computed_with=exprs.ColumnRef(col).apply(lambda x: x, col_type=col.col_type),
             stored=True,
-            col_id=tbl_version.next_col_id,
-            sa_col_type=col.col_type.to_sa_type(),
-            schema_version_add=tbl_version.schema_version,
         )
-        proxy_col.tbl = TableVersionHandle(tbl_version.id, tbl_version.effective_version, tbl_version=tbl_version)
-        tbl_version.next_col_id += 1
         self.stored_proxies[col] = proxy_col
         return proxy_col
 
@@ -213,6 +191,7 @@ class Project(ExternalStore, abc.ABC):
             external (import or export) columns.
         If validation fails, an exception will be raised. If validation succeeds, a new mapping will be returned
         in which the Pixeltable column names are resolved to the corresponding `Column` objects.
+        TODO: return columns as names or qualified ids
         """
         from pixeltable import exprs
 
@@ -223,7 +202,7 @@ class Project(ExternalStore, abc.ABC):
         resolved_col_mapping: dict[Column, str] = {}
 
         # Validate names
-        t_cols = set(table._schema.keys())
+        t_cols = set(table._get_schema().keys())
         for t_col, ext_col in col_mapping.items():
             if t_col not in t_cols:
                 if is_user_specified_col_mapping:
@@ -246,7 +225,7 @@ class Project(ExternalStore, abc.ABC):
             assert isinstance(col_ref, exprs.ColumnRef)
             resolved_col_mapping[col_ref.col] = ext_col
         # Validate column specs
-        t_col_types = table._schema
+        t_col_types = table._get_schema()
         for t_col, ext_col in col_mapping.items():
             t_col_type = t_col_types[t_col]
             if ext_col in export_cols:

pixeltable/io/label_studio.py

@@ -412,8 +412,8 @@ class LabelStudioProject(Project):
             # TODO(aaron-siegel): Simplify this once propagation is properly implemented in batch_update
             ancestor = t
             while local_annotations_col not in ancestor._tbl_version.get().cols:
-                assert ancestor._base_table is not None
-                ancestor = ancestor._base_table
+                assert ancestor._get_base_table is not None
+                ancestor = ancestor._get_base_table()
             update_status = ancestor.batch_update(updates)
             env.Env.get().console_logger.info(f'Updated annotation(s) from {len(updates)} task(s) in {self}.')
             return SyncStatus(pxt_rows_updated=update_status.num_rows, num_excs=update_status.num_excs)
@@ -560,7 +560,7 @@ class LabelStudioProject(Project):
 
         if name is None:
             # Create a default name that's unique to the table
-            all_stores = t.external_stores
+            all_stores = t.external_stores()
             n = 0
             while f'ls_project_{n}' in all_stores:
                 n += 1
@@ -576,8 +576,8 @@ class LabelStudioProject(Project):
                 local_annotations_column = ANNOTATIONS_COLUMN
             else:
                 local_annotations_column = next(k for k, v in col_mapping.items() if v == ANNOTATIONS_COLUMN)
-            if local_annotations_column not in t._schema:
-                t.add_columns({local_annotations_column: ts.JsonType(nullable=True)})
+            if local_annotations_column not in t._get_schema():
+                t.add_columns({local_annotations_column: ts.Json})
 
         resolved_col_mapping = cls.validate_columns(
             t, config.export_columns, {ANNOTATIONS_COLUMN: ts.JsonType(nullable=True)}, col_mapping

pixeltable/io/parquet.py

@@ -14,7 +14,7 @@ import PIL.Image
 
 import pixeltable as pxt
 import pixeltable.exceptions as excs
-from pixeltable.env import Env
+from pixeltable.catalog import Catalog
 from pixeltable.utils.transactional_directory import transactional_directory
 
 if typing.TYPE_CHECKING:
@@ -87,7 +87,7 @@ def export_parquet(
         current_value_batch: dict[str, deque] = {k: deque() for k in df.schema}
         current_byte_estimate = 0
 
-        with Env.get().begin_xact():
+        with Catalog.get().begin_xact(for_write=False):
             for data_row in df._exec():
                 for (col_name, col_type), e in zip(df.schema.items(), df._select_list_exprs):
                     val = data_row[e.slot_idx]
@@ -112,11 +112,11 @@ def export_parquet(
                         length = len(val)
                     elif col_type.is_string_type():
                         length = len(val)
-                    elif col_type.is_video_type():
+                    elif col_type.is_video_type() or col_type.is_audio_type():
                         if data_row.file_paths is not None and data_row.file_paths[e.slot_idx] is not None:
                             val = data_row.file_paths[e.slot_idx]
                         else:
-                            raise excs.Error(f'unknown video type {type(val)}')
+                            raise excs.Error(f'unknown audio/video type {type(val)}')
                         length = len(val)
                     elif col_type.is_json_type():
                         val = json.dumps(val)

pixeltable/io/table_data_conduit.py

@@ -23,7 +23,6 @@ from .utils import normalize_schema_names
 
 _logger = logging.getLogger('pixeltable')
 
-# ---------------------------------------------------------------------------------------------------------
 
 if TYPE_CHECKING:
     import datasets  # type: ignore[import-untyped]
@@ -46,9 +45,6 @@ class TableDataConduitFormat(str, enum.Enum):
         return False
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 @dataclass
 class TableDataConduit:
     source: TableDataSource
@@ -105,7 +101,7 @@ class TableDataConduit:
     def add_table_info(self, table: pxt.Table) -> None:
         """Add information about the table into which we are inserting data"""
         assert isinstance(table, pxt.Table)
-        self.pxt_schema = table._schema
+        self.pxt_schema = table._get_schema()
         self.pxt_pk = table._tbl_version.get().primary_key
         for col in table._tbl_version_path.columns():
             if col.is_required_for_insert:
@@ -129,9 +125,6 @@ class TableDataConduit:
             raise excs.Error(f'Missing required column(s) ({", ".join(missing_cols)})')
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class DFTableDataConduit(TableDataConduit):
     pxt_df: pxt.DataFrame = None
 
@@ -155,9 +148,6 @@ class DFTableDataConduit(TableDataConduit):
         self.check_source_columns_are_insertable(self.pxt_df.schema.keys())
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class RowDataTableDataConduit(TableDataConduit):
     raw_rows: Optional[RowData] = None
     disable_mapping: bool = True
@@ -235,9 +225,6 @@ class RowDataTableDataConduit(TableDataConduit):
             yield self.valid_rows
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class PandasTableDataConduit(TableDataConduit):
     pd_df: pd.DataFrame = None
     batch_count: int = 0
@@ -293,9 +280,6 @@ class PandasTableDataConduit(TableDataConduit):
             yield self.valid_rows
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class CSVTableDataConduit(TableDataConduit):
     @classmethod
     def from_tds(cls, tds: TableDataConduit) -> 'PandasTableDataConduit':
@@ -307,9 +291,6 @@ class CSVTableDataConduit(TableDataConduit):
         return PandasTableDataConduit.from_tds(t)
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class ExcelTableDataConduit(TableDataConduit):
     @classmethod
     def from_tds(cls, tds: TableDataConduit) -> 'PandasTableDataConduit':
@@ -321,9 +302,6 @@ class ExcelTableDataConduit(TableDataConduit):
         return PandasTableDataConduit.from_tds(t)
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class JsonTableDataConduit(TableDataConduit):
     @classmethod
     def from_tds(cls, tds: TableDataConduit) -> RowDataTableDataConduit:
@@ -346,9 +324,6 @@ class JsonTableDataConduit(TableDataConduit):
         return t2
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class HFTableDataConduit(TableDataConduit):
     hf_ds: Optional[Union[datasets.Dataset, datasets.DatasetDict]] = None
     column_name_for_split: Optional[str] = None
@@ -478,9 +453,6 @@ class HFTableDataConduit(TableDataConduit):
                 yield batch
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class ParquetTableDataConduit(TableDataConduit):
     pq_ds: Optional[ParquetDataset] = None
 
@@ -542,9 +514,6 @@ class ParquetTableDataConduit(TableDataConduit):
             raise e
 
 
-# ---------------------------------------------------------------------------------------------------------
-
-
 class UnkTableDataConduit(TableDataConduit):
     """Source type is not known at the time of creation"""
 

pixeltable/metadata/__init__.py

@@ -8,15 +8,17 @@ from typing import Callable
 import sqlalchemy as sql
 from sqlalchemy import orm
 
+import pixeltable as pxt
+import pixeltable.exceptions as excs
 from pixeltable.utils.console_output import ConsoleLogger
 
 from .schema import SystemInfo, SystemInfoMd
 
 _console_logger = ConsoleLogger(logging.getLogger('pixeltable'))
-
+_logger = logging.getLogger('pixeltable')
 
 # current version of the metadata; this is incremented whenever the metadata schema changes
-VERSION = 35
+VERSION = 38
 
 
 def create_system_info(engine: sql.engine.Engine) -> None:
@@ -55,6 +57,13 @@ def upgrade_md(engine: sql.engine.Engine) -> None:
         system_info = session.query(SystemInfo).one().md
         md_version = system_info['schema_version']
         assert isinstance(md_version, int)
+        _logger.info(f'Current database version: {md_version}, installed version: {VERSION}')
+        if md_version > VERSION:
+            raise excs.Error(
+                'This Pixeltable database was created with a newer Pixeltable version '
+                f'than the one currently installed ({pxt.__version__}).\n'
+                'Please update to the latest Pixeltable version by running: pip install --upgrade pixeltable'
+            )
         if md_version == VERSION:
             return
         while md_version < VERSION:

pixeltable/metadata/converters/convert_13.py

@@ -12,9 +12,9 @@ _logger = logging.getLogger('pixeltable')
 @register_converter(version=13)
 def _(engine: sql.engine.Engine) -> None:
     with engine.begin() as conn:
-        for row in conn.execute(sql.select(Table)):
+        for row in conn.execute(sql.select(Table.id, Table.md)):
             id = row[0]
-            md = row[2]
+            md = row[1]
             updated_md = __update_md(md)
             if updated_md != md:
                 _logger.info(f'Updating schema for table: {id}')

pixeltable/metadata/converters/convert_30.py

@@ -1,33 +1,28 @@
 import copy
+from uuid import UUID
 
 import sqlalchemy as sql
 
 from pixeltable.metadata import register_converter
 from pixeltable.metadata.converters.util import (
-    convert_table_record,
+    convert_table_md,
     convert_table_schema_version_record,
     convert_table_version_record,
 )
-from pixeltable.metadata.schema import Table, TableSchemaVersion, TableVersion
+from pixeltable.metadata.schema import TableSchemaVersion, TableVersion
 
 
 @register_converter(version=30)
 def _(engine: sql.engine.Engine) -> None:
-    convert_table_record(engine, table_record_updater=__update_table_record)
+    convert_table_md(engine, table_md_updater=__update_table_md)
     convert_table_version_record(engine, table_version_record_updater=__update_table_version_record)
     convert_table_schema_version_record(
         engine, table_schema_version_record_updater=__update_table_schema_version_record
     )
 
 
-def __update_table_record(record: Table) -> None:
-    """
-    Update TableMd with table_id
-    """
-    assert isinstance(record.md, dict)
-    md = copy.copy(record.md)
-    md['tbl_id'] = str(record.id)
-    record.md = md
+def __update_table_md(md: dict, tbl_id: UUID) -> None:
+    md['tbl_id'] = str(tbl_id)
 
 
 def __update_table_version_record(record: TableVersion) -> None:

pixeltable/metadata/converters/convert_35.py

@@ -0,0 +1,9 @@
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+
+
+@register_converter(version=35)
+def _(engine: sql.engine.Engine) -> None:
+    with engine.begin() as conn:
+        conn.execute(sql.text('ALTER TABLE tables ADD COLUMN lock_dummy int8'))

pixeltable/metadata/converters/convert_36.py

@@ -0,0 +1,38 @@
+import logging
+from typing import Any, Optional
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+_logger = logging.getLogger('pixeltable')
+
+
+@register_converter(version=36)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_md_updater=__update_table_md, substitution_fn=__substitute_md)
+
+
+def __update_table_md(table_md: dict, table_id: UUID) -> None:
+    """Update the view metadata to add the sample_clause field if it is missing
+
+    Args:
+        table_md (dict): copy of the original table metadata. this gets updated in place.
+        table_id (UUID): the table id
+
+    """
+    if table_md['view_md'] is None:
+        return
+    if 'sample_clause' not in table_md['view_md']:
+        table_md['view_md']['sample_clause'] = None
+        _logger.info(f'Updating view metadata for table: {table_id}')
+
+
+def __substitute_md(k: Optional[str], v: Any) -> Optional[tuple[Optional[str], Any]]:
+    if isinstance(v, dict) and (v.get('_classname') == 'DataFrame'):
+        if 'sample_clause' not in v:
+            v['sample_clause'] = None
+        return k, v
+    return None

pixeltable/metadata/converters/convert_37.py

@@ -0,0 +1,15 @@
+from uuid import UUID
+
+import sqlalchemy as sql
+
+from pixeltable.metadata import register_converter
+from pixeltable.metadata.converters.util import convert_table_md
+
+
+@register_converter(version=37)
+def _(engine: sql.engine.Engine) -> None:
+    convert_table_md(engine, table_md_updater=__update_table_md)
+
+
+def __update_table_md(table_md: dict, _: UUID) -> None:
+    table_md['view_sn'] = 0

pixeltable/metadata/converters/util.py

@@ -33,9 +33,10 @@ def convert_table_md(
             the original entry will be replaced, and the traversal will continue with `v'`.
     """
     with engine.begin() as conn:
-        for row in conn.execute(sql.select(Table)):
+        # avoid a SELECT * here, which breaks when we add new columns to Table
+        for row in conn.execute(sql.select(Table.id, Table.md)):
             tbl_id = row[0]
-            table_md = row[2]
+            table_md = row[1]
             assert isinstance(table_md, dict)
             updated_table_md = copy.deepcopy(table_md)
             if table_md_updater is not None:
@@ -145,13 +146,6 @@ def __update_schema_column(table_schema_version_md: dict, schema_column_updater:
         schema_column_updater(schema_col)
 
 
-def convert_table_record(engine: sql.engine.Engine, table_record_updater: Optional[Callable[[Table], None]]) -> None:
-    with sql.orm.Session(engine, future=True) as session:
-        for record in session.query(Table).all():
-            table_record_updater(record)
-        session.commit()
-
-
 def convert_table_version_record(
     engine: sql.engine.Engine, table_version_record_updater: Optional[Callable[[TableVersion], None]]
 ) -> None:

pixeltable/metadata/notes.py

@@ -2,6 +2,9 @@
 # rather than as a comment, so that the existence of a description can be enforced by
 # the unit tests when new versions are added.
 VERSION_NOTES = {
+    38: 'Added TableMd.view_sn',
+    37: 'Add support for the sample() method on DataFrames',
+    36: 'Added Table.lock_dummy',
     35: 'Track reference_tbl in ColumnRef',
     34: 'Set default value for is_pk field in column metadata to False',
     33: 'Add is_replica field to table metadata',

pixeltable/metadata/schema.py

@@ -84,7 +84,8 @@ class Dir(Base):
     )
     parent_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), ForeignKey('dirs.id'), nullable=True)
     md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)  # DirMd
-    # This field is updated to synchronize database operations across multiple sessions
+
+    # used to force acquisition of an X-lock via an Update stmt
     lock_dummy: orm.Mapped[int] = orm.mapped_column(BigInteger, nullable=True)
 
 
@@ -146,6 +147,9 @@ class ViewMd:
     # filter predicate applied to the base table; view-only
     predicate: Optional[dict[str, Any]]
 
+    # sampling predicate applied to the base table; view-only
+    sample_clause: Optional[dict[str, Any]]
+
     # ComponentIterator subclass; only for component views
     iterator_class_fqn: Optional[str]
 
@@ -173,6 +177,11 @@ class TableMd:
     # - every row is assigned a unique and immutable rowid on insertion
     next_row_id: int
 
+    # sequence number to track changes in the set of mutable views of this table (ie, this table = the view base)
+    # - incremented for each add/drop of a mutable view
+    # - only maintained for mutable tables
+    view_sn: int
+
     # Metadata format for external stores:
     # {'class': 'pixeltable.io.label_studio.LabelStudioProject', 'md': {'project_id': 3}}
     external_stores: list[dict[str, Any]]
@@ -200,6 +209,9 @@ class Table(Base):
     dir_id: orm.Mapped[uuid.UUID] = orm.mapped_column(UUID(as_uuid=True), ForeignKey('dirs.id'), nullable=False)
     md: orm.Mapped[dict[str, Any]] = orm.mapped_column(JSONB, nullable=False)  # TableMd
 
+    # used to force acquisition of an X-lock via an Update stmt
+    lock_dummy: orm.Mapped[int] = orm.mapped_column(BigInteger, nullable=True)
+
 
 @dataclasses.dataclass
 class TableVersionMd: