pixeltable 0.4.2__tar.gz → 0.4.3__tar.gz

{pixeltable-0.4.2 → pixeltable-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pixeltable
-Version: 0.4.2
+Version: 0.4.3
 Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
 License: Apache-2.0
 Keywords: data-science,machine-learning,database,ai,computer-vision,chatbot,ml,artificial-intelligence,feature-engineering,multimodal,mlops,feature-store,vector-database,llm,genai

{pixeltable-0.4.2 → pixeltable-0.4.3}/pixeltable/__init__.py

@@ -20,6 +20,7 @@ from .globals import (
     list_dirs,
     list_functions,
     list_tables,
+    ls,
     move,
     tool,
     tools,

{pixeltable-0.4.2 → pixeltable-0.4.3}/pixeltable/__version__.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced during build.
-__version__ = '0.4.2'
-__version_tuple__ = (0, 4, 2)
+__version__ = '0.4.3'
+__version_tuple__ = (0, 4, 3)

{pixeltable-0.4.2 → pixeltable-0.4.3}/pixeltable/catalog/__init__.py

@@ -3,16 +3,7 @@
 from .catalog import Catalog
 from .column import Column
 from .dir import Dir
-from .globals import (
-    IfExistsParam,
-    IfNotExistsParam,
-    MediaValidation,
-    QColumnId,
-    RowCountStats,
-    UpdateStatus,
-    is_valid_identifier,
-    is_valid_path,
-)
+from .globals import IfExistsParam, IfNotExistsParam, MediaValidation, QColumnId, is_valid_identifier, is_valid_path
 from .insertable_table import InsertableTable
 from .named_function import NamedFunction
 from .path import Path
@@ -21,4 +12,5 @@ from .table import Table
 from .table_version import TableVersion
 from .table_version_handle import ColumnHandle, TableVersionHandle
 from .table_version_path import TableVersionPath
+from .update_status import RowCountStats, UpdateStatus
 from .view import View

{pixeltable-0.4.2 → pixeltable-0.4.3}/pixeltable/catalog/catalog.py

@@ -766,56 +766,47 @@ class Catalog:
         self._tbls[view._id] = view
         return view
 
-    @_retry_loop(for_write=True)
-    def create_replica(
-        self, path: Path, md: list[schema.FullTableMd], if_exists: IfExistsParam = IfExistsParam.ERROR
-    ) -> None:
+    def create_replica(self, path: Path, md: list[schema.FullTableMd]) -> None:
         """
         Creates table, table_version, and table_schema_version records for a replica with the given metadata.
         The metadata should be presented in standard "ancestor order", with the table being replicated at
         list position 0 and the (root) base table at list position -1.
-
-        TODO: create_replica() also needs to create the store tables and populate them in order to make
-        replica creation atomic.
         """
+        assert Env.get().in_xact
+
         tbl_id = UUID(md[0].tbl_md.tbl_id)
 
-        # First handle path collisions (if_exists='ignore' or 'replace' or etc).
-        existing = self._handle_path_collision(path, View, False, if_exists)
-        if existing is not None:
-            if existing._id != tbl_id:
-                raise excs.Error(
-                    f"An attempt was made to create a replica table at {path!r} with if_exists='ignore', "
-                    'but a different table already exists at that location.'
-                )
-            assert isinstance(existing, View)
-            return
+        existing = self._handle_path_collision(path, Table, False, if_exists=IfExistsParam.IGNORE)  # type: ignore[type-abstract]
+        if existing is not None and existing._id != tbl_id:
+            raise excs.Error(
+                f'An attempt was made to create a replica table at {path!r}, '
+                'but a different table already exists at that location.'
+            )
 
         # Ensure that the system directory exists.
         self._create_dir(Path('_system', allow_system_paths=True), if_exists=IfExistsParam.IGNORE, parents=False)
 
-        # Now check to see if this table already exists in the catalog.
+        # Now check to see if this table UUID already exists in the catalog.
         existing = Catalog.get().get_table_by_id(tbl_id)
         if existing is not None:
             existing_path = Path(existing._path(), allow_system_paths=True)
-            # It does exist. If it's a non-system table, that's an error: it's already been replicated.
-            if not existing_path.is_system_path:
-                raise excs.Error(
-                    f'That table has already been replicated as {existing._path()!r}. \n'
-                    f'Drop the existing replica if you wish to re-create it.'
-                )
-            # If it's a system table, then this means it was created at some point as the ancestor of some other
-            # table (a snapshot-over-snapshot scenario). In that case, we simply move it to the new (named) location.
-            self._move(existing_path, path)
-
-        # Now store the metadata for this replica. In the case where the table already exists (and was just moved
-        # into a named location), this will be a no-op, but it still serves to validate that the newly received
-        # metadata is identical to what's in the catalog.
-        self.__store_replica_md(path, md[0])
+            if existing_path != path:
+                # It does exist, under a different path from the specified one.
+                if not existing_path.is_system_path:
+                    raise excs.Error(
+                        f'That table has already been replicated as {existing_path!r}.\n'
+                        f'Drop the existing replica if you wish to re-create it.'
+                    )
+                # If it's a system table, then this means it was created at some point as the ancestor of some other
+                # table (a snapshot-over-snapshot scenario). In that case, we simply move it to the new (named)
+                # location.
+                self._move(existing_path, path)
 
-        # Now store the metadata for all of this table's proper ancestors. If one or more proper ancestors
+        # Now store the metadata for this replica's proper ancestors. If one or more proper ancestors
         # do not yet exist in the store, they will be created as anonymous system tables.
-        for ancestor_md in md[1:]:
+        # We instantiate the ancestors starting with the base table and ending with the immediate parent of the
+        # table being replicated.
+        for ancestor_md in md[:0:-1]:
             ancestor_id = UUID(ancestor_md.tbl_md.tbl_id)
             replica = Catalog.get().get_table_by_id(ancestor_id)
             replica_path: Path
@@ -828,12 +819,22 @@ class Catalog:
                 # that was directly replicated by the user at some point). In either case, use the existing path.
                 replica_path = Path(replica._path(), allow_system_paths=True)
 
-            # Store the metadata; it could be a new version (in which case a new record will be created) or a
-            # known version (in which case the newly received metadata will be validated as identical).
+            # Store the metadata; it could be a new version (in which case a new record will be created), or a known
+            # version (in which case the newly received metadata will be validated as identical).
+            # If it's a new version, this will result in a new TableVersion record being created.
             self.__store_replica_md(replica_path, ancestor_md)
 
-        # don't create TableVersion instances at this point, they would be superseded by calls to TV.create_replica()
-        # in TableRestorer.restore()
+            # Now we must clear cached metadata for the ancestor table, to force the next table operation to pick up
+            # the new TableVersion instance. This is necessary because computed columns of descendant tables might
+            # reference columns of the ancestor table that only exist in the new version.
+            replica = Catalog.get().get_table_by_id(ancestor_id)
+            assert replica is not None  # If it didn't exist before, it must have been created by now.
+            replica._tbl_version_path.clear_cached_md()
+
+        # Finally, store the metadata for the table being replicated; as before, it could be a new version or a known
+        # version. If it's a new version, then a TableVersion record will be created, unless the table being replicated
+        # is a pure snapshot.
+        self.__store_replica_md(path, md[0])
 
     def __store_replica_md(self, path: Path, md: schema.FullTableMd) -> None:
         _logger.info(f'Creating replica table at {path!r} with ID: {md.tbl_md.tbl_id}')
@@ -917,10 +918,17 @@ class Catalog:
 
         self.store_tbl_md(UUID(tbl_id), None, new_tbl_md, new_version_md, new_schema_version_md)
 
+        if new_version_md is not None and not md.is_pure_snapshot:
+            # It's a new version of a table that has a physical store, so we need to create a TableVersion instance.
+            TableVersion.create_replica(md)
+
     @_retry_loop(for_write=False)
     def get_table(self, path: Path) -> Table:
         obj = Catalog.get()._get_schema_object(path, expected=Table, raise_if_not_exists=True)
         assert isinstance(obj, Table)
+        # We need to clear cached metadata from tbl_version_path, in case the schema has been changed
+        # by another process.
+        obj._tbl_version_path.clear_cached_md()
         return obj
 
     @_retry_loop(for_write=True)
@@ -1398,6 +1406,24 @@ class Catalog:
             session.add(schema_version_record)
         session.flush()  # Inform SQLAlchemy that we want to write these changes to the DB.
 
+    def update_tbl_version_md(self, version_md: Optional[schema.TableVersionMd]) -> None:
+        """
+        Update the TableVersion.md field in the DB. Typically used to update the cascade row count status.
+
+        Args:
+            version_md: TableVersionMd
+        """
+        assert self._in_write_xact
+        session = Env.get().session
+
+        session.execute(
+            sql.update(schema.TableVersion.__table__)
+            .values({schema.TableVersion.md: dataclasses.asdict(version_md)})
+            .where(schema.TableVersion.tbl_id == version_md.tbl_id, schema.TableVersion.version == version_md.version)
+        )
+
+        session.flush()  # Inform SQLAlchemy that we want to write these changes to the DB.
+
     def delete_tbl_md(self, tbl_id: UUID) -> None:
         """
         Deletes all table metadata from the store for the given table UUID.

{pixeltable-0.4.2 → pixeltable-0.4.3}/pixeltable/catalog/column.py

@@ -36,11 +36,10 @@ class Column:
     _media_validation: Optional[MediaValidation]  # if not set, TableVersion.media_validation applies
     schema_version_add: Optional[int]
     schema_version_drop: Optional[int]
-    _records_errors: Optional[bool]
+    _stores_cellmd: Optional[bool]
     sa_col: Optional[sql.schema.Column]
     sa_col_type: Optional[sql.sqltypes.TypeEngine]
-    sa_errormsg_col: Optional[sql.schema.Column]
-    sa_errortype_col: Optional[sql.schema.Column]
+    sa_cellmd_col: Optional[sql.schema.Column]  # JSON metadata for the cell, e.g. errortype, errormsg for media columns
     _value_expr: Optional[exprs.Expr]
     value_expr_dict: Optional[dict[str, Any]]
     # we store a TableVersion here, not a TableVersionHandle, because this column is owned by that TableVersion instance
@@ -59,7 +58,7 @@ class Column:
         schema_version_add: Optional[int] = None,
         schema_version_drop: Optional[int] = None,
         sa_col_type: Optional[sql.sqltypes.TypeEngine] = None,
-        records_errors: Optional[bool] = None,
+        stores_cellmd: Optional[bool] = None,
         value_expr_dict: Optional[dict[str, Any]] = None,
         tbl: Optional[TableVersion] = None,
     ):
@@ -118,15 +117,14 @@ class Column:
         self.schema_version_add = schema_version_add
         self.schema_version_drop = schema_version_drop
 
-        self._records_errors = records_errors
+        self._stores_cellmd = stores_cellmd
 
         # column in the stored table for the values of this Column
         self.sa_col = None
         self.sa_col_type = sa_col_type
 
         # computed cols also have storage columns for the exception string and type
-        self.sa_errormsg_col = None
-        self.sa_errortype_col = None
+        self.sa_cellmd_col = None
 
     def init_value_expr(self) -> None:
         from pixeltable import exprs
@@ -203,11 +201,11 @@ class Column:
         return self.stored
 
     @property
-    def records_errors(self) -> bool:
+    def stores_cellmd(self) -> bool:
         """True if this column also stores error information."""
         # default: record errors for computed and media columns
-        if self._records_errors is not None:
-            return self._records_errors
+        if self._stores_cellmd is not None:
+            return self._stores_cellmd
         return self.is_stored and (self.is_computed or self.col_type.is_media_type())
 
     @property
@@ -243,28 +241,29 @@ class Column:
         """
         assert self.is_stored
         # all storage columns are nullable (we deal with null errors in Pixeltable directly)
-        self.sa_col = sql.Column(
-            self.store_name(),
-            self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type,
-            nullable=True,
-        )
-        if self.is_computed or self.col_type.is_media_type():
-            self.sa_errormsg_col = sql.Column(self.errormsg_store_name(), ts.StringType().to_sa_type(), nullable=True)
-            self.sa_errortype_col = sql.Column(self.errortype_store_name(), ts.StringType().to_sa_type(), nullable=True)
+        self.sa_col = sql.Column(self.store_name(), self.get_sa_col_type(), nullable=True)
+        if self.stores_cellmd:
+            # JSON metadata for the cell, e.g. errortype, errormsg for media columns
+            self.sa_cellmd_col = sql.Column(self.cellmd_store_name(), self.sa_cellmd_type(), nullable=True)
 
     def get_sa_col_type(self) -> sql.sqltypes.TypeEngine:
         return self.col_type.to_sa_type() if self.sa_col_type is None else self.sa_col_type
 
+    @classmethod
+    def cellmd_type(cls) -> ts.ColumnType:
+        return ts.JsonType(nullable=True)
+
+    @classmethod
+    def sa_cellmd_type(cls) -> sql.sqltypes.TypeEngine:
+        return cls.cellmd_type().to_sa_type()
+
     def store_name(self) -> str:
         assert self.id is not None
         assert self.is_stored
         return f'col_{self.id}'
 
-    def errormsg_store_name(self) -> str:
-        return f'{self.store_name()}_errormsg'
-
-    def errortype_store_name(self) -> str:
-        return f'{self.store_name()}_errortype'
+    def cellmd_store_name(self) -> str:
+        return f'{self.store_name()}_cellmd'
 
     def __str__(self) -> str:
         return f'{self.name}: {self.col_type}'

pixeltable-0.4.3/pixeltable/catalog/globals.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import enum
+import itertools
+import logging
+from dataclasses import dataclass
+from typing import Optional
+from uuid import UUID
+
+import pixeltable.exceptions as excs
+
+_logger = logging.getLogger('pixeltable')
+
+# name of the position column in a component view
+_POS_COLUMN_NAME = 'pos'
+_ROWID_COLUMN_NAME = '_rowid'
+
+# Set of symbols that are predefined in the `InsertableTable` class (and are therefore not allowed as column names).
+# This will be populated lazily to avoid circular imports.
+_PREDEF_SYMBOLS: Optional[set[str]] = None
+
+
+@dataclass(frozen=True)
+class QColumnId:
+    """Qualified column id"""
+
+    tbl_id: UUID
+    col_id: int
+
+
+class MediaValidation(enum.Enum):
+    ON_READ = 0
+    ON_WRITE = 1
+
+    @classmethod
+    def validated(cls, name: str, error_prefix: str) -> MediaValidation:
+        try:
+            return cls[name.upper()]
+        except KeyError:
+            val_strs = ', '.join(f'{s.lower()!r}' for s in cls.__members__)
+            raise excs.Error(f'{error_prefix} must be one of: [{val_strs}]') from None
+
+
+class IfExistsParam(enum.Enum):
+    ERROR = 0
+    IGNORE = 1
+    REPLACE = 2
+    REPLACE_FORCE = 3
+
+    @classmethod
+    def validated(cls, param_val: str, param_name: str) -> IfExistsParam:
+        try:
+            return cls[param_val.upper()]
+        except KeyError:
+            val_strs = ', '.join(f'{s.lower()!r}' for s in cls.__members__)
+            raise excs.Error(f'{param_name} must be one of: [{val_strs}]') from None
+
+
+class IfNotExistsParam(enum.Enum):
+    ERROR = 0
+    IGNORE = 1
+
+    @classmethod
+    def validated(cls, param_val: str, param_name: str) -> IfNotExistsParam:
+        try:
+            return cls[param_val.upper()]
+        except KeyError:
+            val_strs = ', '.join(f'{s.lower()!r}' for s in cls.__members__)
+            raise excs.Error(f'{param_name} must be one of: [{val_strs}]') from None
+
+
+def is_valid_identifier(name: str, allow_system_identifiers: bool = False) -> bool:
+    return name.isidentifier() and (allow_system_identifiers or not name.startswith('_'))
+
+
+def is_valid_path(path: str, empty_is_valid: bool, allow_system_paths: bool = False) -> bool:
+    if path == '':
+        return empty_is_valid
+    return all(is_valid_identifier(part, allow_system_paths) for part in path.split('.'))
+
+
+def is_system_column_name(name: str) -> bool:
+    from pixeltable.catalog import InsertableTable, View
+
+    global _PREDEF_SYMBOLS  # noqa: PLW0603
+    if _PREDEF_SYMBOLS is None:
+        _PREDEF_SYMBOLS = set(itertools.chain(dir(InsertableTable), dir(View)))
+    return name == _POS_COLUMN_NAME or name in _PREDEF_SYMBOLS

{pixeltable-0.4.2 → pixeltable-0.4.3}/pixeltable/catalog/insertable_table.py

@@ -10,11 +10,12 @@ from pixeltable import exceptions as excs, type_system as ts
 from pixeltable.env import Env
 from pixeltable.utils.filecache import FileCache
 
-from .globals import MediaValidation, UpdateStatus
+from .globals import MediaValidation
 from .table import Table
 from .table_version import TableVersion
 from .table_version_handle import TableVersionHandle
 from .table_version_path import TableVersionPath
+from .update_status import UpdateStatus
 
 if TYPE_CHECKING:
     from pixeltable import exprs
@@ -106,6 +107,7 @@ class InsertableTable(Table):
 
     def _get_metadata(self) -> dict[str, Any]:
         md = super()._get_metadata()
+        md['base'] = None
         md['is_view'] = False
         md['is_snapshot'] = False
         return md
@@ -171,14 +173,14 @@ class InsertableTable(Table):
         from pixeltable.catalog import Catalog
         from pixeltable.io.table_data_conduit import DFTableDataConduit
 
-        status = pxt.UpdateStatus()
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
             if isinstance(data_source, DFTableDataConduit):
-                status = self._tbl_version.get().insert(
+                status = pxt.UpdateStatus()
+                status += self._tbl_version.get().insert(
                     rows=None, df=data_source.pxt_df, print_stats=print_stats, fail_on_exception=fail_on_exception
                 )
             else:
-                status = UpdateStatus()
+                status = pxt.UpdateStatus()
                 for row_batch in data_source.valid_row_batch():
                     status += self._tbl_version.get().insert(
                         rows=row_batch, df=None, print_stats=print_stats, fail_on_exception=fail_on_exception

{pixeltable-0.4.2 → pixeltable-0.4.3}/pixeltable/catalog/path.py

@@ -77,5 +77,11 @@ class Path:
     def __str__(self) -> str:
         return '.'.join(self.components)
 
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, Path) and str(self) == str(other)
+
+    def __hash__(self) -> int:
+        return hash(str(self))
+
     def __lt__(self, other: Path) -> bool:
         return str(self) < str(other)