pixeltable 0.4.6__py3-none-any.whl → 0.4.8__py3-none-any.whl

pixeltable/catalog/table_version.py

@@ -14,7 +14,6 @@ import sqlalchemy as sql
 
 import pixeltable as pxt
 import pixeltable.exceptions as excs
-import pixeltable.type_system as ts
 from pixeltable import exprs, index
 from pixeltable.env import Env
 from pixeltable.iterators import ComponentIterator
@@ -223,18 +222,23 @@ class TableVersion:
         view_md: Optional[schema.ViewMd] = None,
     ) -> TableVersionMd:
         user = Env.get().user
+        timestamp = time.time()
 
-        # assign ids
+        # assign ids, create metadata
         cols_by_name: dict[str, Column] = {}
+        column_md: dict[int, schema.ColumnMd] = {}
+        schema_col_md: dict[int, schema.SchemaColumn] = {}
         for pos, col in enumerate(cols):
             col.id = pos
             col.schema_version_add = 0
             cols_by_name[col.name] = col
             if col.is_computed:
                 col.check_value_expr()
+            col_md, sch_md = col.to_md(pos)
+            assert sch_md is not None
+            column_md[col.id] = col_md
+            schema_col_md[col.id] = sch_md
 
-        timestamp = time.time()
-        column_md = cls._create_column_md(cols)
         tbl_id = uuid.uuid4()
         tbl_id_str = str(tbl_id)
         tbl_md = schema.TableMd(
@@ -256,18 +260,15 @@ class TableVersion:
         )
 
         table_version_md = schema.TableVersionMd(
-            tbl_id=tbl_id_str, created_at=timestamp, version=0, schema_version=0, additional_md={}
+            tbl_id=tbl_id_str,
+            created_at=timestamp,
+            version=0,
+            schema_version=0,
+            user=user,
+            update_status=None,
+            additional_md={},
         )
 
-        schema_col_md: dict[int, schema.SchemaColumn] = {}
-        for pos, col in enumerate(cols):
-            md = schema.SchemaColumn(
-                pos=pos,
-                name=col.name,
-                media_validation=col._media_validation.name.lower() if col._media_validation is not None else None,
-            )
-            schema_col_md[col.id] = md
-
         schema_version_md = schema.TableSchemaVersionMd(
             tbl_id=tbl_id_str,
             schema_version=0,
@@ -290,76 +291,11 @@ class TableVersion:
         comment: str,
         media_validation: MediaValidation,
     ) -> tuple[UUID, Optional[TableVersion]]:
-        user = Env.get().user
-
-        # assign ids
-        cols_by_name: dict[str, Column] = {}
-        for pos, col in enumerate(cols):
-            col.id = pos
-            col.schema_version_add = 0
-            cols_by_name[col.name] = col
-            if col.is_computed:
-                col.check_value_expr()
-
-        timestamp = time.time()
-        # create schema.Table
-        # Column.dependent_cols for existing cols is wrong at this point, but init() will set it correctly
-        column_md = cls._create_column_md(cols)
-        tbl_id = uuid.uuid4()
-        tbl_id_str = str(tbl_id)
-        table_md = schema.TableMd(
-            tbl_id=tbl_id_str,
-            name=name,
-            user=user,
-            is_replica=False,
-            current_version=0,
-            current_schema_version=0,
-            next_col_id=len(cols),
-            next_idx_id=0,
-            next_row_id=0,
-            view_sn=0,
-            column_md=column_md,
-            index_md={},
-            external_stores=[],
-            view_md=None,
-            additional_md={},
-        )
-
-        # create schema.TableVersion of the initial version
-        table_version_md = schema.TableVersionMd(
-            tbl_id=tbl_id_str,
-            created_at=timestamp,
-            version=0,
-            schema_version=0,
-            user=user,
-            update_status=None,
-            additional_md={},
-        )
-
-        # create schema.TableSchemaVersion
-        schema_col_md: dict[int, schema.SchemaColumn] = {}
-        for pos, col in enumerate(cols):
-            md = schema.SchemaColumn(
-                pos=pos,
-                name=col.name,
-                media_validation=col._media_validation.name.lower() if col._media_validation is not None else None,
-            )
-            schema_col_md[col.id] = md
-
-        schema_version_md = schema.TableSchemaVersionMd(
-            tbl_id=tbl_id_str,
-            schema_version=0,
-            preceding_schema_version=None,
-            columns=schema_col_md,
-            num_retained_versions=num_retained_versions,
-            comment=comment,
-            media_validation=media_validation.name.lower(),
-            additional_md={},
-        )
-
+        inital_md = cls.create_initial_md(name, cols, num_retained_versions, comment, media_validation, view_md=None)
         cat = pxt.catalog.Catalog.get()
 
-        tbl_version = cls(tbl_id, table_md, table_version_md, None, schema_version_md, [])
+        tbl_id = UUID(hex=inital_md.tbl_md.tbl_id)
+        tbl_version = cls(tbl_id, inital_md.tbl_md, inital_md.version_md, None, inital_md.schema_version_md, [])
         # TODO: break this up, so that Catalog.create_table() registers tbl_version
         cat._tbl_versions[tbl_id, None] = tbl_version
         tbl_version.init()
@@ -373,8 +309,8 @@ class TableVersion:
             tbl_id=tbl_id,
             dir_id=dir_id,
             tbl_md=tbl_version.tbl_md,
-            version_md=table_version_md,
-            schema_version_md=schema_version_md,
+            version_md=inital_md.version_md,
+            schema_version_md=inital_md.schema_version_md,
         )
         return tbl_id, tbl_version
 
@@ -391,7 +327,7 @@ class TableVersion:
             from .table_version_path import TableVersionPath
 
             # clear out any remaining media files from an aborted previous attempt
-            MediaStore.delete(self.id)
+            MediaStore.get().delete(self.id)
             view_path = TableVersionPath.from_dict(op.load_view_op.view_path)
             plan, _ = Planner.create_view_load_plan(view_path)
             _, row_counts = self.store_tbl.insert_rows(plan, v_min=self.version)
@@ -438,7 +374,7 @@ class TableVersion:
         #     if self.base.get().is_mutable:
         #         self.base.get().mutable_views.remove(TableVersionHandle.create(self))
 
-        MediaStore.delete(self.id)
+        MediaStore.get().delete(self.id)
         FileCache.get().clear(tbl_id=self.id)
         self.store_tbl.drop()
 
@@ -480,25 +416,7 @@ class TableVersion:
         sorted_column_md = sorted(self.tbl_md.column_md.values(), key=lambda item: item.id)
         for col_md in sorted_column_md:
             schema_col_md = self.schema_version_md.columns.get(col_md.id)
-            col_name = schema_col_md.name if schema_col_md is not None else None
-            media_val = (
-                MediaValidation[schema_col_md.media_validation.upper()]
-                if schema_col_md is not None and schema_col_md.media_validation is not None
-                else None
-            )
-            col = Column(
-                col_id=col_md.id,
-                name=col_name,
-                col_type=ts.ColumnType.from_dict(col_md.col_type),
-                is_pk=col_md.is_pk,
-                stored=col_md.stored,
-                media_validation=media_val,
-                schema_version_add=col_md.schema_version_add,
-                schema_version_drop=col_md.schema_version_drop,
-                value_expr_dict=col_md.value_expr,
-                tbl=self,
-            )
-            col.tbl = self
+            col = Column.from_md(col_md, self, schema_col_md)
             self.cols.append(col)
 
             # populate the lookup structures before Expr.from_dict()
@@ -783,31 +701,22 @@ class TableVersion:
         num_excs = 0
         cols_with_excs: list[Column] = []
         for col in cols_to_add:
+            assert col.id is not None, 'Column id must be set before adding the column'
             excs_per_col = 0
             col.schema_version_add = self.schema_version
             # add the column to the lookup structures now, rather than after the store changes executed successfully,
             # because it might be referenced by the next column's value_expr
             self.cols.append(col)
-            if col.name is not None:
-                self.cols_by_name[col.name] = col
             self.cols_by_id[col.id] = col
-
-            # also add to stored md
-            self._tbl_md.column_md[col.id] = schema.ColumnMd(
-                id=col.id,
-                col_type=col.col_type.as_dict(),
-                is_pk=col.is_pk,
-                schema_version_add=col.schema_version_add,
-                schema_version_drop=col.schema_version_drop,
-                value_expr=col.value_expr.as_dict() if col.value_expr is not None else None,
-                stored=col.stored,
-            )
             if col.name is not None:
-                self._schema_version_md.columns[col.id] = schema.SchemaColumn(
-                    name=col.name,
-                    pos=len(self.cols_by_name),
-                    media_validation=col._media_validation.name.lower() if col._media_validation is not None else None,
-                )
+                self.cols_by_name[col.name] = col
+                col_md, sch_md = col.to_md(len(self.cols_by_name))
+                assert sch_md is not None, 'Schema column metadata must be created for user-facing columns'
+                self._tbl_md.column_md[col.id] = col_md
+                self._schema_version_md.columns[col.id] = sch_md
+            else:
+                col_md, _ = col.to_md()
+                self._tbl_md.column_md[col.id] = col_md
 
             if col.is_stored:
                 self.store_tbl.add_column(col)
@@ -918,14 +827,17 @@ class TableVersion:
 
     def rename_column(self, old_name: str, new_name: str) -> None:
         """Rename a column."""
-        assert self.is_mutable
-        if old_name not in self.cols_by_name:
+        if not self.is_mutable:
+            raise excs.Error(f'Cannot rename column for immutable table {self.name!r}')
+        col = self.path.get_column(old_name)
+        if col is None:
             raise excs.Error(f'Unknown column: {old_name}')
+        if col.tbl.id != self.id:
+            raise excs.Error(f'Cannot rename base table column {col.name!r}')
         if not is_valid_identifier(new_name):
             raise excs.Error(f"Invalid column name: '{new_name}'")
         if new_name in self.cols_by_name:
             raise excs.Error(f'Column {new_name} already exists')
-        col = self.cols_by_name[old_name]
         del self.cols_by_name[old_name]
         col.name = new_name
         self.cols_by_name[new_name] = col
@@ -1115,10 +1027,11 @@ class TableVersion:
                 for el in val:
                     assert isinstance(el, int)
                 continue
-            col = self.path.get_column(col_name, include_bases=False)
+            col = self.path.get_column(col_name)
             if col is None:
-                # TODO: return more informative error if this is trying to update a base column
                 raise excs.Error(f'Column {col_name} unknown')
+            if col.tbl.id != self.id:
+                raise excs.Error(f'Column {col.name!r} is a base table column and cannot be updated')
             if col.is_computed:
                 raise excs.Error(f'Column {col_name} is computed and cannot be updated')
             if col.is_pk and not allow_pk:
@@ -1326,7 +1239,7 @@ class TableVersion:
             )
 
         # delete newly-added data
-        MediaStore.delete(self.id, tbl_version=self.version)
+        MediaStore.get().delete(self.id, tbl_version=self.version)
         conn.execute(sql.delete(self.store_tbl.sa_tbl).where(self.store_tbl.sa_tbl.c.v_min == self.version))
 
         # revert new deletions
@@ -1628,23 +1541,6 @@ class TableVersion:
             return 1 + self.base.get().num_rowid_columns()
         return 1
 
-    @classmethod
-    def _create_column_md(cls, cols: list[Column]) -> dict[int, schema.ColumnMd]:
-        column_md: dict[int, schema.ColumnMd] = {}
-        for col in cols:
-            value_expr_dict = col.value_expr.as_dict() if col.value_expr is not None else None
-            assert col.is_pk is not None
-            column_md[col.id] = schema.ColumnMd(
-                id=col.id,
-                col_type=col.col_type.as_dict(),
-                is_pk=col.is_pk,
-                schema_version_add=col.schema_version_add,
-                schema_version_drop=col.schema_version_drop,
-                value_expr=value_expr_dict,
-                stored=col.stored,
-            )
-        return column_md
-
     @classmethod
     def _create_stores_md(cls, stores: Iterable[pxt.io.ExternalStore]) -> list[dict[str, Any]]:
         return [

pixeltable/catalog/table_version_path.py

@@ -184,13 +184,13 @@ class TableVersionPath:
         cols = self.columns()
         return {col.id: col for col in cols}
 
-    def get_column(self, name: str, include_bases: Optional[bool] = None) -> Optional[Column]:
+    def get_column(self, name: str) -> Optional[Column]:
         """Return the column with the given name, or None if not found"""
         self.refresh_cached_md()
         col = self._cached_tbl_version.cols_by_name.get(name)
         if col is not None:
             return col
-        elif self.base is not None and (include_bases or self._cached_tbl_version.include_base_columns):
+        elif self.base is not None and self._cached_tbl_version.include_base_columns:
             return self.base.get_column(name)
         else:
             return None
@@ -206,10 +206,11 @@ class TableVersionPath:
         else:
             return None
 
-    def has_column(self, col: Column, include_bases: bool = True) -> bool:
+    def has_column(self, col: Column) -> bool:
         """Return True if this table has the given column."""
-        self.refresh_cached_md()
         assert col.tbl is not None
+        self.refresh_cached_md()
+
         if (
             col.tbl.id == self.tbl_version.id
             and col.tbl.effective_version == self.tbl_version.effective_version
@@ -217,7 +218,7 @@ class TableVersionPath:
         ):
             # the column is visible in this table version
             return True
-        elif self.base is not None and include_bases:
+        elif self.base is not None:
             return self.base.has_column(col)
         else:
             return False

pixeltable/catalog/view.py

@@ -25,6 +25,7 @@ from .tbl_ops import CreateStoreTableOp, LoadViewOp, TableOp
 from .update_status import UpdateStatus
 
 if TYPE_CHECKING:
+    from pixeltable.catalog.table import TableMetadata
     from pixeltable.globals import TableDataSource
 
 _logger = logging.getLogger('pixeltable')
@@ -261,7 +262,7 @@ class View(Table):
         """
         return self._snapshot_only and self._id == self._tbl_version_path.tbl_id
 
-    def _get_metadata(self) -> dict[str, Any]:
+    def _get_metadata(self) -> 'TableMetadata':
         md = super()._get_metadata()
         md['is_view'] = True
         md['is_snapshot'] = self._tbl_version_path.is_snapshot()

pixeltable/config.py

@@ -111,10 +111,19 @@ class Config:
         return default
 
     def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
-        value = self.lookup_env(section, key)  # Try to get from environment first
+        value: Any = self.lookup_env(section, key)  # Try to get from environment first
         # Next try the config file
-        if value is None and section in self.__config_dict and key in self.__config_dict[section]:
-            value = self.__config_dict[section][key]
+        if value is None:
+            # Resolve nested section dicts
+            lookup_elems = [*section.split('.'), key]
+            value = self.__config_dict
+            for el in lookup_elems:
+                if isinstance(value, dict):
+                    if el not in value:
+                        return None
+                    value = value[el]
+                else:
+                    return None
 
         if value is None:
             return None  # Not specified
@@ -155,19 +164,25 @@ KNOWN_CONFIG_OPTIONS = {
     },
     'anthropic': {'api_key': 'Anthropic API key'},
     'bedrock': {'api_key': 'AWS Bedrock API key'},
-    'deepseek': {'api_key': 'Deepseek API key'},
-    'fireworks': {'api_key': 'Fireworks API key'},
-    'gemini': {'api_key': 'Gemini API key'},
-    'groq': {'api_key': 'Groq API key'},
+    'deepseek': {'api_key': 'Deepseek API key', 'rate_limit': 'Rate limit for Deepseek API requests'},
+    'fireworks': {'api_key': 'Fireworks API key', 'rate_limit': 'Rate limit for Fireworks API requests'},
+    'gemini': {'api_key': 'Gemini API key', 'rate_limits': 'Per-model rate limits for Gemini API requests'},
+    'imagen': {'rate_limits': 'Per-model rate limits for Imagen API requests'},
+    'veo': {'rate_limits': 'Per-model rate limits for Veo API requests'},
+    'groq': {'api_key': 'Groq API key', 'rate_limit': 'Rate limit for Groq API requests'},
     'label_studio': {'api_key': 'Label Studio API key', 'url': 'Label Studio server URL'},
-    'mistral': {'api_key': 'Mistral API key'},
+    'mistral': {'api_key': 'Mistral API key', 'rate_limit': 'Rate limit for Mistral API requests'},
     'openai': {
         'api_key': 'OpenAI API key',
         'base_url': 'OpenAI API base URL',
         'api_version': 'API version if using Azure OpenAI',
+        'rate_limits': 'Per-model rate limits for OpenAI API requests',
     },
     'replicate': {'api_token': 'Replicate API token'},
-    'together': {'api_key': 'Together API key'},
+    'together': {
+        'api_key': 'Together API key',
+        'rate_limits': 'Per-model category rate limits for Together API requests',
+    },
     'pypi': {'api_key': 'PyPI API key (for internal use only)'},
 }
 

pixeltable/dataframe.py

@@ -19,7 +19,6 @@ from typing import (
     Optional,
     Sequence,
     TypeVar,
-    Union,
 )
 
 import pandas as pd
@@ -766,7 +765,7 @@ class DataFrame:
         )
 
     def _create_join_predicate(
-        self, other: catalog.TableVersionPath, on: Union[exprs.Expr, Sequence[exprs.ColumnRef]]
+        self, other: catalog.TableVersionPath, on: exprs.Expr | Sequence[exprs.ColumnRef]
     ) -> exprs.Expr:
         """Verifies user-specified 'on' argument and converts it into a join predicate."""
         col_refs: list[exprs.ColumnRef] = []
@@ -796,19 +795,19 @@ class DataFrame:
         assert len(col_refs) > 0 and len(joined_tbls) >= 2
         for col_ref in col_refs:
             # identify the referenced column by name in 'other'
-            rhs_col = other.get_column(col_ref.col.name, include_bases=True)
+            rhs_col = other.get_column(col_ref.col.name)
             if rhs_col is None:
                 raise excs.Error(f"'on': column {col_ref.col.name!r} not found in joined table")
             rhs_col_ref = exprs.ColumnRef(rhs_col)
 
             lhs_col_ref: Optional[exprs.ColumnRef] = None
-            if any(tbl.has_column(col_ref.col, include_bases=True) for tbl in self._from_clause.tbls):
+            if any(tbl.has_column(col_ref.col) for tbl in self._from_clause.tbls):
                 # col_ref comes from the existing from_clause, we use that directly
                 lhs_col_ref = col_ref
             else:
                 # col_ref comes from other, we need to look for a match in the existing from_clause by name
                 for tbl in self._from_clause.tbls:
-                    col = tbl.get_column(col_ref.col.name, include_bases=True)
+                    col = tbl.get_column(col_ref.col.name)
                     if col is None:
                         continue
                     if lhs_col_ref is not None:
@@ -829,7 +828,7 @@ class DataFrame:
     def join(
         self,
         other: catalog.Table,
-        on: Optional[Union[exprs.Expr, Sequence[exprs.ColumnRef]]] = None,
+        on: exprs.Expr | Sequence[exprs.ColumnRef] | None = None,
         how: plan.JoinType.LiteralType = 'inner',
     ) -> DataFrame:
         """