pixeltable 0.4.17__py3-none-any.whl → 0.4.19__py3-none-any.whl

pixeltable/catalog/table_version_handle.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import logging
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 from uuid import UUID
 
 from pixeltable import exceptions as excs
@@ -21,10 +21,10 @@ class TableVersionHandle:
     """
 
     id: UUID
-    effective_version: Optional[int]
-    _tbl_version: Optional[TableVersion]
+    effective_version: int | None
+    _tbl_version: TableVersion | None
 
-    def __init__(self, tbl_id: UUID, effective_version: Optional[int], tbl_version: Optional[TableVersion] = None):
+    def __init__(self, tbl_id: UUID, effective_version: int | None, tbl_version: TableVersion | None = None):
         self.id = tbl_id
         self.effective_version = effective_version
         self._tbl_version = tbl_version

pixeltable/catalog/table_version_path.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-from typing import Optional
 from uuid import UUID
 
 from pixeltable.env import Env
@@ -39,10 +38,10 @@ class TableVersionPath:
     """
 
     tbl_version: TableVersionHandle
-    base: Optional[TableVersionPath]
-    _cached_tbl_version: Optional[TableVersion]
+    base: TableVersionPath | None
+    _cached_tbl_version: TableVersion | None
 
-    def __init__(self, tbl_version: TableVersionHandle, base: Optional[TableVersionPath] = None):
+    def __init__(self, tbl_version: TableVersionHandle, base: TableVersionPath | None = None):
         assert tbl_version is not None
         self.tbl_version = tbl_version
         self.base = base
@@ -51,7 +50,7 @@ class TableVersionPath:
     @classmethod
     def from_md(cls, path: schema.TableVersionPath) -> TableVersionPath:
         assert len(path) > 0
-        result: Optional[TableVersionPath] = None
+        result: TableVersionPath | None = None
         for tbl_id_str, effective_version in path[::-1]:
             tbl_id = UUID(tbl_id_str)
             result = TableVersionPath(TableVersionHandle(tbl_id, effective_version), base=result)
@@ -156,7 +155,7 @@ class TableVersionPath:
             return []
         return self.base.get_tbl_versions()
 
-    def find_tbl_version(self, id: UUID) -> Optional[TableVersionHandle]:
+    def find_tbl_version(self, id: UUID) -> TableVersionHandle | None:
         """Return the matching TableVersion in the chain of TableVersions, starting with this one"""
         if self.tbl_version.id == id:
             return self.tbl_version
@@ -184,7 +183,7 @@ class TableVersionPath:
         cols = self.columns()
         return {col.id: col for col in cols}
 
-    def get_column(self, name: str) -> Optional[Column]:
+    def get_column(self, name: str) -> Column | None:
         """Return the column with the given name, or None if not found"""
         self.refresh_cached_md()
         col = self._cached_tbl_version.cols_by_name.get(name)
@@ -197,12 +196,12 @@ class TableVersionPath:
 
     def has_column(self, col: Column) -> bool:
         """Return True if this table has the given column."""
-        assert col.tbl is not None
+        assert col.get_tbl() is not None
         self.refresh_cached_md()
 
         if (
-            col.tbl.id == self.tbl_version.id
-            and col.tbl.effective_version == self.tbl_version.effective_version
+            col.get_tbl().id == self.tbl_version.id
+            and col.get_tbl().effective_version == self.tbl_version.effective_version
             and col.id in self._cached_tbl_version.cols_by_id
         ):
             # the column is visible in this table version

pixeltable/catalog/tbl_ops.py

@@ -5,7 +5,7 @@
 #   TableVersion
 
 import dataclasses
-from typing import Any, Optional
+from typing import Any
 
 
 @dataclasses.dataclass
@@ -13,6 +13,11 @@ class CreateStoreTableOp:
     pass
 
 
+@dataclasses.dataclass
+class CreateIndexOp:
+    idx_id: int
+
+
 @dataclasses.dataclass
 class LoadViewOp:
     view_path: dict[str, Any]  # needed to create the view load plan
@@ -40,5 +45,6 @@ class TableOp:
     num_ops: int  # total number of ops forming the update operation
     needs_xact: bool  # if True, op must be run as part of a transaction
 
-    create_store_table_op: Optional[CreateStoreTableOp] = None
-    load_view_op: Optional[LoadViewOp] = None
+    create_store_table_op: CreateStoreTableOp | None = None
+    create_index_op: CreateIndexOp | None = None
+    load_view_op: LoadViewOp | None = None

pixeltable/catalog/view.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import inspect
 import logging
-from typing import TYPE_CHECKING, Any, List, Literal, Optional
+from typing import TYPE_CHECKING, Any, List, Literal
 from uuid import UUID
 
 import pixeltable.exceptions as excs
@@ -11,10 +11,6 @@ import pixeltable.type_system as ts
 from pixeltable import catalog, exprs, func
 from pixeltable.iterators import ComponentIterator
 
-if TYPE_CHECKING:
-    from pixeltable.plan import SampleClause
-
-
 from .column import Column
 from .globals import _POS_COLUMN_NAME, MediaValidation
 from .table import Table
@@ -27,6 +23,7 @@ from .update_status import UpdateStatus
 if TYPE_CHECKING:
     from pixeltable.catalog.table import TableMetadata
     from pixeltable.globals import TableDataSource
+    from pixeltable.plan import SampleClause
 
 _logger = logging.getLogger('pixeltable')
 
@@ -56,7 +53,7 @@ class View(Table):
         return 'table'
 
     @classmethod
-    def select_list_to_additional_columns(cls, select_list: list[tuple[exprs.Expr, Optional[str]]]) -> dict[str, dict]:
+    def select_list_to_additional_columns(cls, select_list: list[tuple[exprs.Expr, str | None]]) -> dict[str, dict]:
         """Returns a list of columns in the same format as the additional_columns parameter of View.create.
         The source is the list of expressions from a select() statement on a DataFrame.
         If the column is a ColumnRef, to a base table column, it is marked to not be stored.sy
@@ -76,17 +73,18 @@ class View(Table):
         dir_id: UUID,
         name: str,
         base: TableVersionPath,
-        select_list: Optional[list[tuple[exprs.Expr, Optional[str]]]],
+        select_list: list[tuple[exprs.Expr, str | None]] | None,
         additional_columns: dict[str, Any],
-        predicate: Optional['exprs.Expr'],
-        sample_clause: Optional['SampleClause'],
+        predicate: 'exprs.Expr' | None,
+        sample_clause: 'SampleClause' | None,
         is_snapshot: bool,
+        create_default_idxs: bool,
         num_retained_versions: int,
         comment: str,
         media_validation: MediaValidation,
-        iterator_cls: Optional[type[ComponentIterator]],
-        iterator_args: Optional[dict],
-    ) -> tuple[TableVersionMd, Optional[list[TableOp]]]:
+        iterator_cls: type[ComponentIterator] | None,
+        iterator_args: dict | None,
+    ) -> tuple[TableVersionMd, list[TableOp] | None]:
         from pixeltable.plan import SampleClause
 
         # Convert select_list to more additional_columns if present
@@ -103,7 +101,7 @@ class View(Table):
         # verify that filters can be evaluated in the context of the base
         if predicate is not None:
             if not predicate.is_bound_by([base]):
-                raise excs.Error(f'Filter cannot be computed in the context of the base {base.tbl_name()}')
+                raise excs.Error(f'View filter cannot be computed in the context of the base table {base.tbl_name()!r}')
             # create a copy that we can modify and store
             predicate = predicate.copy()
         if sample_clause is not None:
@@ -111,7 +109,9 @@ class View(Table):
             if sample_clause.stratify_exprs is not None and not all(
                 stratify_expr.is_bound_by([base]) for stratify_expr in sample_clause.stratify_exprs
             ):
-                raise excs.Error(f'Sample clause cannot be computed in the context of the base {base.tbl_name()}')
+                raise excs.Error(
+                    f'View sample clause cannot be computed in the context of the base table {base.tbl_name()!r}'
+                )
             # create a copy that we can modify and store
             sc = sample_clause
             sample_clause = SampleClause(
@@ -125,8 +125,8 @@ class View(Table):
             # make sure that the value can be computed in the context of the base
             if col.value_expr is not None and not col.value_expr.is_bound_by([base]):
                 raise excs.Error(
-                    f'Column {col.name}: value expression cannot be computed in the context of the '
-                    f'base {base.tbl_name()}'
+                    f'Column {col.name!r}: Value expression cannot be computed in the context of the '
+                    f'base table {base.tbl_name()!r}'
                 )
 
         if iterator_cls is not None:
@@ -153,18 +153,18 @@ class View(Table):
             sig = func.Signature(ts.InvalidType(), params)
 
             expr_args = {k: exprs.Expr.from_object(v) for k, v in bound_args.items()}
-            sig.validate_args(expr_args, context=f'in iterator {iterator_cls.__name__!r}')
+            sig.validate_args(expr_args, context=f'in iterator of type `{iterator_cls.__name__}`')
             literal_args = {k: v.val if isinstance(v, exprs.Literal) else v for k, v in expr_args.items()}
 
             # prepend pos and output_schema columns to cols:
             # a component view exposes the pos column of its rowid;
             # we create that column here, so it gets assigned a column id;
             # stored=False: it is not stored separately (it's already stored as part of the rowid)
-            iterator_cols = [Column(_POS_COLUMN_NAME, ts.IntType(), stored=False)]
+            iterator_cols = [Column(_POS_COLUMN_NAME, ts.IntType(), is_iterator_col=True, stored=False)]
             output_dict, unstored_cols = iterator_cls.output_schema(**literal_args)
             iterator_cols.extend(
                 [
-                    Column(col_name, col_type, stored=col_name not in unstored_cols)
+                    Column(col_name, col_type, is_iterator_col=True, stored=col_name not in unstored_cols)
                     for col_name, col_type in output_dict.items()
                 ]
             )
@@ -206,7 +206,13 @@ class View(Table):
         )
 
         md = TableVersion.create_initial_md(
-            name, columns, num_retained_versions, comment, media_validation=media_validation, view_md=view_md
+            name,
+            columns,
+            num_retained_versions,
+            comment,
+            media_validation=media_validation,
+            view_md=view_md,
+            create_default_idxs=create_default_idxs,
         )
         if md.tbl_md.is_pure_snapshot:
             # this is purely a snapshot: no store table to create or load
@@ -230,7 +236,7 @@ class View(Table):
     def _verify_column(cls, col: Column) -> None:
         # make sure that columns are nullable or have a default
         if not col.col_type.nullable and not col.is_computed:
-            raise excs.Error(f'Column {col.name}: non-computed columns in views must be nullable')
+            raise excs.Error(f'Column {col.name!r}: Non-computed columns in views must be nullable')
         super()._verify_column(col)
 
     @classmethod
@@ -282,22 +288,22 @@ class View(Table):
 
     def insert(
         self,
-        source: Optional[TableDataSource] = None,
+        source: TableDataSource | None = None,
         /,
         *,
-        source_format: Optional[Literal['csv', 'excel', 'parquet', 'json']] = None,
-        schema_overrides: Optional[dict[str, ts.ColumnType]] = None,
+        source_format: Literal['csv', 'excel', 'parquet', 'json'] | None = None,
+        schema_overrides: dict[str, ts.ColumnType] | None = None,
         on_error: Literal['abort', 'ignore'] = 'abort',
         print_stats: bool = False,
         **kwargs: Any,
     ) -> UpdateStatus:
         raise excs.Error(f'{self._display_str()}: Cannot insert into a {self._display_name()}.')
 
-    def delete(self, where: Optional[exprs.Expr] = None) -> UpdateStatus:
+    def delete(self, where: exprs.Expr | None = None) -> UpdateStatus:
         raise excs.Error(f'{self._display_str()}: Cannot delete from a {self._display_name()}.')
 
     @property
-    def _base_tbl_id(self) -> Optional[UUID]:
+    def _base_tbl_id(self) -> UUID | None:
         if self._tbl_version_path.tbl_id != self._id:
             # _tbl_version_path represents a different schema object from this one. This can only happen if this is a
             # named pure snapshot.
@@ -306,14 +312,14 @@ class View(Table):
             return None
         return self._tbl_version_path.base.tbl_id
 
-    def _get_base_table(self) -> Optional['Table']:
+    def _get_base_table(self) -> 'Table' | None:
         """Returns None if there is no base table, or if the base table is hidden."""
         base_tbl_id = self._base_tbl_id
         with catalog.Catalog.get().begin_xact(tbl_id=base_tbl_id, for_write=False):
             return catalog.Catalog.get().get_table_by_id(base_tbl_id)
 
     @property
-    def _effective_base_versions(self) -> list[Optional[int]]:
+    def _effective_base_versions(self) -> list[int | None]:
         effective_versions = [tv.effective_version for tv in self._tbl_version_path.get_tbl_versions()]
         if self._snapshot_only and not self._is_anonymous_snapshot():
             return effective_versions  # Named pure snapshot

pixeltable/config.py

@@ -4,7 +4,7 @@ import logging
 import os
 import shutil
 from pathlib import Path
-from typing import Any, ClassVar, Optional, TypeVar
+from typing import Any, ClassVar, TypeVar
 
 import toml
 
@@ -21,7 +21,7 @@ class Config:
     configuration values, which can be set in the config file or as environment variables.
     """
 
-    __instance: ClassVar[Optional[Config]] = None
+    __instance: ClassVar[Config | None] = None
 
     __home: Path
     __config_file: Path
@@ -110,7 +110,7 @@ class Config:
             return os.environ[env_var]
         return default
 
-    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> Optional[T]:
+    def get_value(self, key: str, expected_type: type[T], section: str = 'pixeltable') -> T | None:
         value: Any = self.lookup_env(section, key)  # Try to get from environment first
         # Next try the config file
         if value is None:
@@ -131,22 +131,22 @@ class Config:
         try:
             if expected_type is bool and isinstance(value, str):
                 if value.lower() not in ('true', 'false'):
-                    raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}')
+                    raise excs.Error(f"Invalid value for configuration parameter '{section}.{key}': {value}")
                 return value.lower() == 'true'  # type: ignore[return-value]
             return expected_type(value)  # type: ignore[call-arg]
         except (ValueError, TypeError) as exc:
-            raise excs.Error(f'Invalid value for configuration parameter {section}.{key}: {value}') from exc
+            raise excs.Error(f"Invalid value for configuration parameter '{section}.{key}': {value}") from exc
 
-    def get_string_value(self, key: str, section: str = 'pixeltable') -> Optional[str]:
+    def get_string_value(self, key: str, section: str = 'pixeltable') -> str | None:
         return self.get_value(key, str, section)
 
-    def get_int_value(self, key: str, section: str = 'pixeltable') -> Optional[int]:
+    def get_int_value(self, key: str, section: str = 'pixeltable') -> int | None:
         return self.get_value(key, int, section)
 
-    def get_float_value(self, key: str, section: str = 'pixeltable') -> Optional[float]:
+    def get_float_value(self, key: str, section: str = 'pixeltable') -> float | None:
         return self.get_value(key, float, section)
 
-    def get_bool_value(self, key: str, section: str = 'pixeltable') -> Optional[bool]:
+    def get_bool_value(self, key: str, section: str = 'pixeltable') -> bool | None:
         return self.get_value(key, bool, section)
 
 
@@ -161,18 +161,21 @@ KNOWN_CONFIG_OPTIONS = {
         'hide_warnings': 'Hide warnings from the console',
         'verbosity': 'Verbosity level for console output',
         'api_key': 'API key for Pixeltable cloud',
+        'input_media_dest': 'Default destination URI for input media data',
+        'output_media_dest': 'Default destination URI for output (computed) media data',
         'r2_profile': 'AWS config profile name used to access R2 storage',
         's3_profile': 'AWS config profile name used to access S3 storage',
         'b2_profile': 'S3-compatible profile name used to access Backblaze B2 storage',
     },
     'anthropic': {'api_key': 'Anthropic API key'},
+    'azure': {'storage_account_name': 'Azure storage account name', 'storage_account_key': 'Azure storage account key'},
     'bedrock': {'api_key': 'AWS Bedrock API key'},
     'deepseek': {'api_key': 'Deepseek API key', 'rate_limit': 'Rate limit for Deepseek API requests'},
     'fireworks': {'api_key': 'Fireworks API key', 'rate_limit': 'Rate limit for Fireworks API requests'},
+    'twelvelabs': {'api_key': 'TwelveLabs API key', 'rate_limit': 'Rate limit for TwelveLabs API requests'},
     'gemini': {'api_key': 'Gemini API key', 'rate_limits': 'Per-model rate limits for Gemini API requests'},
     'hf': {'auth_token': 'Hugging Face access token'},
     'imagen': {'rate_limits': 'Per-model rate limits for Imagen API requests'},
-    'veo': {'rate_limits': 'Per-model rate limits for Veo API requests'},
     'groq': {'api_key': 'Groq API key', 'rate_limit': 'Rate limit for Groq API requests'},
     'label_studio': {'api_key': 'Label Studio API key', 'url': 'Label Studio server URL'},
     'mistral': {'api_key': 'Mistral API key', 'rate_limit': 'Rate limit for Mistral API requests'},
@@ -193,6 +196,7 @@ KNOWN_CONFIG_OPTIONS = {
         'api_key': 'Together API key',
         'rate_limits': 'Per-model category rate limits for Together API requests',
     },
+    'veo': {'rate_limits': 'Per-model rate limits for Veo API requests'},
     'pypi': {'api_key': 'PyPI API key (for internal use only)'},
 }