pixeltable 0.4.8__py3-none-any.whl → 0.4.10__py3-none-any.whl

pixeltable/__init__.py

@@ -30,7 +30,7 @@ from .globals import (
 from .type_system import Array, Audio, Bool, Date, Document, Float, Image, Int, Json, Required, String, Timestamp, Video
 
 # This import must go last to avoid circular imports.
-from . import ext, functions, io, iterators  # isort: skip
+from . import functions, io, iterators  # isort: skip
 
 # This is the safest / most maintainable way to construct __all__: start with the default and "blacklist"
 # stuff that we don't want in there. (Using a "whitelist" is considerably harder to maintain.)

pixeltable/catalog/insertable_table.py

@@ -2,13 +2,17 @@ from __future__ import annotations
 
 import enum
 import logging
-from typing import TYPE_CHECKING, Any, Literal, Optional, overload
+from typing import TYPE_CHECKING, Any, Literal, Optional, Sequence, cast, overload
 from uuid import UUID
 
+import pydantic
+import pydantic_core
+
 import pixeltable as pxt
 from pixeltable import exceptions as excs, type_system as ts
 from pixeltable.env import Env
 from pixeltable.utils.filecache import FileCache
+from pixeltable.utils.pydantic import is_json_convertible
 
 from .globals import MediaValidation
 from .table import Table
@@ -137,8 +141,24 @@ class InsertableTable(Table):
         from pixeltable.catalog import Catalog
         from pixeltable.io.table_data_conduit import UnkTableDataConduit
 
+        if source is not None and isinstance(source, Sequence) and len(source) == 0:
+            raise excs.Error('Cannot insert an empty sequence')
+        fail_on_exception = OnErrorParameter.fail_on_exception(on_error)
+
         with Catalog.get().begin_xact(tbl=self._tbl_version_path, for_write=True, lock_mutable_tree=True):
             table = self
+
+            # TODO: unify with TableDataConduit
+            if source is not None and isinstance(source, Sequence) and isinstance(source[0], pydantic.BaseModel):
+                status = self._insert_pydantic(
+                    cast(Sequence[pydantic.BaseModel], source),  # needed for mypy
+                    print_stats=print_stats,
+                    fail_on_exception=fail_on_exception,
+                )
+                Env.get().console_logger.info(status.insert_msg)
+                FileCache.get().emit_eviction_warnings()
+                return status
+
             if source is None:
                 source = [kwargs]
                 kwargs = None
@@ -154,7 +174,6 @@ class InsertableTable(Table):
             data_source.add_table_info(table)
             data_source.prepare_for_insert_into_table()
 
-            fail_on_exception = OnErrorParameter.fail_on_exception(on_error)
             return table.insert_table_data_source(
                 data_source=data_source, fail_on_exception=fail_on_exception, print_stats=print_stats
             )
@@ -184,32 +203,110 @@ class InsertableTable(Table):
         FileCache.get().emit_eviction_warnings()
         return status
 
-    def _validate_input_rows(self, rows: list[dict[str, Any]]) -> None:
-        """Verify that the input rows match the table schema"""
-        valid_col_names = set(self._get_schema().keys())
-        reqd_col_names = set(self._tbl_version_path.tbl_version.get().get_required_col_names())
-        computed_col_names = set(self._tbl_version_path.tbl_version.get().get_computed_col_names())
-        for row in rows:
-            assert isinstance(row, dict)
-            col_names = set(row.keys())
-            if len(reqd_col_names - col_names) > 0:
-                raise excs.Error(f'Missing required column(s) ({", ".join(reqd_col_names - col_names)}) in row {row}')
-
-            for col_name, val in row.items():
-                if col_name not in valid_col_names:
-                    raise excs.Error(f'Unknown column name {col_name!r} in row {row}')
-                if col_name in computed_col_names:
-                    raise excs.Error(f'Value for computed column {col_name!r} in row {row}')
-
-                # validate data
-                col = self._tbl_version_path.get_column(col_name)
-                try:
-                    # basic sanity checks here
-                    checked_val = col.col_type.create_literal(val)
-                    row[col_name] = checked_val
-                except TypeError as e:
-                    msg = str(e)
-                    raise excs.Error(f'Error in column {col.name}: {msg[0].lower() + msg[1:]}\nRow: {row}') from e
+    def _insert_pydantic(
+        self, rows: Sequence[pydantic.BaseModel], print_stats: bool = False, fail_on_exception: bool = True
+    ) -> UpdateStatus:
+        model_class = type(rows[0])
+        self._validate_pydantic_model(model_class)
+        # convert rows one-by-one in order to be able to print meaningful error messages
+        pxt_rows: list[dict[str, Any]] = []
+        for i, row in enumerate(rows):
+            try:
+                pxt_rows.append(row.model_dump(mode='json'))
+            except pydantic_core.PydanticSerializationError as e:
+                raise excs.Error(f'Row {i}: error serializing pydantic model to JSON:\n{e!s}') from e
+
+        # explicitly check that all required columns are present and non-None in the rows,
+        # because we ignore nullability when validating the pydantic model
+        reqd_col_names = [col.name for col in self._tbl_version_path.columns() if col.is_required_for_insert]
+        for i, pxt_row in enumerate(pxt_rows):
+            if type(rows[i]) is not model_class:
+                raise excs.Error(
+                    f'Expected {model_class.__name__!r} instance, got {type(rows[i]).__name__!r} (in row {i})'
+                )
+            for col_name in reqd_col_names:
+                if pxt_row.get(col_name) is None:
+                    raise excs.Error(f'Missing required column {col_name!r} in row {i}')
+
+        status = self._tbl_version.get().insert(
+            rows=pxt_rows, df=None, print_stats=print_stats, fail_on_exception=fail_on_exception
+        )
+        return status
+
+    def _validate_pydantic_model(self, model: type[pydantic.BaseModel]) -> None:
+        """
+        Check if a Pydantic model is compatible with this table for insert operations.
+
+        A model is compatible if:
+        - All required table columns have corresponding model fields with compatible types
+        - Model does not define fields for computed columns
+        - Model field types are compatible with table column types
+        """
+        assert isinstance(model, type) and issubclass(model, pydantic.BaseModel)
+
+        schema = self._get_schema()
+        required_cols = set(self._tbl_version.get().get_required_col_names())
+        computed_cols = set(self._tbl_version.get().get_computed_col_names())
+        model_fields = model.model_fields
+        model_field_names = set(model_fields.keys())
+
+        missing_required = required_cols - model_field_names
+        if missing_required:
+            raise excs.Error(
+                f'Pydantic model {model.__name__!r} is missing required columns: '
+                f'{", ".join(f"{col_name!r}" for col_name in missing_required)}'
+            )
+
+        computed_in_model = computed_cols & model_field_names
+        if computed_in_model:
+            raise excs.Error(
+                f'Pydantic model {model.__name__!r} has fields for computed columns: '
+                f'{", ".join(f"{col_name!r}" for col_name in computed_in_model)}'
+            )
+
+        # validate type compatibility
+        common_fields = model_field_names & set(schema.keys())
+        if len(common_fields) == 0:
+            raise excs.Error(
+                f'Pydantic model {model.__name__!r} has no fields that map to columns in table {self._name!r}'
+            )
+        for field_name in common_fields:
+            pxt_col_type = schema[field_name]
+            model_field = model_fields[field_name]
+            model_type = model_field.annotation
+
+            # we ignore nullability: we want to accept optional model fields for required table columns, as long as
+            # the model instances provide a non-null value
+            # allow_enum=True: model_dump(mode='json') converts enums to their values
+            inferred_pxt_type = ts.ColumnType.from_python_type(model_type, infer_pydantic_json=True)
+            if inferred_pxt_type is None:
+                raise excs.Error(
+                    f'Pydantic model {model.__name__!r}: cannot infer Pixeltable type for column {field_name!r}'
+                )
+
+            if pxt_col_type.is_media_type():
+                # media types require file paths, either as str or Path
+                if not inferred_pxt_type.is_string_type():
+                    raise excs.Error(
+                        f"Column {field_name!r} requires a 'str' or 'Path' field in {model.__name__!r}, but it is "
+                        f'{model_type.__name__!r}'
+                    )
+            else:
+                if not pxt_col_type.is_supertype_of(inferred_pxt_type, ignore_nullable=True):
+                    raise excs.Error(
+                        f'Pydantic model {model.__name__!r} has incompatible type ({model_type.__name__}) '
+                        f'for column {field_name!r} ({pxt_col_type})'
+                    )
+
+                if (
+                    isinstance(model_type, type)
+                    and issubclass(model_type, pydantic.BaseModel)
+                    and not is_json_convertible(model_type)
+                ):
+                    raise excs.Error(
+                        f'Pydantic model {model.__name__!r} has field {field_name!r} with nested model '
+                        f'{model_type.__name__!r}, which is not JSON-convertible'
+                    )
 
     def delete(self, where: Optional['exprs.Expr'] = None) -> UpdateStatus:
         """Delete rows in this table.

pixeltable/catalog/table.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import abc
 import builtins
+import datetime
 import json
 import logging
 from keyword import iskeyword as is_python_keyword
@@ -9,7 +10,6 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Iterable, Literal, Optional, TypedDict, overload
 
 from typing import _GenericAlias  # type: ignore[attr-defined]  # isort: skip
-import datetime
 from uuid import UUID
 
 import pandas as pd
@@ -1356,6 +1356,15 @@ class Table(SchemaObject):
             Insert rows from a CSV file:
 
             >>> tbl.insert(source='path/to/file.csv')
+
+            Insert Pydantic model instances into a table with two `pxt.Int` columns `a` and `b`:
+
+            >>> class MyModel(pydantic.BaseModel):
+            ...     a: int
+            ...     b: int
+            ...
+            ... models = [MyModel(a=1, b=2), MyModel(a=3, b=4)]
+            ... tbl.insert(models)
         """
         raise NotImplementedError
 

pixeltable/config.py

@@ -167,6 +167,7 @@ KNOWN_CONFIG_OPTIONS = {
     'deepseek': {'api_key': 'Deepseek API key', 'rate_limit': 'Rate limit for Deepseek API requests'},
     'fireworks': {'api_key': 'Fireworks API key', 'rate_limit': 'Rate limit for Fireworks API requests'},
     'gemini': {'api_key': 'Gemini API key', 'rate_limits': 'Per-model rate limits for Gemini API requests'},
+    'hf': {'auth_token': 'Hugging Face access token'},
     'imagen': {'rate_limits': 'Per-model rate limits for Imagen API requests'},
     'veo': {'rate_limits': 'Per-model rate limits for Veo API requests'},
     'groq': {'api_key': 'Groq API key', 'rate_limit': 'Rate limit for Groq API requests'},

pixeltable/env.py

@@ -11,6 +11,7 @@ import logging
 import os
 import platform
 import shutil
+import subprocess
 import sys
 import threading
 import types
@@ -27,6 +28,7 @@ import nest_asyncio  # type: ignore[import-untyped]
 import pixeltable_pgserver
 import sqlalchemy as sql
 from pillow_heif import register_heif_opener  # type: ignore[import-untyped]
+from tenacity import retry, stop_after_attempt, wait_exponential_jitter
 from tqdm import TqdmWarning
 
 from pixeltable import exceptions as excs
@@ -81,6 +83,7 @@ class Env:
     _file_cache_size_g: float
     _pxt_api_key: Optional[str]
     _stdout_handler: logging.StreamHandler
+    _default_video_encoder: str | None
     _initialized: bool
 
     _resource_pool_info: dict[str, Any]
@@ -104,10 +107,14 @@ class Env:
             cls._instance._clean_up()
         cls._instance = None
         env = Env()
-        env._set_up(reinit_db=reinit_db)
-        env._upgrade_metadata()
-        cls._instance = env
-        cls.__initializing = False
+        try:
+            env._set_up(reinit_db=reinit_db)
+            env._upgrade_metadata()
+            cls._instance = env
+        finally:
+            # Reset the initializing flag, even if setup fails.
+            # This prevents the environment from being left in a broken state.
+            cls.__initializing = False
 
     def __init__(self) -> None:
         assert self._instance is None, 'Env is a singleton; use Env.get() to access the instance'
@@ -127,6 +134,7 @@ class Env:
         self._spacy_nlp = None
         self._httpd = None
         self._http_address = None
+        self._default_video_encoder = None
 
         # logging-related state
         self._logger = logging.getLogger('pixeltable')
@@ -500,14 +508,24 @@ class Env:
         assert self._db_url is not None
         assert self._db_name is not None
 
+    @retry(
+        stop=stop_after_attempt(3),  # Stop after 3 attempts
+        wait=wait_exponential_jitter(initial=0.2, max=1.0, jitter=0.2),  # Exponential backoff with jitter
+    )
     def _init_metadata(self) -> None:
         """
         Create pixeltable metadata tables and system metadata.
         This is an idempotent operation.
+
+        Retry logic handles race conditions when multiple Pixeltable processes
+        attempt to initialize metadata tables simultaneously. The first process may succeed
+        in creating tables while others encounter database constraints (e.g., "table already exists").
+        Exponential backoff with jitter reduces contention between competing processes.
         """
         assert self._sa_engine is not None
         from pixeltable import metadata
 
+        self._logger.debug('Creating pixeltable metadata')
         metadata.schema.base_metadata.create_all(self._sa_engine, checkfirst=True)
         metadata.create_system_info(self._sa_engine)
 
@@ -662,6 +680,41 @@ class Env:
         self._start_web_server()
         self.__register_packages()
 
+    @property
+    def default_video_encoder(self) -> str | None:
+        if self._default_video_encoder is None:
+            self._default_video_encoder = self._determine_default_video_encoder()
+        return self._default_video_encoder
+
+    def _determine_default_video_encoder(self) -> str | None:
+        """
+        Returns the first available encoder from a list of candidates.
+
+        TODO:
+        - the user might prefer a hardware-accelerated encoder (eg, h264_nvenc or h264_videotoolbox)
+        - allow user override via a config option 'video_encoder'
+        """
+        # look for available encoders, in this order
+        candidates = [
+            'libx264',  # GPL, best quality
+            'libopenh264',  # BSD
+        ]
+
+        try:
+            # Get list of available encoders
+            result = subprocess.run(['ffmpeg', '-encoders'], capture_output=True, text=True, timeout=10, check=True)
+
+            if result.returncode == 0:
+                available_encoders = result.stdout
+                for encoder in candidates:
+                    # ffmpeg -encoders output format: " V..... encoder_name  description"
+                    if f' {encoder} ' in available_encoders:
+                        _logger.debug(f'Using H.264 encoder: {encoder}')
+                        return encoder
+        except Exception:
+            pass
+        return None
+
     def __register_packages(self) -> None:
         """Declare optional packages that are utilized by some parts of the code."""
         self.__register_package('anthropic')

pixeltable/functions/__init__.py

@@ -26,6 +26,8 @@ from . import (
     video,
     vision,
     whisper,
+    whisperx,
+    yolox,
 )
 from .globals import count, map, max, mean, min, sum
 

pixeltable/functions/audio.py

@@ -3,6 +3,7 @@ Pixeltable [UDFs](https://pixeltable.readme.io/docs/user-defined-functions-udfs)
 """
 
 import pixeltable as pxt
+import pixeltable.utils.av as av_utils
 from pixeltable.utils.code import local_public_names
 
 
@@ -47,7 +48,7 @@ def get_metadata(audio: pxt.Audio) -> dict:
 
         >>> tbl.select(tbl.audio_col.get_metadata()).collect()
     """
-    return pxt.functions.video._get_metadata(audio)
+    return av_utils.get_metadata(audio)
 
 
 __all__ = local_public_names(__name__)

pixeltable/functions/gemini.py

@@ -14,6 +14,7 @@ import PIL.Image
 
 import pixeltable as pxt
 from pixeltable import env, exceptions as excs, exprs
+from pixeltable.utils.code import local_public_names
 from pixeltable.utils.media_store import TempStore
 
 if TYPE_CHECKING:
@@ -232,3 +233,10 @@ async def generate_videos(
 @generate_videos.resource_pool
 def _(model: str) -> str:
     return f'request-rate:veo:{model}'
+
+
+__all__ = local_public_names(__name__)
+
+
+def __dir__() -> list[str]:
+    return __all__