pixeltable 0.4.5__py3-none-any.whl → 0.4.7__py3-none-any.whl

pixeltable/store.py

@@ -4,7 +4,7 @@ import abc
 import logging
 import sys
 import warnings
-from typing import Any, Iterable, Iterator, Optional, Union
+from typing import Any, Iterable, Iterator, Optional
 
 import more_itertools
 import psycopg
@@ -17,7 +17,6 @@ from pixeltable.env import Env
 from pixeltable.exec import ExecNode
 from pixeltable.metadata import schema
 from pixeltable.utils.exception_handler import run_cleanup
-from pixeltable.utils.media_store import MediaStore
 from pixeltable.utils.sql import log_explain, log_stmt
 
 _logger = logging.getLogger('pixeltable')
@@ -123,21 +122,6 @@ class StoreBase:
     def _storage_name(self) -> str:
         """Return the name of the data store table"""
 
-    def _move_tmp_media_file(self, file_url: Optional[str], col: catalog.Column) -> str:
-        src_path = MediaStore.resolve_tmp_url(file_url)
-        if src_path is None:
-            return file_url
-        assert col.tbl.id == self.tbl_version.id  # Ensure the column belongs to the same table as this store
-        new_file_url = MediaStore.relocate_local_media_file(src_path, col)
-        return new_file_url
-
-    def _move_tmp_media_files(
-        self, table_row: list[Any], media_cols_by_sql_idx: dict[int, catalog.Column], v_min: int
-    ) -> None:
-        """Move tmp media files that we generated to a permanent location"""
-        for n, col in media_cols_by_sql_idx.items():
-            table_row[n] = self._move_tmp_media_file(table_row[n], col)
-
     def count(self) -> int:
         """Return the number of rows visible in self.tbl_version"""
         stmt = (
@@ -235,7 +219,6 @@ class StoreBase:
         # create temp table to store output of exec_plan, with the same primary key as the store table
         tmp_name = f'temp_{self._storage_name()}'
         tmp_pk_cols = tuple(sql.Column(col.name, col.type, primary_key=True) for col in self.pk_columns())
-        tmp_val_col_sql_idx = len(tmp_pk_cols)
         tmp_val_col = sql.Column(col.sa_col.name, col.sa_col.type)
         tmp_cols = [*tmp_pk_cols, tmp_val_col]
         # add error columns if the store column records errors
@@ -262,9 +245,7 @@ class StoreBase:
                     if abort_on_exc and row.has_exc():
                         exc = row.get_first_exc()
                         raise excs.Error(f'Error while evaluating computed column {col.name!r}:\n{exc}') from exc
-                    table_row, num_row_exc = row_builder.create_table_row(row, None, row.pk)
-                    if col.col_type.is_media_type():
-                        table_row[tmp_val_col_sql_idx] = self._move_tmp_media_file(table_row[tmp_val_col_sql_idx], col)
+                    table_row, num_row_exc = row_builder.create_store_table_row(row, None, row.pk)
                     num_excs += num_row_exc
                     batch_table_rows.append(tuple(table_row))
 
@@ -317,7 +298,7 @@ class StoreBase:
         progress_bar: Optional[tqdm] = None  # create this only after we started executing
         row_builder = exec_plan.row_builder
 
-        store_col_names, media_cols_by_idx = row_builder.store_column_names()
+        store_col_names = row_builder.store_column_names()
 
         try:
             table_rows: list[tuple[Any]] = []
@@ -337,7 +318,7 @@ class StoreBase:
                     rowid = (next(rowids),) if rowids is not None else row.pk[:-1]
                     pk = (*rowid, v_min)
                     assert len(pk) == len(self._pk_cols)
-                    table_row, num_row_exc = row_builder.create_table_row(row, cols_with_excs, pk)
+                    table_row, num_row_exc = row_builder.create_store_table_row(row, cols_with_excs, pk)
                     num_excs += num_row_exc
 
                     if show_progress:
@@ -351,7 +332,6 @@ class StoreBase:
                             )
                         progress_bar.update(1)
 
-                    self._move_tmp_media_files(table_row, media_cols_by_idx, v_min)
                     batch_table_rows.append(tuple(table_row))
 
                 table_rows.extend(batch_table_rows)
@@ -427,7 +407,7 @@ class StoreBase:
         base_versions_clause = (
             sql.true() if len(base_versions) == 0 else self.base._versions_clause(base_versions, match_on_vmin)
         )
-        set_clause: dict[sql.Column, Union[int, sql.Column]] = {self.v_max_col: current_version}
+        set_clause: dict[sql.Column, int | sql.Column] = {self.v_max_col: current_version}
         for index_info in self.tbl_version.get().idxs_by_name.values():
             # copy value column to undo column
             set_clause[index_info.undo_col.sa_col] = index_info.val_col.sa_col

pixeltable/type_system.py

@@ -292,7 +292,7 @@ class ColumnType:
 
     @classmethod
     def from_python_type(
-        cls, t: Union[type, _GenericAlias], nullable_default: bool = False, allow_builtin_types: bool = True
+        cls, t: type | _GenericAlias, nullable_default: bool = False, allow_builtin_types: bool = True
     ) -> Optional[ColumnType]:
         """
         Convert a Python type into a Pixeltable `ColumnType` instance.
@@ -311,7 +311,7 @@ class ColumnType:
         if origin in (typing.Union, types.UnionType):
             # Check if `t` has the form Optional[T].
             if len(type_args) == 2 and type(None) in type_args:
-                # `t` is a type of the form Optional[T] (equivalently, Union[T, None] or Union[None, T]).
+                # `t` is a type of the form Optional[T] (equivalently, T | None or None | T).
                 # We treat it as the underlying type but with nullable=True.
                 underlying_py_type = type_args[0] if type_args[1] is type(None) else type_args[1]
                 underlying = cls.from_python_type(underlying_py_type, allow_builtin_types=allow_builtin_types)
@@ -361,10 +361,7 @@ class ColumnType:
 
     @classmethod
     def normalize_type(
-        cls,
-        t: Union[ColumnType, type, _AnnotatedAlias],
-        nullable_default: bool = False,
-        allow_builtin_types: bool = True,
+        cls, t: ColumnType | type | _AnnotatedAlias, nullable_default: bool = False, allow_builtin_types: bool = True
     ) -> ColumnType:
         """
         Convert any type recognizable by Pixeltable to its corresponding ColumnType.
@@ -389,7 +386,7 @@ class ColumnType:
     ]
 
     @classmethod
-    def __raise_exc_for_invalid_type(cls, t: Union[type, _AnnotatedAlias]) -> None:
+    def __raise_exc_for_invalid_type(cls, t: type | _AnnotatedAlias) -> None:
         for builtin_type, suggestion in cls.__TYPE_SUGGESTIONS:
             if t is builtin_type or (isinstance(t, type) and issubclass(t, builtin_type)):
                 name = t.__name__ if t.__module__ == 'builtins' else f'{t.__module__}.{t.__name__}'
@@ -405,7 +402,7 @@ class ColumnType:
         return cls.from_python_type(py_type) if py_type is not None else None
 
     @classmethod
-    def __json_schema_to_py_type(cls, schema: dict[str, Any]) -> Union[type, _GenericAlias, None]:
+    def __json_schema_to_py_type(cls, schema: dict[str, Any]) -> type | _GenericAlias | None:
         if 'type' in schema:
             if schema['type'] == 'null':
                 return type(None)

pixeltable/utils/__init__.py

@@ -2,7 +2,7 @@ import hashlib
 import urllib.parse
 import urllib.request
 from pathlib import Path
-from typing import Optional, Union
+from typing import Optional
 
 
 def print_perf_counter_delta(delta: float) -> str:
@@ -24,7 +24,7 @@ def print_perf_counter_delta(delta: float) -> str:
         return f'{delta:.2f} s'
 
 
-def sha256sum(path: Union[Path, str]) -> str:
+def sha256sum(path: Path | str) -> str:
     """
     Compute the SHA256 hash of a file.
     """

pixeltable/utils/arrow.py

@@ -1,5 +1,5 @@
 import datetime
-from typing import Any, Iterator, Optional, Union
+from typing import Any, Iterator, Optional
 
 import numpy as np
 import pyarrow as pa
@@ -88,11 +88,11 @@ def to_arrow_schema(pixeltable_schema: dict[str, Any]) -> pa.Schema:
     return pa.schema((name, to_arrow_type(typ)) for name, typ in pixeltable_schema.items())  # type: ignore[misc]
 
 
-def to_pydict(batch: Union[pa.Table, pa.RecordBatch]) -> dict[str, Union[list, np.ndarray]]:
+def to_pydict(batch: pa.Table | pa.RecordBatch) -> dict[str, list | np.ndarray]:
     """Convert a RecordBatch to a dictionary of lists, unlike pa.lib.RecordBatch.to_pydict,
     this function will not convert numpy arrays to lists, and will preserve the original numpy dtype.
     """
-    out: dict[str, Union[list, np.ndarray]] = {}
+    out: dict[str, list | np.ndarray] = {}
     for k, name in enumerate(batch.schema.names):
         col = batch.column(k)
         if isinstance(col.type, pa.FixedShapeTensorType):
@@ -105,7 +105,7 @@ def to_pydict(batch: Union[pa.Table, pa.RecordBatch]) -> dict[str, Union[list, n
     return out
 
 
-def iter_tuples(batch: Union[pa.Table, pa.RecordBatch]) -> Iterator[dict[str, Any]]:
+def iter_tuples(batch: pa.Table | pa.RecordBatch) -> Iterator[dict[str, Any]]:
     """Convert a RecordBatch to an iterator of dictionaries. also works with pa.Table and pa.RowGroup"""
     pydict = to_pydict(batch)
     assert len(pydict) > 0, 'empty record batch'
@@ -145,7 +145,7 @@ def _ar_val_to_pxt_val(val: Any, pxt_type: ts.ColumnType) -> Any:
 
 
 def iter_tuples2(
-    batch: Union[pa.Table, pa.RecordBatch], col_mapping: Optional[dict[str, str]], schema: dict[str, ts.ColumnType]
+    batch: pa.Table | pa.RecordBatch, col_mapping: Optional[dict[str, str]], schema: dict[str, ts.ColumnType]
 ) -> Iterator[dict[str, Any]]:
     """Convert a RecordBatch to an iterator of dictionaries. also works with pa.Table and pa.RowGroup"""
     pydict = to_pydict(batch)

pixeltable/utils/description_helper.py

@@ -1,5 +1,5 @@
 import dataclasses
-from typing import Optional, Union
+from typing import Optional
 
 import pandas as pd
 from pandas.io.formats.style import Styler
@@ -7,7 +7,7 @@ from pandas.io.formats.style import Styler
 
 @dataclasses.dataclass
 class _Descriptor:
-    body: Union[str, pd.DataFrame]
+    body: str | pd.DataFrame
     # The remaining fields only affect the behavior if `body` is a pd.DataFrame.
     show_index: bool
     show_header: bool
@@ -33,7 +33,7 @@ class DescriptionHelper:
 
     def append(
         self,
-        descriptor: Union[str, pd.DataFrame],
+        descriptor: str | pd.DataFrame,
         show_index: bool = False,
         show_header: bool = True,
         styler: Optional[Styler] = None,

pixeltable/utils/iceberg.py

@@ -1,10 +1,9 @@
 from pathlib import Path
-from typing import Union
 
 from pyiceberg.catalog.sql import SqlCatalog
 
 
-def sqlite_catalog(warehouse_path: Union[str, Path], name: str = 'pixeltable') -> SqlCatalog:
+def sqlite_catalog(warehouse_path: str | Path, name: str = 'pixeltable') -> SqlCatalog:
     """
     Instantiate a sqlite Iceberg catalog at the specified path. If no catalog exists, one will be created.
     """

{pixeltable-0.4.5.dist-info → pixeltable-0.4.7.dist-info}/METADATA

@@ -1,12 +1,14 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: pixeltable
-Version: 0.4.5
+Version: 0.4.7
 Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
-License: Apache-2.0
-Keywords: data-science,machine-learning,database,ai,computer-vision,chatbot,ml,artificial-intelligence,feature-engineering,multimodal,mlops,feature-store,vector-database,llm,genai
-Author: Pixeltable, Inc.
-Author-email: contact@pixeltable.com>
-Requires-Python: >=3.10
+Project-URL: homepage, https://pixeltable.com/
+Project-URL: repository, https://github.com/pixeltable/pixeltable
+Project-URL: documentation, https://docs.pixeltable.com/
+Author-email: "Pixeltable, Inc." <contact@pixeltable.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: ai,artificial-intelligence,chatbot,computer-vision,data-science,database,feature-engineering,feature-store,genai,llm,machine-learning,ml,mlops,multimodal,vector-database
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: Apache Software License
@@ -20,39 +22,37 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Database
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Dist: av (>=10.0.0)
-Requires-Dist: beautifulsoup4 (>=4.0.0)
-Requires-Dist: cloudpickle (>=2.2.1)
-Requires-Dist: ftfy (>=6.2.0)
-Requires-Dist: httpcore (>=1.0.3)
-Requires-Dist: httpx (>=0.27)
-Requires-Dist: jinja2 (>=3.1.3)
-Requires-Dist: jmespath (>=1.0.1)
-Requires-Dist: jsonschema (>=4.1.0)
-Requires-Dist: lxml (>=5.0)
-Requires-Dist: more-itertools (>=10.2)
-Requires-Dist: nest_asyncio (>=1.5)
-Requires-Dist: numpy (>=1.25)
-Requires-Dist: pandas (>=2.0,<3.0)
-Requires-Dist: pgvector (>=0.2.1)
-Requires-Dist: pillow (>=9.3.0)
-Requires-Dist: pillow-heif (>=0.15.0)
-Requires-Dist: pixeltable-pgserver (==0.3.1)
-Requires-Dist: psutil (>=5.9.5)
-Requires-Dist: psycopg[binary] (>=3.1.18)
-Requires-Dist: puremagic (>=1.20)
-Requires-Dist: pyarrow (>=13.0.0)
-Requires-Dist: pydantic (>=2.7.4)
-Requires-Dist: pymupdf (>=1.24.1)
-Requires-Dist: pyyaml (>=6.0.1)
-Requires-Dist: requests (>=2.31.0)
-Requires-Dist: sqlalchemy (>=2.0.23)
-Requires-Dist: tenacity (>=8.2)
-Requires-Dist: toml (>=0.10)
-Requires-Dist: tqdm (>=4.64)
-Project-URL: Documentation, https://docs.pixeltable.com/
-Project-URL: Homepage, https://pixeltable.com/
-Project-URL: Repository, https://github.com/pixeltable/pixeltable
+Requires-Python: >=3.10
+Requires-Dist: av>=10.0.0
+Requires-Dist: beautifulsoup4>=4.10
+Requires-Dist: cloudpickle>=2.2.1
+Requires-Dist: ftfy>=6.2.0
+Requires-Dist: httpcore>=1.0.3
+Requires-Dist: httpx>=0.27
+Requires-Dist: jinja2>=3.1.3
+Requires-Dist: jmespath>=1.0.1
+Requires-Dist: jsonschema>=4.1.0
+Requires-Dist: lxml>=5.1
+Requires-Dist: more-itertools>=10.2
+Requires-Dist: nest-asyncio>=1.5
+Requires-Dist: numpy>=1.25
+Requires-Dist: pandas>=2.0
+Requires-Dist: pgvector>=0.2.1
+Requires-Dist: pillow-heif>=0.15.0
+Requires-Dist: pillow>=9.3.0
+Requires-Dist: pixeltable-pgserver==0.3.1
+Requires-Dist: psutil>=5.9.5
+Requires-Dist: psycopg[binary]>=3.1.18
+Requires-Dist: puremagic>=1.20
+Requires-Dist: pyarrow>=13.0.0
+Requires-Dist: pydantic>=2.7.4
+Requires-Dist: pymupdf>=1.24.1
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: requests>=2.31.0
+Requires-Dist: sqlalchemy>=2.0.23
+Requires-Dist: tenacity>=8.2
+Requires-Dist: toml>=0.10
+Requires-Dist: tqdm>=4.64
 Description-Content-Type: text/markdown
 
 <div align="center">
@@ -62,6 +62,8 @@ Description-Content-Type: text/markdown
 
 <h2>Declarative Data Infrastructure for Multimodal AI Apps</h2>
 
+Pixeltable is the only Python library providing incremental storage, transformation, indexing, and orchestration of multimodal data.
+
 [![License](https://img.shields.io/badge/License-Apache%202.0-0530AD.svg)](https://opensource.org/licenses/Apache-2.0)
 ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pixeltable?logo=python&logoColor=white&)
 ![Platform Support](https://img.shields.io/badge/platform-Linux%20%7C%20macOS%20%7C%20Windows-E5DDD4)
@@ -82,31 +84,68 @@ Description-Content-Type: text/markdown
 
 ---
 
-Pixeltable is the only Python framework that provides incremental storage, transformation, indexing, and orchestration of your multimodal data.
+## 💾 Installation
 
-## 😩 Maintaining Production-Ready Multimodal AI Apps is Still Too Hard
+```python
+pip install pixeltable
+```
 
-Building robust AI applications, especially [multimodal](https://docs.pixeltable.com/docs/datastore/bringing-data) ones, requires stitching together numerous tools:
-*   ETL pipelines for data loading and transformation.
-*   Vector databases for semantic search.
-*   Feature stores for ML models.
-*   Orchestrators for scheduling.
-*   Model serving infrastructure for inference.
-*   Separate systems for parallelization, caching, versioning, and lineage tracking.
+**Pixeltable unifies multimodal data storage, retrieval and orchestration.** It stores metadata and computed results persistently, typically in a `.pixeltable` directory in your workspace. See [configuration](https://docs.pixeltable.com/docs/overview/configuration) options for your setup. All media  (videos, images, audio) resides in ext. files, and Pixeltable stores references to those. Files can be local/remote (e.g. S3). For the latter, Pixeltable caches the [files locally on access](https://github.com/pixeltable/pixeltable/blob/main/docs/notebooks/feature-guides/working-with-external-files.ipynb).
 
-This complex "data plumbing" slows down development, increases costs, and makes applications brittle and hard to reproduce.
+https://github.com/user-attachments/assets/b50fd6df-5169-4881-9dbe-1b6e5d06cede
 
-## 💾 Installation
+## Quick Start
+
+With Pixeltable, you define your *entire* data processing and AI workflow declaratively using **[computed columns](https://docs.pixeltable.com/docs/datastore/computed-columns)** on **[tables](https://docs.pixeltable.com/docs/datastore/tables-and-operations)**.
 
 ```python
-pip install pixeltable
-```
 
-**Pixeltable is a database.** It stores metadata and computed results persistently, typically in a `.pixeltable` directory in your workspace. See [configuration](https://docs.pixeltable.com/docs/overview/configuration) options for your setup.
+# Installation
+pip install -qU torch transformers openai pixeltable
+
+# Basic setup
+import pixeltable as pxt
+
+# Table with multimodal column types (Image, Video, Audio, Document)
+t = pxt.create_table('images', {'input_image': pxt.Image})
+
+# Computed columns: define transformation logic once, runs on all data
+from pixeltable.functions import huggingface
 
-## ✨ What is Pixeltable?
+# Object detection with automatic model management
+t.add_computed_column(
+    detections=huggingface.detr_for_object_detection(
+        t.input_image, 
+        model_id='facebook/detr-resnet-50'
+    )
+)
+
+# Extract specific fields from detection results
+t.add_computed_column(detections_text=t.detections.label_text)
+
+# OpenAI Vision API integration with built-in rate limiting and async managemennt
+from pixeltable.functions import openai         
+
+t.add_computed_column(
+    vision=openai.vision(
+        prompt="Describe what's in this image.",
+        image=t.input_image,
+        model='gpt-4o-mini'
+    )
+)
+
+# Insert data - automatically triggers computation of all computed columns
+t.insert(input_image='https://raw.github.com/pixeltable/pixeltable/release/docs/resources/images/000000000025.jpg')
+
+# Query - All data, metadata, and computed results are persistently stored
+results = t.select(
+    t.input_image,
+    t.detections_text,
+    t.vision
+).collect()
+```
 
-With Pixeltable, you define your *entire* data processing and AI workflow declaratively using **[computed columns](https://docs.pixeltable.com/docs/datastore/computed-columns)** on **[tables](https://docs.pixeltable.com/docs/datastore/tables-and-operations)**. Pixeltable's engine then automatically handles:
+## ✨ What Happened?
 
 *   **Data Ingestion & Storage:** References [files](https://docs.pixeltable.com/docs/datastore/bringing-data) (images, videos, audio, docs) in place, handles structured data.
 *   **Transformation & Processing:** Applies *any* Python function ([UDFs](https://docs.pixeltable.com/docs/datastore/custom-functions)) or built-in operations ([chunking, frame extraction](https://docs.pixeltable.com/docs/datastore/iterators)) automatically.
@@ -118,7 +157,7 @@ With Pixeltable, you define your *entire* data processing and AI workflow declar
 **Focus on your application logic, not the infrastructure.**
 
 
-## 🚀 Key Features
+## ⚖️ Key Principles
 
 * **[Unified Multimodal Interface:](https://docs.pixeltable.com/docs/datastore/tables-and-operations)** `pxt.Image`, `pxt.Video`, `pxt.Audio`, `pxt.Document`, etc. – manage diverse data consistently.
   ```python
@@ -416,12 +455,24 @@ Explore Pixeltable's capabilities interactively:
 | Object Detection | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/notebooks/use-cases/object-detection-in-videos.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | Image/Text Search | <a target="_blank" href="https://github.com/pixeltable/pixeltable/tree/main/docs/sample-apps/text-and-image-similarity-search-nextjs-fastapi">  <img src="https://img.shields.io/badge/🖥️%20App-black.svg" alt="GitHub App"/> |
 | Audio Transcription | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/notebooks/use-cases/audio-transcriptions.ipynb">  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> | Discord Bot | <a target="_blank" href="https://github.com/pixeltable/pixeltable/blob/main/docs/sample-apps/context-aware-discord-bot"> <img src="https://img.shields.io/badge/%F0%9F%92%AC%20Bot-%235865F2.svg" alt="GitHub App"/></a> |
 
+## 🚨 Maintaining Production-Ready Multimodal AI Apps is Still Too Hard
+
+Building robust AI applications, especially [multimodal](https://docs.pixeltable.com/docs/datastore/bringing-data) ones, requires stitching together numerous tools:
+*   ETL pipelines for data loading and transformation.
+*   Vector databases for semantic search.
+*   Feature stores for ML models.
+*   Orchestrators for scheduling.
+*   Model serving infrastructure for inference.
+*   Separate systems for parallelization, caching, versioning, and lineage tracking.
+
+This complex "data plumbing" slows down development, increases costs, and makes applications brittle and hard to reproduce.
+
 ## 🔮 Roadmap (2025)
 
 ### Cloud Infrastructure and Deployment
 We're working on a hosted Pixeltable service that will:
 
-- Enable Multimodal Data Sharing of Pixeltable Tables and Views
+- Enable Multimodal Data Sharing of Pixeltable Tables and Views | [Waitlist](https://www.pixeltable.com/waitlist)
 - Provide a persistent cloud instance
 - Turn Pixeltable workflows (Tables, Queries, UDFs) into API endpoints/[MCP Servers](https://github.com/pixeltable/pixeltable-mcp-server)
 
@@ -432,4 +483,3 @@ We love contributions! Whether it's reporting bugs, suggesting features, improvi
 ## 🏢 License
 
 Pixeltable is licensed under the Apache 2.0 License.
-