pixeltable 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

pixeltable/utils/parquet.py

@@ -1,14 +1,24 @@
+import io
 import json
+import logging
+from collections import deque
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import numpy as np
 import PIL.Image
-import io
-import pyarrow.parquet as pq
 import pyarrow as pa
-import numpy as np
-from pathlib import Path
-from collections import deque
+import pyarrow.parquet
 
-from typing import List, Tuple, Any, Dict
+import pixeltable.type_system as ts
+from pixeltable.utils.arrow import iter_tuples, to_arrow_schema, to_pixeltable_schema
 from pixeltable.utils.transactional_directory import transactional_directory
+import pixeltable.exceptions as exc
+
+import random
+
+_logger = logging.getLogger(__name__)
+
 
 def _write_batch(value_batch : Dict[str, deque], schema : pa.Schema, output_path : Path) -> None:
     pydict = {}
@@ -20,18 +30,18 @@ def _write_batch(value_batch : Dict[str, deque], schema : pa.Schema, output_path
             pydict[field.name] = value_batch[field.name]
 
     tab = pa.Table.from_pydict(pydict, schema=schema)
-    pq.write_table(tab, output_path)
+    pa.parquet.write_table(tab, output_path)
 
 def save_parquet(df: 'pixeltable.DataFrame', dest_path: Path, partition_size_bytes: int = 100_000_000) -> None:
     """
         Internal method to stream dataframe data to parquet format.
         Does not materialize the dataset to memory.
 
-        It preserves pixeltable type metadata in a json file, which would otherwise 
+        It preserves pixeltable type metadata in a json file, which would otherwise
         not be available in the parquet format.
 
         Images are stored inline in a compressed format in their parquet file.
-    
+
         Args:
             df : dataframe to save.
             dest_path : path to directory to save the parquet files to.
@@ -39,10 +49,9 @@ def save_parquet(df: 'pixeltable.DataFrame', dest_path: Path, partition_size_byt
     """
     column_names = df.get_column_names()
     column_types = df.get_column_types()
-
     type_dict = {k: v.as_dict() for k, v in zip(column_names, column_types)}
+    arrow_schema = to_arrow_schema(dict(zip(column_names, column_types)))
 
-    arrow_schema = pa.schema([pa.field(name, column_types[i].to_arrow_type()) for i, name in enumerate(column_names)])
     # store the changes atomically
     with transactional_directory(dest_path) as temp_path:
         # dump metadata json file so we can inspect what was the source of the parquet file later on.
@@ -65,8 +74,9 @@ def save_parquet(df: 'pixeltable.DataFrame', dest_path: Path, partition_size_byt
                     # images get inlined into the parquet file
                     if data_row.file_paths is not None and data_row.file_paths[e.slot_idx] is not None:
                         # if there is a file, read directly to preserve information
-                        val = open(data_row.file_paths[e.slot_idx], 'rb').read()
-                    elif isinstance(val, PIL.Image.Image): 
+                        with open(data_row.file_paths[e.slot_idx], 'rb') as f:
+                            val = f.read()
+                    elif isinstance(val, PIL.Image.Image):
                         # if no file available, eg. bc it is computed, convert to png
                         buf = io.BytesIO()
                         val.save(buf, format='PNG')
@@ -109,18 +119,49 @@ def save_parquet(df: 'pixeltable.DataFrame', dest_path: Path, partition_size_byt
 
         _write_batch(current_value_batch, arrow_schema, temp_path / f'part-{batch_num:05d}.parquet')
 
-def get_part_metadata(path : Path) -> List[Tuple[str, int]]:
-    """
-        Args:
-            path: path to directory containing parquet files.
-        Returns:
-            A list of (file_name, num_rows) tuples for the parquet files in file name order.
-    """
-    parts = sorted([f for f in path.iterdir() if f.suffix == '.parquet'])
-    rows_per_file = {}
-
-    for part in parts:
-        parquet_file = pq.ParquetFile(str(part))
-        rows_per_file[part] = parquet_file.metadata.num_rows
 
-    return [(file, num_rows) for file, num_rows in rows_per_file.items()]
+def parquet_schema_to_pixeltable_schema(parquet_path: str) -> Dict[str, Optional[ts.ColumnType]]:
+    """Generate a default pixeltable schema for the given parquet file. Returns None for unknown types."""
+
+    input_path = Path(parquet_path).expanduser()
+    parquet_dataset = pa.parquet.ParquetDataset(input_path)
+    return to_pixeltable_schema(parquet_dataset.schema)
+
+
+def import_parquet(
+    cl: 'pixeltable.Client',
+    table_path: str,
+    *,
+    parquet_path: str,
+    schema_override: Optional[Dict[str, ts.ColumnType]],
+    **kwargs,
+) -> 'catalog.InsertableTable':
+    """See `pixeltable.Client.import_parquet` for documentation"""
+    input_path = Path(parquet_path).expanduser()
+    parquet_dataset = pa.parquet.ParquetDataset(input_path)
+
+    schema = parquet_schema_to_pixeltable_schema(parquet_path)
+    if schema_override is None:
+        schema_override = {}
+
+    schema.update(schema_override)
+    for k, v in schema.items():
+        if v is None:
+            raise exc.Error(f'Could not infer pixeltable type for column {k} from parquet file')
+
+    if table_path in cl.list_tables():
+        raise exc.Error(f'Table {table_path} already exists')
+
+    try:
+        tmp_name = f'{table_path}_tmp_{random.randint(0, 100000000)}'
+        tab = cl.create_table(tmp_name, schema, **kwargs)
+        for fragment in parquet_dataset.fragments:
+            for batch in fragment.to_batches():
+                dict_batch = list(iter_tuples(batch))
+                tab.insert(dict_batch)
+    except Exception as e:
+        _logger.error(f'Error while inserting Parquet file into table: {e}')
+        raise e
+
+    cl.move(tmp_name, table_path)
+    return cl.get_table(table_path)

pixeltable/utils/pytorch.py

@@ -1,25 +1,17 @@
 import io
 import pyarrow as pa
-import pyarrow.parquet as pq
+import pyarrow.parquet
 import torch
 import torch.utils.data
-import math
 from pathlib import Path
 import PIL.Image
 import json
-from typing import Dict, Tuple, Generator, Any
+from typing import Dict, Iterator, Any
 import datetime
 
 from pixeltable.type_system import ColumnType
-from pixeltable.utils.parquet import get_part_metadata
 import numpy as np
 
-def _cumsum(lst):
-    acc = [0]
-    for x in lst:
-        acc.append(acc[-1] + x)
-    return acc
-
 class PixeltablePytorchDataset(torch.utils.data.IterableDataset):
     """
     PyTorch dataset interface for pixeltable data.
@@ -47,24 +39,7 @@ class PixeltablePytorchDataset(torch.utils.data.IterableDataset):
         with column_type_path.open() as f:
             column_types = json.load(f)
         self.column_types = {k: ColumnType.from_dict(v) for k, v in column_types.items()}
-        self.part_metadata = get_part_metadata(self.path)
-        self._totals = _cumsum([x[1] for x in self.part_metadata])
-
-    def _get_start_position(self, row_start: int) -> Tuple[int, int]:
-        """
-        Returns the starting parquet file and row within that file for a given 'global' row number.
-        based on the individual sizes of each part
-        """
-        assert row_start >= self._totals[0]
-        assert row_start < self._totals[-1]
-
-        prev_acc = 0
-        for i, acc in enumerate(self._totals[1:], start=1):
-            if acc > row_start:
-                return (i - 1, row_start - prev_acc)
-            prev_acc = acc
-
-        assert False, "unreachable"
+        self.part_metadata = pa.parquet.ParquetDataset(path).files
 
     def _unmarshall(self, k: str, v: Any) -> Any:
         if self.column_types[k].is_image_type():
@@ -88,13 +63,10 @@ class PixeltablePytorchDataset(torch.utils.data.IterableDataset):
             return json.loads(v)
         elif self.column_types[k].is_array_type():
             assert isinstance(v, np.ndarray)
-            # WRITEABLE is required for torch collate function, or undefined behavior
             if not v.flags["WRITEABLE"]:
-                vout = v.copy()
-                assert vout.flags["WRITEABLE"]
-                return vout
-            else:
-                return v
+                v = v.copy()
+            assert v.flags["WRITEABLE"]
+            return v
         elif self.column_types[k].is_timestamp_type():
             # pytorch default collation only supports numeric types
             assert isinstance(v, datetime.datetime)
@@ -103,70 +75,17 @@ class PixeltablePytorchDataset(torch.utils.data.IterableDataset):
             assert not isinstance(v, np.ndarray) # all array outputs should be handled above
             return v
 
-    def __iter__(self) -> Generator[Dict[str, Any], None, None]:
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
+        import pixeltable.utils.arrow as arrow
         worker_info = torch.utils.data.get_worker_info()
-        if worker_info is None:  # single-process data loading, return the full iterator
-            start_row = 0
-            end_row = self._totals[-1]
-        else:  # in a worker process
-            num_workers = [
-                math.floor(self._totals[-1] / float(worker_info.num_workers))
-                for _ in range(worker_info.num_workers)
-            ]
-            assert self._totals[-1] - sum(num_workers) < worker_info.num_workers
-            for i in range(self._totals[-1] - sum(num_workers)):
-                num_workers[i] += 1
-
-            assert sum(num_workers) == self._totals[-1]
-            start_rows = _cumsum(num_workers)
-            start_row = start_rows[worker_info.id]
-            end_row = start_rows[worker_info.id + 1]
 
-        if start_row == end_row:
-            return iter([])  # type: ignore
+        if worker_info is None:
+            part_list = range(len(self.part_metadata))
         else:
-            return self._iter_range(start_row, end_row)
-
-    def _iter_range(self, start_row : int, end_row : int) -> Generator[Dict[str, Any], None, None]:
-        (part_no, iter_start) = self._get_start_position(start_row)
-        total = end_row - start_row
-
-        acc = 0
-        part_pos = part_no
-        iter_pos = iter_start
-
-        def _to_column_dict(tab : pa.Table) -> Dict[str, Any]:
-            column_dict = {}
-            for k in tab.column_names:
-                if self.column_types[k].is_array_type():
-                    # treat array columns as numpy arrays to easily preserve numpy type
-                    column_dict[k] = tab.column(k).to_numpy()
-                else:
-                    # for the rest, use pydict to preserve python types
-                    column_dict[k] = tab.column(k).to_pylist()
-            return column_dict
-
-        tab: pa.Table = pq.read_table(self.path / self.part_metadata[part_no][0])
-        column_dict = _to_column_dict(tab)
-        assert tab.num_rows == self.part_metadata[part_no][1]
-
-        while True:
-            while iter_pos < tab.num_rows and acc < total:
-                next_tup = {}
-                for col_name, col_vals in column_dict.items():
-                    raw_val = col_vals[iter_pos]
-                    next_tup[col_name] = self._unmarshall(col_name, raw_val)
-
-                yield next_tup
-                acc += 1
-                iter_pos += 1
-
-            if acc == total:
-                break
+            part_list = [ i for i in part_list if (i % worker_info.num_workers) == worker_info.id ]
 
-            # move on to next part
-            part_pos += 1
-            assert part_pos < len(self.part_metadata)
-            iter_pos = 0
-            tab = pq.read_table(self.path / self.part_metadata[part_pos][0])
-            column_dict = _to_column_dict(tab)
+        for part_no in part_list:
+            pqf = pa.parquet.ParquetFile(self.part_metadata[part_no])
+            for batch in pqf.iter_batches():
+                for tup in arrow.iter_tuples(batch):
+                    yield {k: self._unmarshall(k, v) for k, v in tup.items()}

{pixeltable-0.2.3.dist-info → pixeltable-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.3
+Version: 0.2.5
 Summary: Pixeltable: The Multimodal AI Data Plane
 Author: Marcel Kornacker
 Author-email: marcelk@gmail.com
@@ -15,12 +15,12 @@ Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
 Requires-Dist: cloudpickle (>=2.2.1,<3.0.0)
 Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
 Requires-Dist: jmespath (>=1.0.1,<2.0.0)
-Requires-Dist: numpy (>=1.26,<2.0)
+Requires-Dist: numpy (>=1.26)
 Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
 Requires-Dist: pandas (>=2.0,<3.0)
-Requires-Dist: pgserver (==0.0.9)
+Requires-Dist: pgserver (==0.1.2)
 Requires-Dist: pgvector (>=0.2.1,<0.3.0)
-Requires-Dist: pillow (>=9.4.0,<10.0.0)
+Requires-Dist: pillow (>=10.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: psycopg2-binary (>=2.9.5,<3.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
@@ -28,38 +28,44 @@ Requires-Dist: regex (>=2022.10.31,<2023.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: sqlalchemy-utils (>=0.41.1,<0.42.0)
 Requires-Dist: sqlalchemy[mypy] (>=2.0.23,<3.0.0)
+Requires-Dist: tenacity (>=8.2,<9.0)
 Requires-Dist: tqdm (>=4.64.1,<5.0.0)
 Description-Content-Type: text/markdown
 
+<div align="center">
 <img src="docs/pixeltable-banner.png" width="45%"/>
 
-# Pixeltable: The Multimodal AI Data Plane
+# Unifying Data, Models, and Orchestration for AI Products
 
 [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 &nbsp;&nbsp;
 ![pytest status](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml/badge.svg)
 
-Pixeltable is a Python library that lets AI engineers and data scientists focus on
-exploration, modeling, and app development without having to deal with the customary
-data plumbing.
+[Installation](https://pixeltable.github.io/pixeltable/getting-started/) | [Documentation](https://pixeltable.github.io/pixeltable/)
+</div>
 
-**Pixeltable redefines data infrastructure and workflow orchestration for AI development.**
-It brings together data storage, versioning, and indexing with orchestration and model
-versioning under a declarative table interface, with transformations, model inference,
-and custom logic represented as computed columns.
+Pixeltable is a Python library that lets AI engineers and data scientists focus on exploration, modeling, and app development without dealing with the customary data plumbing.
 
-## Quick Start
+## What problems does Pixeltable solve?
+
+Today’s solutions for AI app development require extensive custom coding and infrastructure
+plumbing. Tracking lineage and versions between and across data transformations, models, and
+deployment is cumbersome. Pixeltable is a replacement for traditional data plumbing, providing
+a unified plane for data, models, and orchestration. It removes the data plumbing overhead in
+building and productionizing AI applications.
+
+## ⚡Quick Start
+Learn the basics of Pixeltable through interactive examples. View the notebooks on Google Colab or Kaggle, for free.
 
-If you just want to play around with Pixeltable to see what it's capable of, the easiest way is to run
-the Pixeltable Basics tutorial in colab:
+### Pixeltable Basics
+In this tutorial, we'll survey how to create tables, populate them with data, and enhance them with built-in and user-defined transformations and AI operations.
 
-<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb">
-  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
-</a>
+[![Kaggle](https://kaggle.com/static/images/open-in-kaggle.svg)](https://kaggle.com/kernels/welcome?src=https://github.com/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb)&nbsp;&nbsp;
+<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a>
 
-## Installation
 
-Pixeltable works with Python 3.9, 3.10, or 3.11 running on Linux or MacOS.
+## 💾 Installation
+Pixeltable works with Python 3.9, 3.10, 3.11, or 3.12 running on Linux, MacOS, or Windows.
 
 ```
 pip install pixeltable
@@ -78,14 +84,6 @@ guide. Then, check out the
 [Pixeltable Basics](https://pixeltable.github.io/pixeltable/tutorials/pixeltable-basics/)
 tutorial for a tour of its most important features.
 
-## What problems does Pixeltable solve?
-
-Today’s solutions for AI app development require extensive custom coding and infrastructure
-plumbing. Tracking lineage and versions between and across data transformations, models, and
-deployment is cumbersome. Pixeltable is a replacement for traditional data plumbing, providing
-a unified plane for data, models, and orchestration. It removes the data plumbing overhead in
-building and productionizing AI applications.
-
 ## Why should you use Pixeltable?
 
 - It gives you transparency and reproducibility
@@ -119,3 +117,12 @@ get cost projections before adding new data and new augmentations.
 * Rely on Pixeltable's automatic versioning and snapshot functionality to protect against regressions
 and to ensure reproducibility.
 
+## Contributions & Feedback
+
+Are you experiencing issues or bugs with Pixeltable? File an [Issue](https://github.com/pixeltable/pixeltable/issues).
+</br>Do you want to contribute? Feel free to open a [PR](https://github.com/pixeltable/pixeltable/pulls).
+
+## :classical_building: License
+
+This library is licensed under the Apache 2.0 License.
+

{pixeltable-0.2.3.dist-info → pixeltable-0.2.5.dist-info}/RECORD

@@ -1,31 +1,31 @@
 pixeltable/__init__.py,sha256=enh38lhZ_38Ys3rCy_XC_BlPNPK9H2CKV4KePaqnV4E,959
 pixeltable/catalog/__init__.py,sha256=E41bxaPeQIcgRYzTWc2vkDOboQhRymrJf4IcHQO7o_8,453
 pixeltable/catalog/catalog.py,sha256=0TYWB1R6YBp9qCkWF7kCcX2Yw70UuburKKIemv5L1Js,7908
-pixeltable/catalog/column.py,sha256=7PU5yECislpR_PNZTT3Z5sZiE34DnI6SR3WDsI-WgCg,8661
+pixeltable/catalog/column.py,sha256=2K3rUrcfdF-eXiRB9WhIrFVj1XmjrK51lQA616OYfLQ,7887
 pixeltable/catalog/dir.py,sha256=pG1nMpG123POo6WMSHhAmnwXOQ26uUJfUcbzL-Jb4ws,919
 pixeltable/catalog/globals.py,sha256=yLEGNbsSnLzjWNHVJacfjA9hbw13Q6QXLOSCRmdTlq0,943
-pixeltable/catalog/insertable_table.py,sha256=1JGZqmx5tKbm_UmJLIalntFDs81s-mvTv0gOqk98b3Q,8280
+pixeltable/catalog/insertable_table.py,sha256=B9QLN2eoCTjwqH5KLBRy3YQ7_l1nd8hqJd83eIdE1Qc,8303
 pixeltable/catalog/named_function.py,sha256=a96gnKtx-nz5_MzDIiD4t4Hxqdjkg9ZtijRQxvWA5WQ,1147
 pixeltable/catalog/path.py,sha256=QgccEi_QOfaKt8YsR2zLtd_z7z7QQkU_1kprJFi2SPQ,1677
 pixeltable/catalog/path_dict.py,sha256=xfvxg1Ze5jZCARUGASF2DRbQPh7pRVTYhuJ_u82gYUo,5941
 pixeltable/catalog/schema_object.py,sha256=-UxmPLbuEBqJiJi_GGRbFdr7arAFxTqs4bt6TFmSt3M,1059
-pixeltable/catalog/table.py,sha256=CJQc2NnebfL1ekMcaZhJEZ0CI408-F7R90_czjDKy-g,25499
-pixeltable/catalog/table_version.py,sha256=Q1NSaeLmFw7gYgARyyW6JJ79GRhjtp0_Q410KH27e1Q,35680
-pixeltable/catalog/table_version_path.py,sha256=kn5T0m5XBqphlDqCKsqD-s159diCjKvrSrtzYZHwKuw,5461
+pixeltable/catalog/table.py,sha256=6OKvbrFFVu_xoKs68QC7mt-N99rKrBPwos4Bozs9rvg,30675
+pixeltable/catalog/table_version.py,sha256=1KBk1A_iqXtQbUs3P-5bxedOccj2kcryMLmlwxZcLF4,48818
+pixeltable/catalog/table_version_path.py,sha256=2Ofzd0n36flcNm86KWwIWDBAfgnV5Z-FxAHdMSPgMLc,5482
 pixeltable/catalog/view.py,sha256=BIL3s4DV3tWbOcqtqnhn46B2UvLaBhppfJUlNEt5nec,9734
-pixeltable/client.py,sha256=HiqGNRQoyBPXF2rAhTJFE3ZSUUaSLpGo28pLZjubh8Y,20548
-pixeltable/dataframe.py,sha256=vBc4T-QU-lTTtnljqDbBFTHwm2CecLHIjJumEv8lQ0s,28339
-pixeltable/env.py,sha256=sY-VUY9WFs4li7CidyRXubfjmBh3cbR-cFj-wZr4xY0,16398
+pixeltable/client.py,sha256=Ypah43hdEByGZ7WGS5-2LWKnsWAuhofq84TMRZD90uQ,23442
+pixeltable/dataframe.py,sha256=GyeutsjdKSAzRorzIsSZ_hElGJM7bRSRv28ynB4qFGY,30014
+pixeltable/env.py,sha256=Qk_Br2gJKDykvz0AoWJ45nDZTjhl-MStxVWWFI4pU3A,16303
 pixeltable/exceptions.py,sha256=MSP9zeL0AmXT93XqjdvgGN4rzno1_KRrGriq6hpemnw,376
 pixeltable/exec/__init__.py,sha256=FOQBSMQ3buGg5kRzymkfDCk5fnVfICn3LrsJtbyNC6E,412
 pixeltable/exec/aggregation_node.py,sha256=cf6rVAgrGh_uaMrCIgXJIwQTmbcboJlnrH_MmPIQSd0,3321
-pixeltable/exec/cache_prefetch_node.py,sha256=zDYYc0vmdLzB82YqgJRdCMriaMHX07LS3o04G2U3NmI,5212
+pixeltable/exec/cache_prefetch_node.py,sha256=d5pEuR6AtJQkEVy9X3XeYFI_q0szMtoNAH96vYdtBE0,5241
 pixeltable/exec/component_iteration_node.py,sha256=Uz6zEeaJMcbvF3S0W0qmLI_uWsZsaSspHKNzuAMrasg,4069
 pixeltable/exec/data_row_batch.py,sha256=ZaS_ciyYhoP50Aoszhgtnh_yDRR6z9H43crnaJdP-xA,3505
 pixeltable/exec/exec_context.py,sha256=E82Q2bJMJ1ulud5L5D9dh2Z8vEUQ659SgT614YKDO34,924
 pixeltable/exec/exec_node.py,sha256=Hji5NCPHfa50IWyjladXrBm4I0zseV7AV4cVdx0Q8Ew,2170
 pixeltable/exec/expr_eval_node.py,sha256=mqACyEy48fsiWkbHSkkMc8gG2nL-WT6eJSG3nltM85c,10810
-pixeltable/exec/in_memory_data_node.py,sha256=x2Jn_AGx52YcDPORH8sZZRD0s64Ng1NvfMpqAW-bVwY,2949
+pixeltable/exec/in_memory_data_node.py,sha256=SNM2AbMQSjmGDWMNJUf_5MmlXWE3P80lsuUjNfzQckA,3171
 pixeltable/exec/media_validation_node.py,sha256=OKfRyKpcn7AZdACy_HD4NsDC87ZfNFs1tdrQz2NiIVw,1514
 pixeltable/exec/sql_scan_node.py,sha256=zKJ2_iM0ah748Vz8jD0Lkm7TRyUChIejnGUPNmYPgbs,10318
 pixeltable/exprs/__init__.py,sha256=EkpjeEW-8rriE9hVb9PdiknHQ3-Y8jPp37yJ-NC_oWA,935
@@ -33,9 +33,9 @@ pixeltable/exprs/arithmetic_expr.py,sha256=sWBYCBKI6IHj9ASwDcm2BlkQ5gleVtKtmpiPv
 pixeltable/exprs/array_slice.py,sha256=VmWc6iFusrM85MjyEBBCfXG1Jnt8-Gr6-J88BXxNoOE,2131
 pixeltable/exprs/column_property_ref.py,sha256=0PHiBys0fxe2LgjaMId5UHob4E-ZggyPLnnW41RgA0E,2706
 pixeltable/exprs/column_ref.py,sha256=5-mbWV8-cWWu0ynxPSQ0L8oLewDrLp17xh8DOCmQ36s,4794
-pixeltable/exprs/comparison.py,sha256=jwwvKG0nLOHiNyjZm8AWKn59qBtTYH98vTF_hhwn0Og,3000
+pixeltable/exprs/comparison.py,sha256=rAlGUF0AuzkYGspewJPu-6aaQZa4dVMJYGbMwqKyBIc,2964
 pixeltable/exprs/compound_predicate.py,sha256=Gh22MKi625m5A_RunVRd-a1XFi-fitikqBVz2VNXKrs,3830
-pixeltable/exprs/data_row.py,sha256=TRbC2lmPm-U_Az2edlVaHbaYn-dm0G3IQp6gQ6HH5DQ,8046
+pixeltable/exprs/data_row.py,sha256=2kGnZhDna4bkgzb2y9iDnkLFe8lXSk59QAf9zW2Z-Y0,8278
 pixeltable/exprs/expr.py,sha256=JyEmP8F9LXmxOrVswih1Aq0SrpGcboSRinOYIMWYfKM,23748
 pixeltable/exprs/expr_set.py,sha256=Q64Q2yI0CTq2Ma_E-BUYlMotSstVuMm4OFZnBCedHRk,1222
 pixeltable/exprs/function_call.py,sha256=QFzkb2gKjeALGTgCet-o0HETwTXoFBzsncbghgScmOE,16961
@@ -47,80 +47,93 @@ pixeltable/exprs/inline_dict.py,sha256=Lb3VS3Mkxb72mWL4qipOQjUgY8AMxk1POj225Xqjt
 pixeltable/exprs/is_null.py,sha256=nvpOXtQj1UeYJpkCWzbaGuQElzrA2HSG3XNQugOv-pw,1041
 pixeltable/exprs/json_mapper.py,sha256=I60VNgus64ai80gnFCIsRn0VRWYXMkqH5VNvnATsN9s,4559
 pixeltable/exprs/json_path.py,sha256=Wz_5zFsyc9TPhsSbsDjDmQ3Nb0uVIwMCx5nh-cQYBiE,6526
-pixeltable/exprs/literal.py,sha256=LOeD0aN_V7Kg332xzCHWUlzKPupXiptpU_rg-X0bEt0,1885
+pixeltable/exprs/literal.py,sha256=5NNza-WL1dd3hNznwwkr_yAcTGXSIRYUszGfy30lruI,2396
 pixeltable/exprs/object_ref.py,sha256=eTcx84aWRI59fIiGvbdv3_cfL0XW4xEFQ4lwpLpJkM8,1250
 pixeltable/exprs/predicate.py,sha256=OSDgjfSqiK7J_5GZMUXMvjfyomKEGi0JNxeB073SGXw,1859
-pixeltable/exprs/row_builder.py,sha256=sOW2zG6MwqOT8BFgmlveqIiRGPZinuK0_dY6IgmCWT0,16992
+pixeltable/exprs/row_builder.py,sha256=72bbPPynmzFrWuZoqsCCBmtWAgf3XFyKqUQaoMusmeo,15321
 pixeltable/exprs/rowid_ref.py,sha256=74w4rEy21YysTVbyKNc3op-pYFqDAx8VJdtl7ZPpxHs,4268
 pixeltable/exprs/type_cast.py,sha256=JMg8p1qYoFfiAXfJPSbTEnfrK7lRO_JMaqlPHOrhNQU,1793
 pixeltable/exprs/variable.py,sha256=Kg_O4ytcHYZFijIyMHYBJn063cTKU1-YE583FAz8Qaw,1361
+pixeltable/ext/__init__.py,sha256=0uugfuME1FybVo-MdxaVNGagRjhcvNTnv5MZUem6Cyo,269
+pixeltable/ext/functions/yolox.py,sha256=LwrOtXMT57AP6-IkmRZ_12yN5-EiFRpTuh4Sexm8x24,3131
 pixeltable/func/__init__.py,sha256=4qvDnK_S5yljwtIrxCkeQlz6vuTVfwSU4zl-MBT2TMU,457
-pixeltable/func/aggregate_function.py,sha256=E11lVKBR4MZQ6VT0G4oo5xWR907BqHqJog-widXcJt0,9177
+pixeltable/func/aggregate_function.py,sha256=OC5dmWKOjNHcgH5KrsxZpISoP5RFXfHFkzHQk8PnOUQ,9267
 pixeltable/func/batched_function.py,sha256=nrfmykJps4QkgzeC4ZroGy8mJ-BACNKbRJABHHG6-Ac,2350
 pixeltable/func/callable_function.py,sha256=WGUwjNEzXGDJ1jzdZ-mrn2cnfEHmFZIh_DRl_Yk89D0,2357
-pixeltable/func/expr_template_function.py,sha256=Nj-FDDozSuvcoQ7KsJcQrzzQn7foeCY7zSEFkusiLqQ,3392
+pixeltable/func/expr_template_function.py,sha256=aBIjO5ibd2t56lNKKEmv-YFVcxUW_jqHIiYaslG1u2A,3807
 pixeltable/func/function.py,sha256=NcGyKp7ds6bMzg1qzuaK8pBQH_DhknmSiZzsfXi-z2k,4023
 pixeltable/func/function_registry.py,sha256=1ibSQxEPm3Zd3r497vSlckQiDG9sfCnyJx3zcSm9t7c,11456
-pixeltable/func/globals.py,sha256=UP4hAkEVKD_3lr7l_VVUUCetJP85sydUYiVAU4m5-aw,1220
+pixeltable/func/globals.py,sha256=sEwn6lGgHMp6VQORb_P5qRd_-Q2_bUSqvqM9-XPN_ec,1483
 pixeltable/func/nos_function.py,sha256=HzIKK4XjTo1E6pML-EbhuX3u_LYibFWUuTkIxoIih7c,9650
-pixeltable/func/signature.py,sha256=5Xo3hIVSXoYtkU-BvUi21b6FghIQABm48UfVjQr9QYU,7107
-pixeltable/func/udf.py,sha256=XlB7dzOc98XinUrs4tMGUAetV0SmQ_Z2oKPDBsx46o0,6457
-pixeltable/functions/__init__.py,sha256=kFepRTpg-YOeeXG9076s9v-lm8ZUNqicaPUfX-CrW5c,3419
-pixeltable/functions/eval.py,sha256=qLc_55VTwi_IDnAJFq79Hy7FLi7Gb26DxdykYiBTMrM,8453
-pixeltable/functions/fireworks.py,sha256=7h1gxcGhG8jepS_Gi1nwp8zCw1gdRaTzufS6U08mmLc,1596
-pixeltable/functions/huggingface.py,sha256=Wg8AgCsoGFx8AX3AThko66L7uSg9cUEjL8_QxKoA4dg,4804
+pixeltable/func/signature.py,sha256=aQRb-znfblRlMlUN7YmbFHF6-PRAGbYRpqDNd7nawII,7343
+pixeltable/func/udf.py,sha256=DEaXgKrx-Nrszf9cSC_ql9GF016DRbKI5JXAHODUU0I,6711
+pixeltable/functions/__init__.py,sha256=uO-XB4QUbx3Jjs9GoaTXoJY2jn0AuXTL32YLkL_3_CI,3297
+pixeltable/functions/eval.py,sha256=_2FANDJqwtIDzTxtcKc0Yacf7b4LTAjyy2fPDw1FG_s,8404
+pixeltable/functions/fireworks.py,sha256=e_rCITg18yNndNI8TJPXRSN6DR0hYWT-_dUavoPuyfc,908
+pixeltable/functions/huggingface.py,sha256=_Z-MqqMRVDw1GOS5KonaeEmh5qsU-dfam9EODddiHw4,5693
 pixeltable/functions/image.py,sha256=xR_S_0BuX6Ycc5E366GpOfP0JptD7beQwHE_fLl8ZVM,431
-pixeltable/functions/openai.py,sha256=SkwUBnnTT6olQCMHzQGO0vpvApgAaBk_88W7YM8QQuo,2863
+pixeltable/functions/openai.py,sha256=o5-qh_T4JgIyMR8Ky1Y6HbK9yg88fJSkkQCIaLAf9Qg,6870
 pixeltable/functions/pil/image.py,sha256=8gItSXXuJaCkq9FHEJE9qFpRM3WAoa59x89Xa0DgksQ,6217
 pixeltable/functions/string.py,sha256=RYOgZwifjC943YloEMi3PdflnjFqOYB2FddrUvzgtXs,516
-pixeltable/functions/together.py,sha256=TGo72RJuHqofwJHSrHqsJoO6RQm6Iipqhy49V3JiBss,650
-pixeltable/functions/util.py,sha256=v094FgixHchzKMj4YvfHExDRGIVhCL2D-NL3pl-92oo,1596
+pixeltable/functions/together.py,sha256=sG23nLMScmp4wRA4K1EIJDY9peqE1IPgxlK3fhNrbgw,3423
+pixeltable/functions/util.py,sha256=djVqro_W5M_jUgYWzZZaXXH3lWaAWj6q-hrpzFl_Ko8,1860
 pixeltable/functions/video.py,sha256=WZF4G3tV-_LfRQHUinXe_rnu1-4N68Ht60JCR_s7Bew,2403
+pixeltable/index/__init__.py,sha256=tlJENOzEq6p_8xu-nX1mN4Zt9asw4481Znl5ZXYIKwc,72
+pixeltable/index/base.py,sha256=LwBXLOBWkQhAOSOo2gtgtgSJglnRUrf0RtG5JmcwwEQ,1362
+pixeltable/index/embedding_index.py,sha256=THFue7f4fKCUXYlxqqRGNcoc1o22GqYcYg6s5vvQFFg,4372
 pixeltable/iterators/__init__.py,sha256=sfsasCypAq5rNOTMlr4j2ROXxzdl4M8L2KvQIEbd0cQ,70
 pixeltable/iterators/base.py,sha256=sugU9mG19xf9I4c_lEQDI_xrXPLuyuIYlKc-9OiS1HQ,1545
 pixeltable/iterators/document.py,sha256=bWK-3oRD9Sbc5vIcxccMyFh8zTP40233NK3ch6uyLI0,13105
 pixeltable/iterators/video.py,sha256=mUdHev0f_XJIfHTB-mejjToURh10Ud7l096eZm2aPjM,3444
 pixeltable/metadata/__init__.py,sha256=6rLGFpWzQMCk0EcSKLp4Bnbk8tLK1cxuONBPqwDCMzU,2083
 pixeltable/metadata/converters/convert_10.py,sha256=0mSGCn7vqtef63riPi9msUaaUvsSQIj-NFj9QFDYPdA,733
-pixeltable/metadata/schema.py,sha256=oTxtFSvbmCEcr54F929_NYFVp76N8gpN4nHDr9VnA9M,7761
-pixeltable/plan.py,sha256=BQMCxOCy7lO4dqBCt5W9GUFdBWkuhr13-T8_2E97fUk,34710
-pixeltable/store.py,sha256=ICqfHiotyMyTi2gsQ-Qw-ed1ooNkAx0vgMSMri4xfqk,19511
-pixeltable/tests/conftest.py,sha256=2RyRWIW2Ky9VQ41Hj0ChNr_Oh5vTx4x95tUdJWX5ktE,6677
+pixeltable/metadata/schema.py,sha256=uuk3rzCpYr99PzEO1pIXe8nMaOoTJtwRfhnqgQ_MdDs,8335
+pixeltable/plan.py,sha256=sZe_K_5VUGISPIiRoogXhfYMSx6tU58YPGcx1HCKACo,33549
+pixeltable/store.py,sha256=IAurcG5vyryl4GMIl6nH0vM-R-IXTj5ZaczgtMKsEx0,19426
+pixeltable/tests/conftest.py,sha256=mVZmWdQWNRJPmeyYTkdE2AcOx861A-JsfaZkRmZ5_wE,6339
+pixeltable/tests/ext/test_yolox.py,sha256=hZRCq1bue3PO86zdBdfUa-3I8_pZ4i91J25cLPcmRX0,1085
+pixeltable/tests/functions/test_fireworks.py,sha256=d6-bgB4PUByJ_9pfn7_jcI8weMpUgYM1Bln0ohBu49I,1690
+pixeltable/tests/functions/test_functions.py,sha256=SRE4zRwfJ1Hetu8uMp5X-Iqg-pawn87q07JHa7w_8l0,2884
+pixeltable/tests/functions/test_huggingface.py,sha256=hmxM-2skPe1xu0uniI2N_IL8djDpBis86wdEh4Oc_84,7657
+pixeltable/tests/functions/test_openai.py,sha256=YT34H6knG-pSypGFxGGjqebjjcYF1Z7BDm1Vo9Ar970,8659
+pixeltable/tests/functions/test_together.py,sha256=sNySFn0tgWAVz99htLA3UvrKFoHevyKOfBQ6uJb4AFM,4812
 pixeltable/tests/test_audio.py,sha256=92PMPrMtYzLoaLiW92MupEfWPEcfoLlYlZKlxmWQXdI,3186
 pixeltable/tests/test_catalog.py,sha256=Npxt72g5aZkfg1fqE-19L8rGilzicAiTC0ithIy3woI,1189
 pixeltable/tests/test_client.py,sha256=9acrjElh1YVX8WXWO04VQ-S6mNx9ZtqAHoh4uBc81y4,531
-pixeltable/tests/test_component_view.py,sha256=VfcBgdG1QKeo0AOnHmO21J3qdm_8-06RljB7kkTeTok,17732
-pixeltable/tests/test_dataframe.py,sha256=EH8pVeDpthH8J8561xZtn_Lz4Tb_8uycw4JjLKjZEJI,18007
+pixeltable/tests/test_component_view.py,sha256=j0Ri9li_gqEgu_X_MDlHq3SrglwSb-TFt-VgAPiXk1M,18221
+pixeltable/tests/test_dataframe.py,sha256=7MNVGKg7gD__K_EH5PRx2wSDkykpQF2Uoahr_JvQwD4,17906
 pixeltable/tests/test_dirs.py,sha256=WuWGOcpUFPCl1PGZgOMcU5bpzv1ClUqZ5AxavrDeCic,3611
 pixeltable/tests/test_document.py,sha256=5G52cMbkkX2lk5SwyYQ1KYjQjgUfVhjcsjPtL9xjGXU,5810
-pixeltable/tests/test_exprs.py,sha256=ZwImlJpTu9YQ2AnAplurjxDmabYqG7rB7aU6TxQAJdI,32743
-pixeltable/tests/test_function.py,sha256=6sA_3Oe1p4sUhL5L5hYyW-mG1TPGfFDmW1BPVPnGXek,12817
-pixeltable/tests/test_functions.py,sha256=pKRsb8503vkYWzp_BOGuxd-PDtOb04NKW-T6_4sJ3B0,14394
-pixeltable/tests/test_migration.py,sha256=UjJhhg061TtyTAa1CAB-BdlT9YwLJkl8yIGSetnqcKw,1535
+pixeltable/tests/test_exprs.py,sha256=3F2btTIdT6TymaQLtiDIo3sAntW1WLY5oU-ud267VhY,32181
+pixeltable/tests/test_function.py,sha256=zzW7IYTipe23ao5_dNMmGEv2xJOsDUe0DoHI8pfuPDI,12987
+pixeltable/tests/test_index.py,sha256=6ONEKsxHdUJuLTYoeIVqEJzbp_sANzr43D1xre2b-6o,5410
+pixeltable/tests/test_migration.py,sha256=RGpQOjkkjhT4oO9U5X6dG2q8YHTeJPDq_dRlQZY7YQM,1599
 pixeltable/tests/test_nos.py,sha256=ITE7FNEaNreJ_XTz4wYLWuidFSUQMFp3ShuHTz05OrE,2649
-pixeltable/tests/test_snapshot.py,sha256=J8wepLOaOrJkqDWq1dkou8MIR5Hf1DEseAqlUrb7BFg,9297
-pixeltable/tests/test_table.py,sha256=KGPny8KQ4wVYT-N78dqsoMahd_1GyQAGZaCqT7sB61E,48484
+pixeltable/tests/test_snapshot.py,sha256=uTTxnBNZODhhbH2pAxtzKdGrNmbiozKmya-LvK8OQ6g,10331
+pixeltable/tests/test_table.py,sha256=a7BCBsSzlLmIEXtmrLiIFP6B4KoG4B9MuHm_JIwFEic,56557
 pixeltable/tests/test_transactional_directory.py,sha256=Jx55PZgrkHTI0Eli09puwMckvwfCM9arPTQUT9g6Tbg,1308
-pixeltable/tests/test_types.py,sha256=4_LR5Poc8Fa5fOpRBaAc_qAN21SsVxFP54Npq_6n9G8,970
-pixeltable/tests/test_video.py,sha256=j9vgxp9f2WdH0rCoMXqhgTW3XG-czTBU4rrnuR95y8E,7782
-pixeltable/tests/test_view.py,sha256=vbKEk9kWF5U6B78uNLSQsGw46s2oOOhHnJy8p9NmNTo,21859
-pixeltable/tests/utils.py,sha256=2EpA2KQmE7fdNDGsOBmjpOwHmGpfNIv_1880itvcaoE,10378
-pixeltable/tool/create_test_db_dump.py,sha256=ARlLUfOs4stxIPFl-hGl_KX-ck6JRWmUiXqTwTtS1GU,5148
+pixeltable/tests/test_types.py,sha256=psRqCDInUPH99W6-FlZhqUoN5FTlfZaUUS7YR3UzvIw,2045
+pixeltable/tests/test_video.py,sha256=zN9h9KM2V8U-Rf3tFxYimp_QoT4hLbbguyEu2o9_PLU,7740
+pixeltable/tests/test_view.py,sha256=EWo_Z-UEbilayXK8iEziPg5fedGtT6o4745S04Z4Lb4,22108
+pixeltable/tests/utils.py,sha256=LNHP9O0vqtKV4_WngD6V3e6iI7dKxLkII2UInR4xKHQ,15106
+pixeltable/tool/create_test_db_dump.py,sha256=mKX42BHgx1JggmG2vupT0kzDM0N2HG-z8ObKcgBRMls,5842
 pixeltable/tool/create_test_video.py,sha256=OLfccymYReIpzE8osZn4rQvLXxxiPC_l0vc06U74hVM,2899
-pixeltable/type_system.py,sha256=Qxx-LzE88QJPC528z8wnof66BwKBT7GPYOJ4bE4BV9o,31798
+pixeltable/type_system.py,sha256=VXVxgclkW3_wX818qZ6NnST29M8R4fgQ9_LiEfVOo8k,29929
 pixeltable/utils/__init__.py,sha256=UYlrf6TIWJT0g-Hac0b34-dEk478B5Qx8dGco34YlIk,439
+pixeltable/utils/arrow.py,sha256=83_7aG5UR2qtTktw_otLkQs-RQbLk0VVM0JLJkbweNU,3692
 pixeltable/utils/clip.py,sha256=HXXWFBJXW9XysdMk9_3hP1V1S-3B8Hwd5rNMbJFjjnI,720
 pixeltable/utils/coco.py,sha256=mk1cxjKYQC0ABm2ZQ9SNu9MvBPECmmKvnASpxnFXdL0,5604
 pixeltable/utils/documents.py,sha256=MGtWZs93Bt_pdExxT9yjH7VWbawSd5A2O5_YdxEZlMk,1094
 pixeltable/utils/filecache.py,sha256=UoNONG2VaAc2IBB0e3sQdsvyOPOes2XSDc5_CsA4qek,7839
 pixeltable/utils/help.py,sha256=cCnxJ4VP9MJ57iDqExmnDcM-JG3a1lw_q7g-D7bpSVI,252
+pixeltable/utils/hf_datasets.py,sha256=TxN0H9d25VfXiWwT1XAziNFJMKLSK9Y7x1oPejbuFok,7005
 pixeltable/utils/media_store.py,sha256=x71wnJDZDHcdd13VCfL4AkHQ6IJB41gNA-zBvXJwFos,3116
-pixeltable/utils/parquet.py,sha256=3cjC1vibPHQ24cQS0xPwTjkhxK9TVkITkSxkf29bdaQ,5775
-pixeltable/utils/pytorch.py,sha256=h7q-7q3gXThDoTVA8k9vIvMc4VgnNAQy3ScDIU6h3IE,6515
+pixeltable/utils/parquet.py,sha256=DmiF74UYNvnGOOgHUHk_108QZALsQY7HnZoJw6-H3xc,7120
+pixeltable/utils/pytorch.py,sha256=BR4tgfUWw-2rwWTOgzXj5qdMBpe1Arpp5SK4ax6jjpk,3483
 pixeltable/utils/s3.py,sha256=rkanuhk9DWvSfmbOLQW1j1Iov4sl2KhxGGKN-AJ8LSE,432
 pixeltable/utils/sql.py,sha256=5n5_OmXAGtqFdL6z5XvgnU-vlx6Ba6f1WJrO1ZwUle8,765
 pixeltable/utils/transactional_directory.py,sha256=UGzCrGtLR3hEEf8sYGuWBzLVFAEQml3vdIavigWeTBM,1349
-pixeltable-0.2.3.dist-info/LICENSE,sha256=0UNMmwuqWPC0xDY1NWMm4uNJ2_MyA1pnTNRgQTvuBiQ,746
-pixeltable-0.2.3.dist-info/METADATA,sha256=n_jCwEwEIRpA78qdeOTZBgQzlTFAljHgsyOZPrg-o58,5473
-pixeltable-0.2.3.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-pixeltable-0.2.3.dist-info/RECORD,,
+pixeltable-0.2.5.dist-info/LICENSE,sha256=0UNMmwuqWPC0xDY1NWMm4uNJ2_MyA1pnTNRgQTvuBiQ,746
+pixeltable-0.2.5.dist-info/METADATA,sha256=Fl2vtfThiLTgAhZMSioMssMmq7DvnSe-7V42AfBrT7g,6081
+pixeltable-0.2.5.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+pixeltable-0.2.5.dist-info/RECORD,,