pixeltable 0.2.3__py3-none-any.whl → 0.2.4__py3-none-any.whl

pixeltable/utils/pytorch.py

@@ -1,25 +1,17 @@
 import io
 import pyarrow as pa
-import pyarrow.parquet as pq
+import pyarrow.parquet
 import torch
 import torch.utils.data
-import math
 from pathlib import Path
 import PIL.Image
 import json
-from typing import Dict, Tuple, Generator, Any
+from typing import Dict, Iterator, Any
 import datetime
 
 from pixeltable.type_system import ColumnType
-from pixeltable.utils.parquet import get_part_metadata
 import numpy as np
 
-def _cumsum(lst):
-    acc = [0]
-    for x in lst:
-        acc.append(acc[-1] + x)
-    return acc
-
 class PixeltablePytorchDataset(torch.utils.data.IterableDataset):
     """
     PyTorch dataset interface for pixeltable data.
@@ -47,24 +39,7 @@ class PixeltablePytorchDataset(torch.utils.data.IterableDataset):
         with column_type_path.open() as f:
             column_types = json.load(f)
         self.column_types = {k: ColumnType.from_dict(v) for k, v in column_types.items()}
-        self.part_metadata = get_part_metadata(self.path)
-        self._totals = _cumsum([x[1] for x in self.part_metadata])
-
-    def _get_start_position(self, row_start: int) -> Tuple[int, int]:
-        """
-        Returns the starting parquet file and row within that file for a given 'global' row number.
-        based on the individual sizes of each part
-        """
-        assert row_start >= self._totals[0]
-        assert row_start < self._totals[-1]
-
-        prev_acc = 0
-        for i, acc in enumerate(self._totals[1:], start=1):
-            if acc > row_start:
-                return (i - 1, row_start - prev_acc)
-            prev_acc = acc
-
-        assert False, "unreachable"
+        self.part_metadata = pa.parquet.ParquetDataset(path).files
 
     def _unmarshall(self, k: str, v: Any) -> Any:
         if self.column_types[k].is_image_type():
@@ -88,13 +63,10 @@ class PixeltablePytorchDataset(torch.utils.data.IterableDataset):
             return json.loads(v)
         elif self.column_types[k].is_array_type():
             assert isinstance(v, np.ndarray)
-            # WRITEABLE is required for torch collate function, or undefined behavior
             if not v.flags["WRITEABLE"]:
-                vout = v.copy()
-                assert vout.flags["WRITEABLE"]
-                return vout
-            else:
-                return v
+                v = v.copy()
+            assert v.flags["WRITEABLE"]
+            return v
         elif self.column_types[k].is_timestamp_type():
             # pytorch default collation only supports numeric types
             assert isinstance(v, datetime.datetime)
@@ -103,70 +75,17 @@ class PixeltablePytorchDataset(torch.utils.data.IterableDataset):
             assert not isinstance(v, np.ndarray) # all array outputs should be handled above
             return v
 
-    def __iter__(self) -> Generator[Dict[str, Any], None, None]:
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
+        import pixeltable.utils.arrow as arrow
         worker_info = torch.utils.data.get_worker_info()
-        if worker_info is None:  # single-process data loading, return the full iterator
-            start_row = 0
-            end_row = self._totals[-1]
-        else:  # in a worker process
-            num_workers = [
-                math.floor(self._totals[-1] / float(worker_info.num_workers))
-                for _ in range(worker_info.num_workers)
-            ]
-            assert self._totals[-1] - sum(num_workers) < worker_info.num_workers
-            for i in range(self._totals[-1] - sum(num_workers)):
-                num_workers[i] += 1
-
-            assert sum(num_workers) == self._totals[-1]
-            start_rows = _cumsum(num_workers)
-            start_row = start_rows[worker_info.id]
-            end_row = start_rows[worker_info.id + 1]
 
-        if start_row == end_row:
-            return iter([])  # type: ignore
+        if worker_info is None:
+            part_list = range(len(self.part_metadata))
         else:
-            return self._iter_range(start_row, end_row)
-
-    def _iter_range(self, start_row : int, end_row : int) -> Generator[Dict[str, Any], None, None]:
-        (part_no, iter_start) = self._get_start_position(start_row)
-        total = end_row - start_row
-
-        acc = 0
-        part_pos = part_no
-        iter_pos = iter_start
-
-        def _to_column_dict(tab : pa.Table) -> Dict[str, Any]:
-            column_dict = {}
-            for k in tab.column_names:
-                if self.column_types[k].is_array_type():
-                    # treat array columns as numpy arrays to easily preserve numpy type
-                    column_dict[k] = tab.column(k).to_numpy()
-                else:
-                    # for the rest, use pydict to preserve python types
-                    column_dict[k] = tab.column(k).to_pylist()
-            return column_dict
-
-        tab: pa.Table = pq.read_table(self.path / self.part_metadata[part_no][0])
-        column_dict = _to_column_dict(tab)
-        assert tab.num_rows == self.part_metadata[part_no][1]
-
-        while True:
-            while iter_pos < tab.num_rows and acc < total:
-                next_tup = {}
-                for col_name, col_vals in column_dict.items():
-                    raw_val = col_vals[iter_pos]
-                    next_tup[col_name] = self._unmarshall(col_name, raw_val)
-
-                yield next_tup
-                acc += 1
-                iter_pos += 1
-
-            if acc == total:
-                break
+            part_list = [ i for i in part_list if (i % worker_info.num_workers) == worker_info.id ]
 
-            # move on to next part
-            part_pos += 1
-            assert part_pos < len(self.part_metadata)
-            iter_pos = 0
-            tab = pq.read_table(self.path / self.part_metadata[part_pos][0])
-            column_dict = _to_column_dict(tab)
+        for part_no in part_list:
+            pqf = pa.parquet.ParquetFile(self.part_metadata[part_no])
+            for batch in pqf.iter_batches():
+                for tup in arrow.iter_tuples(batch):
+                    yield {k: self._unmarshall(k, v) for k, v in tup.items()}

{pixeltable-0.2.3.dist-info → pixeltable-0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pixeltable
-Version: 0.2.3
+Version: 0.2.4
 Summary: Pixeltable: The Multimodal AI Data Plane
 Author: Marcel Kornacker
 Author-email: marcelk@gmail.com
@@ -15,12 +15,12 @@ Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
 Requires-Dist: cloudpickle (>=2.2.1,<3.0.0)
 Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
 Requires-Dist: jmespath (>=1.0.1,<2.0.0)
-Requires-Dist: numpy (>=1.26,<2.0)
+Requires-Dist: numpy (>=1.26)
 Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
 Requires-Dist: pandas (>=2.0,<3.0)
-Requires-Dist: pgserver (==0.0.9)
+Requires-Dist: pgserver (==0.1.0)
 Requires-Dist: pgvector (>=0.2.1,<0.3.0)
-Requires-Dist: pillow (>=9.4.0,<10.0.0)
+Requires-Dist: pillow (>=10.0)
 Requires-Dist: psutil (>=5.9.5,<6.0.0)
 Requires-Dist: psycopg2-binary (>=2.9.5,<3.0.0)
 Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
@@ -28,37 +28,42 @@ Requires-Dist: regex (>=2022.10.31,<2023.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: sqlalchemy-utils (>=0.41.1,<0.42.0)
 Requires-Dist: sqlalchemy[mypy] (>=2.0.23,<3.0.0)
+Requires-Dist: tenacity (>=8.2,<9.0)
 Requires-Dist: tqdm (>=4.64.1,<5.0.0)
 Description-Content-Type: text/markdown
 
+<div align="center">
 <img src="docs/pixeltable-banner.png" width="45%"/>
 
-# Pixeltable: The Multimodal AI Data Plane
+# Unifying Data, Models, and Orchestration for AI Products
 
 [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 &nbsp;&nbsp;
 ![pytest status](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml/badge.svg)
 
-Pixeltable is a Python library that lets AI engineers and data scientists focus on
-exploration, modeling, and app development without having to deal with the customary
-data plumbing.
+[Installation](https://pixeltable.github.io/pixeltable/getting-started/) | [Documentation](https://pixeltable.github.io/pixeltable/)
+</div>
 
-**Pixeltable redefines data infrastructure and workflow orchestration for AI development.**
-It brings together data storage, versioning, and indexing with orchestration and model
-versioning under a declarative table interface, with transformations, model inference,
-and custom logic represented as computed columns.
+Pixeltable is a Python library that lets AI engineers and data scientists focus on exploration, modeling, and app development without dealing with the customary data plumbing.
 
-## Quick Start
+## What problems does Pixeltable solve?
+
+Today’s solutions for AI app development require extensive custom coding and infrastructure
+plumbing. Tracking lineage and versions between and across data transformations, models, and
+deployment is cumbersome. Pixeltable is a replacement for traditional data plumbing, providing
+a unified plane for data, models, and orchestration. It removes the data plumbing overhead in
+building and productionizing AI applications.
+
+## ⚡Quick Start
+Learn the basics of Pixeltable through interactive examples. View the notebooks on Google Colab or Kaggle, for free.
 
-If you just want to play around with Pixeltable to see what it's capable of, the easiest way is to run
-the Pixeltable Basics tutorial in colab:
+### Pixeltable Basics
+In this tutorial, we'll survey how to create tables, populate them with data, and enhance them with built-in and user-defined transformations and AI operations.
 
-<a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb">
-  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
-</a>
+[![Open in Kaggle](https://kaggle.com/static/images/open-in-kaggle.svg)](https://www.kaggle.com/code/brunep/pixeltable-basics) <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a>
 
-## Installation
 
+## 💾 Installation
 Pixeltable works with Python 3.9, 3.10, or 3.11 running on Linux or MacOS.
 
 ```
@@ -78,14 +83,6 @@ guide. Then, check out the
 [Pixeltable Basics](https://pixeltable.github.io/pixeltable/tutorials/pixeltable-basics/)
 tutorial for a tour of its most important features.
 
-## What problems does Pixeltable solve?
-
-Today’s solutions for AI app development require extensive custom coding and infrastructure
-plumbing. Tracking lineage and versions between and across data transformations, models, and
-deployment is cumbersome. Pixeltable is a replacement for traditional data plumbing, providing
-a unified plane for data, models, and orchestration. It removes the data plumbing overhead in
-building and productionizing AI applications.
-
 ## Why should you use Pixeltable?
 
 - It gives you transparency and reproducibility
@@ -119,3 +116,12 @@ get cost projections before adding new data and new augmentations.
 * Rely on Pixeltable's automatic versioning and snapshot functionality to protect against regressions
 and to ensure reproducibility.
 
+## Contributions & Feedback
+
+Are you experiencing issues or bugs with Pixeltable? File an [Issue](https://github.com/pixeltable/pixeltable/issues).
+</br>Do you want to contribute? Feel free to open a [PR](https://github.com/pixeltable/pixeltable/pulls).
+
+## :classical_building: License
+
+This library is licensed under the Apache 2.0 License.
+

{pixeltable-0.2.3.dist-info → pixeltable-0.2.4.dist-info}/RECORD

@@ -1,7 +1,7 @@
 pixeltable/__init__.py,sha256=enh38lhZ_38Ys3rCy_XC_BlPNPK9H2CKV4KePaqnV4E,959
 pixeltable/catalog/__init__.py,sha256=E41bxaPeQIcgRYzTWc2vkDOboQhRymrJf4IcHQO7o_8,453
 pixeltable/catalog/catalog.py,sha256=0TYWB1R6YBp9qCkWF7kCcX2Yw70UuburKKIemv5L1Js,7908
-pixeltable/catalog/column.py,sha256=7PU5yECislpR_PNZTT3Z5sZiE34DnI6SR3WDsI-WgCg,8661
+pixeltable/catalog/column.py,sha256=rlWhKh_k9zLXus0Slcrxzsl6tJpyOxylu2qfUk0waVI,8663
 pixeltable/catalog/dir.py,sha256=pG1nMpG123POo6WMSHhAmnwXOQ26uUJfUcbzL-Jb4ws,919
 pixeltable/catalog/globals.py,sha256=yLEGNbsSnLzjWNHVJacfjA9hbw13Q6QXLOSCRmdTlq0,943
 pixeltable/catalog/insertable_table.py,sha256=1JGZqmx5tKbm_UmJLIalntFDs81s-mvTv0gOqk98b3Q,8280
@@ -13,9 +13,9 @@ pixeltable/catalog/table.py,sha256=CJQc2NnebfL1ekMcaZhJEZ0CI408-F7R90_czjDKy-g,2
 pixeltable/catalog/table_version.py,sha256=Q1NSaeLmFw7gYgARyyW6JJ79GRhjtp0_Q410KH27e1Q,35680
 pixeltable/catalog/table_version_path.py,sha256=kn5T0m5XBqphlDqCKsqD-s159diCjKvrSrtzYZHwKuw,5461
 pixeltable/catalog/view.py,sha256=BIL3s4DV3tWbOcqtqnhn46B2UvLaBhppfJUlNEt5nec,9734
-pixeltable/client.py,sha256=HiqGNRQoyBPXF2rAhTJFE3ZSUUaSLpGo28pLZjubh8Y,20548
+pixeltable/client.py,sha256=RweoJoUGap_uzAcrhSwU6yOH7j_fsSM-VgyKSCYGvv8,23616
 pixeltable/dataframe.py,sha256=vBc4T-QU-lTTtnljqDbBFTHwm2CecLHIjJumEv8lQ0s,28339
-pixeltable/env.py,sha256=sY-VUY9WFs4li7CidyRXubfjmBh3cbR-cFj-wZr4xY0,16398
+pixeltable/env.py,sha256=A38md5r-hE6i-6Og1Nptlzo0ci3Oyn62ogPs6hlNLZE,15744
 pixeltable/exceptions.py,sha256=MSP9zeL0AmXT93XqjdvgGN4rzno1_KRrGriq6hpemnw,376
 pixeltable/exec/__init__.py,sha256=FOQBSMQ3buGg5kRzymkfDCk5fnVfICn3LrsJtbyNC6E,412
 pixeltable/exec/aggregation_node.py,sha256=cf6rVAgrGh_uaMrCIgXJIwQTmbcboJlnrH_MmPIQSd0,3321
@@ -65,15 +65,15 @@ pixeltable/func/globals.py,sha256=UP4hAkEVKD_3lr7l_VVUUCetJP85sydUYiVAU4m5-aw,12
 pixeltable/func/nos_function.py,sha256=HzIKK4XjTo1E6pML-EbhuX3u_LYibFWUuTkIxoIih7c,9650
 pixeltable/func/signature.py,sha256=5Xo3hIVSXoYtkU-BvUi21b6FghIQABm48UfVjQr9QYU,7107
 pixeltable/func/udf.py,sha256=XlB7dzOc98XinUrs4tMGUAetV0SmQ_Z2oKPDBsx46o0,6457
-pixeltable/functions/__init__.py,sha256=kFepRTpg-YOeeXG9076s9v-lm8ZUNqicaPUfX-CrW5c,3419
+pixeltable/functions/__init__.py,sha256=T8FZuE9YSh4KeRg7tBgOEdg8LLyvn89xWLIRylpiP-I,3390
 pixeltable/functions/eval.py,sha256=qLc_55VTwi_IDnAJFq79Hy7FLi7Gb26DxdykYiBTMrM,8453
-pixeltable/functions/fireworks.py,sha256=7h1gxcGhG8jepS_Gi1nwp8zCw1gdRaTzufS6U08mmLc,1596
+pixeltable/functions/fireworks.py,sha256=e_rCITg18yNndNI8TJPXRSN6DR0hYWT-_dUavoPuyfc,908
 pixeltable/functions/huggingface.py,sha256=Wg8AgCsoGFx8AX3AThko66L7uSg9cUEjL8_QxKoA4dg,4804
 pixeltable/functions/image.py,sha256=xR_S_0BuX6Ycc5E366GpOfP0JptD7beQwHE_fLl8ZVM,431
-pixeltable/functions/openai.py,sha256=SkwUBnnTT6olQCMHzQGO0vpvApgAaBk_88W7YM8QQuo,2863
+pixeltable/functions/openai.py,sha256=xh40TvNanhXPXWGte_ZKxuyD6TYWzmC_1G3qRNDkNv8,6861
 pixeltable/functions/pil/image.py,sha256=8gItSXXuJaCkq9FHEJE9qFpRM3WAoa59x89Xa0DgksQ,6217
 pixeltable/functions/string.py,sha256=RYOgZwifjC943YloEMi3PdflnjFqOYB2FddrUvzgtXs,516
-pixeltable/functions/together.py,sha256=TGo72RJuHqofwJHSrHqsJoO6RQm6Iipqhy49V3JiBss,650
+pixeltable/functions/together.py,sha256=sG23nLMScmp4wRA4K1EIJDY9peqE1IPgxlK3fhNrbgw,3423
 pixeltable/functions/util.py,sha256=v094FgixHchzKMj4YvfHExDRGIVhCL2D-NL3pl-92oo,1596
 pixeltable/functions/video.py,sha256=WZF4G3tV-_LfRQHUinXe_rnu1-4N68Ht60JCR_s7Bew,2403
 pixeltable/iterators/__init__.py,sha256=sfsasCypAq5rNOTMlr4j2ROXxzdl4M8L2KvQIEbd0cQ,70
@@ -85,42 +85,48 @@ pixeltable/metadata/converters/convert_10.py,sha256=0mSGCn7vqtef63riPi9msUaaUvsS
 pixeltable/metadata/schema.py,sha256=oTxtFSvbmCEcr54F929_NYFVp76N8gpN4nHDr9VnA9M,7761
 pixeltable/plan.py,sha256=BQMCxOCy7lO4dqBCt5W9GUFdBWkuhr13-T8_2E97fUk,34710
 pixeltable/store.py,sha256=ICqfHiotyMyTi2gsQ-Qw-ed1ooNkAx0vgMSMri4xfqk,19511
-pixeltable/tests/conftest.py,sha256=2RyRWIW2Ky9VQ41Hj0ChNr_Oh5vTx4x95tUdJWX5ktE,6677
+pixeltable/tests/conftest.py,sha256=qTfWzGHH35gSkHPOaCAzdDtWWJDEktNIPprLh1wjHwU,6684
+pixeltable/tests/functions/test_fireworks.py,sha256=G-54dABoRbmXApyjWZKKrKKooIp7sWcpAT6rJWVESWk,1666
+pixeltable/tests/functions/test_functions.py,sha256=SRE4zRwfJ1Hetu8uMp5X-Iqg-pawn87q07JHa7w_8l0,2884
+pixeltable/tests/functions/test_huggingface.py,sha256=jOnE51qCHosB6FYIBUWf4VsK4m9mLO3zaBxnKZ5cmHI,7739
+pixeltable/tests/functions/test_openai.py,sha256=ptt3QDAQttddqWjmeGMmOkedJg14KFr_b2FT9Chmwlw,8172
+pixeltable/tests/functions/test_together.py,sha256=0XGAO2J17tefSlhtMbOFN92bp1SgUB9kw_almcz6fyQ,4788
 pixeltable/tests/test_audio.py,sha256=92PMPrMtYzLoaLiW92MupEfWPEcfoLlYlZKlxmWQXdI,3186
 pixeltable/tests/test_catalog.py,sha256=Npxt72g5aZkfg1fqE-19L8rGilzicAiTC0ithIy3woI,1189
 pixeltable/tests/test_client.py,sha256=9acrjElh1YVX8WXWO04VQ-S6mNx9ZtqAHoh4uBc81y4,531
 pixeltable/tests/test_component_view.py,sha256=VfcBgdG1QKeo0AOnHmO21J3qdm_8-06RljB7kkTeTok,17732
-pixeltable/tests/test_dataframe.py,sha256=EH8pVeDpthH8J8561xZtn_Lz4Tb_8uycw4JjLKjZEJI,18007
+pixeltable/tests/test_dataframe.py,sha256=KytKHjsyHluB9DmxmI0g3YPC7UTvbnv5YhysaUMfVzA,17975
 pixeltable/tests/test_dirs.py,sha256=WuWGOcpUFPCl1PGZgOMcU5bpzv1ClUqZ5AxavrDeCic,3611
 pixeltable/tests/test_document.py,sha256=5G52cMbkkX2lk5SwyYQ1KYjQjgUfVhjcsjPtL9xjGXU,5810
 pixeltable/tests/test_exprs.py,sha256=ZwImlJpTu9YQ2AnAplurjxDmabYqG7rB7aU6TxQAJdI,32743
 pixeltable/tests/test_function.py,sha256=6sA_3Oe1p4sUhL5L5hYyW-mG1TPGfFDmW1BPVPnGXek,12817
-pixeltable/tests/test_functions.py,sha256=pKRsb8503vkYWzp_BOGuxd-PDtOb04NKW-T6_4sJ3B0,14394
 pixeltable/tests/test_migration.py,sha256=UjJhhg061TtyTAa1CAB-BdlT9YwLJkl8yIGSetnqcKw,1535
 pixeltable/tests/test_nos.py,sha256=ITE7FNEaNreJ_XTz4wYLWuidFSUQMFp3ShuHTz05OrE,2649
 pixeltable/tests/test_snapshot.py,sha256=J8wepLOaOrJkqDWq1dkou8MIR5Hf1DEseAqlUrb7BFg,9297
-pixeltable/tests/test_table.py,sha256=KGPny8KQ4wVYT-N78dqsoMahd_1GyQAGZaCqT7sB61E,48484
+pixeltable/tests/test_table.py,sha256=zthAzzO9DGmOSsnFD2Xuy40t4-_1ZzchAMKg9JXs1kY,53303
 pixeltable/tests/test_transactional_directory.py,sha256=Jx55PZgrkHTI0Eli09puwMckvwfCM9arPTQUT9g6Tbg,1308
 pixeltable/tests/test_types.py,sha256=4_LR5Poc8Fa5fOpRBaAc_qAN21SsVxFP54Npq_6n9G8,970
 pixeltable/tests/test_video.py,sha256=j9vgxp9f2WdH0rCoMXqhgTW3XG-czTBU4rrnuR95y8E,7782
 pixeltable/tests/test_view.py,sha256=vbKEk9kWF5U6B78uNLSQsGw46s2oOOhHnJy8p9NmNTo,21859
-pixeltable/tests/utils.py,sha256=2EpA2KQmE7fdNDGsOBmjpOwHmGpfNIv_1880itvcaoE,10378
+pixeltable/tests/utils.py,sha256=OY8r6SM9MAEiXehqBymF8BkHYmlAd7-D09nfzTZMCsA,14359
 pixeltable/tool/create_test_db_dump.py,sha256=ARlLUfOs4stxIPFl-hGl_KX-ck6JRWmUiXqTwTtS1GU,5148
 pixeltable/tool/create_test_video.py,sha256=OLfccymYReIpzE8osZn4rQvLXxxiPC_l0vc06U74hVM,2899
-pixeltable/type_system.py,sha256=Qxx-LzE88QJPC528z8wnof66BwKBT7GPYOJ4bE4BV9o,31798
+pixeltable/type_system.py,sha256=oUrqqux-YJrzNpPFp_3WkSdI3lHm7AcZByTCnWa97VQ,30374
 pixeltable/utils/__init__.py,sha256=UYlrf6TIWJT0g-Hac0b34-dEk478B5Qx8dGco34YlIk,439
+pixeltable/utils/arrow.py,sha256=83_7aG5UR2qtTktw_otLkQs-RQbLk0VVM0JLJkbweNU,3692
 pixeltable/utils/clip.py,sha256=HXXWFBJXW9XysdMk9_3hP1V1S-3B8Hwd5rNMbJFjjnI,720
 pixeltable/utils/coco.py,sha256=mk1cxjKYQC0ABm2ZQ9SNu9MvBPECmmKvnASpxnFXdL0,5604
 pixeltable/utils/documents.py,sha256=MGtWZs93Bt_pdExxT9yjH7VWbawSd5A2O5_YdxEZlMk,1094
 pixeltable/utils/filecache.py,sha256=UoNONG2VaAc2IBB0e3sQdsvyOPOes2XSDc5_CsA4qek,7839
 pixeltable/utils/help.py,sha256=cCnxJ4VP9MJ57iDqExmnDcM-JG3a1lw_q7g-D7bpSVI,252
+pixeltable/utils/hf_datasets.py,sha256=TxN0H9d25VfXiWwT1XAziNFJMKLSK9Y7x1oPejbuFok,7005
 pixeltable/utils/media_store.py,sha256=x71wnJDZDHcdd13VCfL4AkHQ6IJB41gNA-zBvXJwFos,3116
-pixeltable/utils/parquet.py,sha256=3cjC1vibPHQ24cQS0xPwTjkhxK9TVkITkSxkf29bdaQ,5775
-pixeltable/utils/pytorch.py,sha256=h7q-7q3gXThDoTVA8k9vIvMc4VgnNAQy3ScDIU6h3IE,6515
+pixeltable/utils/parquet.py,sha256=DmiF74UYNvnGOOgHUHk_108QZALsQY7HnZoJw6-H3xc,7120
+pixeltable/utils/pytorch.py,sha256=BR4tgfUWw-2rwWTOgzXj5qdMBpe1Arpp5SK4ax6jjpk,3483
 pixeltable/utils/s3.py,sha256=rkanuhk9DWvSfmbOLQW1j1Iov4sl2KhxGGKN-AJ8LSE,432
 pixeltable/utils/sql.py,sha256=5n5_OmXAGtqFdL6z5XvgnU-vlx6Ba6f1WJrO1ZwUle8,765
 pixeltable/utils/transactional_directory.py,sha256=UGzCrGtLR3hEEf8sYGuWBzLVFAEQml3vdIavigWeTBM,1349
-pixeltable-0.2.3.dist-info/LICENSE,sha256=0UNMmwuqWPC0xDY1NWMm4uNJ2_MyA1pnTNRgQTvuBiQ,746
-pixeltable-0.2.3.dist-info/METADATA,sha256=n_jCwEwEIRpA78qdeOTZBgQzlTFAljHgsyOZPrg-o58,5473
-pixeltable-0.2.3.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-pixeltable-0.2.3.dist-info/RECORD,,
+pixeltable-0.2.4.dist-info/LICENSE,sha256=0UNMmwuqWPC0xDY1NWMm4uNJ2_MyA1pnTNRgQTvuBiQ,746
+pixeltable-0.2.4.dist-info/METADATA,sha256=tUA1jMlooZRLprmQniFTby9KDCuOHFr5b1FVv4fCZdM,5983
+pixeltable-0.2.4.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+pixeltable-0.2.4.dist-info/RECORD,,