pixeltable 0.2.21__py3-none-any.whl → 0.2.23__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pixeltable/__init__.py +2 -2
- pixeltable/__version__.py +2 -2
- pixeltable/catalog/__init__.py +1 -1
- pixeltable/catalog/column.py +41 -29
- pixeltable/catalog/globals.py +18 -0
- pixeltable/catalog/insertable_table.py +30 -10
- pixeltable/catalog/table.py +198 -86
- pixeltable/catalog/table_version.py +47 -53
- pixeltable/catalog/table_version_path.py +2 -2
- pixeltable/catalog/view.py +17 -18
- pixeltable/dataframe.py +27 -36
- pixeltable/env.py +7 -0
- pixeltable/exec/__init__.py +0 -1
- pixeltable/exec/aggregation_node.py +6 -3
- pixeltable/exec/cache_prefetch_node.py +189 -43
- pixeltable/exec/data_row_batch.py +5 -22
- pixeltable/exec/exec_context.py +2 -2
- pixeltable/exec/exec_node.py +3 -2
- pixeltable/exec/expr_eval_node.py +23 -16
- pixeltable/exec/in_memory_data_node.py +6 -3
- pixeltable/exec/sql_node.py +24 -25
- pixeltable/exprs/arithmetic_expr.py +12 -5
- pixeltable/exprs/array_slice.py +7 -7
- pixeltable/exprs/column_property_ref.py +37 -10
- pixeltable/exprs/column_ref.py +97 -14
- pixeltable/exprs/comparison.py +10 -5
- pixeltable/exprs/compound_predicate.py +8 -7
- pixeltable/exprs/data_row.py +27 -18
- pixeltable/exprs/expr.py +53 -52
- pixeltable/exprs/expr_set.py +5 -0
- pixeltable/exprs/function_call.py +32 -16
- pixeltable/exprs/globals.py +4 -1
- pixeltable/exprs/in_predicate.py +8 -7
- pixeltable/exprs/inline_expr.py +4 -4
- pixeltable/exprs/is_null.py +4 -4
- pixeltable/exprs/json_mapper.py +11 -12
- pixeltable/exprs/json_path.py +6 -11
- pixeltable/exprs/literal.py +5 -5
- pixeltable/exprs/method_ref.py +5 -4
- pixeltable/exprs/object_ref.py +2 -1
- pixeltable/exprs/row_builder.py +88 -36
- pixeltable/exprs/rowid_ref.py +12 -11
- pixeltable/exprs/similarity_expr.py +12 -7
- pixeltable/exprs/sql_element_cache.py +7 -5
- pixeltable/exprs/type_cast.py +8 -6
- pixeltable/exprs/variable.py +5 -4
- pixeltable/func/aggregate_function.py +9 -9
- pixeltable/func/expr_template_function.py +6 -5
- pixeltable/func/function.py +11 -10
- pixeltable/func/udf.py +6 -11
- pixeltable/functions/__init__.py +2 -2
- pixeltable/functions/globals.py +5 -7
- pixeltable/functions/huggingface.py +155 -45
- pixeltable/functions/llama_cpp.py +107 -0
- pixeltable/functions/mistralai.py +1 -1
- pixeltable/functions/ollama.py +147 -0
- pixeltable/functions/openai.py +1 -1
- pixeltable/functions/replicate.py +72 -0
- pixeltable/functions/string.py +9 -0
- pixeltable/functions/together.py +1 -1
- pixeltable/functions/util.py +5 -2
- pixeltable/globals.py +67 -26
- pixeltable/index/btree.py +16 -3
- pixeltable/index/embedding_index.py +4 -4
- pixeltable/io/__init__.py +1 -2
- pixeltable/io/fiftyone.py +178 -0
- pixeltable/io/globals.py +96 -2
- pixeltable/iterators/base.py +3 -2
- pixeltable/iterators/document.py +1 -1
- pixeltable/iterators/video.py +120 -63
- pixeltable/metadata/__init__.py +1 -1
- pixeltable/metadata/converters/convert_21.py +34 -0
- pixeltable/metadata/converters/util.py +45 -4
- pixeltable/metadata/notes.py +1 -0
- pixeltable/metadata/schema.py +8 -0
- pixeltable/plan.py +17 -15
- pixeltable/py.typed +0 -0
- pixeltable/store.py +7 -2
- pixeltable/tool/create_test_db_dump.py +1 -1
- pixeltable/tool/create_test_video.py +1 -1
- pixeltable/tool/embed_udf.py +1 -1
- pixeltable/tool/mypy_plugin.py +28 -5
- pixeltable/type_system.py +100 -36
- pixeltable/utils/coco.py +5 -5
- pixeltable/utils/documents.py +15 -1
- pixeltable/utils/formatter.py +12 -13
- pixeltable/utils/s3.py +6 -3
- {pixeltable-0.2.21.dist-info → pixeltable-0.2.23.dist-info}/METADATA +158 -49
- pixeltable-0.2.23.dist-info/RECORD +153 -0
- pixeltable/exec/media_validation_node.py +0 -43
- pixeltable-0.2.21.dist-info/RECORD +0 -148
- {pixeltable-0.2.21.dist-info → pixeltable-0.2.23.dist-info}/LICENSE +0 -0
- {pixeltable-0.2.21.dist-info → pixeltable-0.2.23.dist-info}/WHEEL +0 -0
- {pixeltable-0.2.21.dist-info → pixeltable-0.2.23.dist-info}/entry_points.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pixeltable
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.23
|
|
4
4
|
Summary: Pixeltable: The Multimodal AI Data Plane
|
|
5
5
|
Author: Pixeltable, Inc.
|
|
6
6
|
Author-email: contact@pixeltable.com
|
|
@@ -16,9 +16,9 @@ Requires-Dist: cloudpickle (>=2.2.1,<3.0.0)
|
|
|
16
16
|
Requires-Dist: ftfy (>=6.2.0,<7.0.0)
|
|
17
17
|
Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
|
|
18
18
|
Requires-Dist: jmespath (>=1.0.1,<2.0.0)
|
|
19
|
+
Requires-Dist: lxml (>=5.0)
|
|
19
20
|
Requires-Dist: more-itertools (>=10.2,<11.0)
|
|
20
21
|
Requires-Dist: numpy (>=1.25,<2.0)
|
|
21
|
-
Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
|
|
22
22
|
Requires-Dist: pandas (>=2.0,<3.0)
|
|
23
23
|
Requires-Dist: pgvector (>=0.2.1,<0.3.0)
|
|
24
24
|
Requires-Dist: pillow (>=9.3.0)
|
|
@@ -36,54 +36,81 @@ Requires-Dist: tqdm (>=4.64)
|
|
|
36
36
|
Description-Content-Type: text/markdown
|
|
37
37
|
|
|
38
38
|
<div align="center">
|
|
39
|
-
<img src="https://raw.githubusercontent.com/pixeltable/pixeltable/main/docs/source/data/pixeltable-logo-large.png"
|
|
39
|
+
<img src="https://raw.githubusercontent.com/pixeltable/pixeltable/main/docs/source/data/pixeltable-logo-large.png"
|
|
40
|
+
alt="Pixeltable" width="50%" />
|
|
40
41
|
<br></br>
|
|
41
42
|
|
|
43
|
+
<h2>AI Data Insfrastructure — Declarative, Multimodal, and Incremental</h2>
|
|
44
|
+
|
|
42
45
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
43
46
|

|
|
44
47
|

|
|
45
48
|
<br>
|
|
46
49
|
[](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml)
|
|
47
50
|
[](https://github.com/pixeltable/pixeltable/actions/workflows/nightly.yml)
|
|
51
|
+
[](https://app.fossa.com/projects/git%2Bgithub.com%2Fpixeltable%2Fpixeltable?ref=badge_shield&issueType=security)
|
|
48
52
|
[](https://pypi.org/project/pixeltable/)
|
|
49
|
-
<a target="_blank" href="https://huggingface.co/Pixeltable">
|
|
50
|
-
|
|
51
|
-
|
|
53
|
+
<a target="_blank" href="https://huggingface.co/Pixeltable">
|
|
54
|
+
<img src="https://img.shields.io/badge/🤗-HF Space-FF7D04" alt="Visit our Hugging Face space"/>
|
|
55
|
+
</a>
|
|
56
|
+
|
|
57
|
+
[Installation](https://docs.pixeltable.com/docs/installation) |
|
|
58
|
+
[Documentation](https://pixeltable.readme.io/) |
|
|
59
|
+
[API Reference](https://pixeltable.github.io/pixeltable/) |
|
|
60
|
+
[Code Samples](https://github.com/pixeltable/pixeltable?tab=readme-ov-file#-code-samples) |
|
|
61
|
+
[Computer Vision](https://docs.pixeltable.com/docs/object-detection-in-videos) |
|
|
62
|
+
[LLM](https://docs.pixeltable.com/docs/document-indexing-and-rag)
|
|
52
63
|
</div>
|
|
53
64
|
|
|
54
|
-
Pixeltable is a Python library providing a declarative interface for multimodal data (text, images, audio, video).
|
|
65
|
+
Pixeltable is a Python library providing a declarative interface for multimodal data (text, images, audio, video).
|
|
66
|
+
It features built-in versioning, lineage tracking, and incremental updates, enabling users to **store**, **transform**,
|
|
67
|
+
**index**, and **iterate** on data for their ML workflows.
|
|
55
68
|
|
|
56
69
|
Data transformations, model inference, and custom logic are embedded as **computed columns**.
|
|
57
|
-
|
|
58
|
-
- **
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
70
|
+
|
|
71
|
+
- **Load/Query all data types**: Interact with
|
|
72
|
+
[video data](https://github.com/pixeltable/pixeltable?tab=readme-ov-file#import-media-data-into-pixeltable-videos-images-audio)
|
|
73
|
+
at the [frame level](https://github.com/pixeltable/pixeltable?tab=readme-ov-file#text-and-image-similarity-search-on-video-frames-with-embedding-indexes)
|
|
74
|
+
and documents at the [chunk level](https://github.com/pixeltable/pixeltable?tab=readme-ov-file#automate-data-operations-with-views-eg-split-documents-into-chunks)
|
|
75
|
+
- **Incremental updates for data transformation**: Maintain an
|
|
76
|
+
[embedding index](https://docs.pixeltable.com/docs/embedding-vector-indexes) colocated with your data
|
|
77
|
+
- **Lazy evaluation and cache management**: Eliminates the need for
|
|
78
|
+
[manual frame extraction](https://docs.pixeltable.com/docs/object-detection-in-videos)
|
|
79
|
+
- **Integrates with any Python libraries**: Use
|
|
80
|
+
[built-in and custom functions (UDFs)](https://docs.pixeltable.com/docs/user-defined-functions-udfs)
|
|
81
|
+
without complex pipelines
|
|
82
|
+
- **Data format agnostic and extensibility**: Access tables as Parquet files,
|
|
83
|
+
[PyTorch datasets](https://pixeltable.github.io/pixeltable/api/data-frame/#pixeltable.DataFrame.to_pytorch_dataset),
|
|
84
|
+
or [COCO annotations](https://pixeltable.github.io/pixeltable/api/table/#pixeltable.Table.to_coco_dataset)
|
|
62
85
|
|
|
63
86
|
## 💾 Installation
|
|
64
87
|
|
|
65
88
|
```python
|
|
66
89
|
pip install pixeltable
|
|
67
90
|
```
|
|
91
|
+
|
|
68
92
|
**Pixeltable is persistent. Unlike in-memory Python libraries such as Pandas, Pixeltable is a database.**
|
|
69
93
|
|
|
70
94
|
## 💡 Getting Started
|
|
95
|
+
|
|
71
96
|
Learn how to create tables, populate them with data, and enhance them with built-in or user-defined transformations.
|
|
72
97
|
|
|
73
98
|
| Topic | Notebook | Topic | Notebook |
|
|
74
99
|
|:----------|:-----------------|:-------------------------|:---------------------------------:|
|
|
75
|
-
| 10-Minute Tour of Pixeltable | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/tutorials/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | Tables and Data Operations | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/fundamentals/tables-and-data-operations.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a>
|
|
76
|
-
| User-Defined Functions (UDFs) | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/howto/udfs-in-pixeltable.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | Object Detection Models | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/tutorials/object-detection-in-videos.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a>
|
|
77
|
-
|
|
|
78
|
-
| Integrating with Label Studio | <a target="_blank" href="https://pixeltable.readme.io/docs/label-studio"> <img src="https://img.shields.io/badge/
|
|
100
|
+
| 10-Minute Tour of Pixeltable | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/tutorials/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | Tables and Data Operations | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/fundamentals/tables-and-data-operations.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> |
|
|
101
|
+
| User-Defined Functions (UDFs) | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/howto/udfs-in-pixeltable.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | Object Detection Models | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/tutorials/object-detection-in-videos.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> |
|
|
102
|
+
| Incremental Prompt Engineering | <a target="_blank" href="https://colab.research.google.com/github/mistralai/cookbook/blob/main/third_party/Pixeltable/incremental_prompt_engineering_and_model_comparison.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Github"/> | Working with External Files | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/howto/working-with-external-files.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> |
|
|
103
|
+
| Integrating with Label Studio | <a target="_blank" href="https://pixeltable.readme.io/docs/label-studio"> <img src="https://img.shields.io/badge/Documentation-013056" alt="Visit our documentation"/></a> | Audio/Video Transcript Indexing | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/tutorials/audio-transcriptions.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> |
|
|
104
|
+
| Multimodal Application | <a target="_blank" href="https://huggingface.co/spaces/Pixeltable/Multimodal-Powerhouse"> <img src="https://img.shields.io/badge/Hugging Face-FF7D04" alt="Visit our documentation"/></a> | Document Indexing and RAG | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/release/tutorials/rag-demo.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> |
|
|
79
105
|
|
|
80
106
|
## 🧱 Code Samples
|
|
81
107
|
|
|
82
108
|
### Import media data into Pixeltable (videos, images, audio...)
|
|
109
|
+
|
|
83
110
|
```python
|
|
84
111
|
import pixeltable as pxt
|
|
85
112
|
|
|
86
|
-
v = pxt.create_table('external_data.videos', {'video': pxt.
|
|
113
|
+
v = pxt.create_table('external_data.videos', {'video': pxt.Video})
|
|
87
114
|
|
|
88
115
|
prefix = 's3://multimedia-commons/'
|
|
89
116
|
paths = [
|
|
@@ -93,15 +120,17 @@ paths = [
|
|
|
93
120
|
]
|
|
94
121
|
v.insert({'video': prefix + p} for p in paths)
|
|
95
122
|
```
|
|
123
|
+
|
|
96
124
|
Learn how to [work with data in Pixeltable](https://pixeltable.readme.io/docs/working-with-external-files).
|
|
97
125
|
|
|
98
126
|
### Object detection in images using DETR model
|
|
127
|
+
|
|
99
128
|
```python
|
|
100
129
|
import pixeltable as pxt
|
|
101
130
|
from pixeltable.functions import huggingface
|
|
102
131
|
|
|
103
132
|
# Create a table to store data persistently
|
|
104
|
-
t = pxt.create_table('image', {'image': pxt.
|
|
133
|
+
t = pxt.create_table('image', {'image': pxt.Image})
|
|
105
134
|
|
|
106
135
|
# Insert some images
|
|
107
136
|
prefix = 'https://upload.wikimedia.org/wikipedia/commons'
|
|
@@ -113,18 +142,22 @@ paths = [
|
|
|
113
142
|
t.insert({'image': prefix + p} for p in paths)
|
|
114
143
|
|
|
115
144
|
# Add a computed column for image classification
|
|
116
|
-
t
|
|
117
|
-
|
|
118
|
-
|
|
145
|
+
t.add_computed_column(classification=huggingface.detr_for_object_detection(
|
|
146
|
+
t.image,
|
|
147
|
+
model_id='facebook/detr-resnet-50'
|
|
148
|
+
))
|
|
119
149
|
|
|
120
150
|
# Retrieve the rows where cats have been identified
|
|
121
151
|
t.select(animal = t.image,
|
|
122
152
|
classification = t.classification.label_text[0]) \
|
|
123
153
|
.where(t.classification.label_text[0]=='cat').head()
|
|
124
154
|
```
|
|
125
|
-
|
|
155
|
+
|
|
156
|
+
Learn about computed columns and object detection:
|
|
157
|
+
[Comparing object detection models](https://pixeltable.readme.io/docs/object-detection-in-videos).
|
|
126
158
|
|
|
127
159
|
### Extend Pixeltable's capabilities with user-defined functions
|
|
160
|
+
|
|
128
161
|
```python
|
|
129
162
|
@pxt.udf
|
|
130
163
|
def draw_boxes(img: PIL.Image.Image, boxes: list[list[float]]) -> PIL.Image.Image:
|
|
@@ -134,9 +167,12 @@ def draw_boxes(img: PIL.Image.Image, boxes: list[list[float]]) -> PIL.Image.Imag
|
|
|
134
167
|
d.rectangle(box, width=3) # Draw bounding box rectangles on the copied image
|
|
135
168
|
return result
|
|
136
169
|
```
|
|
137
|
-
|
|
170
|
+
|
|
171
|
+
Learn more about user-defined functions:
|
|
172
|
+
[UDFs in Pixeltable](https://pixeltable.readme.io/docs/user-defined-functions-udfs).
|
|
138
173
|
|
|
139
174
|
### Automate data operations with views, e.g., split documents into chunks
|
|
175
|
+
|
|
140
176
|
```python
|
|
141
177
|
# In this example, the view is defined by iteration over the chunks of a DocumentSplitter
|
|
142
178
|
chunks_table = pxt.create_view(
|
|
@@ -147,36 +183,45 @@ chunks_table = pxt.create_view(
|
|
|
147
183
|
separators='token_limit', limit=300)
|
|
148
184
|
)
|
|
149
185
|
```
|
|
150
|
-
|
|
186
|
+
|
|
187
|
+
Learn how to leverage views to build your
|
|
188
|
+
[RAG workflow](https://pixeltable.readme.io/docs/document-indexing-and-rag).
|
|
151
189
|
|
|
152
190
|
### Evaluate model performance
|
|
191
|
+
|
|
153
192
|
```python
|
|
154
193
|
# The computation of the mAP metric can become a query over the evaluation output
|
|
155
194
|
frames_view.select(mean_ap(frames_view.eval_yolox_tiny), mean_ap(frames_view.eval_yolox_m)).show()
|
|
156
195
|
```
|
|
196
|
+
|
|
157
197
|
Learn how to leverage Pixeltable for [Model analytics](https://pixeltable.readme.io/docs/object-detection-in-videos).
|
|
158
198
|
|
|
159
199
|
### Working with inference services
|
|
200
|
+
|
|
160
201
|
```python
|
|
161
|
-
chat_table = pxt.create_table('together_demo.chat', {'input': pxt.
|
|
202
|
+
chat_table = pxt.create_table('together_demo.chat', {'input': pxt.String})
|
|
162
203
|
|
|
163
204
|
# The chat-completions API expects JSON-formatted input:
|
|
164
205
|
messages = [{'role': 'user', 'content': chat_table.input}]
|
|
165
206
|
|
|
166
207
|
# This example shows how additional parameters from the Together API can be used in Pixeltable
|
|
167
|
-
chat_table
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
208
|
+
chat_table.add_computed_column(
|
|
209
|
+
output=chat_completions(
|
|
210
|
+
messages=messages,
|
|
211
|
+
model='mistralai/Mixtral-8x7B-Instruct-v0.1',
|
|
212
|
+
max_tokens=300,
|
|
213
|
+
stop=['\n'],
|
|
214
|
+
temperature=0.7,
|
|
215
|
+
top_p=0.9,
|
|
216
|
+
top_k=40,
|
|
217
|
+
repetition_penalty=1.1,
|
|
218
|
+
logprobs=1,
|
|
219
|
+
echo=True
|
|
220
|
+
)
|
|
221
|
+
)
|
|
222
|
+
chat_table.add_computed_column(
|
|
223
|
+
response=chat_table.output.choices[0].message.content
|
|
178
224
|
)
|
|
179
|
-
chat_table['response'] = chat_table.output.choices[0].message.content
|
|
180
225
|
|
|
181
226
|
# Start a conversation
|
|
182
227
|
chat_table.insert([
|
|
@@ -185,16 +230,18 @@ chat_table.insert([
|
|
|
185
230
|
])
|
|
186
231
|
chat_table.select(chat_table.input, chat_table.response).head()
|
|
187
232
|
```
|
|
233
|
+
|
|
188
234
|
Learn how to interact with inference services such as [Together AI](https://pixeltable.readme.io/docs/together-ai) in Pixeltable.
|
|
189
235
|
|
|
190
236
|
### Text and image similarity search on video frames with embedding indexes
|
|
237
|
+
|
|
191
238
|
```python
|
|
192
239
|
import pixeltable as pxt
|
|
193
240
|
from pixeltable.functions.huggingface import clip_image, clip_text
|
|
194
241
|
from pixeltable.iterators import FrameIterator
|
|
195
242
|
import PIL.Image
|
|
196
243
|
|
|
197
|
-
video_table = pxt.create_table('videos', {'video': pxt.
|
|
244
|
+
video_table = pxt.create_table('videos', {'video': pxt.Video})
|
|
198
245
|
|
|
199
246
|
video_table.insert([{'video': '/video.mp4'}])
|
|
200
247
|
|
|
@@ -221,19 +268,55 @@ frames_view.order_by(sim, asc=False).limit(5).select(frames_view.frame, sim=sim)
|
|
|
221
268
|
sample_text = 'red truck'
|
|
222
269
|
sim = frames_view.frame.similarity(sample_text)
|
|
223
270
|
frames_view.order_by(sim, asc=False).limit(5).select(frames_view.frame, sim=sim).collect()
|
|
224
|
-
|
|
225
271
|
```
|
|
272
|
+
|
|
226
273
|
Learn how to work with [Embedding and Vector Indexes](https://docs.pixeltable.com/docs/embedding-vector-indexes).
|
|
227
274
|
|
|
275
|
+
## 🔄 AI Stack Comparison
|
|
276
|
+
|
|
277
|
+
### 🎯 Computer Vision Workflows
|
|
278
|
+
|
|
279
|
+
| Requirement | Traditional | Pixeltable |
|
|
280
|
+
|-------------|---------------------|------------|
|
|
281
|
+
| Frame Extraction | ffmpeg + custom code | Automatic via FrameIterator |
|
|
282
|
+
| Object Detection | Multiple scripts + caching | Single computed column |
|
|
283
|
+
| Video Indexing | Custom pipelines + Vector DB | Native similarity search |
|
|
284
|
+
| Annotation Management | Separate tools + custom code | Label Studio integration |
|
|
285
|
+
| Model Evaluation | Custom metrics pipeline | Built-in mAP computation |
|
|
286
|
+
|
|
287
|
+
### 🤖 LLM Workflows
|
|
288
|
+
|
|
289
|
+
| Requirement | Traditional | Pixeltable |
|
|
290
|
+
|-------------|---------------------|------------|
|
|
291
|
+
| Document Chunking | Tool + custom code | Native DocumentSplitter |
|
|
292
|
+
| Embedding Generation | Separate pipeline + caching | Computed columns |
|
|
293
|
+
| Vector Search | External vector DB | Built-in vector indexing |
|
|
294
|
+
| Prompt Management | Custom tracking solution | Version-controlled columns |
|
|
295
|
+
| Chain Management | Tool + custom code | Computed column DAGs |
|
|
296
|
+
|
|
297
|
+
### 🎨 Multimodal Workflows
|
|
298
|
+
|
|
299
|
+
| Requirement | Traditional | Pixeltable |
|
|
300
|
+
|-------------|---------------------|------------|
|
|
301
|
+
| Data Types | Multiple storage systems | Unified table interface |
|
|
302
|
+
| Cross-Modal Search | Complex integration | Native similarity support |
|
|
303
|
+
| Pipeline Orchestration | Multiple tools (Airflow, etc.) | Single declarative interface |
|
|
304
|
+
| Asset Management | Custom tracking system | Automatic lineage |
|
|
305
|
+
| Quality Control | Multiple validation tools | Computed validation columns |
|
|
306
|
+
|
|
228
307
|
## ❓ FAQ
|
|
229
308
|
|
|
230
309
|
### What is Pixeltable?
|
|
231
310
|
|
|
232
|
-
Pixeltable unifies data storage, versioning, and indexing with orchestration and model versioning under a declarative
|
|
311
|
+
Pixeltable unifies data storage, versioning, and indexing with orchestration and model versioning under a declarative
|
|
312
|
+
table interface, with transformations, model inference, and custom logic represented as computed columns.
|
|
233
313
|
|
|
234
314
|
### What problems does Pixeltable solve?
|
|
235
315
|
|
|
236
|
-
Today's solutions for AI app development require extensive custom coding and infrastructure plumbing.
|
|
316
|
+
Today's solutions for AI app development require extensive custom coding and infrastructure plumbing.
|
|
317
|
+
Tracking lineage and versions between and across data transformations, models, and deployments is cumbersome.
|
|
318
|
+
Pixeltable lets ML Engineers and Data Scientists focus on exploration, modeling, and app development without
|
|
319
|
+
dealing with the customary data plumbing.
|
|
237
320
|
|
|
238
321
|
### What does Pixeltable provide me with? Pixeltable provides:
|
|
239
322
|
|
|
@@ -254,22 +337,48 @@ Today's solutions for AI app development require extensive custom coding and inf
|
|
|
254
337
|
- You never need to re-run pipelines from scratch because you’re adding data
|
|
255
338
|
- **It integrates with any existing Python code or libraries**
|
|
256
339
|
- Bring your ever-changing code and workloads
|
|
257
|
-
- You choose the models, tools, and AI practices (e.g., your embedding model for a vector index);
|
|
340
|
+
- You choose the models, tools, and AI practices (e.g., your embedding model for a vector index);
|
|
341
|
+
Pixeltable orchestrates the data
|
|
258
342
|
|
|
259
343
|
### What is Pixeltable not providing?
|
|
260
344
|
|
|
261
|
-
- Pixeltable is not a low-code, prescriptive AI solution. We empower you to use the best frameworks and techniques for
|
|
262
|
-
|
|
345
|
+
- Pixeltable is not a low-code, prescriptive AI solution. We empower you to use the best frameworks and techniques for
|
|
346
|
+
your specific needs.
|
|
347
|
+
- We do not aim to replace your existing AI toolkit, but rather enhance it by streamlining the underlying data
|
|
348
|
+
infrastructure and orchestration.
|
|
263
349
|
|
|
264
350
|
> [!TIP]
|
|
265
|
-
> Check out the [Integrations](https://pixeltable.readme.io/docs/working-with-openai) section, and feel free to submit
|
|
351
|
+
> Check out the [Integrations](https://pixeltable.readme.io/docs/working-with-openai) section, and feel free to submit
|
|
352
|
+
> a request for additional ones.
|
|
353
|
+
|
|
354
|
+
## 🤝 Contributing to Pixeltable
|
|
355
|
+
|
|
356
|
+
We're excited to welcome contributions from the community! Here's how you can get involved:
|
|
357
|
+
|
|
358
|
+
### 🐛 Report Issues
|
|
359
|
+
|
|
360
|
+
- Found a bug? [Open an issue](https://github.com/pixeltable/pixeltable/issues)
|
|
361
|
+
- Include steps to reproduce and environment details
|
|
362
|
+
|
|
363
|
+
### 💡 Submit Changes
|
|
364
|
+
|
|
365
|
+
- Fork the repository
|
|
366
|
+
- Create a feature branch
|
|
367
|
+
- Submit a [pull request](https://github.com/pixeltable/pixeltable/pulls)
|
|
368
|
+
- See our [Contributing Guide](CONTRIBUTING.md) for detailed instructions
|
|
369
|
+
|
|
370
|
+
### 💬 Join the Discussion
|
|
371
|
+
|
|
372
|
+
- Have questions? Start a [Discussion](https://github.com/pixeltable/pixeltable/discussions)
|
|
373
|
+
- Share your Pixeltable projects and use cases
|
|
374
|
+
- Help others in the community
|
|
266
375
|
|
|
267
|
-
|
|
376
|
+
### 📝 Improve Documentation
|
|
268
377
|
|
|
269
|
-
|
|
270
|
-
|
|
378
|
+
- Suggest examples and tutorials
|
|
379
|
+
- Propose improvements
|
|
271
380
|
|
|
272
|
-
##
|
|
381
|
+
## 🏢 License
|
|
273
382
|
|
|
274
383
|
This library is licensed under the Apache 2.0 License.
|
|
275
384
|
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
pixeltable/__init__.py,sha256=gv2jvZ7H5tEjLear10E7hSH9jF5Mw6iSeryvRp88bOE,1391
|
|
2
|
+
pixeltable/__version__.py,sha256=7KY-iWvvGLR_f-tQMjxiOTfMCj-z3KM_23EsiPZSTl8,114
|
|
3
|
+
pixeltable/catalog/__init__.py,sha256=Ar6_F_6C7tkznIlCPBHVHDop5YssBDjKQr2NPQ21QCI,484
|
|
4
|
+
pixeltable/catalog/catalog.py,sha256=tyDyI5wQw7vV6_FChrp9qgGCRClcjiSdW3eygYT0p9s,7849
|
|
5
|
+
pixeltable/catalog/column.py,sha256=ezeKoGl6aBTzSZBihDA6vdETcvyCguAD4OsmrxWs73A,9595
|
|
6
|
+
pixeltable/catalog/dir.py,sha256=fG_BQM-fLuABpTstMVH-9dvZPx7kqi3sgTQgKveVXJI,922
|
|
7
|
+
pixeltable/catalog/globals.py,sha256=2P9cEfgnYQ19MFS4OsbnA8ZQkWLZPqJmmknDYWCXjWw,2415
|
|
8
|
+
pixeltable/catalog/insertable_table.py,sha256=hOsdYhkhtRcDOrRkweIGFUfjweWF3fLUErkUTlGYoSU,7172
|
|
9
|
+
pixeltable/catalog/named_function.py,sha256=W8vikP_3jMJ9pQQsksO2EfQAlaVxuQHBlo65M4924dc,1150
|
|
10
|
+
pixeltable/catalog/path.py,sha256=QgccEi_QOfaKt8YsR2zLtd_z7z7QQkU_1kprJFi2SPQ,1677
|
|
11
|
+
pixeltable/catalog/path_dict.py,sha256=4b9_Ax7Q8tkmoCYPaKNedpQkU17pE0oGDd2XB53eNZA,5979
|
|
12
|
+
pixeltable/catalog/schema_object.py,sha256=qhpeeUPOYT5doDbsyUNBcPm5QzAQPCAsikqh1PQ6d1k,2226
|
|
13
|
+
pixeltable/catalog/table.py,sha256=7UwLGydwfvzBoQXjqZdn8toPU6-6t0OySPFIiwj6KkA,48477
|
|
14
|
+
pixeltable/catalog/table_version.py,sha256=W4yrvn-mF63CQ8cQGmjbjYTCf1xYuV6k6rI__ubJOIc,57898
|
|
15
|
+
pixeltable/catalog/table_version_path.py,sha256=CczGbcz5ESq663arreri_p4chMZHozgG6k3y-ajkJN4,5787
|
|
16
|
+
pixeltable/catalog/view.py,sha256=7KQeHiqKmGrS_hjt6QjIjZhOvl9-XKBAi7A9_9_Worg,10715
|
|
17
|
+
pixeltable/dataframe.py,sha256=SGge2DxcjT0gaKpWsmQJhqlqX6b75nefoKAh1Ld4S8Y,33639
|
|
18
|
+
pixeltable/env.py,sha256=oIFeLJVDQuQgW8REAoNkY45BF2o02NxtysgAlJNEBc8,30520
|
|
19
|
+
pixeltable/exceptions.py,sha256=NuFY2WtkQpLfLHT_J70kOw9Tr0kEDkkgo-u7As4Gaq4,410
|
|
20
|
+
pixeltable/exec/__init__.py,sha256=QmaKzkZ704nbJ1BiAFwjYvIX3R2_EHzpO8R7HxcEY-g,481
|
|
21
|
+
pixeltable/exec/aggregation_node.py,sha256=0LdoPp_CR_UbcS60XkDw66SqlrQnw6Dy6KDWqi4PJ6k,4005
|
|
22
|
+
pixeltable/exec/cache_prefetch_node.py,sha256=hrJA5PxLSmFRuZdZO4Nc-vy6_PSb9YO4rnJc6MSTPHE,12017
|
|
23
|
+
pixeltable/exec/component_iteration_node.py,sha256=ABuXGbDRQWLGuaBnfK7bvOxCrz81vMMiAvXHHI8SX4c,4930
|
|
24
|
+
pixeltable/exec/data_row_batch.py,sha256=qPN0GtBGt72Jnap-UN53VlUPOpYEvqJhp1nuTeJ_8Co,2925
|
|
25
|
+
pixeltable/exec/exec_context.py,sha256=-FbfYLcGOL7mviru5dE1A7x4YxLbdKoXBHN3OSbqbcg,1084
|
|
26
|
+
pixeltable/exec/exec_node.py,sha256=glESCX17lcVYE0SzDb1O2HzKiWwaiXZRsB5DHYwMwnQ,3248
|
|
27
|
+
pixeltable/exec/expr_eval_node.py,sha256=nEMnVTikQyTa3C9xfe6tuWLpQceM_7XJqfMPB_oI0wU,11890
|
|
28
|
+
pixeltable/exec/in_memory_data_node.py,sha256=uK3puLZDzUSJUJwGwkTMYoPqNIvE67jtQodhE2sCz2M,3452
|
|
29
|
+
pixeltable/exec/row_update_node.py,sha256=b8yuKtkmI2Q6U-7svKbkjdM3Z85ddZoJgJgUa17j-YE,2773
|
|
30
|
+
pixeltable/exec/sql_node.py,sha256=9huPLpMqq4J5JhSh5vtt-T4K1Tu3EXJ9omv6mMi-zD4,17616
|
|
31
|
+
pixeltable/exprs/__init__.py,sha256=toKJm6y1_2fD8Fo-ld5feD-nMN-yZGMWwMHYyb8vsrg,967
|
|
32
|
+
pixeltable/exprs/arithmetic_expr.py,sha256=oUe7r3CQo3VJOzFUpmDDuYZMWc7zEcngNdb17GvTseQ,6321
|
|
33
|
+
pixeltable/exprs/array_slice.py,sha256=HgB9BV8a9Qg6Y-t7PRjFszr40jDArensZ75-wDyFpcQ,2179
|
|
34
|
+
pixeltable/exprs/column_property_ref.py,sha256=rCYNv9o_UMVhIz-U0-bhTxLf4c7_CTYWH6VB9LLo0F8,3787
|
|
35
|
+
pixeltable/exprs/column_ref.py,sha256=Od4UAvSr--TtsWkJhqNBEkUWmOGTNEhLJRAZQerQ44c,9817
|
|
36
|
+
pixeltable/exprs/comparison.py,sha256=s1f4fGytnrhXN5wfuGlKUoNpp5EO4LLrFNA321OH1Kk,5100
|
|
37
|
+
pixeltable/exprs/compound_predicate.py,sha256=H35SB053aCHigtPp_JYrOTLRC2AGDQK3NAkOYOBPDQg,3845
|
|
38
|
+
pixeltable/exprs/data_row.py,sha256=rLtKxlst9mK6684A5y-nsjBcalyKEcKAWcYCtNpK10w,9852
|
|
39
|
+
pixeltable/exprs/expr.py,sha256=QlVslNfAOPxPVvsNiEv_k_S2QWp4-8dc3eocT6zclNU,29986
|
|
40
|
+
pixeltable/exprs/expr_dict.py,sha256=xkvo_iVPOLMq3WkBZQ2FOoXqYoebuV6XGlidPJxdOkY,1588
|
|
41
|
+
pixeltable/exprs/expr_set.py,sha256=kAV-PbudbAlW0dqhKqfstXnxKbK_FWKXyrzkpvDrpjQ,2330
|
|
42
|
+
pixeltable/exprs/function_call.py,sha256=Sp23zxpzA4f7ztd71sn_B0sgos1hyMPKRR8wICzxOZ4,21365
|
|
43
|
+
pixeltable/exprs/globals.py,sha256=5pwn5vdi-EEpYBpPty658YV45myY7W0iFIfTH7QIzak,2032
|
|
44
|
+
pixeltable/exprs/in_predicate.py,sha256=eR6qFf_0lsqM0jPRI9IxTsDlXM6Bt9RBY9a05bZ6bDs,3782
|
|
45
|
+
pixeltable/exprs/inline_expr.py,sha256=OF-yLIoU7LfHqFdbZkZnC1oQ2_X26WyHbfxbxrdZr-k,7104
|
|
46
|
+
pixeltable/exprs/is_null.py,sha256=x9-CijQf1JuUioUAv1_8OvP9nK9Ahjc1wJfgE1QEOL8,1091
|
|
47
|
+
pixeltable/exprs/json_mapper.py,sha256=OJ8ohAZhuLxTGtmmbB8voVG9AzUuCHLHuNBi-3hJZEg,4544
|
|
48
|
+
pixeltable/exprs/json_path.py,sha256=BUpjc7u-GOHcmQ1d3FzIZlGVE6VLioGOfMwA_gpWvEE,6810
|
|
49
|
+
pixeltable/exprs/literal.py,sha256=TTKb0gw6qck9D61SwVDuBrLrBrGwEhkCB-m0ILFpWFk,3764
|
|
50
|
+
pixeltable/exprs/method_ref.py,sha256=x9rQzlRQlVnbTpBQoV0COFsiOPstJcOifXl0lJC-roI,2614
|
|
51
|
+
pixeltable/exprs/object_ref.py,sha256=GVg6uxZnKwFVTC0kouJq-wMFP-gUPb_ld_syHrsGMdE,1283
|
|
52
|
+
pixeltable/exprs/row_builder.py,sha256=7f-h4y8xv0ktkk6GYqGrMJvLSwkVYtMPHsBBIskmQLw,18435
|
|
53
|
+
pixeltable/exprs/rowid_ref.py,sha256=hjGrbk9zHH3H-00uFAopyRvLTeQeB2e44kAJUAxiy3k,4400
|
|
54
|
+
pixeltable/exprs/similarity_expr.py,sha256=snOseawC4ySvyHo8TCqbh_bDxIrENfIzO_0lXlzL-BA,4016
|
|
55
|
+
pixeltable/exprs/sql_element_cache.py,sha256=DRW5Aa0WQZ-yMf5anlUg-_Rmq3Oz3G6Us1X_KseMC68,1229
|
|
56
|
+
pixeltable/exprs/type_cast.py,sha256=BTnhgoll7LVZdOU_282QlzGZ4EEMTzPYcNdDcFOfkTs,1837
|
|
57
|
+
pixeltable/exprs/variable.py,sha256=VATAmLaPrQipv2AXqg-m6FYMLNGyhPtys8c5Et8Ba0g,1414
|
|
58
|
+
pixeltable/ext/__init__.py,sha256=iO0J_Jfnv38F5y40sDAW54gpXjIyZgOGgoWQJAwjQec,423
|
|
59
|
+
pixeltable/ext/functions/__init__.py,sha256=hIjPEKC5E5uJOXlQqUyhP9yn9ZqWOCJAlj0kXWDlhlE,159
|
|
60
|
+
pixeltable/ext/functions/whisperx.py,sha256=jojjNhazcYiAh1scwUl-erhIDRr4kOTkcLrjy0xcp6g,2325
|
|
61
|
+
pixeltable/ext/functions/yolox.py,sha256=k-pQTelv4Tea3AXvDB7Kc7YCIa1uexjVGqxETP0B_hc,5351
|
|
62
|
+
pixeltable/func/__init__.py,sha256=WjftUGyNkd6bF_qSxqZ5Gd7Elf8oExb3dUlpydhdFTo,407
|
|
63
|
+
pixeltable/func/aggregate_function.py,sha256=LM6DMldmaUJNlQgEqPFQIKVmwjl3IpBxS4Y61G8-zG0,9477
|
|
64
|
+
pixeltable/func/callable_function.py,sha256=PO5Mn5WL2cd7y5LcKr_K0AaYHf2-1NfuXP2IPOfsiVs,4933
|
|
65
|
+
pixeltable/func/expr_template_function.py,sha256=2co1I9Epzh2ROyQ2WYJAfEbw8OxTbnc0Wp3GOvBLRBU,4043
|
|
66
|
+
pixeltable/func/function.py,sha256=BLJbahyKteGemp0EcG7joTrgrG2hnIfwHkuKoLZm4mo,6941
|
|
67
|
+
pixeltable/func/function_registry.py,sha256=fBXe7NKyk8_JzZz6fsS0LF-WHTdMnmIP_XzrICuj9fA,12328
|
|
68
|
+
pixeltable/func/globals.py,sha256=sEwn6lGgHMp6VQORb_P5qRd_-Q2_bUSqvqM9-XPN_ec,1483
|
|
69
|
+
pixeltable/func/query_template_function.py,sha256=pGqwtWiPsEmo7phVoJJODiuD1Sh0gZoW4BpKnZV9cRE,3537
|
|
70
|
+
pixeltable/func/signature.py,sha256=vBKs3igtijTQGK7rEGTxBiOznKo6Tj9Ut6OrSfiVcA0,8609
|
|
71
|
+
pixeltable/func/udf.py,sha256=_883xbGujeIhYOz-lWH5SvLn22F5JVpP6O_MRvpU4hU,7362
|
|
72
|
+
pixeltable/functions/__init__.py,sha256=EtR9M3ewYpmtHeshNULqZVBd87bNeKSFAdpOuWCMl6o,389
|
|
73
|
+
pixeltable/functions/anthropic.py,sha256=P1E5o4-8QP1LTIUsWVgo_wMJ4WOnxtXUUXuFWUagChU,3032
|
|
74
|
+
pixeltable/functions/audio.py,sha256=7213nTnqKJ6vM9kalaoJ283OwX5SGEJN10vDhaRNZ6E,644
|
|
75
|
+
pixeltable/functions/fireworks.py,sha256=qwFC_eIaDs-glxyJ_IVXaNGkpgPzeRsQ_SdpzueBxq0,2605
|
|
76
|
+
pixeltable/functions/globals.py,sha256=pCFX2a_N87SwG9GxyPjSOC3TVMowMB6XIHSWKfFOuGE,3917
|
|
77
|
+
pixeltable/functions/huggingface.py,sha256=hcScloxprIuyf89_bSnQi-N3VZRQosmZfBhmDJF_eTc,20814
|
|
78
|
+
pixeltable/functions/image.py,sha256=3Qm4ybAT_o4YUl3bzhEXy8dKOwgZ7RCUV-ky-dbL_jc,13836
|
|
79
|
+
pixeltable/functions/json.py,sha256=ehCnBA5WiIl-crV9PFVgmxrsWsiO8FpRs9LDwcSpLa4,879
|
|
80
|
+
pixeltable/functions/llama_cpp.py,sha256=1awALuAXVpQH64l7vQlM8gvxLDix4c1-6DV3nG5RHu4,3881
|
|
81
|
+
pixeltable/functions/mistralai.py,sha256=zljm0ZfucnJFaRyMQHkjyD51vEe0OlurACin7kCNmxk,5569
|
|
82
|
+
pixeltable/functions/ollama.py,sha256=eZh461HvChjlr0CvQyd93m7qrv889PAoM-Z1IKierY0,4335
|
|
83
|
+
pixeltable/functions/openai.py,sha256=e1-NZP5v0h9Axlo1oxUYzWfVRxKZVhOrKnqwcy0jH88,15583
|
|
84
|
+
pixeltable/functions/replicate.py,sha256=j8ZedScOMInmHWmriQSUOviw6tp8gQr-W6n21PNNL2g,2188
|
|
85
|
+
pixeltable/functions/string.py,sha256=VqzhVildxTt_XblW89Kl5Zd6MVOU71eaX2LTMY5jkUg,20366
|
|
86
|
+
pixeltable/functions/timestamp.py,sha256=Q5l2iEscrS3ZfKAa4R940bSM_x4FsmF-PF2i-wQ_4_c,9096
|
|
87
|
+
pixeltable/functions/together.py,sha256=xbgQMlA2G2FXdoQR4O9DlkmBsm2mivAdiOnxtxcyPwo,9440
|
|
88
|
+
pixeltable/functions/util.py,sha256=GgKTzCjvzUQNjWtSObTkfxkvJ9GVUOzKimY45WhE25M,759
|
|
89
|
+
pixeltable/functions/video.py,sha256=yW1Lwqu4_jYXp1aAOUThKB5-_Qxy-In_vTgB5cuW7Lg,6809
|
|
90
|
+
pixeltable/functions/vision.py,sha256=K_E1Q-n2plPuFoOPlbKWRMiJp9dPgftIJ2T_o3TNL3I,15594
|
|
91
|
+
pixeltable/functions/whisper.py,sha256=f2wqRd0n9jSBqRZN3W93UaetiAHtbsK0j9jXR2j2kkQ,2913
|
|
92
|
+
pixeltable/globals.py,sha256=GDA5T6jpXlFHLMKraLZxZ9jTC99IiQfW85jkbTCPKKc,20002
|
|
93
|
+
pixeltable/index/__init__.py,sha256=XBwetNQQwnz0fiKwonOKhyy_U32l_cjt77kNvEIdjWs,102
|
|
94
|
+
pixeltable/index/base.py,sha256=zo0YvJI3oXiK6hZJztB36ZftKKhLfO75Zq3t-PeQA6M,1556
|
|
95
|
+
pixeltable/index/btree.py,sha256=JFerLyyLoBaD0FSF_jJ6iJFBVa-z_et--KdNR02xjRg,2264
|
|
96
|
+
pixeltable/index/embedding_index.py,sha256=g5bajq8Dn82o8ZlkDwPT5kBUClR4ZCuH6dttwCW6KWI,7793
|
|
97
|
+
pixeltable/io/__init__.py,sha256=PHqbiEJXFtCzTsia7LmsHLfBIkA41tzII3n9L4UkfJ8,547
|
|
98
|
+
pixeltable/io/external_store.py,sha256=H1jt7MDn464QRgBvU-PmcPcFlo3EZBCG7fKWEZXOfyc,16676
|
|
99
|
+
pixeltable/io/fiftyone.py,sha256=hH-FahW6BuMQY8lGa2atnNnJto_pK8kWrP_y_EMsq6g,6965
|
|
100
|
+
pixeltable/io/globals.py,sha256=9S9wnlIAuhZq7eC_GklTM_UX0UATK9fEagk8-SRCeXQ,17794
|
|
101
|
+
pixeltable/io/hf_datasets.py,sha256=o5fqm2CJAjhFd3z-NYGxN0jM1tfrp4szuUX0TGnyNRY,8316
|
|
102
|
+
pixeltable/io/label_studio.py,sha256=7KTro1H-AlVbwWuoYwU-mxH3zejZWTpQbz56uX-Wnjs,31078
|
|
103
|
+
pixeltable/io/pandas.py,sha256=7eHg7wnAfRA9eBk4iC0iSSVTKOM59Ne4pXokKWdt3dY,9793
|
|
104
|
+
pixeltable/io/parquet.py,sha256=4bAQNCahtLGuHRF669kLGx2MhOFmuwTkUYYHLC-qKcs,7862
|
|
105
|
+
pixeltable/iterators/__init__.py,sha256=sjldFckkT8aVRiKgEP6faeAK2NQBdzbmpwAeRhI1FkM,366
|
|
106
|
+
pixeltable/iterators/base.py,sha256=ZC0ZvXL4iw6AmT8cu-Mdx-T2UG9nmJYV1C6LK4efAfw,1669
|
|
107
|
+
pixeltable/iterators/document.py,sha256=AsvEmZ5RGRi3AFGCrH2_-UNx5rTCFA-0WmMYQBTQI20,19679
|
|
108
|
+
pixeltable/iterators/string.py,sha256=NG_fWc_GAITDfzl6MvrDOMrSoMcZdMZf6hPQztCSatE,1305
|
|
109
|
+
pixeltable/iterators/video.py,sha256=CKx6jHFW-393r4xN-ulVDDZFETSkhhMwT7bPX2NHVU8,9442
|
|
110
|
+
pixeltable/metadata/__init__.py,sha256=CI0ZWcxsCbXEWhdbbByNJFdSmvIBrMEbf_vqrVb0b-Q,2209
|
|
111
|
+
pixeltable/metadata/converters/convert_10.py,sha256=J1_r7LNNAWTdb042AwqFpJ4sEB-i4qhUdk5iOjcZk34,719
|
|
112
|
+
pixeltable/metadata/converters/convert_12.py,sha256=Ci-qyZW1gqci-8wnjeOB5afdq7KTuN-hVSV9OqSPx8g,162
|
|
113
|
+
pixeltable/metadata/converters/convert_13.py,sha256=yFR6lD3pOrZ46ZQBFKYvxiIYa7rRxh46Bsq7yiCBNak,1356
|
|
114
|
+
pixeltable/metadata/converters/convert_14.py,sha256=9e_JNm3a35Gs4dvFFaV-_jtCqp3ud6OEOqMIhTQmasE,428
|
|
115
|
+
pixeltable/metadata/converters/convert_15.py,sha256=jMfL5wGil0-gZeIfmCbgtR3LSVNyOcxVp6YRhBECqY4,1741
|
|
116
|
+
pixeltable/metadata/converters/convert_16.py,sha256=SvcWOYgLwRw_gLTnLbCSI9f2cpdkXazYOmmtJUOOzv4,476
|
|
117
|
+
pixeltable/metadata/converters/convert_17.py,sha256=vJg4y2lg53WSj9OSntWsdUiCr6yRgMQm0eFbs_Geqjg,861
|
|
118
|
+
pixeltable/metadata/converters/convert_18.py,sha256=NxSroQntVLgmvXfae1f0-jYJIhM2W7dhRY3I7g0RxPY,1482
|
|
119
|
+
pixeltable/metadata/converters/convert_19.py,sha256=QgUDjNoH87KZg_f3cx4k0ZR67NqWRhZQKIIojbqxSkg,2080
|
|
120
|
+
pixeltable/metadata/converters/convert_20.py,sha256=NLMeke9QUGqIJUe5MNqKmVdaLs6pPFrjyigImhogaT4,2768
|
|
121
|
+
pixeltable/metadata/converters/convert_21.py,sha256=YTztkbqOC2zQcTWrXfhrP8diUbfxy5DHwsu_IT-bBok,1115
|
|
122
|
+
pixeltable/metadata/converters/util.py,sha256=nycZk_UecJcrVZsIyxQrz5ngbke8-yfY-_UcERuzhPk,5983
|
|
123
|
+
pixeltable/metadata/notes.py,sha256=HJmeA9fo37iREFIhlbGbxWnwbwIDcvdqvJO-BavIvxE,597
|
|
124
|
+
pixeltable/metadata/schema.py,sha256=CnEsMqLn4hzLDaAr5lyd-NqiOUFQdvhxdCoXMR4Qcs4,9352
|
|
125
|
+
pixeltable/plan.py,sha256=GRR1HvQKxYfpFJzeuPa1Ku3aE3tR3g3bcRu7KlfsAqQ,37436
|
|
126
|
+
pixeltable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
127
|
+
pixeltable/store.py,sha256=IG33KEu0eHGD2CGMBVV-HmPj7xGUMCLmLGdJn49_4Cs,21990
|
|
128
|
+
pixeltable/tool/create_test_db_dump.py,sha256=Qm9uq-PbcwNhj4Iczw5_uYJXh4TIyOp4RVKXssVFAiU,11972
|
|
129
|
+
pixeltable/tool/create_test_video.py,sha256=4cQmqoKjn3juy7Ilty75gWBygqBxTZ1E9XPlrsC0Ssk,2931
|
|
130
|
+
pixeltable/tool/doc_plugins/griffe.py,sha256=J5zxyEUchfR3mkWmhx4Vjl_iSodL_pHiuOyD2eczbNU,2182
|
|
131
|
+
pixeltable/tool/doc_plugins/mkdocstrings.py,sha256=afq7XOaSC5WRmugkh-FMFMK8PqOgIlDIsJdD8cuPhtE,207
|
|
132
|
+
pixeltable/tool/doc_plugins/templates/material/udf.html.jinja,sha256=R-7Q57nmDd5BUea-F1-MjwjK3pq7uBHXNoSo8_QjZG4,4890
|
|
133
|
+
pixeltable/tool/embed_udf.py,sha256=EXvfvuzZm0uTgH-aAATSrKV8ixCU8OMwpzXlJMg845Y,299
|
|
134
|
+
pixeltable/tool/mypy_plugin.py,sha256=__oTFElirrK25jCX1z_asD_gxGnGxtD2TaU6r1if-Ic,1784
|
|
135
|
+
pixeltable/type_system.py,sha256=6U3bSQpXCnrxrpO4uCQ6jTO-OIbgnQB8A2agXnnlvkA,41935
|
|
136
|
+
pixeltable/utils/__init__.py,sha256=UYlrf6TIWJT0g-Hac0b34-dEk478B5Qx8dGco34YlIk,439
|
|
137
|
+
pixeltable/utils/arrow.py,sha256=UQkMxyU4G_ikUF9OnEHcaBXHjVqdPXHSY4JXGmuARhA,3776
|
|
138
|
+
pixeltable/utils/coco.py,sha256=WCUyoj0dTyJFbPB7frEQUvY92SlEPxQ068r-3QAwROE,7285
|
|
139
|
+
pixeltable/utils/code.py,sha256=AOw1u2r8_DQXpX-lxJhyHWARGrCRDXOJHFVgKOi54Uc,1231
|
|
140
|
+
pixeltable/utils/documents.py,sha256=B984nVigJgHZ5Rm-zX4LLuHuMnfmz-zr24bbAsc_y3w,2511
|
|
141
|
+
pixeltable/utils/filecache.py,sha256=6HKQdItqSSTQvj2HkSJulyhfBedi4PgC7umwxXGOVG8,10637
|
|
142
|
+
pixeltable/utils/formatter.py,sha256=5E_gDg11ClFI-5SthwkiqyE3hAok3JHDj4OSK9cJklM,9257
|
|
143
|
+
pixeltable/utils/http_server.py,sha256=xYPTvmYrkUpKfOaLDq08D-eHswkcgDf4qAt76ZFH6lM,2411
|
|
144
|
+
pixeltable/utils/media_store.py,sha256=YwvTjbVqC_aLbDvLuqnDSL8xeIVMZcmzp0ANuM6uMbw,3092
|
|
145
|
+
pixeltable/utils/pytorch.py,sha256=6RvOCjy_QV4gc-aht-3d0zoASkuv-warfpl87vgmuKw,3450
|
|
146
|
+
pixeltable/utils/s3.py,sha256=huA5hxDGkPIu18zWet76o0FsO7Vbtp-iPmnOzCe-MvA,586
|
|
147
|
+
pixeltable/utils/sql.py,sha256=j_tj0h4ffm-DhUIJbvGphxrVyBKlNTwDKqWGhRQ5_PY,795
|
|
148
|
+
pixeltable/utils/transactional_directory.py,sha256=UGzCrGtLR3hEEf8sYGuWBzLVFAEQml3vdIavigWeTBM,1349
|
|
149
|
+
pixeltable-0.2.23.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
150
|
+
pixeltable-0.2.23.dist-info/METADATA,sha256=WYtMLp2PmlhHkQl6WUyvJFuJ2X-_lSB1VUO3i56dKwc,18112
|
|
151
|
+
pixeltable-0.2.23.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
152
|
+
pixeltable-0.2.23.dist-info/entry_points.txt,sha256=TNI1Gb5vPwFrTdw6TimSYjO8FeK8c_HuPr28vcf7o_I,108
|
|
153
|
+
pixeltable-0.2.23.dist-info/RECORD,,
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
from typing import Iterable, Optional
|
|
3
|
-
|
|
4
|
-
from .data_row_batch import DataRowBatch
|
|
5
|
-
from .exec_node import ExecNode
|
|
6
|
-
import pixeltable.exprs as exprs
|
|
7
|
-
import pixeltable.exceptions as excs
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class MediaValidationNode(ExecNode):
|
|
11
|
-
"""Validation of selected media slots
|
|
12
|
-
Records exceptions in the rows of the input batch
|
|
13
|
-
"""
|
|
14
|
-
def __init__(
|
|
15
|
-
self, row_builder: exprs.RowBuilder, media_slots: Iterable[exprs.ColumnSlotIdx],
|
|
16
|
-
input: Optional[ExecNode]):
|
|
17
|
-
super().__init__(row_builder, [], [], input)
|
|
18
|
-
self.row_builder = row_builder
|
|
19
|
-
self.input = input
|
|
20
|
-
for col in [c.col for c in media_slots]:
|
|
21
|
-
assert col.col_type.is_media_type()
|
|
22
|
-
self.media_slots = media_slots
|
|
23
|
-
|
|
24
|
-
def __next__(self) -> DataRowBatch:
|
|
25
|
-
assert self.input is not None
|
|
26
|
-
row_batch = next(self.input)
|
|
27
|
-
for row in row_batch:
|
|
28
|
-
for slot_idx, col in [(c.slot_idx, c.col) for c in self.media_slots]:
|
|
29
|
-
if row.has_exc(slot_idx):
|
|
30
|
-
continue
|
|
31
|
-
assert row.has_val[slot_idx]
|
|
32
|
-
path = row.file_paths[slot_idx]
|
|
33
|
-
if path is None:
|
|
34
|
-
continue
|
|
35
|
-
|
|
36
|
-
try:
|
|
37
|
-
col.col_type.validate_media(path)
|
|
38
|
-
except excs.Error as exc:
|
|
39
|
-
self.row_builder.set_exc(row, slot_idx, exc)
|
|
40
|
-
if not self.ctx.ignore_errors:
|
|
41
|
-
raise exc
|
|
42
|
-
|
|
43
|
-
return row_batch
|