pixeltable 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pixeltable might be problematic. Click here for more details.

Files changed (110) hide show
  1. pixeltable/__init__.py +20 -9
  2. pixeltable/__version__.py +3 -0
  3. pixeltable/catalog/column.py +23 -7
  4. pixeltable/catalog/insertable_table.py +32 -19
  5. pixeltable/catalog/table.py +210 -20
  6. pixeltable/catalog/table_version.py +272 -111
  7. pixeltable/catalog/table_version_path.py +6 -1
  8. pixeltable/dataframe.py +184 -110
  9. pixeltable/datatransfer/__init__.py +1 -0
  10. pixeltable/datatransfer/label_studio.py +526 -0
  11. pixeltable/datatransfer/remote.py +113 -0
  12. pixeltable/env.py +213 -79
  13. pixeltable/exec/__init__.py +2 -1
  14. pixeltable/exec/data_row_batch.py +6 -7
  15. pixeltable/exec/expr_eval_node.py +28 -28
  16. pixeltable/exec/sql_scan_node.py +7 -6
  17. pixeltable/exprs/__init__.py +4 -3
  18. pixeltable/exprs/column_ref.py +11 -2
  19. pixeltable/exprs/comparison.py +39 -1
  20. pixeltable/exprs/data_row.py +7 -0
  21. pixeltable/exprs/expr.py +26 -19
  22. pixeltable/exprs/function_call.py +17 -18
  23. pixeltable/exprs/globals.py +14 -2
  24. pixeltable/exprs/image_member_access.py +9 -28
  25. pixeltable/exprs/in_predicate.py +96 -0
  26. pixeltable/exprs/inline_array.py +13 -11
  27. pixeltable/exprs/inline_dict.py +15 -13
  28. pixeltable/exprs/row_builder.py +7 -1
  29. pixeltable/exprs/similarity_expr.py +67 -0
  30. pixeltable/ext/functions/whisperx.py +30 -0
  31. pixeltable/ext/functions/yolox.py +16 -0
  32. pixeltable/func/__init__.py +0 -2
  33. pixeltable/func/aggregate_function.py +5 -2
  34. pixeltable/func/callable_function.py +57 -13
  35. pixeltable/func/expr_template_function.py +14 -3
  36. pixeltable/func/function.py +35 -4
  37. pixeltable/func/signature.py +5 -15
  38. pixeltable/func/udf.py +8 -12
  39. pixeltable/functions/fireworks.py +9 -4
  40. pixeltable/functions/huggingface.py +48 -5
  41. pixeltable/functions/openai.py +49 -11
  42. pixeltable/functions/pil/image.py +61 -64
  43. pixeltable/functions/together.py +32 -6
  44. pixeltable/functions/util.py +0 -43
  45. pixeltable/functions/video.py +46 -8
  46. pixeltable/globals.py +443 -0
  47. pixeltable/index/__init__.py +1 -0
  48. pixeltable/index/base.py +9 -2
  49. pixeltable/index/btree.py +54 -0
  50. pixeltable/index/embedding_index.py +91 -15
  51. pixeltable/io/__init__.py +4 -0
  52. pixeltable/io/globals.py +59 -0
  53. pixeltable/{utils → io}/hf_datasets.py +48 -17
  54. pixeltable/io/pandas.py +148 -0
  55. pixeltable/{utils → io}/parquet.py +58 -33
  56. pixeltable/iterators/__init__.py +1 -1
  57. pixeltable/iterators/base.py +8 -4
  58. pixeltable/iterators/document.py +225 -93
  59. pixeltable/iterators/video.py +16 -9
  60. pixeltable/metadata/__init__.py +8 -4
  61. pixeltable/metadata/converters/convert_12.py +3 -0
  62. pixeltable/metadata/converters/convert_13.py +41 -0
  63. pixeltable/metadata/converters/convert_14.py +13 -0
  64. pixeltable/metadata/converters/convert_15.py +29 -0
  65. pixeltable/metadata/converters/util.py +63 -0
  66. pixeltable/metadata/schema.py +12 -6
  67. pixeltable/plan.py +11 -24
  68. pixeltable/store.py +16 -23
  69. pixeltable/tool/create_test_db_dump.py +49 -14
  70. pixeltable/type_system.py +27 -58
  71. pixeltable/utils/coco.py +94 -0
  72. pixeltable/utils/documents.py +42 -12
  73. pixeltable/utils/http_server.py +70 -0
  74. pixeltable-0.2.7.dist-info/METADATA +137 -0
  75. pixeltable-0.2.7.dist-info/RECORD +126 -0
  76. {pixeltable-0.2.5.dist-info → pixeltable-0.2.7.dist-info}/WHEEL +1 -1
  77. pixeltable/client.py +0 -600
  78. pixeltable/exprs/image_similarity_predicate.py +0 -58
  79. pixeltable/func/batched_function.py +0 -53
  80. pixeltable/func/nos_function.py +0 -202
  81. pixeltable/tests/conftest.py +0 -171
  82. pixeltable/tests/ext/test_yolox.py +0 -21
  83. pixeltable/tests/functions/test_fireworks.py +0 -43
  84. pixeltable/tests/functions/test_functions.py +0 -60
  85. pixeltable/tests/functions/test_huggingface.py +0 -158
  86. pixeltable/tests/functions/test_openai.py +0 -162
  87. pixeltable/tests/functions/test_together.py +0 -112
  88. pixeltable/tests/test_audio.py +0 -65
  89. pixeltable/tests/test_catalog.py +0 -27
  90. pixeltable/tests/test_client.py +0 -21
  91. pixeltable/tests/test_component_view.py +0 -379
  92. pixeltable/tests/test_dataframe.py +0 -440
  93. pixeltable/tests/test_dirs.py +0 -107
  94. pixeltable/tests/test_document.py +0 -120
  95. pixeltable/tests/test_exprs.py +0 -802
  96. pixeltable/tests/test_function.py +0 -332
  97. pixeltable/tests/test_index.py +0 -138
  98. pixeltable/tests/test_migration.py +0 -44
  99. pixeltable/tests/test_nos.py +0 -54
  100. pixeltable/tests/test_snapshot.py +0 -231
  101. pixeltable/tests/test_table.py +0 -1343
  102. pixeltable/tests/test_transactional_directory.py +0 -42
  103. pixeltable/tests/test_types.py +0 -52
  104. pixeltable/tests/test_video.py +0 -159
  105. pixeltable/tests/test_view.py +0 -535
  106. pixeltable/tests/utils.py +0 -442
  107. pixeltable/utils/clip.py +0 -18
  108. pixeltable-0.2.5.dist-info/METADATA +0 -128
  109. pixeltable-0.2.5.dist-info/RECORD +0 -139
  110. {pixeltable-0.2.5.dist-info → pixeltable-0.2.7.dist-info}/LICENSE +0 -0
pixeltable/utils/coco.py CHANGED
@@ -134,3 +134,97 @@ def write_coco_dataset(df: 'pixeltable.DataFrame', dest_path: Path) -> Path:
134
134
  json.dump(result, f)
135
135
  return output_path
136
136
 
137
+
138
+ COCO_2017_CATEGORIES = {
139
+ 0: 'N/A',
140
+ 1: 'person',
141
+ 2: 'bicycle',
142
+ 3: 'car',
143
+ 4: 'motorcycle',
144
+ 5: 'airplane',
145
+ 6: 'bus',
146
+ 7: 'train',
147
+ 8: 'truck',
148
+ 9: 'boat',
149
+ 10: 'traffic light',
150
+ 11: 'fire hydrant',
151
+ 12: 'N/A',
152
+ 13: 'stop sign',
153
+ 14: 'parking meter',
154
+ 15: 'bench',
155
+ 16: 'bird',
156
+ 17: 'cat',
157
+ 18: 'dog',
158
+ 19: 'horse',
159
+ 20: 'sheep',
160
+ 21: 'cow',
161
+ 22: 'elephant',
162
+ 23: 'bear',
163
+ 24: 'zebra',
164
+ 25: 'giraffe',
165
+ 26: 'N/A',
166
+ 27: 'backpack',
167
+ 28: 'umbrella',
168
+ 29: 'N/A',
169
+ 30: 'N/A',
170
+ 31: 'handbag',
171
+ 32: 'tie',
172
+ 33: 'suitcase',
173
+ 34: 'frisbee',
174
+ 35: 'skis',
175
+ 36: 'snowboard',
176
+ 37: 'sports ball',
177
+ 38: 'kite',
178
+ 39: 'baseball bat',
179
+ 40: 'baseball glove',
180
+ 41: 'skateboard',
181
+ 42: 'surfboard',
182
+ 43: 'tennis racket',
183
+ 44: 'bottle',
184
+ 45: 'N/A',
185
+ 46: 'wine glass',
186
+ 47: 'cup',
187
+ 48: 'fork',
188
+ 49: 'knife',
189
+ 50: 'spoon',
190
+ 51: 'bowl',
191
+ 52: 'banana',
192
+ 53: 'apple',
193
+ 54: 'sandwich',
194
+ 55: 'orange',
195
+ 56: 'broccoli',
196
+ 57: 'carrot',
197
+ 58: 'hot dog',
198
+ 59: 'pizza',
199
+ 60: 'donut',
200
+ 61: 'cake',
201
+ 62: 'chair',
202
+ 63: 'couch',
203
+ 64: 'potted plant',
204
+ 65: 'bed',
205
+ 66: 'N/A',
206
+ 67: 'dining table',
207
+ 68: 'N/A',
208
+ 69: 'N/A',
209
+ 70: 'toilet',
210
+ 71: 'N/A',
211
+ 72: 'tv',
212
+ 73: 'laptop',
213
+ 74: 'mouse',
214
+ 75: 'remote',
215
+ 76: 'keyboard',
216
+ 77: 'cell phone',
217
+ 78: 'microwave',
218
+ 79: 'oven',
219
+ 80: 'toaster',
220
+ 81: 'sink',
221
+ 82: 'refrigerator',
222
+ 83: 'N/A',
223
+ 84: 'book',
224
+ 85: 'clock',
225
+ 86: 'vase',
226
+ 87: 'scissors',
227
+ 88: 'teddy bear',
228
+ 89: 'hair drier',
229
+ 90: 'toothbrush'
230
+ }
@@ -9,31 +9,61 @@ class DocumentHandle:
9
9
  format: ts.DocumentType.DocumentFormat
10
10
  bs_doc: Optional['bs4.BeautifulSoup'] = None
11
11
  md_ast: Optional[Dict] = None
12
+ pdf_doc: Optional['fitz.Document'] = None
12
13
 
14
+ def get_document_handle(path: str) -> Optional[DocumentHandle]:
15
+ # try pdf first, because a correct PDF is a binary format that
16
+ # would trigger encoding exceptions if oppened as utf8.
17
+ pdf_doc = get_pdf_handle(path)
18
+ if pdf_doc is not None:
19
+ return DocumentHandle(format=ts.DocumentType.DocumentFormat.PDF, pdf_doc=pdf_doc)
20
+ # currently the rest of the types are text-based, so we can open them in utf8 mode once
21
+ try:
22
+ with open(path, 'r', encoding='utf8') as file:
23
+ contents = file.read()
24
+ except UnicodeDecodeError:
25
+ # not pdf, and also not valid text file
26
+ return None
13
27
 
14
- def get_document_handle(s: str) -> Optional[DocumentHandle]:
15
- bs_doc = get_html_handle(s)
28
+ # bs4 will appear to succeed for md files as well.
29
+ # this will break most markdown files at the moment.
30
+ bs_doc = get_html_handle(contents)
16
31
  if bs_doc is not None:
17
32
  return DocumentHandle(format=ts.DocumentType.DocumentFormat.HTML, bs_doc=bs_doc)
18
- md_ast = get_markdown_handle(s)
33
+
34
+ md_ast = get_markdown_handle(contents)
19
35
  if md_ast is not None:
20
36
  return DocumentHandle(format=ts.DocumentType.DocumentFormat.MD, md_ast=md_ast)
37
+
21
38
  return None
22
39
 
23
- def get_html_handle(s: str) -> Optional['bs4.BeautifulSoup']:
40
+ def get_html_handle(text: str) -> Optional['bs4.BeautifulSoup']:
24
41
  import bs4
25
42
  try:
26
- doc = bs4.BeautifulSoup(s, 'html.parser')
27
- except Exception as e:
43
+ doc = bs4.BeautifulSoup(text, 'html.parser')
44
+ if doc.find() is None:
45
+ return None
46
+ return doc
47
+ except Exception:
28
48
  return None
29
- if doc.find() is None:
30
- return None
31
- return doc
32
49
 
33
- def get_markdown_handle(s: str) -> Optional[Dict]:
50
+ def get_markdown_handle(text: str) -> Optional[Dict]:
34
51
  import mistune
35
52
  try:
36
53
  md_ast = mistune.create_markdown(renderer=None)
37
- return md_ast(s)
38
- except Exception as e:
54
+ return md_ast(text)
55
+ except Exception:
39
56
  return None
57
+
58
+ def get_pdf_handle(path : str) -> Optional['fitz.Document']:
59
+ import fitz # aka pymupdf
60
+ try:
61
+ doc = fitz.open(path)
62
+ # check pdf (bc it will work for images)
63
+ if not doc.is_pdf:
64
+ return None
65
+ # try to read one page
66
+ next(page for page in doc)
67
+ return doc
68
+ except Exception:
69
+ return None
@@ -0,0 +1,70 @@
1
+ import http
2
+ import http.server
3
+ import logging
4
+ import urllib
5
+ import posixpath
6
+ import pathlib
7
+ import os
8
+ import string
9
+
10
+ _logger = logging.getLogger('pixeltable.http.server')
11
+
12
+
13
+ def get_file_uri(http_address: str, file_path: str) -> str:
14
+ """Get the URI for a file path, with the given prefix.
15
+ Used in the client to generate a URI
16
+ """
17
+ abs_path = pathlib.Path(file_path)
18
+ assert abs_path.is_absolute()
19
+ url = urllib.request.pathname2url(str(abs_path))
20
+ return f'{http_address}{url}'
21
+
22
+
23
+ class AbsolutePathHandler(http.server.SimpleHTTPRequestHandler):
24
+ """Serves all absolute paths, not just the current directory"""
25
+ def translate_path(self, path: str) -> str:
26
+ """
27
+ Translate a /-separated PATH to the local filename syntax.
28
+ overrides http.server.SimpleHTTPRequestHandler.translate_path
29
+
30
+ This is only useful for file serving.
31
+
32
+ Code initially taken from there:
33
+ https://github.com/python/cpython/blob/f5406ef454662b98df107775d18ff71ae6849618/Lib/http/server.py#L834
34
+ """
35
+ _logger.info(f'translate path {path=}')
36
+ # abandon query parameters, taken from http.server.SimpleHTTPRequestHandler
37
+ path = path.split('?', 1)[0]
38
+ path = path.split('#', 1)[0]
39
+
40
+ path = pathlib.Path(urllib.request.url2pathname(path))
41
+ return str(path)
42
+
43
+ def log_message(self, format, *args) -> None:
44
+ """override logging to stderr in http.server.BaseHTTPRequestHandler"""
45
+ message = format % args
46
+ _logger.info(message.translate(self._control_char_table))
47
+
48
+
49
+ class LoggingHTTPServer(http.server.ThreadingHTTPServer):
50
+ """Avoids polluting stdout and stderr"""
51
+
52
+ def handle_error(self, request, client_address) -> None:
53
+ """override socketserver.TCPServer.handle_error which prints directly to sys.stderr"""
54
+ import traceback
55
+
56
+ _logger.error(
57
+ f'Exception occurred during processing of {request=} from {client_address=}\
58
+ \nbacktrace:\n{traceback.format_exc()}\n----\n'
59
+ )
60
+
61
+
62
+ def make_server(address: str, port: int) -> http.server.HTTPServer:
63
+ """Create a file server with pixeltable specific config """
64
+ return LoggingHTTPServer((address, port), AbsolutePathHandler)
65
+
66
+
67
+ if __name__ == '__main__':
68
+ httpd = make_server('127.0.0.1', 8000)
69
+ print(f'about to server HTTP on {httpd.server_address}')
70
+ httpd.serve_forever()
@@ -0,0 +1,137 @@
1
+ Metadata-Version: 2.1
2
+ Name: pixeltable
3
+ Version: 0.2.7
4
+ Summary: Pixeltable: The Multimodal AI Data Plane
5
+ Author: Marcel Kornacker
6
+ Author-email: marcelk@gmail.com
7
+ Requires-Python: >=3.9,<4.0
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: Programming Language :: Python :: 3.9
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Requires-Dist: av (>=10.0.0)
14
+ Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
15
+ Requires-Dist: cloudpickle (>=2.2.1,<3.0.0)
16
+ Requires-Dist: ftfy (>=6.2.0,<7.0.0)
17
+ Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
18
+ Requires-Dist: jmespath (>=1.0.1,<2.0.0)
19
+ Requires-Dist: mistune (>=3.0.2,<4.0.0)
20
+ Requires-Dist: more-itertools (>=10.2,<11.0)
21
+ Requires-Dist: numpy (>=1.25)
22
+ Requires-Dist: opencv-python-headless (>=4.7.0.68,<5.0.0.0)
23
+ Requires-Dist: pandas (>=2.0,<3.0)
24
+ Requires-Dist: pgserver (==0.1.3)
25
+ Requires-Dist: pgvector (>=0.2.1,<0.3.0)
26
+ Requires-Dist: pillow (>=9.3.0)
27
+ Requires-Dist: psutil (>=5.9.5,<6.0.0)
28
+ Requires-Dist: psycopg2-binary (>=2.9.5,<3.0.0)
29
+ Requires-Dist: pymupdf (>=1.24.1,<2.0.0)
30
+ Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
31
+ Requires-Dist: requests (>=2.31.0,<3.0.0)
32
+ Requires-Dist: setuptools (==69.1.1)
33
+ Requires-Dist: sqlalchemy[mypy] (>=2.0.23,<3.0.0)
34
+ Requires-Dist: tenacity (>=8.2,<9.0)
35
+ Requires-Dist: tqdm (>=4.64)
36
+ Description-Content-Type: text/markdown
37
+
38
+ <div align="center">
39
+ <img src="https://raw.githubusercontent.com/pixeltable/pixeltable/master/docs/release/pixeltable-banner.png" alt="Pixeltable" width="45%" />
40
+
41
+ # Unifying Data, Models, and Orchestration for AI Products
42
+
43
+ [![License](https://img.shields.io/badge/License-Apache%202.0-darkblue.svg)](https://opensource.org/licenses/Apache-2.0)
44
+ ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pixeltable?logo=python&logoColor=white)
45
+ [![Platform Support](https://img.shields.io/badge/platform-Linux%20%7C%20macOS%20%7C%20Windows-8A2BE2)]()
46
+ [![pytest status](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml/badge.svg)](https://github.com/pixeltable/pixeltable/actions)
47
+ [![PyPI Package](https://img.shields.io/pypi/v/pixeltable?color=darkorange)](https://pypi.org/project/pixeltable/)
48
+
49
+ [Installation](https://pixeltable.github.io/pixeltable/getting-started/) | [Documentation](https://pixeltable.readme.io/) | [API Reference](https://pixeltable.github.io/pixeltable/) | [Code Samples](https://pixeltable.readme.io/recipes) | [Examples](https://github.com/pixeltable/pixeltable/tree/master/docs/release/tutorials)
50
+ </div>
51
+
52
+ Pixeltable is a Python library that lets AI engineers and data scientists focus on exploration, modeling, and app development without dealing with the customary data plumbing.
53
+
54
+ ## What problems does Pixeltable solve?
55
+
56
+ Today’s solutions for AI app development require extensive custom coding and infrastructure plumbing. Tracking lineage and versions between and across data transformations, models, and deployment is cumbersome. With Pixeltable you can store, transform, index, and iterate on your data within the same table interface, whether it's text, images, embeddings, or even video. Built-in lineage and versioning ensure transparency and reproducibility, while the development-to-production mirror streamlines deployment.
57
+
58
+ ## 💾 Installation
59
+
60
+ ```python
61
+ %pip install pixeltable
62
+ ```
63
+
64
+ To verify that it's working:
65
+
66
+ ```python
67
+ import pixeltable as pxt
68
+ pxt.init()
69
+ ```
70
+ > [!NOTE]
71
+ > Check out the [Pixeltable Basics](https://pixeltable.readme.io/docs/pixeltable-basics) tutorial for a tour of its most important features.
72
+
73
+ ## 💡 Get Started
74
+ Learn how to create tables, populate them with data, and enhance them with built-in or user-defined transformations and AI operations.
75
+
76
+ | Topic | Notebook | API |
77
+ |:--------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------:|
78
+ | Get Started | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/tutorials/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | [![API](https://img.shields.io/badge/api-reference-blue.svg)](https://pixeltable.github.io/pixeltable/api/pixeltable/) |
79
+ | User-Defined Functions (UDFs) | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/release/howto/udfs-in-pixeltable.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | [![API](https://img.shields.io/badge/api-reference-blue.svg)](https://pixeltable.github.io/pixeltable/api/iterators/document-splitter/) |
80
+ | Comparing Object Detection Models | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/release/tutorials/object-detection-in-videos.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | [![API](https://img.shields.io/badge/api-reference-blue.svg)](https://pixeltable.github.io/pixeltable/api-cheat-sheet/#frame-extraction-for-video-data) |
81
+ | Experimenting with Chunking (RAG) | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/release/tutorials/rag-operations.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> | [![API](https://img.shields.io/badge/api-reference-blue.svg)](https://pixeltable.github.io/pixeltable/api/iterators/document-splitter/) |
82
+ | Working with External Files | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/master/docs/release/howto/working-with-external-files.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | [![API](https://img.shields.io/badge/api-reference-blue.svg)](https://pixeltable.github.io/pixeltable/api-cheat-sheet/#inserting-data-into-a-table) |
83
+
84
+ ## ❓ FAQ
85
+
86
+ ### What does Pixeltable provide me with? Pixeltable provides:
87
+
88
+ - Data storage and versioning
89
+ - Combined Data and Model Lineage
90
+ - Indexing (e.g. embedding vectors) and Data Retrieval
91
+ - Orchestration of multimodal workloads
92
+ - Incremental updates
93
+ - Code is automatically production-ready
94
+
95
+ ### Why should you use Pixeltable?
96
+
97
+ - **It gives you transparency and reproducibility**
98
+ - All generated data is automatically recorded and versioned
99
+ - You will never need to re-run a workload because you lost track of the input data
100
+ - **It saves you money**
101
+ - All data changes are automatically incremental
102
+ - You never need to re-run pipelines from scratch because you’re adding data
103
+ - **It integrates with any existing Python code or libraries**
104
+ - Bring your ever-changing code and workloads
105
+ - You choose the models, tools, and AI practices (e.g., your embedding model for a vector index); Pixeltable orchestrates the data
106
+
107
+ ### What is Pixeltable not providing?
108
+
109
+ - Pixeltable is not a low-code, prescriptive AI solution. We empower you to use the best frameworks and techniques for your specific needs.
110
+ - We do not aim to replace your existing AI toolkit, but rather enhance it by streamlining the underlying data infrastructure and orchestration.
111
+
112
+ > [!TIP]
113
+ > Check out the [Integrations](https://pixeltable.readme.io/docs/working-with-openai) section, and feel free to submit a request for additional ones.
114
+
115
+ ## 📙 Example of Use Cases
116
+
117
+ - **Interact with video data at the frame level** without having to think about frame extraction, intermediate file storage, or storage space explosion.
118
+ - **Augment your data incrementally and interactively with built-in functions and UDFs**, such as image transformations, model inference, and visualizations, without having to think about data pipelines, incremental updates, or capturing function output.
119
+ - **Interact with all the data relevant to your AI application** (video, images, documents, audio, structured data, JSON) through a simple dataframe-style API directly in Python. This includes:
120
+ - similarity search on embeddings, supported by high-dimensional vector indexing;
121
+ - path expressions and transformations on JSON data;
122
+ - PIL and OpenCV image operations;
123
+ - assembling frames into videos.
124
+ - **Perform keyword and image similarity search at the video frame level** without having to worry about frame storage.
125
+ - **Access all Pixeltable-resident data directly as a PyTorch dataset** in your training scripts.
126
+ - **Understand the compute and storage costs of your data at the granularity** of individual augmentations and get cost projections before adding new data and new augmentations.
127
+ - **Rely on Pixeltable's automatic versioning and snapshot functionality** to protect against regressions and to ensure reproducibility.
128
+
129
+ ## 🐛 Contributions & Feedback
130
+
131
+ Are you experiencing issues or bugs with Pixeltable? File an [Issue](https://github.com/pixeltable/pixeltable/issues).
132
+ </br>Do you want to contribute? Feel free to open a [PR](https://github.com/pixeltable/pixeltable/pulls).
133
+
134
+ ## :classical_building: License
135
+
136
+ This library is licensed under the Apache 2.0 License.
137
+
@@ -0,0 +1,126 @@
1
+ pixeltable/__init__.py,sha256=DzVevwic1g8Tp4QYrcCIzPFFwPZ66KGWeYprlRC9Uwc,1142
2
+ pixeltable/__version__.py,sha256=QpFd23iUxkd6iv60M7Xgh5GNlwHpgZV9y3kI5aNgy58,112
3
+ pixeltable/catalog/__init__.py,sha256=E41bxaPeQIcgRYzTWc2vkDOboQhRymrJf4IcHQO7o_8,453
4
+ pixeltable/catalog/catalog.py,sha256=0TYWB1R6YBp9qCkWF7kCcX2Yw70UuburKKIemv5L1Js,7908
5
+ pixeltable/catalog/column.py,sha256=EW2nDHDcHiCSYN6Gboh5D39c0_NqheWsxSl62IrFWgg,8584
6
+ pixeltable/catalog/dir.py,sha256=pG1nMpG123POo6WMSHhAmnwXOQ26uUJfUcbzL-Jb4ws,919
7
+ pixeltable/catalog/globals.py,sha256=yLEGNbsSnLzjWNHVJacfjA9hbw13Q6QXLOSCRmdTlq0,943
8
+ pixeltable/catalog/insertable_table.py,sha256=vMa_XUWT3DG3ZlxkScDZ9-mYNw31G8XB4ODUlxXt7NU,8927
9
+ pixeltable/catalog/named_function.py,sha256=a96gnKtx-nz5_MzDIiD4t4Hxqdjkg9ZtijRQxvWA5WQ,1147
10
+ pixeltable/catalog/path.py,sha256=QgccEi_QOfaKt8YsR2zLtd_z7z7QQkU_1kprJFi2SPQ,1677
11
+ pixeltable/catalog/path_dict.py,sha256=xfvxg1Ze5jZCARUGASF2DRbQPh7pRVTYhuJ_u82gYUo,5941
12
+ pixeltable/catalog/schema_object.py,sha256=-UxmPLbuEBqJiJi_GGRbFdr7arAFxTqs4bt6TFmSt3M,1059
13
+ pixeltable/catalog/table.py,sha256=FbLDJGFuvHtLT7rFTbFaayNKhmK1kzcr4lzdqyRjGBg,39507
14
+ pixeltable/catalog/table_version.py,sha256=U-GWWSAvVvNinEDSAvN-m2dOO8SviuN1py442OO0hlo,57459
15
+ pixeltable/catalog/table_version_path.py,sha256=P-HL9H1UIiJrKfa0wIRsESGrG3f815nGvHOEpVLJ3no,5718
16
+ pixeltable/catalog/view.py,sha256=BIL3s4DV3tWbOcqtqnhn46B2UvLaBhppfJUlNEt5nec,9734
17
+ pixeltable/dataframe.py,sha256=lzSzR7mi9C4BO39fNXYo64k3KxILyG_Z7eET6DXTgKY,31922
18
+ pixeltable/datatransfer/__init__.py,sha256=cRWdQ_LUNkJgmionI1RrYD71A1CSI92P4o8_XXOnFmU,27
19
+ pixeltable/datatransfer/label_studio.py,sha256=7E70MdgbAKW4AkZcR0uBuYXjAlUzruYgsOIR7FEbZY4,23802
20
+ pixeltable/datatransfer/remote.py,sha256=PaxdVggZMbHyG9FROfKXQxTmGAa8PiVhETl7lp5CNYg,3801
21
+ pixeltable/env.py,sha256=6IEbaG7a-Jz_hJ7nqA32wPmejF9Njl1UIzadubq0SjI,21487
22
+ pixeltable/exceptions.py,sha256=MSP9zeL0AmXT93XqjdvgGN4rzno1_KRrGriq6hpemnw,376
23
+ pixeltable/exec/__init__.py,sha256=RK7SKvrQ7Ky3G_LXDP4Bf7lHmMM_uYZl8dJaZYs0FjY,454
24
+ pixeltable/exec/aggregation_node.py,sha256=cf6rVAgrGh_uaMrCIgXJIwQTmbcboJlnrH_MmPIQSd0,3321
25
+ pixeltable/exec/cache_prefetch_node.py,sha256=d5pEuR6AtJQkEVy9X3XeYFI_q0szMtoNAH96vYdtBE0,5241
26
+ pixeltable/exec/component_iteration_node.py,sha256=Uz6zEeaJMcbvF3S0W0qmLI_uWsZsaSspHKNzuAMrasg,4069
27
+ pixeltable/exec/data_row_batch.py,sha256=1IDYHBkSQ60dwOnAGnS-Wpp3AsnbMqKcY40zUT7ku-Q,3392
28
+ pixeltable/exec/exec_context.py,sha256=E82Q2bJMJ1ulud5L5D9dh2Z8vEUQ659SgT614YKDO34,924
29
+ pixeltable/exec/exec_node.py,sha256=Hji5NCPHfa50IWyjladXrBm4I0zseV7AV4cVdx0Q8Ew,2170
30
+ pixeltable/exec/expr_eval_node.py,sha256=fEzbeZ0J-kylRQ2M0nSlUeLFRTHlwNzlvBo1yqWQ2rg,10856
31
+ pixeltable/exec/in_memory_data_node.py,sha256=SNM2AbMQSjmGDWMNJUf_5MmlXWE3P80lsuUjNfzQckA,3171
32
+ pixeltable/exec/media_validation_node.py,sha256=OKfRyKpcn7AZdACy_HD4NsDC87ZfNFs1tdrQz2NiIVw,1514
33
+ pixeltable/exec/sql_scan_node.py,sha256=s2PVls7gfSL0zQsdDWz8dC7MAE6KWeV_EDBCilx8Ros,10250
34
+ pixeltable/exprs/__init__.py,sha256=7dwrdk-NpF66OT-m5yNtFEhq-o1T476dnXHjluw2K1s,951
35
+ pixeltable/exprs/arithmetic_expr.py,sha256=sWBYCBKI6IHj9ASwDcm2BlkQ5gleVtKtmpiPvzFNBJM,4386
36
+ pixeltable/exprs/array_slice.py,sha256=VmWc6iFusrM85MjyEBBCfXG1Jnt8-Gr6-J88BXxNoOE,2131
37
+ pixeltable/exprs/column_property_ref.py,sha256=0PHiBys0fxe2LgjaMId5UHob4E-ZggyPLnnW41RgA0E,2706
38
+ pixeltable/exprs/column_ref.py,sha256=5GoddEyH1nLeeKYqX2g2WOBoCqnmghwt3Hg3D6JdzvI,5430
39
+ pixeltable/exprs/comparison.py,sha256=hP3M_lMWcFgENBICFosZPw2lRm1R6_qM_O9bKPmWJGI,4789
40
+ pixeltable/exprs/compound_predicate.py,sha256=Gh22MKi625m5A_RunVRd-a1XFi-fitikqBVz2VNXKrs,3830
41
+ pixeltable/exprs/data_row.py,sha256=TN4WeAz7D3xiSCDYgCinkVpTaSkkJhGgNkNqviBHVqs,8425
42
+ pixeltable/exprs/expr.py,sha256=NvkJmxAbAxfQYfoegXyBaD0_fm-3lnotUrvf_R6Z0Gk,24163
43
+ pixeltable/exprs/expr_set.py,sha256=Q64Q2yI0CTq2Ma_E-BUYlMotSstVuMm4OFZnBCedHRk,1222
44
+ pixeltable/exprs/function_call.py,sha256=d3TXU2MbfgQyth1-GEeZJcEZs_BX9548gN5TCwGLNX0,17118
45
+ pixeltable/exprs/globals.py,sha256=KhK4xwkLHv4NsXXcLdjRu2OFSvEnlC7GG-8Gs_IbQtI,1858
46
+ pixeltable/exprs/image_member_access.py,sha256=KSYdTIaLh53dNRjv3SJFchPMPo7o5diJSQkV1NsyB4Y,3547
47
+ pixeltable/exprs/in_predicate.py,sha256=burxrBCH1MXqU-wrNWJvD0PRGzJdWy85intOSftQK54,3696
48
+ pixeltable/exprs/inline_array.py,sha256=293WuUEhYXrcp8-AnPDVIWQBPQMrPviB88A619Ls_Es,4499
49
+ pixeltable/exprs/inline_dict.py,sha256=TWYokJ14Nq-evODcYFVO471WSEDbz6cJqIdRb2PkbZQ,3885
50
+ pixeltable/exprs/is_null.py,sha256=nvpOXtQj1UeYJpkCWzbaGuQElzrA2HSG3XNQugOv-pw,1041
51
+ pixeltable/exprs/json_mapper.py,sha256=I60VNgus64ai80gnFCIsRn0VRWYXMkqH5VNvnATsN9s,4559
52
+ pixeltable/exprs/json_path.py,sha256=Wz_5zFsyc9TPhsSbsDjDmQ3Nb0uVIwMCx5nh-cQYBiE,6526
53
+ pixeltable/exprs/literal.py,sha256=5NNza-WL1dd3hNznwwkr_yAcTGXSIRYUszGfy30lruI,2396
54
+ pixeltable/exprs/object_ref.py,sha256=eTcx84aWRI59fIiGvbdv3_cfL0XW4xEFQ4lwpLpJkM8,1250
55
+ pixeltable/exprs/predicate.py,sha256=OSDgjfSqiK7J_5GZMUXMvjfyomKEGi0JNxeB073SGXw,1859
56
+ pixeltable/exprs/row_builder.py,sha256=cpQa7GHR2dZYxhCAwZBfz-MqO0oP-NS44mAYoVUOt7A,15662
57
+ pixeltable/exprs/rowid_ref.py,sha256=74w4rEy21YysTVbyKNc3op-pYFqDAx8VJdtl7ZPpxHs,4268
58
+ pixeltable/exprs/similarity_expr.py,sha256=IvSeUFMjyj-ZFZTae8UJKa2YlDGn6512RyTIvLIcG7w,2829
59
+ pixeltable/exprs/type_cast.py,sha256=JMg8p1qYoFfiAXfJPSbTEnfrK7lRO_JMaqlPHOrhNQU,1793
60
+ pixeltable/exprs/variable.py,sha256=Kg_O4ytcHYZFijIyMHYBJn063cTKU1-YE583FAz8Qaw,1361
61
+ pixeltable/ext/__init__.py,sha256=0uugfuME1FybVo-MdxaVNGagRjhcvNTnv5MZUem6Cyo,269
62
+ pixeltable/ext/functions/whisperx.py,sha256=CnpSPZJgufXa01vgUubVkyxQuZIdublJzkwbm5kS1YQ,1078
63
+ pixeltable/ext/functions/yolox.py,sha256=JCG57Kbu0hNc1wRGDY3Mhhr6LQUY0fOgSSh-JV1plkw,3644
64
+ pixeltable/func/__init__.py,sha256=LCB5iB2aZyMrX-hn_oNBYnB1SE60t50hE23av_v2F50,348
65
+ pixeltable/func/aggregate_function.py,sha256=fB2kwLaeBKWCAOihOUdACR-2XDbMN14zJQIpQBVP0Bk,9398
66
+ pixeltable/func/callable_function.py,sha256=nEEmXFvd8TW9TBPbDnC3q8phj9ARokAsB-OJ1_hTkGo,4612
67
+ pixeltable/func/expr_template_function.py,sha256=7PNIzqkNOsCfU0KXCzrh_fczKMpZihu6FHV4NOgeVDM,4349
68
+ pixeltable/func/function.py,sha256=fANPfafLwY0Mq6CF21VYbuF-hRxxsPLHn5waoj1mOGY,5611
69
+ pixeltable/func/function_registry.py,sha256=1ibSQxEPm3Zd3r497vSlckQiDG9sfCnyJx3zcSm9t7c,11456
70
+ pixeltable/func/globals.py,sha256=sEwn6lGgHMp6VQORb_P5qRd_-Q2_bUSqvqM9-XPN_ec,1483
71
+ pixeltable/func/signature.py,sha256=erOPFuSuaxkXnRyFd3nCYLuprUWcYFox3Hk3ZKUPWfM,6697
72
+ pixeltable/func/udf.py,sha256=eu6Dt1SGlMvIBDNUxSc467-VBns5X12UU47FihlO_to,6540
73
+ pixeltable/functions/__init__.py,sha256=uO-XB4QUbx3Jjs9GoaTXoJY2jn0AuXTL32YLkL_3_CI,3297
74
+ pixeltable/functions/eval.py,sha256=_2FANDJqwtIDzTxtcKc0Yacf7b4LTAjyy2fPDw1FG_s,8404
75
+ pixeltable/functions/fireworks.py,sha256=dVGFZ6Kb2sahyySoLkSatLHsSnEXjAvLc0_hCHXUxXg,985
76
+ pixeltable/functions/huggingface.py,sha256=SlzPc01TS4oeFhJlsbwYCtG3h_LF1r-CnWisIfO37qc,7224
77
+ pixeltable/functions/image.py,sha256=xR_S_0BuX6Ycc5E366GpOfP0JptD7beQwHE_fLl8ZVM,431
78
+ pixeltable/functions/openai.py,sha256=lDmp9v7k-TCHUyMsPVTIwfjygWEihrlhnnJuOhvek1I,8046
79
+ pixeltable/functions/pil/image.py,sha256=6eNdMy2lZliFb8Lw12aBRUaShH07VEsFmhHSG21Jjt4,5992
80
+ pixeltable/functions/string.py,sha256=RYOgZwifjC943YloEMi3PdflnjFqOYB2FddrUvzgtXs,516
81
+ pixeltable/functions/together.py,sha256=Iu2_pZjOH6jXq6Sdqmu8LqtG1ELTtBEQ4eSOdYkig9c,4249
82
+ pixeltable/functions/util.py,sha256=DW7SwsftTRYo5h6iYl2Ec3WGmUA1D1Hrv4bPt-j7fKM,262
83
+ pixeltable/functions/video.py,sha256=wanwNQ1dyo5mP0NZ5F6gf6MEMUX40lcHXkz04WyOsbA,3876
84
+ pixeltable/globals.py,sha256=-2_ndBGunTXYm0A99nocSfKXRsQtiiesKVl2pBv0We4,14157
85
+ pixeltable/index/__init__.py,sha256=XBwetNQQwnz0fiKwonOKhyy_U32l_cjt77kNvEIdjWs,102
86
+ pixeltable/index/base.py,sha256=YAQ5Dz1mfI0dfu9rxWHWroE8TjB90yKfPtXAzoADq38,1568
87
+ pixeltable/index/btree.py,sha256=NE4GYhcJWYJhdKyeHI0sQBlFvUaIgGOF9KLyCZOfFjE,1822
88
+ pixeltable/index/embedding_index.py,sha256=AYphEggN-0B4GNrm4nMmi46CEtrQw5tguyk67BK2sWo,7627
89
+ pixeltable/io/__init__.py,sha256=Io5ZLrcvRPeqRQwnU2iODvWMqkYroWErkbp7dLxE4Kk,197
90
+ pixeltable/io/globals.py,sha256=5EdMVoRVvk1AeLaq-P4_LNdNlEDVfsl9bRTFuoTgexQ,2735
91
+ pixeltable/io/hf_datasets.py,sha256=h5M1NkXOvEU8kaeT3AON1A18Vmhnc1lVo5a3TZ5AAic,8004
92
+ pixeltable/io/pandas.py,sha256=cDHUDW2CGiBbsEJB9zE5vkXopTKxDdI-CZxNcp0OnIk,6478
93
+ pixeltable/io/parquet.py,sha256=Z1b92gsPeCBf4P9_jgWWHAEHtu51nhuC8nSJgoKiywQ,8150
94
+ pixeltable/iterators/__init__.py,sha256=kokLguXBY_nxBTqUiXZVvCxTv-vGsX4cK8tgIbsW5G8,108
95
+ pixeltable/iterators/base.py,sha256=cnEh1tNN2JAxRzrLTg3dhun3N1oNQ8vifCm6ts3_UiE,1687
96
+ pixeltable/iterators/document.py,sha256=netSCJatG8NcgbHZ69BvQVICdAorQlYi8OlcpqwLQD4,19436
97
+ pixeltable/iterators/video.py,sha256=xtxODL1AfZwTfHVzWekhTCLA8gwTJIvJFdxC0KecD9Q,3836
98
+ pixeltable/metadata/__init__.py,sha256=k3aptna__92VIJ4C6bIHLQrfYE1MOpBCKKMJ_b03nxA,2228
99
+ pixeltable/metadata/converters/convert_10.py,sha256=0mSGCn7vqtef63riPi9msUaaUvsSQIj-NFj9QFDYPdA,733
100
+ pixeltable/metadata/converters/convert_12.py,sha256=g9rHTcKlDQZbM3_k4eBv0FBdWmQXHWCnMwx1_l6KpMI,107
101
+ pixeltable/metadata/converters/convert_13.py,sha256=FEgOH5PKf05xVoCaioDDDHOSuoWPyBzodojmsSMMZ5U,1366
102
+ pixeltable/metadata/converters/convert_14.py,sha256=UAWHEipZ-NrQtI5zZN1u9C5AD24ZpYXsdpC3Te0t-qE,402
103
+ pixeltable/metadata/converters/convert_15.py,sha256=WFpIOMRZDgb-_bh1V07EbLv5Hshh1aArdavSnCsUwoQ,1004
104
+ pixeltable/metadata/converters/util.py,sha256=97efM1Hx1qKMIOaEI4bjmX93POie7XvBEF_HJhFhzE0,2400
105
+ pixeltable/metadata/schema.py,sha256=IdvV_UIyQqQL25cf36Rz6dMhHRWXvsApKt_uFSHW5kk,8497
106
+ pixeltable/plan.py,sha256=A_kPsX3bjLyfYbeQ6eCgDbrb_Oldk4w8HdFRqRSDpPY,32653
107
+ pixeltable/store.py,sha256=foQe9y8rRbl35f3naL7rbYVrD8LO00cmD53vWP2A9XI,18850
108
+ pixeltable/tool/create_test_db_dump.py,sha256=W5XNaKpi_-YnOCDVYjNs3g_p2CnRQp0xMtIwyu2rP14,7607
109
+ pixeltable/tool/create_test_video.py,sha256=OLfccymYReIpzE8osZn4rQvLXxxiPC_l0vc06U74hVM,2899
110
+ pixeltable/type_system.py,sha256=nljZs4O_dsVFMs4aB3z7Szc9LgFtl5eOuloxJkk7tPE,29503
111
+ pixeltable/utils/__init__.py,sha256=UYlrf6TIWJT0g-Hac0b34-dEk478B5Qx8dGco34YlIk,439
112
+ pixeltable/utils/arrow.py,sha256=83_7aG5UR2qtTktw_otLkQs-RQbLk0VVM0JLJkbweNU,3692
113
+ pixeltable/utils/coco.py,sha256=ISpFBhR4eO1jOcg_SPb0thVI4KdS6H0RyNQauZIA5A4,7287
114
+ pixeltable/utils/documents.py,sha256=Q7e5U2Hk0go83MdKzD_MIiMscwbcFsLMgRw2IU_vQF4,2213
115
+ pixeltable/utils/filecache.py,sha256=UoNONG2VaAc2IBB0e3sQdsvyOPOes2XSDc5_CsA4qek,7839
116
+ pixeltable/utils/help.py,sha256=cCnxJ4VP9MJ57iDqExmnDcM-JG3a1lw_q7g-D7bpSVI,252
117
+ pixeltable/utils/http_server.py,sha256=WQ5ILMzlz4TlwI9j5YqAPgEZyhrN1GytMNDbLD9occk,2422
118
+ pixeltable/utils/media_store.py,sha256=x71wnJDZDHcdd13VCfL4AkHQ6IJB41gNA-zBvXJwFos,3116
119
+ pixeltable/utils/pytorch.py,sha256=BR4tgfUWw-2rwWTOgzXj5qdMBpe1Arpp5SK4ax6jjpk,3483
120
+ pixeltable/utils/s3.py,sha256=rkanuhk9DWvSfmbOLQW1j1Iov4sl2KhxGGKN-AJ8LSE,432
121
+ pixeltable/utils/sql.py,sha256=5n5_OmXAGtqFdL6z5XvgnU-vlx6Ba6f1WJrO1ZwUle8,765
122
+ pixeltable/utils/transactional_directory.py,sha256=UGzCrGtLR3hEEf8sYGuWBzLVFAEQml3vdIavigWeTBM,1349
123
+ pixeltable-0.2.7.dist-info/LICENSE,sha256=0UNMmwuqWPC0xDY1NWMm4uNJ2_MyA1pnTNRgQTvuBiQ,746
124
+ pixeltable-0.2.7.dist-info/METADATA,sha256=umL9Z-G7tgcumv96Wu89_5i2S_PXLEb9wuvGLoSo2SQ,9806
125
+ pixeltable-0.2.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
126
+ pixeltable-0.2.7.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: poetry-core 1.8.1
2
+ Generator: poetry-core 1.9.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any