pixeltable 0.2.24__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pixeltable might be problematic. Click here for more details.
- pixeltable/__init__.py +2 -2
- pixeltable/__version__.py +2 -2
- pixeltable/catalog/__init__.py +1 -1
- pixeltable/catalog/dir.py +6 -0
- pixeltable/catalog/globals.py +25 -0
- pixeltable/catalog/named_function.py +4 -0
- pixeltable/catalog/path_dict.py +37 -11
- pixeltable/catalog/schema_object.py +6 -0
- pixeltable/catalog/table.py +531 -251
- pixeltable/catalog/table_version.py +22 -8
- pixeltable/catalog/view.py +8 -7
- pixeltable/dataframe.py +439 -105
- pixeltable/env.py +19 -5
- pixeltable/exec/__init__.py +1 -1
- pixeltable/exec/exec_node.py +6 -7
- pixeltable/exec/expr_eval_node.py +1 -1
- pixeltable/exec/sql_node.py +92 -45
- pixeltable/exprs/__init__.py +1 -0
- pixeltable/exprs/arithmetic_expr.py +1 -1
- pixeltable/exprs/array_slice.py +1 -1
- pixeltable/exprs/column_property_ref.py +1 -1
- pixeltable/exprs/column_ref.py +29 -2
- pixeltable/exprs/comparison.py +1 -1
- pixeltable/exprs/compound_predicate.py +1 -1
- pixeltable/exprs/expr.py +12 -5
- pixeltable/exprs/expr_set.py +8 -0
- pixeltable/exprs/function_call.py +147 -39
- pixeltable/exprs/in_predicate.py +1 -1
- pixeltable/exprs/inline_expr.py +25 -5
- pixeltable/exprs/is_null.py +1 -1
- pixeltable/exprs/json_mapper.py +1 -1
- pixeltable/exprs/json_path.py +1 -1
- pixeltable/exprs/method_ref.py +1 -1
- pixeltable/exprs/row_builder.py +1 -1
- pixeltable/exprs/rowid_ref.py +1 -1
- pixeltable/exprs/similarity_expr.py +17 -7
- pixeltable/exprs/sql_element_cache.py +4 -0
- pixeltable/exprs/type_cast.py +2 -2
- pixeltable/exprs/variable.py +3 -0
- pixeltable/func/__init__.py +5 -4
- pixeltable/func/aggregate_function.py +151 -68
- pixeltable/func/callable_function.py +48 -16
- pixeltable/func/expr_template_function.py +64 -23
- pixeltable/func/function.py +227 -23
- pixeltable/func/function_registry.py +2 -1
- pixeltable/func/query_template_function.py +51 -9
- pixeltable/func/signature.py +65 -7
- pixeltable/func/tools.py +153 -0
- pixeltable/func/udf.py +57 -35
- pixeltable/functions/__init__.py +2 -2
- pixeltable/functions/anthropic.py +51 -4
- pixeltable/functions/gemini.py +85 -0
- pixeltable/functions/globals.py +54 -34
- pixeltable/functions/huggingface.py +10 -28
- pixeltable/functions/json.py +3 -8
- pixeltable/functions/math.py +67 -0
- pixeltable/functions/mistralai.py +0 -2
- pixeltable/functions/ollama.py +8 -8
- pixeltable/functions/openai.py +51 -4
- pixeltable/functions/timestamp.py +1 -1
- pixeltable/functions/video.py +3 -9
- pixeltable/functions/vision.py +1 -1
- pixeltable/globals.py +374 -89
- pixeltable/index/embedding_index.py +106 -29
- pixeltable/io/__init__.py +1 -1
- pixeltable/io/label_studio.py +1 -1
- pixeltable/io/parquet.py +39 -19
- pixeltable/iterators/__init__.py +1 -0
- pixeltable/iterators/document.py +12 -0
- pixeltable/iterators/image.py +100 -0
- pixeltable/iterators/video.py +7 -8
- pixeltable/metadata/__init__.py +1 -1
- pixeltable/metadata/converters/convert_16.py +2 -1
- pixeltable/metadata/converters/convert_17.py +2 -1
- pixeltable/metadata/converters/convert_22.py +17 -0
- pixeltable/metadata/converters/convert_23.py +35 -0
- pixeltable/metadata/converters/convert_24.py +56 -0
- pixeltable/metadata/converters/convert_25.py +19 -0
- pixeltable/metadata/converters/util.py +4 -2
- pixeltable/metadata/notes.py +4 -0
- pixeltable/metadata/schema.py +1 -0
- pixeltable/plan.py +129 -51
- pixeltable/store.py +1 -1
- pixeltable/type_system.py +196 -54
- pixeltable/utils/arrow.py +8 -3
- pixeltable/utils/description_helper.py +89 -0
- pixeltable/utils/documents.py +14 -0
- {pixeltable-0.2.24.dist-info → pixeltable-0.3.0.dist-info}/METADATA +32 -22
- pixeltable-0.3.0.dist-info/RECORD +155 -0
- {pixeltable-0.2.24.dist-info → pixeltable-0.3.0.dist-info}/WHEEL +1 -1
- pixeltable-0.3.0.dist-info/entry_points.txt +3 -0
- pixeltable/tool/create_test_db_dump.py +0 -308
- pixeltable/tool/create_test_video.py +0 -81
- pixeltable/tool/doc_plugins/griffe.py +0 -50
- pixeltable/tool/doc_plugins/mkdocstrings.py +0 -6
- pixeltable/tool/doc_plugins/templates/material/udf.html.jinja +0 -135
- pixeltable/tool/embed_udf.py +0 -9
- pixeltable/tool/mypy_plugin.py +0 -55
- pixeltable-0.2.24.dist-info/RECORD +0 -153
- pixeltable-0.2.24.dist-info/entry_points.txt +0 -3
- {pixeltable-0.2.24.dist-info → pixeltable-0.3.0.dist-info}/LICENSE +0 -0
|
@@ -1,31 +1,46 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pixeltable
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: AI Data Infrastructure: Declarative, Multimodal, and Incremental
|
|
5
|
+
Home-page: https://pixeltable.com/
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Keywords: data-science,machine-learning,database,ai,computer-vision,chatbot,ml,artificial-intelligence,feature-engineering,multimodal,mlops,feature-store,vector-database,llm,genai
|
|
5
8
|
Author: Pixeltable, Inc.
|
|
6
9
|
Author-email: contact@pixeltable.com
|
|
7
10
|
Requires-Python: >=3.9,<4.0
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
8
17
|
Classifier: Programming Language :: Python :: 3
|
|
9
18
|
Classifier: Programming Language :: Python :: 3.9
|
|
10
19
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Database
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
13
26
|
Requires-Dist: av (>=10.0.0)
|
|
14
27
|
Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
|
|
15
28
|
Requires-Dist: cloudpickle (>=2.2.1,<3.0.0)
|
|
16
29
|
Requires-Dist: ftfy (>=6.2.0,<7.0.0)
|
|
17
30
|
Requires-Dist: jinja2 (>=3.1.3,<4.0.0)
|
|
18
31
|
Requires-Dist: jmespath (>=1.0.1,<2.0.0)
|
|
32
|
+
Requires-Dist: jsonschema (>=4.1.0)
|
|
19
33
|
Requires-Dist: lxml (>=5.0)
|
|
20
34
|
Requires-Dist: more-itertools (>=10.2,<11.0)
|
|
21
35
|
Requires-Dist: numpy (>=1.25,<2.0)
|
|
22
36
|
Requires-Dist: pandas (>=2.0,<3.0)
|
|
23
37
|
Requires-Dist: pgvector (>=0.2.1,<0.3.0)
|
|
24
38
|
Requires-Dist: pillow (>=9.3.0)
|
|
25
|
-
Requires-Dist: pixeltable-pgserver (==0.2.
|
|
39
|
+
Requires-Dist: pixeltable-pgserver (==0.2.9)
|
|
26
40
|
Requires-Dist: psutil (>=5.9.5,<6.0.0)
|
|
27
|
-
Requires-Dist: psycopg[binary] (
|
|
41
|
+
Requires-Dist: psycopg[binary] (>=3.1.18)
|
|
28
42
|
Requires-Dist: puremagic (>=1.20)
|
|
43
|
+
Requires-Dist: pydantic (>=2.7.4)
|
|
29
44
|
Requires-Dist: pymupdf (>=1.24.1,<2.0.0)
|
|
30
45
|
Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
|
|
31
46
|
Requires-Dist: requests (>=2.31.0,<3.0.0)
|
|
@@ -33,14 +48,16 @@ Requires-Dist: sqlalchemy (>=2.0.23,<3.0.0)
|
|
|
33
48
|
Requires-Dist: tenacity (>=8.2,<9.0)
|
|
34
49
|
Requires-Dist: toml (>=0.10)
|
|
35
50
|
Requires-Dist: tqdm (>=4.64)
|
|
51
|
+
Project-URL: Documentation, https://docs.pixeltable.com/
|
|
52
|
+
Project-URL: Repository, https://github.com/pixeltable/pixeltable
|
|
36
53
|
Description-Content-Type: text/markdown
|
|
37
54
|
|
|
38
55
|
<div align="center">
|
|
39
|
-
<img src="https://raw.githubusercontent.com/pixeltable/pixeltable/main/docs/
|
|
56
|
+
<img src="https://raw.githubusercontent.com/pixeltable/pixeltable/main/docs/resources/pixeltable-logo-large.png"
|
|
40
57
|
alt="Pixeltable" width="50%" />
|
|
41
58
|
<br></br>
|
|
42
59
|
|
|
43
|
-
<h2>AI Data
|
|
60
|
+
<h2>AI Data Infrastructure — Declarative, Multimodal, and Incremental</h2>
|
|
44
61
|
|
|
45
62
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
46
63
|

|
|
@@ -48,8 +65,8 @@ Description-Content-Type: text/markdown
|
|
|
48
65
|
<br>
|
|
49
66
|
[](https://github.com/pixeltable/pixeltable/actions/workflows/pytest.yml)
|
|
50
67
|
[](https://github.com/pixeltable/pixeltable/actions/workflows/nightly.yml)
|
|
51
|
-
[](https://app.fossa.com/projects/git%2Bgithub.com%2Fpixeltable%2Fpixeltable?ref=badge_shield&issueType=security)
|
|
52
68
|
[](https://pypi.org/project/pixeltable/)
|
|
69
|
+
[](https://discord.gg/QPyqFYx2UN)
|
|
53
70
|
<a target="_blank" href="https://huggingface.co/Pixeltable">
|
|
54
71
|
<img src="https://img.shields.io/badge/🤗-HF Space-FF7D04" alt="Visit our Hugging Face space"/>
|
|
55
72
|
</a>
|
|
@@ -97,11 +114,12 @@ Learn how to create tables, populate them with data, and enhance them with built
|
|
|
97
114
|
|
|
98
115
|
| Topic | Notebook | Topic | Notebook |
|
|
99
116
|
|:----------|:-----------------|:-------------------------|:---------------------------------:|
|
|
100
|
-
| 10-Minute Tour of Pixeltable | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/
|
|
101
|
-
| User-Defined Functions (UDFs) | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/
|
|
102
|
-
| Incremental Prompt Engineering | <a target="_blank" href="https://colab.research.google.com/github/mistralai/cookbook/blob/main/third_party/Pixeltable/incremental_prompt_engineering_and_model_comparison.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Github"/> | Working with External Files | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/
|
|
103
|
-
| Integrating with Label Studio | <a target="_blank" href="https://pixeltable.readme.io/docs/label-studio"> <img src="https://img.shields.io/badge
|
|
104
|
-
| Multimodal Application | <a target="_blank" href="https://huggingface.co/spaces/Pixeltable/Multimodal-Powerhouse"> <img src="https://img.shields.io/badge
|
|
117
|
+
| 10-Minute Tour of Pixeltable | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/notebooks/pixeltable-basics.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | Tables and Data Operations | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/notebooks/fundamentals/tables-and-data-operations.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> |
|
|
118
|
+
| User-Defined Functions (UDFs) | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/notebooks/feature-guides/udfs-in-pixeltable.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> | Object Detection Models | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/notebooks/use-cases/object-detection-in-videos.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> |
|
|
119
|
+
| Incremental Prompt Engineering | <a target="_blank" href="https://colab.research.google.com/github/mistralai/cookbook/blob/main/third_party/Pixeltable/incremental_prompt_engineering_and_model_comparison.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Github"/> | Working with External Files | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/notebooks/feature-guides/working-with-external-files.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> </a> |
|
|
120
|
+
| Integrating with Label Studio | <a target="_blank" href="https://pixeltable.readme.io/docs/label-studio"> <img src="https://img.shields.io/badge/📚 Documentation-013056" alt="Visit our documentation"/></a> | Audio/Video Transcript Indexing | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/notebooks/use-cases/audio-transcriptions.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> |
|
|
121
|
+
| Multimodal Application | <a target="_blank" href="https://huggingface.co/spaces/Pixeltable/Multimodal-Powerhouse"> <img src="https://img.shields.io/badge/🤗-Gradio App-FF7D04" alt="Visit our Hugging Face Space"/></a> | Document Indexing and RAG | <a target="_blank" href="https://colab.research.google.com/github/pixeltable/pixeltable/blob/release/docs/notebooks/use-cases/rag-demo.ipynb"> <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/> |
|
|
122
|
+
| Context-Aware Discord Bot | <a target="_blank" href="https://github.com/pixeltable/pixeltable/blob/main/docs/sample-apps/context-aware-discord-bot"> <img src="https://img.shields.io/badge/%F0%9F%92%AC-Discord Bot-%235865F2.svg" alt="Visit our documentation"/></a> | Image/Text Similarity Search | <a target="_blank" href="https://github.com/pixeltable/pixeltable/tree/main/docs/sample-apps/text-and-image-similarity-search-nextjs-fastapi"> <img src="https://img.shields.io/badge/🖥️-Next.js + FastAPI-black.svg" alt="Open In Colab"/> |
|
|
105
123
|
|
|
106
124
|
## 🧱 Code Samples
|
|
107
125
|
|
|
@@ -237,7 +255,7 @@ Learn how to interact with inference services such as [Together AI](https://pixe
|
|
|
237
255
|
|
|
238
256
|
```python
|
|
239
257
|
import pixeltable as pxt
|
|
240
|
-
from pixeltable.functions.huggingface import
|
|
258
|
+
from pixeltable.functions.huggingface import clip
|
|
241
259
|
from pixeltable.iterators import FrameIterator
|
|
242
260
|
import PIL.Image
|
|
243
261
|
|
|
@@ -248,16 +266,8 @@ video_table.insert([{'video': '/video.mp4'}])
|
|
|
248
266
|
frames_view = pxt.create_view(
|
|
249
267
|
'frames', video_table, iterator=FrameIterator.create(video=video_table.video))
|
|
250
268
|
|
|
251
|
-
@pxt.expr_udf
|
|
252
|
-
def embed_image(img: PIL.Image.Image):
|
|
253
|
-
return clip_image(img, model_id='openai/clip-vit-base-patch32')
|
|
254
|
-
|
|
255
|
-
@pxt.expr_udf
|
|
256
|
-
def str_embed(s: str):
|
|
257
|
-
return clip_text(s, model_id='openai/clip-vit-base-patch32')
|
|
258
|
-
|
|
259
269
|
# Create an index on the 'frame' column that allows text and image search
|
|
260
|
-
frames_view.add_embedding_index('frame',
|
|
270
|
+
frames_view.add_embedding_index('frame', embed=clip.using('openai/clip-vit-base-patch32'))
|
|
261
271
|
|
|
262
272
|
# Now we will retrieve images based on a sample image
|
|
263
273
|
sample_image = '/image.jpeg'
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
pixeltable/__init__.py,sha256=c-Z21TqJUbzGQKsO4CS4lVFwWlZnVhpIR94cL6RNmDo,1411
|
|
2
|
+
pixeltable/__version__.py,sha256=zP_zQgwZByDmFisB1NLZeZLdsIy7Xh24k7e1Y59IAY4,112
|
|
3
|
+
pixeltable/catalog/__init__.py,sha256=u8Ods4ncTY7DI5w0jyHVC0QDVDLHAy8Rqia3qsM5OH8,517
|
|
4
|
+
pixeltable/catalog/catalog.py,sha256=tyDyI5wQw7vV6_FChrp9qgGCRClcjiSdW3eygYT0p9s,7849
|
|
5
|
+
pixeltable/catalog/column.py,sha256=ezeKoGl6aBTzSZBihDA6vdETcvyCguAD4OsmrxWs73A,9595
|
|
6
|
+
pixeltable/catalog/dir.py,sha256=-dV0-gn5GfqRmVIJbuUBOXqPuNLDm219cud_kBTNzuk,1203
|
|
7
|
+
pixeltable/catalog/globals.py,sha256=VgcmwtKYYDQtvtHShVde52n4FzGBtAIFYxoS-WD7OBw,3242
|
|
8
|
+
pixeltable/catalog/insertable_table.py,sha256=hOsdYhkhtRcDOrRkweIGFUfjweWF3fLUErkUTlGYoSU,7172
|
|
9
|
+
pixeltable/catalog/named_function.py,sha256=cWf9WxAagSY9uqE7mM0IwWSsDOvQUkJlcHlGqgVciJg,1225
|
|
10
|
+
pixeltable/catalog/path.py,sha256=QgccEi_QOfaKt8YsR2zLtd_z7z7QQkU_1kprJFi2SPQ,1677
|
|
11
|
+
pixeltable/catalog/path_dict.py,sha256=V7YQM0QhUWr4FbgUiDUIweGbmMJr687QCm2p614Xrs0,6473
|
|
12
|
+
pixeltable/catalog/schema_object.py,sha256=NUE6Fx6Km0UUJ6WcE9mOgL2UQCLBMQjINQLp_mljzKA,2382
|
|
13
|
+
pixeltable/catalog/table.py,sha256=Gqb1tT9l8oB5eeeFuj45fy0_1fdCqizmRWvyjFH-U9Y,62404
|
|
14
|
+
pixeltable/catalog/table_version.py,sha256=hbnAtd-0EsVFCuWE1yrmm_CLar6NNQDRevgDjduh2CY,58581
|
|
15
|
+
pixeltable/catalog/table_version_path.py,sha256=CczGbcz5ESq663arreri_p4chMZHozgG6k3y-ajkJN4,5787
|
|
16
|
+
pixeltable/catalog/view.py,sha256=qUjou_z_YwKTqTIRUY4Q3JxWUEiKvl2pDi7DNOnw4fw,10709
|
|
17
|
+
pixeltable/dataframe.py,sha256=Xd1xJ0Drj2WBC1B9fJ2TCFMI23HeMV7L7RZJi7Wm2ew,47735
|
|
18
|
+
pixeltable/env.py,sha256=jxnHxliUpWegRbsT6lRz83nNqDG29bWAy5wyAYImNGg,31338
|
|
19
|
+
pixeltable/exceptions.py,sha256=NuFY2WtkQpLfLHT_J70kOw9Tr0kEDkkgo-u7As4Gaq4,410
|
|
20
|
+
pixeltable/exec/__init__.py,sha256=bWbIA7U7y1hC40VOSv_p9doaMuOB0i3r1O2UFNw0VEs,494
|
|
21
|
+
pixeltable/exec/aggregation_node.py,sha256=0LdoPp_CR_UbcS60XkDw66SqlrQnw6Dy6KDWqi4PJ6k,4005
|
|
22
|
+
pixeltable/exec/cache_prefetch_node.py,sha256=hrJA5PxLSmFRuZdZO4Nc-vy6_PSb9YO4rnJc6MSTPHE,12017
|
|
23
|
+
pixeltable/exec/component_iteration_node.py,sha256=ABuXGbDRQWLGuaBnfK7bvOxCrz81vMMiAvXHHI8SX4c,4930
|
|
24
|
+
pixeltable/exec/data_row_batch.py,sha256=qPN0GtBGt72Jnap-UN53VlUPOpYEvqJhp1nuTeJ_8Co,2925
|
|
25
|
+
pixeltable/exec/exec_context.py,sha256=-FbfYLcGOL7mviru5dE1A7x4YxLbdKoXBHN3OSbqbcg,1084
|
|
26
|
+
pixeltable/exec/exec_node.py,sha256=3t_Ri_FQfc-dV91YGSq17Av3vPJth5mHC-LAVc-1Whc,3222
|
|
27
|
+
pixeltable/exec/expr_eval_node.py,sha256=8IzcxHvXCj69RKFfrt4XEol88SGAlLWMl0bjAijX7Ug,11887
|
|
28
|
+
pixeltable/exec/in_memory_data_node.py,sha256=uK3puLZDzUSJUJwGwkTMYoPqNIvE67jtQodhE2sCz2M,3452
|
|
29
|
+
pixeltable/exec/row_update_node.py,sha256=b8yuKtkmI2Q6U-7svKbkjdM3Z85ddZoJgJgUa17j-YE,2773
|
|
30
|
+
pixeltable/exec/sql_node.py,sha256=TOeSI7TIUh8SG3esExj5zaWgLrjmpqMN74iJlPq16D0,19546
|
|
31
|
+
pixeltable/exprs/__init__.py,sha256=zx5OTxfUpFs_U36CAO-81v6RCX49wqRw3eHaLfytvkE,1044
|
|
32
|
+
pixeltable/exprs/arithmetic_expr.py,sha256=I_OESFmUHlMSDmMy699-2JnFb7KbRjvh1GF06u8rBQ8,6322
|
|
33
|
+
pixeltable/exprs/array_slice.py,sha256=eSqtQRxvweW_CwTRgkDD189i-BmTJ48dsW3znRPDebg,2180
|
|
34
|
+
pixeltable/exprs/column_property_ref.py,sha256=ycHqZ7_LTFz_gag_53T5KRN6RfZckpw83DvvtxmcynM,3788
|
|
35
|
+
pixeltable/exprs/column_ref.py,sha256=Z2YAiZ0nMv_ecbw91qTc9VN6DB2hZj-_RNkogZcu4ng,10784
|
|
36
|
+
pixeltable/exprs/comparison.py,sha256=w1FuhnGsJRoCLYX3ztIHZDywf8nZeT2tKKwRjIQa6Lc,5101
|
|
37
|
+
pixeltable/exprs/compound_predicate.py,sha256=6N_zN4A_hKGLordNY0si-8KHeFWG1UtyeiTHcs69IrM,3846
|
|
38
|
+
pixeltable/exprs/data_row.py,sha256=rLtKxlst9mK6684A5y-nsjBcalyKEcKAWcYCtNpK10w,9852
|
|
39
|
+
pixeltable/exprs/expr.py,sha256=E1V_fU1qA4nkHLHTqGkxr8exJE6I9-b9LwNVXxGNTXo,30463
|
|
40
|
+
pixeltable/exprs/expr_dict.py,sha256=xkvo_iVPOLMq3WkBZQ2FOoXqYoebuV6XGlidPJxdOkY,1588
|
|
41
|
+
pixeltable/exprs/expr_set.py,sha256=GeUQXadzJbAqQOGdsO6Z5hPAp0A03YUr2hyIvfZDYEE,2576
|
|
42
|
+
pixeltable/exprs/function_call.py,sha256=5lMTj4G6jOarabZ0b3xpgAMJe3rgDdCSXbYwnrZEjxs,26196
|
|
43
|
+
pixeltable/exprs/globals.py,sha256=5pwn5vdi-EEpYBpPty658YV45myY7W0iFIfTH7QIzak,2032
|
|
44
|
+
pixeltable/exprs/in_predicate.py,sha256=VxSn9TA_3UDHTC6rqQ12No5HbZO7SuE4DglpwAoI58s,3783
|
|
45
|
+
pixeltable/exprs/inline_expr.py,sha256=oALVAM5BVFskEQgkfgGIhKFm6V2FUlrPtocGtqtXM0w,7867
|
|
46
|
+
pixeltable/exprs/is_null.py,sha256=rnusy1j33o48Z-qjguj09G7AmsBHFJI5SWQXOAjgKQw,1092
|
|
47
|
+
pixeltable/exprs/json_mapper.py,sha256=yhefrgvy76czfyTER_oRXnfxOVQb42Q4Us1KkXhSsLM,4545
|
|
48
|
+
pixeltable/exprs/json_path.py,sha256=UTf3VUcyEmEBJs6yU4kONGcZ8fy1o6mIe9MlntnUd6M,6811
|
|
49
|
+
pixeltable/exprs/literal.py,sha256=TTKb0gw6qck9D61SwVDuBrLrBrGwEhkCB-m0ILFpWFk,3764
|
|
50
|
+
pixeltable/exprs/method_ref.py,sha256=3O_uFMP6wWGiwJWri8on-47EVl-QD3AiAc7hXJMADxs,2615
|
|
51
|
+
pixeltable/exprs/object_ref.py,sha256=GVg6uxZnKwFVTC0kouJq-wMFP-gUPb_ld_syHrsGMdE,1283
|
|
52
|
+
pixeltable/exprs/row_builder.py,sha256=fJEb8XU7j8M4PIc67O35a5Pzhoh_fs-i77lYxJPrA6o,18444
|
|
53
|
+
pixeltable/exprs/rowid_ref.py,sha256=iNBZ0wIhBAGkGCrNP9UQ2FeK8OajVX-eI4dtDA_bfHU,4401
|
|
54
|
+
pixeltable/exprs/similarity_expr.py,sha256=GhiopGxN3wu5MOEqfWN4PNbqgSszg2VIC1_bqVPpm3Q,4333
|
|
55
|
+
pixeltable/exprs/sql_element_cache.py,sha256=8i9ZslKWKTd1lUN7JvwRtQ-PFD1QMi5LBYdwcGA2-p0,1364
|
|
56
|
+
pixeltable/exprs/type_cast.py,sha256=DzVVpMo3msAJm42SAGOV4P0ECyM6LEao3zdfmob4MQo,1862
|
|
57
|
+
pixeltable/exprs/variable.py,sha256=3ZV3HzqULk05LKGG1p1T_xIlBfhqHIK4KJSCgoCDDSY,1488
|
|
58
|
+
pixeltable/ext/__init__.py,sha256=iO0J_Jfnv38F5y40sDAW54gpXjIyZgOGgoWQJAwjQec,423
|
|
59
|
+
pixeltable/ext/functions/__init__.py,sha256=hIjPEKC5E5uJOXlQqUyhP9yn9ZqWOCJAlj0kXWDlhlE,159
|
|
60
|
+
pixeltable/ext/functions/whisperx.py,sha256=jojjNhazcYiAh1scwUl-erhIDRr4kOTkcLrjy0xcp6g,2325
|
|
61
|
+
pixeltable/ext/functions/yolox.py,sha256=k-pQTelv4Tea3AXvDB7Kc7YCIa1uexjVGqxETP0B_hc,5351
|
|
62
|
+
pixeltable/func/__init__.py,sha256=Tmy5srLdxcWDwxwnXXj2mTNUMimdYY-MdU6fJ9SpePc,457
|
|
63
|
+
pixeltable/func/aggregate_function.py,sha256=SWqU3stHBzrZbzoNCNHBhMHqQbT9rpVTelWY6otboSg,12649
|
|
64
|
+
pixeltable/func/callable_function.py,sha256=AK2sO-nOlDqDSDJvJ5zOp2UNwDJY-2keJlfPycGzRDw,6500
|
|
65
|
+
pixeltable/func/expr_template_function.py,sha256=xoi7yngicX4F_-qa64a38_dvX3Bs_4BDphodTEul06o,5381
|
|
66
|
+
pixeltable/func/function.py,sha256=1bGC7e93QrRZhSij-DdMVL94Lz-A7OWwbncmPhJj1ps,16799
|
|
67
|
+
pixeltable/func/function_registry.py,sha256=Ps09iPDRJRKj-NQtancrI07zqFBIPpASGZ636mpBiI0,12333
|
|
68
|
+
pixeltable/func/globals.py,sha256=sEwn6lGgHMp6VQORb_P5qRd_-Q2_bUSqvqM9-XPN_ec,1483
|
|
69
|
+
pixeltable/func/query_template_function.py,sha256=p3zp5aNIxlr0QR1ROVJLISDnZXpfOhqAeMIwr8l2b5g,5170
|
|
70
|
+
pixeltable/func/signature.py,sha256=wWf07OawOwDmqZOjUPgHJHAhAQ-fd61PGBx82aWLBgA,11212
|
|
71
|
+
pixeltable/func/tools.py,sha256=7OAHVb9KbmcPtfMWsbKAoRu3kx-_japSTE-hgRQJefM,5988
|
|
72
|
+
pixeltable/func/udf.py,sha256=a8FqOcDqPvUICvELpPOQVsrvr0sgZBQuV4aVHXuexac,8603
|
|
73
|
+
pixeltable/functions/__init__.py,sha256=2vdpJ3S1pUle8dBzCVM70o8Lj1c0lgccZPJgPYms_CQ,403
|
|
74
|
+
pixeltable/functions/anthropic.py,sha256=FzMK4zhz4HvPqJmOM4CBIlpVa8JLiRShi8jEd2abHsg,4710
|
|
75
|
+
pixeltable/functions/audio.py,sha256=7213nTnqKJ6vM9kalaoJ283OwX5SGEJN10vDhaRNZ6E,644
|
|
76
|
+
pixeltable/functions/fireworks.py,sha256=qwFC_eIaDs-glxyJ_IVXaNGkpgPzeRsQ_SdpzueBxq0,2605
|
|
77
|
+
pixeltable/functions/gemini.py,sha256=RQ3I25nXSXUXdF0rmhnv5XOgZXb_6SIgMc_hEyF83SI,2783
|
|
78
|
+
pixeltable/functions/globals.py,sha256=Q5S84Vk_ovCADLivXkn4vfUnX8oSgW8v1dZ0aPoz7C4,4844
|
|
79
|
+
pixeltable/functions/huggingface.py,sha256=fXcj6NC5Hz_nYE-0CjvSJ1sD3Jq726hQXzueqnQ6Kr4,20576
|
|
80
|
+
pixeltable/functions/image.py,sha256=3Qm4ybAT_o4YUl3bzhEXy8dKOwgZ7RCUV-ky-dbL_jc,13836
|
|
81
|
+
pixeltable/functions/json.py,sha256=cptFpuxQED5wSXXsH8steeHu3pCkZ_zXRE7lccjXybU,756
|
|
82
|
+
pixeltable/functions/llama_cpp.py,sha256=1awALuAXVpQH64l7vQlM8gvxLDix4c1-6DV3nG5RHu4,3881
|
|
83
|
+
pixeltable/functions/math.py,sha256=WPoH9zD9_GdwvBs-FSC3Sqb70gOPNouhPcBZABsuLwI,1541
|
|
84
|
+
pixeltable/functions/mistralai.py,sha256=GpxtT-a8ltx1kQC8XTJRflxkh-17VhzzFTkxqIUsba8,5494
|
|
85
|
+
pixeltable/functions/ollama.py,sha256=z-g0cCJ-WMf6RI3SXIVVbbOlrWNR8nxGTyzET1rratU,4320
|
|
86
|
+
pixeltable/functions/openai.py,sha256=Jikl9oYPiKy61r5Ohe1xqjvb6SLCbNTYFr4yAHFkYME,17101
|
|
87
|
+
pixeltable/functions/replicate.py,sha256=j8ZedScOMInmHWmriQSUOviw6tp8gQr-W6n21PNNL2g,2188
|
|
88
|
+
pixeltable/functions/string.py,sha256=VqzhVildxTt_XblW89Kl5Zd6MVOU71eaX2LTMY5jkUg,20366
|
|
89
|
+
pixeltable/functions/timestamp.py,sha256=KOm5eVF51PqOCTqwq8cKCAEJNWMgu2xN_yTtGf7yixU,9095
|
|
90
|
+
pixeltable/functions/together.py,sha256=xbgQMlA2G2FXdoQR4O9DlkmBsm2mivAdiOnxtxcyPwo,9440
|
|
91
|
+
pixeltable/functions/util.py,sha256=GgKTzCjvzUQNjWtSObTkfxkvJ9GVUOzKimY45WhE25M,759
|
|
92
|
+
pixeltable/functions/video.py,sha256=12jnOdU0G-Hk42rJx-S5QC77-bDkUflkxfkjej8n5pM,6702
|
|
93
|
+
pixeltable/functions/vision.py,sha256=MEgkNp-4tUUeQS05VEJBrMgQFM48aNsNlksfAl9rH7w,15494
|
|
94
|
+
pixeltable/functions/whisper.py,sha256=f2wqRd0n9jSBqRZN3W93UaetiAHtbsK0j9jXR2j2kkQ,2913
|
|
95
|
+
pixeltable/globals.py,sha256=W5-4KEVqg9tZwfL6EffkjQYtRNXDLKXj_Bv32liZeHI,33895
|
|
96
|
+
pixeltable/index/__init__.py,sha256=XBwetNQQwnz0fiKwonOKhyy_U32l_cjt77kNvEIdjWs,102
|
|
97
|
+
pixeltable/index/base.py,sha256=zo0YvJI3oXiK6hZJztB36ZftKKhLfO75Zq3t-PeQA6M,1556
|
|
98
|
+
pixeltable/index/btree.py,sha256=JFerLyyLoBaD0FSF_jJ6iJFBVa-z_et--KdNR02xjRg,2264
|
|
99
|
+
pixeltable/index/embedding_index.py,sha256=xXqUlDGss-a-nA1TFhixF_NY7HOmE3YZGpT-4PG3O74,11177
|
|
100
|
+
pixeltable/io/__init__.py,sha256=j3qDyGO1ejLce-UzIncK3eRqyOCLoOlDqClecMBSJGc,563
|
|
101
|
+
pixeltable/io/external_store.py,sha256=H1jt7MDn464QRgBvU-PmcPcFlo3EZBCG7fKWEZXOfyc,16676
|
|
102
|
+
pixeltable/io/fiftyone.py,sha256=hH-FahW6BuMQY8lGa2atnNnJto_pK8kWrP_y_EMsq6g,6965
|
|
103
|
+
pixeltable/io/globals.py,sha256=9S9wnlIAuhZq7eC_GklTM_UX0UATK9fEagk8-SRCeXQ,17794
|
|
104
|
+
pixeltable/io/hf_datasets.py,sha256=o5fqm2CJAjhFd3z-NYGxN0jM1tfrp4szuUX0TGnyNRY,8316
|
|
105
|
+
pixeltable/io/label_studio.py,sha256=Yx2ES_Y32eZA8JvJ1k17tSU0GiRTcU0_R116c7RCbZo,31091
|
|
106
|
+
pixeltable/io/pandas.py,sha256=7eHg7wnAfRA9eBk4iC0iSSVTKOM59Ne4pXokKWdt3dY,9793
|
|
107
|
+
pixeltable/io/parquet.py,sha256=bvwTRIKhS99M3bAcAP63jnh5-R6Qww1SrYBGWjZvN1g,8800
|
|
108
|
+
pixeltable/iterators/__init__.py,sha256=qA9cJTwPO3gk-7y8mUXw2anWSbnUAVGwzl52SxiUjNU,398
|
|
109
|
+
pixeltable/iterators/base.py,sha256=ZC0ZvXL4iw6AmT8cu-Mdx-T2UG9nmJYV1C6LK4efAfw,1669
|
|
110
|
+
pixeltable/iterators/document.py,sha256=e5EMUAW4f5SqfC5mRN7Oo2E3gw_vcMSzq2707NVLByU,20254
|
|
111
|
+
pixeltable/iterators/image.py,sha256=7eJupbyLEl_q4bemEozL8WgTgkxjl-DYnSXyHTQ7Rck,3628
|
|
112
|
+
pixeltable/iterators/string.py,sha256=NG_fWc_GAITDfzl6MvrDOMrSoMcZdMZf6hPQztCSatE,1305
|
|
113
|
+
pixeltable/iterators/video.py,sha256=anedQpSOnjYhHLa8UiEx-6ROagb14rpXh9am8oTriUg,9382
|
|
114
|
+
pixeltable/metadata/__init__.py,sha256=S99VPq09bEYI7nF6HQ55Fyp9bJ0TUhk9L9gsOQi5XT4,2209
|
|
115
|
+
pixeltable/metadata/converters/convert_10.py,sha256=J1_r7LNNAWTdb042AwqFpJ4sEB-i4qhUdk5iOjcZk34,719
|
|
116
|
+
pixeltable/metadata/converters/convert_12.py,sha256=Ci-qyZW1gqci-8wnjeOB5afdq7KTuN-hVSV9OqSPx8g,162
|
|
117
|
+
pixeltable/metadata/converters/convert_13.py,sha256=yFR6lD3pOrZ46ZQBFKYvxiIYa7rRxh46Bsq7yiCBNak,1356
|
|
118
|
+
pixeltable/metadata/converters/convert_14.py,sha256=9e_JNm3a35Gs4dvFFaV-_jtCqp3ud6OEOqMIhTQmasE,428
|
|
119
|
+
pixeltable/metadata/converters/convert_15.py,sha256=jMfL5wGil0-gZeIfmCbgtR3LSVNyOcxVp6YRhBECqY4,1741
|
|
120
|
+
pixeltable/metadata/converters/convert_16.py,sha256=W5pQAcW4JX56NBXAtqdBeKQxzJuFFbwItxLyTa1ZbYg,514
|
|
121
|
+
pixeltable/metadata/converters/convert_17.py,sha256=Z1FiMoN0N5M7Tf4_0LrT6lDJWLBy9U4Lmt1sIns_VFY,899
|
|
122
|
+
pixeltable/metadata/converters/convert_18.py,sha256=NxSroQntVLgmvXfae1f0-jYJIhM2W7dhRY3I7g0RxPY,1482
|
|
123
|
+
pixeltable/metadata/converters/convert_19.py,sha256=QgUDjNoH87KZg_f3cx4k0ZR67NqWRhZQKIIojbqxSkg,2080
|
|
124
|
+
pixeltable/metadata/converters/convert_20.py,sha256=NLMeke9QUGqIJUe5MNqKmVdaLs6pPFrjyigImhogaT4,2768
|
|
125
|
+
pixeltable/metadata/converters/convert_21.py,sha256=YTztkbqOC2zQcTWrXfhrP8diUbfxy5DHwsu_IT-bBok,1115
|
|
126
|
+
pixeltable/metadata/converters/convert_22.py,sha256=f_oauWGEiefr9tZOtGBJMcqqIsTmJwhLNr6DqKlZZkg,590
|
|
127
|
+
pixeltable/metadata/converters/convert_23.py,sha256=ih_3st9Y6A4ZbI8G5HTaUE9xDhiClN-QaT1k1iATIb4,1235
|
|
128
|
+
pixeltable/metadata/converters/convert_24.py,sha256=cBzfCqbPgYrWDO0DmJ019VVkbe2CZXTS4xF6q4LZYKM,2401
|
|
129
|
+
pixeltable/metadata/converters/convert_25.py,sha256=3sCXMbHiLJWMvbghj-XaeW4_1xSECivlbsdGtGSmntE,620
|
|
130
|
+
pixeltable/metadata/converters/util.py,sha256=NgzoKralRIXwOqtguifVsBbpEN2M2X1v8lvOnsdGnPs,6100
|
|
131
|
+
pixeltable/metadata/notes.py,sha256=RsyIuwl3KhFRstB6cLX8ySWfIx67LAyr05HYXovu7Yc,779
|
|
132
|
+
pixeltable/metadata/schema.py,sha256=H9t51cbhXOhNu9Xog2VTjZlkTCx4cjQAlwOXi3HSd2E,9448
|
|
133
|
+
pixeltable/plan.py,sha256=dzG3iD9ClggFyIYmDuIK7H3eBC8HV3ft__f4ZULQvow,40687
|
|
134
|
+
pixeltable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
135
|
+
pixeltable/store.py,sha256=m-OeilqOFai5wUXvtXTV9txzn-GhK2lfA7fkvzrfWFg,21974
|
|
136
|
+
pixeltable/type_system.py,sha256=IbUXeaqsguRAneiXZc2WrAXskUiN1IPJ3O9xHBGGQhk,48154
|
|
137
|
+
pixeltable/utils/__init__.py,sha256=UYlrf6TIWJT0g-Hac0b34-dEk478B5Qx8dGco34YlIk,439
|
|
138
|
+
pixeltable/utils/arrow.py,sha256=4AyCAkkNfNChVTaPAVS5j1ZO7BtTxn11duonolfktV8,3931
|
|
139
|
+
pixeltable/utils/coco.py,sha256=WCUyoj0dTyJFbPB7frEQUvY92SlEPxQ068r-3QAwROE,7285
|
|
140
|
+
pixeltable/utils/code.py,sha256=AOw1u2r8_DQXpX-lxJhyHWARGrCRDXOJHFVgKOi54Uc,1231
|
|
141
|
+
pixeltable/utils/description_helper.py,sha256=P-8EE2pRFP8s3coe73frgV68yt4Dp3saErCUehMxKUw,3759
|
|
142
|
+
pixeltable/utils/documents.py,sha256=pytTYY167wYc1rGaDd9HPK6GOrz1ML78eAD3hIHrG68,2930
|
|
143
|
+
pixeltable/utils/filecache.py,sha256=6HKQdItqSSTQvj2HkSJulyhfBedi4PgC7umwxXGOVG8,10637
|
|
144
|
+
pixeltable/utils/formatter.py,sha256=5E_gDg11ClFI-5SthwkiqyE3hAok3JHDj4OSK9cJklM,9257
|
|
145
|
+
pixeltable/utils/http_server.py,sha256=xYPTvmYrkUpKfOaLDq08D-eHswkcgDf4qAt76ZFH6lM,2411
|
|
146
|
+
pixeltable/utils/media_store.py,sha256=YwvTjbVqC_aLbDvLuqnDSL8xeIVMZcmzp0ANuM6uMbw,3092
|
|
147
|
+
pixeltable/utils/pytorch.py,sha256=6RvOCjy_QV4gc-aht-3d0zoASkuv-warfpl87vgmuKw,3450
|
|
148
|
+
pixeltable/utils/s3.py,sha256=huA5hxDGkPIu18zWet76o0FsO7Vbtp-iPmnOzCe-MvA,586
|
|
149
|
+
pixeltable/utils/sql.py,sha256=j_tj0h4ffm-DhUIJbvGphxrVyBKlNTwDKqWGhRQ5_PY,795
|
|
150
|
+
pixeltable/utils/transactional_directory.py,sha256=UGzCrGtLR3hEEf8sYGuWBzLVFAEQml3vdIavigWeTBM,1349
|
|
151
|
+
pixeltable-0.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
152
|
+
pixeltable-0.3.0.dist-info/METADATA,sha256=GiNmcGREs3AUhpt4tKPImEGYzzD4XHMR1StiJSEkWhI,19304
|
|
153
|
+
pixeltable-0.3.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
154
|
+
pixeltable-0.3.0.dist-info/entry_points.txt,sha256=ToOd-pRgG7AitEBgYoBCRRB4-KVDQ0pj_9T4a1LgwA4,97
|
|
155
|
+
pixeltable-0.3.0.dist-info/RECORD,,
|
|
@@ -1,308 +0,0 @@
|
|
|
1
|
-
import datetime
|
|
2
|
-
import json
|
|
3
|
-
import logging
|
|
4
|
-
import os
|
|
5
|
-
import pathlib
|
|
6
|
-
import subprocess
|
|
7
|
-
import sys
|
|
8
|
-
from typing import Any
|
|
9
|
-
from zoneinfo import ZoneInfo
|
|
10
|
-
|
|
11
|
-
import pixeltable_pgserver
|
|
12
|
-
import toml
|
|
13
|
-
|
|
14
|
-
import pixeltable as pxt
|
|
15
|
-
import pixeltable.metadata as metadata
|
|
16
|
-
from pixeltable.env import Env
|
|
17
|
-
from pixeltable.func import Batch
|
|
18
|
-
from pixeltable.io.external_store import Project
|
|
19
|
-
from pixeltable.tool import embed_udf
|
|
20
|
-
from pixeltable.type_system import BoolType, FloatType, ImageType, IntType, JsonType, StringType, TimestampType
|
|
21
|
-
|
|
22
|
-
_logger = logging.getLogger('pixeltable')
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class Dumper:
|
|
26
|
-
|
|
27
|
-
def __init__(self, output_dir='target', db_name='pxtdump') -> None:
|
|
28
|
-
if sys.version_info >= (3, 10):
|
|
29
|
-
raise RuntimeError(
|
|
30
|
-
'This script must be run on Python 3.9. '
|
|
31
|
-
'DB dumps are incompatible across versions due to issues with pickling anonymous UDFs.'
|
|
32
|
-
)
|
|
33
|
-
|
|
34
|
-
self.output_dir = pathlib.Path(output_dir)
|
|
35
|
-
shared_home = pathlib.Path(os.environ.get('PIXELTABLE_HOME', '~/.pixeltable')).expanduser()
|
|
36
|
-
mock_home_dir = self.output_dir / '.pixeltable'
|
|
37
|
-
mock_home_dir.mkdir(parents=True, exist_ok=True)
|
|
38
|
-
os.environ['PIXELTABLE_HOME'] = str(mock_home_dir)
|
|
39
|
-
os.environ['PIXELTABLE_CONFIG'] = str(shared_home / 'config.toml')
|
|
40
|
-
os.environ['PIXELTABLE_DB'] = db_name
|
|
41
|
-
os.environ['PIXELTABLE_PGDATA'] = str(shared_home / 'pgdata')
|
|
42
|
-
|
|
43
|
-
Env._init_env(reinit_db=True)
|
|
44
|
-
|
|
45
|
-
Env.get().configure_logging(level=logging.DEBUG, to_stdout=True)
|
|
46
|
-
|
|
47
|
-
def dump_db(self) -> None:
|
|
48
|
-
md_version = metadata.VERSION
|
|
49
|
-
dump_file = self.output_dir / f'pixeltable-v{md_version:03d}-test.dump.gz'
|
|
50
|
-
_logger.info(f'Creating database dump at: {dump_file}')
|
|
51
|
-
pg_package_dir = os.path.dirname(pixeltable_pgserver.__file__)
|
|
52
|
-
pg_dump_binary = f'{pg_package_dir}/pginstall/bin/pg_dump'
|
|
53
|
-
_logger.info(f'Using pg_dump binary at: {pg_dump_binary}')
|
|
54
|
-
# We need the raw DB URL, without a driver qualifier. (The driver qualifier is needed by
|
|
55
|
-
# SQLAlchemy, but command-line Postgres won't know how to interpret it.)
|
|
56
|
-
db_url = Env.get()._db_server.get_uri(Env.get()._db_name)
|
|
57
|
-
with open(dump_file, 'wb') as dump:
|
|
58
|
-
pg_dump_process = subprocess.Popen(
|
|
59
|
-
(pg_dump_binary, db_url, '-U', 'postgres', '-Fc'),
|
|
60
|
-
stdout=subprocess.PIPE
|
|
61
|
-
)
|
|
62
|
-
subprocess.run(
|
|
63
|
-
('gzip', '-9'),
|
|
64
|
-
stdin=pg_dump_process.stdout,
|
|
65
|
-
stdout=dump,
|
|
66
|
-
check=True
|
|
67
|
-
)
|
|
68
|
-
if pg_dump_process.poll() != 0:
|
|
69
|
-
raise RuntimeError(f'pg_dump failed with return code {pg_dump_process.returncode}')
|
|
70
|
-
info_file = self.output_dir / f'pixeltable-v{md_version:03d}-test-info.toml'
|
|
71
|
-
git_sha = subprocess.check_output(('git', 'rev-parse', 'HEAD')).decode('ascii').strip()
|
|
72
|
-
user = os.environ.get('USER', os.environ.get('USERNAME'))
|
|
73
|
-
info_dict = {'pixeltable-dump': {
|
|
74
|
-
'metadata-version': md_version,
|
|
75
|
-
'git-sha': git_sha,
|
|
76
|
-
'datetime': datetime.datetime.now(tz=datetime.timezone.utc),
|
|
77
|
-
'user': user
|
|
78
|
-
}}
|
|
79
|
-
with open(info_file, 'w') as info:
|
|
80
|
-
toml.dump(info_dict, info)
|
|
81
|
-
|
|
82
|
-
# Expression types, predicate types, embedding indices, views on views
|
|
83
|
-
def create_tables(self) -> None:
|
|
84
|
-
schema = {
|
|
85
|
-
'c1': StringType(nullable=False),
|
|
86
|
-
'c1n': StringType(nullable=True),
|
|
87
|
-
'c2': IntType(nullable=False),
|
|
88
|
-
'c3': FloatType(nullable=False),
|
|
89
|
-
'c4': BoolType(nullable=False),
|
|
90
|
-
'c5': TimestampType(nullable=False),
|
|
91
|
-
'c6': JsonType(nullable=False),
|
|
92
|
-
'c7': JsonType(nullable=False),
|
|
93
|
-
'c8': ImageType(nullable=True)
|
|
94
|
-
}
|
|
95
|
-
t = pxt.create_table('base_table', schema, primary_key='c2')
|
|
96
|
-
|
|
97
|
-
num_rows = 20
|
|
98
|
-
d1 = {
|
|
99
|
-
'f1': 'test string 1',
|
|
100
|
-
'f2': 1,
|
|
101
|
-
'f3': 1.0,
|
|
102
|
-
'f4': True,
|
|
103
|
-
'f5': [1.0, 2.0, 3.0, 4.0],
|
|
104
|
-
'f6': {
|
|
105
|
-
'f7': 'test string 2',
|
|
106
|
-
'f8': [1.0, 2.0, 3.0, 4.0],
|
|
107
|
-
},
|
|
108
|
-
}
|
|
109
|
-
d2 = [d1, d1]
|
|
110
|
-
|
|
111
|
-
c1_data = [f'test string {i}' for i in range(num_rows)]
|
|
112
|
-
c2_data = [i for i in range(num_rows)]
|
|
113
|
-
c3_data = [float(i) for i in range(num_rows)]
|
|
114
|
-
c4_data = [bool(i % 2) for i in range(num_rows)]
|
|
115
|
-
c5_data = [datetime.datetime.now()] * num_rows
|
|
116
|
-
c6_data = [
|
|
117
|
-
{
|
|
118
|
-
'f1': f'test string {i}',
|
|
119
|
-
'f2': i,
|
|
120
|
-
'f3': float(i),
|
|
121
|
-
'f4': bool(i % 2),
|
|
122
|
-
'f5': [1.0, 2.0, 3.0, 4.0],
|
|
123
|
-
'f6': {
|
|
124
|
-
'f7': 'test string 2',
|
|
125
|
-
'f8': [1.0, 2.0, 3.0, 4.0],
|
|
126
|
-
},
|
|
127
|
-
}
|
|
128
|
-
for i in range(num_rows)
|
|
129
|
-
]
|
|
130
|
-
c7_data = [d2] * num_rows
|
|
131
|
-
rows = [
|
|
132
|
-
{
|
|
133
|
-
'c1': c1_data[i],
|
|
134
|
-
'c1n': c1_data[i] if i % 10 != 0 else None,
|
|
135
|
-
'c2': c2_data[i],
|
|
136
|
-
'c3': c3_data[i],
|
|
137
|
-
'c4': c4_data[i],
|
|
138
|
-
'c5': c5_data[i],
|
|
139
|
-
'c6': c6_data[i],
|
|
140
|
-
'c7': c7_data[i],
|
|
141
|
-
'c8': None
|
|
142
|
-
}
|
|
143
|
-
for i in range(num_rows)
|
|
144
|
-
]
|
|
145
|
-
|
|
146
|
-
self.__add_expr_columns(t, 'base_table')
|
|
147
|
-
t.insert(rows)
|
|
148
|
-
|
|
149
|
-
pxt.create_dir('views')
|
|
150
|
-
|
|
151
|
-
# simple view
|
|
152
|
-
v = pxt.create_view('views.view', t.where(t.c2 < 50))
|
|
153
|
-
self.__add_expr_columns(v, 'view')
|
|
154
|
-
|
|
155
|
-
# snapshot
|
|
156
|
-
_ = pxt.create_snapshot('views.snapshot', t.where(t.c2 >= 75))
|
|
157
|
-
|
|
158
|
-
# view of views
|
|
159
|
-
vv = pxt.create_view('views.view_of_views', v.where(t.c2 >= 25))
|
|
160
|
-
self.__add_expr_columns(vv, 'view_of_views')
|
|
161
|
-
|
|
162
|
-
# empty view
|
|
163
|
-
e = pxt.create_view('views.empty_view', t.where(t.c2 == 4171780))
|
|
164
|
-
assert e.count() == 0
|
|
165
|
-
self.__add_expr_columns(e, 'empty_view', include_expensive_functions=True)
|
|
166
|
-
|
|
167
|
-
# Add external stores
|
|
168
|
-
from pixeltable.io.external_store import MockProject
|
|
169
|
-
v._link_external_store(
|
|
170
|
-
MockProject.create(
|
|
171
|
-
v,
|
|
172
|
-
'project',
|
|
173
|
-
{'int_field': pxt.IntType()},
|
|
174
|
-
{'str_field': pxt.StringType()},
|
|
175
|
-
{'view_test_udf': 'int_field', 'c1': 'str_field'}
|
|
176
|
-
)
|
|
177
|
-
)
|
|
178
|
-
# We're just trying to test metadata here, so it's ok to link a false Label Studio project.
|
|
179
|
-
# We include a computed image column in order to ensure the creation of a stored proxy.
|
|
180
|
-
from pixeltable.io.label_studio import LabelStudioProject
|
|
181
|
-
col_mapping = Project.validate_columns(
|
|
182
|
-
v, {'str_field': pxt.StringType(), 'img_field': pxt.ImageType()}, {},
|
|
183
|
-
{'view_function_call': 'str_field', 'base_table_image_rot': 'img_field'}
|
|
184
|
-
)
|
|
185
|
-
project = LabelStudioProject('ls_project_0', 4171780, media_import_method='file', col_mapping=col_mapping)
|
|
186
|
-
v._link_external_store(project)
|
|
187
|
-
# Sanity check that the stored proxy column did get created
|
|
188
|
-
assert len(project.stored_proxies) == 1
|
|
189
|
-
assert t.base_table_image_rot.col in project.stored_proxies
|
|
190
|
-
|
|
191
|
-
def __add_expr_columns(self, t: pxt.Table, col_prefix: str, include_expensive_functions=False) -> None:
|
|
192
|
-
def add_column(col_name: str, col_expr: Any, stored: bool = True) -> None:
|
|
193
|
-
t.add_column(**{f'{col_prefix}_{col_name}': col_expr}, stored=stored)
|
|
194
|
-
|
|
195
|
-
# arithmetic_expr
|
|
196
|
-
add_column('plus', t.c2 + 6)
|
|
197
|
-
add_column('minus', t.c2 - 5)
|
|
198
|
-
add_column('times', t.c3 * 1.2)
|
|
199
|
-
add_column('div', t.c3 / 1.7)
|
|
200
|
-
add_column('mod', t.c2 % 11)
|
|
201
|
-
|
|
202
|
-
# column_property_ref
|
|
203
|
-
add_column('fileurl', t.c8.fileurl)
|
|
204
|
-
add_column('localpath', t.c8.localpath)
|
|
205
|
-
|
|
206
|
-
# comparison
|
|
207
|
-
add_column('lt', t.c2 < t.c3)
|
|
208
|
-
add_column('le', t.c2 <= t.c3)
|
|
209
|
-
add_column('gt', t.c2 > t.c3)
|
|
210
|
-
add_column('ge', t.c2 >= t.c3)
|
|
211
|
-
add_column('ne', t.c2 != t.c3)
|
|
212
|
-
add_column('eq', t.c2 == t.c3)
|
|
213
|
-
|
|
214
|
-
# compound_predicate
|
|
215
|
-
add_column('and', (t.c2 >= 5) & (t.c2 < 8))
|
|
216
|
-
add_column('or', (t.c2 > 1) | t.c4)
|
|
217
|
-
add_column('not', ~(t.c2 > 20))
|
|
218
|
-
|
|
219
|
-
# function_call
|
|
220
|
-
add_column('function_call', pxt.functions.string.format('{0} {key}', t.c1, key=t.c1)) # library function
|
|
221
|
-
add_column('test_udf', test_udf_stored(t.c2)) # stored udf
|
|
222
|
-
add_column('test_udf_batched', test_udf_stored_batched(t.c1, upper=False)) # batched stored udf
|
|
223
|
-
if include_expensive_functions:
|
|
224
|
-
# batched library function
|
|
225
|
-
add_column('batched', pxt.functions.huggingface.clip_text(t.c1, model_id='openai/clip-vit-base-patch32'))
|
|
226
|
-
|
|
227
|
-
# image_member_access
|
|
228
|
-
add_column('image_mode', t.c8.mode)
|
|
229
|
-
add_column('image_rot', t.c8.rotate(180), stored=False)
|
|
230
|
-
|
|
231
|
-
# in_predicate
|
|
232
|
-
add_column('isin_1', t.c1.isin(['test string 1', 'test string 2', 'test string 3']))
|
|
233
|
-
add_column('isin_2', t.c2.isin([1, 2, 3, 4, 5]))
|
|
234
|
-
add_column('isin_3', t.c2.isin(t.c6.f5))
|
|
235
|
-
|
|
236
|
-
# inline_array, inline_list, inline_dict
|
|
237
|
-
add_column('inline_array_1', pxt.array([[1, 2, 3], [4, 5, 6]]))
|
|
238
|
-
add_column('inline_array_2', pxt.array([['a', 'b', 'c'], ['d', 'e', 'f']]))
|
|
239
|
-
add_column('inline_array_exprs', pxt.array([[t.c2, t.c2 + 1], [t.c2 + 2, t.c2]]))
|
|
240
|
-
add_column('inline_array_mixed', pxt.array([[1, t.c2], [3, t.c2]]))
|
|
241
|
-
add_column('inline_list_1', [[1, 2, 3], [4, 5, 6]])
|
|
242
|
-
add_column('inline_list_2', [['a', 'b', 'c'], ['d', 'e', 'f']])
|
|
243
|
-
add_column('inline_list_exprs', [t.c1, [t.c1n, t.c2]])
|
|
244
|
-
add_column('inline_list_mixed', [1, 'a', t.c1, [1, 'a', t.c1n], 1, 'a'])
|
|
245
|
-
add_column('inline_dict', {'int': 22, 'dict': {'key': 'val'}, 'expr': t.c1})
|
|
246
|
-
|
|
247
|
-
# is_null
|
|
248
|
-
add_column('isnull', t.c1 == None)
|
|
249
|
-
|
|
250
|
-
# json_mapper and json_path
|
|
251
|
-
add_column('json_mapper', t.c6[3])
|
|
252
|
-
add_column('json_path', t.c6.f1)
|
|
253
|
-
add_column('json_path_nested', t.c6.f6.f7)
|
|
254
|
-
add_column('json_path_star', t.c6.f5['*'])
|
|
255
|
-
add_column('json_path_idx', t.c6.f5[3])
|
|
256
|
-
add_column('json_path_slice', t.c6.f5[1:3:2])
|
|
257
|
-
|
|
258
|
-
# literal
|
|
259
|
-
add_column('str_const', 'str')
|
|
260
|
-
add_column('int_const', 5)
|
|
261
|
-
add_column('float_const', 5.0)
|
|
262
|
-
add_column('timestamp_const_1', datetime.datetime.now())
|
|
263
|
-
add_column('timestamp_const_2', datetime.datetime.now().astimezone(ZoneInfo('America/Anchorage')))
|
|
264
|
-
|
|
265
|
-
# type_cast
|
|
266
|
-
add_column('astype', t.c2.astype(FloatType()))
|
|
267
|
-
|
|
268
|
-
# .apply
|
|
269
|
-
add_column('c2_to_string', t.c2.apply(str))
|
|
270
|
-
add_column('c6_to_string', t.c6.apply(json.dumps))
|
|
271
|
-
add_column('c6_back_to_json', t[f'{col_prefix}_c6_to_string'].apply(json.loads))
|
|
272
|
-
|
|
273
|
-
t.add_embedding_index(f'{col_prefix}_function_call', string_embed=embed_udf.clip_text_embed)
|
|
274
|
-
|
|
275
|
-
# query()
|
|
276
|
-
@t.query
|
|
277
|
-
def q1(i: int):
|
|
278
|
-
# this breaks; TODO: why?
|
|
279
|
-
#return t.where(t.c2 < i)
|
|
280
|
-
return t.where(t.c2 < i).select(t.c1, t.c2)
|
|
281
|
-
add_column('query_output', t.queries.q1(t.c2))
|
|
282
|
-
|
|
283
|
-
@t.query
|
|
284
|
-
def q2(s: str):
|
|
285
|
-
sim = t[f'{col_prefix}_function_call'].similarity(s)
|
|
286
|
-
return t.order_by(sim, asc=False).select(t[f'{col_prefix}_function_call']).limit(5)
|
|
287
|
-
add_column('sim_output', t.queries.q2(t.c1))
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
@pxt.udf(_force_stored=True)
|
|
291
|
-
def test_udf_stored(n: int) -> int:
|
|
292
|
-
return n + 1
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
@pxt.udf(batch_size=4, _force_stored=True)
|
|
296
|
-
def test_udf_stored_batched(strings: Batch[str], *, upper: bool = True) -> Batch[str]:
|
|
297
|
-
return [string.upper() if upper else string.lower() for string in strings]
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
def main() -> None:
|
|
301
|
-
_logger.info("Creating pixeltable test artifact.")
|
|
302
|
-
dumper = Dumper()
|
|
303
|
-
dumper.create_tables()
|
|
304
|
-
dumper.dump_db()
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
if __name__ == "__main__":
|
|
308
|
-
main()
|