pixeltable 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pixeltable might be problematic. Click here for more details.

Files changed (47) hide show
  1. pixeltable/__version__.py +2 -2
  2. pixeltable/catalog/table_version.py +2 -1
  3. pixeltable/dataframe.py +52 -27
  4. pixeltable/env.py +92 -4
  5. pixeltable/exec/__init__.py +1 -1
  6. pixeltable/exec/aggregation_node.py +3 -3
  7. pixeltable/exec/cache_prefetch_node.py +13 -7
  8. pixeltable/exec/component_iteration_node.py +3 -9
  9. pixeltable/exec/data_row_batch.py +17 -5
  10. pixeltable/exec/exec_node.py +32 -12
  11. pixeltable/exec/expr_eval/__init__.py +1 -0
  12. pixeltable/exec/expr_eval/evaluators.py +245 -0
  13. pixeltable/exec/expr_eval/expr_eval_node.py +404 -0
  14. pixeltable/exec/expr_eval/globals.py +114 -0
  15. pixeltable/exec/expr_eval/row_buffer.py +76 -0
  16. pixeltable/exec/expr_eval/schedulers.py +232 -0
  17. pixeltable/exec/in_memory_data_node.py +2 -2
  18. pixeltable/exec/row_update_node.py +14 -14
  19. pixeltable/exec/sql_node.py +2 -2
  20. pixeltable/exprs/column_ref.py +5 -1
  21. pixeltable/exprs/data_row.py +50 -40
  22. pixeltable/exprs/expr.py +57 -12
  23. pixeltable/exprs/function_call.py +54 -19
  24. pixeltable/exprs/inline_expr.py +12 -21
  25. pixeltable/exprs/literal.py +25 -8
  26. pixeltable/exprs/row_builder.py +23 -0
  27. pixeltable/func/aggregate_function.py +4 -0
  28. pixeltable/func/callable_function.py +54 -4
  29. pixeltable/func/expr_template_function.py +5 -1
  30. pixeltable/func/function.py +48 -7
  31. pixeltable/func/query_template_function.py +16 -7
  32. pixeltable/func/udf.py +7 -1
  33. pixeltable/functions/__init__.py +1 -1
  34. pixeltable/functions/anthropic.py +95 -21
  35. pixeltable/functions/gemini.py +2 -6
  36. pixeltable/functions/openai.py +207 -28
  37. pixeltable/globals.py +1 -1
  38. pixeltable/plan.py +24 -9
  39. pixeltable/store.py +6 -0
  40. pixeltable/type_system.py +3 -3
  41. pixeltable/utils/arrow.py +3 -3
  42. {pixeltable-0.3.0.dist-info → pixeltable-0.3.1.dist-info}/METADATA +3 -1
  43. {pixeltable-0.3.0.dist-info → pixeltable-0.3.1.dist-info}/RECORD +46 -41
  44. pixeltable/exec/expr_eval_node.py +0 -232
  45. {pixeltable-0.3.0.dist-info → pixeltable-0.3.1.dist-info}/LICENSE +0 -0
  46. {pixeltable-0.3.0.dist-info → pixeltable-0.3.1.dist-info}/WHEEL +0 -0
  47. {pixeltable-0.3.0.dist-info → pixeltable-0.3.1.dist-info}/entry_points.txt +0 -0
@@ -1,5 +1,5 @@
1
1
  pixeltable/__init__.py,sha256=c-Z21TqJUbzGQKsO4CS4lVFwWlZnVhpIR94cL6RNmDo,1411
2
- pixeltable/__version__.py,sha256=zP_zQgwZByDmFisB1NLZeZLdsIy7Xh24k7e1Y59IAY4,112
2
+ pixeltable/__version__.py,sha256=zVI6ltlHGwKITzBi4TSc2jGBnWUb4mnqjF_PYoGqTuw,112
3
3
  pixeltable/catalog/__init__.py,sha256=u8Ods4ncTY7DI5w0jyHVC0QDVDLHAy8Rqia3qsM5OH8,517
4
4
  pixeltable/catalog/catalog.py,sha256=tyDyI5wQw7vV6_FChrp9qgGCRClcjiSdW3eygYT0p9s,7849
5
5
  pixeltable/catalog/column.py,sha256=ezeKoGl6aBTzSZBihDA6vdETcvyCguAD4OsmrxWs73A,9595
@@ -11,45 +11,50 @@ pixeltable/catalog/path.py,sha256=QgccEi_QOfaKt8YsR2zLtd_z7z7QQkU_1kprJFi2SPQ,16
11
11
  pixeltable/catalog/path_dict.py,sha256=V7YQM0QhUWr4FbgUiDUIweGbmMJr687QCm2p614Xrs0,6473
12
12
  pixeltable/catalog/schema_object.py,sha256=NUE6Fx6Km0UUJ6WcE9mOgL2UQCLBMQjINQLp_mljzKA,2382
13
13
  pixeltable/catalog/table.py,sha256=Gqb1tT9l8oB5eeeFuj45fy0_1fdCqizmRWvyjFH-U9Y,62404
14
- pixeltable/catalog/table_version.py,sha256=hbnAtd-0EsVFCuWE1yrmm_CLar6NNQDRevgDjduh2CY,58581
14
+ pixeltable/catalog/table_version.py,sha256=wuvFYbr6WrwhDWC202tUIgF8T61P69WOtduBUDq5EsI,58601
15
15
  pixeltable/catalog/table_version_path.py,sha256=CczGbcz5ESq663arreri_p4chMZHozgG6k3y-ajkJN4,5787
16
16
  pixeltable/catalog/view.py,sha256=qUjou_z_YwKTqTIRUY4Q3JxWUEiKvl2pDi7DNOnw4fw,10709
17
- pixeltable/dataframe.py,sha256=Xd1xJ0Drj2WBC1B9fJ2TCFMI23HeMV7L7RZJi7Wm2ew,47735
18
- pixeltable/env.py,sha256=jxnHxliUpWegRbsT6lRz83nNqDG29bWAy5wyAYImNGg,31338
17
+ pixeltable/dataframe.py,sha256=KrAB-l_qdArlT3pnlSH69k0mqs0AK2wFhQ4V_7Wcjyw,48639
18
+ pixeltable/env.py,sha256=VMS4Kj-RVEeFgmLzuQ2sFRQFtG4dLRbBZ9VzVzvO7hk,34993
19
19
  pixeltable/exceptions.py,sha256=NuFY2WtkQpLfLHT_J70kOw9Tr0kEDkkgo-u7As4Gaq4,410
20
- pixeltable/exec/__init__.py,sha256=bWbIA7U7y1hC40VOSv_p9doaMuOB0i3r1O2UFNw0VEs,494
21
- pixeltable/exec/aggregation_node.py,sha256=0LdoPp_CR_UbcS60XkDw66SqlrQnw6Dy6KDWqi4PJ6k,4005
22
- pixeltable/exec/cache_prefetch_node.py,sha256=hrJA5PxLSmFRuZdZO4Nc-vy6_PSb9YO4rnJc6MSTPHE,12017
23
- pixeltable/exec/component_iteration_node.py,sha256=ABuXGbDRQWLGuaBnfK7bvOxCrz81vMMiAvXHHI8SX4c,4930
24
- pixeltable/exec/data_row_batch.py,sha256=qPN0GtBGt72Jnap-UN53VlUPOpYEvqJhp1nuTeJ_8Co,2925
20
+ pixeltable/exec/__init__.py,sha256=E_ub_ftAJVebEmKHZs47egeJqMfAbbx4SS1l7jR-Nd0,489
21
+ pixeltable/exec/aggregation_node.py,sha256=iE3UHHW0IbkHsY7kYJAg6m1UKyKUu2_tJuzgL_rQUVE,4038
22
+ pixeltable/exec/cache_prefetch_node.py,sha256=9jwCzWuAhymff6xwvBietw2S_eYno6y2n59QWbzowns,12220
23
+ pixeltable/exec/component_iteration_node.py,sha256=4DkEzekeuaKok-VjI0Y3FIMF7BdoKHNmX2tAM10hsOY,4730
24
+ pixeltable/exec/data_row_batch.py,sha256=5WhdS3ahtCP3wROw4NPJjtvaQUAWKwyN1rHKbfdT6TU,3308
25
25
  pixeltable/exec/exec_context.py,sha256=-FbfYLcGOL7mviru5dE1A7x4YxLbdKoXBHN3OSbqbcg,1084
26
- pixeltable/exec/exec_node.py,sha256=3t_Ri_FQfc-dV91YGSq17Av3vPJth5mHC-LAVc-1Whc,3222
27
- pixeltable/exec/expr_eval_node.py,sha256=8IzcxHvXCj69RKFfrt4XEol88SGAlLWMl0bjAijX7Ug,11887
28
- pixeltable/exec/in_memory_data_node.py,sha256=uK3puLZDzUSJUJwGwkTMYoPqNIvE67jtQodhE2sCz2M,3452
29
- pixeltable/exec/row_update_node.py,sha256=b8yuKtkmI2Q6U-7svKbkjdM3Z85ddZoJgJgUa17j-YE,2773
30
- pixeltable/exec/sql_node.py,sha256=TOeSI7TIUh8SG3esExj5zaWgLrjmpqMN74iJlPq16D0,19546
26
+ pixeltable/exec/exec_node.py,sha256=qDp63PFlMQUVwav0JIjuP0r0BH9EwW7s_NgqFE_Zn20,3735
27
+ pixeltable/exec/expr_eval/__init__.py,sha256=t0o2T7MxfH09TI7KlzxEjPUySejn2vkK06LC8t4FjsQ,41
28
+ pixeltable/exec/expr_eval/evaluators.py,sha256=HtcrWivzoGWzpEAYRgyKvFRGVKq4rMO-QpK7fZq4fE8,11349
29
+ pixeltable/exec/expr_eval/expr_eval_node.py,sha256=37c5eevFMAvvYmSbqzcO4Wjxouy3R5a5vSnLLjifeic,19786
30
+ pixeltable/exec/expr_eval/globals.py,sha256=UAlOAOJNHw4sJ3CXXk2Gh0qhOxEm2cGtNlkvvvebx1g,3895
31
+ pixeltable/exec/expr_eval/row_buffer.py,sha256=Wjc6ZklxSW4-sCAMEGIzESII1uuC6Urzmq6JeUO3ALA,2737
32
+ pixeltable/exec/expr_eval/schedulers.py,sha256=MqO0r5TaQhywozpRPL87S8YwZmKEc8sdEerX2LHDaCY,10962
33
+ pixeltable/exec/in_memory_data_node.py,sha256=CwuwkA2xdyApMfPfSQamlLdbb-IQn7R7zeBBJiI8I7g,3479
34
+ pixeltable/exec/row_update_node.py,sha256=MEvI3Yzw6YcUDTufSNVNVtyG83Hj6zHzEI9fvKtAgRQ,2859
35
+ pixeltable/exec/sql_node.py,sha256=vMHNHzhrlk2Ej58vMZzk7hYAeHFHlBtkLwYZsOVn3CQ,19573
31
36
  pixeltable/exprs/__init__.py,sha256=zx5OTxfUpFs_U36CAO-81v6RCX49wqRw3eHaLfytvkE,1044
32
37
  pixeltable/exprs/arithmetic_expr.py,sha256=I_OESFmUHlMSDmMy699-2JnFb7KbRjvh1GF06u8rBQ8,6322
33
38
  pixeltable/exprs/array_slice.py,sha256=eSqtQRxvweW_CwTRgkDD189i-BmTJ48dsW3znRPDebg,2180
34
39
  pixeltable/exprs/column_property_ref.py,sha256=ycHqZ7_LTFz_gag_53T5KRN6RfZckpw83DvvtxmcynM,3788
35
- pixeltable/exprs/column_ref.py,sha256=Z2YAiZ0nMv_ecbw91qTc9VN6DB2hZj-_RNkogZcu4ng,10784
40
+ pixeltable/exprs/column_ref.py,sha256=win5w5oU0isvpa6bcB9c3I-ufI50RvZ2YOnNcSq-NX0,10886
36
41
  pixeltable/exprs/comparison.py,sha256=w1FuhnGsJRoCLYX3ztIHZDywf8nZeT2tKKwRjIQa6Lc,5101
37
42
  pixeltable/exprs/compound_predicate.py,sha256=6N_zN4A_hKGLordNY0si-8KHeFWG1UtyeiTHcs69IrM,3846
38
- pixeltable/exprs/data_row.py,sha256=rLtKxlst9mK6684A5y-nsjBcalyKEcKAWcYCtNpK10w,9852
39
- pixeltable/exprs/expr.py,sha256=E1V_fU1qA4nkHLHTqGkxr8exJE6I9-b9LwNVXxGNTXo,30463
43
+ pixeltable/exprs/data_row.py,sha256=11WAmLmkhrjsKvKfPwj0dg8IkbeLwVcbFpK6eq3VUsM,10577
44
+ pixeltable/exprs/expr.py,sha256=JYjdweVm6z5Fw37_WhefGDaVN_2jQtUV9BiVwt20oYE,32203
40
45
  pixeltable/exprs/expr_dict.py,sha256=xkvo_iVPOLMq3WkBZQ2FOoXqYoebuV6XGlidPJxdOkY,1588
41
46
  pixeltable/exprs/expr_set.py,sha256=GeUQXadzJbAqQOGdsO6Z5hPAp0A03YUr2hyIvfZDYEE,2576
42
- pixeltable/exprs/function_call.py,sha256=5lMTj4G6jOarabZ0b3xpgAMJe3rgDdCSXbYwnrZEjxs,26196
47
+ pixeltable/exprs/function_call.py,sha256=DrBFDU2cKVBj7TiDsAJFuElFMClg1RO9W8klnLNz8l0,28033
43
48
  pixeltable/exprs/globals.py,sha256=5pwn5vdi-EEpYBpPty658YV45myY7W0iFIfTH7QIzak,2032
44
49
  pixeltable/exprs/in_predicate.py,sha256=VxSn9TA_3UDHTC6rqQ12No5HbZO7SuE4DglpwAoI58s,3783
45
- pixeltable/exprs/inline_expr.py,sha256=oALVAM5BVFskEQgkfgGIhKFm6V2FUlrPtocGtqtXM0w,7867
50
+ pixeltable/exprs/inline_expr.py,sha256=PRiW4kvxBmvbJD27Rr-xQzcL7f5YWmnHd3GuhwOGocw,7707
46
51
  pixeltable/exprs/is_null.py,sha256=rnusy1j33o48Z-qjguj09G7AmsBHFJI5SWQXOAjgKQw,1092
47
52
  pixeltable/exprs/json_mapper.py,sha256=yhefrgvy76czfyTER_oRXnfxOVQb42Q4Us1KkXhSsLM,4545
48
53
  pixeltable/exprs/json_path.py,sha256=UTf3VUcyEmEBJs6yU4kONGcZ8fy1o6mIe9MlntnUd6M,6811
49
- pixeltable/exprs/literal.py,sha256=TTKb0gw6qck9D61SwVDuBrLrBrGwEhkCB-m0ILFpWFk,3764
54
+ pixeltable/exprs/literal.py,sha256=Nhh-fXYMqGL4Tfm2YiFUd4ALMjzG7nJ2H4jHZe9XS1Y,4394
50
55
  pixeltable/exprs/method_ref.py,sha256=3O_uFMP6wWGiwJWri8on-47EVl-QD3AiAc7hXJMADxs,2615
51
56
  pixeltable/exprs/object_ref.py,sha256=GVg6uxZnKwFVTC0kouJq-wMFP-gUPb_ld_syHrsGMdE,1283
52
- pixeltable/exprs/row_builder.py,sha256=fJEb8XU7j8M4PIc67O35a5Pzhoh_fs-i77lYxJPrA6o,18444
57
+ pixeltable/exprs/row_builder.py,sha256=hpvmQyBie5pRtOkMKQu0xCxB18EK8iw_aUbtxYlzWcs,19862
53
58
  pixeltable/exprs/rowid_ref.py,sha256=iNBZ0wIhBAGkGCrNP9UQ2FeK8OajVX-eI4dtDA_bfHU,4401
54
59
  pixeltable/exprs/similarity_expr.py,sha256=GhiopGxN3wu5MOEqfWN4PNbqgSszg2VIC1_bqVPpm3Q,4333
55
60
  pixeltable/exprs/sql_element_cache.py,sha256=8i9ZslKWKTd1lUN7JvwRtQ-PFD1QMi5LBYdwcGA2-p0,1364
@@ -60,21 +65,21 @@ pixeltable/ext/functions/__init__.py,sha256=hIjPEKC5E5uJOXlQqUyhP9yn9ZqWOCJAlj0k
60
65
  pixeltable/ext/functions/whisperx.py,sha256=jojjNhazcYiAh1scwUl-erhIDRr4kOTkcLrjy0xcp6g,2325
61
66
  pixeltable/ext/functions/yolox.py,sha256=k-pQTelv4Tea3AXvDB7Kc7YCIa1uexjVGqxETP0B_hc,5351
62
67
  pixeltable/func/__init__.py,sha256=Tmy5srLdxcWDwxwnXXj2mTNUMimdYY-MdU6fJ9SpePc,457
63
- pixeltable/func/aggregate_function.py,sha256=SWqU3stHBzrZbzoNCNHBhMHqQbT9rpVTelWY6otboSg,12649
64
- pixeltable/func/callable_function.py,sha256=AK2sO-nOlDqDSDJvJ5zOp2UNwDJY-2keJlfPycGzRDw,6500
65
- pixeltable/func/expr_template_function.py,sha256=xoi7yngicX4F_-qa64a38_dvX3Bs_4BDphodTEul06o,5381
66
- pixeltable/func/function.py,sha256=1bGC7e93QrRZhSij-DdMVL94Lz-A7OWwbncmPhJj1ps,16799
68
+ pixeltable/func/aggregate_function.py,sha256=eRPKCX2gdpEokyn5tKWxgbrC2UneaC0o2zIUp39Gd-w,12717
69
+ pixeltable/func/callable_function.py,sha256=KL69czs2EZ9Fv5EoDT4xaXILePhaV9owTBNBgtNU4RU,8941
70
+ pixeltable/func/expr_template_function.py,sha256=prKOIfO5rO_M3y6lSOB2zTYqZYb2nycbJ5m5FlJQ27E,5454
71
+ pixeltable/func/function.py,sha256=Ida2SOKQzryiuOqQDF5l9Q-8LmmGx1rNvzxHwuOrtsg,18853
67
72
  pixeltable/func/function_registry.py,sha256=Ps09iPDRJRKj-NQtancrI07zqFBIPpASGZ636mpBiI0,12333
68
73
  pixeltable/func/globals.py,sha256=sEwn6lGgHMp6VQORb_P5qRd_-Q2_bUSqvqM9-XPN_ec,1483
69
- pixeltable/func/query_template_function.py,sha256=p3zp5aNIxlr0QR1ROVJLISDnZXpfOhqAeMIwr8l2b5g,5170
74
+ pixeltable/func/query_template_function.py,sha256=zaRpthtFDsN0RGsZMebXiCSukW7oBHY4oskUx_93H8Y,5316
70
75
  pixeltable/func/signature.py,sha256=wWf07OawOwDmqZOjUPgHJHAhAQ-fd61PGBx82aWLBgA,11212
71
76
  pixeltable/func/tools.py,sha256=7OAHVb9KbmcPtfMWsbKAoRu3kx-_japSTE-hgRQJefM,5988
72
- pixeltable/func/udf.py,sha256=a8FqOcDqPvUICvELpPOQVsrvr0sgZBQuV4aVHXuexac,8603
73
- pixeltable/functions/__init__.py,sha256=2vdpJ3S1pUle8dBzCVM70o8Lj1c0lgccZPJgPYms_CQ,403
74
- pixeltable/functions/anthropic.py,sha256=FzMK4zhz4HvPqJmOM4CBIlpVa8JLiRShi8jEd2abHsg,4710
77
+ pixeltable/func/udf.py,sha256=p7imgwk7S03q3lHZxqAZUKr63XSeSsrD8kWTbwOLQWo,8882
78
+ pixeltable/functions/__init__.py,sha256=C7Okwst3JdhzwIFWXhyJ3F0EHKGKor2R8dRMvQ3y6Us,428
79
+ pixeltable/functions/anthropic.py,sha256=rSgMWmR2YVRfrdbbAPY5OvKkbyV1DHBJIHFGdDwVnP4,8800
75
80
  pixeltable/functions/audio.py,sha256=7213nTnqKJ6vM9kalaoJ283OwX5SGEJN10vDhaRNZ6E,644
76
81
  pixeltable/functions/fireworks.py,sha256=qwFC_eIaDs-glxyJ_IVXaNGkpgPzeRsQ_SdpzueBxq0,2605
77
- pixeltable/functions/gemini.py,sha256=RQ3I25nXSXUXdF0rmhnv5XOgZXb_6SIgMc_hEyF83SI,2783
82
+ pixeltable/functions/gemini.py,sha256=0aqsDpxI0AgeWu6zXQxtFbp3nhhuwXqzulZyot6UKdg,2613
78
83
  pixeltable/functions/globals.py,sha256=Q5S84Vk_ovCADLivXkn4vfUnX8oSgW8v1dZ0aPoz7C4,4844
79
84
  pixeltable/functions/huggingface.py,sha256=fXcj6NC5Hz_nYE-0CjvSJ1sD3Jq726hQXzueqnQ6Kr4,20576
80
85
  pixeltable/functions/image.py,sha256=3Qm4ybAT_o4YUl3bzhEXy8dKOwgZ7RCUV-ky-dbL_jc,13836
@@ -83,7 +88,7 @@ pixeltable/functions/llama_cpp.py,sha256=1awALuAXVpQH64l7vQlM8gvxLDix4c1-6DV3nG5
83
88
  pixeltable/functions/math.py,sha256=WPoH9zD9_GdwvBs-FSC3Sqb70gOPNouhPcBZABsuLwI,1541
84
89
  pixeltable/functions/mistralai.py,sha256=GpxtT-a8ltx1kQC8XTJRflxkh-17VhzzFTkxqIUsba8,5494
85
90
  pixeltable/functions/ollama.py,sha256=z-g0cCJ-WMf6RI3SXIVVbbOlrWNR8nxGTyzET1rratU,4320
86
- pixeltable/functions/openai.py,sha256=Jikl9oYPiKy61r5Ohe1xqjvb6SLCbNTYFr4yAHFkYME,17101
91
+ pixeltable/functions/openai.py,sha256=Sh2zqwW72CYiMoyEELLpY1fwTmxuLGnAQbtEkmTac4c,24030
87
92
  pixeltable/functions/replicate.py,sha256=j8ZedScOMInmHWmriQSUOviw6tp8gQr-W6n21PNNL2g,2188
88
93
  pixeltable/functions/string.py,sha256=VqzhVildxTt_XblW89Kl5Zd6MVOU71eaX2LTMY5jkUg,20366
89
94
  pixeltable/functions/timestamp.py,sha256=KOm5eVF51PqOCTqwq8cKCAEJNWMgu2xN_yTtGf7yixU,9095
@@ -92,7 +97,7 @@ pixeltable/functions/util.py,sha256=GgKTzCjvzUQNjWtSObTkfxkvJ9GVUOzKimY45WhE25M,
92
97
  pixeltable/functions/video.py,sha256=12jnOdU0G-Hk42rJx-S5QC77-bDkUflkxfkjej8n5pM,6702
93
98
  pixeltable/functions/vision.py,sha256=MEgkNp-4tUUeQS05VEJBrMgQFM48aNsNlksfAl9rH7w,15494
94
99
  pixeltable/functions/whisper.py,sha256=f2wqRd0n9jSBqRZN3W93UaetiAHtbsK0j9jXR2j2kkQ,2913
95
- pixeltable/globals.py,sha256=W5-4KEVqg9tZwfL6EffkjQYtRNXDLKXj_Bv32liZeHI,33895
100
+ pixeltable/globals.py,sha256=9UfD7FOn71dbzPZjV-LxzhxXBhwPcpLt46J0_RVPynk,33899
96
101
  pixeltable/index/__init__.py,sha256=XBwetNQQwnz0fiKwonOKhyy_U32l_cjt77kNvEIdjWs,102
97
102
  pixeltable/index/base.py,sha256=zo0YvJI3oXiK6hZJztB36ZftKKhLfO75Zq3t-PeQA6M,1556
98
103
  pixeltable/index/btree.py,sha256=JFerLyyLoBaD0FSF_jJ6iJFBVa-z_et--KdNR02xjRg,2264
@@ -130,12 +135,12 @@ pixeltable/metadata/converters/convert_25.py,sha256=3sCXMbHiLJWMvbghj-XaeW4_1xSE
130
135
  pixeltable/metadata/converters/util.py,sha256=NgzoKralRIXwOqtguifVsBbpEN2M2X1v8lvOnsdGnPs,6100
131
136
  pixeltable/metadata/notes.py,sha256=RsyIuwl3KhFRstB6cLX8ySWfIx67LAyr05HYXovu7Yc,779
132
137
  pixeltable/metadata/schema.py,sha256=H9t51cbhXOhNu9Xog2VTjZlkTCx4cjQAlwOXi3HSd2E,9448
133
- pixeltable/plan.py,sha256=dzG3iD9ClggFyIYmDuIK7H3eBC8HV3ft__f4ZULQvow,40687
138
+ pixeltable/plan.py,sha256=DgzP5kDmBooaUioZ0KBlSUcx_jpqy456CnbDRVzFBJM,41501
134
139
  pixeltable/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
135
- pixeltable/store.py,sha256=m-OeilqOFai5wUXvtXTV9txzn-GhK2lfA7fkvzrfWFg,21974
136
- pixeltable/type_system.py,sha256=IbUXeaqsguRAneiXZc2WrAXskUiN1IPJ3O9xHBGGQhk,48154
140
+ pixeltable/store.py,sha256=-A7oRGHDpvhUZzx68QSde0t2NvUjeM1rOw9KdwukQ9s,22497
141
+ pixeltable/type_system.py,sha256=nR004iZjvA5BIBSXsBXaHbpUPbddtiUCUzB0To1Bakw,48115
137
142
  pixeltable/utils/__init__.py,sha256=UYlrf6TIWJT0g-Hac0b34-dEk478B5Qx8dGco34YlIk,439
138
- pixeltable/utils/arrow.py,sha256=4AyCAkkNfNChVTaPAVS5j1ZO7BtTxn11duonolfktV8,3931
143
+ pixeltable/utils/arrow.py,sha256=iuZhueel1FT_3UDAMITFXxodLB7w8TrUpsMpie3IDXg,3991
139
144
  pixeltable/utils/coco.py,sha256=WCUyoj0dTyJFbPB7frEQUvY92SlEPxQ068r-3QAwROE,7285
140
145
  pixeltable/utils/code.py,sha256=AOw1u2r8_DQXpX-lxJhyHWARGrCRDXOJHFVgKOi54Uc,1231
141
146
  pixeltable/utils/description_helper.py,sha256=P-8EE2pRFP8s3coe73frgV68yt4Dp3saErCUehMxKUw,3759
@@ -148,8 +153,8 @@ pixeltable/utils/pytorch.py,sha256=6RvOCjy_QV4gc-aht-3d0zoASkuv-warfpl87vgmuKw,3
148
153
  pixeltable/utils/s3.py,sha256=huA5hxDGkPIu18zWet76o0FsO7Vbtp-iPmnOzCe-MvA,586
149
154
  pixeltable/utils/sql.py,sha256=j_tj0h4ffm-DhUIJbvGphxrVyBKlNTwDKqWGhRQ5_PY,795
150
155
  pixeltable/utils/transactional_directory.py,sha256=UGzCrGtLR3hEEf8sYGuWBzLVFAEQml3vdIavigWeTBM,1349
151
- pixeltable-0.3.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
152
- pixeltable-0.3.0.dist-info/METADATA,sha256=GiNmcGREs3AUhpt4tKPImEGYzzD4XHMR1StiJSEkWhI,19304
153
- pixeltable-0.3.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
154
- pixeltable-0.3.0.dist-info/entry_points.txt,sha256=ToOd-pRgG7AitEBgYoBCRRB4-KVDQ0pj_9T4a1LgwA4,97
155
- pixeltable-0.3.0.dist-info/RECORD,,
156
+ pixeltable-0.3.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
157
+ pixeltable-0.3.1.dist-info/METADATA,sha256=tRC6nn3plpwtLqC5TZsNDfDRB9SCj2NyNJIvzK6kWMA,19370
158
+ pixeltable-0.3.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
159
+ pixeltable-0.3.1.dist-info/entry_points.txt,sha256=ToOd-pRgG7AitEBgYoBCRRB4-KVDQ0pj_9T4a1LgwA4,97
160
+ pixeltable-0.3.1.dist-info/RECORD,,
@@ -1,232 +0,0 @@
1
- import logging
2
- import sys
3
- import time
4
- import warnings
5
- from dataclasses import dataclass
6
- from typing import Iterable, Optional
7
-
8
- from tqdm import TqdmWarning, tqdm
9
-
10
- from pixeltable import exprs
11
- from pixeltable.func import CallableFunction
12
-
13
- from .data_row_batch import DataRowBatch
14
- from .exec_node import ExecNode
15
-
16
- _logger = logging.getLogger('pixeltable')
17
-
18
-
19
- class ExprEvalNode(ExecNode):
20
- """Materializes expressions
21
- """
22
- @dataclass
23
- class Cohort:
24
- """List of exprs that form an evaluation context and contain calls to at most one external function"""
25
- exprs_: list[exprs.Expr]
26
- batched_fn: Optional[CallableFunction]
27
- segment_ctxs: list['exprs.RowBuilder.EvalCtx']
28
- target_slot_idxs: list[int]
29
- batch_size: int = 8
30
-
31
- def __init__(
32
- self, row_builder: exprs.RowBuilder, output_exprs: Iterable[exprs.Expr], input_exprs: Iterable[exprs.Expr],
33
- input: ExecNode
34
- ):
35
- super().__init__(row_builder, output_exprs, input_exprs, input)
36
- self.input_exprs = input_exprs
37
- input_slot_idxs = {e.slot_idx for e in input_exprs}
38
- # we're only materializing exprs that are not already in the input
39
- self.target_exprs = [e for e in output_exprs if e.slot_idx not in input_slot_idxs]
40
- self.pbar: Optional[tqdm] = None
41
- self.cohorts: list[ExprEvalNode.Cohort] = []
42
- self._create_cohorts()
43
-
44
- def __next__(self) -> DataRowBatch:
45
- input_batch = next(self.input)
46
- # compute target exprs
47
- for cohort in self.cohorts:
48
- self._exec_cohort(cohort, input_batch)
49
- _logger.debug(f'ExprEvalNode: returning {len(input_batch)} rows')
50
- return input_batch
51
-
52
- def _open(self) -> None:
53
- warnings.simplefilter("ignore", category=TqdmWarning)
54
- # This is a temporary hack. When B-tree indices on string columns were implemented (via computed columns
55
- # that invoke the `BtreeIndex.str_filter` udf), it resulted in frivolous progress bars appearing on every
56
- # insertion. This special-cases the `str_filter` call to suppress the corresponding progress bar.
57
- # TODO(aaron-siegel) Remove this hack once we clean up progress bars more generally.
58
- is_str_filter_node = all(
59
- isinstance(expr, exprs.FunctionCall) and expr.fn.name == 'str_filter' for expr in self.output_exprs
60
- )
61
- if self.ctx.show_pbar and not is_str_filter_node:
62
- self.pbar = tqdm(
63
- total=len(self.target_exprs) * self.ctx.num_rows,
64
- desc='Computing cells',
65
- unit=' cells',
66
- ncols=100,
67
- file=sys.stdout
68
- )
69
-
70
- def _close(self) -> None:
71
- if self.pbar is not None:
72
- self.pbar.close()
73
-
74
- def _get_batched_fn(self, expr: exprs.Expr) -> Optional[CallableFunction]:
75
- if isinstance(expr, exprs.FunctionCall) and isinstance(expr.fn, CallableFunction) and expr.fn.is_batched:
76
- return expr.fn
77
- return None
78
-
79
- def _is_batched_fn_call(self, expr: exprs.Expr) -> bool:
80
- return self._get_batched_fn(expr) is not None
81
-
82
- def _create_cohorts(self) -> None:
83
- all_exprs = self.row_builder.get_dependencies(self.target_exprs)
84
- # break up all_exprs into cohorts such that each cohort contains calls to at most one external function;
85
- # seed the cohorts with only the ext fn calls
86
- cohorts: list[list[exprs.Expr]] = []
87
- current_batched_fn: Optional[CallableFunction] = None
88
- for e in all_exprs:
89
- if not self._is_batched_fn_call(e):
90
- continue
91
- assert isinstance(e, exprs.FunctionCall)
92
- assert isinstance(e.fn, CallableFunction)
93
- if current_batched_fn is None or current_batched_fn != e.fn:
94
- # create a new cohort
95
- cohorts.append([])
96
- current_batched_fn = e.fn
97
- cohorts[-1].append(e)
98
-
99
- # expand the cohorts to include all exprs that are in the same evaluation context as the external calls;
100
- # cohorts are evaluated in order, so we can exclude the target slots from preceding cohorts and input slots
101
- exclude = set(e.slot_idx for e in self.input_exprs)
102
- all_target_slot_idxs = set(e.slot_idx for e in self.target_exprs)
103
- target_slot_idxs: list[list[int]] = [] # the ones materialized by each cohort
104
- for i in range(len(cohorts)):
105
- cohorts[i] = self.row_builder.get_dependencies(
106
- cohorts[i], exclude=[self.row_builder.unique_exprs[slot_idx] for slot_idx in exclude])
107
- target_slot_idxs.append(
108
- [e.slot_idx for e in cohorts[i] if e.slot_idx in all_target_slot_idxs])
109
- exclude.update(target_slot_idxs[-1])
110
-
111
- all_cohort_slot_idxs = set(e.slot_idx for cohort in cohorts for e in cohort)
112
- remaining_slot_idxs = set(all_target_slot_idxs) - all_cohort_slot_idxs
113
- if len(remaining_slot_idxs) > 0:
114
- cohorts.append(self.row_builder.get_dependencies(
115
- [self.row_builder.unique_exprs[slot_idx] for slot_idx in remaining_slot_idxs],
116
- exclude=[self.row_builder.unique_exprs[slot_idx] for slot_idx in exclude]))
117
- target_slot_idxs.append(list(remaining_slot_idxs))
118
- # we need to have captured all target slots at this point
119
- assert all_target_slot_idxs == set().union(*target_slot_idxs)
120
-
121
- for i in range(len(cohorts)):
122
- cohort = cohorts[i]
123
- # segment the cohort into sublists that contain either a single ext. function call or no ext. function calls
124
- # (i.e., only computed cols)
125
- assert len(cohort) > 0
126
- # create the first segment here, so we can avoid checking for an empty list in the loop
127
- segments = [[cohort[0]]]
128
- is_batched_segment = self._is_batched_fn_call(cohort[0])
129
- batched_fn: Optional[CallableFunction] = self._get_batched_fn(cohort[0])
130
- for e in cohort[1:]:
131
- if self._is_batched_fn_call(e):
132
- segments.append([e])
133
- is_batched_segment = True
134
- batched_fn = self._get_batched_fn(e)
135
- else:
136
- if is_batched_segment:
137
- # start a new segment
138
- segments.append([])
139
- is_batched_segment = False
140
- segments[-1].append(e)
141
-
142
- # we create the EvalCtxs manually because create_eval_ctx() would repeat the dependencies of each segment
143
- segment_ctxs = [
144
- exprs.RowBuilder.EvalCtx(
145
- slot_idxs=[e.slot_idx for e in s], exprs=s, target_slot_idxs=[], target_exprs=[])
146
- for s in segments
147
- ]
148
- cohort_info = self.Cohort(cohort, batched_fn, segment_ctxs, target_slot_idxs[i])
149
- self.cohorts.append(cohort_info)
150
-
151
- def _exec_cohort(self, cohort: Cohort, rows: DataRowBatch) -> None:
152
- """Compute the cohort for the entire input batch by dividing it up into sub-batches"""
153
- batch_start_idx = 0 # start row of the current sub-batch
154
- # for multi-resolution models, we re-assess the correct ext fn batch size for each input batch
155
- ext_batch_size = cohort.batched_fn.get_batch_size() if cohort.batched_fn is not None else None
156
- if ext_batch_size is not None:
157
- cohort.batch_size = ext_batch_size
158
-
159
- while batch_start_idx < len(rows):
160
- num_batch_rows = min(cohort.batch_size, len(rows) - batch_start_idx)
161
- for segment_ctx in cohort.segment_ctxs:
162
- if not self._is_batched_fn_call(segment_ctx.exprs[0]):
163
- # compute batch row-wise
164
- for row_idx in range(batch_start_idx, batch_start_idx + num_batch_rows):
165
- self.row_builder.eval(
166
- rows[row_idx], segment_ctx, self.ctx.profile, ignore_errors=self.ctx.ignore_errors)
167
- else:
168
- fn_call = segment_ctx.exprs[0]
169
- assert isinstance(fn_call, exprs.FunctionCall)
170
- # make a batched external function call
171
- arg_batches: list[list[exprs.Expr]] = [[] for _ in range(len(fn_call.args))]
172
- kwarg_batches: dict[str, list[exprs.Expr]] = {k: [] for k in fn_call.kwargs.keys()}
173
-
174
- valid_batch_idxs: list[int] = [] # rows with exceptions are not valid
175
- for row_idx in range(batch_start_idx, batch_start_idx + num_batch_rows):
176
- row = rows[row_idx]
177
- if row.has_exc(fn_call.slot_idx):
178
- # one of our inputs had an exception, skip this row
179
- continue
180
- valid_batch_idxs.append(row_idx)
181
- args, kwargs = fn_call._make_args(row)
182
- for i in range(len(args)):
183
- arg_batches[i].append(args[i])
184
- for k in kwargs.keys():
185
- kwarg_batches[k].append(kwargs[k])
186
- num_valid_batch_rows = len(valid_batch_idxs)
187
-
188
- if ext_batch_size is None:
189
- # we need to choose a batch size based on the args
190
- assert isinstance(fn_call.fn, CallableFunction)
191
- sample_args = [arg_batches[i][0] for i in range(len(arg_batches))]
192
- ext_batch_size = fn_call.fn.get_batch_size(*sample_args)
193
-
194
- num_remaining_batch_rows = num_valid_batch_rows
195
- while num_remaining_batch_rows > 0:
196
- # we make ext. fn calls in batches of ext_batch_size
197
- if ext_batch_size is None:
198
- pass
199
- num_ext_batch_rows = min(ext_batch_size, num_remaining_batch_rows)
200
- ext_batch_offset = num_valid_batch_rows - num_remaining_batch_rows # offset into args, not rows
201
- call_args = [
202
- arg_batches[i][ext_batch_offset:ext_batch_offset + num_ext_batch_rows]
203
- for i in range(len(arg_batches))
204
- ]
205
- call_kwargs = {
206
- k: kwarg_batches[k][ext_batch_offset:ext_batch_offset + num_ext_batch_rows]
207
- for k in kwarg_batches.keys()
208
- }
209
- start_ts = time.perf_counter()
210
- assert isinstance(fn_call.fn, CallableFunction)
211
- result_batch = fn_call.fn.exec_batch(call_args, call_kwargs)
212
- self.ctx.profile.eval_time[fn_call.slot_idx] += time.perf_counter() - start_ts
213
- self.ctx.profile.eval_count[fn_call.slot_idx] += num_ext_batch_rows
214
-
215
- # move the result into the row batch
216
- for result_idx in range(len(result_batch)):
217
- row_idx = valid_batch_idxs[ext_batch_offset + result_idx]
218
- row = rows[row_idx]
219
- row[fn_call.slot_idx] = result_batch[result_idx]
220
-
221
- num_remaining_batch_rows -= num_ext_batch_rows
222
-
223
- # switch to the ext fn batch size
224
- cohort.batch_size = ext_batch_size
225
-
226
- # make sure images for stored cols have been saved to files before moving on to the next batch
227
- rows.flush_imgs(
228
- slice(batch_start_idx, batch_start_idx + num_batch_rows), self.stored_img_cols, self.flushed_img_slots)
229
- if self.pbar is not None:
230
- self.pbar.update(num_batch_rows * len(cohort.target_slot_idxs))
231
- batch_start_idx += num_batch_rows
232
-