pixeltable 0.4.18__py3-none-any.whl → 0.4.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pixeltable might be problematic. Click here for more details.
- pixeltable/__init__.py +1 -1
- pixeltable/_version.py +1 -0
- pixeltable/catalog/catalog.py +119 -100
- pixeltable/catalog/column.py +104 -115
- pixeltable/catalog/globals.py +1 -2
- pixeltable/catalog/insertable_table.py +44 -49
- pixeltable/catalog/path.py +3 -4
- pixeltable/catalog/schema_object.py +4 -4
- pixeltable/catalog/table.py +118 -122
- pixeltable/catalog/table_metadata.py +6 -6
- pixeltable/catalog/table_version.py +322 -257
- pixeltable/catalog/table_version_handle.py +4 -4
- pixeltable/catalog/table_version_path.py +9 -10
- pixeltable/catalog/tbl_ops.py +9 -3
- pixeltable/catalog/view.py +34 -28
- pixeltable/config.py +14 -10
- pixeltable/dataframe.py +68 -77
- pixeltable/env.py +74 -64
- pixeltable/exec/aggregation_node.py +6 -6
- pixeltable/exec/cache_prefetch_node.py +10 -10
- pixeltable/exec/data_row_batch.py +3 -3
- pixeltable/exec/exec_context.py +4 -5
- pixeltable/exec/exec_node.py +5 -5
- pixeltable/exec/expr_eval/evaluators.py +6 -6
- pixeltable/exec/expr_eval/expr_eval_node.py +8 -7
- pixeltable/exec/expr_eval/globals.py +6 -6
- pixeltable/exec/expr_eval/row_buffer.py +1 -2
- pixeltable/exec/expr_eval/schedulers.py +11 -11
- pixeltable/exec/in_memory_data_node.py +2 -2
- pixeltable/exec/object_store_save_node.py +14 -17
- pixeltable/exec/sql_node.py +25 -25
- pixeltable/exprs/arithmetic_expr.py +4 -4
- pixeltable/exprs/array_slice.py +2 -2
- pixeltable/exprs/column_property_ref.py +3 -3
- pixeltable/exprs/column_ref.py +61 -74
- pixeltable/exprs/comparison.py +5 -5
- pixeltable/exprs/compound_predicate.py +3 -3
- pixeltable/exprs/data_row.py +12 -12
- pixeltable/exprs/expr.py +41 -31
- pixeltable/exprs/expr_dict.py +3 -3
- pixeltable/exprs/expr_set.py +3 -3
- pixeltable/exprs/function_call.py +14 -14
- pixeltable/exprs/in_predicate.py +4 -4
- pixeltable/exprs/inline_expr.py +8 -8
- pixeltable/exprs/is_null.py +1 -3
- pixeltable/exprs/json_mapper.py +8 -8
- pixeltable/exprs/json_path.py +6 -6
- pixeltable/exprs/literal.py +5 -5
- pixeltable/exprs/method_ref.py +2 -2
- pixeltable/exprs/object_ref.py +2 -2
- pixeltable/exprs/row_builder.py +14 -14
- pixeltable/exprs/rowid_ref.py +8 -8
- pixeltable/exprs/similarity_expr.py +50 -25
- pixeltable/exprs/sql_element_cache.py +4 -4
- pixeltable/exprs/string_op.py +2 -2
- pixeltable/exprs/type_cast.py +3 -5
- pixeltable/func/aggregate_function.py +8 -8
- pixeltable/func/callable_function.py +9 -9
- pixeltable/func/expr_template_function.py +3 -3
- pixeltable/func/function.py +15 -17
- pixeltable/func/function_registry.py +6 -7
- pixeltable/func/globals.py +2 -3
- pixeltable/func/mcp.py +2 -2
- pixeltable/func/query_template_function.py +16 -16
- pixeltable/func/signature.py +14 -14
- pixeltable/func/tools.py +11 -11
- pixeltable/func/udf.py +16 -18
- pixeltable/functions/__init__.py +1 -0
- pixeltable/functions/anthropic.py +7 -7
- pixeltable/functions/audio.py +76 -0
- pixeltable/functions/bedrock.py +6 -6
- pixeltable/functions/deepseek.py +4 -4
- pixeltable/functions/fireworks.py +2 -2
- pixeltable/functions/gemini.py +6 -6
- pixeltable/functions/globals.py +12 -12
- pixeltable/functions/groq.py +4 -4
- pixeltable/functions/huggingface.py +18 -20
- pixeltable/functions/image.py +7 -10
- pixeltable/functions/llama_cpp.py +7 -7
- pixeltable/functions/math.py +2 -3
- pixeltable/functions/mistralai.py +3 -3
- pixeltable/functions/ollama.py +9 -9
- pixeltable/functions/openai.py +21 -21
- pixeltable/functions/openrouter.py +7 -7
- pixeltable/functions/string.py +21 -28
- pixeltable/functions/timestamp.py +7 -8
- pixeltable/functions/together.py +4 -6
- pixeltable/functions/twelvelabs.py +92 -0
- pixeltable/functions/video.py +2 -24
- pixeltable/functions/vision.py +6 -6
- pixeltable/functions/whisper.py +7 -7
- pixeltable/functions/whisperx.py +16 -16
- pixeltable/globals.py +52 -36
- pixeltable/index/base.py +12 -8
- pixeltable/index/btree.py +19 -22
- pixeltable/index/embedding_index.py +30 -39
- pixeltable/io/datarows.py +3 -3
- pixeltable/io/external_store.py +13 -16
- pixeltable/io/fiftyone.py +5 -5
- pixeltable/io/globals.py +5 -5
- pixeltable/io/hf_datasets.py +4 -4
- pixeltable/io/label_studio.py +12 -12
- pixeltable/io/pandas.py +6 -6
- pixeltable/io/parquet.py +2 -2
- pixeltable/io/table_data_conduit.py +12 -12
- pixeltable/io/utils.py +2 -2
- pixeltable/iterators/audio.py +2 -2
- pixeltable/iterators/video.py +8 -13
- pixeltable/metadata/converters/convert_18.py +2 -2
- pixeltable/metadata/converters/convert_19.py +2 -2
- pixeltable/metadata/converters/convert_20.py +2 -2
- pixeltable/metadata/converters/convert_21.py +2 -2
- pixeltable/metadata/converters/convert_22.py +2 -2
- pixeltable/metadata/converters/convert_24.py +2 -2
- pixeltable/metadata/converters/convert_25.py +2 -2
- pixeltable/metadata/converters/convert_26.py +2 -2
- pixeltable/metadata/converters/convert_29.py +4 -4
- pixeltable/metadata/converters/convert_34.py +2 -2
- pixeltable/metadata/converters/convert_36.py +2 -2
- pixeltable/metadata/converters/convert_38.py +2 -2
- pixeltable/metadata/converters/convert_39.py +1 -2
- pixeltable/metadata/converters/util.py +11 -13
- pixeltable/metadata/schema.py +22 -21
- pixeltable/metadata/utils.py +2 -6
- pixeltable/mypy/mypy_plugin.py +5 -5
- pixeltable/plan.py +30 -28
- pixeltable/share/packager.py +7 -7
- pixeltable/share/publish.py +3 -3
- pixeltable/store.py +125 -61
- pixeltable/type_system.py +43 -46
- pixeltable/utils/__init__.py +1 -2
- pixeltable/utils/arrow.py +4 -4
- pixeltable/utils/av.py +8 -0
- pixeltable/utils/azure_store.py +305 -0
- pixeltable/utils/code.py +1 -2
- pixeltable/utils/dbms.py +15 -19
- pixeltable/utils/description_helper.py +2 -3
- pixeltable/utils/documents.py +5 -6
- pixeltable/utils/exception_handler.py +2 -2
- pixeltable/utils/filecache.py +5 -5
- pixeltable/utils/formatter.py +4 -6
- pixeltable/utils/gcs_store.py +9 -9
- pixeltable/utils/local_store.py +17 -17
- pixeltable/utils/object_stores.py +59 -43
- pixeltable/utils/s3_store.py +35 -30
- {pixeltable-0.4.18.dist-info → pixeltable-0.4.19.dist-info}/METADATA +1 -1
- pixeltable-0.4.19.dist-info/RECORD +213 -0
- pixeltable/__version__.py +0 -3
- pixeltable-0.4.18.dist-info/RECORD +0 -211
- {pixeltable-0.4.18.dist-info → pixeltable-0.4.19.dist-info}/WHEEL +0 -0
- {pixeltable-0.4.18.dist-info → pixeltable-0.4.19.dist-info}/entry_points.txt +0 -0
- {pixeltable-0.4.18.dist-info → pixeltable-0.4.19.dist-info}/licenses/LICENSE +0 -0
pixeltable/exprs/expr_set.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import Generic, Iterable, Iterator,
|
|
3
|
+
from typing import Generic, Iterable, Iterator, TypeVar
|
|
4
4
|
|
|
5
5
|
from .expr import Expr
|
|
6
6
|
|
|
@@ -17,7 +17,7 @@ class ExprSet(Generic[T]):
|
|
|
17
17
|
expr_offsets: dict[int, int] # key: Expr.id, value: offset into self.exprs.keys()
|
|
18
18
|
exprs_by_idx: dict[int, T] # key: slot_idx
|
|
19
19
|
|
|
20
|
-
def __init__(self, elements:
|
|
20
|
+
def __init__(self, elements: Iterable[T] | None = None):
|
|
21
21
|
self.exprs = {}
|
|
22
22
|
self.expr_offsets = {}
|
|
23
23
|
self.exprs_by_idx = {}
|
|
@@ -51,7 +51,7 @@ class ExprSet(Generic[T]):
|
|
|
51
51
|
def __iter__(self) -> Iterator[T]:
|
|
52
52
|
return iter(self.exprs.values())
|
|
53
53
|
|
|
54
|
-
def __getitem__(self, index: object) ->
|
|
54
|
+
def __getitem__(self, index: object) -> T | None:
|
|
55
55
|
"""Indexed lookup by slot_idx or Expr.id."""
|
|
56
56
|
assert isinstance(index, (int, Expr))
|
|
57
57
|
if isinstance(index, int):
|
|
@@ -4,7 +4,7 @@ import inspect
|
|
|
4
4
|
import logging
|
|
5
5
|
import sys
|
|
6
6
|
from textwrap import dedent
|
|
7
|
-
from typing import Any,
|
|
7
|
+
from typing import Any, Sequence
|
|
8
8
|
|
|
9
9
|
import sqlalchemy as sql
|
|
10
10
|
|
|
@@ -24,7 +24,7 @@ class FunctionCall(Expr):
|
|
|
24
24
|
fn: func.Function
|
|
25
25
|
is_method_call: bool
|
|
26
26
|
agg_init_args: dict[str, Any]
|
|
27
|
-
resource_pool:
|
|
27
|
+
resource_pool: str | None
|
|
28
28
|
|
|
29
29
|
# These collections hold the component indices corresponding to the args and kwargs
|
|
30
30
|
# that were passed to the FunctionCall. They're 1:1 with the original call pattern.
|
|
@@ -43,10 +43,10 @@ class FunctionCall(Expr):
|
|
|
43
43
|
group_by_stop_idx: int
|
|
44
44
|
fn_expr_idx: int
|
|
45
45
|
order_by_start_idx: int
|
|
46
|
-
aggregator:
|
|
47
|
-
current_partition_vals:
|
|
46
|
+
aggregator: Any | None
|
|
47
|
+
current_partition_vals: list[Any] | None
|
|
48
48
|
|
|
49
|
-
_validation_error:
|
|
49
|
+
_validation_error: str | None
|
|
50
50
|
|
|
51
51
|
def __init__(
|
|
52
52
|
self,
|
|
@@ -54,10 +54,10 @@ class FunctionCall(Expr):
|
|
|
54
54
|
args: list[Expr],
|
|
55
55
|
kwargs: dict[str, Expr],
|
|
56
56
|
return_type: ts.ColumnType,
|
|
57
|
-
order_by_clause:
|
|
58
|
-
group_by_clause:
|
|
57
|
+
order_by_clause: list[Any] | None = None,
|
|
58
|
+
group_by_clause: list[Any] | None = None,
|
|
59
59
|
is_method_call: bool = False,
|
|
60
|
-
validation_error:
|
|
60
|
+
validation_error: str | None = None,
|
|
61
61
|
):
|
|
62
62
|
assert not fn.is_polymorphic
|
|
63
63
|
assert all(isinstance(arg, Expr) for arg in args)
|
|
@@ -149,7 +149,7 @@ class FunctionCall(Expr):
|
|
|
149
149
|
target = tbl._tbl_version_path.tbl_version
|
|
150
150
|
return [RowidRef(target, i) for i in range(target.get().num_rowid_columns())]
|
|
151
151
|
|
|
152
|
-
def default_column_name(self) ->
|
|
152
|
+
def default_column_name(self) -> str | None:
|
|
153
153
|
return self.fn.name
|
|
154
154
|
|
|
155
155
|
def _equals(self, other: FunctionCall) -> bool:
|
|
@@ -178,7 +178,7 @@ class FunctionCall(Expr):
|
|
|
178
178
|
return self.display_str()
|
|
179
179
|
|
|
180
180
|
@property
|
|
181
|
-
def validation_error(self) ->
|
|
181
|
+
def validation_error(self) -> str | None:
|
|
182
182
|
return self._validation_error or super().validation_error
|
|
183
183
|
|
|
184
184
|
def display_str(self, inline: bool = True) -> str:
|
|
@@ -245,7 +245,7 @@ class FunctionCall(Expr):
|
|
|
245
245
|
assert self.is_agg_fn_call
|
|
246
246
|
return self.order_by
|
|
247
247
|
|
|
248
|
-
def sql_expr(self, sql_elements: SqlElementCache) ->
|
|
248
|
+
def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
|
|
249
249
|
assert self.is_valid
|
|
250
250
|
|
|
251
251
|
# we currently can't translate aggregate functions with grouping and/or ordering to SQL
|
|
@@ -321,7 +321,7 @@ class FunctionCall(Expr):
|
|
|
321
321
|
args, kwargs = self.make_args(data_row)
|
|
322
322
|
self.aggregator.update(*args, **kwargs)
|
|
323
323
|
|
|
324
|
-
def make_args(self, data_row: DataRow) ->
|
|
324
|
+
def make_args(self, data_row: DataRow) -> tuple[list[Any], dict[str, Any]] | None:
|
|
325
325
|
"""Return args and kwargs, constructed for data_row; returns None if any non-nullable arg is None."""
|
|
326
326
|
args: list[Any] = []
|
|
327
327
|
parameters_by_pos = self.fn.signature.parameters_by_pos
|
|
@@ -448,7 +448,7 @@ class FunctionCall(Expr):
|
|
|
448
448
|
group_by_exprs = components[group_by_start_idx:group_by_stop_idx]
|
|
449
449
|
order_by_exprs = components[order_by_start_idx:]
|
|
450
450
|
|
|
451
|
-
validation_error:
|
|
451
|
+
validation_error: str | None = None
|
|
452
452
|
|
|
453
453
|
if isinstance(fn, func.InvalidFunction):
|
|
454
454
|
validation_error = (
|
|
@@ -489,7 +489,7 @@ class FunctionCall(Expr):
|
|
|
489
489
|
).strip()
|
|
490
490
|
else:
|
|
491
491
|
# Evaluate the call_return_type as defined in the current codebase.
|
|
492
|
-
call_return_type:
|
|
492
|
+
call_return_type: ts.ColumnType | None = None
|
|
493
493
|
|
|
494
494
|
if isinstance(resolved_fn, func.ExprTemplateFunction) and not resolved_fn.template.expr.is_valid:
|
|
495
495
|
# The FunctionCall is based on an ExprTemplateFunction, but the template expression is not valid
|
pixeltable/exprs/in_predicate.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import Any, Iterable
|
|
3
|
+
from typing import Any, Iterable
|
|
4
4
|
|
|
5
5
|
import sqlalchemy as sql
|
|
6
6
|
|
|
@@ -16,13 +16,13 @@ from .sql_element_cache import SqlElementCache
|
|
|
16
16
|
class InPredicate(Expr):
|
|
17
17
|
"""Predicate corresponding to the SQL IN operator."""
|
|
18
18
|
|
|
19
|
-
def __init__(self, lhs: Expr, value_set_literal:
|
|
19
|
+
def __init__(self, lhs: Expr, value_set_literal: Iterable | None = None, value_set_expr: Expr | None = None):
|
|
20
20
|
assert (value_set_literal is None) != (value_set_expr is None)
|
|
21
21
|
if not lhs.col_type.is_scalar_type():
|
|
22
22
|
raise excs.Error(f'isin(): only supported for scalar types, not {lhs.col_type}')
|
|
23
23
|
super().__init__(ts.BoolType())
|
|
24
24
|
|
|
25
|
-
self.value_list:
|
|
25
|
+
self.value_list: list | None = None # only contains values of the correct type
|
|
26
26
|
if value_set_expr is not None:
|
|
27
27
|
if not value_set_expr.col_type.is_json_type():
|
|
28
28
|
raise excs.Error(
|
|
@@ -73,7 +73,7 @@ class InPredicate(Expr):
|
|
|
73
73
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
74
74
|
return [*super()._id_attrs(), ('value_list', self.value_list)]
|
|
75
75
|
|
|
76
|
-
def sql_expr(self, sql_elements: SqlElementCache) ->
|
|
76
|
+
def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
|
|
77
77
|
lhs_sql_exprs = sql_elements.get(self.components[0])
|
|
78
78
|
if lhs_sql_exprs is None or self.value_list is None:
|
|
79
79
|
return None
|
pixeltable/exprs/inline_expr.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import Any, Iterable
|
|
3
|
+
from typing import Any, Iterable
|
|
4
4
|
|
|
5
5
|
import numpy as np
|
|
6
6
|
import sqlalchemy as sql
|
|
@@ -30,7 +30,7 @@ class InlineArray(Expr):
|
|
|
30
30
|
else:
|
|
31
31
|
exprs.append(Literal(el))
|
|
32
32
|
|
|
33
|
-
inferred_element_type:
|
|
33
|
+
inferred_element_type: ts.ColumnType | None = ts.InvalidType()
|
|
34
34
|
for i, expr in enumerate(exprs):
|
|
35
35
|
supertype = inferred_element_type.supertype(expr.col_type)
|
|
36
36
|
if supertype is None:
|
|
@@ -61,7 +61,7 @@ class InlineArray(Expr):
|
|
|
61
61
|
def _equals(self, _: InlineArray) -> bool:
|
|
62
62
|
return True # Always true if components match
|
|
63
63
|
|
|
64
|
-
def sql_expr(self, _: SqlElementCache) ->
|
|
64
|
+
def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
|
|
65
65
|
return None
|
|
66
66
|
|
|
67
67
|
def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
|
|
@@ -81,7 +81,7 @@ class InlineArray(Expr):
|
|
|
81
81
|
# loaded and their types are known.
|
|
82
82
|
return InlineList(components) # type: ignore[return-value]
|
|
83
83
|
|
|
84
|
-
def as_literal(self) ->
|
|
84
|
+
def as_literal(self) -> Literal | None:
|
|
85
85
|
assert isinstance(self.col_type, ts.ArrayType)
|
|
86
86
|
if not all(isinstance(comp, Literal) for comp in self.components):
|
|
87
87
|
return None
|
|
@@ -109,7 +109,7 @@ class InlineList(Expr):
|
|
|
109
109
|
def _equals(self, _: InlineList) -> bool:
|
|
110
110
|
return True # Always true if components match
|
|
111
111
|
|
|
112
|
-
def sql_expr(self, _: SqlElementCache) ->
|
|
112
|
+
def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
|
|
113
113
|
return None
|
|
114
114
|
|
|
115
115
|
def eval(self, data_row: DataRow, _: RowBuilder) -> None:
|
|
@@ -122,7 +122,7 @@ class InlineList(Expr):
|
|
|
122
122
|
def _from_dict(cls, _: dict, components: list[Expr]) -> InlineList:
|
|
123
123
|
return cls(components)
|
|
124
124
|
|
|
125
|
-
def as_literal(self) ->
|
|
125
|
+
def as_literal(self) -> Literal | None:
|
|
126
126
|
if not all(isinstance(comp, Literal) for comp in self.components):
|
|
127
127
|
return None
|
|
128
128
|
return Literal([c.as_literal().val for c in self.components], self.col_type)
|
|
@@ -159,7 +159,7 @@ class InlineDict(Expr):
|
|
|
159
159
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
160
160
|
return [*super()._id_attrs(), ('keys', self.keys)]
|
|
161
161
|
|
|
162
|
-
def sql_expr(self, _: SqlElementCache) ->
|
|
162
|
+
def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
|
|
163
163
|
return None
|
|
164
164
|
|
|
165
165
|
def eval(self, data_row: DataRow, _: RowBuilder) -> None:
|
|
@@ -191,7 +191,7 @@ class InlineDict(Expr):
|
|
|
191
191
|
arg = dict(zip(d['keys'], components))
|
|
192
192
|
return InlineDict(arg)
|
|
193
193
|
|
|
194
|
-
def as_literal(self) ->
|
|
194
|
+
def as_literal(self) -> Literal | None:
|
|
195
195
|
if not all(isinstance(comp, Literal) for comp in self.components):
|
|
196
196
|
return None
|
|
197
197
|
return Literal(dict(zip(self.keys, (c.as_literal().val for c in self.components))), self.col_type)
|
pixeltable/exprs/is_null.py
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import Optional
|
|
4
|
-
|
|
5
3
|
import sqlalchemy as sql
|
|
6
4
|
|
|
7
5
|
import pixeltable.type_system as ts
|
|
@@ -24,7 +22,7 @@ class IsNull(Expr):
|
|
|
24
22
|
def _equals(self, other: IsNull) -> bool:
|
|
25
23
|
return True
|
|
26
24
|
|
|
27
|
-
def sql_expr(self, sql_elements: SqlElementCache) ->
|
|
25
|
+
def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
|
|
28
26
|
e = sql_elements.get(self.components[0])
|
|
29
27
|
if e is None:
|
|
30
28
|
return None
|
pixeltable/exprs/json_mapper.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
4
|
|
|
5
5
|
import sqlalchemy as sql
|
|
6
6
|
|
|
@@ -29,10 +29,10 @@ class JsonMapper(Expr):
|
|
|
29
29
|
"""
|
|
30
30
|
|
|
31
31
|
target_expr_scope: ExprScope
|
|
32
|
-
parent_mapper:
|
|
33
|
-
target_expr_eval_ctx:
|
|
32
|
+
parent_mapper: JsonMapper | None
|
|
33
|
+
target_expr_eval_ctx: RowBuilder.EvalCtx | None
|
|
34
34
|
|
|
35
|
-
def __init__(self, src_expr:
|
|
35
|
+
def __init__(self, src_expr: Expr | None, target_expr: Expr | None):
|
|
36
36
|
# TODO: type spec should be list[target_expr.col_type]
|
|
37
37
|
super().__init__(ts.JsonType())
|
|
38
38
|
|
|
@@ -54,7 +54,7 @@ class JsonMapper(Expr):
|
|
|
54
54
|
def _equals(self, _: JsonMapper) -> bool:
|
|
55
55
|
return True
|
|
56
56
|
|
|
57
|
-
def sql_expr(self, _: SqlElementCache) ->
|
|
57
|
+
def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
|
|
58
58
|
return None
|
|
59
59
|
|
|
60
60
|
def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
|
|
@@ -92,8 +92,8 @@ class JsonMapperDispatch(Expr):
|
|
|
92
92
|
"""
|
|
93
93
|
|
|
94
94
|
target_expr_scope: ExprScope
|
|
95
|
-
parent_mapper:
|
|
96
|
-
target_expr_eval_ctx:
|
|
95
|
+
parent_mapper: JsonMapperDispatch | None
|
|
96
|
+
target_expr_eval_ctx: RowBuilder.EvalCtx | None
|
|
97
97
|
|
|
98
98
|
def __init__(self, src_expr: Expr, target_expr: Expr):
|
|
99
99
|
super().__init__(ts.InvalidType())
|
|
@@ -116,7 +116,7 @@ class JsonMapperDispatch(Expr):
|
|
|
116
116
|
scope_anchor = ObjectRef(self.target_expr_scope, self)
|
|
117
117
|
self.components.append(scope_anchor)
|
|
118
118
|
|
|
119
|
-
def _bind_rel_paths(self, mapper:
|
|
119
|
+
def _bind_rel_paths(self, mapper: JsonMapperDispatch | None = None) -> None:
|
|
120
120
|
self.src_expr._bind_rel_paths(mapper)
|
|
121
121
|
self.target_expr._bind_rel_paths(self)
|
|
122
122
|
self.parent_mapper = mapper
|
pixeltable/exprs/json_path.py
CHANGED
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import io
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any
|
|
6
6
|
|
|
7
7
|
import jmespath
|
|
8
8
|
import sqlalchemy as sql
|
|
@@ -32,7 +32,7 @@ class JsonPath(Expr):
|
|
|
32
32
|
file_handles: dict[Path, io.BufferedReader] # key: file path
|
|
33
33
|
|
|
34
34
|
def __init__(
|
|
35
|
-
self, anchor:
|
|
35
|
+
self, anchor: Expr | None, path_elements: list[str | int | slice] | None = None, scope_idx: int = 0
|
|
36
36
|
) -> None:
|
|
37
37
|
if path_elements is None:
|
|
38
38
|
path_elements = []
|
|
@@ -81,7 +81,7 @@ class JsonPath(Expr):
|
|
|
81
81
|
return cls(anchor, path_elements, d['scope_idx'])
|
|
82
82
|
|
|
83
83
|
@property
|
|
84
|
-
def anchor(self) ->
|
|
84
|
+
def anchor(self) -> Expr | None:
|
|
85
85
|
return None if len(self.components) == 0 else self.components[0]
|
|
86
86
|
|
|
87
87
|
def set_anchor(self, anchor: Expr) -> None:
|
|
@@ -94,7 +94,7 @@ class JsonPath(Expr):
|
|
|
94
94
|
def _has_relative_path(self) -> bool:
|
|
95
95
|
return self.is_relative_path() or super()._has_relative_path()
|
|
96
96
|
|
|
97
|
-
def _bind_rel_paths(self, mapper:
|
|
97
|
+
def _bind_rel_paths(self, mapper: 'JsonMapperDispatch' | None = None) -> None:
|
|
98
98
|
if self.is_relative_path():
|
|
99
99
|
# TODO: take scope_idx into account
|
|
100
100
|
self.set_anchor(mapper.scope_anchor)
|
|
@@ -120,7 +120,7 @@ class JsonPath(Expr):
|
|
|
120
120
|
return JsonPath(self.anchor, [*self.path_elements, index])
|
|
121
121
|
raise excs.Error(f'Invalid json list index: {index}')
|
|
122
122
|
|
|
123
|
-
def default_column_name(self) ->
|
|
123
|
+
def default_column_name(self) -> str | None:
|
|
124
124
|
anchor_name = self.anchor.default_column_name() if self.anchor is not None else ''
|
|
125
125
|
ret_name = f'{anchor_name}.{self._json_path()}'
|
|
126
126
|
|
|
@@ -148,7 +148,7 @@ class JsonPath(Expr):
|
|
|
148
148
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
149
149
|
return [*super()._id_attrs(), ('path_elements', self.path_elements)]
|
|
150
150
|
|
|
151
|
-
def sql_expr(self, _: SqlElementCache) ->
|
|
151
|
+
def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
|
|
152
152
|
"""
|
|
153
153
|
Postgres appears to have a bug: jsonb_path_query('{a: [{b: 0}, {b: 1}]}', '$.a.b') returns
|
|
154
154
|
*two* rows (each containing col val 0), not a single row with [0, 0].
|
pixeltable/exprs/literal.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import datetime
|
|
4
|
-
from typing import Any
|
|
4
|
+
from typing import Any
|
|
5
5
|
|
|
6
6
|
import numpy as np
|
|
7
7
|
import sqlalchemy as sql
|
|
@@ -18,7 +18,7 @@ from .sql_element_cache import SqlElementCache
|
|
|
18
18
|
class Literal(Expr):
|
|
19
19
|
val: Any
|
|
20
20
|
|
|
21
|
-
def __init__(self, val: Any, col_type:
|
|
21
|
+
def __init__(self, val: Any, col_type: ts.ColumnType | None = None):
|
|
22
22
|
if col_type is not None:
|
|
23
23
|
val = col_type.create_literal(val)
|
|
24
24
|
else:
|
|
@@ -42,7 +42,7 @@ class Literal(Expr):
|
|
|
42
42
|
self.val = val
|
|
43
43
|
self.id = self._create_id()
|
|
44
44
|
|
|
45
|
-
def default_column_name(self) ->
|
|
45
|
+
def default_column_name(self) -> str | None:
|
|
46
46
|
return 'Literal'
|
|
47
47
|
|
|
48
48
|
def __str__(self) -> str:
|
|
@@ -69,7 +69,7 @@ class Literal(Expr):
|
|
|
69
69
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
70
70
|
return [*super()._id_attrs(), ('val', self.val)]
|
|
71
71
|
|
|
72
|
-
def sql_expr(self, _: SqlElementCache) ->
|
|
72
|
+
def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
|
|
73
73
|
# Return a sql object so that constants can participate in SQL expressions
|
|
74
74
|
return sql.sql.expression.literal(self.val, type_=self.col_type.to_sa_type())
|
|
75
75
|
|
|
@@ -97,7 +97,7 @@ class Literal(Expr):
|
|
|
97
97
|
else:
|
|
98
98
|
return {'val': self.val, **super()._as_dict()}
|
|
99
99
|
|
|
100
|
-
def as_literal(self) ->
|
|
100
|
+
def as_literal(self) -> Literal | None:
|
|
101
101
|
return self
|
|
102
102
|
|
|
103
103
|
@classmethod
|
pixeltable/exprs/method_ref.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any
|
|
1
|
+
from typing import Any
|
|
2
2
|
|
|
3
3
|
import sqlalchemy as sql
|
|
4
4
|
|
|
@@ -55,7 +55,7 @@ class MethodRef(Expr):
|
|
|
55
55
|
def _id_attrs(self) -> list[tuple[str, Any]]:
|
|
56
56
|
return [*super()._id_attrs(), ('method_name', self.method_name)]
|
|
57
57
|
|
|
58
|
-
def sql_expr(self, _: SqlElementCache) ->
|
|
58
|
+
def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
|
|
59
59
|
return None
|
|
60
60
|
|
|
61
61
|
def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
|
pixeltable/exprs/object_ref.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from typing import Any
|
|
3
|
+
from typing import Any
|
|
4
4
|
|
|
5
5
|
import sqlalchemy as sql
|
|
6
6
|
|
|
@@ -43,7 +43,7 @@ class ObjectRef(Expr):
|
|
|
43
43
|
def _equals(self, other: ObjectRef) -> bool:
|
|
44
44
|
return self.id == other.id
|
|
45
45
|
|
|
46
|
-
def sql_expr(self, _: SqlElementCache) ->
|
|
46
|
+
def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
|
|
47
47
|
return None
|
|
48
48
|
|
|
49
49
|
def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
|
pixeltable/exprs/row_builder.py
CHANGED
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import dataclasses
|
|
4
4
|
import sys
|
|
5
5
|
import time
|
|
6
|
-
from typing import Any, Iterable, NamedTuple,
|
|
6
|
+
from typing import Any, Iterable, NamedTuple, Sequence
|
|
7
7
|
from uuid import UUID
|
|
8
8
|
|
|
9
9
|
import numpy as np
|
|
@@ -69,7 +69,7 @@ class RowBuilder:
|
|
|
69
69
|
|
|
70
70
|
input_exprs: ExprSet
|
|
71
71
|
|
|
72
|
-
tbl:
|
|
72
|
+
tbl: catalog.TableVersion | None # reference table of the RowBuilder; used to identify pk columns for writes
|
|
73
73
|
table_columns: dict[catalog.Column, int | None] # value: slot idx, if the result of an expr
|
|
74
74
|
default_eval_ctx: EvalCtx
|
|
75
75
|
unstored_iter_args: dict[UUID, Expr]
|
|
@@ -110,7 +110,7 @@ class RowBuilder:
|
|
|
110
110
|
output_exprs: Sequence[Expr],
|
|
111
111
|
columns: Sequence[catalog.Column],
|
|
112
112
|
input_exprs: Iterable[Expr],
|
|
113
|
-
tbl:
|
|
113
|
+
tbl: catalog.TableVersion | None = None,
|
|
114
114
|
):
|
|
115
115
|
self.unique_exprs: ExprSet[Expr] = ExprSet() # dependencies precede their dependents
|
|
116
116
|
self.next_slot_idx = 0
|
|
@@ -183,18 +183,18 @@ class RowBuilder:
|
|
|
183
183
|
col_refs = [e for e in self.unique_exprs if isinstance(e, ColumnRef)]
|
|
184
184
|
|
|
185
185
|
def refs_unstored_iter_col(col_ref: ColumnRef) -> bool:
|
|
186
|
-
tbl = col_ref.col.
|
|
186
|
+
tbl = col_ref.col.get_tbl()
|
|
187
187
|
return tbl.is_component_view and tbl.is_iterator_column(col_ref.col) and not col_ref.col.is_stored
|
|
188
188
|
|
|
189
189
|
unstored_iter_col_refs = [col_ref for col_ref in col_refs if refs_unstored_iter_col(col_ref)]
|
|
190
|
-
component_views = [col_ref.col.
|
|
190
|
+
component_views = [col_ref.col.get_tbl() for col_ref in unstored_iter_col_refs]
|
|
191
191
|
unstored_iter_args = {view.id: view.iterator_args.copy() for view in component_views}
|
|
192
192
|
self.unstored_iter_args = {
|
|
193
193
|
id: self._record_unique_expr(arg, recursive=True) for id, arg in unstored_iter_args.items()
|
|
194
194
|
}
|
|
195
195
|
|
|
196
196
|
for col_ref in unstored_iter_col_refs:
|
|
197
|
-
iter_arg_ctx = self.create_eval_ctx([unstored_iter_args[col_ref.col.
|
|
197
|
+
iter_arg_ctx = self.create_eval_ctx([unstored_iter_args[col_ref.col.get_tbl().id]])
|
|
198
198
|
col_ref.set_iter_arg_ctx(iter_arg_ctx)
|
|
199
199
|
|
|
200
200
|
# we guarantee that we can compute the expr DAG in a single front-to-back pass
|
|
@@ -308,7 +308,7 @@ class RowBuilder:
|
|
|
308
308
|
self._record_output_expr_id(d, output_expr_id)
|
|
309
309
|
|
|
310
310
|
def _compute_dependencies(
|
|
311
|
-
self, target_slot_idxs: list[int], excluded_slot_idxs: list[int], target_scope:
|
|
311
|
+
self, target_slot_idxs: list[int], excluded_slot_idxs: list[int], target_scope: ExprScope | None = None
|
|
312
312
|
) -> list[int]:
|
|
313
313
|
"""Compute exprs needed to materialize the given target slots, excluding 'excluded_slot_idxs'
|
|
314
314
|
|
|
@@ -362,7 +362,7 @@ class RowBuilder:
|
|
|
362
362
|
self.__set_slot_idxs_aux(c)
|
|
363
363
|
|
|
364
364
|
def get_dependencies(
|
|
365
|
-
self, targets: Iterable[Expr], exclude:
|
|
365
|
+
self, targets: Iterable[Expr], exclude: Iterable[Expr] | None = None, limit_scope: bool = True
|
|
366
366
|
) -> list[Expr]:
|
|
367
367
|
"""
|
|
368
368
|
Return list of dependencies needed to evaluate the given target exprs (expressed as slot idxs).
|
|
@@ -380,7 +380,7 @@ class RowBuilder:
|
|
|
380
380
|
return []
|
|
381
381
|
# make sure we only refer to recorded exprs
|
|
382
382
|
targets = [self.unique_exprs[e] for e in targets]
|
|
383
|
-
target_scope:
|
|
383
|
+
target_scope: ExprScope | None = None
|
|
384
384
|
if limit_scope:
|
|
385
385
|
# make sure all targets are from the same scope
|
|
386
386
|
target_scopes = {e.scope() for e in targets}
|
|
@@ -398,7 +398,7 @@ class RowBuilder:
|
|
|
398
398
|
return [self.unique_exprs[id] for id in result_ids]
|
|
399
399
|
|
|
400
400
|
def create_eval_ctx(
|
|
401
|
-
self, targets: Iterable[Expr], exclude:
|
|
401
|
+
self, targets: Iterable[Expr], exclude: Iterable[Expr] | None = None, limit_scope: bool = True
|
|
402
402
|
) -> EvalCtx:
|
|
403
403
|
"""Return EvalCtx for targets"""
|
|
404
404
|
targets = list(targets)
|
|
@@ -427,9 +427,9 @@ class RowBuilder:
|
|
|
427
427
|
self,
|
|
428
428
|
data_row: DataRow,
|
|
429
429
|
ctx: EvalCtx,
|
|
430
|
-
profile:
|
|
430
|
+
profile: ExecProfile | None = None,
|
|
431
431
|
ignore_errors: bool = False,
|
|
432
|
-
force_eval:
|
|
432
|
+
force_eval: ExprScope | None = None,
|
|
433
433
|
) -> None:
|
|
434
434
|
"""
|
|
435
435
|
Populates the slots in data_row given in ctx.
|
|
@@ -459,7 +459,7 @@ class RowBuilder:
|
|
|
459
459
|
) from exc
|
|
460
460
|
|
|
461
461
|
def create_store_table_row(
|
|
462
|
-
self, data_row: DataRow, cols_with_excs:
|
|
462
|
+
self, data_row: DataRow, cols_with_excs: set[int] | None, pk: tuple[int, ...]
|
|
463
463
|
) -> tuple[list[Any], int]:
|
|
464
464
|
"""Create a store table row from the slots that have an output column assigned
|
|
465
465
|
|
|
@@ -499,7 +499,7 @@ class RowBuilder:
|
|
|
499
499
|
# exceptions get stored in the errortype/-msg properties of the cellmd column
|
|
500
500
|
table_row.append(ColumnPropertyRef.create_cellmd_exc(exc))
|
|
501
501
|
else:
|
|
502
|
-
val = data_row.get_stored_val(slot_idx, col.
|
|
502
|
+
val = data_row.get_stored_val(slot_idx, col.sa_col_type)
|
|
503
503
|
table_row.append(val)
|
|
504
504
|
if col.stores_cellmd:
|
|
505
505
|
table_row.append(sql.sql.null()) # placeholder for cellmd column
|
pixeltable/exprs/rowid_ref.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
-
from typing import Any,
|
|
4
|
+
from typing import Any, cast
|
|
5
5
|
from uuid import UUID
|
|
6
6
|
|
|
7
7
|
import sqlalchemy as sql
|
|
@@ -25,18 +25,18 @@ class RowidRef(Expr):
|
|
|
25
25
|
(with and without a TableVersion).
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
-
tbl:
|
|
29
|
-
normalized_base:
|
|
28
|
+
tbl: catalog.TableVersionHandle | None
|
|
29
|
+
normalized_base: catalog.TableVersionHandle | None
|
|
30
30
|
tbl_id: UUID
|
|
31
31
|
normalized_base_id: UUID
|
|
32
32
|
rowid_component_idx: int
|
|
33
33
|
|
|
34
34
|
def __init__(
|
|
35
35
|
self,
|
|
36
|
-
tbl:
|
|
36
|
+
tbl: catalog.TableVersionHandle | None,
|
|
37
37
|
idx: int,
|
|
38
|
-
tbl_id:
|
|
39
|
-
normalized_base_id:
|
|
38
|
+
tbl_id: UUID | None = None,
|
|
39
|
+
normalized_base_id: UUID | None = None,
|
|
40
40
|
):
|
|
41
41
|
super().__init__(ts.IntType(nullable=False))
|
|
42
42
|
self.tbl = tbl
|
|
@@ -57,7 +57,7 @@ class RowidRef(Expr):
|
|
|
57
57
|
self.rowid_component_idx = idx
|
|
58
58
|
self.id = self._create_id()
|
|
59
59
|
|
|
60
|
-
def default_column_name(self) ->
|
|
60
|
+
def default_column_name(self) -> str | None:
|
|
61
61
|
return str(self)
|
|
62
62
|
|
|
63
63
|
def _equals(self, other: RowidRef) -> bool:
|
|
@@ -98,7 +98,7 @@ class RowidRef(Expr):
|
|
|
98
98
|
self.tbl = tbl.tbl_version
|
|
99
99
|
self.tbl_id = self.tbl.id
|
|
100
100
|
|
|
101
|
-
def sql_expr(self, _: SqlElementCache) ->
|
|
101
|
+
def sql_expr(self, _: SqlElementCache) -> sql.ColumnElement | None:
|
|
102
102
|
tbl = self.tbl.get() if self.tbl is not None else catalog.Catalog.get().get_tbl_version(self.tbl_id, None)
|
|
103
103
|
assert tbl.is_validated
|
|
104
104
|
rowid_cols = tbl.store_tbl.rowid_columns()
|