pixeltable 0.2.27__py3-none-any.whl → 0.2.29__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pixeltable might be problematic. Click here for more details.
- pixeltable/__version__.py +2 -2
- pixeltable/catalog/__init__.py +1 -1
- pixeltable/catalog/dir.py +6 -0
- pixeltable/catalog/globals.py +13 -0
- pixeltable/catalog/named_function.py +4 -0
- pixeltable/catalog/path_dict.py +37 -11
- pixeltable/catalog/schema_object.py +6 -0
- pixeltable/catalog/table.py +22 -5
- pixeltable/catalog/table_version.py +22 -8
- pixeltable/dataframe.py +201 -3
- pixeltable/env.py +9 -3
- pixeltable/exec/expr_eval_node.py +1 -1
- pixeltable/exec/sql_node.py +2 -2
- pixeltable/exprs/expr.py +1 -0
- pixeltable/exprs/function_call.py +134 -24
- pixeltable/exprs/inline_expr.py +22 -2
- pixeltable/exprs/row_builder.py +1 -1
- pixeltable/exprs/similarity_expr.py +9 -2
- pixeltable/func/aggregate_function.py +148 -68
- pixeltable/func/callable_function.py +49 -13
- pixeltable/func/expr_template_function.py +55 -24
- pixeltable/func/function.py +183 -22
- pixeltable/func/function_registry.py +2 -1
- pixeltable/func/query_template_function.py +11 -6
- pixeltable/func/signature.py +64 -7
- pixeltable/func/udf.py +57 -35
- pixeltable/functions/globals.py +54 -34
- pixeltable/functions/json.py +3 -8
- pixeltable/functions/ollama.py +4 -4
- pixeltable/functions/timestamp.py +1 -1
- pixeltable/functions/video.py +3 -9
- pixeltable/functions/vision.py +1 -1
- pixeltable/globals.py +218 -59
- pixeltable/index/embedding_index.py +44 -24
- pixeltable/metadata/__init__.py +1 -1
- pixeltable/metadata/converters/convert_16.py +2 -1
- pixeltable/metadata/converters/convert_17.py +2 -1
- pixeltable/metadata/converters/convert_23.py +35 -0
- pixeltable/metadata/converters/convert_24.py +47 -0
- pixeltable/metadata/converters/util.py +4 -2
- pixeltable/metadata/notes.py +2 -0
- pixeltable/metadata/schema.py +1 -0
- pixeltable/tool/create_test_db_dump.py +11 -0
- pixeltable/tool/doc_plugins/griffe.py +4 -3
- pixeltable/type_system.py +182 -47
- {pixeltable-0.2.27.dist-info → pixeltable-0.2.29.dist-info}/METADATA +3 -2
- {pixeltable-0.2.27.dist-info → pixeltable-0.2.29.dist-info}/RECORD +50 -48
- {pixeltable-0.2.27.dist-info → pixeltable-0.2.29.dist-info}/LICENSE +0 -0
- {pixeltable-0.2.27.dist-info → pixeltable-0.2.29.dist-info}/WHEEL +0 -0
- {pixeltable-0.2.27.dist-info → pixeltable-0.2.29.dist-info}/entry_points.txt +0 -0
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import inspect
|
|
4
|
-
from typing import Any, Callable, Optional
|
|
4
|
+
from typing import Any, Callable, Optional, Sequence
|
|
5
5
|
from uuid import UUID
|
|
6
6
|
|
|
7
7
|
import cloudpickle # type: ignore[import-untyped]
|
|
8
8
|
|
|
9
|
+
import pixeltable.exceptions as excs
|
|
10
|
+
|
|
9
11
|
from .function import Function
|
|
10
12
|
from .signature import Signature
|
|
11
13
|
|
|
@@ -20,26 +22,39 @@ class CallableFunction(Function):
|
|
|
20
22
|
|
|
21
23
|
def __init__(
|
|
22
24
|
self,
|
|
23
|
-
|
|
24
|
-
|
|
25
|
+
signatures: list[Signature],
|
|
26
|
+
py_fns: list[Callable],
|
|
25
27
|
self_path: Optional[str] = None,
|
|
26
28
|
self_name: Optional[str] = None,
|
|
27
29
|
batch_size: Optional[int] = None,
|
|
28
30
|
is_method: bool = False,
|
|
29
31
|
is_property: bool = False
|
|
30
32
|
):
|
|
31
|
-
assert
|
|
32
|
-
|
|
33
|
+
assert len(signatures) > 0
|
|
34
|
+
assert len(signatures) == len(py_fns)
|
|
35
|
+
if self_path is None and len(signatures) > 1:
|
|
36
|
+
raise excs.Error('Multiple signatures are only allowed for module UDFs (not locally defined UDFs)')
|
|
37
|
+
self.py_fns = py_fns
|
|
33
38
|
self.self_name = self_name
|
|
34
39
|
self.batch_size = batch_size
|
|
35
|
-
self.__doc__ =
|
|
36
|
-
super().__init__(
|
|
40
|
+
self.__doc__ = self.py_fns[0].__doc__
|
|
41
|
+
super().__init__(signatures, self_path=self_path, is_method=is_method, is_property=is_property)
|
|
42
|
+
|
|
43
|
+
def _update_as_overload_resolution(self, signature_idx: int) -> None:
|
|
44
|
+
assert len(self.py_fns) > signature_idx
|
|
45
|
+
self.py_fns = [self.py_fns[signature_idx]]
|
|
37
46
|
|
|
38
47
|
@property
|
|
39
48
|
def is_batched(self) -> bool:
|
|
40
49
|
return self.batch_size is not None
|
|
41
50
|
|
|
42
|
-
|
|
51
|
+
@property
|
|
52
|
+
def py_fn(self) -> Callable:
|
|
53
|
+
assert not self.is_polymorphic
|
|
54
|
+
return self.py_fns[0]
|
|
55
|
+
|
|
56
|
+
def exec(self, args: Sequence[Any], kwargs: dict[str, Any]) -> Any:
|
|
57
|
+
assert not self.is_polymorphic
|
|
43
58
|
if self.is_batched:
|
|
44
59
|
# Pack the batched parameters into singleton lists
|
|
45
60
|
constant_param_names = [p.name for p in self.signature.constant_parameters]
|
|
@@ -52,13 +67,14 @@ class CallableFunction(Function):
|
|
|
52
67
|
else:
|
|
53
68
|
return self.py_fn(*args, **kwargs)
|
|
54
69
|
|
|
55
|
-
def exec_batch(self,
|
|
70
|
+
def exec_batch(self, args: list[Any], kwargs: dict[str, Any]) -> list:
|
|
56
71
|
"""Execute the function with the given arguments and return the result.
|
|
57
72
|
The arguments are expected to be batched: if the corresponding parameter has type T,
|
|
58
73
|
then the argument should have type T if it's a constant parameter, or list[T] if it's
|
|
59
74
|
a batched parameter.
|
|
60
75
|
"""
|
|
61
76
|
assert self.is_batched
|
|
77
|
+
assert not self.is_polymorphic
|
|
62
78
|
# Unpack the constant parameters
|
|
63
79
|
constant_param_names = [p.name for p in self.signature.constant_parameters]
|
|
64
80
|
constant_kwargs = {k: v[0] for k, v in kwargs.items() if k in constant_param_names}
|
|
@@ -77,9 +93,25 @@ class CallableFunction(Function):
|
|
|
77
93
|
def name(self) -> str:
|
|
78
94
|
return self.self_name
|
|
79
95
|
|
|
96
|
+
def overload(self, fn: Callable) -> CallableFunction:
|
|
97
|
+
if self.self_path is None:
|
|
98
|
+
raise excs.Error('`overload` can only be used with module UDFs (not locally defined UDFs)')
|
|
99
|
+
if self.is_batched:
|
|
100
|
+
raise excs.Error('`overload` cannot be used with batched functions')
|
|
101
|
+
if self.is_method or self.is_property:
|
|
102
|
+
raise excs.Error('`overload` cannot be used with `is_method` or `is_property`')
|
|
103
|
+
if self._has_resolved_fns:
|
|
104
|
+
raise excs.Error('New `overload` not allowed after the UDF has already been called')
|
|
105
|
+
if self._conditional_return_type is not None:
|
|
106
|
+
raise excs.Error('New `overload` not allowed after a conditional return type has been specified')
|
|
107
|
+
sig = Signature.create(fn)
|
|
108
|
+
self.signatures.append(sig)
|
|
109
|
+
self.py_fns.append(fn)
|
|
110
|
+
return self
|
|
111
|
+
|
|
80
112
|
def help_str(self) -> str:
|
|
81
113
|
res = super().help_str()
|
|
82
|
-
res += '\n\n' + inspect.getdoc(self.
|
|
114
|
+
res += '\n\n' + inspect.getdoc(self.py_fns[0])
|
|
83
115
|
return res
|
|
84
116
|
|
|
85
117
|
def _as_dict(self) -> dict:
|
|
@@ -99,6 +131,7 @@ class CallableFunction(Function):
|
|
|
99
131
|
return super()._from_dict(d)
|
|
100
132
|
|
|
101
133
|
def to_store(self) -> tuple[dict, bytes]:
|
|
134
|
+
assert not self.is_polymorphic # multi-signature UDFs not allowed for stored fns
|
|
102
135
|
md = {
|
|
103
136
|
'signature': self.signature.as_dict(),
|
|
104
137
|
'batch_size': self.batch_size,
|
|
@@ -111,12 +144,15 @@ class CallableFunction(Function):
|
|
|
111
144
|
assert callable(py_fn)
|
|
112
145
|
sig = Signature.from_dict(md['signature'])
|
|
113
146
|
batch_size = md['batch_size']
|
|
114
|
-
return CallableFunction(sig, py_fn, self_name=name, batch_size=batch_size)
|
|
147
|
+
return CallableFunction([sig], [py_fn], self_name=name, batch_size=batch_size)
|
|
115
148
|
|
|
116
149
|
def validate_call(self, bound_args: dict[str, Any]) -> None:
|
|
117
|
-
|
|
150
|
+
from pixeltable import exprs
|
|
151
|
+
|
|
152
|
+
assert not self.is_polymorphic
|
|
118
153
|
if self.is_batched:
|
|
119
|
-
|
|
154
|
+
signature = self.signatures[0]
|
|
155
|
+
for param in signature.constant_parameters:
|
|
120
156
|
if param.name in bound_args and isinstance(bound_args[param.name], exprs.Expr):
|
|
121
157
|
raise ValueError(
|
|
122
158
|
f'{self.display_name}(): '
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import inspect
|
|
2
|
-
from typing import Any, Optional
|
|
2
|
+
from typing import Any, Optional, Sequence
|
|
3
3
|
|
|
4
4
|
import pixeltable
|
|
5
5
|
import pixeltable.exceptions as excs
|
|
@@ -8,15 +8,22 @@ from .function import Function
|
|
|
8
8
|
from .signature import Signature
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
class
|
|
12
|
-
"""
|
|
11
|
+
class ExprTemplate:
|
|
12
|
+
"""
|
|
13
|
+
Encapsulates a single signature of an `ExprTemplateFunction` and its associated parameterized expression,
|
|
14
|
+
along with various precomputed metadata. (This is analogous to a `Callable`-`Signature` pair in a
|
|
15
|
+
`CallableFunction`.)
|
|
16
|
+
"""
|
|
17
|
+
expr: 'pixeltable.exprs.Expr'
|
|
18
|
+
signature: Signature
|
|
19
|
+
param_exprs: list['pixeltable.exprs.Variable']
|
|
20
|
+
|
|
21
|
+
def __init__(self, expr: 'pixeltable.exprs.Expr', signature: Signature):
|
|
22
|
+
from pixeltable import exprs
|
|
13
23
|
|
|
14
|
-
def __init__(
|
|
15
|
-
self, expr: 'pixeltable.exprs.Expr', signature: Signature, self_path: Optional[str] = None,
|
|
16
|
-
name: Optional[str] = None):
|
|
17
|
-
import pixeltable.exprs as exprs
|
|
18
24
|
self.expr = expr
|
|
19
|
-
self.
|
|
25
|
+
self.signature = signature
|
|
26
|
+
|
|
20
27
|
self.param_exprs = list(set(expr.subexprs(expr_class=exprs.Variable)))
|
|
21
28
|
# make sure there are no duplicate names
|
|
22
29
|
assert len(self.param_exprs) == len(set(p.name for p in self.param_exprs))
|
|
@@ -24,7 +31,7 @@ class ExprTemplateFunction(Function):
|
|
|
24
31
|
|
|
25
32
|
# verify default values
|
|
26
33
|
self.defaults: dict[str, exprs.Literal] = {} # key: param name, value: default value converted to a Literal
|
|
27
|
-
for param in signature.parameters.values():
|
|
34
|
+
for param in self.signature.parameters.values():
|
|
28
35
|
if param.default is inspect.Parameter.empty:
|
|
29
36
|
continue
|
|
30
37
|
param_expr = self.param_exprs_by_name[param.name]
|
|
@@ -35,18 +42,39 @@ class ExprTemplateFunction(Function):
|
|
|
35
42
|
msg = str(e)
|
|
36
43
|
raise excs.Error(f"Default value for parameter '{param.name}': {msg[0].lower() + msg[1:]}")
|
|
37
44
|
|
|
38
|
-
super().__init__(signature, self_path=self_path)
|
|
39
45
|
|
|
40
|
-
|
|
46
|
+
class ExprTemplateFunction(Function):
|
|
47
|
+
"""A parameterized expression from which an executable Expr is created with a function call."""
|
|
48
|
+
templates: list[ExprTemplate]
|
|
49
|
+
self_name: str
|
|
50
|
+
|
|
51
|
+
def __init__(self, templates: list[ExprTemplate], self_path: Optional[str] = None, name: Optional[str] = None):
|
|
52
|
+
self.templates = templates
|
|
53
|
+
self.self_name = name
|
|
54
|
+
|
|
55
|
+
super().__init__([t.signature for t in templates], self_path=self_path)
|
|
56
|
+
|
|
57
|
+
def _update_as_overload_resolution(self, signature_idx: int) -> None:
|
|
58
|
+
self.templates = [self.templates[signature_idx]]
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def template(self) -> ExprTemplate:
|
|
62
|
+
assert not self.is_polymorphic
|
|
63
|
+
return self.templates[0]
|
|
64
|
+
|
|
65
|
+
def instantiate(self, args: Sequence[Any], kwargs: dict[str, Any]) -> 'pixeltable.exprs.Expr':
|
|
66
|
+
from pixeltable import exprs
|
|
67
|
+
|
|
68
|
+
assert not self.is_polymorphic
|
|
69
|
+
template = self.template
|
|
41
70
|
bound_args = self.signature.py_signature.bind(*args, **kwargs).arguments
|
|
42
71
|
# apply defaults, otherwise we might have Parameters left over
|
|
43
72
|
bound_args.update(
|
|
44
|
-
{param_name: default for param_name, default in
|
|
45
|
-
result =
|
|
46
|
-
import pixeltable.exprs as exprs
|
|
73
|
+
{param_name: default for param_name, default in template.defaults.items() if param_name not in bound_args})
|
|
74
|
+
result = template.expr.copy()
|
|
47
75
|
arg_exprs: dict[exprs.Expr, exprs.Expr] = {}
|
|
48
76
|
for param_name, arg in bound_args.items():
|
|
49
|
-
param_expr =
|
|
77
|
+
param_expr = template.param_exprs_by_name[param_name]
|
|
50
78
|
if not isinstance(arg, exprs.Expr):
|
|
51
79
|
# TODO: use the available param_expr.col_type
|
|
52
80
|
arg_expr = exprs.Expr.from_object(arg)
|
|
@@ -56,15 +84,15 @@ class ExprTemplateFunction(Function):
|
|
|
56
84
|
arg_expr = arg
|
|
57
85
|
arg_exprs[param_expr] = arg_expr
|
|
58
86
|
result = result.substitute(arg_exprs)
|
|
59
|
-
import pixeltable.exprs as exprs
|
|
60
87
|
assert not result._contains(exprs.Variable)
|
|
61
88
|
return result
|
|
62
89
|
|
|
63
|
-
def exec(self,
|
|
64
|
-
|
|
65
|
-
|
|
90
|
+
def exec(self, args: Sequence[Any], kwargs: dict[str, Any]) -> Any:
|
|
91
|
+
from pixeltable import exec, exprs
|
|
92
|
+
|
|
93
|
+
assert not self.is_polymorphic
|
|
94
|
+
expr = self.instantiate(args, kwargs)
|
|
66
95
|
row_builder = exprs.RowBuilder(output_exprs=[expr], columns=[], input_exprs=[])
|
|
67
|
-
import pixeltable.exec as exec
|
|
68
96
|
row_batch = exec.DataRowBatch(tbl=None, row_builder=row_builder, len=1)
|
|
69
97
|
row = row_batch[0]
|
|
70
98
|
row_builder.eval(row, ctx=row_builder.default_eval_ctx)
|
|
@@ -79,13 +107,15 @@ class ExprTemplateFunction(Function):
|
|
|
79
107
|
return self.self_name
|
|
80
108
|
|
|
81
109
|
def __str__(self) -> str:
|
|
82
|
-
return str(self.expr)
|
|
110
|
+
return str(self.templates[0].expr)
|
|
83
111
|
|
|
84
112
|
def _as_dict(self) -> dict:
|
|
85
113
|
if self.self_path is not None:
|
|
86
114
|
return super()._as_dict()
|
|
115
|
+
assert not self.is_polymorphic
|
|
116
|
+
assert len(self.templates) == 1
|
|
87
117
|
return {
|
|
88
|
-
'expr': self.expr.as_dict(),
|
|
118
|
+
'expr': self.template.expr.as_dict(),
|
|
89
119
|
'signature': self.signature.as_dict(),
|
|
90
120
|
'name': self.name,
|
|
91
121
|
}
|
|
@@ -93,7 +123,8 @@ class ExprTemplateFunction(Function):
|
|
|
93
123
|
@classmethod
|
|
94
124
|
def _from_dict(cls, d: dict) -> Function:
|
|
95
125
|
if 'expr' not in d:
|
|
96
|
-
|
|
126
|
+
return super()._from_dict(d)
|
|
97
127
|
assert 'signature' in d and 'name' in d
|
|
98
128
|
import pixeltable.exprs as exprs
|
|
99
|
-
|
|
129
|
+
template = ExprTemplate(exprs.Expr.from_dict(d['expr']), Signature.from_dict(d['signature']))
|
|
130
|
+
return cls([template], name=d['name'])
|
pixeltable/func/function.py
CHANGED
|
@@ -3,9 +3,11 @@ from __future__ import annotations
|
|
|
3
3
|
import abc
|
|
4
4
|
import importlib
|
|
5
5
|
import inspect
|
|
6
|
-
from
|
|
6
|
+
from copy import copy
|
|
7
|
+
from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence, cast
|
|
7
8
|
|
|
8
9
|
import sqlalchemy as sql
|
|
10
|
+
from typing_extensions import Self
|
|
9
11
|
|
|
10
12
|
import pixeltable as pxt
|
|
11
13
|
import pixeltable.exceptions as excs
|
|
@@ -15,7 +17,7 @@ from .globals import resolve_symbol
|
|
|
15
17
|
from .signature import Signature
|
|
16
18
|
|
|
17
19
|
if TYPE_CHECKING:
|
|
18
|
-
from .expr_template_function import ExprTemplateFunction
|
|
20
|
+
from .expr_template_function import ExprTemplate, ExprTemplateFunction
|
|
19
21
|
|
|
20
22
|
|
|
21
23
|
class Function(abc.ABC):
|
|
@@ -26,30 +28,42 @@ class Function(abc.ABC):
|
|
|
26
28
|
via the member self_path.
|
|
27
29
|
"""
|
|
28
30
|
|
|
29
|
-
|
|
31
|
+
signatures: list[Signature]
|
|
30
32
|
self_path: Optional[str]
|
|
31
33
|
is_method: bool
|
|
32
34
|
is_property: bool
|
|
33
35
|
_conditional_return_type: Optional[Callable[..., ts.ColumnType]]
|
|
34
36
|
|
|
37
|
+
# We cache the overload resolutions in self._resolutions. This ensures that each resolution is represented
|
|
38
|
+
# globally by a single Python object. We do this dynamically rather than pre-constructing them in order to
|
|
39
|
+
# avoid circular complexity in the `Function` initialization logic.
|
|
40
|
+
__resolved_fns: list[Self]
|
|
41
|
+
|
|
35
42
|
# Translates a call to this function with the given arguments to its SQLAlchemy equivalent.
|
|
36
43
|
# Overriden for specific Function instances via the to_sql() decorator. The override must accept the same
|
|
37
44
|
# parameter names as the original function. Each parameter is going to be of type sql.ColumnElement.
|
|
38
45
|
_to_sql: Callable[..., Optional[sql.ColumnElement]]
|
|
39
46
|
|
|
40
|
-
|
|
41
47
|
def __init__(
|
|
42
|
-
self,
|
|
48
|
+
self,
|
|
49
|
+
signatures: list[Signature],
|
|
50
|
+
self_path: Optional[str] = None,
|
|
51
|
+
is_method: bool = False,
|
|
52
|
+
is_property: bool = False
|
|
43
53
|
):
|
|
44
54
|
# Check that stored functions cannot be declared using `is_method` or `is_property`:
|
|
45
55
|
assert not ((is_method or is_property) and self_path is None)
|
|
46
|
-
|
|
56
|
+
assert isinstance(signatures, list)
|
|
57
|
+
assert len(signatures) > 0
|
|
58
|
+
self.signatures = signatures
|
|
47
59
|
self.self_path = self_path # fully-qualified path to self
|
|
48
60
|
self.is_method = is_method
|
|
49
61
|
self.is_property = is_property
|
|
50
62
|
self._conditional_return_type = None
|
|
51
63
|
self._to_sql = self.__default_to_sql
|
|
52
64
|
|
|
65
|
+
self.__resolved_fns = []
|
|
66
|
+
|
|
53
67
|
@property
|
|
54
68
|
def name(self) -> str:
|
|
55
69
|
assert self.self_path is not None
|
|
@@ -64,37 +78,121 @@ class Function(abc.ABC):
|
|
|
64
78
|
return self.self_path[len(ptf_prefix):]
|
|
65
79
|
return self.self_path
|
|
66
80
|
|
|
81
|
+
@property
|
|
82
|
+
def is_polymorphic(self) -> bool:
|
|
83
|
+
return len(self.signatures) > 1
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def signature(self) -> Signature:
|
|
87
|
+
assert not self.is_polymorphic
|
|
88
|
+
return self.signatures[0]
|
|
89
|
+
|
|
67
90
|
@property
|
|
68
91
|
def arity(self) -> int:
|
|
92
|
+
assert not self.is_polymorphic
|
|
69
93
|
return len(self.signature.parameters)
|
|
70
94
|
|
|
95
|
+
@property
|
|
96
|
+
def _resolved_fns(self) -> list[Self]:
|
|
97
|
+
"""
|
|
98
|
+
Return the list of overload resolutions for this `Function`, constructing it first if necessary.
|
|
99
|
+
Each resolution is a new `Function` instance that retains just the single signature at index `signature_idx`,
|
|
100
|
+
and is otherwise identical to this `Function`.
|
|
101
|
+
"""
|
|
102
|
+
if len(self.__resolved_fns) == 0:
|
|
103
|
+
# The list of overload resolutions hasn't been constructed yet; do so now.
|
|
104
|
+
if len(self.signatures) == 1:
|
|
105
|
+
# Only one signature: no need to construct separate resolutions
|
|
106
|
+
self.__resolved_fns.append(self)
|
|
107
|
+
else:
|
|
108
|
+
# Multiple signatures: construct a resolution for each signature
|
|
109
|
+
for idx in range(len(self.signatures)):
|
|
110
|
+
resolution = cast(Self, copy(self))
|
|
111
|
+
resolution.signatures = [self.signatures[idx]]
|
|
112
|
+
resolution._update_as_overload_resolution(idx)
|
|
113
|
+
self.__resolved_fns.append(resolution)
|
|
114
|
+
|
|
115
|
+
return self.__resolved_fns
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def _has_resolved_fns(self) -> bool:
|
|
119
|
+
"""
|
|
120
|
+
Returns true if the resolved_fns for this `Function` have been constructed (i.e., if self._resolved_fns
|
|
121
|
+
has been accessed).
|
|
122
|
+
"""
|
|
123
|
+
return len(self.__resolved_fns) > 0
|
|
124
|
+
|
|
125
|
+
def _update_as_overload_resolution(self, signature_idx: int) -> None:
|
|
126
|
+
"""
|
|
127
|
+
Subclasses must implement this in order to do any additional work when creating a resolution, beyond
|
|
128
|
+
simply updating `self.signatures`.
|
|
129
|
+
"""
|
|
130
|
+
raise NotImplementedError()
|
|
131
|
+
|
|
71
132
|
def help_str(self) -> str:
|
|
72
|
-
return self.display_name + str(self.
|
|
133
|
+
return self.display_name + str(self.signatures[0])
|
|
73
134
|
|
|
74
135
|
def __call__(self, *args: Any, **kwargs: Any) -> 'pxt.exprs.FunctionCall':
|
|
75
136
|
from pixeltable import exprs
|
|
76
|
-
|
|
77
|
-
self.
|
|
78
|
-
|
|
137
|
+
|
|
138
|
+
resolved_fn, bound_args = self._bind_to_matching_signature(args, kwargs)
|
|
139
|
+
return_type = resolved_fn.call_return_type(args, kwargs)
|
|
140
|
+
return exprs.FunctionCall(resolved_fn, bound_args, return_type)
|
|
141
|
+
|
|
142
|
+
def _bind_to_matching_signature(self, args: Sequence[Any], kwargs: dict[str, Any]) -> tuple[Self, dict[str, Any]]:
|
|
143
|
+
result: int = -1
|
|
144
|
+
bound_args: Optional[dict[str, Any]] = None
|
|
145
|
+
assert len(self.signatures) > 0
|
|
146
|
+
if len(self.signatures) == 1:
|
|
147
|
+
# Only one signature: call _bind_to_signature() and surface any errors directly
|
|
148
|
+
result = 0
|
|
149
|
+
bound_args = self._bind_to_signature(0, args, kwargs)
|
|
150
|
+
else:
|
|
151
|
+
# Multiple signatures: try each signature in declaration order and trap any errors.
|
|
152
|
+
# If none of them succeed, raise a generic error message.
|
|
153
|
+
for i in range(len(self.signatures)):
|
|
154
|
+
try:
|
|
155
|
+
bound_args = self._bind_to_signature(i, args, kwargs)
|
|
156
|
+
except (TypeError, excs.Error):
|
|
157
|
+
continue
|
|
158
|
+
result = i
|
|
159
|
+
break
|
|
160
|
+
if result == -1:
|
|
161
|
+
raise excs.Error(f'Function {self.name!r} has no matching signature for arguments')
|
|
162
|
+
assert result >= 0
|
|
163
|
+
assert bound_args is not None
|
|
164
|
+
return self._resolved_fns[result], bound_args
|
|
165
|
+
|
|
166
|
+
def _bind_to_signature(self, signature_idx: int, args: Sequence[Any], kwargs: dict[str, Any]) -> dict[str, Any]:
|
|
167
|
+
from pixeltable import exprs
|
|
168
|
+
|
|
169
|
+
signature = self.signatures[signature_idx]
|
|
170
|
+
bound_args = signature.py_signature.bind(*args, **kwargs).arguments
|
|
171
|
+
self._resolved_fns[signature_idx].validate_call(bound_args)
|
|
172
|
+
exprs.FunctionCall.normalize_args(self.name, signature, bound_args)
|
|
173
|
+
return bound_args
|
|
79
174
|
|
|
80
175
|
def validate_call(self, bound_args: dict[str, Any]) -> None:
|
|
81
176
|
"""Override this to do custom validation of the arguments"""
|
|
82
|
-
|
|
177
|
+
assert not self.is_polymorphic
|
|
83
178
|
|
|
84
|
-
def call_return_type(self, kwargs: dict[str, Any]) -> ts.ColumnType:
|
|
179
|
+
def call_return_type(self, args: Sequence[Any], kwargs: dict[str, Any]) -> ts.ColumnType:
|
|
85
180
|
"""Return the type of the value returned by calling this function with the given arguments"""
|
|
181
|
+
assert not self.is_polymorphic
|
|
86
182
|
if self._conditional_return_type is None:
|
|
87
183
|
return self.signature.return_type
|
|
88
|
-
bound_args = self.signature.py_signature.bind(**kwargs)
|
|
184
|
+
bound_args = self.signature.py_signature.bind(*args, **kwargs).arguments
|
|
89
185
|
kw_args: dict[str, Any] = {}
|
|
90
186
|
sig = inspect.signature(self._conditional_return_type)
|
|
91
187
|
for param in sig.parameters.values():
|
|
92
|
-
if param.name in bound_args
|
|
93
|
-
kw_args[param.name] = bound_args
|
|
188
|
+
if param.name in bound_args:
|
|
189
|
+
kw_args[param.name] = bound_args[param.name]
|
|
94
190
|
return self._conditional_return_type(**kw_args)
|
|
95
191
|
|
|
96
192
|
def conditional_return_type(self, fn: Callable[..., ts.ColumnType]) -> Callable[..., ts.ColumnType]:
|
|
97
193
|
"""Instance decorator for specifying a conditional return type for this function"""
|
|
194
|
+
if self.is_polymorphic:
|
|
195
|
+
raise excs.Error('`conditional_return_type` is not supported for functions with multiple signatures')
|
|
98
196
|
# verify that call_return_type only has parameters that are also present in the signature
|
|
99
197
|
sig = inspect.signature(fn)
|
|
100
198
|
for param in sig.parameters.values():
|
|
@@ -104,9 +202,35 @@ class Function(abc.ABC):
|
|
|
104
202
|
return fn
|
|
105
203
|
|
|
106
204
|
def using(self, **kwargs: Any) -> 'ExprTemplateFunction':
|
|
205
|
+
from .expr_template_function import ExprTemplateFunction
|
|
206
|
+
|
|
207
|
+
assert len(self.signatures) > 0
|
|
208
|
+
if len(self.signatures) == 1:
|
|
209
|
+
# Only one signature: call _bind_and_create_template() and surface any errors directly
|
|
210
|
+
template = self._bind_and_create_template(kwargs)
|
|
211
|
+
return ExprTemplateFunction([template])
|
|
212
|
+
else:
|
|
213
|
+
# Multiple signatures: iterate over each signature and generate a template for each
|
|
214
|
+
# successful binding. If there are no successful bindings, raise a generic error.
|
|
215
|
+
# (Note that the resulting ExprTemplateFunction may have strictly fewer signatures than
|
|
216
|
+
# this Function, in the event that only some of the signatures are successfully bound.)
|
|
217
|
+
templates: list['ExprTemplate'] = []
|
|
218
|
+
for i in range(len(self.signatures)):
|
|
219
|
+
try:
|
|
220
|
+
template = self._resolved_fns[i]._bind_and_create_template(kwargs)
|
|
221
|
+
templates.append(template)
|
|
222
|
+
except (TypeError, excs.Error):
|
|
223
|
+
continue
|
|
224
|
+
if len(templates) == 0:
|
|
225
|
+
raise excs.Error(f'Function {self.name!r} has no matching signature for arguments')
|
|
226
|
+
return ExprTemplateFunction(templates)
|
|
227
|
+
|
|
228
|
+
def _bind_and_create_template(self, kwargs: dict[str, Any]) -> 'ExprTemplate':
|
|
107
229
|
from pixeltable import exprs
|
|
108
230
|
|
|
109
|
-
from .expr_template_function import
|
|
231
|
+
from .expr_template_function import ExprTemplate
|
|
232
|
+
|
|
233
|
+
assert not self.is_polymorphic
|
|
110
234
|
|
|
111
235
|
# Resolve each kwarg into a parameter binding
|
|
112
236
|
bindings: dict[str, exprs.Expr] = {}
|
|
@@ -127,16 +251,18 @@ class Function(abc.ABC):
|
|
|
127
251
|
for param in residual_params:
|
|
128
252
|
bindings[param.name] = exprs.Variable(param.name, param.col_type)
|
|
129
253
|
|
|
130
|
-
|
|
254
|
+
return_type = self.call_return_type([], bindings)
|
|
255
|
+
call = exprs.FunctionCall(self, bindings, return_type)
|
|
131
256
|
|
|
132
257
|
# Construct the (n-k)-ary signature of the new function. We use `call.col_type` for this, rather than
|
|
133
258
|
# `self.signature.return_type`, because the return type of the new function may be specialized via a
|
|
134
259
|
# conditional return type.
|
|
135
260
|
new_signature = Signature(call.col_type, residual_params, self.signature.is_batched)
|
|
136
|
-
|
|
261
|
+
|
|
262
|
+
return ExprTemplate(call, new_signature)
|
|
137
263
|
|
|
138
264
|
@abc.abstractmethod
|
|
139
|
-
def exec(self,
|
|
265
|
+
def exec(self, args: Sequence[Any], kwargs: dict[str, Any]) -> Any:
|
|
140
266
|
"""Execute the function with the given arguments and return the result."""
|
|
141
267
|
pass
|
|
142
268
|
|
|
@@ -164,13 +290,18 @@ class Function(abc.ABC):
|
|
|
164
290
|
to an instance with from_dict().
|
|
165
291
|
Subclasses can override _as_dict().
|
|
166
292
|
"""
|
|
293
|
+
# We currently only ever serialize a function that has a specific signature (not a polymorphic form).
|
|
294
|
+
assert not self.is_polymorphic
|
|
167
295
|
classpath = f'{self.__class__.__module__}.{self.__class__.__qualname__}'
|
|
168
296
|
return {'_classpath': classpath, **self._as_dict()}
|
|
169
297
|
|
|
170
298
|
def _as_dict(self) -> dict:
|
|
171
|
-
"""Default serialization: store the path to self (which includes the module path)"""
|
|
299
|
+
"""Default serialization: store the path to self (which includes the module path) and signature."""
|
|
172
300
|
assert self.self_path is not None
|
|
173
|
-
return {
|
|
301
|
+
return {
|
|
302
|
+
'path': self.self_path,
|
|
303
|
+
'signature': self.signature.as_dict(),
|
|
304
|
+
}
|
|
174
305
|
|
|
175
306
|
@classmethod
|
|
176
307
|
def from_dict(cls, d: dict) -> Function:
|
|
@@ -181,15 +312,45 @@ class Function(abc.ABC):
|
|
|
181
312
|
module_path, class_name = d['_classpath'].rsplit('.', 1)
|
|
182
313
|
class_module = importlib.import_module(module_path)
|
|
183
314
|
func_class = getattr(class_module, class_name)
|
|
315
|
+
assert isinstance(func_class, type) and issubclass(func_class, Function)
|
|
184
316
|
return func_class._from_dict(d)
|
|
185
317
|
|
|
186
318
|
@classmethod
|
|
187
319
|
def _from_dict(cls, d: dict) -> Function:
|
|
188
320
|
"""Default deserialization: load the symbol indicated by the stored symbol_path"""
|
|
189
321
|
assert 'path' in d and d['path'] is not None
|
|
322
|
+
assert 'signature' in d and d['signature'] is not None
|
|
190
323
|
instance = resolve_symbol(d['path'])
|
|
191
324
|
assert isinstance(instance, Function)
|
|
192
|
-
|
|
325
|
+
|
|
326
|
+
# Load the signature from the DB and check that it is still valid (i.e., is still consistent with a signature
|
|
327
|
+
# in the code).
|
|
328
|
+
signature = Signature.from_dict(d['signature'])
|
|
329
|
+
idx = instance.__find_matching_overload(signature)
|
|
330
|
+
if idx is None:
|
|
331
|
+
# No match; generate an informative error message.
|
|
332
|
+
signature_note_str = 'any of its signatures' if instance.is_polymorphic else 'its signature as'
|
|
333
|
+
instance_signature_str = (
|
|
334
|
+
f'{len(instance.signatures)} signatures' if instance.is_polymorphic else str(instance.signature)
|
|
335
|
+
)
|
|
336
|
+
# TODO: Handle this more gracefully (instead of failing the DB load, allow the DB load to succeed, but
|
|
337
|
+
# mark any enclosing FunctionCall as unusable). It's the same issue as dealing with a renamed UDF or
|
|
338
|
+
# FunctionCall return type mismatch.
|
|
339
|
+
raise excs.Error(
|
|
340
|
+
f'The signature stored in the database for the UDF `{instance.self_path}` no longer matches '
|
|
341
|
+
f'{signature_note_str} as currently defined in the code.\nThis probably means that the code for '
|
|
342
|
+
f'`{instance.self_path}` has changed in a backward-incompatible way.\n'
|
|
343
|
+
f'Signature in database: {signature}\n'
|
|
344
|
+
f'Signature in code: {instance_signature_str}'
|
|
345
|
+
)
|
|
346
|
+
# We found a match; specialize to the appropriate overload resolution (non-polymorphic form) and return that.
|
|
347
|
+
return instance._resolved_fns[idx]
|
|
348
|
+
|
|
349
|
+
def __find_matching_overload(self, sig: Signature) -> Optional[int]:
|
|
350
|
+
for idx, overload_sig in enumerate(self.signatures):
|
|
351
|
+
if sig.is_consistent_with(overload_sig):
|
|
352
|
+
return idx
|
|
353
|
+
return None
|
|
193
354
|
|
|
194
355
|
def to_store(self) -> tuple[dict, bytes]:
|
|
195
356
|
"""
|
|
@@ -13,6 +13,7 @@ import pixeltable.env as env
|
|
|
13
13
|
import pixeltable.exceptions as excs
|
|
14
14
|
import pixeltable.type_system as ts
|
|
15
15
|
from pixeltable.metadata import schema
|
|
16
|
+
|
|
16
17
|
from .function import Function
|
|
17
18
|
|
|
18
19
|
_logger = logging.getLogger('pixeltable')
|
|
@@ -68,7 +69,7 @@ class FunctionRegistry:
|
|
|
68
69
|
raise excs.Error(f'A UDF with that name already exists: {fqn}')
|
|
69
70
|
self.module_fns[fqn] = fn
|
|
70
71
|
if fn.is_method or fn.is_property:
|
|
71
|
-
base_type = fn.
|
|
72
|
+
base_type = fn.signatures[0].parameters_by_pos[0].col_type.type_enum
|
|
72
73
|
if base_type not in self.type_methods:
|
|
73
74
|
self.type_methods[base_type] = {}
|
|
74
75
|
if fn.name in self.type_methods[base_type]:
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import inspect
|
|
4
|
-
from typing import Any, Callable, Optional
|
|
4
|
+
from typing import Any, Callable, Optional, Sequence
|
|
5
5
|
|
|
6
6
|
import sqlalchemy as sql
|
|
7
7
|
|
|
@@ -33,10 +33,11 @@ class QueryTemplateFunction(Function):
|
|
|
33
33
|
return QueryTemplateFunction(template_df, sig, path=path, name=name)
|
|
34
34
|
|
|
35
35
|
def __init__(
|
|
36
|
-
self, template_df: Optional['pxt.DataFrame'], sig:
|
|
36
|
+
self, template_df: Optional['pxt.DataFrame'], sig: Signature, path: Optional[str] = None,
|
|
37
37
|
name: Optional[str] = None,
|
|
38
38
|
):
|
|
39
|
-
|
|
39
|
+
assert sig is not None
|
|
40
|
+
super().__init__([sig], self_path=path)
|
|
40
41
|
self.self_name = name
|
|
41
42
|
self.template_df = template_df
|
|
42
43
|
|
|
@@ -48,16 +49,20 @@ class QueryTemplateFunction(Function):
|
|
|
48
49
|
# convert defaults to Literals
|
|
49
50
|
self.defaults: dict[str, exprs.Literal] = {} # key: param name, value: default value converted to a Literal
|
|
50
51
|
param_types = self.template_df.parameters()
|
|
51
|
-
for param in [p for p in
|
|
52
|
+
for param in [p for p in sig.parameters.values() if p.has_default()]:
|
|
52
53
|
assert param.name in param_types
|
|
53
54
|
param_type = param_types[param.name]
|
|
54
55
|
literal_default = exprs.Literal(param.default, col_type=param_type)
|
|
55
56
|
self.defaults[param.name] = literal_default
|
|
56
57
|
|
|
58
|
+
def _update_as_overload_resolution(self, signature_idx: int) -> None:
|
|
59
|
+
pass # only one signature supported for QueryTemplateFunction
|
|
60
|
+
|
|
57
61
|
def set_conn(self, conn: Optional[sql.engine.Connection]) -> None:
|
|
58
62
|
self.conn = conn
|
|
59
63
|
|
|
60
|
-
def exec(self,
|
|
64
|
+
def exec(self, args: Sequence[Any], kwargs: dict[str, Any]) -> Any:
|
|
65
|
+
assert not self.is_polymorphic
|
|
61
66
|
bound_args = self.signature.py_signature.bind(*args, **kwargs).arguments
|
|
62
67
|
# apply defaults, otherwise we might have Parameters left over
|
|
63
68
|
bound_args.update(
|
|
@@ -75,7 +80,7 @@ class QueryTemplateFunction(Function):
|
|
|
75
80
|
return self.self_name
|
|
76
81
|
|
|
77
82
|
def _as_dict(self) -> dict:
|
|
78
|
-
return {'name': self.name, 'signature': self.
|
|
83
|
+
return {'name': self.name, 'signature': self.signatures[0].as_dict(), 'df': self.template_df.as_dict()}
|
|
79
84
|
|
|
80
85
|
@classmethod
|
|
81
86
|
def _from_dict(cls, d: dict) -> Function:
|