pixeltable 0.2.26__py3-none-any.whl → 0.5.7__py3-none-any.whl

pixeltable/exprs/function_call.py

@@ -1,126 +1,94 @@
 from __future__ import annotations
 
 import inspect
-import json
+import logging
 import sys
-from typing import Any, Optional
+from textwrap import dedent
+from typing import Any, Sequence
 
 import sqlalchemy as sql
 
-import pixeltable.catalog as catalog
-import pixeltable.exceptions as excs
-import pixeltable.func as func
-import pixeltable.type_system as ts
+from pixeltable import catalog, exceptions as excs, func, type_system as ts
 
 from .data_row import DataRow
 from .expr import Expr
-from .inline_expr import InlineDict, InlineList
+from .literal import Literal
 from .row_builder import RowBuilder
 from .rowid_ref import RowidRef
 from .sql_element_cache import SqlElementCache
 
+_logger = logging.getLogger('pixeltable')
 
-class FunctionCall(Expr):
 
+class FunctionCall(Expr):
     fn: func.Function
     is_method_call: bool
     agg_init_args: dict[str, Any]
+    resource_pool: str | None
+
+    # These collections hold the component indices corresponding to the args and kwargs
+    # that were passed to the FunctionCall. They're 1:1 with the original call pattern.
+    arg_idxs: list[int]
+    kwarg_idxs: dict[str, int]
 
-    # tuple[Optional[int], Optional[Any]]:
-    # - for Exprs: (index into components, None)
-    # - otherwise: (None, val)
-    args: list[tuple[Optional[int], Optional[Any]]]
-    kwargs: dict[str, tuple[Optional[int], Optional[Any]]]
+    # A "bound" version of the FunctionCall arguments, mapping each specified parameter name
+    # to one of three types of bindings:
+    # - a component index, if the parameter is a non-variadic parameter
+    # - a list of component indices, if the parameter is a variadic positional parameter
+    # - a dict mapping keyword names to component indices, if the parameter is a variadic keyword parameter
+    bound_idxs: dict[str, int | list[int] | dict[str, int]]
 
-    arg_types: list[ts.ColumnType]
-    kwarg_types: dict[str, ts.ColumnType]
+    return_type: ts.ColumnType
     group_by_start_idx: int
     group_by_stop_idx: int
     fn_expr_idx: int
     order_by_start_idx: int
-    constant_args: set[str]
-    aggregator: Optional[Any]
-    current_partition_vals: Optional[list[Any]]
+    aggregator: Any | None
+    current_partition_vals: list[Any] | None
+
+    _validation_error: str | None
 
     def __init__(
-            self, fn: func.Function, bound_args: dict[str, Any], order_by_clause: Optional[list[Any]] = None,
-            group_by_clause: Optional[list[Any]] = None, is_method_call: bool = False):
+        self,
+        fn: func.Function,
+        args: list[Expr],
+        kwargs: dict[str, Expr],
+        return_type: ts.ColumnType,
+        order_by_clause: list[Any] | None = None,
+        group_by_clause: list[Any] | None = None,
+        is_method_call: bool = False,
+        validation_error: str | None = None,
+    ):
+        assert not fn.is_polymorphic
+        assert all(isinstance(arg, Expr) for arg in args)
+        assert all(isinstance(arg, Expr) for arg in kwargs.values())
+
         if order_by_clause is None:
             order_by_clause = []
         if group_by_clause is None:
             group_by_clause = []
-        signature = fn.signature
-        return_type = fn.call_return_type(bound_args)
-        self.fn = fn
-        self.is_method_call = is_method_call
-        self.normalize_args(fn.name, signature, bound_args)
-
-        # If `return_type` is non-nullable, but the function call has a nullable input to any of its non-nullable
-        # parameters, then we need to make it nullable. This is because Pixeltable defaults a function output to
-        # `None` when any of its non-nullable inputs are `None`.
-        for arg_name, arg in bound_args.items():
-            param = signature.parameters[arg_name]
-            if (
-                param.col_type is not None and not param.col_type.nullable
-                and isinstance(arg, Expr) and arg.col_type.nullable
-            ):
-                return_type = return_type.copy(nullable=True)
-                break
 
         super().__init__(return_type)
 
-        self.agg_init_args = {}
-        if self.is_agg_fn_call:
-            # we separate out the init args for the aggregator
-            assert isinstance(fn, func.AggregateFunction)
-            self.agg_init_args = {
-                arg_name: arg for arg_name, arg in bound_args.items() if arg_name in fn.init_param_names
-            }
-            bound_args = {arg_name: arg for arg_name, arg in bound_args.items() if arg_name not in fn.init_param_names}
-
-        # construct components, args, kwargs
-        self.args = []
-        self.kwargs = {}
-
-        # we record the types of non-variable parameters for runtime type checks
-        self.arg_types = []
-        self.kwarg_types = {}
-
-        # the prefix of parameters that are bound can be passed by position
-        processed_args: set[str] = set()
-        for py_param in fn.signature.py_signature.parameters.values():
-            if py_param.name not in bound_args or py_param.kind == inspect.Parameter.KEYWORD_ONLY:
-                break
-            arg = bound_args[py_param.name]
-            if isinstance(arg, Expr):
-                self.args.append((len(self.components), None))
-                self.components.append(arg.copy())
-            else:
-                self.args.append((None, arg))
-            if py_param.kind != inspect.Parameter.VAR_POSITIONAL and py_param.kind != inspect.Parameter.VAR_KEYWORD:
-                self.arg_types.append(signature.parameters[py_param.name].col_type)
-            processed_args.add(py_param.name)
-
-        # the remaining args are passed as keywords
-        for param_name in bound_args.keys():
-            if param_name not in processed_args:
-                arg = bound_args[param_name]
-                if isinstance(arg, Expr):
-                    self.kwargs[param_name] = (len(self.components), None)
-                    self.components.append(arg.copy())
-                else:
-                    self.kwargs[param_name] = (None, arg)
-                if fn.signature.py_signature.parameters[param_name].kind != inspect.Parameter.VAR_KEYWORD:
-                    self.kwarg_types[param_name] = signature.parameters[param_name].col_type
+        self.fn = fn
+        self.return_type = return_type
+        self.is_method_call = is_method_call
+
+        # Build the components list from the specified args and kwargs, and note the component_idx of each argument.
+        self.components.extend(arg.copy() for arg in args)
+        self.arg_idxs = list(range(len(self.components)))
+        self.components.extend(arg.copy() for arg in kwargs.values())
+        self.kwarg_idxs = {name: i + len(args) for i, name in enumerate(kwargs.keys())}
 
         # window function state:
         # self.components[self.group_by_start_idx:self.group_by_stop_idx] contains group_by exprs
         self.group_by_start_idx, self.group_by_stop_idx = 0, 0
         if len(group_by_clause) > 0:
             if isinstance(group_by_clause[0], catalog.Table):
+                assert len(group_by_clause) == 1
                 group_by_exprs = self._create_rowid_refs(group_by_clause[0])
             else:
-                assert isinstance(group_by_clause[0], Expr)
+                assert all(isinstance(expr, Expr) for expr in group_by_clause)
                 group_by_exprs = group_by_clause
             # record grouping exprs in self.components, we need to evaluate them to get partition vals
             self.group_by_start_idx = len(self.components)
@@ -129,9 +97,9 @@ class FunctionCall(Expr):
 
         if isinstance(self.fn, func.ExprTemplateFunction):
             # we instantiate the template to create an Expr that can be evaluated and record that as a component
-            fn_expr = self.fn.instantiate(**bound_args)
+            fn_expr = self.fn.instantiate(args, kwargs)
+            self.fn_expr_idx = len(self.components)
             self.components.append(fn_expr)
-            self.fn_expr_idx = len(self.components) - 1
         else:
             self.fn_expr_idx = sys.maxsize
 
@@ -139,12 +107,38 @@ class FunctionCall(Expr):
         # (that's done in SQL)
         if len(order_by_clause) > 0 and not isinstance(order_by_clause[0], Expr):
             raise excs.Error(
-                f'order_by argument needs to be a Pixeltable expression, but instead is a {type(order_by_clause[0])}')
-        # don't add components after this, everthing after order_by_start_idx is part of the order_by clause
+                f'order_by argument needs to be a Pixeltable expression, but instead is a {type(order_by_clause[0])}'
+            )
         self.order_by_start_idx = len(self.components)
         self.components.extend(order_by_clause)
 
-        self.constant_args = {param_name for param_name, arg in bound_args.items() if not isinstance(arg, Expr)}
+        self._validation_error = validation_error
+
+        if validation_error is not None:
+            self.bound_idxs = {}
+            self.resource_pool = None
+            return
+
+        # Now generate bound_idxs for the args and kwargs indices.
+        # This is guaranteed to work, because at this point the call has already been validated.
+        # These will be used later to dereference specific parameter values.
+        bindings = fn.signature.py_signature.bind(*self.arg_idxs, **self.kwarg_idxs)
+        self.bound_idxs = bindings.arguments
+
+        # Separately generate bound_args for purposes of determining the resource pool.
+        bindings = fn.signature.py_signature.bind(*args, **kwargs)
+        bound_args = bindings.arguments
+        self.resource_pool = fn.call_resource_pool(bound_args)
+
+        self.agg_init_args = {}
+        if self.is_agg_fn_call:
+            # We separate out the init args for the aggregator. Unpack Literals in init args.
+            assert isinstance(fn, func.AggregateFunction)
+            for arg_name, arg in bound_args.items():
+                if arg_name in fn.init_param_names[0]:
+                    assert isinstance(arg, Literal)  # This was checked during validate_call
+                    self.agg_init_args[arg_name] = arg.val
+
         # execution state for aggregate functions
         self.aggregator = None
         self.current_partition_vals = None
@@ -153,131 +147,58 @@ class FunctionCall(Expr):
 
     def _create_rowid_refs(self, tbl: catalog.Table) -> list[Expr]:
         target = tbl._tbl_version_path.tbl_version
-        return [RowidRef(target, i) for i in range(target.num_rowid_columns())]
+        return [RowidRef(target, i) for i in range(target.get().num_rowid_columns())]
 
-    def default_column_name(self) -> Optional[str]:
+    def default_column_name(self) -> str | None:
         return self.fn.name
 
-    @classmethod
-    def normalize_args(cls, fn_name: str, signature: func.Signature, bound_args: dict[str, Any]) -> None:
-        """Converts args to Exprs where appropriate and checks that they are compatible with signature.
-
-        Updates bound_args in place, where necessary.
-        """
-        for param_name, arg in bound_args.items():
-            param = signature.parameters[param_name]
-            is_var_param = param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
-
-            if isinstance(arg, dict):
-                try:
-                    arg = InlineDict(arg)
-                    bound_args[param_name] = arg
-                    continue
-                except excs.Error:
-                    # this didn't work, but it might be a literal
-                    pass
-
-            if isinstance(arg, list) or isinstance(arg, tuple):
-                try:
-                    arg = InlineList(arg)
-                    bound_args[param_name] = arg
-                    continue
-                except excs.Error:
-                    # this didn't work, but it might be a literal
-                    pass
-
-            if not isinstance(arg, Expr):
-                # make sure that non-Expr args are json-serializable and are literals of the correct type
-                try:
-                    _ = json.dumps(arg)
-                except TypeError:
-                    raise excs.Error(f'Argument for parameter {param_name!r} is not json-serializable: {arg} (of type {type(arg)})')
-                if arg is not None:
-                    try:
-                        param_type = param.col_type
-                        bound_args[param_name] = param_type.create_literal(arg)
-                    except TypeError as e:
-                        msg = str(e)
-                        raise excs.Error(f'Argument for parameter {param_name!r}: {msg[0].lower() + msg[1:]}')
-                continue
-
-            # these checks break the db migration test, because InlineArray isn't serialized correctly (it looses
-            # the type information)
-            # if is_var_param:
-            #     if param.kind == inspect.Parameter.VAR_POSITIONAL:
-            #         if not isinstance(arg, InlineArray) or not arg.col_type.is_json_type():
-            #             pass
-            #         assert isinstance(arg, InlineArray), type(arg)
-            #         assert arg.col_type.is_json_type()
-            #     if param.kind == inspect.Parameter.VAR_KEYWORD:
-            #         if not isinstance(arg, InlineDict):
-            #             pass
-            #         assert isinstance(arg, InlineDict), type(arg)
-            if is_var_param:
-                pass
-            else:
-                assert param.col_type is not None
-                # Check that the argument is consistent with the expected parameter type, with the allowance that
-                # non-nullable parameters can still accept nullable arguments (since function calls with Nones
-                # assigned to non-nullable parameters will always return None)
-                if not (
-                    param.col_type.is_supertype_of(arg.col_type, ignore_nullable=True)
-                    # TODO: this is a hack to allow JSON columns to be passed to functions that accept scalar
-                    # types. It's necessary to avoid littering notebooks with `apply(str)` calls or equivalent.
-                    # (Previously, this wasn't necessary because `is_supertype_of()` was improperly implemented.)
-                    # We need to think through the right way to handle this scenario.
-                    or (arg.col_type.is_json_type() and param.col_type.is_scalar_type())
-                ):
-                    raise excs.Error(
-                        f'Parameter {param_name} (in function {fn_name}): argument type {arg.col_type} does not match parameter type '
-                        f'{param.col_type}')
-
     def _equals(self, other: FunctionCall) -> bool:
-        if self.fn != other.fn:
-            return False
-        if len(self.args) != len(other.args):
-            return False
-        for i in range(len(self.args)):
-            if self.args[i] != other.args[i]:
-                return False
-        if self.group_by_start_idx != other.group_by_start_idx:
-            return False
-        if self.group_by_stop_idx != other.group_by_stop_idx:
-            return False
-        if self.order_by_start_idx != other.order_by_start_idx:
-            return False
-        return True
+        return (
+            self.fn == other.fn
+            and self.arg_idxs == other.arg_idxs
+            and self.kwarg_idxs == other.kwarg_idxs
+            and self.group_by_start_idx == other.group_by_start_idx
+            and self.group_by_stop_idx == other.group_by_stop_idx
+            and self.order_by_start_idx == other.order_by_start_idx
+        )
 
     def _id_attrs(self) -> list[tuple[str, Any]]:
-        return super()._id_attrs() + [
+        return [
+            *super()._id_attrs(),
             ('fn', id(self.fn)),  # use the function pointer, not the fqn, which isn't set for lambdas
-            ('args', self.args),
-            ('kwargs', self.kwargs),
+            ('args', self.arg_idxs),
+            ('kwargs', self.kwarg_idxs),
             ('group_by_start_idx', self.group_by_start_idx),
             ('group_by_stop_idx', self.group_by_stop_idx),
-            ('order_by_start_idx', self.order_by_start_idx)
+            ('fn_expr_idx', self.fn_expr_idx),
+            ('order_by_start_idx', self.order_by_start_idx),
         ]
 
     def __repr__(self) -> str:
         return self.display_str()
 
+    # def __repr__(self) -> str:
+    #     return f'FunctionCall(fn={self.fn!r}, args={self.args!r}, kwargs={self.kwargs!r})'
+
+    @property
+    def validation_error(self) -> str | None:
+        return self._validation_error or super().validation_error
+
     def display_str(self, inline: bool = True) -> str:
+        if isinstance(self.fn, func.ExprTemplateFunction) and isinstance(self.fn.template.expr, FunctionCall):
+            # If this FunctionCall uses an ExprTemplateFunction with a nested FunctionCall, then resolve the
+            # indirection by substitution into the ExprTemplateFunction.
+            subst = self.fn.instantiate(self.args, self.kwargs)
+            return subst.display_str(inline)
         if self.is_method_call:
             return f'{self.components[0]}.{self.fn.name}({self._print_args(1, inline)})'
         else:
-            fn_name = self.fn.display_name if self.fn.display_name != '' else 'anonymous_fn'
+            fn_name = self.fn.display_name or 'anonymous_fn'
             return f'{fn_name}({self._print_args()})'
 
     def _print_args(self, start_idx: int = 0, inline: bool = True) -> str:
-        def print_arg(arg: Any) -> str:
-            return repr(arg) if isinstance(arg, str) else str(arg)
-        arg_strs = [
-            print_arg(arg) if idx is None else str(self.components[idx]) for idx, arg in self.args[start_idx:]
-        ]
-        arg_strs.extend([
-            f'{param_name}={print_arg(arg) if idx is None else str(self.components[idx])}'
-            for param_name, (idx, arg) in self.kwargs.items()
-        ])
+        arg_strs = [str(self.components[idx]) for idx in self.arg_idxs[start_idx:]]
+        arg_strs.extend([f'{param_name}={self.components[idx]}' for param_name, idx in self.kwarg_idxs.items()])
         if len(self.order_by) > 0:
             assert isinstance(self.fn, func.AggregateFunction)
             if self.fn.requires_order_by:
@@ -293,20 +214,28 @@ class FunctionCall(Expr):
     def has_group_by(self) -> bool:
         return self.group_by_stop_idx != 0
 
+    @property
+    def is_async(self) -> bool:
+        return self.fn.is_async
+
     @property
     def group_by(self) -> list[Expr]:
-        return self.components[self.group_by_start_idx:self.group_by_stop_idx]
+        return self.components[self.group_by_start_idx : self.group_by_stop_idx]
 
     @property
     def order_by(self) -> list[Expr]:
-        return self.components[self.order_by_start_idx:]
+        return self.components[self.order_by_start_idx :]
 
     @property
     def is_window_fn_call(self) -> bool:
-        return isinstance(self.fn, func.AggregateFunction) and self.fn.allows_window and (
-            not self.fn.allows_std_agg
-            or self.has_group_by()
-            or (len(self.order_by) > 0 and not self.fn.requires_order_by)
+        return (
+            isinstance(self.fn, func.AggregateFunction)
+            and self.fn.allows_window
+            and (
+                not self.fn.allows_std_agg
+                or self.has_group_by()
+                or (len(self.order_by) > 0 and not self.fn.requires_order_by)
+            )
         )
 
     def get_window_sort_exprs(self) -> tuple[list[Expr], list[Expr]]:
@@ -324,35 +253,29 @@ class FunctionCall(Expr):
         assert self.is_agg_fn_call
         return self.order_by
 
-    def sql_expr(self, sql_elements: SqlElementCache) -> Optional[sql.ColumnElement]:
+    def sql_expr(self, sql_elements: SqlElementCache) -> sql.ColumnElement | None:
+        assert self.is_valid
+
         # we currently can't translate aggregate functions with grouping and/or ordering to SQL
         if self.has_group_by() or len(self.order_by) > 0:
             return None
 
         # try to construct args and kwargs to call self.fn._to_sql()
+        args: list[sql.ColumnElement] = []
+        for component_idx in self.arg_idxs:
+            arg_element = sql_elements.get(self.components[component_idx])
+            if arg_element is None:
+                return None
+            args.append(arg_element)
+
         kwargs: dict[str, sql.ColumnElement] = {}
-        for param_name, (component_idx, arg) in self.kwargs.items():
-            param = self.fn.signature.parameters[param_name]
-            assert param.kind != inspect.Parameter.VAR_POSITIONAL and param.kind != inspect.Parameter.VAR_KEYWORD
-            if component_idx is None:
-                kwargs[param_name] = sql.literal(arg)
-            else:
-                arg_element = sql_elements.get(self.components[component_idx])
-                if arg_element is None:
-                    return None
-                kwargs[param_name] = arg_element
+        for param_name, component_idx in self.kwarg_idxs.items():
+            arg_element = sql_elements.get(self.components[component_idx])
+            if arg_element is None:
+                return None
+            kwargs[param_name] = arg_element
 
-        args: list[sql.ColumnElement] = []
-        for _, (component_idx, arg) in enumerate(self.args):
-            if component_idx is None:
-                args.append(sql.literal(arg))
-            else:
-                arg_element = sql_elements.get(self.components[component_idx])
-                if arg_element is None:
-                    return None
-                args.append(arg_element)
-        result = self.fn._to_sql(*args, **kwargs)
-        return result
+        return self.fn._to_sql(*args, **kwargs)
 
     def reset_agg(self) -> None:
         """
@@ -360,46 +283,127 @@ class FunctionCall(Expr):
         """
         assert self.is_agg_fn_call
         assert isinstance(self.fn, func.AggregateFunction)
-        self.aggregator = self.fn.agg_cls(**self.agg_init_args)
+        self.aggregator = self.fn.agg_class(**self.agg_init_args)
+
+    @property
+    def bound_args(self) -> dict[str, Expr]:
+        """
+        Reconstructs bound arguments from the components of this FunctionCall.
+        """
+        bound_args: dict[str, Expr] = {}
+        for name, idx in self.bound_idxs.items():
+            if isinstance(idx, int):
+                bound_args[name] = self.components[idx]
+            elif isinstance(idx, Sequence):
+                bound_args[name] = Expr.from_object([self.components[i] for i in idx])
+            elif isinstance(idx, dict):
+                bound_args[name] = Expr.from_object({k: self.components[i] for k, i in idx.items()})
+            else:
+                raise AssertionError(f'{name}: {idx} (of type `{type(idx)}`)')
+        return bound_args
+
+    def substitute(self, spec: dict[Expr, Expr]) -> Expr:
+        """
+        Substitution of FunctionCall arguments could cause the return value to become more specific, in the case
+        where a variable is replaced with a specific value.
+        """
+        res = super().substitute(spec)
+        assert res is self
+        if self.is_valid:
+            # If this FunctionCall is valid, re-evaluate the call_return_type of the substituted expression. If the
+            # FunctionCall is not valid, it isn't safe to do this. (Really we should be asserting that it *is* valid,
+            # but we still need to be able to do substitutions on invalid FunctionCalls, because loading an
+            # EmbeddingIndex from the db involves reconstructing the requisite (substituted) FunctionCalls. We could
+            # fix this by separately persisting the FunctionCall instances held by EmbeddingIndex to the db. That's
+            # probably a good idea, but it's also probably not urgent, since it only affects Functions that have a
+            # conditional_return_type implemented.)
+            self.return_type = self.fn.call_return_type(self.bound_args)
+            self.col_type = self.return_type
+        return self
+
+    @property
+    def args(self) -> list[Expr]:
+        return [self.components[idx] for idx in self.arg_idxs]
+
+    @property
+    def kwargs(self) -> dict[str, Expr]:
+        return {name: self.components[idx] for name, idx in self.kwarg_idxs.items()}
+
+    @property
+    def fn_expr(self) -> Expr | None:
+        if self.fn_expr_idx != sys.maxsize:
+            return self.components[self.fn_expr_idx]
+        return None
 
     def update(self, data_row: DataRow) -> None:
         """
         Update agg state
         """
         assert self.is_agg_fn_call
-        args, kwargs = self._make_args(data_row)
+        args, kwargs = self.make_args(data_row)
         self.aggregator.update(*args, **kwargs)
 
-    def _make_args(self, data_row: DataRow) -> tuple[list[Any], dict[str, Any]]:
-        """Return args and kwargs, constructed for data_row"""
+    def make_args(self, data_row: DataRow) -> tuple[list[Any], dict[str, Any]] | None:
+        """Return args and kwargs, constructed for data_row; returns None if any non-nullable arg is None."""
+        args: list[Any] = []
+        parameters_by_pos = self.fn.signature.parameters_by_pos
+        for idx in self.arg_idxs:
+            val = data_row[self.components[idx].slot_idx]
+            if (
+                val is None
+                and parameters_by_pos[idx].kind
+                in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+                and not parameters_by_pos[idx].col_type.nullable
+            ):
+                return None
+            args.append(val)
+
         kwargs: dict[str, Any] = {}
-        for param_name, (component_idx, arg) in self.kwargs.items():
-            val = arg if component_idx is None else data_row[self.components[component_idx].slot_idx]
-            param = self.fn.signature.parameters[param_name]
-            if param.kind == inspect.Parameter.VAR_KEYWORD:
-                # expand **kwargs parameter
-                kwargs.update(val)
-            else:
-                assert param.kind != inspect.Parameter.VAR_POSITIONAL
-                kwargs[param_name] = val
+        parameters = self.fn.signature.parameters
+        for param_name, idx in self.kwarg_idxs.items():
+            val = data_row[self.components[idx].slot_idx]
+            if (
+                val is None
+                and parameters[param_name].kind
+                in (inspect.Parameter.KEYWORD_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+                and not parameters[param_name].col_type.nullable
+            ):
+                return None
+            kwargs[param_name] = val
 
-        args: list[Any] = []
-        for param_idx, (component_idx, arg) in enumerate(self.args):
-            val = arg if component_idx is None else data_row[self.components[component_idx].slot_idx]
-            param = self.fn.signature.parameters_by_pos[param_idx]
-            if param.kind == inspect.Parameter.VAR_POSITIONAL:
-                # expand *args parameter
-                assert isinstance(val, list)
-                args.extend(val)
-            elif param.kind == inspect.Parameter.VAR_KEYWORD:
-                # expand **kwargs parameter
-                assert isinstance(val, dict)
-                kwargs.update(val)
-            else:
-                args.append(val)
         return args, kwargs
 
+    def get_param_values(self, param_names: Sequence[str], data_rows: list[DataRow]) -> list[dict[str, Any]]:
+        """
+        Returns a list of dicts mapping each param name to its value when this FunctionCall is evaluated against
+        data_rows
+        """
+        assert self.is_valid
+        assert all(name in self.fn.signature.parameters for name in param_names), f'{param_names}, {self.fn.signature}'
+        result: list[dict[str, Any]] = []
+        for row in data_rows:
+            d: dict[str, Any] = {}
+            for param_name in param_names:
+                val = self.bound_idxs.get(param_name)
+                if isinstance(val, int):
+                    d[param_name] = row[self.components[val].slot_idx]
+                elif isinstance(val, list):
+                    # var_positional
+                    d[param_name] = [row[self.components[idx].slot_idx] for idx in val]
+                elif isinstance(val, dict):
+                    # var_keyword
+                    d[param_name] = {k: row[self.components[idx].slot_idx] for k, idx in val.items()}
+                else:
+                    assert val is None
+                    default = self.fn.signature.parameters[param_name].default
+                    assert default is not None
+                    d[param_name] = default.val
+            result.append(d)
+        return result
+
     def eval(self, data_row: DataRow, row_builder: RowBuilder) -> None:
+        assert self.is_valid
+
         if isinstance(self.fn, func.ExprTemplateFunction):
             # we need to evaluate the template
             # TODO: can we get rid of this extra copy?
@@ -412,64 +416,159 @@ class FunctionCall(Expr):
             data_row[self.slot_idx] = self.aggregator.value()
             return
 
-        args, kwargs = self._make_args(data_row)
-        signature = self.fn.signature
-        if signature.parameters is not None:
-            # check for nulls
-            for i in range(len(self.arg_types)):
-                if args[i] is None and not self.arg_types[i].nullable:
-                    # we can't evaluate this function
-                    data_row[self.slot_idx] = None
-                    return
-            for param_name, param_type in self.kwarg_types.items():
-                if kwargs[param_name] is None and not param_type.nullable:
-                    # we can't evaluate this function
-                    data_row[self.slot_idx] = None
-                    return
-
-        if isinstance(self.fn, func.CallableFunction) and not self.fn.is_batched:
-            # optimization: avoid additional level of indirection we'd get from calling Function.exec()
-            data_row[self.slot_idx] = self.fn.py_fn(*args, **kwargs)
-        elif self.is_window_fn_call:
+        args_kwargs = self.make_args(data_row)
+        if args_kwargs is None:
+            # we can't evaluate this function
+            data_row[self.slot_idx] = None
+            return
+        args, kwargs = args_kwargs
+
+        if self.is_window_fn_call:
             assert isinstance(self.fn, func.AggregateFunction)
+            agg_cls = self.fn.agg_class
             if self.has_group_by():
                 if self.current_partition_vals is None:
                     self.current_partition_vals = [None] * len(self.group_by)
                 partition_vals = [data_row[e.slot_idx] for e in self.group_by]
                 if partition_vals != self.current_partition_vals:
                     # new partition
-                    self.aggregator = self.fn.agg_cls(**self.agg_init_args)
+                    self.aggregator = agg_cls(**self.agg_init_args)
                     self.current_partition_vals = partition_vals
             elif self.aggregator is None:
-                self.aggregator = self.fn.agg_cls(**self.agg_init_args)
+                self.aggregator = agg_cls(**self.agg_init_args)
             self.aggregator.update(*args)
             data_row[self.slot_idx] = self.aggregator.value()
         else:
-            data_row[self.slot_idx] = self.fn.exec(*args, **kwargs)
+            data_row[self.slot_idx] = self.fn.exec(args, kwargs)
 
     def _as_dict(self) -> dict:
-        result = {
-            'fn': self.fn.as_dict(), 'args': self.args, 'kwargs': self.kwargs,
-            'group_by_start_idx': self.group_by_start_idx, 'group_by_stop_idx': self.group_by_stop_idx,
+        return {
+            'fn': self.fn.as_dict(),
+            'return_type': self.return_type.as_dict(),
+            'arg_idxs': self.arg_idxs,
+            'kwarg_idxs': self.kwarg_idxs,
+            'group_by_start_idx': self.group_by_start_idx,
+            'group_by_stop_idx': self.group_by_stop_idx,
             'order_by_start_idx': self.order_by_start_idx,
-            **super()._as_dict()
+            'is_method_call': self.is_method_call,
+            **super()._as_dict(),
         }
-        return result
 
     @classmethod
     def _from_dict(cls, d: dict, components: list[Expr]) -> FunctionCall:
-        assert 'fn' in d
-        assert 'args' in d
-        assert 'kwargs' in d
-        # reassemble bound args
         fn = func.Function.from_dict(d['fn'])
-        param_names = list(fn.signature.parameters.keys())
-        bound_args = {param_names[i]: arg if idx is None else components[idx] for i, (idx, arg) in enumerate(d['args'])}
-        bound_args.update(
-            {param_name: val if idx is None else components[idx] for param_name, (idx, val) in d['kwargs'].items()})
-        group_by_exprs = components[d['group_by_start_idx']:d['group_by_stop_idx']]
-        order_by_exprs = components[d['order_by_start_idx']:]
+        return_type = ts.ColumnType.from_dict(d['return_type']) if 'return_type' in d else None
+        arg_idxs: list[int] = d['arg_idxs']
+        kwarg_idxs: dict[str, int] = d['kwarg_idxs']
+        group_by_start_idx: int = d['group_by_start_idx']
+        group_by_stop_idx: int = d['group_by_stop_idx']
+        order_by_start_idx: int = d['order_by_start_idx']
+        is_method_call: bool = d['is_method_call']
+
+        args = [components[idx] for idx in arg_idxs]
+        kwargs = {name: components[idx] for name, idx in kwarg_idxs.items()}
+        group_by_exprs = components[group_by_start_idx:group_by_stop_idx]
+        order_by_exprs = components[order_by_start_idx:]
+
+        validation_error: str | None = None
+
+        if isinstance(fn, func.InvalidFunction):
+            validation_error = (
+                dedent(
+                    f"""
+                    The UDF '{fn.self_path}' cannot be located, because
+                    {{error_msg}}
+                    """
+                )
+                .strip()
+                .format(error_msg=fn.error_msg)
+            )
+            return cls(fn, args, kwargs, return_type, is_method_call=is_method_call, validation_error=validation_error)
+
+        # Now re-bind args and kwargs using the version of `fn` that is currently represented in code. This ensures
+        # that we get a valid binding even if the signatures of `fn` have changed since the FunctionCall was
+        # serialized.
+
+        resolved_fn: func.Function = fn
+
+        try:
+            # Bind args and kwargs to the function signature in the current codebase.
+            resolved_fn, bound_args = fn._bind_to_matching_signature(args, kwargs)
+        except (TypeError, excs.Error):
+            signature_note_str = 'any of its signatures' if fn.is_polymorphic else 'its signature'
+            args_str = [f'pxt.{arg.col_type}' for arg in args]
+            args_str.extend(f'{name}: pxt.{arg.col_type}' for name, arg in kwargs.items())
+            call_signature_str = f'({", ".join(args_str)}) -> pxt.{return_type}'
+            fn_signature_str = f'{len(fn.signatures)} signatures' if fn.is_polymorphic else str(fn.signature)
+            validation_error = dedent(
+                f"""
+                The signature stored in the database for a UDF call to {fn.self_path!r} no longer
+                matches {signature_note_str} as currently defined in the code. This probably means that the
+                code for {fn.self_path!r} has changed in a backward-incompatible way.
+                Signature of UDF call in the database: {call_signature_str}
+                Signature of UDF as currently defined in code: {fn_signature_str}
+                """
+            ).strip()
+        else:
+            # Evaluate the call_return_type as defined in the current codebase.
+            call_return_type: ts.ColumnType | None = None
+
+            if isinstance(resolved_fn, func.ExprTemplateFunction) and not resolved_fn.template.expr.is_valid:
+                # The FunctionCall is based on an ExprTemplateFunction, but the template expression is not valid
+                # (because it in turn contains an invalid FunctionCall). In this case, inherit the validation error
+                # from the template expression.
+                validation_error = resolved_fn.template.expr.validation_error
+            else:
+                try:
+                    call_return_type = resolved_fn.call_return_type(bound_args)
+                except ImportError as exc:
+                    validation_error = dedent(
+                        f"""
+                        A UDF call to {fn.self_path!r} could not be fully resolved, because a module required
+                        by the UDF could not be imported:
+                        {exc}
+                        """
+                    )
+
+            assert (call_return_type is None) != (validation_error is None)
+
+            if call_return_type is None and return_type is None:
+                # Schema versions prior to 25 did not store the return_type in metadata, and there is no obvious
+                # way to infer it during DB migration, so we might encounter a stored return_type of None. If the
+                # resolution of call_return_type also fails, then we're out of luck; we have no choice but to
+                # fail-fast.
+                raise excs.Error(validation_error)
+
+            if call_return_type is not None:
+                # call_return_type resolution succeeded.
+                if return_type is None:
+                    # Schema versions prior to 25 did not store the return_type in metadata (as mentioned above), so
+                    # fall back on the call_return_type.
+                    return_type = call_return_type
+                elif not return_type.is_supertype_of(call_return_type, ignore_nullable=True):
+                    # There is a return_type stored in metadata (schema version >= 25),
+                    # and the stored return_type of the UDF call doesn't match the column type of the FunctionCall.
+                    validation_error = dedent(
+                        f"""
+                        The return type stored in the database for a UDF call to {fn.self_path!r} no longer
+                        matches its return type as currently defined in the code. This probably means that the
+                        code for {fn.self_path!r} has changed in a backward-incompatible way.
+                        Return type of UDF call in the database: {return_type}
+                        Return type of UDF as currently defined in code: {call_return_type}
+                        """
+                    ).strip()
+
+        assert return_type is not None  # Guaranteed by the above logic.
+
         fn_call = cls(
-            func.Function.from_dict(d['fn']), bound_args, group_by_clause=group_by_exprs,
-            order_by_clause=order_by_exprs)
+            resolved_fn,
+            args,
+            kwargs,
+            return_type,
+            group_by_clause=group_by_exprs,
+            order_by_clause=order_by_exprs,
+            is_method_call=is_method_call,
+            validation_error=validation_error,
+        )
+
         return fn_call