metaflow-functions 0.2.1.dev1__tar.gz → 0.2.3__tar.gz

metaflow_functions-0.2.3/MANIFEST.in

@@ -0,0 +1,2 @@
+include VERSION
+include README.md

{metaflow_functions-0.2.1.dev1/metaflow_functions.egg-info → metaflow_functions-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow-functions
-Version: 0.2.1.dev1
+Version: 0.2.3
 Summary: Metaflow Functions
 Author: Netflix Metaflow Developers
 Author-email: metaflow-dev@netflix.com

metaflow_functions-0.2.3/VERSION

@@ -0,0 +1 @@
+0.2.3

{metaflow_functions-0.2.1.dev1 → metaflow_functions-0.2.3}/metaflow_extensions/nflx/plugins/functions/core/function_parameters.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import types
 from collections.abc import Iterator, Mapping
-from typing import TYPE_CHECKING, Any, Dict, Optional, TypeVar, Generic
+from typing import TYPE_CHECKING, Any, Dict, Optional, Type, TypeVar, Generic
 
 if TYPE_CHECKING:
     from metaflow import Task
@@ -92,6 +92,7 @@ class LazyArtifactMapping(Mapping[str, Any]):
 
 
 T_FP_co = TypeVar("T_FP_co", covariant=True)
+T_FP = TypeVar("T_FP", bound="FunctionParameters")
 
 
 class FunctionParameters(Generic[T_FP_co]):
@@ -414,3 +415,28 @@ class FunctionParameters(Generic[T_FP_co]):
     def __len__(self) -> int:
         """Return number of artifacts."""
         return len(self._lazy_mapping)
+
+    def as_typed(self, typed_class: Type[T_FP]) -> T_FP:
+        """
+        Create a typed view of this FunctionParameters instance.
+
+        The new instance is pre-populated from this instance's already-fetched
+        cache, so previously prefetched artifacts are not re-fetched. Any
+        artifacts not yet in the cache are loaded lazily from the same
+        function spec.
+
+        Parameters
+        ----------
+        typed_class : Type[FunctionParameters]
+            A FunctionParameters subclass to instantiate.
+
+        Returns
+        -------
+        FunctionParameters
+            An instance of typed_class pre-populated from this instance's cache.
+        """
+        cached = dict(self._lazy_mapping._cache)
+        return typed_class(
+            function_spec=self._lazy_mapping._function_spec,
+            **cached,
+        )

{metaflow_functions-0.2.1.dev1 → metaflow_functions-0.2.3}/metaflow_extensions/nflx/plugins/functions/core/function_pipeline.py

@@ -33,6 +33,10 @@ from metaflow_extensions.nflx.plugins.functions.exceptions import (
     MetaflowFunctionTypeException,
     MetaflowFunctionUserException,
 )
+from metaflow_extensions.nflx.plugins.functions.core.function_parameters import (
+    FunctionParameters,
+)
+from metaflow_extensions.nflx.plugins.functions.utils import load_type_from_string
 
 if TYPE_CHECKING:
     from metaflow_extensions.nflx.plugins.functions.memory.memory import (
@@ -68,6 +72,9 @@ class FunctionPipeline(MetaflowFunction):
 
         self.functions = functions
         self._name = name
+        self._scoped_params: Optional[List["FunctionParameters"]] = (
+            None  # populated on first execute call
+        )
 
         # Pipeline creates its own backend instance
         from metaflow_extensions.nflx.plugins.functions.backends import get_backend
@@ -172,8 +179,14 @@ class FunctionPipeline(MetaflowFunction):
                 self.functions[0].spec.task_code_path if self.functions else None
             ),  # Use first function's task code path
             artifacts=(
-                self.functions[0].spec.artifacts if self.functions else None
-            ),  # Use first function's artifacts metadata
+                {
+                    k: v
+                    for func in self.functions
+                    if func.spec.artifacts
+                    for k, v in func.spec.artifacts.items()
+                }
+                or None
+            ),  # Union of all constituent functions' artifacts metadata
             user=_lazy_get_username(),
             timestamp_utc=int(time.time()),
             system_metadata={
@@ -239,6 +252,44 @@ class FunctionPipeline(MetaflowFunction):
         """
         return self.backend.apply(self, data, **kwargs)
 
+    def _make_func_params(
+        self, pipeline_params: "FunctionParameters", func: MetaflowFunction
+    ) -> "FunctionParameters":
+        """
+        Create a typed FunctionParameters instance for a constituent function.
+
+        Uses the function's parameter_schema to instantiate the correct typed class,
+        pre-populated from the pipeline-level cache (already prefetched). This means
+        isinstance checks work correctly and no re-fetching is needed.
+
+        Returns pipeline params as-is for Optional[FunctionParameters] and base
+        FunctionParameters functions, and a typed instance for typed subclasses.
+        """
+        param_schema = None
+        if (
+            func.spec
+            and func.spec.function
+            and hasattr(func.spec.function, "parameter_schema")
+        ):
+            param_schema = func.spec.function.parameter_schema
+
+        # Optional[FunctionParameters] — artifacts not required but pass through anyway;
+        # the function can check and use them if present
+        if param_schema is None:
+            return pipeline_params
+
+        # Base FunctionParameters — pass pipeline params as-is, all artifacts accessible
+        if not param_schema or "type" not in param_schema:
+            return pipeline_params
+
+        # Typed subclass — create a typed view so isinstance works and
+        # no re-fetching occurs for already-prefetched artifacts
+        typed_class = load_type_from_string(param_schema["type"])
+        if typed_class is None or not issubclass(typed_class, FunctionParameters):
+            return pipeline_params
+
+        return pipeline_params.as_typed(typed_class)
+
     def execute(self, data: Any, params: "FunctionParameters", **kwargs: Any) -> Any:  # type: ignore
         """
         Execute the pipeline by chaining all functions in the same process.
@@ -257,10 +308,16 @@ class FunctionPipeline(MetaflowFunction):
         Any
             The result of the pipeline execution
         """
-        # Chain all functions together in the same process
+        # Per-function params are computed once on the first call and reused —
+        # params is the same object for every request in a runtime lifetime.
+        if self._scoped_params is None:
+            self._scoped_params = [
+                self._make_func_params(params, func) for func in self.functions
+            ]
+
         result = data
-        for func in self.functions:
-            result = func.execute(result, params, **kwargs)
+        for func, func_params in zip(self.functions, self._scoped_params):
+            result = func.execute(result, func_params, **kwargs)
         return result
 
     def is_compatible_with(self, other: MetaflowFunction) -> bool:
@@ -522,6 +579,7 @@ class FunctionPipeline(MetaflowFunction):
         if spec.uuid is None:
             raise MetaflowFunctionException("Pipeline spec is missing a uuid.")
         pipeline._name = spec.name
+        pipeline._scoped_params = None  # type: ignore[assignment]  # populated on first execute call
 
         # Pipeline creates its own backend instance
         from metaflow_extensions.nflx.plugins.functions.backends import get_backend

{metaflow_functions-0.2.1.dev1 → metaflow_functions-0.2.3}/metaflow_extensions/nflx/plugins/functions/factory.py

@@ -324,10 +324,15 @@ def create_function_type(
             type_hints = get_type_hints(self.func)
             parameter_schema = None
 
+            from metaflow_extensions.nflx.plugins.functions.core.function_parameters import (
+                FunctionParameters,
+            )
+
             params_type = type_hints.get("params")
             if params_type and is_optional_function_parameters_type(params_type):
                 # Extract the actual concrete type, unwrapping Optional if needed
                 concrete_type = params_type
+                is_optional_wrapper = False
                 if get_origin(params_type) is Union:
                     # Handle Optional[T] which is Union[T, None]
                     args = get_args(params_type)
@@ -335,8 +340,14 @@ def create_function_type(
                         if arg is not type(None):
                             concrete_type = arg
                             break
+                    is_optional_wrapper = True
 
-                parameter_schema = {"type": get_type_string(concrete_type)}
+                # Optional[FunctionParameters] (base class) signals no artifacts needed.
+                # A typed subclass or bare FunctionParameters still carries artifacts.
+                if is_optional_wrapper and concrete_type is FunctionParameters:
+                    parameter_schema = None
+                else:
+                    parameter_schema = {"type": get_type_string(concrete_type)}
 
             return GeneratedDecoratorSpec(
                 name=self.func.__name__,
@@ -401,6 +412,36 @@ def create_function_type(
             """Build function spec and populate serializer configs."""
             func_spec = super()._build_function_spec(**kwargs)
 
+            # Filter artifacts based on what the function's FunctionParameters declares:
+            #   - parameter_schema is None (Optional[FunctionParameters]): no artifacts needed
+            #   - parameter_schema has a typed subclass: only declared artifacts
+            #   - parameter_schema has base FunctionParameters: keep all artifacts
+            if func_spec.function is not None:
+                if func_spec.function.parameter_schema is None:
+                    # Optional[FunctionParameters] — function needs no task artifacts
+                    func_spec.artifacts = {}
+                elif "type" in func_spec.function.parameter_schema:
+                    from metaflow_extensions.nflx.plugins.functions.utils import (
+                        load_type_from_string,
+                    )
+                    from metaflow_extensions.nflx.plugins.functions.core.function_parameters import (
+                        FunctionParameters,
+                    )
+
+                    params_class = load_type_from_string(
+                        func_spec.function.parameter_schema["type"]
+                    )
+                    if (
+                        params_class is not None
+                        and issubclass(params_class, FunctionParameters)
+                        and params_class._expected_artifacts is not None
+                    ):
+                        func_spec.artifacts = {
+                            k: v
+                            for k, v in (func_spec.artifacts or {}).items()
+                            if k in params_class._expected_artifacts
+                        }
+
             # Add serializer configs for this function's types
             from metaflow_extensions.nflx.plugins.functions.serializers.registry import (
                 get_global_registry,

metaflow_functions-0.2.3/metaflow_extensions/nflx/toplevel/functions_version.py

@@ -0,0 +1 @@
+_ext_version = "0.2.3"

{metaflow_functions-0.2.1.dev1 → metaflow_functions-0.2.3/metaflow_functions.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow-functions
-Version: 0.2.1.dev1
+Version: 0.2.3
 Summary: Metaflow Functions
 Author: Netflix Metaflow Developers
 Author-email: metaflow-dev@netflix.com

{metaflow_functions-0.2.1.dev1 → metaflow_functions-0.2.3}/metaflow_functions.egg-info/SOURCES.txt

@@ -1,5 +1,7 @@
 LICENSE
+MANIFEST.in
 README.md
+VERSION
 setup.py
 metaflow_extensions/nflx/config/mfextinit_functions.py
 metaflow_extensions/nflx/plugins/mfextinit_functions.py

metaflow_functions-0.2.1.dev1/metaflow_extensions/nflx/toplevel/functions_version.py

@@ -1 +0,0 @@
-_ext_version = "0.2.1.dev1"