ob-metaflow 2.15.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

metaflow/runner/click_api.py

@@ -43,10 +43,10 @@ from metaflow._vendor.click.types import (
 )
 from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
+from metaflow.flowspec import FlowStateItems
 from metaflow.includefile import FilePathClass
 from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 from metaflow.parameters import JSONTypeClass, flow_context
-from metaflow.user_configs.config_decorators import CustomFlowDecorator
 from metaflow.user_configs.config_options import (
     ConfigValue,
     ConvertDictOrStr,
@@ -55,6 +55,7 @@ from metaflow.user_configs.config_options import (
     MultipleTuple,
     config_options_with_config_input,
 )
+from metaflow.user_decorators.user_flow_decorator import FlowMutator
 
 # Define a recursive type alias for JSON
 JSON = Union[Dict[str, "JSON"], List["JSON"], str, int, float, bool, None]
@@ -165,13 +166,23 @@ def _method_sanity_check(
     return method_params
 
 
+def _cleanup_flow_parameters(cmd_obj: Union[click.Command, click.Group]):
+    if hasattr(cmd_obj, "original_params"):
+        cmd_obj.params = list(cmd_obj.original_params)
+
+    if isinstance(cmd_obj, click.Group):
+        for sub_cmd_name in cmd_obj.list_commands(None):
+            sub_cmd = cmd_obj.get_command(None, sub_cmd_name)
+            if sub_cmd:
+                _cleanup_flow_parameters(sub_cmd)
+
+
 def _lazy_load_command(
     cli_collection: click.Group,
     flow_parameters: Union[str, List[Parameter]],
     _self,
     name: str,
 ):
-
     # Context is not used in get_command so we can pass None. Since we pin click,
     # this won't change from under us.
 
@@ -181,6 +192,7 @@ def _lazy_load_command(
         flow_parameters = getattr(_self, flow_parameters)()
     cmd_obj = cli_collection.get_command(None, name)
     if cmd_obj:
+        _cleanup_flow_parameters(cmd_obj)
         if isinstance(cmd_obj, click.Group):
             # TODO: possibly check for fake groups with cmd_obj.name in ["cli", "main"]
             result = functools.partial(extract_group(cmd_obj, flow_parameters), _self)
@@ -264,12 +276,12 @@ def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
             loaded_modules[flow_file] = module
 
         classes = inspect.getmembers(
-            module, lambda x: inspect.isclass(x) or isinstance(x, CustomFlowDecorator)
+            module, lambda x: inspect.isclass(x) or isinstance(x, FlowMutator)
         )
         flow_cls = None
 
         for _, kls in classes:
-            if isinstance(kls, CustomFlowDecorator):
+            if isinstance(kls, FlowMutator):
                 kls = kls._flow_cls
             if (
                 kls is not FlowSpec
@@ -339,8 +351,10 @@ class MetaflowAPI(object):
         class_dict = {
             "__module__": "metaflow",
             "_API_NAME": flow_file,
-            "_internal_getattr": functools.partial(
-                _lazy_load_command, cli_collection, "_compute_flow_parameters"
+            "_internal_getattr": staticmethod(
+                functools.partial(
+                    _lazy_load_command, cli_collection, "_compute_flow_parameters"
+                )
             ),
             "__getattr__": getattr_wrapper,
         }
@@ -460,31 +474,62 @@ class MetaflowAPI(object):
 
                 ds = opts.get("datastore", defaults["datastore"])
                 quiet = opts.get("quiet", defaults["quiet"])
+
+                # Order to find config or config_value:
+                # 1. Passed directly to the Click API
+                # 2. If not found, check if passed through an environment variable
+                # 3. If not found, use the default value
                 is_default = False
                 config_file = opts.get("config")
                 if config_file is None:
-                    is_default = True
-                    config_file = defaults.get("config")
+                    # Check if it was set through an environment variable -- we
+                    # don't have click process them here so we need to "fake" it.
+                    env_config_file = os.environ.get("METAFLOW_FLOW_CONFIG")
+                    if env_config_file:
+                        # Convert dict items to list of tuples
+                        config_file = list(json.loads(env_config_file).items())
+                        is_default = False
+                    else:
+                        is_default = True
+                        config_file = defaults.get("config")
 
                 if config_file:
-                    config_file = map(
-                        lambda x: (x[0], ConvertPath.convert_value(x[1], is_default)),
-                        config_file,
+                    config_file = dict(
+                        map(
+                            lambda x: (
+                                x[0],
+                                ConvertPath.convert_value(x[1], is_default),
+                            ),
+                            config_file,
+                        )
                     )
 
                 is_default = False
                 config_value = opts.get("config-value")
                 if config_value is None:
-                    is_default = True
-                    config_value = defaults.get("config_value")
+                    env_config_value = os.environ.get("METAFLOW_FLOW_CONFIG_VALUE")
+                    if env_config_value:
+                        # Parse environment variable using MultipleTuple logic
+                        loaded = json.loads(env_config_value)
+                        # Convert dict items to list of tuples with JSON-serialized values
+                        config_value = [
+                            (k, json.dumps(v) if not isinstance(v, str) else v)
+                            for k, v in loaded.items()
+                        ]
+                        is_default = False
+                    else:
+                        is_default = True
+                        config_value = defaults.get("config_value")
 
                 if config_value:
-                    config_value = map(
-                        lambda x: (
-                            x[0],
-                            ConvertDictOrStr.convert_value(x[1], is_default),
-                        ),
-                        config_value,
+                    config_value = dict(
+                        map(
+                            lambda x: (
+                                x[0],
+                                ConvertDictOrStr.convert_value(x[1], is_default),
+                            ),
+                            config_value,
+                        )
                     )
 
                 if (config_file is None) ^ (config_value is None):
@@ -505,10 +550,15 @@ class MetaflowAPI(object):
 
         # At this point, we are like in start() in cli.py -- we obtained the
         # properly processed config_options which we can now use to process
-        # the config decorators (including CustomStep/FlowDecorators)
+        # the config decorators (including StepMutator/FlowMutator)
         # Note that if CLICK_API_PROCESS_CONFIG is False, we still do this because
         # it will init all parameters (config_options will be None)
         # We ignore any errors if we don't check the configs in the click API.
+
+        # Init all values in the flow mutators and then process them
+        for decorator in self._flow_cls._flow_state[FlowStateItems.FLOW_MUTATORS]:
+            decorator.external_init()
+
         new_cls = self._flow_cls._process_config_decorators(
             config_options, process_configs=CLICK_API_PROCESS_CONFIG
         )
@@ -534,14 +584,16 @@ def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
 
     for each_param in cmd_obj.params:
         if isinstance(each_param, click.Argument):
-            arg_params_sigs[each_param.name], annotations[each_param.name] = (
-                get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
-            )
+            (
+                arg_params_sigs[each_param.name],
+                annotations[each_param.name],
+            ) = get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
             arg_parameters[each_param.name] = each_param
         elif isinstance(each_param, click.Option):
-            opt_params_sigs[each_param.name], annotations[each_param.name] = (
-                get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
-            )
+            (
+                opt_params_sigs[each_param.name],
+                annotations[each_param.name],
+            ) = get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
             opt_parameters[each_param.name] = each_param
 
         defaults[each_param.name] = each_param.default
@@ -651,6 +703,7 @@ if __name__ == "__main__":
         .kubernetes()
         .step(
             step_name="process",
+            code_package_metadata="some_version",
             code_package_sha="some_sha",
             code_package_url="some_url",
         )

metaflow/runner/deployer.py

@@ -7,6 +7,57 @@ from typing import ClassVar, Dict, Optional, TYPE_CHECKING
 from metaflow.exception import MetaflowNotFound
 from metaflow.metaflow_config import DEFAULT_FROM_DEPLOYMENT_IMPL
 
+
+def generate_fake_flow_file_contents(
+    flow_name: str, param_info: dict, project_name: Optional[str] = None
+):
+    params_code = ""
+    for _, param_details in param_info.items():
+        param_python_var_name = param_details.get(
+            "python_var_name", param_details["name"]
+        )
+        param_name = param_details["name"]
+        param_type = param_details["type"]
+        param_help = param_details["description"]
+        param_required = param_details["is_required"]
+
+        if param_type == "JSON":
+            params_code += (
+                f"    {param_python_var_name} = Parameter('{param_name}', "
+                f"type=JSONType, help='''{param_help}''', required={param_required})\n"
+            )
+        elif param_type == "FilePath":
+            is_text = param_details.get("is_text", True)
+            encoding = param_details.get("encoding", "utf-8")
+            params_code += (
+                f"    {param_python_var_name} = IncludeFile('{param_name}', "
+                f"is_text={is_text}, encoding='{encoding}', help='''{param_help}''', "
+                f"required={param_required})\n"
+            )
+        else:
+            params_code += (
+                f"    {param_python_var_name} = Parameter('{param_name}', "
+                f"type={param_type}, help='''{param_help}''', required={param_required})\n"
+            )
+
+    project_decorator = f"@project(name='{project_name}')\n" if project_name else ""
+
+    contents = f"""\
+from metaflow import FlowSpec, Parameter, IncludeFile, JSONType, step, project
+{project_decorator}class {flow_name}(FlowSpec):
+{params_code}
+    @step
+    def start(self):
+        self.next(self.end)
+    @step
+    def end(self):
+        pass
+if __name__ == '__main__':
+    {flow_name}()
+"""
+    return contents
+
+
 if TYPE_CHECKING:
     import metaflow
     import metaflow.runner.deployer_impl
@@ -180,7 +231,68 @@ class DeployedFlowMeta(type):
                 }
             )
 
-            def _default_injected_method():
+            def _get_triggered_run_injected_method():
+                def f(
+                    cls,
+                    identifier: str,
+                    run_id: str,
+                    metadata: Optional[str] = None,
+                    impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
+                ) -> "TriggeredRun":
+                    """
+                    Retrieves a `TriggeredRun` object from an identifier, a run id and optional
+                    metadata. The `impl` parameter specifies the deployer implementation
+                    to use (like `argo-workflows`).
+
+                    Parameters
+                    ----------
+                    identifier : str
+                        Deployer specific identifier for the workflow to retrieve
+                    run_id : str
+                        Run ID for the which to fetch the triggered run object
+                    metadata : str, optional, default None
+                        Optional deployer specific metadata.
+                    impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
+                        The default implementation to use if not specified
+
+                    Returns
+                    -------
+                    TriggeredRun
+                        A `TriggeredRun` object representing the triggered run corresponding
+                        to the identifier and the run id.
+                    """
+                    if impl in allowed_providers:
+                        return (
+                            allowed_providers[impl]
+                            .deployed_flow_type()
+                            .get_triggered_run(identifier, run_id, metadata)
+                        )
+                    else:
+                        raise ValueError(
+                            f"No deployer '{impl}' exists; valid deployers are: "
+                            f"{list(allowed_providers.keys())}"
+                        )
+
+                f.__name__ = "get_triggered_run"
+                return f
+
+            def _per_type_get_triggered_run_injected_method(method_name, impl):
+                def f(
+                    cls,
+                    identifier: str,
+                    run_id: str,
+                    metadata: Optional[str] = None,
+                ):
+                    return (
+                        allowed_providers[impl]
+                        .deployed_flow_type()
+                        .get_triggered_run(identifier, run_id, metadata)
+                    )
+
+                f.__name__ = method_name
+                return f
+
+            def _from_deployment_injected_method():
                 def f(
                     cls,
                     identifier: str,
@@ -222,7 +334,7 @@ class DeployedFlowMeta(type):
                 f.__name__ = "from_deployment"
                 return f
 
-            def _per_type_injected_method(method_name, impl):
+            def _per_type_from_deployment_injected_method(method_name, impl):
                 def f(
                     cls,
                     identifier: str,
@@ -237,14 +349,104 @@ class DeployedFlowMeta(type):
                 f.__name__ = method_name
                 return f
 
-            setattr(cls, "from_deployment", classmethod(_default_injected_method()))
+            def _list_deployed_flows_injected_method():
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                    impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
+                ):
+                    """
+                    List all deployed flows for the specified implementation.
+
+                    Parameters
+                    ----------
+                    flow_name : str, optional, default None
+                        If specified, only list deployed flows for this specific flow name.
+                        If None, list all deployed flows.
+                    impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
+                        The default implementation to use if not specified
+
+                    Yields
+                    ------
+                    DeployedFlow
+                        `DeployedFlow` objects representing deployed flows.
+                    """
+                    if impl in allowed_providers:
+                        return (
+                            allowed_providers[impl]
+                            .deployed_flow_type()
+                            .list_deployed_flows(flow_name)
+                        )
+                    else:
+                        raise ValueError(
+                            f"No deployer '{impl}' exists; valid deployers are: "
+                            f"{list(allowed_providers.keys())}"
+                        )
+
+                f.__name__ = "list_deployed_flows"
+                return f
+
+            def _per_type_list_deployed_flows_injected_method(method_name, impl):
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                ):
+                    return (
+                        allowed_providers[impl]
+                        .deployed_flow_type()
+                        .list_deployed_flows(flow_name)
+                    )
+
+                f.__name__ = method_name
+                return f
+
+            setattr(
+                cls, "from_deployment", classmethod(_from_deployment_injected_method())
+            )
+            setattr(
+                cls,
+                "list_deployed_flows",
+                classmethod(_list_deployed_flows_injected_method()),
+            )
+            setattr(
+                cls,
+                "get_triggered_run",
+                classmethod(_get_triggered_run_injected_method()),
+            )
 
             for impl in allowed_providers:
-                method_name = f"from_{impl}"
+                from_deployment_method_name = f"from_{impl}"
+                list_deployed_flows_method_name = f"list_{impl}"
+                get_triggered_run_method_name = f"get_triggered_{impl}_run"
+
+                setattr(
+                    cls,
+                    from_deployment_method_name,
+                    classmethod(
+                        _per_type_from_deployment_injected_method(
+                            from_deployment_method_name, impl
+                        )
+                    ),
+                )
+
                 setattr(
                     cls,
-                    method_name,
-                    classmethod(_per_type_injected_method(method_name, impl)),
+                    list_deployed_flows_method_name,
+                    classmethod(
+                        _per_type_list_deployed_flows_injected_method(
+                            list_deployed_flows_method_name, impl
+                        )
+                    ),
+                )
+
+                setattr(
+                    cls,
+                    get_triggered_run_method_name,
+                    classmethod(
+                        _per_type_get_triggered_run_injected_method(
+                            get_triggered_run_method_name, impl
+                        )
+                    ),
                 )
 
         return cls

metaflow/runner/deployer_impl.py

@@ -3,10 +3,12 @@ import json
 import os
 import sys
 
-from typing import Any, ClassVar, Dict, Optional, TYPE_CHECKING, Type
+from typing import Any, ClassVar, Dict, Optional, TYPE_CHECKING, Type, List
+
+from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 
 from .subprocess_manager import SubprocessManager
-from .utils import get_lower_level_group, handle_timeout, temporary_fifo
+from .utils import get_lower_level_group, handle_timeout, temporary_fifo, with_dir
 
 if TYPE_CHECKING:
     import metaflow.runner.deployer
@@ -65,15 +67,10 @@ class DeployerImpl(object):
 
         # Reload the CLI with an "empty" flow -- this will remove any configuration
         # and parameter options. They are re-added in from_cli (called below).
-        to_reload = [
-            "metaflow.cli",
-            "metaflow.cli_components.run_cmds",
-            "metaflow.cli_components.init_cmd",
-        ]
         with flow_context(None):
             [
                 importlib.reload(sys.modules[module])
-                for module in to_reload
+                for module in self.to_reload
                 if module in sys.modules
             ]
 
@@ -88,7 +85,7 @@ class DeployerImpl(object):
         self.show_output = show_output
         self.profile = profile
         self.env = env
-        self.cwd = cwd
+        self.cwd = cwd or os.getcwd()
         self.file_read_timeout = file_read_timeout
 
         self.env_vars = os.environ.copy()
@@ -100,6 +97,19 @@ class DeployerImpl(object):
         self.top_level_kwargs = kwargs
         self.api = MetaflowAPI.from_cli(self.flow_file, start)
 
+    @property
+    def to_reload(self) -> List[str]:
+        """
+        List of modules to reload when the deployer is initialized.
+        This is used to ensure that the CLI is in a clean state before
+        deploying the flow.
+        """
+        return [
+            "metaflow.cli",
+            "metaflow.cli_components.run_cmds",
+            "metaflow.cli_components.init_cmd",
+        ]
+
     @property
     def deployer_kwargs(self) -> Dict[str, Any]:
         raise NotImplementedError
@@ -140,9 +150,19 @@ class DeployerImpl(object):
     ) -> "metaflow.runner.deployer.DeployedFlow":
         with temporary_fifo() as (attribute_file_path, attribute_file_fd):
             # every subclass needs to have `self.deployer_kwargs`
-            command = get_lower_level_group(
-                self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
-            ).create(deployer_attribute_file=attribute_file_path, **kwargs)
+            # TODO: Get rid of CLICK_API_PROCESS_CONFIG in the near future
+            if CLICK_API_PROCESS_CONFIG:
+                # We need to run this in the cwd because configs depend on files
+                # that may be located in paths relative to the directory the user
+                # wants to run in
+                with with_dir(self.cwd):
+                    command = get_lower_level_group(
+                        self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
+                    ).create(deployer_attribute_file=attribute_file_path, **kwargs)
+            else:
+                command = get_lower_level_group(
+                    self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
+                ).create(deployer_attribute_file=attribute_file_path, **kwargs)
 
             pid = self.spm.run_command(
                 [sys.executable, *command],