metaflow 2.12.36__py2.py3-none-any.whl → 2.12.38__py2.py3-none-any.whl

metaflow/plugins/argo/argo_workflows.py

@@ -60,6 +60,7 @@ from metaflow.plugins.kubernetes.kubernetes import (
 )
 from metaflow.plugins.kubernetes.kubernetes_jobsets import KubernetesArgoJobSet
 from metaflow.unbounded_foreach import UBF_CONTROL, UBF_TASK
+from metaflow.user_configs.config_options import ConfigInput
 from metaflow.util import (
     compress_list,
     dict_to_cli_options,
@@ -168,6 +169,7 @@ class ArgoWorkflows(object):
         self.enable_heartbeat_daemon = enable_heartbeat_daemon
         self.enable_error_msg_capture = enable_error_msg_capture
         self.parameters = self._process_parameters()
+        self.config_parameters = self._process_config_parameters()
         self.triggers, self.trigger_options = self._process_triggers()
         self._schedule, self._timezone = self._get_schedule()
 
@@ -455,6 +457,10 @@ class ArgoWorkflows(object):
                     "case-insensitive." % param.name
                 )
             seen.add(norm)
+            # NOTE: We skip config parameters as these do not have dynamic values,
+            # and need to be treated differently.
+            if param.IS_CONFIG_PARAMETER:
+                continue
 
             extra_attrs = {}
             if param.kwargs.get("type") == JSONType:
@@ -488,6 +494,7 @@ class ArgoWorkflows(object):
             # execution - which needs to be fixed imminently.
             if not is_required or default_value is not None:
                 default_value = json.dumps(default_value)
+
             parameters[param.name] = dict(
                 name=param.name,
                 value=default_value,
@@ -498,6 +505,27 @@ class ArgoWorkflows(object):
             )
         return parameters
 
+    def _process_config_parameters(self):
+        parameters = []
+        seen = set()
+        for var, param in self.flow._get_parameters():
+            if not param.IS_CONFIG_PARAMETER:
+                continue
+            # Throw an exception if the parameter is specified twice.
+            norm = param.name.lower()
+            if norm in seen:
+                raise MetaflowException(
+                    "Parameter *%s* is specified twice. "
+                    "Note that parameter names are "
+                    "case-insensitive." % param.name
+                )
+            seen.add(norm)
+
+            parameters.append(
+                dict(name=param.name, kv_name=ConfigInput.make_key_name(param.name))
+            )
+        return parameters
+
     def _process_triggers(self):
         # Impute triggers for Argo Workflow Template specified through @trigger and
         # @trigger_on_finish decorators
@@ -520,8 +548,13 @@ class ArgoWorkflows(object):
             # convert them to lower case since Metaflow parameters are case
             # insensitive.
             seen = set()
+            # NOTE: We skip config parameters as their values can not be set through event payloads
             params = set(
-                [param.name.lower() for var, param in self.flow._get_parameters()]
+                [
+                    param.name.lower()
+                    for var, param in self.flow._get_parameters()
+                    if not param.IS_CONFIG_PARAMETER
+                ]
             )
             trigger_deco = self.flow._flow_decorators.get("trigger")[0]
             trigger_deco.format_deploytime_value()
@@ -1720,6 +1753,13 @@ class ArgoWorkflows(object):
             metaflow_version["production_token"] = self.production_token
             env["METAFLOW_VERSION"] = json.dumps(metaflow_version)
 
+            # map config values
+            cfg_env = {
+                param["name"]: param["kv_name"] for param in self.config_parameters
+            }
+            if cfg_env:
+                env["METAFLOW_FLOW_CONFIG_VALUE"] = json.dumps(cfg_env)
+
             # Set the template inputs and outputs for passing state. Very simply,
             # the container template takes in input-paths as input and outputs
             # the task-id (which feeds in as input-paths to the subsequent task).

metaflow/plugins/argo/argo_workflows_cli.py

@@ -470,6 +470,7 @@ def make_flow(
     decorators._attach_decorators(
         obj.flow, [KubernetesDecorator.name, EnvironmentDecorator.name]
     )
+    decorators._init(obj.flow)
 
     decorators._init_step_decorators(
         obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger

metaflow/plugins/aws/batch/batch_decorator.py

@@ -138,8 +138,8 @@ class BatchDecorator(StepDecorator):
     supports_conda_environment = True
     target_platform = "linux-64"
 
-    def __init__(self, attributes=None, statically_defined=False):
-        super(BatchDecorator, self).__init__(attributes, statically_defined)
+    def init(self):
+        super(BatchDecorator, self).init()
 
         # If no docker image is explicitly specified, impute a default image.
         if not self.attributes["image"]:

metaflow/plugins/aws/step_functions/step_functions.py

@@ -18,6 +18,7 @@ from metaflow.metaflow_config import (
     SFN_S3_DISTRIBUTED_MAP_OUTPUT_PATH,
 )
 from metaflow.parameters import deploy_time_eval
+from metaflow.user_configs.config_options import ConfigInput
 from metaflow.util import dict_to_cli_options, to_pascalcase
 
 from ..batch.batch import Batch
@@ -71,6 +72,7 @@ class StepFunctions(object):
         self.username = username
         self.max_workers = max_workers
         self.workflow_timeout = workflow_timeout
+        self.config_parameters = self._process_config_parameters()
 
         # https://aws.amazon.com/blogs/aws/step-functions-distributed-map-a-serverless-solution-for-large-scale-parallel-data-processing/
         self.use_distributed_map = use_distributed_map
@@ -485,6 +487,10 @@ class StepFunctions(object):
                     "case-insensitive." % param.name
                 )
             seen.add(norm)
+            # NOTE: We skip config parameters as these do not have dynamic values,
+            # and need to be treated differently.
+            if param.IS_CONFIG_PARAMETER:
+                continue
 
             is_required = param.kwargs.get("required", False)
             # Throw an exception if a schedule is set for a flow with required
@@ -501,6 +507,27 @@ class StepFunctions(object):
             parameters.append(dict(name=param.name, value=value))
         return parameters
 
+    def _process_config_parameters(self):
+        parameters = []
+        seen = set()
+        for var, param in self.flow._get_parameters():
+            if not param.IS_CONFIG_PARAMETER:
+                continue
+            # Throw an exception if the parameter is specified twice.
+            norm = param.name.lower()
+            if norm in seen:
+                raise MetaflowException(
+                    "Parameter *%s* is specified twice. "
+                    "Note that parameter names are "
+                    "case-insensitive." % param.name
+                )
+            seen.add(norm)
+
+            parameters.append(
+                dict(name=param.name, kv_name=ConfigInput.make_key_name(param.name))
+            )
+        return parameters
+
     def _batch(self, node):
         attrs = {
             # metaflow.user is only used for setting the AWS Job Name.
@@ -747,6 +774,11 @@ class StepFunctions(object):
         metaflow_version["production_token"] = self.production_token
         env["METAFLOW_VERSION"] = json.dumps(metaflow_version)
 
+        # map config values
+        cfg_env = {param["name"]: param["kv_name"] for param in self.config_parameters}
+        if cfg_env:
+            env["METAFLOW_FLOW_CONFIG_VALUE"] = json.dumps(cfg_env)
+
         # Set AWS DynamoDb Table Name for state tracking for for-eaches.
         # There are three instances when metaflow runtime directly interacts
         # with AWS DynamoDB.

metaflow/plugins/aws/step_functions/step_functions_cli.py

@@ -326,6 +326,7 @@ def make_flow(
 
     # Attach AWS Batch decorator to the flow
     decorators._attach_decorators(obj.flow, [BatchDecorator.name])
+    decorators._init(obj.flow)
     decorators._init_step_decorators(
         obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
     )

metaflow/plugins/datatools/s3/s3op.py

@@ -722,8 +722,8 @@ def cli():
     pass
 
 
-@tracing.cli_entrypoint("s3op/list")
 @cli.command("list", help="List S3 objects")
+@tracing.cli_entrypoint("s3op/list")
 @click.option(
     "--recursive/--no-recursive",
     default=False,
@@ -782,8 +782,8 @@ def lst(
             print(format_result_line(idx, url.prefix, url.url, str(size)))
 
 
-@tracing.cli_entrypoint("s3op/put")
 @cli.command(help="Upload files to S3")
+@tracing.cli_entrypoint("s3op/put")
 @click.option(
     "--file",
     "files",
@@ -977,8 +977,8 @@ def _populate_prefixes(prefixes, inputs):
     return prefixes, is_transient_retry
 
 
-@tracing.cli_entrypoint("s3op/get")
 @cli.command(help="Download files from S3")
+@tracing.cli_entrypoint("s3op/get")
 @click.option(
     "--recursive/--no-recursive",
     default=False,

metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -33,12 +33,12 @@ def kubernetes():
     pass
 
 
-@tracing.cli_entrypoint("kubernetes/step")
 @kubernetes.command(
     help="Execute a single task on Kubernetes. This command calls the top-level step "
     "command inside a Kubernetes pod with the given options. Typically you do not call "
     "this command directly; it is used internally by Metaflow."
 )
+@tracing.cli_entrypoint("kubernetes/step")
 @click.argument("step-name")
 @click.argument("code-package-sha")
 @click.argument("code-package-url")

metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -151,8 +151,8 @@ class KubernetesDecorator(StepDecorator):
     supports_conda_environment = True
     target_platform = "linux-64"
 
-    def __init__(self, attributes=None, statically_defined=False):
-        super(KubernetesDecorator, self).__init__(attributes, statically_defined)
+    def init(self):
+        super(KubernetesDecorator, self).init()
 
         if not self.attributes["namespace"]:
             self.attributes["namespace"] = KUBERNETES_NAMESPACE

metaflow/plugins/pypi/conda_decorator.py

@@ -55,6 +55,17 @@ class CondaStepDecorator(StepDecorator):
         )
         super(CondaStepDecorator, self).__init__(attributes, statically_defined)
 
+    def init(self):
+        super(CondaStepDecorator, self).init()
+
+        # We have to go back and fixup _user_defined_attributes for potential
+        # config resolution
+        self._user_defined_attributes = {
+            k: v
+            for k, v in self.attributes.items()
+            if k in self._user_defined_attributes
+        }
+
         # Support legacy 'libraries=' attribute for the decorator.
         self.attributes["packages"] = {
             **self.attributes["libraries"],
@@ -338,6 +349,17 @@ class CondaFlowDecorator(FlowDecorator):
         )
         super(CondaFlowDecorator, self).__init__(attributes, statically_defined)
 
+    def init(self):
+        super(CondaFlowDecorator, self).init()
+
+        # We have to go back and fixup _user_defined_attributes for potential
+        # config resolution
+        self._user_defined_attributes = {
+            k: v
+            for k, v in self.attributes.items()
+            if k in self._user_defined_attributes
+        }
+
         # Support legacy 'libraries=' attribute for the decorator.
         self.attributes["packages"] = {
             **self.attributes["libraries"],

metaflow/plugins/pypi/pypi_decorator.py

@@ -140,6 +140,7 @@ class PyPIFlowDecorator(FlowDecorator):
         from metaflow import decorators
 
         decorators._attach_decorators(flow, ["pypi"])
+        decorators._init(flow)
 
         # @pypi uses a conda environment to create a virtual environment.
         # The conda environment can be created through micromamba.

metaflow/plugins/timeout_decorator.py

@@ -37,8 +37,8 @@ class TimeoutDecorator(StepDecorator):
     name = "timeout"
     defaults = {"seconds": 0, "minutes": 0, "hours": 0}
 
-    def __init__(self, *args, **kwargs):
-        super(TimeoutDecorator, self).__init__(*args, **kwargs)
+    def init(self):
+        super().init()
         # Initialize secs in __init__ so other decorators could safely use this
         # value without worrying about decorator order.
         # Convert values in attributes to type:int since they can be type:str

metaflow/runner/click_api.py

@@ -9,6 +9,7 @@ if sys.version_info < (3, 7):
     )
 
 import datetime
+import functools
 import importlib
 import inspect
 import itertools
@@ -38,6 +39,7 @@ from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
 from metaflow.includefile import FilePathClass
 from metaflow.parameters import JSONTypeClass, flow_context
+from metaflow.user_configs.config_options import LocalFileInput
 
 # Define a recursive type alias for JSON
 JSON = Union[Dict[str, "JSON"], List["JSON"], str, int, float, bool, None]
@@ -55,6 +57,7 @@ click_to_python_types = {
     File: str,
     JSONTypeClass: JSON,
     FilePathClass: str,
+    LocalFileInput: str,
 }
 
 
@@ -124,6 +127,37 @@ def _method_sanity_check(
     return method_params
 
 
+def _lazy_load_command(
+    cli_collection: click.Group,
+    flow_parameters: Union[str, List[Parameter]],
+    _self,
+    name: str,
+):
+
+    # Context is not used in get_command so we can pass None. Since we pin click,
+    # this won't change from under us.
+
+    if isinstance(flow_parameters, str):
+        # Resolve flow_parameters -- for start, this is a function which we
+        # need to call to figure out the actual parameters (may be changed by configs)
+        flow_parameters = getattr(_self, flow_parameters)()
+    cmd_obj = cli_collection.get_command(None, name)
+    if cmd_obj:
+        if isinstance(cmd_obj, click.Group):
+            # TODO: possibly check for fake groups with cmd_obj.name in ["cli", "main"]
+            result = functools.partial(extract_group(cmd_obj, flow_parameters), _self)
+        elif isinstance(cmd_obj, click.Command):
+            result = functools.partial(extract_command(cmd_obj, flow_parameters), _self)
+        else:
+            raise RuntimeError(
+                "Cannot handle %s of type %s" % (cmd_obj.name, type(cmd_obj))
+            )
+        setattr(_self, name, result)
+        return result
+    else:
+        raise AttributeError()
+
+
 def get_annotation(param: Union[click.Argument, click.Option]):
     py_type = click_to_python_types[type(param.type)]
     if not param.required:
@@ -179,9 +213,11 @@ def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
 
 
 class MetaflowAPI(object):
-    def __init__(self, parent=None, **kwargs):
+    def __init__(self, parent=None, flow_cls=None, **kwargs):
         self._parent = parent
         self._chain = [{self._API_NAME: kwargs}]
+        self._flow_cls = flow_cls
+        self._cached_computed_parameters = None
 
     @property
     def parent(self):
@@ -200,23 +236,22 @@ class MetaflowAPI(object):
     @classmethod
     def from_cli(cls, flow_file: str, cli_collection: Callable) -> Callable:
         flow_cls = extract_flow_class_from_file(flow_file)
-        flow_parameters = [p for _, p in flow_cls._get_parameters()]
+
         with flow_context(flow_cls) as _:
             add_decorator_options(cli_collection)
 
-        class_dict = {"__module__": "metaflow", "_API_NAME": flow_file}
-        command_groups = cli_collection.sources
-        for each_group in command_groups:
-            for _, cmd_obj in each_group.commands.items():
-                if isinstance(cmd_obj, click.Group):
-                    # TODO: possibly check for fake groups with cmd_obj.name in ["cli", "main"]
-                    class_dict[cmd_obj.name] = extract_group(cmd_obj, flow_parameters)
-                elif isinstance(cmd_obj, click.Command):
-                    class_dict[cmd_obj.name] = extract_command(cmd_obj, flow_parameters)
-                else:
-                    raise RuntimeError(
-                        "Cannot handle %s of type %s" % (cmd_obj.name, type(cmd_obj))
-                    )
+        def getattr_wrapper(_self, name):
+            # Functools.partial do not automatically bind self (no __get__)
+            return _self._internal_getattr(_self, name)
+
+        class_dict = {
+            "__module__": "metaflow",
+            "_API_NAME": flow_file,
+            "_internal_getattr": functools.partial(
+                _lazy_load_command, cli_collection, "_compute_flow_parameters"
+            ),
+            "__getattr__": getattr_wrapper,
+        }
 
         to_return = type(flow_file, (MetaflowAPI,), class_dict)
         to_return.__name__ = flow_file
@@ -237,11 +272,11 @@ class MetaflowAPI(object):
                 defaults,
                 **kwargs,
             )
-            return to_return(parent=None, **method_params)
+            return to_return(parent=None, flow_cls=flow_cls, **method_params)
 
         m = _method
-        m.__name__ = cmd_obj.name
-        m.__doc__ = getattr(cmd_obj, "help", None)
+        m.__name__ = cli_collection.name
+        m.__doc__ = getattr(cli_collection, "help", None)
         m.__signature__ = inspect.signature(_method).replace(
             parameters=params_sigs.values()
         )
@@ -287,6 +322,25 @@ class MetaflowAPI(object):
 
         return components
 
+    def _compute_flow_parameters(self):
+        if self._flow_cls is None or self._parent is not None:
+            raise RuntimeError(
+                "Computing flow-level parameters for a non start API. "
+                "Please report to the Metaflow team."
+            )
+        # TODO: We need to actually compute the new parameters (based on configs) which
+        # would involve processing the options at least partially. We will do this
+        # before GA but for now making it work for regular parameters
+        if self._cached_computed_parameters is not None:
+            return self._cached_computed_parameters
+        self._cached_computed_parameters = []
+        for _, param in self._flow_cls._get_parameters():
+            if param.IS_CONFIG_PARAMETER:
+                continue
+            param.init()
+            self._cached_computed_parameters.append(param)
+        return self._cached_computed_parameters
+
 
 def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
     arg_params_sigs = OrderedDict()
@@ -351,7 +405,7 @@ def extract_group(cmd_obj: click.Group, flow_parameters: List[Parameter]) -> Cal
         method_params = _method_sanity_check(
             possible_arg_params, possible_opt_params, annotations, defaults, **kwargs
         )
-        return resulting_class(parent=_self, **method_params)
+        return resulting_class(parent=_self, flow_cls=None, **method_params)
 
     m = _method
     m.__name__ = cmd_obj.name