metaflow 2.12.38__py2.py3-none-any.whl → 2.13__py2.py3-none-any.whl

metaflow/lint.py

@@ -52,7 +52,7 @@ def check_reserved_words(graph):
     msg = "Step name *%s* is a reserved word. Choose another name for the " "step."
     for node in graph:
         if node.name in RESERVED:
-            raise LintWarn(msg % node.name)
+            raise LintWarn(msg % node.name, node.func_lineno, node.source_file)
 
 
 @linter.ensure_fundamentals
@@ -76,9 +76,9 @@ def check_that_end_is_end(graph):
     node = graph["end"]
 
     if node.has_tail_next or node.invalid_tail_next:
-        raise LintWarn(msg0, node.tail_next_lineno)
+        raise LintWarn(msg0, node.tail_next_lineno, node.source_file)
     if node.num_args > 1:
-        raise LintWarn(msg1, node.tail_next_lineno)
+        raise LintWarn(msg1, node.tail_next_lineno, node.source_file)
 
 
 @linter.ensure_fundamentals
@@ -90,7 +90,7 @@ def check_step_names(graph):
     )
     for node in graph:
         if re.search("[^a-z0-9_]", node.name) or node.name[0] == "_":
-            raise LintWarn(msg.format(node), node.func_lineno)
+            raise LintWarn(msg.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_fundamentals
@@ -108,11 +108,11 @@ def check_num_args(graph):
     msg2 = "Step *{0.name}* is missing the 'self' argument."
     for node in graph:
         if node.num_args > 2:
-            raise LintWarn(msg0.format(node), node.func_lineno)
+            raise LintWarn(msg0.format(node), node.func_lineno, node.source_file)
         elif node.num_args == 2 and node.type != "join":
-            raise LintWarn(msg1.format(node), node.func_lineno)
+            raise LintWarn(msg1.format(node), node.func_lineno, node.source_file)
         elif node.num_args == 0:
-            raise LintWarn(msg2.format(node), node.func_lineno)
+            raise LintWarn(msg2.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -125,7 +125,7 @@ def check_static_transitions(graph):
     )
     for node in graph:
         if node.type != "end" and not node.has_tail_next:
-            raise LintWarn(msg.format(node), node.func_lineno)
+            raise LintWarn(msg.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -138,7 +138,7 @@ def check_valid_transitions(graph):
     )
     for node in graph:
         if node.type != "end" and node.has_tail_next and node.invalid_tail_next:
-            raise LintWarn(msg.format(node), node.tail_next_lineno)
+            raise LintWarn(msg.format(node), node.tail_next_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -151,7 +151,11 @@ def check_unknown_transitions(graph):
     for node in graph:
         unknown = [n for n in node.out_funcs if n not in graph]
         if unknown:
-            raise LintWarn(msg.format(node, step=unknown[0]), node.tail_next_lineno)
+            raise LintWarn(
+                msg.format(node, step=unknown[0]),
+                node.tail_next_lineno,
+                node.source_file,
+            )
 
 
 @linter.ensure_acyclicity
@@ -167,7 +171,9 @@ def check_for_acyclicity(graph):
         for n in node.out_funcs:
             if n in seen:
                 path = "->".join(seen + [n])
-                raise LintWarn(msg.format(path), node.tail_next_lineno)
+                raise LintWarn(
+                    msg.format(path), node.tail_next_lineno, node.source_file
+                )
             else:
                 check_path(graph[n], seen + [n])
 
@@ -195,7 +201,7 @@ def check_for_orphans(graph):
     orphans = nodeset - seen
     if orphans:
         orphan = graph[list(orphans)[0]]
-        raise LintWarn(msg.format(orphan), orphan.func_lineno)
+        raise LintWarn(msg.format(orphan), orphan.func_lineno, orphan.source_file)
 
 
 @linter.ensure_static_graph
@@ -230,7 +236,9 @@ def check_split_join_balance(graph):
             if split_stack:
                 _, split_roots = split_stack.pop()
                 roots = ", ".join(split_roots)
-                raise LintWarn(msg0.format(roots=roots))
+                raise LintWarn(
+                    msg0.format(roots=roots), node.func_lineno, node.source_file
+                )
         elif node.type == "join":
             if split_stack:
                 _, split_roots = split_stack[-1]
@@ -243,9 +251,10 @@ def check_split_join_balance(graph):
                             node, paths=paths, num_roots=len(split_roots), roots=roots
                         ),
                         node.func_lineno,
+                        node.source_file,
                     )
             else:
-                raise LintWarn(msg2.format(node), node.func_lineno)
+                raise LintWarn(msg2.format(node), node.func_lineno, node.source_file)
 
             # check that incoming steps come from the same lineage
             # (no cross joins)
@@ -256,7 +265,7 @@ def check_split_join_balance(graph):
                     return tuple(graph[n].split_parents)
 
             if not all_equal(map(parents, node.in_funcs)):
-                raise LintWarn(msg3.format(node), node.func_lineno)
+                raise LintWarn(msg3.format(node), node.func_lineno, node.source_file)
 
         for n in node.out_funcs:
             traverse(graph[n], new_stack)
@@ -276,7 +285,9 @@ def check_empty_foreaches(graph):
         if node.type == "foreach":
             joins = [n for n in node.out_funcs if graph[n].type == "join"]
             if joins:
-                raise LintWarn(msg.format(node, join=joins[0]))
+                raise LintWarn(
+                    msg.format(node, join=joins[0]), node.func_lineno, node.source_file
+                )
 
 
 @linter.ensure_static_graph
@@ -290,7 +301,7 @@ def check_parallel_step_after_next(graph):
         if node.parallel_foreach and not all(
             graph[out_node].parallel_step for out_node in node.out_funcs
         ):
-            raise LintWarn(msg.format(node))
+            raise LintWarn(msg.format(node), node.func_lineno, node.source_file)
 
 
 @linter.ensure_static_graph
@@ -303,7 +314,9 @@ def check_join_followed_by_parallel_step(graph):
     )
     for node in graph:
         if node.parallel_step and not graph[node.out_funcs[0]].type == "join":
-            raise LintWarn(msg.format(node.out_funcs[0]))
+            raise LintWarn(
+                msg.format(node.out_funcs[0]), node.func_lineno, node.source_file
+            )
 
 
 @linter.ensure_static_graph
@@ -318,7 +331,9 @@ def check_parallel_foreach_calls_parallel_step(graph):
             for node2 in graph:
                 if node2.out_funcs and node.name in node2.out_funcs:
                     if not node2.parallel_foreach:
-                        raise LintWarn(msg.format(node, node2))
+                        raise LintWarn(
+                            msg.format(node, node2), node.func_lineno, node.source_file
+                        )
 
 
 @linter.ensure_non_nested_foreach
@@ -331,4 +346,4 @@ def check_nested_foreach(graph):
     for node in graph:
         if node.type == "foreach":
             if any(graph[p].type == "foreach" for p in node.split_parents):
-                raise LintWarn(msg.format(node))
+                raise LintWarn(msg.format(node), node.func_lineno, node.source_file)

metaflow/metaflow_config.py

@@ -508,6 +508,11 @@ DISABLE_TRACING = bool(os.environ.get("DISABLE_TRACING", False))
 # lexicographic ordering of attempts. This won't work if MAX_ATTEMPTS > 99.
 MAX_ATTEMPTS = 6
 
+# Feature flag (experimental features that are *explicitly* unsupported)
+
+# Process configs even when using the click_api for Runner/Deployer
+CLICK_API_PROCESS_CONFIG = from_conf("CLICK_API_PROCESS_CONFIG", False)
+
 
 # PINNED_CONDA_LIBS are the libraries that metaflow depends on for execution
 # and are needed within a conda environment

metaflow/parameters.py

@@ -359,7 +359,7 @@ class Parameter(object):
             "show_default": show_default,
         }
 
-    def init(self):
+    def init(self, ignore_errors=False):
         # Prevent circular import
         from .user_configs.config_parameters import (
             resolve_delayed_evaluator,
@@ -367,14 +367,21 @@ class Parameter(object):
         )
 
         # Resolve any value from configurations
-        self.kwargs = unpack_delayed_evaluator(self.kwargs)
-        self.kwargs = resolve_delayed_evaluator(self.kwargs)
+        self.kwargs = unpack_delayed_evaluator(self.kwargs, ignore_errors=ignore_errors)
+        # Do it one item at a time so errors are ignored at that level (as opposed to
+        # at the entire kwargs leve)
+        self.kwargs = {
+            k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors)
+            for k, v in self.kwargs.items()
+        }
 
         # This was the behavior before configs: values specified in args would override
         # stuff in kwargs which is what we implement here as well
         for key, value in self._override_kwargs.items():
             if value is not None:
-                self.kwargs[key] = value
+                self.kwargs[key] = resolve_delayed_evaluator(
+                    value, ignore_errors=ignore_errors
+                )
         # Set two default values if no-one specified them
         self.kwargs.setdefault("required", False)
         self.kwargs.setdefault("show_default", True)

metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -1,5 +1,6 @@
 import sys
 import json
+import time
 import tempfile
 from typing import ClassVar, Optional
 
@@ -97,6 +98,7 @@ class ArgoWorkflowsTriggeredRun(TriggeredRun):
         )
 
         command_obj = self.deployer.spm.get(pid)
+        command_obj.sync_wait()
         return command_obj.process.returncode == 0
 
     def unsuspend(self, **kwargs) -> bool:
@@ -131,6 +133,7 @@ class ArgoWorkflowsTriggeredRun(TriggeredRun):
         )
 
         command_obj = self.deployer.spm.get(pid)
+        command_obj.sync_wait()
         return command_obj.process.returncode == 0
 
     def terminate(self, **kwargs) -> bool:
@@ -165,8 +168,50 @@ class ArgoWorkflowsTriggeredRun(TriggeredRun):
         )
 
         command_obj = self.deployer.spm.get(pid)
+        command_obj.sync_wait()
         return command_obj.process.returncode == 0
 
+    def wait_for_completion(self, timeout: Optional[int] = None):
+        """
+        Wait for the workflow to complete or timeout.
+
+        Parameters
+        ----------
+        timeout : int, optional, default None
+            Maximum time in seconds to wait for workflow completion.
+            If None, waits indefinitely.
+
+        Raises
+        ------
+        TimeoutError
+            If the workflow does not complete within the specified timeout period.
+        """
+        start_time = time.time()
+        check_interval = 5
+        while self.is_running:
+            if timeout is not None and (time.time() - start_time) > timeout:
+                raise TimeoutError(
+                    "Workflow did not complete within specified timeout."
+                )
+            time.sleep(check_interval)
+
+    @property
+    def is_running(self):
+        """
+        Check if the workflow is currently running.
+
+        Returns
+        -------
+        bool
+            True if the workflow status is either 'Pending' or 'Running',
+            False otherwise.
+        """
+        workflow_status = self.status
+        # full list of all states present here:
+        # https://github.com/argoproj/argo-workflows/blob/main/pkg/apis/workflow/v1alpha1/workflow_types.go#L54
+        # we only consider non-terminal states to determine if the workflow has not finished
+        return workflow_status is not None and workflow_status in ["Pending", "Running"]
+
     @property
     def status(self) -> Optional[str]:
         """
@@ -319,6 +364,7 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
         )
 
         command_obj = self.deployer.spm.get(pid)
+        command_obj.sync_wait()
         return command_obj.process.returncode == 0
 
     def trigger(self, **kwargs) -> ArgoWorkflowsTriggeredRun:
@@ -361,7 +407,7 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
             content = handle_timeout(
                 attribute_file_fd, command_obj, self.deployer.file_read_timeout
             )
-
+            command_obj.sync_wait()
             if command_obj.process.returncode == 0:
                 return ArgoWorkflowsTriggeredRun(
                     deployer=self.deployer, content=content

metaflow/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.py

@@ -50,24 +50,27 @@ class AwsSecretsManagerSecretsProvider(SecretsProvider):
         The secret payload from AWS is EITHER a string OR a binary blob.
 
         If the secret contains a string payload ("SecretString"):
-        - if the `parse_secret_string_as_json` option is True (default):
+        - if the `json` option is True (default):
             {SecretString} will be parsed as a JSON. If successfully parsed, AND the JSON contains a
             top-level object, each entry K/V in the object will also be converted to an entry in the result. V will
             always be casted to a string (if not already a string).
-        - If `parse_secret_string_as_json` option is False:
-            {SecretString} will be returned as a single entry in the result, with the key being the secret_id.
+        - If `json` option is False:
+            {SecretString} will be returned as a single entry in the result, where the key is either:
+                - the `secret_id`, OR
+                - the value set by `options={"env_var_name": custom_env_var_name}`.
 
-        Otherwise, the secret contains a binary blob payload ("SecretBinary"). In this case
-        - The result dic contains '{SecretName}': '{SecretBinary}', where {SecretBinary} is a base64-encoded string
+        Otherwise, if the secret contains a binary blob payload ("SecretBinary"):
+        - The result dict contains '{SecretName}': '{SecretBinary}', where {SecretBinary} is a base64-encoded string.
 
-        All keys in the result are sanitized to be more valid environment variable names. This is done on a best effort
+        All keys in the result are sanitized to be more valid environment variable names. This is done on a best-effort
         basis. Further validation is expected to be done by the invoking @secrets decorator itself.
 
-        :param secret_id: ARN or friendly name of the secret
-        :param options: unused
-        :param role: AWS IAM Role ARN to assume before reading the secret
-        :return: dict of environment variables. All keys and values are strings.
+        :param secret_id: ARN or friendly name of the secret.
+        :param options: Dictionary of additional options. E.g., `options={"env_var_name": custom_env_var_name}`.
+        :param role: AWS IAM Role ARN to assume before reading the secret.
+        :return: Dictionary of environment variables. All keys and values are strings.
         """
+
         import botocore
         from metaflow.plugins.aws.aws_client import get_aws_client
 

metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py

@@ -46,6 +46,7 @@ class StepFunctionsTriggeredRun(TriggeredRun):
         )
 
         command_obj = self.deployer.spm.get(pid)
+        command_obj.sync_wait()
         return command_obj.process.returncode == 0
 
 
@@ -174,6 +175,7 @@ class StepFunctionsDeployedFlow(DeployedFlow):
         )
 
         command_obj = self.deployer.spm.get(pid)
+        command_obj.sync_wait()
         return command_obj.process.returncode == 0
 
     def trigger(self, **kwargs) -> StepFunctionsTriggeredRun:
@@ -217,6 +219,7 @@ class StepFunctionsDeployedFlow(DeployedFlow):
                 attribute_file_fd, command_obj, self.deployer.file_read_timeout
             )
 
+            command_obj.sync_wait()
             if command_obj.process.returncode == 0:
                 return StepFunctionsTriggeredRun(
                     deployer=self.deployer, content=content

metaflow/plugins/cards/card_creator.py

@@ -122,6 +122,7 @@ class CardCreator:
             executable,
             sys.argv[0],
         ]
+
         cmd += self._top_level_options + [
             "card",
             "create",

metaflow/plugins/cards/card_decorator.py

@@ -1,13 +1,16 @@
+import json
+import os
+import re
+import tempfile
+
 from metaflow.decorators import StepDecorator
 from metaflow.metaflow_current import current
+from metaflow.user_configs.config_options import ConfigInput
+from metaflow.user_configs.config_parameters import dump_config_values
 from metaflow.util import to_unicode
+
 from .component_serializer import CardComponentCollector, get_card_class
 from .card_creator import CardCreator
-
-
-# from metaflow import get_metadata
-import re
-
 from .exception import CARD_ID_PATTERN, TYPE_CHECK_REGEX
 
 ASYNC_TIMEOUT = 30
@@ -111,6 +114,14 @@ class CardDecorator(StepDecorator):
         self._logger = logger
         self.card_options = None
 
+        # We check for configuration options. We do this here before they are
+        # converted to properties.
+        self._config_values = [
+            (config.name, ConfigInput.make_key_name(config.name))
+            for _, config in flow._get_parameters()
+            if config.IS_CONFIG_PARAMETER
+        ]
+
         self.card_options = self.attributes["options"]
 
         evt_name = "step-init"
@@ -146,6 +157,18 @@ class CardDecorator(StepDecorator):
         self._task_datastore = task_datastore
         self._metadata = metadata
 
+        # If we have configs, we need to dump them to a file so we can re-use them
+        # when calling the card creation subprocess.
+        if self._config_values:
+            with tempfile.NamedTemporaryFile(
+                mode="w", encoding="utf-8", delete=False
+            ) as config_file:
+                config_value = dump_config_values(flow)
+                json.dump(config_value, config_file)
+                self._config_file_name = config_file.name
+        else:
+            self._config_file_name = None
+
         card_type = self.attributes["type"]
         card_class = get_card_class(card_type)
 
@@ -179,7 +202,7 @@ class CardDecorator(StepDecorator):
         # we need to ensure that `current.card` has `CardComponentCollector` instantiated only once.
         if not self._is_event_registered("pre-step"):
             self._register_event("pre-step")
-            self._set_card_creator(CardCreator(self._create_top_level_args()))
+            self._set_card_creator(CardCreator(self._create_top_level_args(flow)))
 
             current._update_env(
                 {"card": CardComponentCollector(self._logger, self.card_creator)}
@@ -223,6 +246,13 @@ class CardDecorator(StepDecorator):
             self.card_creator.create(mode="render", final=True, **create_options)
             self.card_creator.create(mode="refresh", final=True, **create_options)
 
+        # Unlink the config file if it exists
+        if self._config_file_name:
+            try:
+                os.unlink(self._config_file_name)
+            except Exception as e:
+                pass
+
     @staticmethod
     def _options(mapping):
         for k, v in mapping.items():
@@ -232,9 +262,13 @@ class CardDecorator(StepDecorator):
                 for value in v:
                     yield "--%s" % k
                     if not isinstance(value, bool):
-                        yield to_unicode(value)
+                        if isinstance(value, tuple):
+                            for val in value:
+                                yield to_unicode(val)
+                        else:
+                            yield to_unicode(value)
 
-    def _create_top_level_args(self):
+    def _create_top_level_args(self, flow):
         top_level_options = {
             "quiet": True,
             "metadata": self._metadata.TYPE,
@@ -247,4 +281,8 @@ class CardDecorator(StepDecorator):
             # We don't provide --with as all execution is taking place in
             # the context of the main process
         }
+        if self._config_values:
+            top_level_options["config-value"] = self._config_values
+            top_level_options["local-config-file"] = self._config_file_name
+
         return list(self._options(top_level_options))