metaflow 2.12.7__py2.py3-none-any.whl → 2.12.9__py2.py3-none-any.whl

metaflow/__init__.py

@@ -150,6 +150,8 @@ from . import tuple_util
 if sys.version_info >= (3, 7):
     from .runner.metaflow_runner import Runner
     from .runner.nbrun import NBRunner
+    from .runner.deployer import Deployer
+    from .runner.nbdeploy import NBDeployer
 
 __version_addl__ = []
 _ext_debug("Loading top-level modules")

metaflow/cli.py

@@ -707,10 +707,18 @@ def resume(
             "/".join((obj.flow.name, runtime.run_id)),
         ),
     )
-    if clone_only:
-        runtime.clone_original_run()
-    else:
-        runtime.execute()
+
+    # We may skip clone-only resume if this is not a resume leader,
+    # and clone is already complete.
+    if runtime.should_skip_clone_only_execution():
+        return
+
+    with runtime.run_heartbeat():
+        if clone_only:
+            runtime.clone_original_run()
+        else:
+            runtime.clone_original_run(generate_task_obj=True, verbose=False)
+            runtime.execute()
 
 
 @tracing.cli_entrypoint("cli/run")

metaflow/extension_support/plugins.py

@@ -180,6 +180,7 @@ _plugin_categories = {
     "datastore": lambda x: x.TYPE,
     "secrets_provider": lambda x: x.TYPE,
     "gcp_client_provider": lambda x: x.name,
+    "deployer_impl_provider": lambda x: x.TYPE,
     "azure_client_provider": lambda x: x.name,
     "sidecar": None,
     "logging_sidecar": None,

metaflow/flowspec.py

@@ -28,7 +28,14 @@ except NameError:
 
 from .datastore.inputs import Inputs
 
-INTERNAL_ARTIFACTS_SET = set(["_foreach_values"])
+INTERNAL_ARTIFACTS_SET = set(
+    [
+        "_foreach_values",
+        "_unbounded_foreach",
+        "_control_mapper_tasks",
+        "_control_task_is_mapper_zero",
+    ]
+)
 
 
 class InvalidNextException(MetaflowException):

metaflow/lint.py

@@ -293,6 +293,19 @@ def check_parallel_step_after_next(graph):
             raise LintWarn(msg.format(node))
 
 
+@linter.ensure_static_graph
+@linter.check
+def check_join_followed_by_parallel_step(graph):
+    msg = (
+        "An @parallel step should be followed by a join step. Step *{0}* is called "
+        "after an @parallel step but is not a join step. Please add an extra `inputs` "
+        "argument to the step."
+    )
+    for node in graph:
+        if node.parallel_step and not graph[node.out_funcs[0]].type == "join":
+            raise LintWarn(msg.format(node.out_funcs[0]))
+
+
 @linter.ensure_static_graph
 @linter.check
 def check_parallel_foreach_calls_parallel_step(graph):

metaflow/metaflow_current.py

@@ -260,14 +260,6 @@ class Current(object):
         """
         return self._username
 
-    @property
-    def parallel(self):
-        return Parallel(
-            main_ip=os.environ.get("MF_PARALLEL_MAIN_IP", "127.0.0.1"),
-            num_nodes=int(os.environ.get("MF_PARALLEL_NUM_NODES", "1")),
-            node_index=int(os.environ.get("MF_PARALLEL_NODE_INDEX", "0")),
-        )
-
     @property
     def tags(self):
         """

metaflow/plugins/__init__.py

@@ -1,3 +1,5 @@
+import sys
+
 from metaflow.extension_support.plugins import (
     merge_lists,
     process_plugins,
@@ -139,6 +141,13 @@ AZURE_CLIENT_PROVIDERS_DESC = [
     ("azure-default", ".azure.azure_credential.AzureDefaultClientProvider")
 ]
 
+DEPLOYER_IMPL_PROVIDERS_DESC = [
+    ("argo-workflows", ".argo.argo_workflows_deployer.ArgoWorkflowsDeployer"),
+    (
+        "step-functions",
+        ".aws.step_functions.step_functions_deployer.StepFunctionsDeployer",
+    ),
+]
 
 process_plugins(globals())
 
@@ -164,6 +173,9 @@ SECRETS_PROVIDERS = resolve_plugins("secrets_provider")
 AZURE_CLIENT_PROVIDERS = resolve_plugins("azure_client_provider")
 GCP_CLIENT_PROVIDERS = resolve_plugins("gcp_client_provider")
 
+if sys.version_info >= (3, 7):
+    DEPLOYER_IMPL_PROVIDERS = resolve_plugins("deployer_impl_provider")
+
 from .cards.card_modules import MF_EXTERNAL_CARDS
 
 # Cards; due to the way cards were designed, it is harder to make them fit