metaflow 2.13.1__tar.gz → 2.13.2__tar.gz

{metaflow-2.13.1/metaflow.egg-info → metaflow-2.13.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.13.1
+Version: 2.13.2
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.13.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.13.2; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.13.1 → metaflow-2.13.2}/metaflow/extension_support/plugins.py

@@ -197,6 +197,7 @@ _plugin_categories = {
     "cli": lambda x: (
         list(x.commands)[0] if len(x.commands) == 1 else "too many commands"
     ),
+    "runner_cli": lambda x: x.name,
 }
 
 

{metaflow-2.13.1 → metaflow-2.13.2}/metaflow/parameters.py

@@ -369,7 +369,7 @@ class Parameter(object):
         # Resolve any value from configurations
         self.kwargs = unpack_delayed_evaluator(self.kwargs, ignore_errors=ignore_errors)
         # Do it one item at a time so errors are ignored at that level (as opposed to
-        # at the entire kwargs leve)
+        # at the entire kwargs level)
         self.kwargs = {
             k: resolve_delayed_evaluator(v, ignore_errors=ignore_errors)
             for k, v in self.kwargs.items()

{metaflow-2.13.1 → metaflow-2.13.2}/metaflow/plugins/__init__.py

@@ -19,6 +19,11 @@ CLIS_DESC = [
     ("logs", ".logs_cli.cli"),
 ]
 
+# Add additional commands to the runner here
+# These will be accessed using Runner().<command>()
+RUNNER_CLIS_DESC = []
+
+
 from .test_unbounded_foreach_decorator import InternalTestUnboundedForeachInput
 
 # Add new step decorators here
@@ -168,6 +173,14 @@ def get_plugin_cli_path():
     return resolve_plugins("cli", path_only=True)
 
 
+def get_runner_cli():
+    return resolve_plugins("runner_cli")
+
+
+def get_runner_cli_path():
+    return resolve_plugins("runner_cli", path_only=True)
+
+
 STEP_DECORATORS = resolve_plugins("step_decorator")
 FLOW_DECORATORS = resolve_plugins("flow_decorator")
 ENVIRONMENTS = resolve_plugins("environment")

{metaflow-2.13.1 → metaflow-2.13.2}/metaflow/plugins/argo/argo_workflows.py

@@ -743,6 +743,17 @@ class ArgoWorkflows(object):
                     )
                 }
             )
+        try:
+            # Build the DAG based on the DAGNodes given by the FlowGraph for the found FlowSpec class.
+            _steps_info, graph_structure = self.graph.output_steps()
+            graph_info = {
+                # for the time being, we only need the graph_structure. Being mindful of annotation size limits we do not include anything extra.
+                "graph_structure": graph_structure
+            }
+        except Exception:
+            graph_info = None
+
+        dag_annotation = {"metaflow/dag": json.dumps(graph_info)}
 
         return (
             WorkflowTemplate()
@@ -758,6 +769,7 @@ class ArgoWorkflows(object):
                 .annotations(self._base_annotations)
                 .labels(self._base_labels)
                 .label("app.kubernetes.io/name", "metaflow-flow")
+                .annotations(dag_annotation)
             )
             .spec(
                 WorkflowSpec()

{metaflow-2.13.1 → metaflow-2.13.2}/metaflow/plugins/pypi/micromamba.py

@@ -1,6 +1,7 @@
 import functools
 import json
 import os
+import re
 import subprocess
 import tempfile
 import time
@@ -23,6 +24,8 @@ class MicromambaException(MetaflowException):
 
 GLIBC_VERSION = os.environ.get("CONDA_OVERRIDE_GLIBC", "2.38")
 
+_double_equal_match = re.compile("==(?=[<=>!~])")
+
 
 class Micromamba(object):
     def __init__(self, logger=None):
@@ -101,7 +104,8 @@ class Micromamba(object):
                 cmd.append("--channel=%s" % channel)
 
             for package, version in packages.items():
-                cmd.append("%s==%s" % (package, version))
+                version_string = "%s==%s" % (package, version)
+                cmd.append(_double_equal_match.sub("", version_string))
             if python:
                 cmd.append("python==%s" % python)
             # TODO: Ensure a human readable message is returned when the environment

{metaflow-2.13.1 → metaflow-2.13.2}/metaflow/runner/metaflow_runner.py

@@ -7,6 +7,8 @@ from typing import Dict, Iterator, Optional, Tuple
 
 from metaflow import Run
 
+from metaflow.plugins import get_runner_cli
+
 from .utils import (
     temporary_fifo,
     handle_timeout,
@@ -187,7 +189,27 @@ class ExecutingRun(object):
             yield position, line
 
 
-class Runner(object):
+class RunnerMeta(type):
+    def __new__(mcs, name, bases, dct):
+        cls = super().__new__(mcs, name, bases, dct)
+
+        def _injected_method(subcommand_name, runner_subcommand):
+            def f(self, *args, **kwargs):
+                return runner_subcommand(self, *args, **kwargs)
+
+            f.__doc__ = runner_subcommand.__doc__ or ""
+            f.__name__ = subcommand_name
+
+            return f
+
+        for runner_subcommand in get_runner_cli():
+            method_name = runner_subcommand.name.replace("-", "_")
+            setattr(cls, method_name, _injected_method(method_name, runner_subcommand))
+
+        return cls
+
+
+class Runner(metaclass=RunnerMeta):
     """
     Metaflow's Runner API that presents a programmatic interface
     to run flows and perform other operations either synchronously or asynchronously.
@@ -337,7 +359,7 @@ class Runner(object):
 
             return self.__get_executing_run(attribute_file_fd, command_obj)
 
-    def resume(self, **kwargs):
+    def resume(self, **kwargs) -> ExecutingRun:
         """
         Blocking resume execution of the run.
         This method will wait until the resumed run has completed execution.
@@ -400,7 +422,7 @@ class Runner(object):
 
             return await self.__async_get_executing_run(attribute_file_fd, command_obj)
 
-    async def async_resume(self, **kwargs):
+    async def async_resume(self, **kwargs) -> ExecutingRun:
         """
         Non-blocking resume execution of the run.
         This method will return as soon as the resume has launched.

{metaflow-2.13.1 → metaflow-2.13.2}/metaflow/runtime.py

@@ -125,7 +125,7 @@ class NativeRuntime(object):
         self._clone_run_id = clone_run_id
         self._clone_only = clone_only
         self._cloned_tasks = []
-        self._cloned_task_index = set()
+        self._ran_or_scheduled_task_index = set()
         self._reentrant = reentrant
         self._run_url = None
 
@@ -297,7 +297,7 @@ class NativeRuntime(object):
                     task.ubf_context = ubf_context
                 new_task_id = task.task_id
                 self._cloned_tasks.append(task)
-                self._cloned_task_index.add(cloned_task_pathspec_index)
+                self._ran_or_scheduled_task_index.add(cloned_task_pathspec_index)
                 task_pathspec = "{}/{}/{}".format(self._run_id, step_name, new_task_id)
             else:
                 task_pathspec = "{}/{}/{}".format(self._run_id, step_name, new_task_id)
@@ -384,8 +384,10 @@ class NativeRuntime(object):
                 and step_name != "_parameters"
                 and (step_name not in self._steps_to_rerun)
             ):
-                # "_unbounded_foreach" is a special flag to indicate that the transition is an unbounded foreach.
-                # Both parent and splitted children tasks will have this flag set. The splitted control/mapper tasks
+                # "_unbounded_foreach" is a special flag to indicate that the transition
+                # is an unbounded foreach.
+                # Both parent and splitted children tasks will have this flag set.
+                # The splitted control/mapper tasks
                 # are not foreach types because UBF is always followed by a join step.
                 is_ubf_task = (
                     "_unbounded_foreach" in task_ds and task_ds["_unbounded_foreach"]
@@ -647,10 +649,18 @@ class NativeRuntime(object):
     # Store the parameters needed for task creation, so that pushing on items
     # onto the run_queue is an inexpensive operation.
     def _queue_push(self, step, task_kwargs, index=None):
-        # If the to-be-pushed task is already cloned before, we don't need
-        # to re-run it.
-        if index and index in self._cloned_task_index:
-            return
+        # In the case of cloning, we set all the cloned tasks as the
+        # finished tasks when pushing tasks using _queue_tasks. This means that we
+        # could potentially try to push the same task multiple times (for example
+        # if multiple parents of a join are cloned). We therefore keep track of what
+        # has executed (been cloned) or what has been scheduled and avoid scheduling
+        # it again.
+        if index:
+            if index in self._ran_or_scheduled_task_index:
+                # It has already run or been scheduled
+                return
+            # Note that we are scheduling this to run
+            self._ran_or_scheduled_task_index.add(index)
         self._run_queue.insert(0, (step, task_kwargs))
         # For foreaches, this will happen multiple time but is ok, becomes a no-op
         self._unprocessed_steps.discard(step)

{metaflow-2.13.1 → metaflow-2.13.2}/metaflow/user_configs/config_parameters.py

@@ -183,7 +183,7 @@ class DelayEvaluator(collections.abc.Mapping):
 
     def __getattr__(self, name):
         if self._access is None:
-            raise AttributeError()
+            raise AttributeError(name)
         self._access.append(name)
         return self
 
@@ -336,6 +336,8 @@ class Config(Parameter, collections.abc.Mapping):
         self.parser = parser
         self._computed_value = None
 
+        self._delayed_evaluator = None
+
     def load_parameter(self, v):
         if v is None:
             return None
@@ -344,22 +346,37 @@ class Config(Parameter, collections.abc.Mapping):
     def _store_value(self, v: Any) -> None:
         self._computed_value = v
 
+    def _init_delayed_evaluator(self) -> None:
+        if self._delayed_evaluator is None:
+            self._delayed_evaluator = DelayEvaluator(self.name.lower())
+
     # Support <config>.<var> syntax
     def __getattr__(self, name):
-        return DelayEvaluator(self.name.lower()).__getattr__(name)
+        # Need to return a new DelayEvaluator everytime because the evaluator will
+        # contain the "path" (ie: .name) and can be further accessed.
+        return getattr(DelayEvaluator(self.name.lower()), name)
 
-    # Next three methods are to implement mapping to support **<config> syntax
+    # Next three methods are to implement mapping to support **<config> syntax. We
+    # need to be careful, however, to also support a regular `config["key"]` syntax
+    # which calls into `__getitem__` and therefore behaves like __getattr__ above.
     def __iter__(self):
-        return iter(DelayEvaluator(self.name.lower()))
+        self._init_delayed_evaluator()
+        yield from self._delayed_evaluator
 
     def __len__(self):
-        return len(DelayEvaluator(self.name.lower()))
+        self._init_delayed_evaluator()
+        return len(self._delayed_evaluator)
 
     def __getitem__(self, key):
+        self._init_delayed_evaluator()
+        if key.startswith(UNPACK_KEY):
+            return self._delayed_evaluator[key]
         return DelayEvaluator(self.name.lower())[key]
 
 
 def resolve_delayed_evaluator(v: Any, ignore_errors: bool = False) -> Any:
+    # NOTE: We don't ignore errors in downstream calls because we want to have either
+    # all or nothing for the top-level call by the user.
     try:
         if isinstance(v, DelayEvaluator):
             return v()
@@ -397,7 +414,7 @@ def unpack_delayed_evaluator(
         else:
             # k.startswith(UNPACK_KEY)
             try:
-                result.update(resolve_delayed_evaluator(v[k]))
+                result.update(resolve_delayed_evaluator(v))
             except Exception as e:
                 if ignore_errors:
                     continue

metaflow-2.13.2/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.13.2"

{metaflow-2.13.1 → metaflow-2.13.2/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.13.1
+Version: 2.13.2
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.13.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.13.2; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.13.1 → metaflow-2.13.2}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.13.1
+metaflow-stubs==2.13.2

metaflow-2.13.1/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.13.1"