ob-metaflow 2.16.8.2rc2__py2.py3-none-any.whl → 2.17.1.0__py2.py3-none-any.whl

metaflow/runtime.py

@@ -37,7 +37,7 @@ from .debug import debug
 from .decorators import flow_decorators
 from .flowspec import _FlowState
 from .mflog import mflog, RUNTIME_LOG_SOURCE
-from .util import to_unicode, compress_list, unicode_type, get_split_branch_for_node
+from .util import to_unicode, compress_list, unicode_type
 from .clone_util import clone_task_helper
 from .unbounded_foreach import (
     CONTROL_TASK_TAG,
@@ -820,90 +820,39 @@ class NativeRuntime(object):
             # matching_split is the split-parent of the finished task
             matching_split = self._graph[self._graph[next_step].split_parents[-1]]
             _, foreach_stack = task.finished_id
-
-            direct_parents = set(self._graph[next_step].in_funcs)
-
-            # next step is a foreach join
+            index = ""
             if matching_split.type == "foreach":
-                top_frame = task.finished_id[1][-1]
-                num_splits = top_frame.num_splits
-
-                finished_for_each_index = {}
-                for finished_id, pathspec in self._finished.items():
-                    finished_step, finished_foreach_stack = finished_id
-
-                    if finished_step not in direct_parents:
-                        continue
-
-                    if (
-                        len(finished_foreach_stack) != len(task.finished_id[1])
-                        or finished_foreach_stack[:-1] != task.finished_id[1][:-1]
-                    ):
-                        continue
-
-                    current_index = finished_foreach_stack[-1].index
-                    finished_for_each_index[current_index] = pathspec
+                # next step is a foreach join
 
-                if (
-                    num_splits is not None
-                    and len(finished_for_each_index) == num_splits
-                ):
-                    required_tasks = list(finished_for_each_index.values())
-                    index = self._translate_index(task, next_step, "join")
-                    self._queue_push(
-                        next_step,
-                        {"input_paths": required_tasks, "join_type": "foreach"},
-                        index,
-                    )
-            elif matching_split.type == "split-switch":
-                required_tasks = [task.path]
-                join_type = "split-switch"
+                def siblings(foreach_stack):
+                    top = foreach_stack[-1]
+                    bottom = list(foreach_stack[:-1])
+                    for index in range(top.num_splits):
+                        yield tuple(bottom + [top._replace(index=index)])
+
+                # required tasks are all split-siblings of the finished task
+                required_tasks = [
+                    self._finished.get((task.step, s)) for s in siblings(foreach_stack)
+                ]
+                join_type = "foreach"
+                index = self._translate_index(task, next_step, "join")
+            else:
+                # next step is a split
+                # required tasks are all branches joined by the next step
+                required_tasks = [
+                    self._finished.get((step, foreach_stack))
+                    for step in self._graph[next_step].in_funcs
+                ]
+                join_type = "linear"
                 index = self._translate_index(task, next_step, "linear")
+
+            if all(required_tasks):
+                # all tasks to be joined are ready. Schedule the next join step.
                 self._queue_push(
                     next_step,
                     {"input_paths": required_tasks, "join_type": join_type},
                     index,
                 )
-            else:
-                split_node_name = matching_split.name
-                expected_branches = set(matching_split.out_funcs)
-
-                resolved_and_finished = {}
-                for finished_id, pathspec in self._finished.items():
-                    finished_step, finished_foreach_stack = finished_id
-
-                    if finished_step not in direct_parents:
-                        continue
-
-                    if finished_foreach_stack != foreach_stack:
-                        continue
-
-                    branch = get_split_branch_for_node(
-                        self._graph, finished_step, split_node_name
-                    )
-                    if branch in expected_branches:
-                        resolved_and_finished[branch] = pathspec
-
-                if set(resolved_and_finished.keys()) == expected_branches:
-                    required_tasks = list(resolved_and_finished.values())
-                    index = self._translate_index(task, next_step, "linear")
-                    self._queue_push(
-                        next_step,
-                        {"input_paths": required_tasks, "join_type": "linear"},
-                        index,
-                    )
-
-    def _queue_task_switch(self, task, next_steps):
-        if len(next_steps) != 1:
-            msg = (
-                "Step *{step}* is a switch statement but runtime got {actual} transitions. "
-                "Expected exactly 1 chosen step."
-            )
-            raise Exception(msg.format(step=task.step, actual=len(next_steps)))
-
-        chosen_step = next_steps[0]
-        index = self._translate_index(task, chosen_step, "linear")
-        self._queue_push(chosen_step, {"input_paths": [task.path]}, index)
 
     def _queue_task_foreach(self, task, next_steps):
         # CHECK: this condition should be enforced by the linter but
@@ -981,28 +930,7 @@ class NativeRuntime(object):
                 next_steps = []
                 foreach = None
             expected = self._graph[task.step].out_funcs
-            if self._graph[task.step].type == "split-switch":
-                if len(next_steps) != 1:
-                    msg = (
-                        "Switch step *{step}* should transition to exactly "
-                        "one step at runtime, but got: {actual}"
-                    )
-                    raise MetaflowInternalError(
-                        msg.format(step=task.step, actual=", ".join(next_steps))
-                    )
-                if next_steps[0] not in expected:
-                    msg = (
-                        "Switch step *{step}* transitioned to unexpected "
-                        "step *{actual}*. Expected one of: {expected}"
-                    )
-                    raise MetaflowInternalError(
-                        msg.format(
-                            step=task.step,
-                            actual=next_steps[0],
-                            expected=", ".join(expected),
-                        )
-                    )
-            elif next_steps != expected:
+            if next_steps != expected:
                 msg = (
                     "Based on static analysis of the code, step *{step}* "
                     "was expected to transition to step(s) *{expected}*. "
@@ -1026,9 +954,6 @@ class NativeRuntime(object):
             elif foreach:
                 # Next step is a foreach child
                 self._queue_task_foreach(task, next_steps)
-            elif self._graph[task.step].type == "split-switch":
-                # Next step is switch - queue the chosen step
-                self._queue_task_switch(task, next_steps)
             else:
                 # Next steps are normal linear steps
                 for step in next_steps:

metaflow/task.py

@@ -670,49 +670,31 @@ class MetaflowTask(object):
 
                 if join_type:
                     # Join step:
-                    passdown_params = None
-
-                    # Ensure that we have the right number of inputs.
-                    if join_type not in ("foreach", "split-switch"):
-                        # Find the corresponding split node from the graph.
-                        split_node = self.flow._graph[node.split_parents[-1]]
-                        # The number of expected inputs is the number of branches from that split.
-                        expected_inputs = len(split_node.out_funcs)
-
-                        if len(inputs) != expected_inputs:
-                            raise MetaflowDataMissing(
-                                "Join *%s* expected %d inputs but only %d inputs "
-                                "were found" % (step_name, expected_inputs, len(inputs))
-                            )
-                    if join_type == "split-switch":
-                        # Switch joins only have one input path (the chosen branch from the switch).
-                        # This occurs when a switch leads directly to a step without other converging branches.
-                        if len(inputs) > 1:
-                            raise MetaflowInternalError(
-                                f"Step *{step_name}* is a switch join but gets multiple inputs"
-                            )
-                        # Use input datastore directly - no need to copy parameters since we're not
-                        # creating a new output datastore
-                        self.flow._set_datastore(inputs[0])
-                        passdown_params = False
-                    else:
-                        # Multiple input contexts are passed in as an argument
-                        # to the step function.
-                        input_obj = Inputs(self._clone_flow(inp) for inp in inputs)
-                        self.flow._set_datastore(output)
-                        passdown_params = True
-                        # initialize parameters (if they exist)
-                        # We take Parameter values from the first input,
-                        # which is always safe since parameters are read-only
-                    if passdown_params is not None:
-                        current._update_env(
-                            {
-                                "parameter_names": self._init_parameters(
-                                    inputs[0], passdown=passdown_params
-                                ),
-                                "graph_info": self.flow._graph_info,
-                            }
+
+                    # Ensure that we have the right number of inputs. The
+                    # foreach case is checked above.
+                    if join_type != "foreach" and len(inputs) != len(node.in_funcs):
+                        raise MetaflowDataMissing(
+                            "Join *%s* expected %d "
+                            "inputs but only %d inputs "
+                            "were found" % (step_name, len(node.in_funcs), len(inputs))
                         )
+
+                    # Multiple input contexts are passed in as an argument
+                    # to the step function.
+                    input_obj = Inputs(self._clone_flow(inp) for inp in inputs)
+                    self.flow._set_datastore(output)
+                    # initialize parameters (if they exist)
+                    # We take Parameter values from the first input,
+                    # which is always safe since parameters are read-only
+                    current._update_env(
+                        {
+                            "parameter_names": self._init_parameters(
+                                inputs[0], passdown=True
+                            ),
+                            "graph_info": self.flow._graph_info,
+                        }
+                    )
                 else:
                     # Linear step:
                     # We are running with a single input context.
@@ -770,10 +752,7 @@ class MetaflowTask(object):
                     )
 
                 if join_type:
-                    if join_type == "split-switch":
-                        self._exec_step_function(step_func, orig_step_func)
-                    else:
-                        self._exec_step_function(step_func, orig_step_func, input_obj)
+                    self._exec_step_function(step_func, orig_step_func, input_obj)
                 else:
                     self._exec_step_function(step_func, orig_step_func)
 

metaflow/user_decorators/mutable_flow.py

@@ -347,8 +347,10 @@ class MutableFlow:
                     "Mutable flow adding flow decorator '%s'" % deco_type
                 )
 
+            # self._flow_cls._flow_decorators is a  dictionary of form :
+            # <deco_name> : [deco_instance, deco_instance, ...]
             existing_deco = [
-                d for d in self._flow_cls._flow_decorators if d.name == flow_deco.name
+                d for d in self._flow_cls._flow_decorators if d == flow_deco.name
             ]
 
             if flow_deco.allow_multiple or not existing_deco:
@@ -365,11 +367,11 @@ class MutableFlow:
                     "Mutable flow overriding flow decorator '%s' "
                     "(removing existing decorator and adding new one)" % flow_deco.name
                 )
-                self._flow_cls._flow_decorators = [
-                    d
+                self._flow_cls._flow_decorators = {
+                    d: self._flow_cls._flow_decorators[d]
                     for d in self._flow_cls._flow_decorators
-                    if d.name != flow_deco.name
-                ]
+                    if d != flow_deco.name
+                }
                 _do_add()
             elif duplicates == MutableFlow.ERROR:
                 # If we error, we raise an exception
@@ -470,7 +472,7 @@ class MutableFlow:
             % (len(old_deco_list) - len(new_deco_list))
         )
         if new_deco_list:
-            self._flow_cls._flow_decorators[deconame] = new_deco_list
+            self._flow_cls._flow_decorators[deco_name] = new_deco_list
         else:
             del self._flow_cls._flow_decorators[deco_name]
         return did_remove

metaflow/util.py

@@ -212,35 +212,6 @@ def write_latest_run_id(obj, run_id):
         f.write(str(run_id))
 
 
-def get_split_branch_for_node(graph, node_name, split_node_name):
-    """
-    Walks backwards from a node to find which branch of a given split
-    it belongs to. The branches are the direct children of the split node.
-    """
-
-    # The branches are the direct children of the split node
-    split_branches = set(graph[split_node_name].out_funcs)
-
-    # Use a queue for breadth-first search backwards
-    q = [node_name]
-    visited = {node_name}
-
-    while q:
-        current_name = q.pop(0)
-
-        # If we have reached one of the original branches, we are done
-        if current_name in split_branches:
-            return current_name
-
-        # Add this node's parents to the queue to continue searching
-        for parent_name in graph[current_name].in_funcs:
-            if parent_name not in visited:
-                visited.add(parent_name)
-                q.append(parent_name)
-
-    return None
-
-
 def get_object_package_version(obj):
     """
     Return the top level package name and package version that defines the

metaflow/vendor.py

@@ -63,14 +63,29 @@ def find_vendored_libs(vendor_dir, whitelist, whitelist_dirs):
     return vendored_libs, paths
 
 
-def fetch_licenses(*info_dir, vendor_dir):
-    for file in chain.from_iterable(map(iter_subtree, info_dir)):
-        if "LICENSE" in file.name:
-            library = file.parent.name.split("-")[0]
-            shutil.copy(file, vendor_dir / ("%s.LICENSE" % library))
-        else:
+def fetch_licenses(*info_dirs, vendor_dir):
+    for dist_info in info_dirs:
+        metadata_file = dist_info / "METADATA"
+        if not metadata_file.exists():
+            continue
+
+        project_name = None
+        for line in metadata_file.read_text("utf-8").splitlines():
+            if line.startswith("Name: "):
+                project_name = line.split("Name: ", 1)[1].strip()
+                break
+        if not project_name:
             continue
 
+        for item in dist_info.iterdir():
+            if item.is_file() and re.search(r"(LICENSE|COPYING)", item.name, re.I):
+                shutil.copy(item, vendor_dir / f"{project_name}.LICENSE")
+            elif item.is_dir() and item.name.lower() == "licenses":
+                for license_file in item.iterdir():
+                    if license_file.is_file():
+                        dest_name = f"{project_name}.{license_file.name}"
+                        shutil.copy(license_file, vendor_dir / dest_name)
+
 
 def vendor(vendor_dir):
     # remove everything
@@ -108,6 +123,8 @@ def vendor(vendor_dir):
                 "-r",
                 "_vendor/vendor_%s.txt" % subdir,
                 "--no-compile",
+                "--no-binary",
+                ":all:",
             ]
         )
 

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.16.8.2rc2"
+metaflow_version = "2.17.1.0"

{ob_metaflow-2.16.8.2rc2.data → ob_metaflow-2.17.1.0.data}/data/share/metaflow/devtools/Makefile

@@ -257,10 +257,11 @@ shell: setup-tilt
 		echo "🔎 Using $$user_shell for interactive session."; \
 		echo "🐍 If you installed Metaflow in a virtual environment, activate it now."; \
 		if [ -f "$(DEVTOOLS_DIR)/aws_config" ]; then \
-			env METAFLOW_HOME="$(DEVTOOLS_DIR)" \
+			env -u AWS_PROFILE \
+				-u AWS_SHARED_CREDENTIALS_FILE \
+				METAFLOW_HOME="$(DEVTOOLS_DIR)" \
 				METAFLOW_PROFILE=local \
 				AWS_CONFIG_FILE="$(DEVTOOLS_DIR)/aws_config" \
-				AWS_SHARED_CREDENTIALS_FILE= \
 				"$$user_shell" -i; \
 		else \
 			env METAFLOW_HOME="$(DEVTOOLS_DIR)" \

{ob_metaflow-2.16.8.2rc2.data → ob_metaflow-2.17.1.0.data}/data/share/metaflow/devtools/Tiltfile

@@ -593,7 +593,7 @@ if "ui" in enabled_components:
             'uiBackend.metaflowDatastoreSysRootS3=s3://metaflow-test',
             'uiBackend.metaflowS3EndpointURL=http://minio.default.svc.cluster.local:9000',
             'uiBackend.image.name=public.ecr.aws/outerbounds/metaflow_metadata_service',
-            'uiBackend.image.tag=2.4.13-2-g70af4ed',
+            'uiBackend.image.tag=2.5.0',
             'uiBackend.env[0].name=AWS_ACCESS_KEY_ID',
             'uiBackend.env[0].value=rootuser',
             'uiBackend.env[1].name=AWS_SECRET_ACCESS_KEY',
@@ -603,7 +603,7 @@ if "ui" in enabled_components:
             'uiBackend.resources.requests.memory=256Mi',
             'uiStatic.metaflowUIBackendURL=http://localhost:8083/api',
             'uiStatic.image.name=public.ecr.aws/outerbounds/metaflow_ui',
-            'uiStatic.image.tag=v1.3.13-5-g5dd049e',
+            'uiStatic.image.tag=v1.3.14',
             'uiStatic.resources.requests.cpu=25m',
             'uiStatic.resources.requests.memory=64Mi',
             'uiStatic.resources.limits.cpu=50m',

{ob_metaflow-2.16.8.2rc2.dist-info → ob_metaflow-2.17.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.16.8.2rc2
+Version: 2.17.1.0
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.8.2rc2; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.17.1.0; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description