ob-metaflow 2.16.8.2rc0__py2.py3-none-any.whl → 2.16.8.2rc2__py2.py3-none-any.whl

metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -203,38 +203,6 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "argo-workflows"
 
-    @classmethod
-    def list_deployed_flows(cls, flow_name: Optional[str] = None):
-        """
-        List all deployed Argo Workflow templates.
-
-        Parameters
-        ----------
-        flow_name : str, optional, default None
-            If specified, only list deployed flows for this specific flow name.
-            If None, list all deployed flows.
-
-        Yields
-        ------
-        ArgoWorkflowsDeployedFlow
-            `ArgoWorkflowsDeployedFlow` objects representing deployed
-            workflow templates on Argo Workflows.
-        """
-        from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
-
-        # When flow_name is None, use all=True to get all templates
-        # When flow_name is specified, use all=False to filter by flow_name
-        all_templates = flow_name is None
-        for template_name in ArgoWorkflows.list_templates(
-            flow_name=flow_name, all=all_templates
-        ):
-            try:
-                deployed_flow = cls.from_deployment(template_name)
-                yield deployed_flow
-            except Exception:
-                # Skip templates that can't be converted to DeployedFlow objects
-                continue
-
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """
@@ -310,43 +278,6 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
 
         return cls(deployer=d)
 
-    @classmethod
-    def get_triggered_run(
-        cls, identifier: str, run_id: str, metadata: Optional[str] = None
-    ):
-        """
-        Retrieves a `ArgoWorkflowsTriggeredRun` object from an identifier, a run id and
-        optional metadata.
-
-        Parameters
-        ----------
-        identifier : str
-            Deployer specific identifier for the workflow to retrieve
-        run_id : str
-            Run ID for the which to fetch the triggered run object
-        metadata : str, optional, default None
-            Optional deployer specific metadata.
-
-        Returns
-        -------
-        ArgoWorkflowsTriggeredRun
-            A `ArgoWorkflowsTriggeredRun` object representing the
-            triggered run on argo workflows.
-        """
-        deployed_flow_obj = cls.from_deployment(identifier, metadata)
-        return ArgoWorkflowsTriggeredRun(
-            deployer=deployed_flow_obj.deployer,
-            content=json.dumps(
-                {
-                    "metadata": deployed_flow_obj.deployer.metadata,
-                    "pathspec": "/".join(
-                        (deployed_flow_obj.deployer.flow_name, run_id)
-                    ),
-                    "name": run_id,
-                }
-            ),
-        )
-
     @property
     def production_token(self) -> Optional[str]:
         """

metaflow/plugins/aws/step_functions/step_functions.py

@@ -317,6 +317,12 @@ class StepFunctions(object):
                     "to AWS Step Functions is not supported currently."
                 )
 
+            if node.type == "split-switch":
+                raise StepFunctionsException(
+                    "Deploying flows with switch statement "
+                    "to AWS Step Functions is not supported currently."
+                )
+
             # Assign an AWS Batch job to the AWS Step Functions state
             # and pass the intermediate state by exposing `JobId` and
             # `Parameters` to the child job(s) as outputs. `Index` and

metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py

@@ -56,20 +56,6 @@ class StepFunctionsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "step-functions"
 
-    @classmethod
-    def list_deployed_flows(cls, flow_name: Optional[str] = None):
-        """
-        This method is not currently implemented for Step Functions.
-
-        Raises
-        ------
-        NotImplementedError
-            This method is not implemented for Step Functions.
-        """
-        raise NotImplementedError(
-            "list_deployed_flows is not implemented for StepFunctions"
-        )
-
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """
@@ -84,22 +70,6 @@ class StepFunctionsDeployedFlow(DeployedFlow):
             "from_deployment is not implemented for StepFunctions"
         )
 
-    @classmethod
-    def get_triggered_run(
-        cls, identifier: str, run_id: str, metadata: Optional[str] = None
-    ):
-        """
-        This method is not currently implemented for Step Functions.
-
-        Raises
-        ------
-        NotImplementedError
-            This method is not implemented for Step Functions.
-        """
-        raise NotImplementedError(
-            "get_triggered_run is not implemented for StepFunctions"
-        )
-
     @property
     def production_token(self: DeployedFlow) -> Optional[str]:
         """

metaflow/plugins/cards/card_modules/basic.py

@@ -20,12 +20,15 @@ def transform_flow_graph(step_info):
             return "split"
         elif node_type == "split-parallel" or node_type == "split-foreach":
             return "foreach"
+        elif node_type == "split-switch":
+            return "switch"
         return "unknown"  # Should never happen
 
     graph_dict = {}
     for stepname in step_info:
-        graph_dict[stepname] = {
-            "type": node_to_type(step_info[stepname]["type"]),
+        node_type = node_to_type(step_info[stepname]["type"])
+        node_info = {
+            "type": node_type,
             "box_next": step_info[stepname]["type"] not in ("linear", "join"),
             "box_ends": (
                 None
@@ -35,6 +38,15 @@ def transform_flow_graph(step_info):
             "next": step_info[stepname]["next"],
             "doc": step_info[stepname]["doc"],
         }
+
+        if node_type == "switch":
+            if "condition" in step_info[stepname]:
+                node_info["condition"] = step_info[stepname]["condition"]
+            if "switch_cases" in step_info[stepname]:
+                node_info["switch_cases"] = step_info[stepname]["switch_cases"]
+
+        graph_dict[stepname] = node_info
+
     return graph_dict
 
 

metaflow/plugins/cards/card_modules/convert_to_native_type.py

@@ -146,13 +146,7 @@ class TaskToDict:
             # Python 3.13 removes the standard ``imghdr`` module. Metaflow
             # vendors a copy so we can keep using ``what`` to detect image
             # formats irrespective of the Python version.
-            import warnings
-
-            with warnings.catch_warnings():
-                warnings.filterwarnings(
-                    "ignore", category=DeprecationWarning, module="imghdr"
-                )
-                from metaflow._vendor import imghdr
+            from metaflow._vendor import imghdr
 
             resp = imghdr.what(None, h=data_object)
             # Only accept types supported on the web

metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -98,7 +98,7 @@ class KubernetesDecorator(StepDecorator):
         the scheduled node should not have GPUs.
     gpu_vendor : str, default KUBERNETES_GPU_VENDOR
         The vendor of the GPUs to be used for this step.
-    tolerations : List[Dict[str,str]], default []
+    tolerations : List[str], default []
         The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
         Kubernetes tolerations to use when launching pod in Kubernetes.
     labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS

metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -6,7 +6,6 @@ from collections import namedtuple
 from metaflow.exception import MetaflowException
 from metaflow.metaflow_config import KUBERNETES_JOBSET_GROUP, KUBERNETES_JOBSET_VERSION
 from metaflow.tracing import inject_tracing_vars
-from metaflow._vendor import yaml
 
 from .kube_utils import qos_requests_and_limits
 
@@ -1026,32 +1025,34 @@ class KubernetesArgoJobSet(object):
 
     def dump(self):
         client = self._kubernetes_sdk
-        js_dict = client.ApiClient().sanitize_for_serialization(
-            dict(
-                apiVersion=self._group + "/" + self._version,
-                kind="JobSet",
-                metadata=client.api_client.ApiClient().sanitize_for_serialization(
-                    client.V1ObjectMeta(
-                        name=self.name,
-                        labels=self._labels,
-                        annotations=self._annotations,
-                    )
-                ),
-                spec=dict(
-                    replicatedJobs=[self.control.dump(), self.worker.dump()],
-                    suspend=False,
-                    startupPolicy=None,
-                    successPolicy=None,
-                    # The Failure Policy helps setting the number of retries for the jobset.
-                    # but we don't rely on it and instead rely on either the local scheduler
-                    # or the Argo Workflows to handle retries.
-                    failurePolicy=None,
-                    network=None,
-                ),
-                status=None,
+
+        data = json.dumps(
+            client.ApiClient().sanitize_for_serialization(
+                dict(
+                    apiVersion=self._group + "/" + self._version,
+                    kind="JobSet",
+                    metadata=client.api_client.ApiClient().sanitize_for_serialization(
+                        client.V1ObjectMeta(
+                            name=self.name,
+                            labels=self._labels,
+                            annotations=self._annotations,
+                        )
+                    ),
+                    spec=dict(
+                        replicatedJobs=[self.control.dump(), self.worker.dump()],
+                        suspend=False,
+                        startupPolicy=None,
+                        successPolicy=None,
+                        # The Failure Policy helps setting the number of retries for the jobset.
+                        # but we don't rely on it and instead rely on either the local scheduler
+                        # or the Argo Workflows to handle retries.
+                        failurePolicy=None,
+                        network=None,
+                    ),
+                    status=None,
+                )
             )
         )
-        data = yaml.dump(js_dict, default_flow_style=False, indent=2)
         # The values we populate in the Jobset manifest (for Argo Workflows) piggybacks on the Argo Workflow's templating engine.
         # Even though Argo Workflows's templating helps us constructing all the necessary IDs and populating the fields
         # required by Metaflow, we run into one glitch. When we construct JSON/YAML serializable objects,
@@ -1066,6 +1067,7 @@ class KubernetesArgoJobSet(object):
         # Since the value of `num_parallel` can be dynamic and can change from run to run, we need to ensure that the
         # value can be passed-down dynamically and is **explicitly set as a integer** in the Jobset Manifest submitted as a
         # part of the Argo Workflow
-        quoted_substring = "'{{=asInt(inputs.parameters.workerCount)}}'"
+
+        quoted_substring = '"{{=asInt(inputs.parameters.workerCount)}}"'
         unquoted_substring = "{{=asInt(inputs.parameters.workerCount)}}"
         return data.replace(quoted_substring, unquoted_substring)

metaflow/plugins/pypi/conda_decorator.py

@@ -243,11 +243,9 @@ class CondaStepDecorator(StepDecorator):
         # Ensure local installation of Metaflow is visible to user code
         python_path = self.__class__._metaflow_home.name
         addl_env_vars = {}
-        if self.__class__._addl_env_vars:
+        if self.__class__._addl_env_vars is not None:
             for key, value in self.__class__._addl_env_vars.items():
-                if key.endswith(":"):
-                    addl_env_vars[key[:-1]] = value
-                elif key == "PYTHONPATH":
+                if key == "PYTHONPATH":
                     addl_env_vars[key] = os.pathsep.join([value, python_path])
                 else:
                     addl_env_vars[key] = value

metaflow/runner/click_api.py

@@ -43,7 +43,6 @@ from metaflow._vendor.click.types import (
 )
 from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
-from metaflow.flowspec import _FlowState
 from metaflow.includefile import FilePathClass
 from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 from metaflow.parameters import JSONTypeClass, flow_context
@@ -172,6 +171,7 @@ def _lazy_load_command(
     _self,
     name: str,
 ):
+
     # Context is not used in get_command so we can pass None. Since we pin click,
     # this won't change from under us.
 
@@ -516,11 +516,6 @@ class MetaflowAPI(object):
         # Note that if CLICK_API_PROCESS_CONFIG is False, we still do this because
         # it will init all parameters (config_options will be None)
         # We ignore any errors if we don't check the configs in the click API.
-
-        # Init all values in the flow mutators and then process them
-        for decorator in self._flow_cls._flow_state.get(_FlowState.FLOW_MUTATORS, []):
-            decorator.external_init()
-
         new_cls = self._flow_cls._process_config_decorators(
             config_options, process_configs=CLICK_API_PROCESS_CONFIG
         )
@@ -546,16 +541,14 @@ def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
 
     for each_param in cmd_obj.params:
         if isinstance(each_param, click.Argument):
-            (
-                arg_params_sigs[each_param.name],
-                annotations[each_param.name],
-            ) = get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
+            arg_params_sigs[each_param.name], annotations[each_param.name] = (
+                get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
+            )
             arg_parameters[each_param.name] = each_param
         elif isinstance(each_param, click.Option):
-            (
-                opt_params_sigs[each_param.name],
-                annotations[each_param.name],
-            ) = get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
+            opt_params_sigs[each_param.name], annotations[each_param.name] = (
+                get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
+            )
             opt_parameters[each_param.name] = each_param
 
         defaults[each_param.name] = each_param.default

metaflow/runner/deployer.py

@@ -13,9 +13,7 @@ def generate_fake_flow_file_contents(
 ):
     params_code = ""
     for _, param_details in param_info.items():
-        param_python_var_name = param_details.get(
-            "python_var_name", param_details["name"]
-        )
+        param_python_var_name = param_details["python_var_name"]
         param_name = param_details["name"]
         param_type = param_details["type"]
         param_help = param_details["description"]
@@ -231,68 +229,7 @@ class DeployedFlowMeta(type):
                 }
             )
 
-            def _get_triggered_run_injected_method():
-                def f(
-                    cls,
-                    identifier: str,
-                    run_id: str,
-                    metadata: Optional[str] = None,
-                    impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
-                ) -> "TriggeredRun":
-                    """
-                    Retrieves a `TriggeredRun` object from an identifier, a run id and optional
-                    metadata. The `impl` parameter specifies the deployer implementation
-                    to use (like `argo-workflows`).
-
-                    Parameters
-                    ----------
-                    identifier : str
-                        Deployer specific identifier for the workflow to retrieve
-                    run_id : str
-                        Run ID for the which to fetch the triggered run object
-                    metadata : str, optional, default None
-                        Optional deployer specific metadata.
-                    impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
-                        The default implementation to use if not specified
-
-                    Returns
-                    -------
-                    TriggeredRun
-                        A `TriggeredRun` object representing the triggered run corresponding
-                        to the identifier and the run id.
-                    """
-                    if impl in allowed_providers:
-                        return (
-                            allowed_providers[impl]
-                            .deployed_flow_type()
-                            .get_triggered_run(identifier, run_id, metadata)
-                        )
-                    else:
-                        raise ValueError(
-                            f"No deployer '{impl}' exists; valid deployers are: "
-                            f"{list(allowed_providers.keys())}"
-                        )
-
-                f.__name__ = "get_triggered_run"
-                return f
-
-            def _per_type_get_triggered_run_injected_method(method_name, impl):
-                def f(
-                    cls,
-                    identifier: str,
-                    run_id: str,
-                    metadata: Optional[str] = None,
-                ):
-                    return (
-                        allowed_providers[impl]
-                        .deployed_flow_type()
-                        .get_triggered_run(identifier, run_id, metadata)
-                    )
-
-                f.__name__ = method_name
-                return f
-
-            def _from_deployment_injected_method():
+            def _default_injected_method():
                 def f(
                     cls,
                     identifier: str,
@@ -334,7 +271,7 @@ class DeployedFlowMeta(type):
                 f.__name__ = "from_deployment"
                 return f
 
-            def _per_type_from_deployment_injected_method(method_name, impl):
+            def _per_type_injected_method(method_name, impl):
                 def f(
                     cls,
                     identifier: str,
@@ -349,104 +286,14 @@ class DeployedFlowMeta(type):
                 f.__name__ = method_name
                 return f
 
-            def _list_deployed_flows_injected_method():
-                def f(
-                    cls,
-                    flow_name: Optional[str] = None,
-                    impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
-                ):
-                    """
-                    List all deployed flows for the specified implementation.
-
-                    Parameters
-                    ----------
-                    flow_name : str, optional, default None
-                        If specified, only list deployed flows for this specific flow name.
-                        If None, list all deployed flows.
-                    impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
-                        The default implementation to use if not specified
-
-                    Yields
-                    ------
-                    DeployedFlow
-                        `DeployedFlow` objects representing deployed flows.
-                    """
-                    if impl in allowed_providers:
-                        return (
-                            allowed_providers[impl]
-                            .deployed_flow_type()
-                            .list_deployed_flows(flow_name)
-                        )
-                    else:
-                        raise ValueError(
-                            f"No deployer '{impl}' exists; valid deployers are: "
-                            f"{list(allowed_providers.keys())}"
-                        )
-
-                f.__name__ = "list_deployed_flows"
-                return f
-
-            def _per_type_list_deployed_flows_injected_method(method_name, impl):
-                def f(
-                    cls,
-                    flow_name: Optional[str] = None,
-                ):
-                    return (
-                        allowed_providers[impl]
-                        .deployed_flow_type()
-                        .list_deployed_flows(flow_name)
-                    )
-
-                f.__name__ = method_name
-                return f
-
-            setattr(
-                cls, "from_deployment", classmethod(_from_deployment_injected_method())
-            )
-            setattr(
-                cls,
-                "list_deployed_flows",
-                classmethod(_list_deployed_flows_injected_method()),
-            )
-            setattr(
-                cls,
-                "get_triggered_run",
-                classmethod(_get_triggered_run_injected_method()),
-            )
+            setattr(cls, "from_deployment", classmethod(_default_injected_method()))
 
             for impl in allowed_providers:
-                from_deployment_method_name = f"from_{impl}"
-                list_deployed_flows_method_name = f"list_{impl}"
-                get_triggered_run_method_name = f"get_triggered_{impl}_run"
-
-                setattr(
-                    cls,
-                    from_deployment_method_name,
-                    classmethod(
-                        _per_type_from_deployment_injected_method(
-                            from_deployment_method_name, impl
-                        )
-                    ),
-                )
-
+                method_name = f"from_{impl}"
                 setattr(
                     cls,
-                    list_deployed_flows_method_name,
-                    classmethod(
-                        _per_type_list_deployed_flows_injected_method(
-                            list_deployed_flows_method_name, impl
-                        )
-                    ),
-                )
-
-                setattr(
-                    cls,
-                    get_triggered_run_method_name,
-                    classmethod(
-                        _per_type_get_triggered_run_injected_method(
-                            get_triggered_run_method_name, impl
-                        )
-                    ),
+                    method_name,
+                    classmethod(_per_type_injected_method(method_name, impl)),
                 )
 
         return cls

metaflow/runner/subprocess_manager.py

@@ -152,20 +152,12 @@ class SubprocessManager(object):
         int
             The process ID of the subprocess.
         """
-        env = env or {}
-        installed_root = os.environ.get("METAFLOW_EXTRACTED_ROOT", get_metaflow_root())
-
-        for k, v in MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
-            installed_root
-        ).items():
-            if k.endswith(":"):
-                # Override
-                env[k[:-1]] = v
-            elif k in env:
-                env[k] = "%s:%s" % (v, env[k])
-            else:
-                env[k] = v
-
+        updated_env = MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
+            get_metaflow_root()
+        )
+        if updated_env:
+            env = env or {}
+            env.update(updated_env)
         command_obj = CommandManager(command, env, cwd)
         pid = command_obj.run(show_output=show_output)
         self.commands[pid] = command_obj
@@ -196,12 +188,12 @@ class SubprocessManager(object):
         int
             The process ID of the subprocess.
         """
-        env = env or {}
-        if "PYTHONPATH" in env:
-            env["PYTHONPATH"] = "%s:%s" % (get_metaflow_root(), env["PYTHONPATH"])
-        else:
-            env["PYTHONPATH"] = get_metaflow_root()
-
+        updated_env = MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
+            get_metaflow_root()
+        )
+        if updated_env:
+            env = env or {}
+            env.update(updated_env)
         command_obj = CommandManager(command, env, cwd)
         pid = await command_obj.async_run()
         self.commands[pid] = command_obj