ob-metaflow 2.16.6.4rc0__py2.py3-none-any.whl → 2.16.6.5rc1__py2.py3-none-any.whl

metaflow/plugins/argo/argo_workflows.py

@@ -216,23 +216,14 @@ class ArgoWorkflows(object):
         return name.replace(".", "-")
 
     @staticmethod
-    def list_templates(flow_name, all=False):
+    def list_templates(flow_name, all=False, page_size=100):
         client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
 
-        templates = client.get_workflow_templates()
-        if templates is None:
-            return []
-
-        template_names = [
-            template["metadata"]["name"]
-            for template in templates
-            if all
-            or flow_name
-            == template["metadata"]
-            .get("annotations", {})
-            .get("metaflow/flow_name", None)
-        ]
-        return template_names
+        for template in client.get_workflow_templates(page_size=page_size):
+            if all or flow_name == template["metadata"].get("annotations", {}).get(
+                "metaflow/flow_name", None
+            ):
+                yield template["metadata"]["name"]
 
     @staticmethod
     def delete(name):
@@ -3286,8 +3277,8 @@ class ArgoWorkflows(object):
                     Trigger().template(
                         TriggerTemplate(self.name)
                         # Trigger a deployed workflow template
-                        .argo_workflow_trigger(
-                            ArgoWorkflowTrigger()
+                        .k8s_trigger(
+                            StandardK8STrigger()
                             .source(
                                 {
                                     "resource": {
@@ -4265,6 +4256,10 @@ class TriggerTemplate(object):
         self.payload = tree()
         self.payload["name"] = name
 
+    def k8s_trigger(self, k8s_trigger):
+        self.payload["k8s"] = k8s_trigger.to_json()
+        return self
+
     def argo_workflow_trigger(self, argo_workflow_trigger):
         self.payload["argoWorkflow"] = argo_workflow_trigger.to_json()
         return self
@@ -4339,3 +4334,57 @@ class TriggerParameter(object):
 
     def __str__(self):
         return json.dumps(self.payload, indent=4)
+
+
+class StandardK8STrigger(object):
+    # https://pkg.go.dev/github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1#StandardK8STrigger
+
+    def __init__(self):
+        tree = lambda: defaultdict(tree)
+        self.payload = tree()
+        self.payload["operation"] = "create"
+
+    def operation(self, operation):
+        self.payload["operation"] = operation
+        return self
+
+    def group(self, group):
+        self.payload["group"] = group
+        return self
+
+    def version(self, version):
+        self.payload["version"] = version
+        return self
+
+    def resource(self, resource):
+        self.payload["resource"] = resource
+        return self
+
+    def namespace(self, namespace):
+        self.payload["namespace"] = namespace
+        return self
+
+    def source(self, source):
+        self.payload["source"] = source
+        return self
+
+    def parameters(self, trigger_parameters):
+        if "parameters" not in self.payload:
+            self.payload["parameters"] = []
+        for trigger_parameter in trigger_parameters:
+            self.payload["parameters"].append(trigger_parameter.to_json())
+        return self
+
+    def live_object(self, live_object=True):
+        self.payload["liveObject"] = live_object
+        return self
+
+    def patch_strategy(self, patch_strategy):
+        self.payload["patchStrategy"] = patch_strategy
+        return self
+
+    def to_json(self):
+        return self.payload
+
+    def __str__(self):
+        return json.dumps(self.payload, indent=4)

metaflow/plugins/argo/argo_workflows_cli.py

@@ -1011,8 +1011,7 @@ def terminate(obj, run_id, authorize=None):
 )
 @click.pass_obj
 def list_workflow_templates(obj, all=None):
-    templates = ArgoWorkflows.list_templates(obj.flow.name, all)
-    for template_name in templates:
+    for template_name in ArgoWorkflows.list_templates(obj.flow.name, all):
         obj.echo_always(template_name)
 
 

metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -203,6 +203,38 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "argo-workflows"
 
+    @classmethod
+    def list_deployed_flows(cls, flow_name: Optional[str] = None):
+        """
+        List all deployed Argo Workflow templates.
+
+        Parameters
+        ----------
+        flow_name : str, optional, default None
+            If specified, only list deployed flows for this specific flow name.
+            If None, list all deployed flows.
+
+        Yields
+        ------
+        ArgoWorkflowsDeployedFlow
+            `ArgoWorkflowsDeployedFlow` objects representing deployed
+            workflow templates on Argo Workflows.
+        """
+        from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
+
+        # When flow_name is None, use all=True to get all templates
+        # When flow_name is specified, use all=False to filter by flow_name
+        all_templates = flow_name is None
+        for template_name in ArgoWorkflows.list_templates(
+            flow_name=flow_name, all=all_templates
+        ):
+            try:
+                deployed_flow = cls.from_deployment(template_name)
+                yield deployed_flow
+            except Exception:
+                # Skip templates that can't be converted to DeployedFlow objects
+                continue
+
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """

metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py

@@ -56,6 +56,20 @@ class StepFunctionsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "step-functions"
 
+    @classmethod
+    def list_deployed_flows(cls, flow_name: Optional[str] = None):
+        """
+        This method is not currently implemented for Step Functions.
+
+        Raises
+        ------
+        NotImplementedError
+            This method is not implemented for Step Functions.
+        """
+        raise NotImplementedError(
+            "list_deployed_flows is not implemented for StepFunctions"
+        )
+
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """

metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -98,7 +98,7 @@ class KubernetesDecorator(StepDecorator):
         the scheduled node should not have GPUs.
     gpu_vendor : str, default KUBERNETES_GPU_VENDOR
         The vendor of the GPUs to be used for this step.
-    tolerations : List[str], default []
+    tolerations : List[Dict[str,str]], default []
         The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
         Kubernetes tolerations to use when launching pod in Kubernetes.
     labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS

metaflow/plugins/pypi/conda_decorator.py

@@ -243,11 +243,9 @@ class CondaStepDecorator(StepDecorator):
         # Ensure local installation of Metaflow is visible to user code
         python_path = self.__class__._metaflow_home.name
         addl_env_vars = {}
-        if self.__class__._addl_env_vars:
+        if self.__class__._addl_env_vars is not None:
             for key, value in self.__class__._addl_env_vars.items():
-                if key.endswith(":"):
-                    addl_env_vars[key[:-1]] = value
-                elif key == "PYTHONPATH":
+                if key == "PYTHONPATH":
                     addl_env_vars[key] = os.pathsep.join([value, python_path])
                 else:
                     addl_env_vars[key] = value

metaflow/runner/click_api.py

@@ -43,7 +43,6 @@ from metaflow._vendor.click.types import (
 )
 from metaflow.decorators import add_decorator_options
 from metaflow.exception import MetaflowException
-from metaflow.flowspec import _FlowState
 from metaflow.includefile import FilePathClass
 from metaflow.metaflow_config import CLICK_API_PROCESS_CONFIG
 from metaflow.parameters import JSONTypeClass, flow_context
@@ -172,6 +171,7 @@ def _lazy_load_command(
     _self,
     name: str,
 ):
+
     # Context is not used in get_command so we can pass None. Since we pin click,
     # this won't change from under us.
 
@@ -516,11 +516,6 @@ class MetaflowAPI(object):
         # Note that if CLICK_API_PROCESS_CONFIG is False, we still do this because
         # it will init all parameters (config_options will be None)
         # We ignore any errors if we don't check the configs in the click API.
-
-        # Init all values in the flow mutators and then process them
-        for decorator in self._flow_cls._flow_state.get(_FlowState.FLOW_MUTATORS, []):
-            decorator.external_init()
-
         new_cls = self._flow_cls._process_config_decorators(
             config_options, process_configs=CLICK_API_PROCESS_CONFIG
         )
@@ -546,16 +541,14 @@ def extract_all_params(cmd_obj: Union[click.Command, click.Group]):
 
     for each_param in cmd_obj.params:
         if isinstance(each_param, click.Argument):
-            (
-                arg_params_sigs[each_param.name],
-                annotations[each_param.name],
-            ) = get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
+            arg_params_sigs[each_param.name], annotations[each_param.name] = (
+                get_inspect_param_obj(each_param, inspect.Parameter.POSITIONAL_ONLY)
+            )
             arg_parameters[each_param.name] = each_param
         elif isinstance(each_param, click.Option):
-            (
-                opt_params_sigs[each_param.name],
-                annotations[each_param.name],
-            ) = get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
+            opt_params_sigs[each_param.name], annotations[each_param.name] = (
+                get_inspect_param_obj(each_param, inspect.Parameter.KEYWORD_ONLY)
+            )
             opt_parameters[each_param.name] = each_param
 
         defaults[each_param.name] = each_param.default

metaflow/runner/deployer.py

@@ -13,7 +13,9 @@ def generate_fake_flow_file_contents(
 ):
     params_code = ""
     for _, param_details in param_info.items():
-        param_python_var_name = param_details["python_var_name"]
+        param_python_var_name = param_details.get(
+            "python_var_name", param_details["name"]
+        )
         param_name = param_details["name"]
         param_type = param_details["type"]
         param_help = param_details["description"]
@@ -229,7 +231,7 @@ class DeployedFlowMeta(type):
                 }
             )
 
-            def _default_injected_method():
+            def _from_deployment_injected_method():
                 def f(
                     cls,
                     identifier: str,
@@ -271,7 +273,7 @@ class DeployedFlowMeta(type):
                 f.__name__ = "from_deployment"
                 return f
 
-            def _per_type_injected_method(method_name, impl):
+            def _per_type_from_deployment_injected_method(method_name, impl):
                 def f(
                     cls,
                     identifier: str,
@@ -286,14 +288,88 @@ class DeployedFlowMeta(type):
                 f.__name__ = method_name
                 return f
 
-            setattr(cls, "from_deployment", classmethod(_default_injected_method()))
+            def _list_deployed_flows_injected_method():
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                    impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
+                ):
+                    """
+                    List all deployed flows for the specified implementation.
+
+                    Parameters
+                    ----------
+                    flow_name : str, optional, default None
+                        If specified, only list deployed flows for this specific flow name.
+                        If None, list all deployed flows.
+                    impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
+                        The default implementation to use if not specified
+
+                    Yields
+                    ------
+                    DeployedFlow
+                        `DeployedFlow` objects representing deployed flows.
+                    """
+                    if impl in allowed_providers:
+                        return (
+                            allowed_providers[impl]
+                            .deployed_flow_type()
+                            .list_deployed_flows(flow_name)
+                        )
+                    else:
+                        raise ValueError(
+                            f"No deployer '{impl}' exists; valid deployers are: "
+                            f"{list(allowed_providers.keys())}"
+                        )
+
+                f.__name__ = "list_deployed_flows"
+                return f
+
+            def _per_type_list_deployed_flows_injected_method(method_name, impl):
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                ):
+                    return (
+                        allowed_providers[impl]
+                        .deployed_flow_type()
+                        .list_deployed_flows(flow_name)
+                    )
+
+                f.__name__ = method_name
+                return f
+
+            setattr(
+                cls, "from_deployment", classmethod(_from_deployment_injected_method())
+            )
+            setattr(
+                cls,
+                "list_deployed_flows",
+                classmethod(_list_deployed_flows_injected_method()),
+            )
 
             for impl in allowed_providers:
-                method_name = f"from_{impl}"
+                from_deployment_method_name = f"from_{impl}"
+                list_deployed_flows_method_name = f"list_{impl}"
+
                 setattr(
                     cls,
-                    method_name,
-                    classmethod(_per_type_injected_method(method_name, impl)),
+                    from_deployment_method_name,
+                    classmethod(
+                        _per_type_from_deployment_injected_method(
+                            from_deployment_method_name, impl
+                        )
+                    ),
+                )
+
+                setattr(
+                    cls,
+                    list_deployed_flows_method_name,
+                    classmethod(
+                        _per_type_list_deployed_flows_injected_method(
+                            list_deployed_flows_method_name, impl
+                        )
+                    ),
                 )
 
         return cls

metaflow/runner/subprocess_manager.py

@@ -152,20 +152,12 @@ class SubprocessManager(object):
         int
             The process ID of the subprocess.
         """
-        env = env or {}
-        installed_root = os.environ.get("METAFLOW_EXTRACTED_ROOT", get_metaflow_root())
-
-        for k, v in MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
-            installed_root
-        ).items():
-            if k.endswith(":"):
-                # Override
-                env[k[:-1]] = v
-            elif k in env:
-                env[k] = "%s:%s" % (v, env[k])
-            else:
-                env[k] = v
-
+        updated_env = MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
+            get_metaflow_root()
+        )
+        if updated_env:
+            env = env or {}
+            env.update(updated_env)
         command_obj = CommandManager(command, env, cwd)
         pid = command_obj.run(show_output=show_output)
         self.commands[pid] = command_obj
@@ -196,12 +188,12 @@ class SubprocessManager(object):
         int
             The process ID of the subprocess.
         """
-        env = env or {}
-        if "PYTHONPATH" in env:
-            env["PYTHONPATH"] = "%s:%s" % (get_metaflow_root(), env["PYTHONPATH"])
-        else:
-            env["PYTHONPATH"] = get_metaflow_root()
-
+        updated_env = MetaflowCodeContent.get_env_vars_for_packaged_metaflow(
+            get_metaflow_root()
+        )
+        if updated_env:
+            env = env or {}
+            env.update(updated_env)
         command_obj = CommandManager(command, env, cwd)
         pid = await command_obj.async_run()
         self.commands[pid] = command_obj

metaflow/user_decorators/mutable_flow.py

@@ -347,8 +347,10 @@ class MutableFlow:
                     "Mutable flow adding flow decorator '%s'" % deco_type
                 )
 
+            # self._flow_cls._flow_decorators is a  dictionary of form :
+            # <deco_name> : [deco_instance, deco_instance, ...]
             existing_deco = [
-                d for d in self._flow_cls._flow_decorators if d.name == flow_deco.name
+                d for d in self._flow_cls._flow_decorators if d == flow_deco.name
             ]
 
             if flow_deco.allow_multiple or not existing_deco:

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.16.6.4rc0"
+metaflow_version = "2.16.6.5rc1"

{ob_metaflow-2.16.6.4rc0.data → ob_metaflow-2.16.6.5rc1.data}/data/share/metaflow/devtools/Tiltfile

@@ -14,6 +14,17 @@
 version_settings(constraint='>=0.22.2')
 allow_k8s_contexts('minikube')
 
+# Version configuration for components
+JOBSET_VERSION = os.getenv("JOBSET_VERSION", "v0.6.0")
+
+# Argo Workflows versions
+ARGO_WORKFLOWS_HELM_CHART_VERSION = os.getenv("ARGO_WORKFLOWS_HELM_CHART_VERSION", "0.45.2")  # Helm chart version
+ARGO_WORKFLOWS_IMAGE_TAG = os.getenv("ARGO_WORKFLOWS_IMAGE_TAG", "v3.6.0")  # Argo Workflows application version
+
+# Argo Events versions  
+ARGO_EVENTS_HELM_CHART_VERSION = os.getenv("ARGO_EVENTS_HELM_CHART_VERSION", "2.4.8")  # Helm chart version
+ARGO_EVENTS_IMAGE_TAG = os.getenv("ARGO_EVENTS_IMAGE_TAG", "v1.9.2")  # Argo Events application version
+
 components = {
     "metadata-service": ["postgresql"],
     "ui": ["postgresql", "minio"],
@@ -21,9 +32,10 @@ components = {
     "postgresql": [],
     "argo-workflows": [],
     "argo-events": ["argo-workflows"],
+    "jobset": [],
 }
 
-services_env = os.getenv("SERVICES", "").strip().lower()
+services_env = os.getenv("SERVICES", "all").strip().lower()
 
 if services_env:
     if services_env == "all":
@@ -205,6 +217,7 @@ if "postgresql" in enabled_components:
 if "argo-workflows" in enabled_components:
     helm_remote(
         'argo-workflows',
+        version=ARGO_WORKFLOWS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -220,7 +233,9 @@ if "argo-workflows" in enabled_components:
             'controller.resources.requests.memory=128Mi',
             'controller.resources.requests.cpu=50m',
             'controller.resources.limits.memory=256Mi',
-            'controller.resources.limits.cpu=100m'
+            'controller.resources.limits.cpu=100m',
+            # Image version overrides
+            'images.tag=%s' % ARGO_WORKFLOWS_IMAGE_TAG,
         ]
     )
 
@@ -307,6 +322,7 @@ if "argo-workflows" in enabled_components:
 if "argo-events" in enabled_components:
     helm_remote(
         'argo-events',
+        version=ARGO_EVENTS_HELM_CHART_VERSION,
         repo_name='argo',
         repo_url='https://argoproj.github.io/argo-helm',
         set=[
@@ -334,6 +350,8 @@ if "argo-events" in enabled_components:
             'configs.jetstream.versions[1].natsImage=nats:2.9.15',
             'configs.jetstream.versions[1].startCommand=/nats-server',
             'configs.jetstream.versions[1].version=2.9.15',
+            # Image version overrides
+            'global.image.tag=%s' % ARGO_EVENTS_IMAGE_TAG,
         ]
     )
 
@@ -541,6 +559,29 @@ if "argo-events" in enabled_components:
     config_resources.append('argo-events-controller-manager')
     config_resources.append('argo-events-webhook-eventsource-svc')
 
+#################################################
+# JOBSET
+#################################################
+if "jobset" in enabled_components:
+    # Apply JobSet manifests directly from GitHub releases
+    jobset_manifest_url = "https://github.com/kubernetes-sigs/jobset/releases/download/%s/manifests.yaml" % JOBSET_VERSION
+
+    cmd = "curl -sSL %s" % (jobset_manifest_url)
+    k8s_yaml(
+        local(
+            cmd,
+        )
+    )
+
+    k8s_resource(
+        'jobset-controller-manager',
+        labels=['jobset'],
+    )
+
+    metaflow_config["METAFLOW_KUBERNETES_JOBSET_ENABLED"] = "true"
+    
+    config_resources.append('jobset-controller-manager')
+
 #################################################
 # METADATA SERVICE
 #################################################

{ob_metaflow-2.16.6.4rc0.data → ob_metaflow-2.16.6.5rc1.data}/data/share/metaflow/devtools/pick_services.sh

@@ -18,6 +18,7 @@ SERVICE_OPTIONS=(
     "ui"
     "argo-workflows"
     "argo-events"
+    "jobset"
 )
 
 gum style "$LOGO" \

{ob_metaflow-2.16.6.4rc0.dist-info → ob_metaflow-2.16.6.5rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.16.6.4rc0
+Version: 2.16.6.5rc1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.6.4rc0; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.6.5rc1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description