ob-metaflow 2.16.4.5__tar.gz → 2.16.6.1__tar.gz

{ob_metaflow-2.16.4.5/ob_metaflow.egg-info → ob_metaflow-2.16.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.16.4.5
+Version: 2.16.6.1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.4.5; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.6.1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1}/metaflow/_vendor/imghdr/__init__.py

@@ -1,14 +1,9 @@
 """Recognize image file formats based on their first few bytes."""
 
 from os import PathLike
-import warnings
 
 __all__ = ["what"]
 
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
 #-------------------------#
 # Recognize image headers #
 #-------------------------#

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1}/metaflow/plugins/argo/argo_client.py

@@ -58,21 +58,38 @@ class ArgoClient(object):
                 json.loads(e.body)["message"] if e.body is not None else e.reason
             )
 
-    def get_workflow_templates(self):
+    def get_workflow_templates(self, page_size=100):
         client = self._client.get()
-        try:
-            return client.CustomObjectsApi().list_namespaced_custom_object(
-                group=self._group,
-                version=self._version,
-                namespace=self._namespace,
-                plural="workflowtemplates",
-            )["items"]
-        except client.rest.ApiException as e:
-            if e.status == 404:
-                return None
-            raise ArgoClientException(
-                json.loads(e.body)["message"] if e.body is not None else e.reason
-            )
+        continue_token = None
+
+        while True:
+            try:
+                params = {"limit": page_size}
+                if continue_token:
+                    params["_continue"] = continue_token
+
+                response = client.CustomObjectsApi().list_namespaced_custom_object(
+                    group=self._group,
+                    version=self._version,
+                    namespace=self._namespace,
+                    plural="workflowtemplates",
+                    **params,
+                )
+
+                for item in response.get("items", []):
+                    yield item
+
+                metadata = response.get("metadata", {})
+                continue_token = metadata.get("continue")
+
+                if not continue_token:
+                    break
+            except client.rest.ApiException as e:
+                if e.status == 404:
+                    return None
+                raise ArgoClientException(
+                    json.loads(e.body)["message"] if e.body is not None else e.reason
+                )
 
     def register_workflow_template(self, name, workflow_template):
         # Unfortunately, Kubernetes client does not handle optimistic

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1}/metaflow/plugins/argo/argo_workflows.py

@@ -216,23 +216,14 @@ class ArgoWorkflows(object):
         return name.replace(".", "-")
 
     @staticmethod
-    def list_templates(flow_name, all=False):
+    def list_templates(flow_name, all=False, page_size=100):
         client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
 
-        templates = client.get_workflow_templates()
-        if templates is None:
-            return []
-
-        template_names = [
-            template["metadata"]["name"]
-            for template in templates
-            if all
-            or flow_name
-            == template["metadata"]
-            .get("annotations", {})
-            .get("metaflow/flow_name", None)
-        ]
-        return template_names
+        for template in client.get_workflow_templates(page_size=page_size):
+            if all or flow_name == template["metadata"].get("annotations", {}).get(
+                "metaflow/flow_name", None
+            ):
+                yield template["metadata"]["name"]
 
     @staticmethod
     def delete(name):
@@ -3286,8 +3277,8 @@ class ArgoWorkflows(object):
                     Trigger().template(
                         TriggerTemplate(self.name)
                         # Trigger a deployed workflow template
-                        .k8s_trigger(
-                            StandardK8STrigger()
+                        .argo_workflow_trigger(
+                            ArgoWorkflowTrigger()
                             .source(
                                 {
                                     "resource": {
@@ -4265,10 +4256,6 @@ class TriggerTemplate(object):
         self.payload = tree()
         self.payload["name"] = name
 
-    def k8s_trigger(self, k8s_trigger):
-        self.payload["k8s"] = k8s_trigger.to_json()
-        return self
-
     def argo_workflow_trigger(self, argo_workflow_trigger):
         self.payload["argoWorkflow"] = argo_workflow_trigger.to_json()
         return self
@@ -4343,57 +4330,3 @@ class TriggerParameter(object):
 
     def __str__(self):
         return json.dumps(self.payload, indent=4)
-
-
-class StandardK8STrigger(object):
-    # https://pkg.go.dev/github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1#StandardK8STrigger
-
-    def __init__(self):
-        tree = lambda: defaultdict(tree)
-        self.payload = tree()
-        self.payload["operation"] = "create"
-
-    def operation(self, operation):
-        self.payload["operation"] = operation
-        return self
-
-    def group(self, group):
-        self.payload["group"] = group
-        return self
-
-    def version(self, version):
-        self.payload["version"] = version
-        return self
-
-    def resource(self, resource):
-        self.payload["resource"] = resource
-        return self
-
-    def namespace(self, namespace):
-        self.payload["namespace"] = namespace
-        return self
-
-    def source(self, source):
-        self.payload["source"] = source
-        return self
-
-    def parameters(self, trigger_parameters):
-        if "parameters" not in self.payload:
-            self.payload["parameters"] = []
-        for trigger_parameter in trigger_parameters:
-            self.payload["parameters"].append(trigger_parameter.to_json())
-        return self
-
-    def live_object(self, live_object=True):
-        self.payload["liveObject"] = live_object
-        return self
-
-    def patch_strategy(self, patch_strategy):
-        self.payload["patchStrategy"] = patch_strategy
-        return self
-
-    def to_json(self):
-        return self.payload
-
-    def __str__(self):
-        return json.dumps(self.payload, indent=4)

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -1011,8 +1011,7 @@ def terminate(obj, run_id, authorize=None):
 )
 @click.pass_obj
 def list_workflow_templates(obj, all=None):
-    templates = ArgoWorkflows.list_templates(obj.flow.name, all)
-    for template_name in templates:
+    for template_name in ArgoWorkflows.list_templates(obj.flow.name, all):
         obj.echo_always(template_name)
 
 

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1}/metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -203,6 +203,38 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "argo-workflows"
 
+    @classmethod
+    def list_deployed_flows(cls, flow_name: Optional[str] = None):
+        """
+        List all deployed Argo Workflow templates.
+
+        Parameters
+        ----------
+        flow_name : str, optional, default None
+            If specified, only list deployed flows for this specific flow name.
+            If None, list all deployed flows.
+
+        Yields
+        ------
+        ArgoWorkflowsDeployedFlow
+            `ArgoWorkflowsDeployedFlow` objects representing deployed
+            workflow templates on Argo Workflows.
+        """
+        from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
+
+        # When flow_name is None, use all=True to get all templates
+        # When flow_name is specified, use all=False to filter by flow_name
+        all_templates = flow_name is None
+        for template_name in ArgoWorkflows.list_templates(
+            flow_name=flow_name, all=all_templates
+        ):
+            try:
+                deployed_flow = cls.from_deployment(template_name)
+                yield deployed_flow
+            except Exception:
+                # Skip templates that can't be converted to DeployedFlow objects
+                continue
+
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1}/metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py

@@ -56,6 +56,20 @@ class StepFunctionsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "step-functions"
 
+    @classmethod
+    def list_deployed_flows(cls, flow_name: Optional[str] = None):
+        """
+        This method is not currently implemented for Step Functions.
+
+        Raises
+        ------
+        NotImplementedError
+            This method is not implemented for Step Functions.
+        """
+        raise NotImplementedError(
+            "list_deployed_flows is not implemented for StepFunctions"
+        )
+
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1}/metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -98,7 +98,7 @@ class KubernetesDecorator(StepDecorator):
         the scheduled node should not have GPUs.
     gpu_vendor : str, default KUBERNETES_GPU_VENDOR
         The vendor of the GPUs to be used for this step.
-    tolerations : List[str], default []
+    tolerations : List[Dict[str,str]], default []
         The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
         Kubernetes tolerations to use when launching pod in Kubernetes.
     labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1}/metaflow/runner/deployer.py

@@ -13,7 +13,9 @@ def generate_fake_flow_file_contents(
 ):
     params_code = ""
     for _, param_details in param_info.items():
-        param_python_var_name = param_details["python_var_name"]
+        param_python_var_name = param_details.get(
+            "python_var_name", param_details["name"]
+        )
         param_name = param_details["name"]
         param_type = param_details["type"]
         param_help = param_details["description"]
@@ -229,7 +231,7 @@ class DeployedFlowMeta(type):
                 }
             )
 
-            def _default_injected_method():
+            def _from_deployment_injected_method():
                 def f(
                     cls,
                     identifier: str,
@@ -271,7 +273,7 @@ class DeployedFlowMeta(type):
                 f.__name__ = "from_deployment"
                 return f
 
-            def _per_type_injected_method(method_name, impl):
+            def _per_type_from_deployment_injected_method(method_name, impl):
                 def f(
                     cls,
                     identifier: str,
@@ -286,14 +288,88 @@ class DeployedFlowMeta(type):
                 f.__name__ = method_name
                 return f
 
-            setattr(cls, "from_deployment", classmethod(_default_injected_method()))
+            def _list_deployed_flows_injected_method():
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                    impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
+                ):
+                    """
+                    List all deployed flows for the specified implementation.
+
+                    Parameters
+                    ----------
+                    flow_name : str, optional, default None
+                        If specified, only list deployed flows for this specific flow name.
+                        If None, list all deployed flows.
+                    impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
+                        The default implementation to use if not specified
+
+                    Yields
+                    ------
+                    DeployedFlow
+                        `DeployedFlow` objects representing deployed flows.
+                    """
+                    if impl in allowed_providers:
+                        return (
+                            allowed_providers[impl]
+                            .deployed_flow_type()
+                            .list_deployed_flows(flow_name)
+                        )
+                    else:
+                        raise ValueError(
+                            f"No deployer '{impl}' exists; valid deployers are: "
+                            f"{list(allowed_providers.keys())}"
+                        )
+
+                f.__name__ = "list_deployed_flows"
+                return f
+
+            def _per_type_list_deployed_flows_injected_method(method_name, impl):
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                ):
+                    return (
+                        allowed_providers[impl]
+                        .deployed_flow_type()
+                        .list_deployed_flows(flow_name)
+                    )
+
+                f.__name__ = method_name
+                return f
+
+            setattr(
+                cls, "from_deployment", classmethod(_from_deployment_injected_method())
+            )
+            setattr(
+                cls,
+                "list_deployed_flows",
+                classmethod(_list_deployed_flows_injected_method()),
+            )
 
             for impl in allowed_providers:
-                method_name = f"from_{impl}"
+                from_deployment_method_name = f"from_{impl}"
+                list_deployed_flows_method_name = f"list_{impl}"
+
                 setattr(
                     cls,
-                    method_name,
-                    classmethod(_per_type_injected_method(method_name, impl)),
+                    from_deployment_method_name,
+                    classmethod(
+                        _per_type_from_deployment_injected_method(
+                            from_deployment_method_name, impl
+                        )
+                    ),
+                )
+
+                setattr(
+                    cls,
+                    list_deployed_flows_method_name,
+                    classmethod(
+                        _per_type_list_deployed_flows_injected_method(
+                            list_deployed_flows_method_name, impl
+                        )
+                    ),
                 )
 
         return cls

ob_metaflow-2.16.6.1/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.16.6.1"

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1/ob_metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.16.4.5
+Version: 2.16.6.1
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.4.5; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.6.1; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.16.4.5 → ob_metaflow-2.16.6.1}/ob_metaflow.egg-info/requires.txt

@@ -4,4 +4,4 @@ pylint
 kubernetes
 
 [stubs]
-metaflow-stubs==2.16.4.5
+metaflow-stubs==2.16.6.1

ob_metaflow-2.16.4.5/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.16.4.5"