metaflow 2.16.4__tar.gz → 2.16.5__tar.gz

{metaflow-2.16.4/metaflow.egg-info → metaflow-2.16.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.16.4
+Version: 2.16.5
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.4; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.5; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.16.4 → metaflow-2.16.5}/metaflow/_vendor/imghdr/__init__.py

@@ -1,14 +1,9 @@
 """Recognize image file formats based on their first few bytes."""
 
 from os import PathLike
-import warnings
 
 __all__ = ["what"]
 
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
 #-------------------------#
 # Recognize image headers #
 #-------------------------#

{metaflow-2.16.4 → metaflow-2.16.5}/metaflow/plugins/argo/argo_client.py

@@ -58,21 +58,38 @@ class ArgoClient(object):
                 json.loads(e.body)["message"] if e.body is not None else e.reason
             )
 
-    def get_workflow_templates(self):
+    def get_workflow_templates(self, page_size=100):
         client = self._client.get()
-        try:
-            return client.CustomObjectsApi().list_namespaced_custom_object(
-                group=self._group,
-                version=self._version,
-                namespace=self._namespace,
-                plural="workflowtemplates",
-            )["items"]
-        except client.rest.ApiException as e:
-            if e.status == 404:
-                return None
-            raise ArgoClientException(
-                json.loads(e.body)["message"] if e.body is not None else e.reason
-            )
+        continue_token = None
+
+        while True:
+            try:
+                params = {"limit": page_size}
+                if continue_token:
+                    params["_continue"] = continue_token
+
+                response = client.CustomObjectsApi().list_namespaced_custom_object(
+                    group=self._group,
+                    version=self._version,
+                    namespace=self._namespace,
+                    plural="workflowtemplates",
+                    **params,
+                )
+
+                for item in response.get("items", []):
+                    yield item
+
+                metadata = response.get("metadata", {})
+                continue_token = metadata.get("continue")
+
+                if not continue_token:
+                    break
+            except client.rest.ApiException as e:
+                if e.status == 404:
+                    return None
+                raise ArgoClientException(
+                    json.loads(e.body)["message"] if e.body is not None else e.reason
+                )
 
     def register_workflow_template(self, name, workflow_template):
         # Unfortunately, Kubernetes client does not handle optimistic

{metaflow-2.16.4 → metaflow-2.16.5}/metaflow/plugins/argo/argo_workflows.py

@@ -207,23 +207,14 @@ class ArgoWorkflows(object):
         return name.replace(".", "-")
 
     @staticmethod
-    def list_templates(flow_name, all=False):
+    def list_templates(flow_name, all=False, page_size=100):
         client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
 
-        templates = client.get_workflow_templates()
-        if templates is None:
-            return []
-
-        template_names = [
-            template["metadata"]["name"]
-            for template in templates
-            if all
-            or flow_name
-            == template["metadata"]
-            .get("annotations", {})
-            .get("metaflow/flow_name", None)
-        ]
-        return template_names
+        for template in client.get_workflow_templates(page_size=page_size):
+            if all or flow_name == template["metadata"].get("annotations", {}).get(
+                "metaflow/flow_name", None
+            ):
+                yield template["metadata"]["name"]
 
     @staticmethod
     def delete(name):

{metaflow-2.16.4 → metaflow-2.16.5}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -1011,8 +1011,7 @@ def terminate(obj, run_id, authorize=None):
 )
 @click.pass_obj
 def list_workflow_templates(obj, all=None):
-    templates = ArgoWorkflows.list_templates(obj.flow.name, all)
-    for template_name in templates:
+    for template_name in ArgoWorkflows.list_templates(obj.flow.name, all):
         obj.echo_always(template_name)
 
 

{metaflow-2.16.4 → metaflow-2.16.5}/metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -203,6 +203,38 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "argo-workflows"
 
+    @classmethod
+    def list_deployed_flows(cls, flow_name: Optional[str] = None):
+        """
+        List all deployed Argo Workflow templates.
+
+        Parameters
+        ----------
+        flow_name : str, optional, default None
+            If specified, only list deployed flows for this specific flow name.
+            If None, list all deployed flows.
+
+        Yields
+        ------
+        ArgoWorkflowsDeployedFlow
+            `ArgoWorkflowsDeployedFlow` objects representing deployed
+            workflow templates on Argo Workflows.
+        """
+        from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
+
+        # When flow_name is None, use all=True to get all templates
+        # When flow_name is specified, use all=False to filter by flow_name
+        all_templates = flow_name is None
+        for template_name in ArgoWorkflows.list_templates(
+            flow_name=flow_name, all=all_templates
+        ):
+            try:
+                deployed_flow = cls.from_deployment(template_name)
+                yield deployed_flow
+            except Exception:
+                # Skip templates that can't be converted to DeployedFlow objects
+                continue
+
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """

{metaflow-2.16.4 → metaflow-2.16.5}/metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py

@@ -56,6 +56,20 @@ class StepFunctionsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "step-functions"
 
+    @classmethod
+    def list_deployed_flows(cls, flow_name: Optional[str] = None):
+        """
+        This method is not currently implemented for Step Functions.
+
+        Raises
+        ------
+        NotImplementedError
+            This method is not implemented for Step Functions.
+        """
+        raise NotImplementedError(
+            "list_deployed_flows is not implemented for StepFunctions"
+        )
+
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """

{metaflow-2.16.4 → metaflow-2.16.5}/metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -96,7 +96,7 @@ class KubernetesDecorator(StepDecorator):
         the scheduled node should not have GPUs.
     gpu_vendor : str, default KUBERNETES_GPU_VENDOR
         The vendor of the GPUs to be used for this step.
-    tolerations : List[str], default []
+    tolerations : List[Dict[str,str]], default []
         The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
         Kubernetes tolerations to use when launching pod in Kubernetes.
     labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS

{metaflow-2.16.4 → metaflow-2.16.5}/metaflow/runner/deployer.py

@@ -229,7 +229,7 @@ class DeployedFlowMeta(type):
                 }
             )
 
-            def _default_injected_method():
+            def _from_deployment_injected_method():
                 def f(
                     cls,
                     identifier: str,
@@ -271,7 +271,7 @@ class DeployedFlowMeta(type):
                 f.__name__ = "from_deployment"
                 return f
 
-            def _per_type_injected_method(method_name, impl):
+            def _per_type_from_deployment_injected_method(method_name, impl):
                 def f(
                     cls,
                     identifier: str,
@@ -286,14 +286,88 @@ class DeployedFlowMeta(type):
                 f.__name__ = method_name
                 return f
 
-            setattr(cls, "from_deployment", classmethod(_default_injected_method()))
+            def _list_deployed_flows_injected_method():
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                    impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
+                ):
+                    """
+                    List all deployed flows for the specified implementation.
+
+                    Parameters
+                    ----------
+                    flow_name : str, optional, default None
+                        If specified, only list deployed flows for this specific flow name.
+                        If None, list all deployed flows.
+                    impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
+                        The default implementation to use if not specified
+
+                    Yields
+                    ------
+                    DeployedFlow
+                        `DeployedFlow` objects representing deployed flows.
+                    """
+                    if impl in allowed_providers:
+                        return (
+                            allowed_providers[impl]
+                            .deployed_flow_type()
+                            .list_deployed_flows(flow_name)
+                        )
+                    else:
+                        raise ValueError(
+                            f"No deployer '{impl}' exists; valid deployers are: "
+                            f"{list(allowed_providers.keys())}"
+                        )
+
+                f.__name__ = "list_deployed_flows"
+                return f
+
+            def _per_type_list_deployed_flows_injected_method(method_name, impl):
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                ):
+                    return (
+                        allowed_providers[impl]
+                        .deployed_flow_type()
+                        .list_deployed_flows(flow_name)
+                    )
+
+                f.__name__ = method_name
+                return f
+
+            setattr(
+                cls, "from_deployment", classmethod(_from_deployment_injected_method())
+            )
+            setattr(
+                cls,
+                "list_deployed_flows",
+                classmethod(_list_deployed_flows_injected_method()),
+            )
 
             for impl in allowed_providers:
-                method_name = f"from_{impl}"
+                from_deployment_method_name = f"from_{impl}"
+                list_deployed_flows_method_name = f"list_{impl}"
+
                 setattr(
                     cls,
-                    method_name,
-                    classmethod(_per_type_injected_method(method_name, impl)),
+                    from_deployment_method_name,
+                    classmethod(
+                        _per_type_from_deployment_injected_method(
+                            from_deployment_method_name, impl
+                        )
+                    ),
+                )
+
+                setattr(
+                    cls,
+                    list_deployed_flows_method_name,
+                    classmethod(
+                        _per_type_list_deployed_flows_injected_method(
+                            list_deployed_flows_method_name, impl
+                        )
+                    ),
                 )
 
         return cls

metaflow-2.16.5/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.16.5"

{metaflow-2.16.4 → metaflow-2.16.5/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: metaflow
-Version: 2.16.4
+Version: 2.16.5
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.16.4; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.16.5; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{metaflow-2.16.4 → metaflow-2.16.5}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.16.4
+metaflow-stubs==2.16.5

metaflow-2.16.4/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.16.4"