ob-metaflow 2.15.21.3__py2.py3-none-any.whl → 2.15.21.4__py2.py3-none-any.whl

metaflow/plugins/argo/argo_client.py

@@ -58,21 +58,38 @@ class ArgoClient(object):
                 json.loads(e.body)["message"] if e.body is not None else e.reason
             )
 
-    def get_workflow_templates(self):
+    def get_workflow_templates(self, page_size=100):
         client = self._client.get()
-        try:
-            return client.CustomObjectsApi().list_namespaced_custom_object(
-                group=self._group,
-                version=self._version,
-                namespace=self._namespace,
-                plural="workflowtemplates",
-            )["items"]
-        except client.rest.ApiException as e:
-            if e.status == 404:
-                return None
-            raise ArgoClientException(
-                json.loads(e.body)["message"] if e.body is not None else e.reason
-            )
+        continue_token = None
+
+        while True:
+            try:
+                params = {"limit": page_size}
+                if continue_token:
+                    params["_continue"] = continue_token
+
+                response = client.CustomObjectsApi().list_namespaced_custom_object(
+                    group=self._group,
+                    version=self._version,
+                    namespace=self._namespace,
+                    plural="workflowtemplates",
+                    **params,
+                )
+
+                for item in response.get("items", []):
+                    yield item
+
+                metadata = response.get("metadata", {})
+                continue_token = metadata.get("continue")
+
+                if not continue_token:
+                    break
+            except client.rest.ApiException as e:
+                if e.status == 404:
+                    return None
+                raise ArgoClientException(
+                    json.loads(e.body)["message"] if e.body is not None else e.reason
+                )
 
     def register_workflow_template(self, name, workflow_template):
         # Unfortunately, Kubernetes client does not handle optimistic

metaflow/plugins/argo/argo_workflows.py

@@ -214,23 +214,14 @@ class ArgoWorkflows(object):
         return name.replace(".", "-")
 
     @staticmethod
-    def list_templates(flow_name, all=False):
+    def list_templates(flow_name, all=False, page_size=100):
         client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
 
-        templates = client.get_workflow_templates()
-        if templates is None:
-            return []
-
-        template_names = [
-            template["metadata"]["name"]
-            for template in templates
-            if all
-            or flow_name
-            == template["metadata"]
-            .get("annotations", {})
-            .get("metaflow/flow_name", None)
-        ]
-        return template_names
+        for template in client.get_workflow_templates(page_size=page_size):
+            if all or flow_name == template["metadata"].get("annotations", {}).get(
+                "metaflow/flow_name", None
+            ):
+                yield template["metadata"]["name"]
 
     @staticmethod
     def delete(name):

metaflow/plugins/argo/argo_workflows_cli.py

@@ -998,8 +998,7 @@ def terminate(obj, run_id, authorize=None):
 )
 @click.pass_obj
 def list_workflow_templates(obj, all=None):
-    templates = ArgoWorkflows.list_templates(obj.flow.name, all)
-    for template_name in templates:
+    for template_name in ArgoWorkflows.list_templates(obj.flow.name, all):
         obj.echo_always(template_name)
 
 

metaflow/plugins/argo/argo_workflows_deployer_objects.py

@@ -203,6 +203,38 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "argo-workflows"
 
+    @classmethod
+    def list_deployed_flows(cls, flow_name: Optional[str] = None):
+        """
+        List all deployed Argo Workflow templates.
+
+        Parameters
+        ----------
+        flow_name : str, optional, default None
+            If specified, only list deployed flows for this specific flow name.
+            If None, list all deployed flows.
+
+        Yields
+        ------
+        ArgoWorkflowsDeployedFlow
+            `ArgoWorkflowsDeployedFlow` objects representing deployed
+            workflow templates on Argo Workflows.
+        """
+        from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
+
+        # When flow_name is None, use all=True to get all templates
+        # When flow_name is specified, use all=False to filter by flow_name
+        all_templates = flow_name is None
+        for template_name in ArgoWorkflows.list_templates(
+            flow_name=flow_name, all=all_templates
+        ):
+            try:
+                deployed_flow = cls.from_deployment(template_name)
+                yield deployed_flow
+            except Exception:
+                # Skip templates that can't be converted to DeployedFlow objects
+                continue
+
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """

metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py

@@ -56,6 +56,20 @@ class StepFunctionsDeployedFlow(DeployedFlow):
 
     TYPE: ClassVar[Optional[str]] = "step-functions"
 
+    @classmethod
+    def list_deployed_flows(cls, flow_name: Optional[str] = None):
+        """
+        This method is not currently implemented for Step Functions.
+
+        Raises
+        ------
+        NotImplementedError
+            This method is not implemented for Step Functions.
+        """
+        raise NotImplementedError(
+            "list_deployed_flows is not implemented for StepFunctions"
+        )
+
     @classmethod
     def from_deployment(cls, identifier: str, metadata: Optional[str] = None):
         """

metaflow/runner/deployer.py

@@ -13,7 +13,9 @@ def generate_fake_flow_file_contents(
 ):
     params_code = ""
     for _, param_details in param_info.items():
-        param_python_var_name = param_details["python_var_name"]
+        param_python_var_name = param_details.get(
+            "python_var_name", param_details["name"]
+        )
         param_name = param_details["name"]
         param_type = param_details["type"]
         param_help = param_details["description"]
@@ -229,7 +231,7 @@ class DeployedFlowMeta(type):
                 }
             )
 
-            def _default_injected_method():
+            def _from_deployment_injected_method():
                 def f(
                     cls,
                     identifier: str,
@@ -271,7 +273,7 @@ class DeployedFlowMeta(type):
                 f.__name__ = "from_deployment"
                 return f
 
-            def _per_type_injected_method(method_name, impl):
+            def _per_type_from_deployment_injected_method(method_name, impl):
                 def f(
                     cls,
                     identifier: str,
@@ -286,14 +288,88 @@ class DeployedFlowMeta(type):
                 f.__name__ = method_name
                 return f
 
-            setattr(cls, "from_deployment", classmethod(_default_injected_method()))
+            def _list_deployed_flows_injected_method():
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                    impl: str = DEFAULT_FROM_DEPLOYMENT_IMPL.replace("-", "_"),
+                ):
+                    """
+                    List all deployed flows for the specified implementation.
+
+                    Parameters
+                    ----------
+                    flow_name : str, optional, default None
+                        If specified, only list deployed flows for this specific flow name.
+                        If None, list all deployed flows.
+                    impl : str, optional, default given by METAFLOW_DEFAULT_FROM_DEPLOYMENT_IMPL
+                        The default implementation to use if not specified
+
+                    Yields
+                    ------
+                    DeployedFlow
+                        `DeployedFlow` objects representing deployed flows.
+                    """
+                    if impl in allowed_providers:
+                        return (
+                            allowed_providers[impl]
+                            .deployed_flow_type()
+                            .list_deployed_flows(flow_name)
+                        )
+                    else:
+                        raise ValueError(
+                            f"No deployer '{impl}' exists; valid deployers are: "
+                            f"{list(allowed_providers.keys())}"
+                        )
+
+                f.__name__ = "list_deployed_flows"
+                return f
+
+            def _per_type_list_deployed_flows_injected_method(method_name, impl):
+                def f(
+                    cls,
+                    flow_name: Optional[str] = None,
+                ):
+                    return (
+                        allowed_providers[impl]
+                        .deployed_flow_type()
+                        .list_deployed_flows(flow_name)
+                    )
+
+                f.__name__ = method_name
+                return f
+
+            setattr(
+                cls, "from_deployment", classmethod(_from_deployment_injected_method())
+            )
+            setattr(
+                cls,
+                "list_deployed_flows",
+                classmethod(_list_deployed_flows_injected_method()),
+            )
 
             for impl in allowed_providers:
-                method_name = f"from_{impl}"
+                from_deployment_method_name = f"from_{impl}"
+                list_deployed_flows_method_name = f"list_{impl}"
+
                 setattr(
                     cls,
-                    method_name,
-                    classmethod(_per_type_injected_method(method_name, impl)),
+                    from_deployment_method_name,
+                    classmethod(
+                        _per_type_from_deployment_injected_method(
+                            from_deployment_method_name, impl
+                        )
+                    ),
+                )
+
+                setattr(
+                    cls,
+                    list_deployed_flows_method_name,
+                    classmethod(
+                        _per_type_list_deployed_flows_injected_method(
+                            list_deployed_flows_method_name, impl
+                        )
+                    ),
                 )
 
         return cls

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.15.21.3"
+metaflow_version = "2.15.21.4"

{ob_metaflow-2.15.21.3.dist-info → ob_metaflow-2.15.21.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ob-metaflow
-Version: 2.15.21.3
+Version: 2.15.21.4
 Summary: Metaflow: More AI and ML, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.15.21.3; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.15.21.4; extra == "stubs"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{ob_metaflow-2.15.21.3.dist-info → ob_metaflow-2.15.21.4.dist-info}/RECORD

@@ -37,7 +37,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=MCXCjcGwpuR7y9euBf1GTNRHPtlh6fCpdPMEtbULefw,14554
 metaflow/vendor.py,sha256=EDZokNMrx1PU07jNMiWFMFtC7TL03pMXZ1kKn13k-2g,5139
-metaflow/version.py,sha256=be-sc_5dofoh8el7z-WGvRCpZomzDz1gk9ZlFh1uoxA,31
+metaflow/version.py,sha256=_6ahaL9ST3eCHwsCdRMSQZvyja51QIOxl5SMPwTZciQ,31
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=q9zxWa6p6CzF1zZvSkygSlklduHf_b3K7MCxGz7MJRc,134519
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -205,13 +205,13 @@ metaflow/plugins/airflow/sensors/base_sensor.py,sha256=s-OQBfPWZ_T3wn96Ua59CCEj1
 metaflow/plugins/airflow/sensors/external_task_sensor.py,sha256=zhYlrZnXT20KW8-fVk0fCNtTyNiKJB5PMVASacu30r0,6034
 metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqNA90nARrjjjEEk_x4,3275
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/plugins/argo/argo_client.py,sha256=A1kI9rjVjCadDsBscZ2Wk8xRBI6GNgWV6SU7TyrdfrI,16530
+metaflow/plugins/argo/argo_client.py,sha256=jLz0FjCTBvFLZt-8lZcMQhDcInhgEcGdPrU2Gvh67zA,17080
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=wwB-a-2GtcOB3I3Rh03SwoAkrQlMFDjvKbdNmAN74HU,189319
-metaflow/plugins/argo/argo_workflows_cli.py,sha256=1bdG8BQgwyfHooBidyY1Aw52uwjCxq2Edt0bO7qezWM,38375
+metaflow/plugins/argo/argo_workflows.py,sha256=Bw37HHRHf4jqh8c5-WpRKLGr8OR0l46J25LmPn2YlrY,189191
+metaflow/plugins/argo/argo_workflows_cli.py,sha256=aWaGVsf3c2Xvwpbm6F_iTSKsBuJWUALkMOtdgbIqj7Q,38349
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=ogCSBmwsC2C3eusydrgjuAJd4qK18f1sI4jJwA4Fd-o,7800
 metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
-metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=7OiapcIM_r-aBkuIobhofgLC5NRJHC-p9bvBmxvhqoM,12500
+metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=1qiY69jB2yq0ryIH2SKzFHgUXRYvr0VTXd9TDQWG-s0,13693
 metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
 metaflow/plugins/argo/exit_hooks.py,sha256=nh8IEkzAtQnbKVnh3N9CVnVKZB39Bjm3e0LFrACsLz8,6109
 metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
@@ -237,7 +237,7 @@ metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=VixVaowCRoUIY9u
 metaflow/plugins/aws/step_functions/step_functions_client.py,sha256=DKpNwAIWElvWjFANs5Ku3rgzjxFoqAD6k-EF8Xhkg3Q,4754
 metaflow/plugins/aws/step_functions/step_functions_decorator.py,sha256=jzDHYmgU_XvLffZDazR_1viow_1qQFblx9UKyjtoM_0,3788
 metaflow/plugins/aws/step_functions/step_functions_deployer.py,sha256=JKYtDhKivtXUWPklprZFzkqezh14loGDmk8mNk6QtpI,3714
-metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py,sha256=zDWmrOeCEL2_uBbmdmXbVpHVTjswwjEL_rOux6MAcRI,7303
+metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py,sha256=KWIPpE4ebOYyyXlRVtJF6XMnYcyjMFTxdzJWbP0m5PA,7719
 metaflow/plugins/azure/__init__.py,sha256=GuuhTVC-zSdyAf79a1wiERMq0Zts7fwVT7t9fAf234A,100
 metaflow/plugins/azure/azure_credential.py,sha256=JmdGEbVzgxy8ucqnQDdTTI_atyMX9WSZUw3qYOo7RhE,2174
 metaflow/plugins/azure/azure_exceptions.py,sha256=NnbwpUC23bc61HZjJmeXztY0tBNn_Y_VpIpDDuYWIZ0,433
@@ -351,7 +351,7 @@ metaflow/plugins/uv/bootstrap.py,sha256=PJP-lrFsLUUfiXCNse2Uy5uI3b8-X3-ji3Uwf30I
 metaflow/plugins/uv/uv_environment.py,sha256=6hUaWrTgqHyzS6igGQUXpS6jb_GHv3Wq9ZrWhZgEals,2587
 metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/runner/click_api.py,sha256=u9Y1bu9vN13ektBs-L8lg2oxb5oYvGe1dPJjV2shMAw,23727
-metaflow/runner/deployer.py,sha256=U-hwf4gVzwUlXgnkfTW3y1daGXvo5eP4HTQZwb-vS0g,11058
+metaflow/runner/deployer.py,sha256=JwclsNiqnr0RsrA7-8a1-3Z23A4BDz4SNJubqDPxUuU,13955
 metaflow/runner/deployer_impl.py,sha256=zTING0_fwP44JcGo69DuNrVut5KqdBVzYOM7MYTZgIY,7049
 metaflow/runner/metaflow_runner.py,sha256=uo3BzcAfZ67VT_f-TPe5ZHiWHn6uuojWusOMGksvX14,18178
 metaflow/runner/nbdeploy.py,sha256=Sp5w-6nCZwjHaRBHWxi8udya-RYnJOB76KNLjB4L7Gs,4166
@@ -398,12 +398,12 @@ metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 metaflow/user_configs/config_decorators.py,sha256=n4OTbfuaXejPVLHnXXgVlNJNcrSvWrVOdXCMcqvQlik,20512
 metaflow/user_configs/config_options.py,sha256=yyaiKLB4QWlR5AaTg9I7GoBv-OlffOFdGPe1BL85Llc,21241
 metaflow/user_configs/config_parameters.py,sha256=Eyiqcz4YV_z4algDHAh2gaejGFgIdHk8Vix9AUdPSh0,20989
-ob_metaflow-2.15.21.3.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
-ob_metaflow-2.15.21.3.data/data/share/metaflow/devtools/Tiltfile,sha256=I55XTG4RBnrMfDcYRtREXqqS8T9bF8agkZq0DlvdFLk,21404
-ob_metaflow-2.15.21.3.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
-ob_metaflow-2.15.21.3.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-ob_metaflow-2.15.21.3.dist-info/METADATA,sha256=97WP9Xq9HbwMUULKSS2ELrw6dXFG4pWXqATGdtc-2r4,5937
-ob_metaflow-2.15.21.3.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-ob_metaflow-2.15.21.3.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
-ob_metaflow-2.15.21.3.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-ob_metaflow-2.15.21.3.dist-info/RECORD,,
+ob_metaflow-2.15.21.4.data/data/share/metaflow/devtools/Makefile,sha256=5n89OGIC_kE4wxtEI66VCucN-b-1w5bqvGeZYmeRGz8,13737
+ob_metaflow-2.15.21.4.data/data/share/metaflow/devtools/Tiltfile,sha256=I55XTG4RBnrMfDcYRtREXqqS8T9bF8agkZq0DlvdFLk,21404
+ob_metaflow-2.15.21.4.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
+ob_metaflow-2.15.21.4.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+ob_metaflow-2.15.21.4.dist-info/METADATA,sha256=lblEK6Gow24-pgfLiGy_CP4rHdfdGp2THMmZyZFZYjE,5937
+ob_metaflow-2.15.21.4.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+ob_metaflow-2.15.21.4.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
+ob_metaflow-2.15.21.4.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+ob_metaflow-2.15.21.4.dist-info/RECORD,,