ob-metaflow 2.12.25.1__py2.py3-none-any.whl → 2.12.25.2__py2.py3-none-any.whl

metaflow/metaflow_config.py

@@ -411,6 +411,13 @@ AIRFLOW_KUBERNETES_KUBECONFIG_CONTEXT = from_conf(
 )
 
 
+###
+# From Deployment configuration
+###
+# From Deployment Configuration for DeployedFlow
+FROM_DEPLOYMENT_IMPL = from_conf("FROM_DEPLOYMENT_IMPL", "argo-workflows")
+
+
 ###
 # Conda configuration
 ###

metaflow/plugins/__init__.py

@@ -156,6 +156,11 @@ def get_plugin_cli():
     return resolve_plugins("cli")
 
 
+# just a normal dictionary, not a plugin since there is no class..
+FROM_DEPLOYMENT_PROVIDERS = {
+    "argo-workflows": "metaflow.plugins.argo.argo_workflows_deployer",
+}
+
 STEP_DECORATORS = resolve_plugins("step_decorator")
 FLOW_DECORATORS = resolve_plugins("flow_decorator")
 ENVIRONMENTS = resolve_plugins("environment")

metaflow/plugins/argo/argo_workflows_deployer.py

@@ -1,9 +1,15 @@
 import sys
+import json
 import tempfile
 from typing import Optional, ClassVar
 
+from metaflow.client.core import get_metadata
+from metaflow.exception import MetaflowException
+from metaflow.plugins.argo.argo_client import ArgoClient
+from metaflow.metaflow_config import KUBERNETES_NAMESPACE
 from metaflow.plugins.argo.argo_workflows import ArgoWorkflows
 from metaflow.runner.deployer import (
+    Deployer,
     DeployerImpl,
     DeployedFlow,
     TriggeredRun,
@@ -12,6 +18,100 @@ from metaflow.runner.deployer import (
 )
 
 
+def generate_fake_flow_file_contents(
+    flow_name: str, param_info: dict, project_name: Optional[str] = None
+):
+    params_code = ""
+    for _, param_details in param_info.items():
+        param_name = param_details["name"]
+        param_type = param_details["type"]
+        param_help = param_details["description"]
+        param_required = param_details["is_required"]
+
+        if param_type == "JSON":
+            params_code += f"    {param_name} = Parameter('{param_name}', type=JSONType, help='{param_help}', required={param_required})\n"
+        elif param_type == "FilePath":
+            is_text = param_details.get("is_text", True)
+            encoding = param_details.get("encoding", "utf-8")
+            params_code += f"    {param_name} = IncludeFile('{param_name}', is_text={is_text}, encoding='{encoding}', help='{param_help}', required={param_required})\n"
+        else:
+            params_code += f"    {param_name} = Parameter('{param_name}', type={param_type}, help='{param_help}', required={param_required})\n"
+
+    project_decorator = f"@project(name='{project_name}')\n" if project_name else ""
+
+    contents = f"""\
+from metaflow import FlowSpec, Parameter, IncludeFile, JSONType, step, project
+{project_decorator}class {flow_name}(FlowSpec):
+{params_code}
+    @step
+    def start(self):
+        self.next(self.end)
+    @step
+    def end(self):
+        pass
+if __name__ == '__main__':
+    {flow_name}()
+"""
+    return contents
+
+
+def from_deployment(identifier: str, metadata: Optional[str] = None):
+    client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
+    workflow_template = client.get_workflow_template(identifier)
+
+    if workflow_template is None:
+        raise MetaflowException("No deployed flow found for: %s" % identifier)
+
+    metadata_annotations = workflow_template.get("metadata", {}).get("annotations", {})
+
+    flow_name = metadata_annotations.get("metaflow/flow_name", "")
+    username = metadata_annotations.get("metaflow/owner", "")
+    parameters = json.loads(metadata_annotations.get("metaflow/parameters", {}))
+
+    # these two only exist if @project decorator is used..
+    branch_name = metadata_annotations.get("metaflow/branch_name", None)
+    project_name = metadata_annotations.get("metaflow/project_name", None)
+
+    project_kwargs = {}
+    if branch_name is not None:
+        if branch_name.startswith("prod."):
+            project_kwargs["production"] = True
+            project_kwargs["branch"] = branch_name[len("prod.") :]
+        elif branch_name.startswith("test."):
+            project_kwargs["branch"] = branch_name[len("test.") :]
+        elif branch_name == "prod":
+            project_kwargs["production"] = True
+
+    fake_flow_file_contents = generate_fake_flow_file_contents(
+        flow_name=flow_name, param_info=parameters, project_name=project_name
+    )
+
+    with tempfile.NamedTemporaryFile(suffix=".py", delete=False) as fake_flow_file:
+        with open(fake_flow_file.name, "w") as fp:
+            fp.write(fake_flow_file_contents)
+
+        if branch_name is not None:
+            d = Deployer(
+                fake_flow_file.name, env={"METAFLOW_USER": username}, **project_kwargs
+            ).argo_workflows()
+        else:
+            d = Deployer(
+                fake_flow_file.name, env={"METAFLOW_USER": username}
+            ).argo_workflows(name=identifier)
+
+        d.name = identifier
+        d.flow_name = flow_name
+        if metadata is None:
+            d.metadata = get_metadata()
+        else:
+            d.metadata = metadata
+
+    df = DeployedFlow(deployer=d)
+    d._enrich_deployed_flow(df)
+
+    return df
+
+
 def suspend(instance: TriggeredRun, **kwargs):
     """
     Suspend the running workflow.

metaflow/runner/deployer.py

@@ -214,7 +214,43 @@ class TriggeredRun(object):
             return None
 
 
-class DeployedFlow(object):
+class LazyDeploymentMethod:
+    def __init__(self, module_path, func_name):
+        self.module_path = module_path
+        self.func_name = func_name
+        self.func = None
+
+    def __call__(self, *args, **kwargs):
+        if self.func is None:
+            module = importlib.import_module(self.module_path)
+            self.func = getattr(module, self.func_name)
+        return self.func(*args, **kwargs)
+
+
+class DeploymentMethodsMeta(type):
+    from metaflow.plugins import FROM_DEPLOYMENT_PROVIDERS
+    from metaflow.metaflow_config import FROM_DEPLOYMENT_IMPL
+
+    def __new__(mcs, name, bases, dct):
+        cls = super().__new__(mcs, name, bases, dct)
+
+        def from_deployment(identifier, metadata=None, impl=None):
+            if impl is None:
+                impl = mcs.FROM_DEPLOYMENT_IMPL
+
+            if impl not in mcs.FROM_DEPLOYMENT_PROVIDERS:
+                raise ValueError("This method is not available for: %s" % impl)
+
+            module_path = mcs.FROM_DEPLOYMENT_PROVIDERS[impl]
+            lazy_method = LazyDeploymentMethod(module_path, "from_deployment")
+            return lazy_method(identifier=identifier, metadata=metadata)
+
+        setattr(cls, "from_deployment", staticmethod(from_deployment))
+
+        return cls
+
+
+class DeployedFlow(metaclass=DeploymentMethodsMeta):
     """
     DeployedFlow class represents a flow that has been deployed.
 
@@ -226,6 +262,9 @@ class DeployedFlow(object):
 
     def __init__(self, deployer: "DeployerImpl"):
         self.deployer = deployer
+        self.name = self.deployer.name
+        self.flow_name = self.deployer.flow_name
+        self.metadata = self.deployer.metadata
 
     def _enrich_object(self, env):
         """

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.25.1"
+metaflow_version = "2.12.25.2"

{ob_metaflow-2.12.25.1.dist-info → ob_metaflow-2.12.25.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.12.25.1
+Version: 2.12.25.2
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.25.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.25.2; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob_metaflow-2.12.25.1.dist-info → ob_metaflow-2.12.25.2.dist-info}/RECORD

@@ -16,7 +16,7 @@ metaflow/includefile.py,sha256=yHczcZ_U0SrasxSNhZb3DIBzx8UZnrJCl3FzvpEQLOA,19753
 metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=5rj1MlpluxyPTSINjtMoJ7viotyNzfjtBJSAihlAwMU,10870
-metaflow/metaflow_config.py,sha256=J11vmCjjffr2Koddo1OuymZFheW-0UaTD6rec2Hp0Gk,23017
+metaflow/metaflow_config.py,sha256=ZqNVr3MlauHxYZ5nJX2HRl8c-E3dfdaWcBN5zSC34Nw,23183
 metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
 metaflow/metaflow_current.py,sha256=pC-EMnAsnvBLvLd61W6MvfiCKcboryeui9f6r8z_sg8,7161
 metaflow/metaflow_environment.py,sha256=rojFyGdyY56sN1HaEb1-0XX53Q3XPNnl0SaH-8xXZ8w,7987
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=pu5ibRx5X5exQtS3_CEK5XaDBp0Hh7eQMXagfHHVqoA,31
+metaflow/version.py,sha256=CT8RBvPGLOTlpmf1k4yY6HvScS3c40tcvgeGWTZhKZw,31
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -143,7 +143,7 @@ metaflow/mflog/mflog.py,sha256=VebXxqitOtNAs7VJixnNfziO_i_urG7bsJ5JiB5IXgY,4370
 metaflow/mflog/save_logs.py,sha256=ZBAF4BMukw4FMAC7odpr9OI2BC_2petPtDX0ca6srC4,2352
 metaflow/mflog/save_logs_periodically.py,sha256=2Uvk9hi-zlCqXxOQoXmmjH1SCugfw6eG6w70WgfI-ho,1256
 metaflow/mflog/tee.py,sha256=wTER15qeHuiRpCkOqo-bd-r3Gj-EVlf3IvWRCA4beW4,887
-metaflow/plugins/__init__.py,sha256=EJgoFpti77Niza4G19cgL2Dszk130e4sj06RZiCCJJQ,7227
+metaflow/plugins/__init__.py,sha256=SCRd06keAc_tpUZt8AC38TUYx-5DJL06GLmUZMWf3vg,7398
 metaflow/plugins/catch_decorator.py,sha256=UOM2taN_OL2RPpuJhwEOA9ZALm0-hHD0XS2Hn2GUev0,4061
 metaflow/plugins/debug_logger.py,sha256=mcF5HYzJ0NQmqCMjyVUk3iAP-heroHRIiVWQC6Ha2-I,879
 metaflow/plugins/debug_monitor.py,sha256=Md5X_sDOSssN9pt2D8YcaIjTK5JaQD55UAYTcF6xYF0,1099
@@ -178,7 +178,7 @@ metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-z
 metaflow/plugins/argo/argo_workflows.py,sha256=AFP0ZmtsxoAAnReYEgfkGYOIWDdG6Ee-hu37CLMo4gY,175217
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=0qAGo0YlC1Y9-1zqYAzhVCpCcITotfOI421VOIRpseM,37232
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=yprszMdbE3rBTcEA9VR0IEnPjTprUauZBc4SBb-Q7sA,7878
-metaflow/plugins/argo/argo_workflows_deployer.py,sha256=wSSZtThn_VPvE_Wu6NB1L0Q86LmBJh9g009v_lpvBPM,8125
+metaflow/plugins/argo/argo_workflows_deployer.py,sha256=_buPhlVPRtAuZRYuLRXmalvvBMeOAR0ZAQO9eLhsSsU,11969
 metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
 metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
 metaflow/plugins/argo/jobset_input_paths.py,sha256=_JhZWngA6p9Q_O2fx3pdzKI0WE-HPRHz_zFvY2pHPTQ,525
@@ -303,7 +303,7 @@ metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpL
 metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYAPTO0isYXGvaUwlG8,11273
 metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/runner/click_api.py,sha256=Qfg4BOz5K2LaXTYBsi1y4zTfNIsGGHBVF3UkorX_-o8,13878
-metaflow/runner/deployer.py,sha256=xNqxFIezWz3AcVqR4jeL-JnInGtefwEYMbXttxgB_I8,12276
+metaflow/runner/deployer.py,sha256=l-ofKKvMbFqhV9Md0-kMjt2KVVgmT3ohCbJYHrTIuJc,13689
 metaflow/runner/metaflow_runner.py,sha256=I7Ao0GHHfP55nUCE8g6CpTPGjuWgXedSc9EZX-tIE2c,15001
 metaflow/runner/nbdeploy.py,sha256=fP1s_5MeiDyT_igP82pB5EUqX9rOy2s06Hyc-OUbOvQ,4115
 metaflow/runner/nbrun.py,sha256=lmvhzMCz7iC9LSPGRijifW1wMXxa4RW_jVmpdjQi22E,7261
@@ -345,9 +345,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-ob_metaflow-2.12.25.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-ob_metaflow-2.12.25.1.dist-info/METADATA,sha256=YvcCRA52Tr57kElZwUHgPRc1XxTP63gKsjsIQ0ojl-M,5143
-ob_metaflow-2.12.25.1.dist-info/WHEEL,sha256=AHX6tWk3qWuce7vKLrj7lnulVHEdWoltgauo8bgCXgU,109
-ob_metaflow-2.12.25.1.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-ob_metaflow-2.12.25.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-ob_metaflow-2.12.25.1.dist-info/RECORD,,
+ob_metaflow-2.12.25.2.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+ob_metaflow-2.12.25.2.dist-info/METADATA,sha256=6zJcRHpOc6ffW0m-B4wBnGrjQxxrtvv3qva_DTBSrjs,5143
+ob_metaflow-2.12.25.2.dist-info/WHEEL,sha256=AHX6tWk3qWuce7vKLrj7lnulVHEdWoltgauo8bgCXgU,109
+ob_metaflow-2.12.25.2.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+ob_metaflow-2.12.25.2.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+ob_metaflow-2.12.25.2.dist-info/RECORD,,