metaflow 2.12.28__py2.py3-none-any.whl → 2.12.29__py2.py3-none-any.whl

metaflow/runner/deployer_impl.py

@@ -0,0 +1,167 @@
+import importlib
+import json
+import os
+import sys
+import tempfile
+
+from typing import Any, ClassVar, Dict, Optional, TYPE_CHECKING, Type
+
+from .subprocess_manager import SubprocessManager
+from .utils import get_lower_level_group, handle_timeout
+
+if TYPE_CHECKING:
+    import metaflow.runner.deployer
+
+# NOTE: This file is separate from the deployer.py file to prevent circular imports.
+# This file is needed in any of the DeployerImpl implementations
+# (like argo_workflows_deployer.py) which is in turn needed to create the Deployer
+# class (ie: it uses ArgoWorkflowsDeployer to create the Deployer class).
+
+
+class DeployerImpl(object):
+    """
+    Base class for deployer implementations. Each implementation should define a TYPE
+    class variable that matches the name of the CLI group.
+
+    Parameters
+    ----------
+    flow_file : str
+        Path to the flow file to deploy.
+    show_output : bool, default True
+        Show the 'stdout' and 'stderr' to the console by default.
+    profile : Optional[str], default None
+        Metaflow profile to use for the deployment. If not specified, the default
+        profile is used.
+    env : Optional[Dict], default None
+        Additional environment variables to set for the deployment.
+    cwd : Optional[str], default None
+        The directory to run the subprocess in; if not specified, the current
+        directory is used.
+    file_read_timeout : int, default 3600
+        The timeout until which we try to read the deployer attribute file.
+    **kwargs : Any
+        Additional arguments that you would pass to `python myflow.py` before
+        the deployment command.
+    """
+
+    TYPE: ClassVar[Optional[str]] = None
+
+    def __init__(
+        self,
+        flow_file: str,
+        show_output: bool = True,
+        profile: Optional[str] = None,
+        env: Optional[Dict] = None,
+        cwd: Optional[str] = None,
+        file_read_timeout: int = 3600,
+        **kwargs
+    ):
+        if self.TYPE is None:
+            raise ValueError(
+                "DeployerImpl doesn't have a 'TYPE' to target. Please use a sub-class "
+                "of DeployerImpl."
+            )
+
+        if "metaflow.cli" in sys.modules:
+            importlib.reload(sys.modules["metaflow.cli"])
+        from metaflow.cli import start
+        from metaflow.runner.click_api import MetaflowAPI
+
+        self.flow_file = flow_file
+        self.show_output = show_output
+        self.profile = profile
+        self.env = env
+        self.cwd = cwd
+        self.file_read_timeout = file_read_timeout
+
+        self.env_vars = os.environ.copy()
+        self.env_vars.update(self.env or {})
+        if self.profile:
+            self.env_vars["METAFLOW_PROFILE"] = profile
+
+        self.spm = SubprocessManager()
+        self.top_level_kwargs = kwargs
+        self.api = MetaflowAPI.from_cli(self.flow_file, start)
+
+    @property
+    def deployer_kwargs(self) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    @staticmethod
+    def deployed_flow_type() -> Type["metaflow.runner.deployer.DeployedFlow"]:
+        raise NotImplementedError
+
+    def __enter__(self) -> "DeployerImpl":
+        return self
+
+    def create(self, **kwargs) -> "metaflow.runner.deployer.DeployedFlow":
+        """
+        Create a sub-class of a `DeployedFlow` depending on the deployer implementation.
+
+        Parameters
+        ----------
+        **kwargs : Any
+            Additional arguments to pass to `create` corresponding to the
+            command line arguments of `create`
+
+        Returns
+        -------
+        DeployedFlow
+            DeployedFlow object representing the deployed flow.
+
+        Raises
+        ------
+        Exception
+            If there is an error during deployment.
+        """
+        # Sub-classes should implement this by simply calling _create and pass the
+        # proper class as the DeployedFlow to return.
+        raise NotImplementedError
+
+    def _create(
+        self, create_class: Type["metaflow.runner.deployer.DeployedFlow"], **kwargs
+    ) -> "metaflow.runner.deployer.DeployedFlow":
+        with tempfile.TemporaryDirectory() as temp_dir:
+            tfp_runner_attribute = tempfile.NamedTemporaryFile(
+                dir=temp_dir, delete=False
+            )
+            # every subclass needs to have `self.deployer_kwargs`
+            command = get_lower_level_group(
+                self.api, self.top_level_kwargs, self.TYPE, self.deployer_kwargs
+            ).create(deployer_attribute_file=tfp_runner_attribute.name, **kwargs)
+
+            pid = self.spm.run_command(
+                [sys.executable, *command],
+                env=self.env_vars,
+                cwd=self.cwd,
+                show_output=self.show_output,
+            )
+
+            command_obj = self.spm.get(pid)
+            content = handle_timeout(
+                tfp_runner_attribute, command_obj, self.file_read_timeout
+            )
+            content = json.loads(content)
+            self.name = content.get("name")
+            self.flow_name = content.get("flow_name")
+            self.metadata = content.get("metadata")
+            # Additional info is used to pass additional deployer specific information.
+            # It is used in non-OSS deployers (extensions).
+            self.additional_info = content.get("additional_info", {})
+
+            if command_obj.process.returncode == 0:
+                return create_class(deployer=self)
+
+        raise RuntimeError("Error deploying %s to %s" % (self.flow_file, self.TYPE))
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """
+        Cleanup resources on exit.
+        """
+        self.cleanup()
+
+    def cleanup(self):
+        """
+        Cleanup resources.
+        """
+        self.spm.cleanup()

metaflow/runner/metaflow_runner.py

@@ -67,10 +67,11 @@ class ExecutingRun(object):
 
         Parameters
         ----------
-        timeout : Optional[float], default None
-            The maximum time to wait for the run to finish.
-            If the timeout is reached, the run is terminated
-        stream : Optional[str], default None
+        timeout : float, optional, default None
+            The maximum time, in seconds, to wait for the run to finish.
+            If the timeout is reached, the run is terminated. If not specified, wait
+            forever.
+        stream : str, optional, default None
             If specified, the specified stream is printed to stdout. `stream` can
             be one of `stdout` or `stderr`.
 
@@ -167,7 +168,7 @@ class ExecutingRun(object):
         ----------
         stream : str
             The stream to stream logs from. Can be one of `stdout` or `stderr`.
-        position : Optional[int], default None
+        position : int, optional, default None
             The position in the log file to start streaming from. If None, it starts
             from the beginning of the log file. This allows resuming streaming from
             a previously known position
@@ -207,13 +208,13 @@ class Runner(object):
     show_output : bool, default True
         Show the 'stdout' and 'stderr' to the console by default,
         Only applicable for synchronous 'run' and 'resume' functions.
-    profile : Optional[str], default None
+    profile : str, optional, default None
         Metaflow profile to use to run this run. If not specified, the default
         profile is used (or the one already set using `METAFLOW_PROFILE`)
-    env : Optional[Dict], default None
+    env : Dict[str, str], optional, default None
         Additional environment variables to set for the Run. This overrides the
         environment set for this process.
-    cwd : Optional[str], default None
+    cwd : str, optional, default None
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
@@ -228,7 +229,7 @@ class Runner(object):
         flow_file: str,
         show_output: bool = True,
         profile: Optional[str] = None,
-        env: Optional[Dict] = None,
+        env: Optional[Dict[str, str]] = None,
         cwd: Optional[str] = None,
         file_read_timeout: int = 3600,
         **kwargs

metaflow/runner/nbdeploy.py

@@ -37,13 +37,13 @@ class NBDeployer(object):
         Flow defined in the same cell
     show_output : bool, default True
         Show the 'stdout' and 'stderr' to the console by default,
-    profile : Optional[str], default None
+    profile : str, optional, default None
         Metaflow profile to use to deploy this run. If not specified, the default
         profile is used (or the one already set using `METAFLOW_PROFILE`)
-    env : Optional[Dict[str, str]], default None
+    env : Dict[str, str], optional, default None
         Additional environment variables to set. This overrides the
         environment set for this process.
-    base_dir : Optional[str], default None
+    base_dir : str, optional, default None
         The directory to run the subprocess in; if not specified, the current
         working directory is used.
     **kwargs : Any
@@ -66,10 +66,11 @@ class NBDeployer(object):
             from IPython import get_ipython
 
             ipython = get_ipython()
-        except ModuleNotFoundError:
+        except ModuleNotFoundError as e:
             raise NBDeployerInitializationError(
-                "'NBDeployer' requires an interactive Python environment (such as Jupyter)"
-            )
+                "'NBDeployer' requires an interactive Python environment "
+                "(such as Jupyter)"
+            ) from e
 
         self.cell = get_current_cell(ipython)
         self.flow = flow
@@ -116,13 +117,11 @@ class NBDeployer(object):
             **kwargs,
         )
 
-        from metaflow.plugins import DEPLOYER_IMPL_PROVIDERS
-
-        for provider_class in DEPLOYER_IMPL_PROVIDERS:
-            method_name = provider_class.TYPE.replace("-", "_")
-            setattr(
-                NBDeployer, method_name, self.deployer.__make_function(provider_class)
-            )
+    def __getattr__(self, name):
+        """
+        Forward all attribute access to the underlying `Deployer` instance.
+        """
+        return getattr(self.deployer, name)
 
     def cleanup(self):
         """

metaflow/runner/nbrun.py

@@ -34,13 +34,13 @@ class NBRunner(object):
     show_output : bool, default True
         Show the 'stdout' and 'stderr' to the console by default,
         Only applicable for synchronous 'run' and 'resume' functions.
-    profile : Optional[str], default None
+    profile : str, optional, default None
         Metaflow profile to use to run this run. If not specified, the default
         profile is used (or the one already set using `METAFLOW_PROFILE`)
-    env : Optional[Dict], default None
+    env : Dict[str, str], optional, default None
         Additional environment variables to set for the Run. This overrides the
         environment set for this process.
-    base_dir : Optional[str], default None
+    base_dir : str, optional, default None
         The directory to run the subprocess in; if not specified, the current
         working directory is used.
     file_read_timeout : int, default 3600

metaflow/runner/utils.py

@@ -4,10 +4,12 @@ import time
 import asyncio
 
 from subprocess import CalledProcessError
-from typing import Dict, TYPE_CHECKING
+from typing import Any, Dict, TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from .subprocess_manager import CommandManager
+    import tempfile
+    import metaflow.runner.subprocess_manager
+    import metaflow.runner.click_api
 
 
 def get_current_cell(ipython):
@@ -18,7 +20,8 @@ def get_current_cell(ipython):
 
 def format_flowfile(cell):
     """
-    Formats the given cell content to create a valid Python script that can be executed as a Metaflow flow.
+    Formats the given cell content to create a valid Python script that can be
+    executed as a Metaflow flow.
     """
     flowspec = [
         x
@@ -36,7 +39,9 @@ def format_flowfile(cell):
     return "\n".join(lines)
 
 
-def check_process_status(command_obj: "CommandManager"):
+def check_process_status(
+    command_obj: "metaflow.runner.subprocess_manager.CommandManager",
+):
     if isinstance(command_obj.process, asyncio.subprocess.Process):
         return command_obj.process.returncode is not None
     else:
@@ -44,7 +49,9 @@ def check_process_status(command_obj: "CommandManager"):
 
 
 def read_from_file_when_ready(
-    file_path: str, command_obj: "CommandManager", timeout: float = 5
+    file_path: str,
+    command_obj: "metaflow.runner.subprocess_manager.CommandManager",
+    timeout: float = 5,
 ):
     start_time = time.time()
     with open(file_path, "r", encoding="utf-8") as file_pointer:
@@ -70,7 +77,9 @@ def read_from_file_when_ready(
 
 
 def handle_timeout(
-    tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int
+    tfp_runner_attribute: "tempfile._TemporaryFileWrapper[str]",
+    command_obj: "metaflow.runner.subprocess_manager.CommandManager",
+    file_read_timeout: int,
 ):
     """
     Handle the timeout for a running subprocess command that reads a file
@@ -102,8 +111,8 @@ def handle_timeout(
         )
         return content
     except (CalledProcessError, TimeoutError) as e:
-        stdout_log = open(command_obj.log_files["stdout"]).read()
-        stderr_log = open(command_obj.log_files["stderr"]).read()
+        stdout_log = open(command_obj.log_files["stdout"], encoding="utf-8").read()
+        stderr_log = open(command_obj.log_files["stderr"], encoding="utf-8").read()
         command = " ".join(command_obj.command)
         error_message = "Error executing: '%s':\n" % command
         if stdout_log.strip():
@@ -111,3 +120,41 @@ def handle_timeout(
         if stderr_log.strip():
             error_message += "\nStderr:\n%s\n" % stderr_log
         raise RuntimeError(error_message) from e
+
+
+def get_lower_level_group(
+    api: "metaflow.runner.click_api.MetaflowAPI",
+    top_level_kwargs: Dict[str, Any],
+    sub_command: str,
+    sub_command_kwargs: Dict[str, Any],
+) -> "metaflow.runner.click_api.MetaflowAPI":
+    """
+    Retrieve a lower-level group from the API based on the type and provided arguments.
+
+    Parameters
+    ----------
+    api : MetaflowAPI
+        Metaflow API instance.
+    top_level_kwargs : Dict[str, Any]
+        Top-level keyword arguments to pass to the API.
+    sub_command : str
+        Sub-command of API to get the API for
+    sub_command_kwargs : Dict[str, Any]
+        Sub-command arguments
+
+    Returns
+    -------
+    MetaflowAPI
+        The lower-level group object retrieved from the API.
+
+    Raises
+    ------
+    ValueError
+        If the `_type` is None.
+    """
+    sub_command_obj = getattr(api(**top_level_kwargs), sub_command)
+
+    if sub_command_obj is None:
+        raise ValueError(f"Sub-command '{sub_command}' not found in API '{api.name}'")
+
+    return sub_command_obj(**sub_command_kwargs)

metaflow/runtime.py

@@ -20,7 +20,7 @@ from metaflow.datastore.exceptions import DataException
 from contextlib import contextmanager
 
 from . import get_namespace
-from .metadata import MetaDatum
+from .metadata_provider import MetaDatum
 from .metaflow_config import MAX_ATTEMPTS, UI_URL
 from .exception import (
     MetaflowException,

metaflow/task.py

@@ -12,7 +12,7 @@ from metaflow.sidecar import Message, MessageTypes
 from metaflow.datastore.exceptions import DataException
 
 from .metaflow_config import MAX_ATTEMPTS
-from .metadata import MetaDatum
+from .metadata_provider import MetaDatum
 from .mflog import TASK_LOG_SOURCE
 from .datastore import Inputs, TaskDataStoreSet
 from .exception import (

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.28"
+metaflow_version = "2.12.29"

{metaflow-2.12.28.dist-info → metaflow-2.12.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.28
+Version: 2.12.29
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.28; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.29; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)