ob-metaflow 2.12.26.1__py2.py3-none-any.whl → 2.12.30.1__py2.py3-none-any.whl

metaflow/runner/metaflow_runner.py

@@ -6,7 +6,7 @@ import tempfile
 
 from typing import Dict, Iterator, Optional, Tuple
 
-from metaflow import Run, metadata
+from metaflow import Run
 
 from .utils import handle_timeout
 from .subprocess_manager import CommandManager, SubprocessManager
@@ -67,10 +67,11 @@ class ExecutingRun(object):
 
         Parameters
         ----------
-        timeout : Optional[float], default None
-            The maximum time to wait for the run to finish.
-            If the timeout is reached, the run is terminated
-        stream : Optional[str], default None
+        timeout : float, optional, default None
+            The maximum time, in seconds, to wait for the run to finish.
+            If the timeout is reached, the run is terminated. If not specified, wait
+            forever.
+        stream : str, optional, default None
             If specified, the specified stream is printed to stdout. `stream` can
             be one of `stdout` or `stderr`.
 
@@ -167,7 +168,7 @@ class ExecutingRun(object):
         ----------
         stream : str
             The stream to stream logs from. Can be one of `stdout` or `stderr`.
-        position : Optional[int], default None
+        position : int, optional, default None
             The position in the log file to start streaming from. If None, it starts
             from the beginning of the log file. This allows resuming streaming from
             a previously known position
@@ -207,13 +208,13 @@ class Runner(object):
     show_output : bool, default True
         Show the 'stdout' and 'stderr' to the console by default,
         Only applicable for synchronous 'run' and 'resume' functions.
-    profile : Optional[str], default None
+    profile : str, optional, default None
         Metaflow profile to use to run this run. If not specified, the default
         profile is used (or the one already set using `METAFLOW_PROFILE`)
-    env : Optional[Dict], default None
+    env : Dict[str, str], optional, default None
         Additional environment variables to set for the Run. This overrides the
         environment set for this process.
-    cwd : Optional[str], default None
+    cwd : str, optional, default None
         The directory to run the subprocess in; if not specified, the current
         directory is used.
     file_read_timeout : int, default 3600
@@ -228,7 +229,7 @@ class Runner(object):
         flow_file: str,
         show_output: bool = True,
         profile: Optional[str] = None,
-        env: Optional[Dict] = None,
+        env: Optional[Dict[str, str]] = None,
         cwd: Optional[str] = None,
         file_read_timeout: int = 3600,
         **kwargs
@@ -275,9 +276,10 @@ class Runner(object):
 
         # Set the correct metadata from the runner_attribute file corresponding to this run.
         metadata_for_flow = content.get("metadata")
-        metadata(metadata_for_flow)
 
-        run_object = Run(pathspec, _namespace_check=False)
+        run_object = Run(
+            pathspec, _namespace_check=False, _current_metadata=metadata_for_flow
+        )
         return ExecutingRun(self, command_obj, run_object)
 
     def run(self, **kwargs) -> ExecutingRun:

metaflow/runner/nbdeploy.py

@@ -37,13 +37,13 @@ class NBDeployer(object):
         Flow defined in the same cell
     show_output : bool, default True
         Show the 'stdout' and 'stderr' to the console by default,
-    profile : Optional[str], default None
+    profile : str, optional, default None
         Metaflow profile to use to deploy this run. If not specified, the default
         profile is used (or the one already set using `METAFLOW_PROFILE`)
-    env : Optional[Dict[str, str]], default None
+    env : Dict[str, str], optional, default None
         Additional environment variables to set. This overrides the
         environment set for this process.
-    base_dir : Optional[str], default None
+    base_dir : str, optional, default None
         The directory to run the subprocess in; if not specified, the current
         working directory is used.
     **kwargs : Any
@@ -66,10 +66,11 @@ class NBDeployer(object):
             from IPython import get_ipython
 
             ipython = get_ipython()
-        except ModuleNotFoundError:
+        except ModuleNotFoundError as e:
             raise NBDeployerInitializationError(
-                "'NBDeployer' requires an interactive Python environment (such as Jupyter)"
-            )
+                "'NBDeployer' requires an interactive Python environment "
+                "(such as Jupyter)"
+            ) from e
 
         self.cell = get_current_cell(ipython)
         self.flow = flow
@@ -116,13 +117,11 @@ class NBDeployer(object):
             **kwargs,
         )
 
-        from metaflow.plugins import DEPLOYER_IMPL_PROVIDERS
-
-        for provider_class in DEPLOYER_IMPL_PROVIDERS:
-            method_name = provider_class.TYPE.replace("-", "_")
-            setattr(
-                NBDeployer, method_name, self.deployer.__make_function(provider_class)
-            )
+    def __getattr__(self, name):
+        """
+        Forward all attribute access to the underlying `Deployer` instance.
+        """
+        return getattr(self.deployer, name)
 
     def cleanup(self):
         """

metaflow/runner/nbrun.py

@@ -34,13 +34,13 @@ class NBRunner(object):
     show_output : bool, default True
         Show the 'stdout' and 'stderr' to the console by default,
         Only applicable for synchronous 'run' and 'resume' functions.
-    profile : Optional[str], default None
+    profile : str, optional, default None
         Metaflow profile to use to run this run. If not specified, the default
         profile is used (or the one already set using `METAFLOW_PROFILE`)
-    env : Optional[Dict], default None
+    env : Dict[str, str], optional, default None
         Additional environment variables to set for the Run. This overrides the
         environment set for this process.
-    base_dir : Optional[str], default None
+    base_dir : str, optional, default None
         The directory to run the subprocess in; if not specified, the current
         working directory is used.
     file_read_timeout : int, default 3600

metaflow/runner/utils.py

@@ -4,10 +4,12 @@ import time
 import asyncio
 
 from subprocess import CalledProcessError
-from typing import Dict, TYPE_CHECKING
+from typing import Any, Dict, TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from .subprocess_manager import CommandManager
+    import tempfile
+    import metaflow.runner.subprocess_manager
+    import metaflow.runner.click_api
 
 
 def get_current_cell(ipython):
@@ -18,7 +20,8 @@ def get_current_cell(ipython):
 
 def format_flowfile(cell):
     """
-    Formats the given cell content to create a valid Python script that can be executed as a Metaflow flow.
+    Formats the given cell content to create a valid Python script that can be
+    executed as a Metaflow flow.
     """
     flowspec = [
         x
@@ -36,7 +39,9 @@ def format_flowfile(cell):
     return "\n".join(lines)
 
 
-def check_process_status(command_obj: "CommandManager"):
+def check_process_status(
+    command_obj: "metaflow.runner.subprocess_manager.CommandManager",
+):
     if isinstance(command_obj.process, asyncio.subprocess.Process):
         return command_obj.process.returncode is not None
     else:
@@ -44,7 +49,9 @@ def check_process_status(command_obj: "CommandManager"):
 
 
 def read_from_file_when_ready(
-    file_path: str, command_obj: "CommandManager", timeout: float = 5
+    file_path: str,
+    command_obj: "metaflow.runner.subprocess_manager.CommandManager",
+    timeout: float = 5,
 ):
     start_time = time.time()
     with open(file_path, "r", encoding="utf-8") as file_pointer:
@@ -70,7 +77,9 @@ def read_from_file_when_ready(
 
 
 def handle_timeout(
-    tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int
+    tfp_runner_attribute: "tempfile._TemporaryFileWrapper[str]",
+    command_obj: "metaflow.runner.subprocess_manager.CommandManager",
+    file_read_timeout: int,
 ):
     """
     Handle the timeout for a running subprocess command that reads a file
@@ -102,8 +111,8 @@ def handle_timeout(
         )
         return content
     except (CalledProcessError, TimeoutError) as e:
-        stdout_log = open(command_obj.log_files["stdout"]).read()
-        stderr_log = open(command_obj.log_files["stderr"]).read()
+        stdout_log = open(command_obj.log_files["stdout"], encoding="utf-8").read()
+        stderr_log = open(command_obj.log_files["stderr"], encoding="utf-8").read()
         command = " ".join(command_obj.command)
         error_message = "Error executing: '%s':\n" % command
         if stdout_log.strip():
@@ -111,3 +120,41 @@ def handle_timeout(
         if stderr_log.strip():
             error_message += "\nStderr:\n%s\n" % stderr_log
         raise RuntimeError(error_message) from e
+
+
+def get_lower_level_group(
+    api: "metaflow.runner.click_api.MetaflowAPI",
+    top_level_kwargs: Dict[str, Any],
+    sub_command: str,
+    sub_command_kwargs: Dict[str, Any],
+) -> "metaflow.runner.click_api.MetaflowAPI":
+    """
+    Retrieve a lower-level group from the API based on the type and provided arguments.
+
+    Parameters
+    ----------
+    api : MetaflowAPI
+        Metaflow API instance.
+    top_level_kwargs : Dict[str, Any]
+        Top-level keyword arguments to pass to the API.
+    sub_command : str
+        Sub-command of API to get the API for
+    sub_command_kwargs : Dict[str, Any]
+        Sub-command arguments
+
+    Returns
+    -------
+    MetaflowAPI
+        The lower-level group object retrieved from the API.
+
+    Raises
+    ------
+    ValueError
+        If the `_type` is None.
+    """
+    sub_command_obj = getattr(api(**top_level_kwargs), sub_command)
+
+    if sub_command_obj is None:
+        raise ValueError(f"Sub-command '{sub_command}' not found in API '{api.name}'")
+
+    return sub_command_obj(**sub_command_kwargs)

metaflow/runtime.py

@@ -20,7 +20,7 @@ from metaflow.datastore.exceptions import DataException
 from contextlib import contextmanager
 
 from . import get_namespace
-from .metadata import MetaDatum
+from .metadata_provider import MetaDatum
 from .metaflow_config import MAX_ATTEMPTS, UI_URL
 from .exception import (
     MetaflowException,

metaflow/system/system_logger.py

@@ -7,26 +7,11 @@ class SystemLogger(object):
     def __init__(self):
         self._logger = None
         self._flow_name = None
-        self._context = {}
-        self._is_context_updated = False
 
     def __del__(self):
         if self._flow_name == "not_a_real_flow":
             self.logger.terminate()
 
-    def update_context(self, context: Dict[str, Any]):
-        """
-        Update the global context maintained by the system logger.
-
-        Parameters
-        ----------
-        context : Dict[str, Any]
-            A dictionary containing the context to update.
-
-        """
-        self._is_context_updated = True
-        self._context.update(context)
-
     def init_system_logger(
         self, flow_name: str, logger: "metaflow.event_logger.NullEventLogger"
     ):
@@ -71,7 +56,7 @@ class SystemLogger(object):
             "false",
             "",
         ):
-            print("system monitor: %s" % msg, file=sys.stderr)
+            print("system logger: %s" % msg, file=sys.stderr)
 
     def log_event(
         self, level: str, module: str, name: str, payload: Optional[Any] = None
@@ -96,8 +81,5 @@ class SystemLogger(object):
                 "module": module,
                 "name": name,
                 "payload": payload if payload is not None else {},
-                "context": self._context,
-                "is_context_updated": self._is_context_updated,
             }
         )
-        self._is_context_updated = False

metaflow/system/system_monitor.py

@@ -9,35 +9,11 @@ class SystemMonitor(object):
     def __init__(self):
         self._monitor = None
         self._flow_name = None
-        self._context = {}
 
     def __del__(self):
         if self._flow_name == "not_a_real_flow":
             self.monitor.terminate()
 
-    def update_context(self, context: Dict[str, Any]):
-        """
-        Update the global context maintained by the system monitor.
-
-        Parameters
-        ----------
-        context : Dict[str, Any]
-            A dictionary containing the context to update.
-
-        """
-        from metaflow.sidecar import Message, MessageTypes
-
-        self._context.update(context)
-        self.monitor.send(
-            Message(
-                MessageTypes.MUST_SEND,
-                {
-                    "is_context_updated": True,
-                    **self._context,
-                },
-            )
-        )
-
     def init_system_monitor(
         self, flow_name: str, monitor: "metaflow.monitor.NullMonitor"
     ):

metaflow/task.py

@@ -12,7 +12,7 @@ from metaflow.sidecar import Message, MessageTypes
 from metaflow.datastore.exceptions import DataException
 
 from .metaflow_config import MAX_ATTEMPTS
-from .metadata import MetaDatum
+from .metadata_provider import MetaDatum
 from .mflog import TASK_LOG_SOURCE
 from .datastore import Inputs, TaskDataStoreSet
 from .exception import (
@@ -306,8 +306,6 @@ class MetaflowTask(object):
             "origin_run_id": origin_run_id,
             "origin_task_id": origin_task_id,
         }
-        _system_logger.update_context(task_payload)
-        _system_monitor.update_context(task_payload)
 
         msg = "Cloning task from {}/{}/{}/{} to {}/{}/{}/{}".format(
             self.flow.name,
@@ -512,8 +510,7 @@ class MetaflowTask(object):
             origin_run_id=origin_run_id,
             namespace=resolve_identity(),
             username=get_username(),
-            metadata_str="%s@%s"
-            % (self.metadata.__class__.TYPE, self.metadata.__class__.INFO),
+            metadata_str=self.metadata.metadata_str(),
             is_running=True,
             tags=self.metadata.sticky_tags,
         )
@@ -546,9 +543,6 @@ class MetaflowTask(object):
             "project_flow_name": current.get("project_flow_name"),
             "trace_id": trace_id or None,
         }
-
-        _system_logger.update_context(task_payload)
-        _system_monitor.update_context(task_payload)
         start = time.time()
         self.metadata.start_task_heartbeat(self.flow.name, run_id, step_name, task_id)
         with self.monitor.measure("metaflow.task.duration"):
@@ -593,7 +587,8 @@ class MetaflowTask(object):
                         {
                             "parameter_names": self._init_parameters(
                                 inputs[0], passdown=True
-                            )
+                            ),
+                            "graph_info": self.flow._graph_info,
                         }
                     )
                 else:
@@ -617,7 +612,8 @@ class MetaflowTask(object):
                             {
                                 "parameter_names": self._init_parameters(
                                     inputs[0], passdown=False
-                                )
+                                ),
+                                "graph_info": self.flow._graph_info,
                             }
                         )
 
@@ -709,24 +705,34 @@ class MetaflowTask(object):
                         name="end",
                         payload={**task_payload, "msg": "Task ended"},
                     )
-
-                attempt_ok = str(bool(self.flow._task_ok))
-                self.metadata.register_metadata(
-                    run_id,
-                    step_name,
-                    task_id,
-                    [
-                        MetaDatum(
-                            field="attempt_ok",
-                            value=attempt_ok,
-                            type="internal_attempt_status",
-                            tags=["attempt_id:{0}".format(retry_count)],
-                        )
-                    ],
-                )
+                try:
+                    # persisting might fail due to unpicklable artifacts.
+                    output.persist(self.flow)
+                except Exception as ex:
+                    self.flow._task_ok = False
+                    raise ex
+                finally:
+                    # The attempt_ok metadata is used to determine task status so it is important
+                    # we ensure that it is written even in case of preceding failures.
+                    # f.ex. failing to serialize artifacts leads to a non-zero exit code for the process,
+                    # even if user code finishes successfully. Flow execution will not continue due to the exit,
+                    # so arguably we should mark the task as failed.
+                    attempt_ok = str(bool(self.flow._task_ok))
+                    self.metadata.register_metadata(
+                        run_id,
+                        step_name,
+                        task_id,
+                        [
+                            MetaDatum(
+                                field="attempt_ok",
+                                value=attempt_ok,
+                                type="internal_attempt_status",
+                                tags=["attempt_id:{0}".format(retry_count)],
+                            )
+                        ],
+                    )
 
                 output.save_metadata({"task_end": {}})
-                output.persist(self.flow)
 
                 # this writes a success marker indicating that the
                 # "transaction" is done

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.26.1"
+metaflow_version = "2.12.30.1"

{ob_metaflow-2.12.26.1.dist-info → ob_metaflow-2.12.30.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.12.26.1
+Version: 2.12.30.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.26.1; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.30.1; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)