metaflow 2.12.5__py2.py3-none-any.whl → 2.12.7__py2.py3-none-any.whl

metaflow/cli.py

@@ -23,6 +23,7 @@ from .metaflow_config import (
     DEFAULT_PACKAGE_SUFFIXES,
 )
 from .metaflow_current import current
+from metaflow.system import _system_monitor, _system_logger
 from .metaflow_environment import MetaflowEnvironment
 from .mflog import LOG_SOURCES, mflog
 from .package import MetaflowPackage
@@ -952,11 +953,13 @@ def start(
         flow=ctx.obj.flow, env=ctx.obj.environment
     )
     ctx.obj.event_logger.start()
+    _system_logger.init_system_logger(ctx.obj.flow.name, ctx.obj.event_logger)
 
     ctx.obj.monitor = MONITOR_SIDECARS[monitor](
         flow=ctx.obj.flow, env=ctx.obj.environment
     )
     ctx.obj.monitor.start()
+    _system_monitor.init_system_monitor(ctx.obj.flow.name, ctx.obj.monitor)
 
     ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == metadata][0](
         ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor

metaflow/client/core.py

@@ -276,6 +276,11 @@ class MetaflowObject(object):
         self._attempt = attempt
         self._current_namespace = _current_namespace or get_namespace()
         self._namespace_check = _namespace_check
+        # If the current namespace is False, we disable checking for namespace for this
+        # and all children objects. Not setting namespace_check to False has the consequence
+        # of preventing access to children objects after the namespace changes
+        if self._current_namespace is None:
+            self._namespace_check = False
 
         if self._attempt is not None:
             if self._NAME not in ["task", "artifact"]:

metaflow/plugins/argo/argo_workflows.py

@@ -6,6 +6,7 @@ import shlex
 import sys
 from collections import defaultdict
 from hashlib import sha1
+from math import inf
 
 from metaflow import JSONType, current
 from metaflow.decorators import flow_decorators
@@ -901,7 +902,9 @@ class ArgoWorkflows(object):
                                 "argo-{{workflow.name}}/%s/{{tasks.%s.outputs.parameters.task-id}}"
                                 % (n, self._sanitize(n))
                                 for n in node.in_funcs
-                            ]
+                            ],
+                            # NOTE: We set zlibmin to infinite because zlib compression for the Argo input-paths breaks template value substitution.
+                            zlibmin=inf,
                         )
                     )
                 ]
@@ -2154,7 +2157,8 @@ class ArgoWorkflows(object):
                                                 # everything within the body.
                                                 # NOTE: We need the conditional logic in order to successfully fall back to the default value
                                                 # when the event payload does not contain a key for a parameter.
-                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (.Input.body.payload.%s | toJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                # NOTE: Keys might contain dashes, so use the safer 'get' for fetching the value
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (get $.Input.body.payload "%s" | toRawJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
                                                 % (v, v),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for

metaflow/plugins/pypi/conda_decorator.py

@@ -100,6 +100,10 @@ class CondaStepDecorator(StepDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.
@@ -340,6 +344,10 @@ class CondaFlowDecorator(FlowDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.

metaflow/plugins/pypi/conda_environment.py

@@ -65,7 +65,7 @@ class CondaEnvironment(MetaflowEnvironment):
         micromamba = Micromamba()
         self.solvers = {"conda": micromamba, "pypi": Pip(micromamba)}
 
-    def init_environment(self, echo):
+    def init_environment(self, echo, only_steps=None):
         # The implementation optimizes for latency to ensure as many operations can
         # be turned into cheap no-ops as feasible. Otherwise, we focus on maintaining
         # a balance between latency and maintainability of code without re-implementing
@@ -77,6 +77,8 @@ class CondaEnvironment(MetaflowEnvironment):
         def environments(type_):
             seen = set()
             for step in self.flow:
+                if only_steps and step.name not in only_steps:
+                    continue
                 environment = self.get_environment(step)
                 if type_ in environment and environment["id_"] not in seen:
                     seen.add(environment["id_"])

metaflow/plugins/pypi/pip.py

@@ -9,7 +9,6 @@ from itertools import chain, product
 from urllib.parse import unquote
 
 from metaflow.exception import MetaflowException
-from metaflow.util import which
 
 from .micromamba import Micromamba
 from .utils import pip_tags, wheel_tags
@@ -25,6 +24,23 @@ class PipException(MetaflowException):
         super(PipException, self).__init__(msg)
 
 
+class PipPackageNotFound(Exception):
+    "Wrapper for pip package resolve errors."
+
+    def __init__(self, error):
+        self.error = error
+        try:
+            # Parse the package spec from error message:
+            # ERROR: ERROR: Could not find a version that satisfies the requirement pkg==0.0.1 (from versions: none)
+            # ERROR: No matching distribution found for pkg==0.0.1
+            self.package_spec = re.search(
+                "ERROR: No matching distribution found for (.*)", self.error
+            )[1]
+            self.package_name = re.match("\w*", self.package_spec)[0]
+        except Exception:
+            pass
+
+
 METADATA_FILE = "{prefix}/.pip/metadata"
 INSTALLATION_MARKER = "{prefix}/.pip/id"
 
@@ -81,7 +97,16 @@ class Pip(object):
                     cmd.append(f"{package}{version}")
                 else:
                     cmd.append(f"{package}=={version}")
-            self._call(prefix, cmd)
+            try:
+                self._call(prefix, cmd)
+            except PipPackageNotFound as ex:
+                # pretty print package errors
+                raise PipException(
+                    "Could not find a binary distribution for %s \n"
+                    "for the platform %s\n\n"
+                    "Note that ***@pypi*** does not currently support source distributions"
+                    % (ex.package_spec, platform)
+                )
 
             def _format(dl_info):
                 res = {k: v for k, v in dl_info.items() if k in ["url"]}
@@ -302,11 +327,14 @@ class Pip(object):
                 .strip()
             )
         except subprocess.CalledProcessError as e:
+            errors = e.stderr.decode()
+            if "No matching distribution" in errors:
+                raise PipPackageNotFound(errors)
             raise PipException(
                 "command '{cmd}' returned error ({code}) {output}\n{stderr}".format(
                     cmd=" ".join(e.cmd),
                     code=e.returncode,
                     output=e.output.decode(),
-                    stderr=e.stderr.decode(),
+                    stderr=errors,
                 )
             )

metaflow/plugins/pypi/pypi_decorator.py

@@ -70,6 +70,10 @@ class PyPIStepDecorator(StepDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @pypi
         # implementation.
@@ -119,6 +123,10 @@ class PyPIFlowDecorator(FlowDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.

metaflow/system/__init__.py

@@ -0,0 +1,5 @@
+from .system_monitor import SystemMonitor
+from .system_logger import SystemLogger
+
+_system_logger = SystemLogger()
+_system_monitor = SystemMonitor()

metaflow/system/system_logger.py

@@ -0,0 +1,103 @@
+import os
+import sys
+from typing import Dict, Any, Optional, Union
+
+
+class SystemLogger(object):
+    def __init__(self):
+        self._logger = None
+        self._flow_name = None
+        self._context = {}
+        self._is_context_updated = False
+
+    def __del__(self):
+        if self._flow_name == "not_a_real_flow":
+            self.logger.terminate()
+
+    def update_context(self, context: Dict[str, Any]):
+        """
+        Update the global context maintained by the system logger.
+
+        Parameters
+        ----------
+        context : Dict[str, Any]
+            A dictionary containing the context to update.
+
+        """
+        self._is_context_updated = True
+        self._context.update(context)
+
+    def init_system_logger(
+        self, flow_name: str, logger: "metaflow.event_logger.NullEventLogger"
+    ):
+        self._flow_name = flow_name
+        self._logger = logger
+
+    def _init_logger_outside_flow(self):
+        from .system_utils import DummyFlow
+        from .system_utils import init_environment_outside_flow
+        from metaflow.plugins import LOGGING_SIDECARS
+        from metaflow.metaflow_config import DEFAULT_EVENT_LOGGER
+
+        self._flow_name = "not_a_real_flow"
+        _flow = DummyFlow(self._flow_name)
+        _environment = init_environment_outside_flow(_flow)
+        _logger = LOGGING_SIDECARS[DEFAULT_EVENT_LOGGER](_flow, _environment)
+        return _logger
+
+    @property
+    def logger(self) -> Optional["metaflow.event_logger.NullEventLogger"]:
+        if self._logger is None:
+            # This happens if the logger is being accessed outside of a flow
+            # We start a logger with a dummy flow and a default environment
+            self._debug("Started logger outside of a flow")
+            self._logger = self._init_logger_outside_flow()
+            self._logger.start()
+        return self._logger
+
+    @staticmethod
+    def _debug(msg: str):
+        """
+        Log a debug message to stderr.
+
+        Parameters
+        ----------
+        msg : str
+            Message to log.
+
+        """
+        if os.environ.get("METAFLOW_DEBUG_SIDECAR", "0").lower() not in (
+            "0",
+            "false",
+            "",
+        ):
+            print("system monitor: %s" % msg, file=sys.stderr)
+
+    def log_event(
+        self, level: str, module: str, name: str, payload: Optional[Any] = None
+    ):
+        """
+        Log an event to the event logger.
+
+        Parameters
+        ----------
+        level : str
+            Log level of the event. Can be one of "info", "warning", "error", "critical", "debug".
+        module : str
+            Module of the event. Usually the name of the class, function, or module that the event is being logged from.
+        name : str
+            Name of the event. Used to qualify the event type.
+        payload : Optional[Any], default None
+            Payload of the event. Contains the event data.
+        """
+        self.logger.log(
+            {
+                "level": level,
+                "module": module,
+                "name": name,
+                "payload": payload if payload is not None else {},
+                "context": self._context,
+                "is_context_updated": self._is_context_updated,
+            }
+        )
+        self._is_context_updated = False

metaflow/system/system_monitor.py

@@ -0,0 +1,132 @@
+import os
+import sys
+from ..debug import debug
+from contextlib import contextmanager
+from typing import Optional, Union, Dict, Any
+
+
+class SystemMonitor(object):
+    def __init__(self):
+        self._monitor = None
+        self._flow_name = None
+        self._context = {}
+
+    def __del__(self):
+        if self._flow_name == "not_a_real_flow":
+            self.monitor.terminate()
+
+    def update_context(self, context: Dict[str, Any]):
+        """
+        Update the global context maintained by the system monitor.
+
+        Parameters
+        ----------
+        context : Dict[str, Any]
+            A dictionary containing the context to update.
+
+        """
+        from metaflow.sidecar import Message, MessageTypes
+
+        self._context.update(context)
+        self.monitor.send(
+            Message(
+                MessageTypes.MUST_SEND,
+                {
+                    "is_context_updated": True,
+                    **self._context,
+                },
+            )
+        )
+
+    def init_system_monitor(
+        self, flow_name: str, monitor: "metaflow.monitor.NullMonitor"
+    ):
+        self._flow_name = flow_name
+        self._monitor = monitor
+
+    def _init_system_monitor_outside_flow(self):
+        from .system_utils import DummyFlow
+        from .system_utils import init_environment_outside_flow
+        from metaflow.plugins import MONITOR_SIDECARS
+        from metaflow.metaflow_config import DEFAULT_MONITOR
+
+        self._flow_name = "not_a_real_flow"
+        _flow = DummyFlow(self._flow_name)
+        _environment = init_environment_outside_flow(_flow)
+        _monitor = MONITOR_SIDECARS[DEFAULT_MONITOR](_flow, _environment)
+        return _monitor
+
+    @property
+    def monitor(self) -> Optional["metaflow.monitor.NullMonitor"]:
+        if self._monitor is None:
+            # This happens if the monitor is being accessed outside of a flow
+            self._debug("Started monitor outside of a flow")
+            self._monitor = self._init_system_monitor_outside_flow()
+            self._monitor.start()
+        return self._monitor
+
+    @staticmethod
+    def _debug(msg: str):
+        """
+        Log a debug message to stderr.
+
+        Parameters
+        ----------
+        msg : str
+            Message to log.
+
+        """
+        if os.environ.get("METAFLOW_DEBUG_SIDECAR", "0").lower() not in (
+            "0",
+            "false",
+            "",
+        ):
+            print("system monitor: %s" % msg, file=sys.stderr)
+
+    @contextmanager
+    def measure(self, name: str):
+        """
+        Context manager to measure the execution duration and counter of a block of code.
+
+        Parameters
+        ----------
+        name : str
+            The name to associate with the timer and counter.
+
+        Yields
+        ------
+        None
+        """
+        # Delegating the context management to the monitor's measure method
+        with self.monitor.measure(name):
+            yield
+
+    @contextmanager
+    def count(self, name: str):
+        """
+        Context manager to increment a counter.
+
+        Parameters
+        ----------
+        name : str
+            The name of the counter.
+
+        Yields
+        ------
+        None
+        """
+        # Delegating the context management to the monitor's count method
+        with self.monitor.count(name):
+            yield
+
+    def gauge(self, gauge: "metaflow.monitor.Gauge"):
+        """
+        Log a gauge.
+
+        Parameters
+        ----------
+        gauge : metaflow.monitor.Gauge
+            The gauge to log.
+
+        """
+        self.monitor.gauge(gauge)

metaflow/system/system_utils.py

@@ -0,0 +1,19 @@
+from typing import Union
+
+
+class DummyFlow(object):
+    def __init__(self, name="not_a_real_flow"):
+        self.name = name
+
+
+# This function is used to initialize the environment outside a flow.
+def init_environment_outside_flow(
+    flow: Union["metaflow.flowspec.FlowSpec", "metaflow.sidecar.DummyFlow"]
+) -> "metaflow.metaflow_environment.MetaflowEnvironment":
+    from metaflow.plugins import ENVIRONMENTS
+    from metaflow.metaflow_config import DEFAULT_ENVIRONMENT
+    from metaflow.metaflow_environment import MetaflowEnvironment
+
+    return [
+        e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == DEFAULT_ENVIRONMENT
+    ][0](flow)

metaflow/task.py

@@ -4,9 +4,11 @@ import math
 import sys
 import os
 import time
+import traceback
 
 from types import MethodType, FunctionType
 
+from metaflow.sidecar import Message, MessageTypes
 from metaflow.datastore.exceptions import DataException
 
 from .metaflow_config import MAX_ATTEMPTS
@@ -22,6 +24,7 @@ from .unbounded_foreach import UBF_CONTROL
 from .util import all_equal, get_username, resolve_identity, unicode_type
 from .clone_util import clone_task_helper
 from .metaflow_current import current
+from metaflow.system import _system_logger, _system_monitor
 from metaflow.tracing import get_trace_id
 from metaflow.tuple_util import ForeachFrame
 
@@ -280,25 +283,42 @@ class MetaflowTask(object):
                 "task.clone_only needs a valid clone_origin_task value."
             )
         origin_run_id, _, origin_task_id = clone_origin_task.split("/")
-
-        msg = {
-            "task_id": task_id,
-            "msg": "Cloning task from {}/{}/{}/{} to {}/{}/{}/{}".format(
-                self.flow.name,
-                origin_run_id,
-                step_name,
-                origin_task_id,
-                self.flow.name,
-                run_id,
-                step_name,
-                task_id,
-            ),
-            "step_name": step_name,
+        # Update system logger and monitor context
+        # We also pass this context as part of the task payload to support implementations that
+        # can't access the context directly
+        task_payload = {
             "run_id": run_id,
-            "flow_name": self.flow.name,
-            "ts": round(time.time()),
+            "step_name": step_name,
+            "task_id": task_id,
+            "retry_count": retry_count,
+            "project_name": current.get("project_name"),
+            "branch_name": current.get("branch_name"),
+            "is_user_branch": current.get("is_user_branch"),
+            "is_production": current.get("is_production"),
+            "project_flow_name": current.get("project_flow_name"),
+            "origin_run_id": origin_run_id,
+            "origin_task_id": origin_task_id,
         }
-        self.event_logger.log(msg)
+        _system_logger.update_context(task_payload)
+        _system_monitor.update_context(task_payload)
+
+        msg = "Cloning task from {}/{}/{}/{} to {}/{}/{}/{}".format(
+            self.flow.name,
+            origin_run_id,
+            step_name,
+            origin_task_id,
+            self.flow.name,
+            run_id,
+            step_name,
+            task_id,
+        )
+        with _system_monitor.count("metaflow.task.clone"):
+            _system_logger.log_event(
+                level="info",
+                module="metaflow.task",
+                name="clone",
+                payload={**task_payload, "msg": msg},
+            )
         # If we actually have to do the clone ourselves, proceed...
         clone_task_helper(
             self.flow.name,
@@ -502,204 +522,230 @@ class MetaflowTask(object):
                 }
             }
         )
-        logger = self.event_logger
+
+        # 6. Update system logger and monitor context
+        # We also pass this context as part of the task payload to support implementations that
+        # can't access the context directly
+
+        task_payload = {
+            "run_id": run_id,
+            "step_name": step_name,
+            "task_id": task_id,
+            "retry_count": retry_count,
+            "project_name": current.get("project_name"),
+            "branch_name": current.get("branch_name"),
+            "is_user_branch": current.get("is_user_branch"),
+            "is_production": current.get("is_production"),
+            "project_flow_name": current.get("project_flow_name"),
+            "trace_id": trace_id or None,
+        }
+
+        _system_logger.update_context(task_payload)
+        _system_monitor.update_context(task_payload)
         start = time.time()
         self.metadata.start_task_heartbeat(self.flow.name, run_id, step_name, task_id)
-        try:
-            msg = {
-                "task_id": task_id,
-                "msg": "task starting",
-                "step_name": step_name,
-                "run_id": run_id,
-                "flow_name": self.flow.name,
-                "ts": round(time.time()),
-            }
-            logger.log(msg)
-
-            self.flow._current_step = step_name
-            self.flow._success = False
-            self.flow._task_ok = None
-            self.flow._exception = None
-            # Note: All internal flow attributes (ie: non-user artifacts)
-            # should either be set prior to running the user code or listed in
-            # FlowSpec._EPHEMERAL to allow for proper merging/importing of
-            # user artifacts in the user's step code.
-
-            if join_type:
-                # Join step:
-
-                # Ensure that we have the right number of inputs. The
-                # foreach case is checked above.
-                if join_type != "foreach" and len(inputs) != len(node.in_funcs):
-                    raise MetaflowDataMissing(
-                        "Join *%s* expected %d "
-                        "inputs but only %d inputs "
-                        "were found" % (step_name, len(node.in_funcs), len(inputs))
+        with self.monitor.measure("metaflow.task.duration"):
+            try:
+                with self.monitor.count("metaflow.task.start"):
+                    _system_logger.log_event(
+                        level="info",
+                        module="metaflow.task",
+                        name="start",
+                        payload={**task_payload, "msg": "Task started"},
                     )
 
-                # Multiple input contexts are passed in as an argument
-                # to the step function.
-                input_obj = Inputs(self._clone_flow(inp) for inp in inputs)
-                self.flow._set_datastore(output)
-                # initialize parameters (if they exist)
-                # We take Parameter values from the first input,
-                # which is always safe since parameters are read-only
-                current._update_env(
-                    {"parameter_names": self._init_parameters(inputs[0], passdown=True)}
-                )
-            else:
-                # Linear step:
-                # We are running with a single input context.
-                # The context is embedded in the flow.
-                if len(inputs) > 1:
-                    # This should be captured by static checking but
-                    # let's assert this again
-                    raise MetaflowInternalError(
-                        "Step *%s* is not a join "
-                        "step but it gets multiple "
-                        "inputs." % step_name
-                    )
-                self.flow._set_datastore(inputs[0])
-                if input_paths:
+                self.flow._current_step = step_name
+                self.flow._success = False
+                self.flow._task_ok = None
+                self.flow._exception = None
+                # Note: All internal flow attributes (ie: non-user artifacts)
+                # should either be set prior to running the user code or listed in
+                # FlowSpec._EPHEMERAL to allow for proper merging/importing of
+                # user artifacts in the user's step code.
+
+                if join_type:
+                    # Join step:
+
+                    # Ensure that we have the right number of inputs. The
+                    # foreach case is checked above.
+                    if join_type != "foreach" and len(inputs) != len(node.in_funcs):
+                        raise MetaflowDataMissing(
+                            "Join *%s* expected %d "
+                            "inputs but only %d inputs "
+                            "were found" % (step_name, len(node.in_funcs), len(inputs))
+                        )
+
+                    # Multiple input contexts are passed in as an argument
+                    # to the step function.
+                    input_obj = Inputs(self._clone_flow(inp) for inp in inputs)
+                    self.flow._set_datastore(output)
                     # initialize parameters (if they exist)
                     # We take Parameter values from the first input,
                     # which is always safe since parameters are read-only
                     current._update_env(
                         {
                             "parameter_names": self._init_parameters(
-                                inputs[0], passdown=False
+                                inputs[0], passdown=True
                             )
                         }
                     )
+                else:
+                    # Linear step:
+                    # We are running with a single input context.
+                    # The context is embedded in the flow.
+                    if len(inputs) > 1:
+                        # This should be captured by static checking but
+                        # let's assert this again
+                        raise MetaflowInternalError(
+                            "Step *%s* is not a join "
+                            "step but it gets multiple "
+                            "inputs." % step_name
+                        )
+                    self.flow._set_datastore(inputs[0])
+                    if input_paths:
+                        # initialize parameters (if they exist)
+                        # We take Parameter values from the first input,
+                        # which is always safe since parameters are read-only
+                        current._update_env(
+                            {
+                                "parameter_names": self._init_parameters(
+                                    inputs[0], passdown=False
+                                )
+                            }
+                        )
+
+                for deco in decorators:
+                    deco.task_pre_step(
+                        step_name,
+                        output,
+                        self.metadata,
+                        run_id,
+                        task_id,
+                        self.flow,
+                        self.flow._graph,
+                        retry_count,
+                        max_user_code_retries,
+                        self.ubf_context,
+                        inputs,
+                    )
 
-            for deco in decorators:
-                deco.task_pre_step(
-                    step_name,
-                    output,
-                    self.metadata,
-                    run_id,
-                    task_id,
-                    self.flow,
-                    self.flow._graph,
-                    retry_count,
-                    max_user_code_retries,
-                    self.ubf_context,
-                    inputs,
-                )
-
-            for deco in decorators:
-                # decorators can actually decorate the step function,
-                # or they can replace it altogether. This functionality
-                # is used e.g. by catch_decorator which switches to a
-                # fallback code if the user code has failed too many
-                # times.
-                step_func = deco.task_decorate(
-                    step_func,
-                    self.flow,
-                    self.flow._graph,
-                    retry_count,
-                    max_user_code_retries,
-                    self.ubf_context,
-                )
+                for deco in decorators:
+                    # decorators can actually decorate the step function,
+                    # or they can replace it altogether. This functionality
+                    # is used e.g. by catch_decorator which switches to a
+                    # fallback code if the user code has failed too many
+                    # times.
+                    step_func = deco.task_decorate(
+                        step_func,
+                        self.flow,
+                        self.flow._graph,
+                        retry_count,
+                        max_user_code_retries,
+                        self.ubf_context,
+                    )
 
-            if join_type:
-                self._exec_step_function(step_func, input_obj)
-            else:
-                self._exec_step_function(step_func)
+                if join_type:
+                    self._exec_step_function(step_func, input_obj)
+                else:
+                    self._exec_step_function(step_func)
+
+                for deco in decorators:
+                    deco.task_post_step(
+                        step_name,
+                        self.flow,
+                        self.flow._graph,
+                        retry_count,
+                        max_user_code_retries,
+                    )
 
-            for deco in decorators:
-                deco.task_post_step(
-                    step_name,
-                    self.flow,
-                    self.flow._graph,
-                    retry_count,
-                    max_user_code_retries,
-                )
+                self.flow._task_ok = True
+                self.flow._success = True
+
+            except Exception as ex:
+                with self.monitor.count("metaflow.task.exception"):
+                    _system_logger.log_event(
+                        level="error",
+                        module="metaflow.task",
+                        name="exception",
+                        payload={**task_payload, "msg": traceback.format_exc()},
+                    )
 
-            self.flow._task_ok = True
-            self.flow._success = True
-
-        except Exception as ex:
-            tsk_msg = {
-                "task_id": task_id,
-                "exception_msg": str(ex),
-                "msg": "task failed with exception",
-                "step_name": step_name,
-                "run_id": run_id,
-                "flow_name": self.flow.name,
-            }
-            logger.log(tsk_msg)
+                exception_handled = False
+                for deco in decorators:
+                    res = deco.task_exception(
+                        ex,
+                        step_name,
+                        self.flow,
+                        self.flow._graph,
+                        retry_count,
+                        max_user_code_retries,
+                    )
+                    exception_handled = bool(res) or exception_handled
+
+                if exception_handled:
+                    self.flow._task_ok = True
+                else:
+                    self.flow._task_ok = False
+                    self.flow._exception = MetaflowExceptionWrapper(ex)
+                    print("%s failed:" % self.flow, file=sys.stderr)
+                    raise
+
+            finally:
+                if self.ubf_context == UBF_CONTROL:
+                    self._finalize_control_task()
+
+                # Emit metrics to logger/monitor sidecar implementations
+                with self.monitor.count("metaflow.task.end"):
+                    _system_logger.log_event(
+                        level="info",
+                        module="metaflow.task",
+                        name="end",
+                        payload={**task_payload, "msg": "Task ended"},
+                    )
 
-            exception_handled = False
-            for deco in decorators:
-                res = deco.task_exception(
-                    ex,
+                attempt_ok = str(bool(self.flow._task_ok))
+                self.metadata.register_metadata(
+                    run_id,
                     step_name,
-                    self.flow,
-                    self.flow._graph,
-                    retry_count,
-                    max_user_code_retries,
+                    task_id,
+                    [
+                        MetaDatum(
+                            field="attempt_ok",
+                            value=attempt_ok,
+                            type="internal_attempt_status",
+                            tags=["attempt_id:{0}".format(retry_count)],
+                        )
+                    ],
                 )
-                exception_handled = bool(res) or exception_handled
-
-            if exception_handled:
-                self.flow._task_ok = True
-            else:
-                self.flow._task_ok = False
-                self.flow._exception = MetaflowExceptionWrapper(ex)
-                print("%s failed:" % self.flow, file=sys.stderr)
-                raise
-
-        finally:
-            if self.ubf_context == UBF_CONTROL:
-                self._finalize_control_task()
-
-            end = time.time() - start
-
-            msg = {
-                "task_id": task_id,
-                "msg": "task ending",
-                "step_name": step_name,
-                "run_id": run_id,
-                "flow_name": self.flow.name,
-                "ts": round(time.time()),
-                "runtime": round(end),
-            }
-            logger.log(msg)
 
-            attempt_ok = str(bool(self.flow._task_ok))
-            self.metadata.register_metadata(
-                run_id,
-                step_name,
-                task_id,
-                [
-                    MetaDatum(
-                        field="attempt_ok",
-                        value=attempt_ok,
-                        type="internal_attempt_status",
-                        tags=["attempt_id:{0}".format(retry_count)],
+                output.save_metadata({"task_end": {}})
+                output.persist(self.flow)
+
+                # this writes a success marker indicating that the
+                # "transaction" is done
+                output.done()
+
+                # final decorator hook: The task results are now
+                # queryable through the client API / datastore
+                for deco in decorators:
+                    deco.task_finished(
+                        step_name,
+                        self.flow,
+                        self.flow._graph,
+                        self.flow._task_ok,
+                        retry_count,
+                        max_user_code_retries,
                     )
-                ],
-            )
-
-            output.save_metadata({"task_end": {}})
-            output.persist(self.flow)
 
-            # this writes a success marker indicating that the
-            # "transaction" is done
-            output.done()
-
-            # final decorator hook: The task results are now
-            # queryable through the client API / datastore
-            for deco in decorators:
-                deco.task_finished(
-                    step_name,
-                    self.flow,
-                    self.flow._graph,
-                    self.flow._task_ok,
-                    retry_count,
-                    max_user_code_retries,
+                # terminate side cars
+                self.metadata.stop_heartbeat()
+
+                # Task duration consists of the time taken to run the task as well as the time taken to
+                # persist the task metadata and data to the datastore.
+                duration = time.time() - start
+                _system_logger.log_event(
+                    level="info",
+                    module="metaflow.task",
+                    name="duration",
+                    payload={**task_payload, "msg": str(duration)},
                 )
-
-            # terminate side cars
-            self.metadata.stop_heartbeat()

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.12.5"
+metaflow_version = "2.12.7"

{metaflow-2.12.5.dist-info → metaflow-2.12.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.5
+Version: 2.12.7
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs ==2.12.5 ; extra == 'stubs'
+Requires-Dist: metaflow-stubs ==2.12.7 ; extra == 'stubs'
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.5.dist-info → metaflow-2.12.7.dist-info}/RECORD

@@ -1,7 +1,7 @@
 metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
 metaflow/__init__.py,sha256=3GEqivYycw6mvjn-ndEFGuCdYnGztciQgEWX87vjf6M,5885
 metaflow/cards.py,sha256=tP1_RrtmqdFh741pqE4t98S7SA0MtGRlGvRICRZF1Mg,426
-metaflow/cli.py,sha256=VXNA2qL3-OpKiEjw7-YqHWZqjL0wBcXjcvsIU1Pufkg,33746
+metaflow/cli.py,sha256=YQBqwaPsD4OAkt_UQypZ0spB3LRlgomi-tExT_n2vdQ,33961
 metaflow/cli_args.py,sha256=lcgBGNTvfaiPxiUnejAe60Upt9swG6lRy1_3OqbU6MY,2616
 metaflow/clone_util.py,sha256=XfUX0vssu_hPlyZfhFl1AOnKkLqvt33Qp8xNrmdocGg,2057
 metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
@@ -30,12 +30,12 @@ metaflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/pylint_wrapper.py,sha256=zzBY9YaSUZOGH-ypDKAv2B_7XcoyMZj-zCoCrmYqNRc,2865
 metaflow/runtime.py,sha256=KQbLI4zH9V6L6YwOgYz28CPMOT_akoCWEBb49avSQvo,63993
 metaflow/tagging_util.py,sha256=ctyf0Q1gBi0RyZX6J0e9DQGNkNHblV_CITfy66axXB4,2346
-metaflow/task.py,sha256=tb5LNyx4u-yAQTM5MK1-wMw62dLfBeR9ojIZuYHeb0c,25592
+metaflow/task.py,sha256=uJHl8K4n3jNllWHSsG1vAZtDza0U2QbQcdg9GS_YPBE,28660
 metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=m5womQ7y-jXehuMyHPfByDbZ4HwTJxzs869cPOlMR8s,13057
 metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
-metaflow/version.py,sha256=c_PABqNl34MzEytuF8yQavSkvAjpdC4GMlENKrxredY,28
+metaflow/version.py,sha256=ve6PO5pM0roPNs0Q25TvJGmen3p4hLZdP2jdojcp-sM,28
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
 metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
@@ -110,7 +110,7 @@ metaflow/_vendor/v3_6/importlib_metadata/_meta.py,sha256=_F48Hu_jFxkfKWz5wcYS8vO
 metaflow/_vendor/v3_6/importlib_metadata/_text.py,sha256=HCsFksZpJLeTP3NEk_ngrAeXVRRtTrtyh9eOABoRP4A,2166
 metaflow/_vendor/v3_6/importlib_metadata/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
-metaflow/client/core.py,sha256=XjYNMytrm7pJmcaV2c2lu4NTFAnDRRqFO7WfcA04R4s,73721
+metaflow/client/core.py,sha256=aFAAacCf3F7Iq0ZGMKr_HW2jh-CKV7p2uC1GhZumPxI,74067
 metaflow/client/filecache.py,sha256=QdD1sW6w4Nnza-ioz4I1fEZI843X33AFIV3eSxq-cuU,14868
 metaflow/cmd/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/cmd/configure_cmd.py,sha256=o-DKnUf2FBo_HiMVyoyzQaGBSMtpbEPEdFTQZ0hkU-k,33396
@@ -174,7 +174,7 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=OAbj6S3uuyS-od_Fstrt2eif-8M2mHhPRgbQjCVRXho,130335
+metaflow/plugins/argo/argo_workflows.py,sha256=7WFmAjSbcJagQD8acih6iRZjeC5SPNXuJ2bFggPPoXA,130686
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=sZTpgfmc50eT3e0qIxpVqUgWhTcYlO1HM4gU6Oaya8g,33259
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=K5t4uIk2IXPdK7v7DEjj3buSB8ikLjLycKjbZUYeiaw,6781
 metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
@@ -285,11 +285,11 @@ metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7IT
 metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
 metaflow/plugins/pypi/bootstrap.py,sha256=Hik3PZ_RQC8T6hEf-NE2Xr_jq2ZIUkpgUtJlx-rqJgU,5107
-metaflow/plugins/pypi/conda_decorator.py,sha256=-bPxNtZKjxqOo4sj89uIp8ZVrCIontWhAp7wwRFjYpg,14189
-metaflow/plugins/pypi/conda_environment.py,sha256=7k5GJ6S-EiSkDxnFtNpdKkCOiMKSnjOJ-2N_0fWjMHU,19230
+metaflow/plugins/pypi/conda_decorator.py,sha256=phrUvVC5QrfNwPqIByrXsnpRDg1SNVsfpl1wbAVrykI,14679
+metaflow/plugins/pypi/conda_environment.py,sha256=COybS4bogDm956UzOzbd4JupE7PBbEMqq1dfl-f9DYM,19339
 metaflow/plugins/pypi/micromamba.py,sha256=wlVN2fm4WXFh3jVNtpDfu4XEz6VJKbmFNp0QvqlMIuI,12179
-metaflow/plugins/pypi/pip.py,sha256=MAgdyP7wK7Cp6iusG6S-jeKKDCxlA9k-jMqIGvyi0Ng,12472
-metaflow/plugins/pypi/pypi_decorator.py,sha256=syWk_oSQhIK9Y7OeOINMG2XVyxh9sj5uJhapwAXRBDw,5583
+metaflow/plugins/pypi/pip.py,sha256=uYPEHYV1_PtY4QA3NqUcVSPBAlRucGeY9tuyz7sB7aY,13641
+metaflow/plugins/pypi/pypi_decorator.py,sha256=Plmm4fhLECW-sj1QSFI84Gva7qqqwlJsqJ8laCRKIzw,6073
 metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
 metaflow/plugins/pypi/utils.py,sha256=ds1Mnv_DaxGnLAYp7ozg_K6oyguGyNhvHfE-75Ia1YA,2836
 metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcwd-WpruE,310
@@ -305,6 +305,10 @@ metaflow/sidecar/sidecar.py,sha256=EspKXvPPNiyRToaUZ51PS5TT_PzrBNAurn_wbFnmGr0,1
 metaflow/sidecar/sidecar_messages.py,sha256=zPsCoYgDIcDkkvdC9MEpJTJ3y6TSGm2JWkRc4vxjbFA,1071
 metaflow/sidecar/sidecar_subprocess.py,sha256=f72n5iJJAYfCIbz4D94-RxR37VvM7kVvE3c8E9dYHe8,9708
 metaflow/sidecar/sidecar_worker.py,sha256=4DfpxtnuphngOnIehKjNR_Knhu1hY7DYBcHl4Svpe3Y,2050
+metaflow/system/__init__.py,sha256=SB9Py7Acecqi76MY9MonSHXFuDD1yIJEGJtEQH8cNq4,149
+metaflow/system/system_logger.py,sha256=31noRo2qFdFyVtlyvIV2coUw4x6YdJGCdhctNBbBUso,3299
+metaflow/system/system_monitor.py,sha256=NracI4ITbUZWziGsAvE1910ODZhXGVYro7jD-rLWe48,3578
+metaflow/system/system_utils.py,sha256=E5C66_oeAxEE4-2okKlA-X6rgeYndPFP7KmWKHvvFZ8,657
 metaflow/tracing/__init__.py,sha256=xYTOT5BS5jbwhjk6hskxqNSU9st2LYtfeLN2Hknm3EI,1551
 metaflow/tracing/propagator.py,sha256=AdPeAqoeRauH82pTw01hLFNPRAzm29nlwM7C2iqKFFk,2502
 metaflow/tracing/span_exporter.py,sha256=kbi92INM8yKPqwLEVbWy7OcdsVJoh5nCbwAK9VyPx_w,1520
@@ -332,9 +336,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-metaflow-2.12.5.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.12.5.dist-info/METADATA,sha256=WNxjG7krB0L--6wpxwGN5tzT7B0SoB7wL-P7DZ4WDAI,5906
-metaflow-2.12.5.dist-info/WHEEL,sha256=pOwdCRdxkCaq8tWTvnGIC-q_meWYHwmqvvQPkeaM2I4,109
-metaflow-2.12.5.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.12.5.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.12.5.dist-info/RECORD,,
+metaflow-2.12.7.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.12.7.dist-info/METADATA,sha256=NkSjEHjnJp5ts8eQ8FV1y_z8a_d2nIeGbRGCx3OCW14,5906
+metaflow-2.12.7.dist-info/WHEEL,sha256=0XQbNV6JE5ziJsWjIU8TRRv0N6SohNonLWgP86g5fiI,109
+metaflow-2.12.7.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.12.7.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.12.7.dist-info/RECORD,,

{metaflow-2.12.5.dist-info → metaflow-2.12.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: setuptools (70.2.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any