metaflow 2.12.5__tar.gz → 2.12.7__tar.gz

{metaflow-2.12.5/metaflow.egg-info → metaflow-2.12.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.5
+Version: 2.12.7
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.5; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.7; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.5 → metaflow-2.12.7}/metaflow/cli.py

@@ -23,6 +23,7 @@ from .metaflow_config import (
     DEFAULT_PACKAGE_SUFFIXES,
 )
 from .metaflow_current import current
+from metaflow.system import _system_monitor, _system_logger
 from .metaflow_environment import MetaflowEnvironment
 from .mflog import LOG_SOURCES, mflog
 from .package import MetaflowPackage
@@ -952,11 +953,13 @@ def start(
         flow=ctx.obj.flow, env=ctx.obj.environment
     )
     ctx.obj.event_logger.start()
+    _system_logger.init_system_logger(ctx.obj.flow.name, ctx.obj.event_logger)
 
     ctx.obj.monitor = MONITOR_SIDECARS[monitor](
         flow=ctx.obj.flow, env=ctx.obj.environment
     )
     ctx.obj.monitor.start()
+    _system_monitor.init_system_monitor(ctx.obj.flow.name, ctx.obj.monitor)
 
     ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == metadata][0](
         ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor

{metaflow-2.12.5 → metaflow-2.12.7}/metaflow/client/core.py

@@ -276,6 +276,11 @@ class MetaflowObject(object):
         self._attempt = attempt
         self._current_namespace = _current_namespace or get_namespace()
         self._namespace_check = _namespace_check
+        # If the current namespace is False, we disable checking for namespace for this
+        # and all children objects. Not setting namespace_check to False has the consequence
+        # of preventing access to children objects after the namespace changes
+        if self._current_namespace is None:
+            self._namespace_check = False
 
         if self._attempt is not None:
             if self._NAME not in ["task", "artifact"]:

{metaflow-2.12.5 → metaflow-2.12.7}/metaflow/plugins/argo/argo_workflows.py

@@ -6,6 +6,7 @@ import shlex
 import sys
 from collections import defaultdict
 from hashlib import sha1
+from math import inf
 
 from metaflow import JSONType, current
 from metaflow.decorators import flow_decorators
@@ -901,7 +902,9 @@ class ArgoWorkflows(object):
                                 "argo-{{workflow.name}}/%s/{{tasks.%s.outputs.parameters.task-id}}"
                                 % (n, self._sanitize(n))
                                 for n in node.in_funcs
-                            ]
+                            ],
+                            # NOTE: We set zlibmin to infinite because zlib compression for the Argo input-paths breaks template value substitution.
+                            zlibmin=inf,
                         )
                     )
                 ]
@@ -2154,7 +2157,8 @@ class ArgoWorkflows(object):
                                                 # everything within the body.
                                                 # NOTE: We need the conditional logic in order to successfully fall back to the default value
                                                 # when the event payload does not contain a key for a parameter.
-                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (.Input.body.payload.%s | toJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                # NOTE: Keys might contain dashes, so use the safer 'get' for fetching the value
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (get $.Input.body.payload "%s" | toRawJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
                                                 % (v, v),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for

{metaflow-2.12.5 → metaflow-2.12.7}/metaflow/plugins/pypi/conda_decorator.py

@@ -100,6 +100,10 @@ class CondaStepDecorator(StepDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.
@@ -340,6 +344,10 @@ class CondaFlowDecorator(FlowDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.

{metaflow-2.12.5 → metaflow-2.12.7}/metaflow/plugins/pypi/conda_environment.py

@@ -65,7 +65,7 @@ class CondaEnvironment(MetaflowEnvironment):
         micromamba = Micromamba()
         self.solvers = {"conda": micromamba, "pypi": Pip(micromamba)}
 
-    def init_environment(self, echo):
+    def init_environment(self, echo, only_steps=None):
         # The implementation optimizes for latency to ensure as many operations can
         # be turned into cheap no-ops as feasible. Otherwise, we focus on maintaining
         # a balance between latency and maintainability of code without re-implementing
@@ -77,6 +77,8 @@ class CondaEnvironment(MetaflowEnvironment):
         def environments(type_):
             seen = set()
             for step in self.flow:
+                if only_steps and step.name not in only_steps:
+                    continue
                 environment = self.get_environment(step)
                 if type_ in environment and environment["id_"] not in seen:
                     seen.add(environment["id_"])

{metaflow-2.12.5 → metaflow-2.12.7}/metaflow/plugins/pypi/pip.py

@@ -9,7 +9,6 @@ from itertools import chain, product
 from urllib.parse import unquote
 
 from metaflow.exception import MetaflowException
-from metaflow.util import which
 
 from .micromamba import Micromamba
 from .utils import pip_tags, wheel_tags
@@ -25,6 +24,23 @@ class PipException(MetaflowException):
         super(PipException, self).__init__(msg)
 
 
+class PipPackageNotFound(Exception):
+    "Wrapper for pip package resolve errors."
+
+    def __init__(self, error):
+        self.error = error
+        try:
+            # Parse the package spec from error message:
+            # ERROR: ERROR: Could not find a version that satisfies the requirement pkg==0.0.1 (from versions: none)
+            # ERROR: No matching distribution found for pkg==0.0.1
+            self.package_spec = re.search(
+                "ERROR: No matching distribution found for (.*)", self.error
+            )[1]
+            self.package_name = re.match("\w*", self.package_spec)[0]
+        except Exception:
+            pass
+
+
 METADATA_FILE = "{prefix}/.pip/metadata"
 INSTALLATION_MARKER = "{prefix}/.pip/id"
 
@@ -81,7 +97,16 @@ class Pip(object):
                     cmd.append(f"{package}{version}")
                 else:
                     cmd.append(f"{package}=={version}")
-            self._call(prefix, cmd)
+            try:
+                self._call(prefix, cmd)
+            except PipPackageNotFound as ex:
+                # pretty print package errors
+                raise PipException(
+                    "Could not find a binary distribution for %s \n"
+                    "for the platform %s\n\n"
+                    "Note that ***@pypi*** does not currently support source distributions"
+                    % (ex.package_spec, platform)
+                )
 
             def _format(dl_info):
                 res = {k: v for k, v in dl_info.items() if k in ["url"]}
@@ -302,11 +327,14 @@ class Pip(object):
                 .strip()
             )
         except subprocess.CalledProcessError as e:
+            errors = e.stderr.decode()
+            if "No matching distribution" in errors:
+                raise PipPackageNotFound(errors)
             raise PipException(
                 "command '{cmd}' returned error ({code}) {output}\n{stderr}".format(
                     cmd=" ".join(e.cmd),
                     code=e.returncode,
                     output=e.output.decode(),
-                    stderr=e.stderr.decode(),
+                    stderr=errors,
                 )
             )

{metaflow-2.12.5 → metaflow-2.12.7}/metaflow/plugins/pypi/pypi_decorator.py

@@ -70,6 +70,10 @@ class PyPIStepDecorator(StepDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @pypi
         # implementation.
@@ -119,6 +123,10 @@ class PyPIFlowDecorator(FlowDecorator):
         # --environment=pypi to --environment=conda
         _supported_virtual_envs.extend(["pypi"])
 
+        # TODO: Hardcoded for now to support Docker environment.
+        # We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
+        _supported_virtual_envs.extend(["docker"])
+
         # The --environment= requirement ensures that valid virtual environments are
         # created for every step to execute it, greatly simplifying the @conda
         # implementation.

metaflow-2.12.7/metaflow/system/__init__.py

@@ -0,0 +1,5 @@
+from .system_monitor import SystemMonitor
+from .system_logger import SystemLogger
+
+_system_logger = SystemLogger()
+_system_monitor = SystemMonitor()

metaflow-2.12.7/metaflow/system/system_logger.py

@@ -0,0 +1,103 @@
+import os
+import sys
+from typing import Dict, Any, Optional, Union
+
+
+class SystemLogger(object):
+    def __init__(self):
+        self._logger = None
+        self._flow_name = None
+        self._context = {}
+        self._is_context_updated = False
+
+    def __del__(self):
+        if self._flow_name == "not_a_real_flow":
+            self.logger.terminate()
+
+    def update_context(self, context: Dict[str, Any]):
+        """
+        Update the global context maintained by the system logger.
+
+        Parameters
+        ----------
+        context : Dict[str, Any]
+            A dictionary containing the context to update.
+
+        """
+        self._is_context_updated = True
+        self._context.update(context)
+
+    def init_system_logger(
+        self, flow_name: str, logger: "metaflow.event_logger.NullEventLogger"
+    ):
+        self._flow_name = flow_name
+        self._logger = logger
+
+    def _init_logger_outside_flow(self):
+        from .system_utils import DummyFlow
+        from .system_utils import init_environment_outside_flow
+        from metaflow.plugins import LOGGING_SIDECARS
+        from metaflow.metaflow_config import DEFAULT_EVENT_LOGGER
+
+        self._flow_name = "not_a_real_flow"
+        _flow = DummyFlow(self._flow_name)
+        _environment = init_environment_outside_flow(_flow)
+        _logger = LOGGING_SIDECARS[DEFAULT_EVENT_LOGGER](_flow, _environment)
+        return _logger
+
+    @property
+    def logger(self) -> Optional["metaflow.event_logger.NullEventLogger"]:
+        if self._logger is None:
+            # This happens if the logger is being accessed outside of a flow
+            # We start a logger with a dummy flow and a default environment
+            self._debug("Started logger outside of a flow")
+            self._logger = self._init_logger_outside_flow()
+            self._logger.start()
+        return self._logger
+
+    @staticmethod
+    def _debug(msg: str):
+        """
+        Log a debug message to stderr.
+
+        Parameters
+        ----------
+        msg : str
+            Message to log.
+
+        """
+        if os.environ.get("METAFLOW_DEBUG_SIDECAR", "0").lower() not in (
+            "0",
+            "false",
+            "",
+        ):
+            print("system monitor: %s" % msg, file=sys.stderr)
+
+    def log_event(
+        self, level: str, module: str, name: str, payload: Optional[Any] = None
+    ):
+        """
+        Log an event to the event logger.
+
+        Parameters
+        ----------
+        level : str
+            Log level of the event. Can be one of "info", "warning", "error", "critical", "debug".
+        module : str
+            Module of the event. Usually the name of the class, function, or module that the event is being logged from.
+        name : str
+            Name of the event. Used to qualify the event type.
+        payload : Optional[Any], default None
+            Payload of the event. Contains the event data.
+        """
+        self.logger.log(
+            {
+                "level": level,
+                "module": module,
+                "name": name,
+                "payload": payload if payload is not None else {},
+                "context": self._context,
+                "is_context_updated": self._is_context_updated,
+            }
+        )
+        self._is_context_updated = False

metaflow-2.12.7/metaflow/system/system_monitor.py

@@ -0,0 +1,132 @@
+import os
+import sys
+from ..debug import debug
+from contextlib import contextmanager
+from typing import Optional, Union, Dict, Any
+
+
+class SystemMonitor(object):
+    def __init__(self):
+        self._monitor = None
+        self._flow_name = None
+        self._context = {}
+
+    def __del__(self):
+        if self._flow_name == "not_a_real_flow":
+            self.monitor.terminate()
+
+    def update_context(self, context: Dict[str, Any]):
+        """
+        Update the global context maintained by the system monitor.
+
+        Parameters
+        ----------
+        context : Dict[str, Any]
+            A dictionary containing the context to update.
+
+        """
+        from metaflow.sidecar import Message, MessageTypes
+
+        self._context.update(context)
+        self.monitor.send(
+            Message(
+                MessageTypes.MUST_SEND,
+                {
+                    "is_context_updated": True,
+                    **self._context,
+                },
+            )
+        )
+
+    def init_system_monitor(
+        self, flow_name: str, monitor: "metaflow.monitor.NullMonitor"
+    ):
+        self._flow_name = flow_name
+        self._monitor = monitor
+
+    def _init_system_monitor_outside_flow(self):
+        from .system_utils import DummyFlow
+        from .system_utils import init_environment_outside_flow
+        from metaflow.plugins import MONITOR_SIDECARS
+        from metaflow.metaflow_config import DEFAULT_MONITOR
+
+        self._flow_name = "not_a_real_flow"
+        _flow = DummyFlow(self._flow_name)
+        _environment = init_environment_outside_flow(_flow)
+        _monitor = MONITOR_SIDECARS[DEFAULT_MONITOR](_flow, _environment)
+        return _monitor
+
+    @property
+    def monitor(self) -> Optional["metaflow.monitor.NullMonitor"]:
+        if self._monitor is None:
+            # This happens if the monitor is being accessed outside of a flow
+            self._debug("Started monitor outside of a flow")
+            self._monitor = self._init_system_monitor_outside_flow()
+            self._monitor.start()
+        return self._monitor
+
+    @staticmethod
+    def _debug(msg: str):
+        """
+        Log a debug message to stderr.
+
+        Parameters
+        ----------
+        msg : str
+            Message to log.
+
+        """
+        if os.environ.get("METAFLOW_DEBUG_SIDECAR", "0").lower() not in (
+            "0",
+            "false",
+            "",
+        ):
+            print("system monitor: %s" % msg, file=sys.stderr)
+
+    @contextmanager
+    def measure(self, name: str):
+        """
+        Context manager to measure the execution duration and counter of a block of code.
+
+        Parameters
+        ----------
+        name : str
+            The name to associate with the timer and counter.
+
+        Yields
+        ------
+        None
+        """
+        # Delegating the context management to the monitor's measure method
+        with self.monitor.measure(name):
+            yield
+
+    @contextmanager
+    def count(self, name: str):
+        """
+        Context manager to increment a counter.
+
+        Parameters
+        ----------
+        name : str
+            The name of the counter.
+
+        Yields
+        ------
+        None
+        """
+        # Delegating the context management to the monitor's count method
+        with self.monitor.count(name):
+            yield
+
+    def gauge(self, gauge: "metaflow.monitor.Gauge"):
+        """
+        Log a gauge.
+
+        Parameters
+        ----------
+        gauge : metaflow.monitor.Gauge
+            The gauge to log.
+
+        """
+        self.monitor.gauge(gauge)

metaflow-2.12.7/metaflow/system/system_utils.py

@@ -0,0 +1,19 @@
+from typing import Union
+
+
+class DummyFlow(object):
+    def __init__(self, name="not_a_real_flow"):
+        self.name = name
+
+
+# This function is used to initialize the environment outside a flow.
+def init_environment_outside_flow(
+    flow: Union["metaflow.flowspec.FlowSpec", "metaflow.sidecar.DummyFlow"]
+) -> "metaflow.metaflow_environment.MetaflowEnvironment":
+    from metaflow.plugins import ENVIRONMENTS
+    from metaflow.metaflow_config import DEFAULT_ENVIRONMENT
+    from metaflow.metaflow_environment import MetaflowEnvironment
+
+    return [
+        e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == DEFAULT_ENVIRONMENT
+    ][0](flow)