metaflow 2.12.6__tar.gz → 2.12.7__tar.gz

{metaflow-2.12.6/metaflow.egg-info → metaflow-2.12.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.6
+Version: 2.12.7
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.6; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.7; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.6 → metaflow-2.12.7}/metaflow/cli.py

@@ -23,6 +23,7 @@ from .metaflow_config import (
     DEFAULT_PACKAGE_SUFFIXES,
 )
 from .metaflow_current import current
+from metaflow.system import _system_monitor, _system_logger
 from .metaflow_environment import MetaflowEnvironment
 from .mflog import LOG_SOURCES, mflog
 from .package import MetaflowPackage
@@ -952,11 +953,13 @@ def start(
         flow=ctx.obj.flow, env=ctx.obj.environment
     )
     ctx.obj.event_logger.start()
+    _system_logger.init_system_logger(ctx.obj.flow.name, ctx.obj.event_logger)
 
     ctx.obj.monitor = MONITOR_SIDECARS[monitor](
         flow=ctx.obj.flow, env=ctx.obj.environment
     )
     ctx.obj.monitor.start()
+    _system_monitor.init_system_monitor(ctx.obj.flow.name, ctx.obj.monitor)
 
     ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == metadata][0](
         ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor

{metaflow-2.12.6 → metaflow-2.12.7}/metaflow/client/core.py

@@ -276,6 +276,11 @@ class MetaflowObject(object):
         self._attempt = attempt
         self._current_namespace = _current_namespace or get_namespace()
         self._namespace_check = _namespace_check
+        # If the current namespace is False, we disable checking for namespace for this
+        # and all children objects. Not setting namespace_check to False has the consequence
+        # of preventing access to children objects after the namespace changes
+        if self._current_namespace is None:
+            self._namespace_check = False
 
         if self._attempt is not None:
             if self._NAME not in ["task", "artifact"]:

{metaflow-2.12.6 → metaflow-2.12.7}/metaflow/plugins/argo/argo_workflows.py

@@ -2158,7 +2158,7 @@ class ArgoWorkflows(object):
                                                 # NOTE: We need the conditional logic in order to successfully fall back to the default value
                                                 # when the event payload does not contain a key for a parameter.
                                                 # NOTE: Keys might contain dashes, so use the safer 'get' for fetching the value
-                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (get $.Input.body.payload "%s" | toJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (get $.Input.body.payload "%s" | toRawJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
                                                 % (v, v),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for

metaflow-2.12.7/metaflow/system/__init__.py

@@ -0,0 +1,5 @@
+from .system_monitor import SystemMonitor
+from .system_logger import SystemLogger
+
+_system_logger = SystemLogger()
+_system_monitor = SystemMonitor()

metaflow-2.12.7/metaflow/system/system_logger.py

@@ -0,0 +1,103 @@
+import os
+import sys
+from typing import Dict, Any, Optional, Union
+
+
+class SystemLogger(object):
+    def __init__(self):
+        self._logger = None
+        self._flow_name = None
+        self._context = {}
+        self._is_context_updated = False
+
+    def __del__(self):
+        if self._flow_name == "not_a_real_flow":
+            self.logger.terminate()
+
+    def update_context(self, context: Dict[str, Any]):
+        """
+        Update the global context maintained by the system logger.
+
+        Parameters
+        ----------
+        context : Dict[str, Any]
+            A dictionary containing the context to update.
+
+        """
+        self._is_context_updated = True
+        self._context.update(context)
+
+    def init_system_logger(
+        self, flow_name: str, logger: "metaflow.event_logger.NullEventLogger"
+    ):
+        self._flow_name = flow_name
+        self._logger = logger
+
+    def _init_logger_outside_flow(self):
+        from .system_utils import DummyFlow
+        from .system_utils import init_environment_outside_flow
+        from metaflow.plugins import LOGGING_SIDECARS
+        from metaflow.metaflow_config import DEFAULT_EVENT_LOGGER
+
+        self._flow_name = "not_a_real_flow"
+        _flow = DummyFlow(self._flow_name)
+        _environment = init_environment_outside_flow(_flow)
+        _logger = LOGGING_SIDECARS[DEFAULT_EVENT_LOGGER](_flow, _environment)
+        return _logger
+
+    @property
+    def logger(self) -> Optional["metaflow.event_logger.NullEventLogger"]:
+        if self._logger is None:
+            # This happens if the logger is being accessed outside of a flow
+            # We start a logger with a dummy flow and a default environment
+            self._debug("Started logger outside of a flow")
+            self._logger = self._init_logger_outside_flow()
+            self._logger.start()
+        return self._logger
+
+    @staticmethod
+    def _debug(msg: str):
+        """
+        Log a debug message to stderr.
+
+        Parameters
+        ----------
+        msg : str
+            Message to log.
+
+        """
+        if os.environ.get("METAFLOW_DEBUG_SIDECAR", "0").lower() not in (
+            "0",
+            "false",
+            "",
+        ):
+            print("system monitor: %s" % msg, file=sys.stderr)
+
+    def log_event(
+        self, level: str, module: str, name: str, payload: Optional[Any] = None
+    ):
+        """
+        Log an event to the event logger.
+
+        Parameters
+        ----------
+        level : str
+            Log level of the event. Can be one of "info", "warning", "error", "critical", "debug".
+        module : str
+            Module of the event. Usually the name of the class, function, or module that the event is being logged from.
+        name : str
+            Name of the event. Used to qualify the event type.
+        payload : Optional[Any], default None
+            Payload of the event. Contains the event data.
+        """
+        self.logger.log(
+            {
+                "level": level,
+                "module": module,
+                "name": name,
+                "payload": payload if payload is not None else {},
+                "context": self._context,
+                "is_context_updated": self._is_context_updated,
+            }
+        )
+        self._is_context_updated = False

metaflow-2.12.7/metaflow/system/system_monitor.py

@@ -0,0 +1,132 @@
+import os
+import sys
+from ..debug import debug
+from contextlib import contextmanager
+from typing import Optional, Union, Dict, Any
+
+
+class SystemMonitor(object):
+    def __init__(self):
+        self._monitor = None
+        self._flow_name = None
+        self._context = {}
+
+    def __del__(self):
+        if self._flow_name == "not_a_real_flow":
+            self.monitor.terminate()
+
+    def update_context(self, context: Dict[str, Any]):
+        """
+        Update the global context maintained by the system monitor.
+
+        Parameters
+        ----------
+        context : Dict[str, Any]
+            A dictionary containing the context to update.
+
+        """
+        from metaflow.sidecar import Message, MessageTypes
+
+        self._context.update(context)
+        self.monitor.send(
+            Message(
+                MessageTypes.MUST_SEND,
+                {
+                    "is_context_updated": True,
+                    **self._context,
+                },
+            )
+        )
+
+    def init_system_monitor(
+        self, flow_name: str, monitor: "metaflow.monitor.NullMonitor"
+    ):
+        self._flow_name = flow_name
+        self._monitor = monitor
+
+    def _init_system_monitor_outside_flow(self):
+        from .system_utils import DummyFlow
+        from .system_utils import init_environment_outside_flow
+        from metaflow.plugins import MONITOR_SIDECARS
+        from metaflow.metaflow_config import DEFAULT_MONITOR
+
+        self._flow_name = "not_a_real_flow"
+        _flow = DummyFlow(self._flow_name)
+        _environment = init_environment_outside_flow(_flow)
+        _monitor = MONITOR_SIDECARS[DEFAULT_MONITOR](_flow, _environment)
+        return _monitor
+
+    @property
+    def monitor(self) -> Optional["metaflow.monitor.NullMonitor"]:
+        if self._monitor is None:
+            # This happens if the monitor is being accessed outside of a flow
+            self._debug("Started monitor outside of a flow")
+            self._monitor = self._init_system_monitor_outside_flow()
+            self._monitor.start()
+        return self._monitor
+
+    @staticmethod
+    def _debug(msg: str):
+        """
+        Log a debug message to stderr.
+
+        Parameters
+        ----------
+        msg : str
+            Message to log.
+
+        """
+        if os.environ.get("METAFLOW_DEBUG_SIDECAR", "0").lower() not in (
+            "0",
+            "false",
+            "",
+        ):
+            print("system monitor: %s" % msg, file=sys.stderr)
+
+    @contextmanager
+    def measure(self, name: str):
+        """
+        Context manager to measure the execution duration and counter of a block of code.
+
+        Parameters
+        ----------
+        name : str
+            The name to associate with the timer and counter.
+
+        Yields
+        ------
+        None
+        """
+        # Delegating the context management to the monitor's measure method
+        with self.monitor.measure(name):
+            yield
+
+    @contextmanager
+    def count(self, name: str):
+        """
+        Context manager to increment a counter.
+
+        Parameters
+        ----------
+        name : str
+            The name of the counter.
+
+        Yields
+        ------
+        None
+        """
+        # Delegating the context management to the monitor's count method
+        with self.monitor.count(name):
+            yield
+
+    def gauge(self, gauge: "metaflow.monitor.Gauge"):
+        """
+        Log a gauge.
+
+        Parameters
+        ----------
+        gauge : metaflow.monitor.Gauge
+            The gauge to log.
+
+        """
+        self.monitor.gauge(gauge)

metaflow-2.12.7/metaflow/system/system_utils.py

@@ -0,0 +1,19 @@
+from typing import Union
+
+
+class DummyFlow(object):
+    def __init__(self, name="not_a_real_flow"):
+        self.name = name
+
+
+# This function is used to initialize the environment outside a flow.
+def init_environment_outside_flow(
+    flow: Union["metaflow.flowspec.FlowSpec", "metaflow.sidecar.DummyFlow"]
+) -> "metaflow.metaflow_environment.MetaflowEnvironment":
+    from metaflow.plugins import ENVIRONMENTS
+    from metaflow.metaflow_config import DEFAULT_ENVIRONMENT
+    from metaflow.metaflow_environment import MetaflowEnvironment
+
+    return [
+        e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == DEFAULT_ENVIRONMENT
+    ][0](flow)