ob-metaflow 2.11.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl

metaflow/sidecar/sidecar_worker.py

@@ -48,8 +48,8 @@ def process_messages(worker_type, worker):
         pass
 
 
-@tracing.cli_entrypoint("sidecar")
 @click.command(help="Initialize workers")
+@tracing.cli("sidecar")
 @click.argument("worker-type")
 def main(worker_type):
     sidecar_type = SIDECARS.get(worker_type)

metaflow/system/__init__.py

@@ -0,0 +1,5 @@
+from .system_monitor import SystemMonitor
+from .system_logger import SystemLogger
+
+_system_logger = SystemLogger()
+_system_monitor = SystemMonitor()

metaflow/system/system_logger.py

@@ -0,0 +1,85 @@
+import os
+import sys
+from typing import Dict, Any, Optional, Union
+
+
+class SystemLogger(object):
+    def __init__(self):
+        self._logger = None
+        self._flow_name = None
+
+    def __del__(self):
+        if self._flow_name == "not_a_real_flow":
+            self.logger.terminate()
+
+    def init_system_logger(
+        self, flow_name: str, logger: "metaflow.event_logger.NullEventLogger"
+    ):
+        self._flow_name = flow_name
+        self._logger = logger
+
+    def _init_logger_outside_flow(self):
+        from .system_utils import DummyFlow
+        from .system_utils import init_environment_outside_flow
+        from metaflow.plugins import LOGGING_SIDECARS
+        from metaflow.metaflow_config import DEFAULT_EVENT_LOGGER
+
+        self._flow_name = "not_a_real_flow"
+        _flow = DummyFlow(self._flow_name)
+        _environment = init_environment_outside_flow(_flow)
+        _logger = LOGGING_SIDECARS[DEFAULT_EVENT_LOGGER](_flow, _environment)
+        return _logger
+
+    @property
+    def logger(self) -> Optional["metaflow.event_logger.NullEventLogger"]:
+        if self._logger is None:
+            # This happens if the logger is being accessed outside of a flow
+            # We start a logger with a dummy flow and a default environment
+            self._debug("Started logger outside of a flow")
+            self._logger = self._init_logger_outside_flow()
+            self._logger.start()
+        return self._logger
+
+    @staticmethod
+    def _debug(msg: str):
+        """
+        Log a debug message to stderr.
+
+        Parameters
+        ----------
+        msg : str
+            Message to log.
+
+        """
+        if os.environ.get("METAFLOW_DEBUG_SIDECAR", "0").lower() not in (
+            "0",
+            "false",
+            "",
+        ):
+            print("system logger: %s" % msg, file=sys.stderr)
+
+    def log_event(
+        self, level: str, module: str, name: str, payload: Optional[Any] = None
+    ):
+        """
+        Log an event to the event logger.
+
+        Parameters
+        ----------
+        level : str
+            Log level of the event. Can be one of "info", "warning", "error", "critical", "debug".
+        module : str
+            Module of the event. Usually the name of the class, function, or module that the event is being logged from.
+        name : str
+            Name of the event. Used to qualify the event type.
+        payload : Optional[Any], default None
+            Payload of the event. Contains the event data.
+        """
+        self.logger.log(
+            {
+                "level": level,
+                "module": module,
+                "name": name,
+                "payload": payload if payload is not None else {},
+            }
+        )

metaflow/system/system_monitor.py

@@ -0,0 +1,108 @@
+import os
+import sys
+from ..debug import debug
+from contextlib import contextmanager
+from typing import Optional, Union, Dict, Any
+
+
+class SystemMonitor(object):
+    def __init__(self):
+        self._monitor = None
+        self._flow_name = None
+
+    def __del__(self):
+        if self._flow_name == "not_a_real_flow":
+            self.monitor.terminate()
+
+    def init_system_monitor(
+        self, flow_name: str, monitor: "metaflow.monitor.NullMonitor"
+    ):
+        self._flow_name = flow_name
+        self._monitor = monitor
+
+    def _init_system_monitor_outside_flow(self):
+        from .system_utils import DummyFlow
+        from .system_utils import init_environment_outside_flow
+        from metaflow.plugins import MONITOR_SIDECARS
+        from metaflow.metaflow_config import DEFAULT_MONITOR
+
+        self._flow_name = "not_a_real_flow"
+        _flow = DummyFlow(self._flow_name)
+        _environment = init_environment_outside_flow(_flow)
+        _monitor = MONITOR_SIDECARS[DEFAULT_MONITOR](_flow, _environment)
+        return _monitor
+
+    @property
+    def monitor(self) -> Optional["metaflow.monitor.NullMonitor"]:
+        if self._monitor is None:
+            # This happens if the monitor is being accessed outside of a flow
+            self._debug("Started monitor outside of a flow")
+            self._monitor = self._init_system_monitor_outside_flow()
+            self._monitor.start()
+        return self._monitor
+
+    @staticmethod
+    def _debug(msg: str):
+        """
+        Log a debug message to stderr.
+
+        Parameters
+        ----------
+        msg : str
+            Message to log.
+
+        """
+        if os.environ.get("METAFLOW_DEBUG_SIDECAR", "0").lower() not in (
+            "0",
+            "false",
+            "",
+        ):
+            print("system monitor: %s" % msg, file=sys.stderr)
+
+    @contextmanager
+    def measure(self, name: str):
+        """
+        Context manager to measure the execution duration and counter of a block of code.
+
+        Parameters
+        ----------
+        name : str
+            The name to associate with the timer and counter.
+
+        Yields
+        ------
+        None
+        """
+        # Delegating the context management to the monitor's measure method
+        with self.monitor.measure(name):
+            yield
+
+    @contextmanager
+    def count(self, name: str):
+        """
+        Context manager to increment a counter.
+
+        Parameters
+        ----------
+        name : str
+            The name of the counter.
+
+        Yields
+        ------
+        None
+        """
+        # Delegating the context management to the monitor's count method
+        with self.monitor.count(name):
+            yield
+
+    def gauge(self, gauge: "metaflow.monitor.Gauge"):
+        """
+        Log a gauge.
+
+        Parameters
+        ----------
+        gauge : metaflow.monitor.Gauge
+            The gauge to log.
+
+        """
+        self.monitor.gauge(gauge)

metaflow/system/system_utils.py

@@ -0,0 +1,19 @@
+from typing import Union
+
+
+class DummyFlow(object):
+    def __init__(self, name="not_a_real_flow"):
+        self.name = name
+
+
+# This function is used to initialize the environment outside a flow.
+def init_environment_outside_flow(
+    flow: Union["metaflow.flowspec.FlowSpec", "metaflow.sidecar.DummyFlow"]
+) -> "metaflow.metaflow_environment.MetaflowEnvironment":
+    from metaflow.plugins import ENVIRONMENTS
+    from metaflow.metaflow_config import DEFAULT_ENVIRONMENT
+    from metaflow.metaflow_environment import MetaflowEnvironment
+
+    return [
+        e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == DEFAULT_ENVIRONMENT
+    ][0](flow)