metaflow 2.12.26__tar.gz → 2.12.28__tar.gz

{metaflow-2.12.26/metaflow.egg-info → metaflow-2.12.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.26
+Version: 2.12.28
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.26; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.28; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/cli.py

@@ -701,12 +701,12 @@ def resume(
     runtime.persist_constants()
 
     if runner_attribute_file:
-        with open(runner_attribute_file, "w") as f:
+        with open(runner_attribute_file, "w", encoding="utf-8") as f:
             json.dump(
                 {
                     "run_id": runtime.run_id,
                     "flow_name": obj.flow.name,
-                    "metadata": get_metadata(),
+                    "metadata": obj.metadata.metadata_str(),
                 },
                 f,
             )
@@ -716,6 +716,20 @@ def resume(
     if runtime.should_skip_clone_only_execution():
         return
 
+    current._update_env(
+        {
+            "run_id": runtime.run_id,
+        }
+    )
+    _system_logger.log_event(
+        level="info",
+        module="metaflow.resume",
+        name="start",
+        payload={
+            "msg": "Resuming run",
+        },
+    )
+
     with runtime.run_heartbeat():
         if clone_only:
             runtime.clone_original_run()
@@ -775,16 +789,29 @@ def run(
     write_file(run_id_file, runtime.run_id)
 
     obj.flow._set_constants(obj.graph, kwargs)
+    current._update_env(
+        {
+            "run_id": runtime.run_id,
+        }
+    )
+    _system_logger.log_event(
+        level="info",
+        module="metaflow.run",
+        name="start",
+        payload={
+            "msg": "Starting run",
+        },
+    )
     runtime.print_workflow_info()
     runtime.persist_constants()
 
     if runner_attribute_file:
-        with open(runner_attribute_file, "w") as f:
+        with open(runner_attribute_file, "w", encoding="utf-8") as f:
             json.dump(
                 {
                     "run_id": runtime.run_id,
                     "flow_name": obj.flow.name,
-                    "metadata": get_metadata(),
+                    "metadata": obj.metadata.metadata_str(),
                 },
                 f,
             )

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/client/core.py

@@ -16,6 +16,7 @@ from typing import (
     List,
     NamedTuple,
     Optional,
+    TYPE_CHECKING,
     Tuple,
 )
 
@@ -37,6 +38,9 @@ from metaflow.util import cached_property, is_stringish, resolve_identity, to_un
 from ..info_file import INFO_FILE
 from .filecache import FileCache
 
+if TYPE_CHECKING:
+    from metaflow.metadata import MetadataProvider
+
 try:
     # python2
     import cPickle as pickle
@@ -82,28 +86,16 @@ def metadata(ms: str) -> str:
         get_metadata()).
     """
     global current_metadata
-    infos = ms.split("@", 1)
-    types = [m.TYPE for m in METADATA_PROVIDERS]
-    if infos[0] in types:
-        current_metadata = [m for m in METADATA_PROVIDERS if m.TYPE == infos[0]][0]
-        if len(infos) > 1:
-            current_metadata.INFO = infos[1]
-    else:
-        # Deduce from ms; if starts with http, use service or else use local
-        if ms.startswith("http"):
-            metadata_type = "service"
-        else:
-            metadata_type = "local"
-        res = [m for m in METADATA_PROVIDERS if m.TYPE == metadata_type]
-        if not res:
-            print(
-                "Cannot find a '%s' metadata provider -- "
-                "try specifying one explicitly using <type>@<info>",
-                metadata_type,
-            )
-            return get_metadata()
-        current_metadata = res[0]
-        current_metadata.INFO = ms
+    provider, info = _metadata(ms)
+    if provider is None:
+        print(
+            "Cannot find a metadata provider -- "
+            "try specifying one explicitly using <type>@<info>",
+        )
+        return get_metadata()
+    current_metadata = provider
+    if info:
+        current_metadata.INFO = info
     return get_metadata()
 
 
@@ -127,7 +119,7 @@ def get_metadata() -> str:
     """
     if current_metadata is False:
         default_metadata()
-    return "%s@%s" % (current_metadata.TYPE, current_metadata.INFO)
+    return current_metadata.metadata_str()
 
 
 def default_metadata() -> str:
@@ -268,9 +260,16 @@ class MetaflowObject(object):
         _object: Optional["MetaflowObject"] = None,
         _parent: Optional["MetaflowObject"] = None,
         _namespace_check: bool = True,
+        _metaflow: Optional["Metaflow"] = None,
         _current_namespace: Optional[str] = None,
+        _current_metadata: Optional[str] = None,
     ):
-        self._metaflow = Metaflow()
+        # the default namespace is activated lazily at the first
+        # get_namespace(). The other option of activating
+        # the namespace at the import time is problematic, since there
+        # may be other modules that alter environment variables etc.
+        # which may affect the namespace setting.
+        self._metaflow = Metaflow(_current_metadata) or _metaflow
         self._parent = _parent
         self._path_components = None
         self._attempt = attempt
@@ -390,6 +389,7 @@ class MetaflowObject(object):
                     attempt=self._attempt,
                     _object=obj,
                     _parent=self,
+                    _metaflow=self._metaflow,
                     _namespace_check=self._namespace_check,
                     _current_namespace=(
                         self._current_namespace if self._namespace_check else None
@@ -505,6 +505,7 @@ class MetaflowObject(object):
                 attempt=self._attempt,
                 _object=obj,
                 _parent=self,
+                _metaflow=self._metaflow,
                 _namespace_check=self._namespace_check,
                 _current_namespace=(
                     self._current_namespace if self._namespace_check else None
@@ -552,7 +553,25 @@ class MetaflowObject(object):
             _current_namespace=ns,
         )
 
-    _UNPICKLE_FUNC = {"2.8.4": _unpickle_284, "2.12.4": _unpickle_2124}
+    def _unpickle_21227(self, data):
+        if len(data) != 5:
+            raise MetaflowInternalError(
+                "Unexpected size of array: {}".format(len(data))
+            )
+        pathspec, attempt, md, ns, namespace_check = data
+        self.__init__(
+            pathspec=pathspec,
+            attempt=attempt,
+            _namespace_check=namespace_check,
+            _current_metadata=md,
+            _current_namespace=ns,
+        )
+
+    _UNPICKLE_FUNC = {
+        "2.8.4": _unpickle_284,
+        "2.12.4": _unpickle_2124,
+        "2.12.27": _unpickle_21227,
+    }
 
     def __setstate__(self, state):
         """
@@ -595,10 +614,11 @@ class MetaflowObject(object):
         # checking for the namespace even after unpickling since we will know which
         # namespace to check.
         return {
-            "version": "2.12.4",
+            "version": "2.12.27",
             "data": [
                 self.pathspec,
                 self._attempt,
+                self._metaflow.metadata.metadata_str(),
                 self._current_namespace,
                 self._namespace_check,
             ],
@@ -2288,17 +2308,16 @@ class Metaflow(object):
         if it has at least one run in the namespace.
     """
 
-    def __init__(self):
-        # the default namespace is activated lazily at the first object
-        # invocation or get_namespace(). The other option of activating
-        # the namespace at the import time is problematic, since there
-        # may be other modules that alter environment variables etc.
-        # which may affect the namescape setting.
-        if current_namespace is False:
-            default_namespace()
-        if current_metadata is False:
-            default_metadata()
-        self.metadata = current_metadata
+    def __init__(self, _current_metadata: Optional[str] = None):
+        if _current_metadata:
+            provider, info = _metadata(_current_metadata)
+            self.metadata = provider
+            if info:
+                self.metadata.INFO = info
+        else:
+            if current_metadata is False:
+                default_metadata()
+            self.metadata = current_metadata
 
     @property
     def flows(self) -> List[Flow]:
@@ -2335,7 +2354,7 @@ class Metaflow(object):
         all_flows = all_flows if all_flows else []
         for flow in all_flows:
             try:
-                v = Flow(_object=flow)
+                v = Flow(_object=flow, _metaflow=self)
                 yield v
             except MetaflowNamespaceMismatch:
                 continue
@@ -2359,7 +2378,26 @@ class Metaflow(object):
         Flow
             Flow with the given name.
         """
-        return Flow(name)
+        return Flow(name, _metaflow=self)
+
+
+def _metadata(ms: str) -> Tuple[Optional["MetadataProvider"], Optional[str]]:
+    infos = ms.split("@", 1)
+    types = [m.TYPE for m in METADATA_PROVIDERS]
+    if infos[0] in types:
+        provider = [m for m in METADATA_PROVIDERS if m.TYPE == infos[0]][0]
+        if len(infos) > 1:
+            return provider, infos[1]
+        return provider, None
+    # Deduce from ms; if starts with http, use service or else use local
+    if ms.startswith("http"):
+        metadata_type = "service"
+    else:
+        metadata_type = "local"
+    res = [m for m in METADATA_PROVIDERS if m.TYPE == metadata_type]
+    if not res:
+        return None, None
+    return res[0], ms
 
 
 _CLASSES["flow"] = Flow

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/metadata/metadata.py

@@ -76,6 +76,12 @@ class ObjectOrder:
 
 @with_metaclass(MetadataProviderMeta)
 class MetadataProvider(object):
+    TYPE = None
+
+    @classmethod
+    def metadata_str(cls):
+        return "%s@%s" % (cls.TYPE, cls.INFO)
+
     @classmethod
     def compute_info(cls, val):
         """

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/plugins/argo/argo_workflows_cli.py

@@ -226,12 +226,12 @@ def create(
     validate_tags(tags)
 
     if deployer_attribute_file:
-        with open(deployer_attribute_file, "w") as f:
+        with open(deployer_attribute_file, "w", encoding="utf-8") as f:
             json.dump(
                 {
                     "name": obj.workflow_name,
                     "flow_name": obj.flow.name,
-                    "metadata": get_metadata(),
+                    "metadata": obj.metadata.metadata_str(),
                 },
                 f,
             )
@@ -657,7 +657,7 @@ def trigger(obj, run_id_file=None, deployer_attribute_file=None, **kwargs):
             json.dump(
                 {
                     "name": obj.workflow_name,
-                    "metadata": get_metadata(),
+                    "metadata": obj.metadata.metadata_str(),
                     "pathspec": "/".join((obj.flow.name, run_id)),
                 },
                 f,

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/plugins/aws/step_functions/step_functions_cli.py

@@ -162,7 +162,7 @@ def create(
                 {
                     "name": obj.state_machine_name,
                     "flow_name": obj.flow.name,
-                    "metadata": get_metadata(),
+                    "metadata": obj.metadata.metadata_str(),
                 },
                 f,
             )
@@ -502,7 +502,7 @@ def trigger(obj, run_id_file=None, deployer_attribute_file=None, **kwargs):
             json.dump(
                 {
                     "name": obj.state_machine_name,
-                    "metadata": get_metadata(),
+                    "metadata": obj.metadata.metadata_str(),
                     "pathspec": "/".join((obj.flow.name, run_id)),
                 },
                 f,

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/plugins/kubernetes/kubernetes_jobsets.py

@@ -332,11 +332,8 @@ class RunningJobSet(object):
                     name=self._name,
                 )
 
-                # Suspend the jobset and set the replica's to Zero.
-                #
+                # Suspend the jobset
                 obj["spec"]["suspend"] = True
-                for replicated_job in obj["spec"]["replicatedJobs"]:
-                    replicated_job["replicas"] = 0
 
                 api_instance.replace_namespaced_custom_object(
                     group=self._group,

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/plugins/resources_decorator.py

@@ -23,7 +23,7 @@ class ResourcesDecorator(StepDecorator):
     ----------
     cpu : int, default 1
         Number of CPUs required for this step.
-    gpu : int, default 0
+    gpu : int, optional, default None
         Number of GPUs required for this step.
     disk : int, optional, default None
         Disk size (in MB) required for this step. Only applies on Kubernetes.
@@ -37,7 +37,7 @@ class ResourcesDecorator(StepDecorator):
     name = "resources"
     defaults = {
         "cpu": "1",
-        "gpu": "0",
+        "gpu": None,
         "disk": None,
         "memory": "4096",
         "shared_memory": None,

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/plugins/tag_cli.py

@@ -225,10 +225,7 @@ def _get_client_run_obj(obj, run_id, user_namespace):
 
 
 def _set_current(obj):
-    current._set_env(
-        metadata_str="%s@%s"
-        % (obj.metadata.__class__.TYPE, obj.metadata.__class__.INFO)
-    )
+    current._set_env(metadata_str=obj.metadata.metadata_str())
 
 
 @click.group()

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/runner/metaflow_runner.py

@@ -6,7 +6,7 @@ import tempfile
 
 from typing import Dict, Iterator, Optional, Tuple
 
-from metaflow import Run, metadata
+from metaflow import Run
 
 from .utils import handle_timeout
 from .subprocess_manager import CommandManager, SubprocessManager
@@ -275,9 +275,10 @@ class Runner(object):
 
         # Set the correct metadata from the runner_attribute file corresponding to this run.
         metadata_for_flow = content.get("metadata")
-        metadata(metadata_for_flow)
 
-        run_object = Run(pathspec, _namespace_check=False)
+        run_object = Run(
+            pathspec, _namespace_check=False, _current_metadata=metadata_for_flow
+        )
         return ExecutingRun(self, command_obj, run_object)
 
     def run(self, **kwargs) -> ExecutingRun:

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/system/system_logger.py

@@ -7,26 +7,11 @@ class SystemLogger(object):
     def __init__(self):
         self._logger = None
         self._flow_name = None
-        self._context = {}
-        self._is_context_updated = False
 
     def __del__(self):
         if self._flow_name == "not_a_real_flow":
             self.logger.terminate()
 
-    def update_context(self, context: Dict[str, Any]):
-        """
-        Update the global context maintained by the system logger.
-
-        Parameters
-        ----------
-        context : Dict[str, Any]
-            A dictionary containing the context to update.
-
-        """
-        self._is_context_updated = True
-        self._context.update(context)
-
     def init_system_logger(
         self, flow_name: str, logger: "metaflow.event_logger.NullEventLogger"
     ):
@@ -71,7 +56,7 @@ class SystemLogger(object):
             "false",
             "",
         ):
-            print("system monitor: %s" % msg, file=sys.stderr)
+            print("system logger: %s" % msg, file=sys.stderr)
 
     def log_event(
         self, level: str, module: str, name: str, payload: Optional[Any] = None
@@ -96,8 +81,5 @@ class SystemLogger(object):
                 "module": module,
                 "name": name,
                 "payload": payload if payload is not None else {},
-                "context": self._context,
-                "is_context_updated": self._is_context_updated,
             }
         )
-        self._is_context_updated = False

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/system/system_monitor.py

@@ -9,35 +9,11 @@ class SystemMonitor(object):
     def __init__(self):
         self._monitor = None
         self._flow_name = None
-        self._context = {}
 
     def __del__(self):
         if self._flow_name == "not_a_real_flow":
             self.monitor.terminate()
 
-    def update_context(self, context: Dict[str, Any]):
-        """
-        Update the global context maintained by the system monitor.
-
-        Parameters
-        ----------
-        context : Dict[str, Any]
-            A dictionary containing the context to update.
-
-        """
-        from metaflow.sidecar import Message, MessageTypes
-
-        self._context.update(context)
-        self.monitor.send(
-            Message(
-                MessageTypes.MUST_SEND,
-                {
-                    "is_context_updated": True,
-                    **self._context,
-                },
-            )
-        )
-
     def init_system_monitor(
         self, flow_name: str, monitor: "metaflow.monitor.NullMonitor"
     ):

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow/task.py

@@ -306,8 +306,6 @@ class MetaflowTask(object):
             "origin_run_id": origin_run_id,
             "origin_task_id": origin_task_id,
         }
-        _system_logger.update_context(task_payload)
-        _system_monitor.update_context(task_payload)
 
         msg = "Cloning task from {}/{}/{}/{} to {}/{}/{}/{}".format(
             self.flow.name,
@@ -512,8 +510,7 @@ class MetaflowTask(object):
             origin_run_id=origin_run_id,
             namespace=resolve_identity(),
             username=get_username(),
-            metadata_str="%s@%s"
-            % (self.metadata.__class__.TYPE, self.metadata.__class__.INFO),
+            metadata_str=self.metadata.metadata_str(),
             is_running=True,
             tags=self.metadata.sticky_tags,
         )
@@ -546,9 +543,6 @@ class MetaflowTask(object):
             "project_flow_name": current.get("project_flow_name"),
             "trace_id": trace_id or None,
         }
-
-        _system_logger.update_context(task_payload)
-        _system_monitor.update_context(task_payload)
         start = time.time()
         self.metadata.start_task_heartbeat(self.flow.name, run_id, step_name, task_id)
         with self.monitor.measure("metaflow.task.duration"):
@@ -593,7 +587,8 @@ class MetaflowTask(object):
                         {
                             "parameter_names": self._init_parameters(
                                 inputs[0], passdown=True
-                            )
+                            ),
+                            "graph_info": self.flow._graph_info,
                         }
                     )
                 else:
@@ -617,7 +612,8 @@ class MetaflowTask(object):
                             {
                                 "parameter_names": self._init_parameters(
                                     inputs[0], passdown=False
-                                )
+                                ),
+                                "graph_info": self.flow._graph_info,
                             }
                         )
 
@@ -709,24 +705,34 @@ class MetaflowTask(object):
                         name="end",
                         payload={**task_payload, "msg": "Task ended"},
                     )
-
-                attempt_ok = str(bool(self.flow._task_ok))
-                self.metadata.register_metadata(
-                    run_id,
-                    step_name,
-                    task_id,
-                    [
-                        MetaDatum(
-                            field="attempt_ok",
-                            value=attempt_ok,
-                            type="internal_attempt_status",
-                            tags=["attempt_id:{0}".format(retry_count)],
-                        )
-                    ],
-                )
+                try:
+                    # persisting might fail due to unpicklable artifacts.
+                    output.persist(self.flow)
+                except Exception as ex:
+                    self.flow._task_ok = False
+                    raise ex
+                finally:
+                    # The attempt_ok metadata is used to determine task status so it is important
+                    # we ensure that it is written even in case of preceding failures.
+                    # f.ex. failing to serialize artifacts leads to a non-zero exit code for the process,
+                    # even if user code finishes successfully. Flow execution will not continue due to the exit,
+                    # so arguably we should mark the task as failed.
+                    attempt_ok = str(bool(self.flow._task_ok))
+                    self.metadata.register_metadata(
+                        run_id,
+                        step_name,
+                        task_id,
+                        [
+                            MetaDatum(
+                                field="attempt_ok",
+                                value=attempt_ok,
+                                type="internal_attempt_status",
+                                tags=["attempt_id:{0}".format(retry_count)],
+                            )
+                        ],
+                    )
 
                 output.save_metadata({"task_end": {}})
-                output.persist(self.flow)
 
                 # this writes a success marker indicating that the
                 # "transaction" is done

metaflow-2.12.28/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.12.28"

{metaflow-2.12.26 → metaflow-2.12.28/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.12.26
+Version: 2.12.28
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.12.26; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.12.28; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.12.26 → metaflow-2.12.28}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.12.26
+metaflow-stubs==2.12.28

metaflow-2.12.26/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.12.26"