ob-metaflow 2.11.0.1__tar.gz → 2.11.10.2__tar.gz

{ob-metaflow-2.11.0.1/ob_metaflow.egg-info → ob-metaflow-2.11.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.11.0.1
+Version: 2.11.10.2
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -11,6 +11,8 @@ Requires-Dist: requests
 Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
+Provides-Extra: stubs
+Requires-Dist: ob-metaflow-stubs==2.11.10.2; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob-metaflow-2.11.0.1 → ob-metaflow-2.11.10.2}/metaflow/__init__.py

@@ -98,7 +98,7 @@ from .multicore_utils import parallel_imap_unordered, parallel_map
 from .metaflow_profile import profile
 
 # current runtime singleton
-from .current import current
+from .metaflow_current import current
 
 # Flow spec
 from .flowspec import FlowSpec

{ob-metaflow-2.11.0.1 → ob-metaflow-2.11.10.2}/metaflow/cli.py

@@ -13,7 +13,7 @@ from . import parameters
 from . import decorators
 from . import metaflow_version
 from . import namespace
-from . import current
+from .metaflow_current import current
 from .cli_args import cli_args
 from .tagging_util import validate_tags
 from .util import (
@@ -235,7 +235,6 @@ def output_dot(obj):
 )
 @click.pass_obj
 def dump(obj, input_path, private=None, max_value_size=None, include=None, file=None):
-
     output = {}
     kwargs = {
         "show_private": private,
@@ -474,13 +473,6 @@ def logs(obj, input_path, stdout=None, stderr=None, both=None, timestamps=False)
     help="Pathspec of the origin task for this task to clone. Do "
     "not execute anything.",
 )
-@click.option(
-    "--clone-wait-only/--no-clone-wait-only",
-    default=False,
-    show_default=True,
-    help="If specified, waits for an external process to clone the task",
-    hidden=True,
-)
 @click.option(
     "--clone-run-id",
     default=None,
@@ -520,7 +512,6 @@ def step(
     retry_count=None,
     max_user_code_retries=None,
     clone_only=None,
-    clone_wait_only=False,
     clone_run_id=None,
     decospecs=None,
     ubf_context="none",
@@ -576,7 +567,6 @@ def step(
             task_id,
             clone_only,
             retry_count,
-            wait_only=clone_wait_only,
         )
     else:
         task.run_step(
@@ -720,6 +710,13 @@ def common_run_options(func):
     hidden=True,
     help="If specified, allows this call to be called in parallel",
 )
+@click.option(
+    "--resume-identifier",
+    default=None,
+    show_default=True,
+    hidden=True,
+    help="If specified, it identifies the task that started this resume call. It is in the form of {step_name}-{task_id}",
+)
 @click.argument("step-to-rerun", required=False)
 @cli.command(help="Resume execution of a previous run of this flow.")
 @common_run_options
@@ -737,8 +734,8 @@ def resume(
     max_log_size=None,
     decospecs=None,
     run_id_file=None,
+    resume_identifier=None,
 ):
-
     before_run(obj, tags, decospecs + obj.environment.decospecs())
 
     if origin_run_id is None:
@@ -791,10 +788,15 @@ def resume(
         max_workers=max_workers,
         max_num_splits=max_num_splits,
         max_log_size=max_log_size * 1024 * 1024,
+        resume_identifier=resume_identifier,
     )
     write_run_id(run_id_file, runtime.run_id)
+    runtime.print_workflow_info()
     runtime.persist_constants()
-    runtime.execute()
+    if clone_only:
+        runtime.clone_original_run()
+    else:
+        runtime.execute()
 
 
 @tracing.cli_entrypoint("cli/run")
@@ -824,7 +826,6 @@ def run(
     user_namespace=None,
     **kwargs
 ):
-
     if user_namespace is not None:
         namespace(user_namespace or None)
     before_run(obj, tags, decospecs + obj.environment.decospecs())
@@ -848,6 +849,7 @@ def run(
     write_run_id(run_id_file, runtime.run_id)
 
     obj.flow._set_constants(obj.graph, kwargs)
+    runtime.print_workflow_info()
     runtime.persist_constants()
     runtime.execute()
 

{ob-metaflow-2.11.0.1 → ob-metaflow-2.11.10.2}/metaflow/client/core.py

@@ -7,9 +7,19 @@ from collections import namedtuple
 from datetime import datetime
 from io import BytesIO
 from itertools import chain
-from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Tuple
+from typing import (
+    Any,
+    Dict,
+    FrozenSet,
+    Iterable,
+    Iterator,
+    List,
+    NamedTuple,
+    Optional,
+    Tuple,
+)
 
-from metaflow.current import current
+from metaflow.metaflow_current import current
 from metaflow.events import Trigger
 from metaflow.exception import (
     MetaflowInternalError,
@@ -203,6 +213,9 @@ def default_namespace() -> str:
     return get_namespace()
 
 
+MetaflowArtifacts = NamedTuple
+
+
 class MetaflowObject(object):
     """
     Base class for all Metaflow objects.
@@ -337,17 +350,17 @@ class MetaflowObject(object):
             raise MetaflowNotFound("%s does not exist" % self)
         return result
 
-    def __iter__(self) -> Iterable["MetaflowObject"]:
+    def __iter__(self) -> Iterator["MetaflowObject"]:
         """
         Iterate over all child objects of this object if any.
 
-        Note that only children present in the current namespace are returned iff
-        _namespace_check is set.
+        Note that only children present in the current namespace are returned if and
+        only if _namespace_check is set.
 
-        Returns
-        -------
-        Iterable[MetaflowObject]
-            Iterator over all children
+        Yields
+        ------
+        MetaflowObject
+            Children of this object
         """
         query_filter = {}
 
@@ -703,50 +716,6 @@ class MetaflowObject(object):
         return list(self._path_components)
 
 
-class MetaflowData(object):
-    """
-    Container of data artifacts produced by a `Task`. This object is
-    instantiated through `Task.data`.
-
-    `MetaflowData` allows results to be retrieved by their name
-    through a convenient dot notation:
-
-    ```python
-    Task(...).data.my_object
-    ```
-
-    You can also test the existence of an object
-
-    ```python
-    if 'my_object' in Task(...).data:
-        print('my_object found')
-    ```
-
-    Note that this container relies on the local cache to load all data
-    artifacts. If your `Task` contains a lot of data, a more efficient
-    approach is to load artifacts individually like so
-
-    ```
-    Task(...)['my_object'].data
-    ```
-    """
-
-    def __init__(self, artifacts: Iterable["DataArtifact"]):
-        self._artifacts = dict((art.id, art) for art in artifacts)
-
-    def __getattr__(self, name: str):
-        return self._artifacts[name].data
-
-    def __contains__(self, var):
-        return var in self._artifacts
-
-    def __str__(self):
-        return "<MetaflowData: %s>" % ", ".join(self._artifacts)
-
-    def __repr__(self):
-        return str(self)
-
-
 class MetaflowCode(object):
     """
     Snapshot of the code used to execute this `Run`. Instantiate the object through
@@ -981,6 +950,50 @@ class DataArtifact(MetaflowObject):
         super(DataArtifact, self).__setstate__(state)
 
 
+class MetaflowData(object):
+    """
+    Container of data artifacts produced by a `Task`. This object is
+    instantiated through `Task.data`.
+
+    `MetaflowData` allows results to be retrieved by their name
+    through a convenient dot notation:
+
+    ```python
+    Task(...).data.my_object
+    ```
+
+    You can also test the existence of an object
+
+    ```python
+    if 'my_object' in Task(...).data:
+        print('my_object found')
+    ```
+
+    Note that this container relies on the local cache to load all data
+    artifacts. If your `Task` contains a lot of data, a more efficient
+    approach is to load artifacts individually like so
+
+    ```
+    Task(...)['my_object'].data
+    ```
+    """
+
+    def __init__(self, artifacts: Iterable[DataArtifact]):
+        self._artifacts = dict((art.id, art) for art in artifacts)
+
+    def __getattr__(self, name: str):
+        return self._artifacts[name].data
+
+    def __contains__(self, var):
+        return var in self._artifacts
+
+    def __str__(self):
+        return "<MetaflowData: %s>" % ", ".join(self._artifacts)
+
+    def __repr__(self):
+        return str(self)
+
+
 class Task(MetaflowObject):
     """
     A `Task` represents an execution of a `Step`.
@@ -1172,7 +1185,7 @@ class Task(MetaflowObject):
         return MetaflowData(self)
 
     @property
-    def artifacts(self) -> "MetaflowArtifacts":
+    def artifacts(self) -> MetaflowArtifacts:
         """
         Returns a container of DataArtifacts produced by this task.
 
@@ -1438,8 +1451,11 @@ class Task(MetaflowObject):
             return self._log_size(stream, meta_dict)
 
     def loglines(
-        self, stream: str, as_unicode: bool = True, meta_dict: Dict[str, Any] = None
-    ) -> Iterable[Tuple[datetime, str]]:
+        self,
+        stream: str,
+        as_unicode: bool = True,
+        meta_dict: Optional[Dict[str, Any]] = None,
+    ) -> Iterator[Tuple[datetime, str]]:
         """
         Return an iterator over (utc_timestamp, logline) tuples.
 
@@ -1451,10 +1467,10 @@ class Task(MetaflowObject):
             If as_unicode=False, each logline is returned as a byte object. Otherwise,
             it is returned as a (unicode) string.
 
-        Returns
-        -------
-        Iterable[(datetime, str)]
-            Iterator over timestamp, logline pairs.
+        Yields
+        ------
+        Tuple[datetime, str]
+            Tuple of timestamp, logline pairs.
         """
         from metaflow.mflog.mflog import merge_logs
 
@@ -1525,6 +1541,39 @@ class Task(MetaflowObject):
             ds_type, ds_root, stream, attempt, *self.path_components
         )
 
+    def __iter__(self) -> Iterator[DataArtifact]:
+        """
+        Iterate over all children DataArtifact of this Task
+
+        Yields
+        ------
+        DataArtifact
+            A DataArtifact in this Step
+        """
+        for d in super(Task, self).__iter__():
+            yield d
+
+    def __getitem__(self, name: str) -> DataArtifact:
+        """
+        Returns the DataArtifact object with the artifact name 'name'
+
+        Parameters
+        ----------
+        name : str
+            Data artifact name
+
+        Returns
+        -------
+        DataArtifact
+            DataArtifact for this artifact name in this task
+
+        Raises
+        ------
+        KeyError
+            If the name does not identify a valid DataArtifact object
+        """
+        return super(Task, self).__getitem__(name)
+
     def __getstate__(self):
         return super(Task, self).__getstate__()
 
@@ -1585,10 +1634,10 @@ class Step(MetaflowObject):
         tags : str
             No op (legacy functionality)
 
-        Returns
-        -------
-        Iterable[Task]
-            Iterator over all `Task` objects in this step.
+        Yields
+        ------
+        Task
+            `Task` objects in this step.
         """
         return self._filtered_children(*tags)
 
@@ -1607,7 +1656,7 @@ class Step(MetaflowObject):
         """
         return next(self.control_tasks(), None)
 
-    def control_tasks(self, *tags: str) -> Iterable[Task]:
+    def control_tasks(self, *tags: str) -> Iterator[Task]:
         """
         [Unpublished API - use with caution!]
 
@@ -1619,10 +1668,11 @@ class Step(MetaflowObject):
         ----------
         tags : str
             Tags to match
-        Returns
-        -------
-        Iterable[Task]
-            Iterator over Control Task objects in this step
+
+        Yields
+        ------
+        Task
+            Control Task objects for this step
         """
         children = super(Step, self).__iter__()
         for child in children:
@@ -1643,11 +1693,39 @@ class Step(MetaflowObject):
                     ):
                         yield child
 
-    def __iter__(self):
-        children = super(Step, self).__iter__()
-        for t in children:
+    def __iter__(self) -> Iterator[Task]:
+        """
+        Iterate over all children Task of this Step
+
+        Yields
+        ------
+        Task
+            A Task in this Step
+        """
+        for t in super(Step, self).__iter__():
             yield t
 
+    def __getitem__(self, task_id: str) -> Task:
+        """
+        Returns the Task object with the task ID 'task_id'
+
+        Parameters
+        ----------
+        task_id : str
+            Task ID
+
+        Returns
+        -------
+        Task
+            Task for this task ID in this Step
+
+        Raises
+        ------
+        KeyError
+            If the task_id does not identify a valid Task object
+        """
+        return super(Step, self).__getitem__(task_id)
+
     def __getstate__(self):
         return super(Step, self).__getstate__()
 
@@ -1723,7 +1801,7 @@ class Run(MetaflowObject):
         # exclude _parameters step
         return x.id[0] != "_"
 
-    def steps(self, *tags: str) -> Iterable[Step]:
+    def steps(self, *tags: str) -> Iterator[Step]:
         """
         [Legacy function - do not use]
 
@@ -1738,10 +1816,10 @@ class Run(MetaflowObject):
         tags : str
             No op (legacy functionality)
 
-        Returns
-        -------
-        Iterable[Step]
-            Iterator over `Step` objects in this run.
+        Yields
+        ------
+        Step
+            `Step` objects in this run.
         """
         return self._filtered_children(*tags)
 
@@ -1976,6 +2054,39 @@ class Run(MetaflowObject):
         self._user_tags = frozenset(final_user_tags)
         self._tags = frozenset([*self._user_tags, *self._system_tags])
 
+    def __iter__(self) -> Iterator[Step]:
+        """
+        Iterate over all children Step of this Run
+
+        Yields
+        ------
+        Step
+            A Step in this Run
+        """
+        for s in super(Run, self).__iter__():
+            yield s
+
+    def __getitem__(self, name: str) -> Step:
+        """
+        Returns the Step object with the step name 'name'
+
+        Parameters
+        ----------
+        name : str
+            Step name
+
+        Returns
+        -------
+        Step
+            Step for this step name in this Run
+
+        Raises
+        ------
+        KeyError
+            If the name does not identify a valid Step object
+        """
+        return super(Run, self).__getitem__(name)
+
     def __getstate__(self):
         return super(Run, self).__getstate__()
 
@@ -2051,7 +2162,7 @@ class Flow(MetaflowObject):
             if run.successful:
                 return run
 
-    def runs(self, *tags: str) -> Iterable[Run]:
+    def runs(self, *tags: str) -> Iterator[Run]:
         """
         Returns an iterator over all `Run`s of this flow.
 
@@ -2064,12 +2175,48 @@ class Flow(MetaflowObject):
         tags : str
             Tags to match.
 
+        Yields
+        ------
+        Run
+            `Run` objects in this flow.
+        """
+        return self._filtered_children(*tags)
+
+    def __iter__(self) -> Iterator[Task]:
+        """
+        Iterate over all children Run of this Flow.
+
+        Note that only runs in the current namespace are returned unless
+        _namespace_check is False
+
+        Yields
+        ------
+        Run
+            A Run in this Flow
+        """
+        for r in super(Flow, self).__iter__():
+            yield r
+
+    def __getitem__(self, run_id: str) -> Run:
+        """
+        Returns the Run object with the run ID 'run_id'
+
+        Parameters
+        ----------
+        run_id : str
+            Run OD
+
         Returns
         -------
-        Iterable[Run]
-            Iterator over `Run` objects in this flow.
+        Run
+            Run for this run ID in this Flow
+
+        Raises
+        ------
+        KeyError
+            If the run_id does not identify a valid Run object
         """
-        return self._filtered_children(*tags)
+        return super(Flow, self).__getitem__(run_id)
 
     def __getstate__(self):
         return super(Flow, self).__getstate__()
@@ -2120,12 +2267,12 @@ class Metaflow(object):
         """
         return list(self)
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[Flow]:
         """
         Iterator over all flows present.
 
-        Only flows present in the set namespace are returned. A flow is present in a namespace if
-        it has at least one run that is in the namespace.
+        Only flows present in the set namespace are returned. A flow is present in a
+        namespace if it has at least one run that is in the namespace.
 
         Yields
         -------
@@ -2145,10 +2292,10 @@ class Metaflow(object):
             except MetaflowNamespaceMismatch:
                 continue
 
-    def __str__(self):
+    def __str__(self) -> str:
         return "Metaflow()"
 
-    def __getitem__(self, id: str) -> Flow:
+    def __getitem__(self, name: str) -> Flow:
         """
         Returns a specific flow by name.
 
@@ -2156,13 +2303,13 @@ class Metaflow(object):
 
         Parameters
         ----------
-        id : string
+        name : str
             Name of the Flow
 
         Returns
         -------
         Flow
-            Flow with the given ID.
+            Flow with the given name.
         """
         return Flow(id)
 

ob-metaflow-2.11.10.2/metaflow/clone_util.py

@@ -0,0 +1,71 @@
+import time
+from .metadata import MetaDatum
+
+
+def clone_task_helper(
+    flow_name,
+    clone_run_id,
+    run_id,
+    step_name,
+    clone_task_id,
+    task_id,
+    flow_datastore,
+    metadata_service,
+    origin_ds_set=None,
+    attempt_id=0,
+):
+    # 1. initialize output datastore
+    output = flow_datastore.get_task_datastore(
+        run_id, step_name, task_id, attempt=attempt_id, mode="w"
+    )
+    output.init_task()
+
+    origin_run_id, origin_step_name, origin_task_id = (
+        clone_run_id,
+        step_name,
+        clone_task_id,
+    )
+    # 2. initialize origin datastore
+    origin = None
+    if origin_ds_set:
+        origin = origin_ds_set.get_with_pathspec(
+            "{}/{}/{}".format(origin_run_id, origin_step_name, origin_task_id)
+        )
+    else:
+        origin = flow_datastore.get_task_datastore(
+            origin_run_id, origin_step_name, origin_task_id
+        )
+    metadata_tags = ["attempt_id:{0}".format(attempt_id)]
+    output.clone(origin)
+    _ = metadata_service.register_task_id(
+        run_id,
+        step_name,
+        task_id,
+        attempt_id,
+    )
+    metadata_service.register_metadata(
+        run_id,
+        step_name,
+        task_id,
+        [
+            MetaDatum(
+                field="origin-task-id",
+                value=str(origin_task_id),
+                type="origin-task-id",
+                tags=metadata_tags,
+            ),
+            MetaDatum(
+                field="origin-run-id",
+                value=str(origin_run_id),
+                type="origin-run-id",
+                tags=metadata_tags,
+            ),
+            MetaDatum(
+                field="attempt",
+                value=str(attempt_id),
+                type="attempt",
+                tags=metadata_tags,
+            ),
+        ],
+    )
+    output.done()