ob-metaflow 2.16.8.2rc1__py2.py3-none-any.whl → 2.17.0.1__py2.py3-none-any.whl

metaflow/_vendor/yaml/serializer.py

@@ -0,0 +1,111 @@
+
+__all__ = ['Serializer', 'SerializerError']
+
+from .error import YAMLError
+from .events import *
+from .nodes import *
+
+class SerializerError(YAMLError):
+    pass
+
+class Serializer:
+
+    ANCHOR_TEMPLATE = 'id%03d'
+
+    def __init__(self, encoding=None,
+            explicit_start=None, explicit_end=None, version=None, tags=None):
+        self.use_encoding = encoding
+        self.use_explicit_start = explicit_start
+        self.use_explicit_end = explicit_end
+        self.use_version = version
+        self.use_tags = tags
+        self.serialized_nodes = {}
+        self.anchors = {}
+        self.last_anchor_id = 0
+        self.closed = None
+
+    def open(self):
+        if self.closed is None:
+            self.emit(StreamStartEvent(encoding=self.use_encoding))
+            self.closed = False
+        elif self.closed:
+            raise SerializerError("serializer is closed")
+        else:
+            raise SerializerError("serializer is already opened")
+
+    def close(self):
+        if self.closed is None:
+            raise SerializerError("serializer is not opened")
+        elif not self.closed:
+            self.emit(StreamEndEvent())
+            self.closed = True
+
+    #def __del__(self):
+    #    self.close()
+
+    def serialize(self, node):
+        if self.closed is None:
+            raise SerializerError("serializer is not opened")
+        elif self.closed:
+            raise SerializerError("serializer is closed")
+        self.emit(DocumentStartEvent(explicit=self.use_explicit_start,
+            version=self.use_version, tags=self.use_tags))
+        self.anchor_node(node)
+        self.serialize_node(node, None, None)
+        self.emit(DocumentEndEvent(explicit=self.use_explicit_end))
+        self.serialized_nodes = {}
+        self.anchors = {}
+        self.last_anchor_id = 0
+
+    def anchor_node(self, node):
+        if node in self.anchors:
+            if self.anchors[node] is None:
+                self.anchors[node] = self.generate_anchor(node)
+        else:
+            self.anchors[node] = None
+            if isinstance(node, SequenceNode):
+                for item in node.value:
+                    self.anchor_node(item)
+            elif isinstance(node, MappingNode):
+                for key, value in node.value:
+                    self.anchor_node(key)
+                    self.anchor_node(value)
+
+    def generate_anchor(self, node):
+        self.last_anchor_id += 1
+        return self.ANCHOR_TEMPLATE % self.last_anchor_id
+
+    def serialize_node(self, node, parent, index):
+        alias = self.anchors[node]
+        if node in self.serialized_nodes:
+            self.emit(AliasEvent(alias))
+        else:
+            self.serialized_nodes[node] = True
+            self.descend_resolver(parent, index)
+            if isinstance(node, ScalarNode):
+                detected_tag = self.resolve(ScalarNode, node.value, (True, False))
+                default_tag = self.resolve(ScalarNode, node.value, (False, True))
+                implicit = (node.tag == detected_tag), (node.tag == default_tag)
+                self.emit(ScalarEvent(alias, node.tag, implicit, node.value,
+                    style=node.style))
+            elif isinstance(node, SequenceNode):
+                implicit = (node.tag
+                            == self.resolve(SequenceNode, node.value, True))
+                self.emit(SequenceStartEvent(alias, node.tag, implicit,
+                    flow_style=node.flow_style))
+                index = 0
+                for item in node.value:
+                    self.serialize_node(item, node, index)
+                    index += 1
+                self.emit(SequenceEndEvent())
+            elif isinstance(node, MappingNode):
+                implicit = (node.tag
+                            == self.resolve(MappingNode, node.value, True))
+                self.emit(MappingStartEvent(alias, node.tag, implicit,
+                    flow_style=node.flow_style))
+                for key, value in node.value:
+                    self.serialize_node(key, node, None)
+                    self.serialize_node(value, node, key)
+                self.emit(MappingEndEvent())
+            self.ascend_resolver()
+

metaflow/_vendor/yaml/tokens.py

@@ -0,0 +1,104 @@
+
+class Token(object):
+    def __init__(self, start_mark, end_mark):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+    def __repr__(self):
+        attributes = [key for key in self.__dict__
+                if not key.endswith('_mark')]
+        attributes.sort()
+        arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
+                for key in attributes])
+        return '%s(%s)' % (self.__class__.__name__, arguments)
+
+#class BOMToken(Token):
+#    id = '<byte order mark>'
+
+class DirectiveToken(Token):
+    id = '<directive>'
+    def __init__(self, name, value, start_mark, end_mark):
+        self.name = name
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+class DocumentStartToken(Token):
+    id = '<document start>'
+
+class DocumentEndToken(Token):
+    id = '<document end>'
+
+class StreamStartToken(Token):
+    id = '<stream start>'
+    def __init__(self, start_mark=None, end_mark=None,
+            encoding=None):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.encoding = encoding
+
+class StreamEndToken(Token):
+    id = '<stream end>'
+
+class BlockSequenceStartToken(Token):
+    id = '<block sequence start>'
+
+class BlockMappingStartToken(Token):
+    id = '<block mapping start>'
+
+class BlockEndToken(Token):
+    id = '<block end>'
+
+class FlowSequenceStartToken(Token):
+    id = '['
+
+class FlowMappingStartToken(Token):
+    id = '{'
+
+class FlowSequenceEndToken(Token):
+    id = ']'
+
+class FlowMappingEndToken(Token):
+    id = '}'
+
+class KeyToken(Token):
+    id = '?'
+
+class ValueToken(Token):
+    id = ':'
+
+class BlockEntryToken(Token):
+    id = '-'
+
+class FlowEntryToken(Token):
+    id = ','
+
+class AliasToken(Token):
+    id = '<alias>'
+    def __init__(self, value, start_mark, end_mark):
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+class AnchorToken(Token):
+    id = '<anchor>'
+    def __init__(self, value, start_mark, end_mark):
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+class TagToken(Token):
+    id = '<tag>'
+    def __init__(self, value, start_mark, end_mark):
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+class ScalarToken(Token):
+    id = '<scalar>'
+    def __init__(self, value, plain, start_mark, end_mark, style=None):
+        self.value = value
+        self.plain = plain
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.style = style
+

metaflow/cli.py

@@ -7,6 +7,7 @@ from datetime import datetime
 
 import metaflow.tracing as tracing
 from metaflow._vendor import click
+from metaflow.system import _system_logger, _system_monitor
 
 from . import decorators, lint, metaflow_version, parameters, plugins
 from .cli_args import cli_args
@@ -26,7 +27,6 @@ from .metaflow_config import (
     DEFAULT_PACKAGE_SUFFIXES,
 )
 from .metaflow_current import current
-from metaflow.system import _system_monitor, _system_logger
 from .metaflow_environment import MetaflowEnvironment
 from .packaging_sys import MetaflowCodeContent
 from .plugins import (
@@ -38,9 +38,9 @@ from .plugins import (
 )
 from .pylint_wrapper import PyLint
 from .R import metaflow_r_version, use_r
-from .util import get_latest_run_id, resolve_identity
 from .user_configs.config_options import LocalFileInput, config_options
 from .user_configs.config_parameters import ConfigValue
+from .util import get_latest_run_id, resolve_identity
 
 ERASE_TO_EOL = "\033[K"
 HIGHLIGHT = "red"
@@ -56,6 +56,15 @@ def echo_dev_null(*args, **kwargs):
 
 
 def echo_always(line, **kwargs):
+    if kwargs.pop("wrap", False):
+        import textwrap
+
+        indent_str = INDENT if kwargs.get("indent", None) else ""
+        effective_width = 80 - len(indent_str)
+        wrapped = textwrap.wrap(line, width=effective_width, break_long_words=False)
+        line = "\n".join(indent_str + l for l in wrapped)
+        kwargs["indent"] = False
+
     kwargs["err"] = kwargs.get("err", True)
     if kwargs.pop("indent", None):
         line = "\n".join(INDENT + x for x in line.splitlines())

metaflow/cli_components/run_cmds.py

@@ -13,8 +13,6 @@ from ..package import MetaflowPackage
 from ..runtime import NativeRuntime
 from ..system import _system_logger
 
-# from ..client.core import Run
-
 from ..tagging_util import validate_tags
 from ..util import get_latest_run_id, write_latest_run_id
 
@@ -232,19 +230,6 @@ def resume(
                     step_to_rerun, ",".join(list(obj.graph.nodes.keys()))
                 )
             )
-
-        ## TODO: instead of checking execution path here, can add a warning later
-        ## instead of throwing an error. This is for resuming a step which was not
-        ## taken inside a branch i.e. not present in the execution path.
-
-        # origin_run = Run(f"{obj.flow.name}/{origin_run_id}", _namespace_check=False)
-        # executed_steps = {step.path_components[-1] for step in origin_run}
-        # if step_to_rerun not in executed_steps:
-        #     raise CommandException(
-        #         f"Cannot resume from step '{step_to_rerun}'. This step was not "
-        #         f"part of the original execution path for run '{origin_run_id}'."
-        #     )
-
         steps_to_rerun = {step_to_rerun}
 
     if run_id:

metaflow/client/core.py

@@ -831,10 +831,12 @@ class MetaflowCode(object):
         )
         self._code_obj = BytesIO(blobdata)
         self._info = MetaflowPackage.cls_get_info(self._code_metadata, self._code_obj)
+        self._code_obj.seek(0)
         if self._info:
             self._flowspec = MetaflowPackage.cls_get_content(
                 self._code_metadata, self._code_obj, self._info["script"]
             )
+            self._code_obj.seek(0)
         else:
             raise MetaflowInternalError("Code package metadata is invalid.")
 
@@ -885,7 +887,9 @@ class MetaflowCode(object):
             TarFile for everything in this code package
         """
         if self._backend.type == "tgz":
-            return self._backend.cls_open(self._code_obj)
+            to_return = self._backend.cls_open(self._code_obj)
+            self._code_obj.seek(0)
+            return to_return
         raise RuntimeError("Archive is not a tarball")
 
     def extract(self) -> TemporaryDirectory:
@@ -921,6 +925,7 @@ class MetaflowCode(object):
         MetaflowPackage.cls_extract_into(
             self._code_metadata, self._code_obj, tmp.name, ContentType.USER_CONTENT
         )
+        self._code_obj.seek(0)
         return tmp
 
     @property

metaflow/extension_support/__init__.py

@@ -205,9 +205,10 @@ def package_mfext_all():
     # the packaged metaflow_extensions directory "self-contained" so that
     # python doesn't go and search other parts of the system for more
     # metaflow_extensions.
-    yield os.path.join(
-        os.path.dirname(os.path.abspath(__file__)), "_empty_file.py"
-    ), os.path.join(EXT_PKG, "__init__.py")
+    if _all_packages:
+        yield os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), "_empty_file.py"
+        ), os.path.join(EXT_PKG, "__init__.py")
 
     for p in _all_packages:
         for path_tuple in package_mfext_package(p):

metaflow/flowspec.py

@@ -788,35 +788,6 @@ class FlowSpec(metaclass=FlowSpecMeta):
         value = item if _is_primitive_type(item) else reprlib.Repr().repr(item)
         return basestring(value)[:MAXIMUM_FOREACH_VALUE_CHARS]
 
-    def _validate_switch_cases(self, switch_cases, step):
-        resolved_cases = {}
-        for case_key, step_method in switch_cases.items():
-            if isinstance(case_key, str) and case_key.startswith("config:"):
-                full_path = case_key[len("config:") :]
-                parts = full_path.split(".", 1)
-                if len(parts) == 2:
-                    config_var_name, config_key_name = parts
-                    try:
-                        config_obj = getattr(self, config_var_name)
-                        resolved_key = str(getattr(config_obj, config_key_name))
-                    except AttributeError:
-                        msg = (
-                            "Step *{step}* references unknown config '{path}' "
-                            "in switch case.".format(step=step, path=full_path)
-                        )
-                        raise InvalidNextException(msg)
-                else:
-                    raise MetaflowInternalError(
-                        "Invalid config path format in switch case."
-                    )
-            else:
-                resolved_key = case_key
-
-            func_name = step_method.__func__.__name__
-            resolved_cases[resolved_key] = func_name
-
-        return resolved_cases
-
     def next(self, *dsts: Callable[..., None], **kwargs) -> None:
         """
         Indicates the next step to execute after this step has completed.
@@ -841,15 +812,6 @@ class FlowSpec(metaclass=FlowSpecMeta):
           evaluates to an iterator. A task will be launched for each value in the iterator and
           each task will execute the code specified by the step `foreach_step`.
 
-        - Switch statement:
-          ```
-          self.next({"case1": self.step_a, "case2": self.step_b}, condition='condition_variable')
-          ```
-          In this situation, `step_a` and `step_b` are methods in the current class decorated
-          with the `@step` decorator and `condition_variable` is a variable name in the current
-          class. The value of the condition variable determines which step to execute. If the
-          value doesn't match any of the dictionary keys, a RuntimeError is raised.
-
         Parameters
         ----------
         dsts : Callable[..., None]
@@ -865,7 +827,6 @@ class FlowSpec(metaclass=FlowSpecMeta):
 
         foreach = kwargs.pop("foreach", None)
         num_parallel = kwargs.pop("num_parallel", None)
-        condition = kwargs.pop("condition", None)
         if kwargs:
             kw = next(iter(kwargs))
             msg = (
@@ -882,79 +843,6 @@ class FlowSpec(metaclass=FlowSpecMeta):
             )
             raise InvalidNextException(msg)
 
-        # check: switch case using condition
-        if condition is not None:
-            if len(dsts) != 1 or not isinstance(dsts[0], dict) or not dsts[0]:
-                msg = (
-                    "Step *{step}* has an invalid self.next() transition. "
-                    "When using 'condition', the transition must be to a single, "
-                    "non-empty dictionary mapping condition values to step methods.".format(
-                        step=step
-                    )
-                )
-                raise InvalidNextException(msg)
-
-            if not isinstance(condition, basestring):
-                msg = (
-                    "Step *{step}* has an invalid self.next() transition. "
-                    "The argument to 'condition' must be a string.".format(step=step)
-                )
-                raise InvalidNextException(msg)
-
-            if foreach is not None or num_parallel is not None:
-                msg = (
-                    "Step *{step}* has an invalid self.next() transition. "
-                    "Switch statements cannot be combined with foreach or num_parallel.".format(
-                        step=step
-                    )
-                )
-                raise InvalidNextException(msg)
-
-            switch_cases = dsts[0]
-
-            # Validate that condition variable exists
-            try:
-                condition_value = getattr(self, condition)
-            except AttributeError:
-                msg = (
-                    "Condition variable *self.{var}* in step *{step}* "
-                    "does not exist. Make sure you set self.{var} in this step.".format(
-                        step=step, var=condition
-                    )
-                )
-                raise InvalidNextException(msg)
-
-            resolved_switch_cases = self._validate_switch_cases(switch_cases, step)
-
-            if str(condition_value) not in resolved_switch_cases:
-                available_cases = list(resolved_switch_cases.keys())
-                raise RuntimeError(
-                    f"Switch condition variable '{condition}' has value '{condition_value}' "
-                    f"which is not in the available cases: {available_cases}"
-                )
-
-            # Get the chosen step and set transition directly
-            chosen_step = resolved_switch_cases[str(condition_value)]
-
-            # Validate that the chosen step exists
-            if not hasattr(self, chosen_step):
-                msg = (
-                    "Step *{step}* specifies a switch transition to an "
-                    "unknown step, *{name}*.".format(step=step, name=chosen_step)
-                )
-                raise InvalidNextException(msg)
-
-            self._transition = ([chosen_step], None)
-            return
-
-        # Check for an invalid transition: a dictionary used without a 'condition' parameter.
-        if len(dsts) == 1 and isinstance(dsts[0], dict):
-            msg = (
-                "Step *{step}* has an invalid self.next() transition. "
-                "Dictionary argument requires 'condition' parameter.".format(step=step)
-            )
-            raise InvalidNextException(msg)
-
         # check: all destinations are methods of this object
         funcs = []
         for i, dst in enumerate(dsts):
@@ -1045,7 +933,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
             self._foreach_var = foreach
 
         # check: non-keyword transitions are valid
-        if foreach is None and condition is None:
+        if foreach is None:
             if len(dsts) < 1:
                 msg = (
                     "Step *{step}* has an invalid self.next() transition. "