ob-metaflow 2.15.18.1__py2.py3-none-any.whl → 2.16.0.1__py2.py3-none-any.whl

metaflow/__init__.py

@@ -104,7 +104,13 @@ from .flowspec import FlowSpec
 from .parameters import Parameter, JSONTypeClass, JSONType
 
 from .user_configs.config_parameters import Config, ConfigValue, config_expr
-from .user_configs.config_decorators import CustomFlowDecorator, CustomStepDecorator
+from .user_decorators.user_step_decorator import (
+    UserStepDecorator,
+    StepMutator,
+    user_step_decorator,
+    USER_SKIP_STEP,
+)
+from .user_decorators.user_flow_decorator import FlowMutator
 
 # data layer
 # For historical reasons, we make metaflow.plugins.datatools accessible as

metaflow/_vendor/imghdr/__init__.py

@@ -0,0 +1,180 @@
+"""Recognize image file formats based on their first few bytes."""
+
+from os import PathLike
+import warnings
+
+__all__ = ["what"]
+
+
+warnings._deprecated(__name__, remove=(3, 13))
+
+
+#-------------------------#
+# Recognize image headers #
+#-------------------------#
+
+def what(file, h=None):
+    """Return the type of image contained in a file or byte stream."""
+    f = None
+    try:
+        if h is None:
+            if isinstance(file, (str, PathLike)):
+                f = open(file, 'rb')
+                h = f.read(32)
+            else:
+                location = file.tell()
+                h = file.read(32)
+                file.seek(location)
+        for tf in tests:
+            res = tf(h, f)
+            if res:
+                return res
+    finally:
+        if f: f.close()
+    return None
+
+
+#---------------------------------#
+# Subroutines per image file type #
+#---------------------------------#
+
+tests = []
+
+def test_jpeg(h, f):
+    """Test for JPEG data with JFIF or Exif markers; and raw JPEG."""
+    if h[6:10] in (b'JFIF', b'Exif'):
+        return 'jpeg'
+    elif h[:4] == b'\xff\xd8\xff\xdb':
+        return 'jpeg'
+
+tests.append(test_jpeg)
+
+def test_png(h, f):
+    """Verify if the image is a PNG."""
+    if h.startswith(b'\211PNG\r\n\032\n'):
+        return 'png'
+
+tests.append(test_png)
+
+def test_gif(h, f):
+    """Verify if the image is a GIF ('87 or '89 variants)."""
+    if h[:6] in (b'GIF87a', b'GIF89a'):
+        return 'gif'
+
+tests.append(test_gif)
+
+def test_tiff(h, f):
+    """Verify if the image is a TIFF (can be in Motorola or Intel byte order)."""
+    if h[:2] in (b'MM', b'II'):
+        return 'tiff'
+
+tests.append(test_tiff)
+
+def test_rgb(h, f):
+    """test for the SGI image library."""
+    if h.startswith(b'\001\332'):
+        return 'rgb'
+
+tests.append(test_rgb)
+
+def test_pbm(h, f):
+    """Verify if the image is a PBM (portable bitmap)."""
+    if len(h) >= 3 and \
+        h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r':
+        return 'pbm'
+
+tests.append(test_pbm)
+
+def test_pgm(h, f):
+    """Verify if the image is a PGM (portable graymap)."""
+    if len(h) >= 3 and \
+        h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r':
+        return 'pgm'
+
+tests.append(test_pgm)
+
+def test_ppm(h, f):
+    """Verify if the image is a PPM (portable pixmap)."""
+    if len(h) >= 3 and \
+        h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r':
+        return 'ppm'
+
+tests.append(test_ppm)
+
+def test_rast(h, f):
+    """test for the Sun raster file."""
+    if h.startswith(b'\x59\xA6\x6A\x95'):
+        return 'rast'
+
+tests.append(test_rast)
+
+def test_xbm(h, f):
+    """Verify if the image is a X bitmap (X10 or X11)."""
+    if h.startswith(b'#define '):
+        return 'xbm'
+
+tests.append(test_xbm)
+
+def test_bmp(h, f):
+    """Verify if the image is a BMP file."""
+    if h.startswith(b'BM'):
+        return 'bmp'
+
+tests.append(test_bmp)
+
+def test_webp(h, f):
+    """Verify if the image is a WebP."""
+    if h.startswith(b'RIFF') and h[8:12] == b'WEBP':
+        return 'webp'
+
+tests.append(test_webp)
+
+def test_exr(h, f):
+    """verify is the image ia a OpenEXR fileOpenEXR."""
+    if h.startswith(b'\x76\x2f\x31\x01'):
+        return 'exr'
+
+tests.append(test_exr)
+
+#--------------------#
+# Small test program #
+#--------------------#
+
+def test():
+    import sys
+    recursive = 0
+    if sys.argv[1:] and sys.argv[1] == '-r':
+        del sys.argv[1:2]
+        recursive = 1
+    try:
+        if sys.argv[1:]:
+            testall(sys.argv[1:], recursive, 1)
+        else:
+            testall(['.'], recursive, 1)
+    except KeyboardInterrupt:
+        sys.stderr.write('\n[Interrupted]\n')
+        sys.exit(1)
+
+def testall(list, recursive, toplevel):
+    import sys
+    import os
+    for filename in list:
+        if os.path.isdir(filename):
+            print(filename + '/:', end=' ')
+            if recursive or toplevel:
+                print('recursing down:')
+                import glob
+                names = glob.glob(os.path.join(glob.escape(filename), '*'))
+                testall(names, recursive, 0)
+            else:
+                print('*** directory (use -r) ***')
+        else:
+            print(filename + ':', end=' ')
+            sys.stdout.flush()
+            try:
+                print(what(filename))
+            except OSError:
+                print('*** not found ***')
+
+if __name__ == '__main__':
+    test()

metaflow/cli.py

@@ -28,6 +28,7 @@ from .metaflow_config import (
 from .metaflow_current import current
 from metaflow.system import _system_monitor, _system_logger
 from .metaflow_environment import MetaflowEnvironment
+from .packaging_sys import MetaflowCodeContent
 from .plugins import (
     DATASTORES,
     ENVIRONMENTS,
@@ -152,8 +153,13 @@ def check(obj, warnings=False):
 def show(obj):
     echo_always("\n%s" % obj.graph.doc)
     for node_name in obj.graph.sorted_nodes:
+        echo_always("")
         node = obj.graph[node_name]
-        echo_always("\nStep *%s*" % node.name, err=False)
+        for deco in node.decorators:
+            echo_always("@%s" % deco.name, err=False)
+        for deco in node.wrappers:
+            echo_always("@%s" % deco.decorator_name, err=False)
+        echo_always("Step *%s*" % node.name, err=False)
         echo_always(node.doc if node.doc else "?", indent=True, err=False)
         if node.type != "end":
             echo_always(
@@ -336,6 +342,11 @@ def start(
     echo(" executing *%s*" % ctx.obj.flow.name, fg="magenta", nl=False)
     echo(" for *%s*" % resolve_identity(), fg="magenta")
 
+    # Check if we need to setup the distribution finder (if running )
+    dist_info = MetaflowCodeContent.get_distribution_finder()
+    if dist_info:
+        sys.meta_path.append(dist_info)
+
     # Setup the context
     cli_args._set_top_kwargs(ctx.params)
     ctx.obj.echo = echo
@@ -436,6 +447,10 @@ def start(
         # be raised. For resume, since we ignore those options, we ignore the error.
         raise ctx.obj.delayed_config_exception
 
+    # Init all values in the config decorators and then process them
+    for decorator in ctx.obj.flow._flow_state.get(_FlowState.CONFIG_DECORATORS, []):
+        decorator.external_init()
+
     new_cls = ctx.obj.flow._process_config_decorators(config_options)
     if new_cls:
         ctx.obj.flow = new_cls(use_cli=False)

metaflow/cli_components/init_cmd.py

@@ -46,6 +46,7 @@ def init(obj, run_id=None, task_id=None, tags=None, **kwargs):
         obj.event_logger,
         obj.monitor,
         run_id=run_id,
+        skip_decorator_hooks=True,
     )
     obj.flow._set_constants(obj.graph, kwargs, obj.config_options)
     runtime.persist_constants(task_id=task_id)

metaflow/cli_components/run_cmds.py

@@ -8,7 +8,7 @@ from .. import decorators, namespace, parameters, tracing
 from ..exception import CommandException
 from ..graph import FlowGraph
 from ..metaflow_current import current
-from ..metaflow_config import DEFAULT_DECOSPECS
+from ..metaflow_config import DEFAULT_DECOSPECS, FEAT_ALWAYS_UPLOAD_CODE_PACKAGE
 from ..package import MetaflowPackage
 from ..runtime import NativeRuntime
 from ..system import _system_logger
@@ -61,7 +61,11 @@ def before_run(obj, tags, decospecs):
     # We explicitly avoid doing this in `start` since it is invoked for every
     # step in the run.
     obj.package = MetaflowPackage(
-        obj.flow, obj.environment, obj.echo, obj.package_suffixes
+        obj.flow,
+        obj.environment,
+        obj.echo,
+        suffixes=obj.package_suffixes,
+        flow_datastore=obj.flow_datastore if FEAT_ALWAYS_UPLOAD_CODE_PACKAGE else None,
     )
 
 

metaflow/client/core.py

@@ -32,11 +32,12 @@ from metaflow.exception import (
 from metaflow.includefile import IncludedFile
 from metaflow.metaflow_config import DEFAULT_METADATA, MAX_ATTEMPTS
 from metaflow.metaflow_environment import MetaflowEnvironment
+from metaflow.package import MetaflowPackage
+from metaflow.packaging_sys import ContentType
 from metaflow.plugins import ENVIRONMENTS, METADATA_PROVIDERS
 from metaflow.unbounded_foreach import CONTROL_TASK_TAG
 from metaflow.util import cached_property, is_stringish, resolve_identity, to_unicode
 
-from ..info_file import INFO_FILE
 from .filecache import FileCache
 
 if TYPE_CHECKING:
@@ -816,20 +817,26 @@ class MetaflowCode(object):
         self._path = info["location"]
         self._ds_type = info["ds_type"]
         self._sha = info["sha"]
+        self._code_metadata = info.get(
+            "metadata",
+            '{"version": 0, "archive_format": "tgz", "mfcontent_version": 0}',
+        )
+
+        self._backend = MetaflowPackage.get_backend(self._code_metadata)
 
         if filecache is None:
             filecache = FileCache()
         _, blobdata = filecache.get_data(
             self._ds_type, self._flow_name, self._path, self._sha
         )
-        code_obj = BytesIO(blobdata)
-        self._tar = tarfile.open(fileobj=code_obj, mode="r:gz")
-        # The JSON module in Python3 deals with Unicode. Tar gives bytes.
-        info_str = (
-            self._tar.extractfile(os.path.basename(INFO_FILE)).read().decode("utf-8")
-        )
-        self._info = json.loads(info_str)
-        self._flowspec = self._tar.extractfile(self._info["script"]).read()
+        self._code_obj = BytesIO(blobdata)
+        self._info = MetaflowPackage.cls_get_info(self._code_metadata, self._code_obj)
+        if self._info:
+            self._flowspec = MetaflowPackage.cls_get_content(
+                self._code_metadata, self._code_obj, self._info["script"]
+            )
+        else:
+            raise MetaflowInternalError("Code package metadata is invalid.")
 
     @property
     def path(self) -> str:
@@ -877,7 +884,9 @@ class MetaflowCode(object):
         TarFile
             TarFile for everything in this code package
         """
-        return self._tar
+        if self._backend.type == "tgz":
+            return self._backend.cls_open(self._code_obj)
+        raise RuntimeError("Archive is not a tarball")
 
     def extract(self) -> TemporaryDirectory:
         """
@@ -908,27 +917,10 @@ class MetaflowCode(object):
             The directory and its contents are automatically deleted when
             this object is garbage collected.
         """
-        exclusions = [
-            "metaflow/",
-            "metaflow_extensions/",
-            "INFO",
-            "CONFIG_PARAMETERS",
-            "conda.manifest",
-            # This file is created when using the conda/pypi features available in
-            # nflx-metaflow-extensions: https://github.com/Netflix/metaflow-nflx-extensions
-            "condav2-1.cnd",
-        ]
-        members = [
-            m
-            for m in self.tarball.getmembers()
-            if not any(
-                (x.endswith("/") and m.name.startswith(x)) or (m.name == x)
-                for x in exclusions
-            )
-        ]
-
         tmp = TemporaryDirectory()
-        self.tarball.extractall(tmp.name, members)
+        MetaflowPackage.cls_extract_into(
+            self._code_metadata, self._code_obj, tmp.name, ContentType.USER_CONTENT
+        )
         return tmp
 
     @property

metaflow/cmd/develop/stub_generator.py

@@ -7,7 +7,6 @@ import pathlib
 import re
 import time
 import typing
-
 from datetime import datetime
 from io import StringIO
 from types import ModuleType
@@ -335,6 +334,8 @@ class StubGenerator:
 
         # Imports that are needed at the top of the file
         self._imports = set()  # type: Set[str]
+
+        self._sub_module_imports = set()  # type: Set[Tuple[str, str]]``
         # Typing imports (behind if TYPE_CHECKING) that are needed at the top of the file
         self._typing_imports = set()  # type: Set[str]
         # Typevars that are defined
@@ -643,6 +644,21 @@ class StubGenerator:
                     "deployer"
                 ] = (self._current_module_name + "." + name)
 
+        # Handle TypedDict gracefully for Python 3.7 compatibility
+        # _TypedDictMeta is not available in Python 3.7
+        typed_dict_meta = getattr(typing, "_TypedDictMeta", None)
+        if typed_dict_meta is not None and isinstance(clazz, typed_dict_meta):
+            self._sub_module_imports.add(("typing", "TypedDict"))
+            total_flag = getattr(clazz, "__total__", False)
+            buff = StringIO()
+            # Emit the TypedDict base and total flag
+            buff.write(f"class {name}(TypedDict, total={total_flag}):\n")
+            # Write out each field from __annotations__
+            for field_name, field_type in clazz.__annotations__.items():
+                ann = self._get_element_name_with_module(field_type)
+                buff.write(f"{TAB}{field_name}: {ann}\n")
+            return buff.getvalue()
+
         buff = StringIO()
         # Class prototype
         buff.write("class " + name.split(".")[-1] + "(")
@@ -987,7 +1003,6 @@ class StubGenerator:
         ]
 
         docs = split_docs(raw_doc, section_boundaries)
-
         parameters, no_arg_version = parse_params_from_doc(docs["param_doc"])
 
         if docs["add_to_current_doc"]:
@@ -1515,6 +1530,8 @@ class StubGenerator:
                     f.write("import " + module + "\n")
                     if module == "typing":
                         imported_typing = True
+                for module, sub_module in self._sub_module_imports:
+                    f.write(f"from {module} import {sub_module}\n")
                 if self._typing_imports:
                     if not imported_typing:
                         f.write("import typing\n")

metaflow/datastore/task_datastore.py

@@ -99,7 +99,6 @@ class TaskDataStore(object):
         mode="r",
         allow_not_done=False,
     ):
-
         self._storage_impl = flow_datastore._storage_impl
         self.TYPE = self._storage_impl.TYPE
         self._ca_store = flow_datastore.ca_store

metaflow/debug.py

@@ -42,6 +42,11 @@ class Debug(object):
         filename = inspect.stack()[1][1]
         print("debug[%s %s:%s]: %s" % (typ, filename, lineno, s), file=sys.stderr)
 
+    def __getattr__(self, name):
+        # Small piece of code to get pyright and other linters to recognize that there
+        # are dynamic attributes.
+        return getattr(self, name)
+
     def noop(self, args):
         pass