metaflow 2.15.17__py2.py3-none-any.whl → 2.15.19__py2.py3-none-any.whl

metaflow/_vendor/imghdr/__init__.py

@@ -0,0 +1,180 @@
+"""Recognize image file formats based on their first few bytes."""
+
+from os import PathLike
+import warnings
+
+__all__ = ["what"]
+
+
+warnings._deprecated(__name__, remove=(3, 13))
+
+
+#-------------------------#
+# Recognize image headers #
+#-------------------------#
+
+def what(file, h=None):
+    """Return the type of image contained in a file or byte stream."""
+    f = None
+    try:
+        if h is None:
+            if isinstance(file, (str, PathLike)):
+                f = open(file, 'rb')
+                h = f.read(32)
+            else:
+                location = file.tell()
+                h = file.read(32)
+                file.seek(location)
+        for tf in tests:
+            res = tf(h, f)
+            if res:
+                return res
+    finally:
+        if f: f.close()
+    return None
+
+
+#---------------------------------#
+# Subroutines per image file type #
+#---------------------------------#
+
+tests = []
+
+def test_jpeg(h, f):
+    """Test for JPEG data with JFIF or Exif markers; and raw JPEG."""
+    if h[6:10] in (b'JFIF', b'Exif'):
+        return 'jpeg'
+    elif h[:4] == b'\xff\xd8\xff\xdb':
+        return 'jpeg'
+
+tests.append(test_jpeg)
+
+def test_png(h, f):
+    """Verify if the image is a PNG."""
+    if h.startswith(b'\211PNG\r\n\032\n'):
+        return 'png'
+
+tests.append(test_png)
+
+def test_gif(h, f):
+    """Verify if the image is a GIF ('87 or '89 variants)."""
+    if h[:6] in (b'GIF87a', b'GIF89a'):
+        return 'gif'
+
+tests.append(test_gif)
+
+def test_tiff(h, f):
+    """Verify if the image is a TIFF (can be in Motorola or Intel byte order)."""
+    if h[:2] in (b'MM', b'II'):
+        return 'tiff'
+
+tests.append(test_tiff)
+
+def test_rgb(h, f):
+    """test for the SGI image library."""
+    if h.startswith(b'\001\332'):
+        return 'rgb'
+
+tests.append(test_rgb)
+
+def test_pbm(h, f):
+    """Verify if the image is a PBM (portable bitmap)."""
+    if len(h) >= 3 and \
+        h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r':
+        return 'pbm'
+
+tests.append(test_pbm)
+
+def test_pgm(h, f):
+    """Verify if the image is a PGM (portable graymap)."""
+    if len(h) >= 3 and \
+        h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r':
+        return 'pgm'
+
+tests.append(test_pgm)
+
+def test_ppm(h, f):
+    """Verify if the image is a PPM (portable pixmap)."""
+    if len(h) >= 3 and \
+        h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r':
+        return 'ppm'
+
+tests.append(test_ppm)
+
+def test_rast(h, f):
+    """test for the Sun raster file."""
+    if h.startswith(b'\x59\xA6\x6A\x95'):
+        return 'rast'
+
+tests.append(test_rast)
+
+def test_xbm(h, f):
+    """Verify if the image is a X bitmap (X10 or X11)."""
+    if h.startswith(b'#define '):
+        return 'xbm'
+
+tests.append(test_xbm)
+
+def test_bmp(h, f):
+    """Verify if the image is a BMP file."""
+    if h.startswith(b'BM'):
+        return 'bmp'
+
+tests.append(test_bmp)
+
+def test_webp(h, f):
+    """Verify if the image is a WebP."""
+    if h.startswith(b'RIFF') and h[8:12] == b'WEBP':
+        return 'webp'
+
+tests.append(test_webp)
+
+def test_exr(h, f):
+    """verify is the image ia a OpenEXR fileOpenEXR."""
+    if h.startswith(b'\x76\x2f\x31\x01'):
+        return 'exr'
+
+tests.append(test_exr)
+
+#--------------------#
+# Small test program #
+#--------------------#
+
+def test():
+    import sys
+    recursive = 0
+    if sys.argv[1:] and sys.argv[1] == '-r':
+        del sys.argv[1:2]
+        recursive = 1
+    try:
+        if sys.argv[1:]:
+            testall(sys.argv[1:], recursive, 1)
+        else:
+            testall(['.'], recursive, 1)
+    except KeyboardInterrupt:
+        sys.stderr.write('\n[Interrupted]\n')
+        sys.exit(1)
+
+def testall(list, recursive, toplevel):
+    import sys
+    import os
+    for filename in list:
+        if os.path.isdir(filename):
+            print(filename + '/:', end=' ')
+            if recursive or toplevel:
+                print('recursing down:')
+                import glob
+                names = glob.glob(os.path.join(glob.escape(filename), '*'))
+                testall(names, recursive, 0)
+            else:
+                print('*** directory (use -r) ***')
+        else:
+            print(filename + ':', end=' ')
+            sys.stdout.flush()
+            try:
+                print(what(filename))
+            except OSError:
+                print('*** not found ***')
+
+if __name__ == '__main__':
+    test()

metaflow/cli.py

@@ -1,5 +1,6 @@
 import functools
 import inspect
+import os
 import sys
 import traceback
 from datetime import datetime
@@ -242,6 +243,14 @@ def version(obj):
     type=click.Choice(["local"] + [m.TYPE for m in ENVIRONMENTS]),
     help="Execution environment type",
 )
+@click.option(
+    "--force-rebuild-environments/--no-force-rebuild-environments",
+    is_flag=True,
+    default=False,
+    hidden=True,
+    type=bool,
+    help="Explicitly rebuild the execution environments",
+)
 # See comment for --quiet
 @click.option(
     "--datastore",
@@ -300,6 +309,7 @@ def start(
     quiet=False,
     metadata=None,
     environment=None,
+    force_rebuild_environments=False,
     datastore=None,
     datastore_root=None,
     decospecs=None,
@@ -435,6 +445,8 @@ def start(
     ctx.obj.environment = [
         e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == environment
     ][0](ctx.obj.flow)
+    # set force rebuild flag for environments that support it.
+    ctx.obj.environment._force_rebuild = force_rebuild_environments
     ctx.obj.environment.validate_environment(ctx.obj.logger, datastore)
 
     ctx.obj.event_logger = LOGGING_SIDECARS[event_logger](

metaflow/cmd/develop/stub_generator.py

@@ -7,7 +7,6 @@ import pathlib
 import re
 import time
 import typing
-
 from datetime import datetime
 from io import StringIO
 from types import ModuleType
@@ -335,6 +334,8 @@ class StubGenerator:
 
         # Imports that are needed at the top of the file
         self._imports = set()  # type: Set[str]
+
+        self._sub_module_imports = set()  # type: Set[Tuple[str, str]]``
         # Typing imports (behind if TYPE_CHECKING) that are needed at the top of the file
         self._typing_imports = set()  # type: Set[str]
         # Typevars that are defined
@@ -643,6 +644,21 @@ class StubGenerator:
                     "deployer"
                 ] = (self._current_module_name + "." + name)
 
+        # Handle TypedDict gracefully for Python 3.7 compatibility
+        # _TypedDictMeta is not available in Python 3.7
+        typed_dict_meta = getattr(typing, "_TypedDictMeta", None)
+        if typed_dict_meta is not None and isinstance(clazz, typed_dict_meta):
+            self._sub_module_imports.add(("typing", "TypedDict"))
+            total_flag = getattr(clazz, "__total__", False)
+            buff = StringIO()
+            # Emit the TypedDict base and total flag
+            buff.write(f"class {name}(TypedDict, total={total_flag}):\n")
+            # Write out each field from __annotations__
+            for field_name, field_type in clazz.__annotations__.items():
+                ann = self._get_element_name_with_module(field_type)
+                buff.write(f"{TAB}{field_name}: {ann}\n")
+            return buff.getvalue()
+
         buff = StringIO()
         # Class prototype
         buff.write("class " + name.split(".")[-1] + "(")
@@ -987,7 +1003,6 @@ class StubGenerator:
         ]
 
         docs = split_docs(raw_doc, section_boundaries)
-
         parameters, no_arg_version = parse_params_from_doc(docs["param_doc"])
 
         if docs["add_to_current_doc"]:
@@ -1515,6 +1530,8 @@ class StubGenerator:
                     f.write("import " + module + "\n")
                     if module == "typing":
                         imported_typing = True
+                for module, sub_module in self._sub_module_imports:
+                    f.write(f"from {module} import {sub_module}\n")
                 if self._typing_imports:
                     if not imported_typing:
                         f.write("import typing\n")

metaflow/metaflow_config.py

@@ -214,8 +214,6 @@ CARD_GSROOT = from_conf(
 )
 CARD_NO_WARNING = from_conf("CARD_NO_WARNING", False)
 
-SKIP_CARD_DUALWRITE = from_conf("SKIP_CARD_DUALWRITE", False)
-
 RUNTIME_CARD_RENDER_INTERVAL = from_conf("RUNTIME_CARD_RENDER_INTERVAL", 60)
 
 # Azure storage account URL

metaflow/plugins/__init__.py

@@ -69,6 +69,7 @@ FLOW_DECORATORS_DESC = [
     ("trigger_on_finish", ".events_decorator.TriggerOnFinishDecorator"),
     ("pypi_base", ".pypi.pypi_decorator.PyPIFlowDecorator"),
     ("conda_base", ".pypi.conda_decorator.CondaFlowDecorator"),
+    ("exit_hook", ".exit_hook.exit_hook_decorator.ExitHookDecorator"),
 ]
 
 # Add environments here
@@ -240,6 +241,7 @@ from .cards.card_modules.test_cards import (
     TestTimeoutCard,
     TestRefreshCard,
     TestRefreshComponentCard,
+    TestImageCard,
 )
 
 CARDS = [
@@ -258,6 +260,7 @@ CARDS = [
     DefaultCardJSON,
     TestRefreshCard,
     TestRefreshComponentCard,
+    TestImageCard,
 ]
 merge_lists(CARDS, MF_EXTERNAL_CARDS, "type")
 

metaflow/plugins/airflow/airflow.py

@@ -654,6 +654,12 @@ class Airflow(object):
                 "to Airflow is not supported currently."
             )
 
+        if self.flow._flow_decorators.get("exit_hook"):
+            raise AirflowException(
+                "Deploying flows with the @exit_hook decorator "
+                "to Airflow is not currently supported."
+            )
+
         # Visit every node of the flow and recursively build the state machine.
         def _visit(node, workflow, exit_node=None):
             kube_deco = dict(