metaflow 2.11.10__py2.py3-none-any.whl → 2.11.12__py2.py3-none-any.whl

metaflow/metaflow_config.py

@@ -363,6 +363,12 @@ CONDA_PACKAGE_GSROOT = from_conf("CONDA_PACKAGE_GSROOT")
 # should result in an appreciable speedup in flow environment initialization.
 CONDA_DEPENDENCY_RESOLVER = from_conf("CONDA_DEPENDENCY_RESOLVER", "conda")
 
+###
+# Escape hatch configuration
+###
+# Print out warning if escape hatch is not used for the target packages
+ESCAPE_HATCH_WARNING = from_conf("ESCAPE_HATCH_WARNING", True)
+
 ###
 # Debug configuration
 ###

metaflow/plugins/aws/aws_utils.py

@@ -19,14 +19,30 @@ def get_ec2_instance_metadata():
     # access to this end-point might be blocked on AWS and not available
     # for non-AWS deployments.
     # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html
+    # Set a very aggressive timeout, as the communication is happening in the same subnet,
+    # there should not be any significant delay in the response.
+    # Having a long default timeout here introduces unnecessary delay in launching tasks when the
+    # instance is unreachable.
+    timeout = (1, 10)
+    token = None
     try:
-        # Set a very aggressive timeout, as the communication is happening in the same subnet,
-        # there should not be any significant delay in the response.
-        # Having a long default timeout here introduces unnecessary delay in launching tasks when the
-        # instance is unreachable.
+        # Try to get an IMDSv2 token.
+        token = requests.put(
+            url="http://169.254.169.254/latest/api/token",
+            headers={"X-aws-ec2-metadata-token-ttl-seconds": 100},
+            timeout=timeout,
+        ).text
+    except:
+        pass
+    try:
+        headers = {}
+        # Add IMDSv2 token if available, else fall back to IMDSv1.
+        if token:
+            headers["X-aws-ec2-metadata-token"] = token
         instance_meta = requests.get(
             url="http://169.254.169.254/latest/dynamic/instance-identity/document",
-            timeout=(1, 10),
+            headers=headers,
+            timeout=timeout,
         ).json()
         meta["ec2-instance-id"] = instance_meta.get("instanceId")
         meta["ec2-instance-type"] = instance_meta.get("instanceType")

metaflow/plugins/aws/step_functions/step_functions.py

@@ -439,7 +439,7 @@ class StepFunctions(object):
                             JSONItemReader()
                             .resource("arn:aws:states:::s3:getObject")
                             .parameter("Bucket.$", "$.Body.DestinationBucket")
-                            .parameter("Key.$", "$.Body.ResultFiles.SUCCEEDED.[0].Key")
+                            .parameter("Key.$", "$.Body.ResultFiles.SUCCEEDED[0].Key")
                         )
                         .output_path("$.[0]")
                     )

metaflow/plugins/env_escape/__init__.py

@@ -110,6 +110,7 @@ import importlib
 import os
 import sys
 from metaflow.plugins.env_escape.client_modules import ModuleImporter
+from metaflow.metaflow_config import ESCAPE_HATCH_WARNING
 
 # This is a trampoline file to ensure that the ModuleImporter to handle the emulated
 # modules gets properly loaded. If multiple modules are emulated by a single configuration
@@ -146,7 +147,7 @@ def load():
             # print("Env escape using executable {python_executable}")
         else:
             # Inverse logic as above here.
-            if sys.executable != "{python_executable}":
+            if sys.executable != "{python_executable}" and ESCAPE_HATCH_WARNING:
                 # We use the package locally and warn user.
                 print("Not using environment escape for '%s' as module present" % prefix)
             # In both cases, we don't load our loader since

metaflow/plugins/pypi/conda_environment.py

@@ -266,7 +266,9 @@ class CondaEnvironment(MetaflowEnvironment):
         # Resolve `linux-64` Conda environments if @batch or @kubernetes are in play
         target_platform = conda_platform()
         for decorator in step.decorators:
-            if decorator.name in ["batch", "kubernetes"]:
+            # TODO: rather than relying on decorator names, rely on attributes
+            #       to make them extensible.
+            if decorator.name in ["batch", "kubernetes", "nvidia"]:
                 # TODO: Support arm architectures
                 target_platform = "linux-64"
                 break

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.11.10"
+metaflow_version = "2.11.12"

{metaflow-2.11.10.dist-info → metaflow-2.11.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.11.10
+Version: 2.11.12
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs ==2.11.10 ; extra == 'stubs'
+Requires-Dist: metaflow-stubs ==2.11.12 ; extra == 'stubs'
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.11.10.dist-info → metaflow-2.11.12.dist-info}/RECORD

@@ -15,7 +15,7 @@ metaflow/graph.py,sha256=ZPxyG8uwVMk5YYgX4pQEQaPZtZM5Wy-G4NtJK73IEuA,11818
 metaflow/includefile.py,sha256=yHczcZ_U0SrasxSNhZb3DIBzx8UZnrJCl3FzvpEQLOA,19753
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=_kYAbAtsP7IG1Rd0FqNbo8I8Zs66_0WXbaZJFARO3dE,10394
-metaflow/metaflow_config.py,sha256=7nb97lLxtgynWZNMsmFx43EBy-ngJ4RWKFDlf2cnoQ4,19870
+metaflow/metaflow_config.py,sha256=X3YbYipCTA5ObumydxQXrgZZerOFlUngnTjHW9wOvtI,20043
 metaflow/metaflow_config_funcs.py,sha256=pCaiQ2ez9wXixJI3ehmf3QiW9lUqFrZnBZx1my_0wIg,4874
 metaflow/metaflow_current.py,sha256=sCENPBiji3LcPbwgOG0ukGd_yEc5tST8EowES8DzRtA,7430
 metaflow/metaflow_environment.py,sha256=JdsmQsYp1SDQniQ0-q1mKRrmzSFfYuzrf6jLEHmyaiM,7352
@@ -34,7 +34,7 @@ metaflow/task.py,sha256=ecGaULbK8kXPnyWzH1u6wtGclm0qeJm7K95amEL17sQ,25863
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=RrjsvADLKxSqjL76CxKh_J4OJl840B9Ak3V-vXleGas,13429
 metaflow/vendor.py,sha256=LZgXrh7ZSDmD32D1T5jj3OKKpXIqqxKzdMAOc5V0SD4,5162
-metaflow/version.py,sha256=tdq0nGUyYAyRYN8zn0X6UUGfq_r_tUaaLh61XPNHXyU,29
+metaflow/version.py,sha256=xc9RGVTlcPv4YdjEJQkhp1o3HJkzjUbLq8GEQhX6-3Q,29
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/click/__init__.py,sha256=FkyGDQ-cbiQxP_lxgUspyFYS48f2S_pTcfKPz-d_RMo,2463
 metaflow/_vendor/click/_bashcomplete.py,sha256=9J98IHQYmCAr2Jup6TDshUr5FJEen-AoQCZR0K5nKxQ,12309
@@ -155,7 +155,7 @@ metaflow/plugins/argo/argo_workflows_decorator.py,sha256=kCtwB6grJso5UwxKSirJn7L
 metaflow/plugins/argo/process_input_paths.py,sha256=4SiUoxbnTX4rCt0RSLcxG5jysbyd8oU-5JT0UOgy-vk,555
 metaflow/plugins/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/aws/aws_client.py,sha256=mO8UD6pxFaOnxDb3hTP3HB7Gqb_ZxoR-76LT683WHvI,4036
-metaflow/plugins/aws/aws_utils.py,sha256=pkkH8Cy9sF5tp3HoZ84wkN-84NmksgCdNN4cMSdsLaA,6455
+metaflow/plugins/aws/aws_utils.py,sha256=mSMBTUQ-CELhyPb6w3_Yq6_Hvd_6vbhAojYkrt2RNt8,6941
 metaflow/plugins/aws/batch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/aws/batch/batch.py,sha256=e9ssahWM18GnipPK2sqYB-ztx9w7Eoo7YtWyEtufYxs,17787
 metaflow/plugins/aws/batch/batch_cli.py,sha256=8j5s9RMZu0aJW76GY2lQkJT5tVDzamg9G_iu1AUpW8o,11632
@@ -169,7 +169,7 @@ metaflow/plugins/aws/step_functions/event_bridge_client.py,sha256=U9-tqKdih4KR-Z
 metaflow/plugins/aws/step_functions/production_token.py,sha256=_o4emv3rozYZoWpaj1Y6UfKhTMlYpQc7GDDDBfZ2G7s,1898
 metaflow/plugins/aws/step_functions/schedule_decorator.py,sha256=Ab1rW8O_no4HNZm4__iBmFDCDW0Z8-TgK4lnxHHA6HI,1940
 metaflow/plugins/aws/step_functions/set_batch_environment.py,sha256=ibiGWFHDjKcLfprH3OsX-g2M9lUsh6J-bp7v2cdLhD4,1294
-metaflow/plugins/aws/step_functions/step_functions.py,sha256=zR4QiPs6oWpbv3rWjVU22VHjwW-jrKoxvPTkA7jTnnY,51845
+metaflow/plugins/aws/step_functions/step_functions.py,sha256=vXhtq9BSY9zmyG1Z0bj38Sq8RU3Oftgi6CAfiEj21Ws,51844
 metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=KlH9jJL0VfsT0JqBhLwaWdYjaccU8UEArKAFnIJbSoU,24426
 metaflow/plugins/aws/step_functions/step_functions_client.py,sha256=DKpNwAIWElvWjFANs5Ku3rgzjxFoqAD6k-EF8Xhkg3Q,4754
 metaflow/plugins/aws/step_functions/step_functions_decorator.py,sha256=9hw_MX36RyFp6IowuAYaJzJg9UC5KCe1FNt1PcG7_J0,3791
@@ -217,7 +217,7 @@ metaflow/plugins/datatools/s3/s3.py,sha256=v2vkJxN3qYAHpu5na2V0GoXjItg8DGeQe71P3
 metaflow/plugins/datatools/s3/s3op.py,sha256=ZQFSxlaQUt-Ko_kIXMbHOKJc8q4FPXogS3xI6xsDR7Y,43390
 metaflow/plugins/datatools/s3/s3tail.py,sha256=boQjQGQMI-bvTqcMP2y7uSlSYLcvWOy7J3ZUaF78NAA,2597
 metaflow/plugins/datatools/s3/s3util.py,sha256=FgRgaVmEq7-i2dV7q8XK5w5PfFt-xJjZa8WrK8IJfdI,3769
-metaflow/plugins/env_escape/__init__.py,sha256=8NhVmk_OncuzVIT_TdwDcZeunfcjlI8iGfi6cgPO_xQ,8771
+metaflow/plugins/env_escape/__init__.py,sha256=Q2Sgnzi-x-BfGxIb8IC_dHjlEqw4zRnN3Yi-bCRsDc4,8854
 metaflow/plugins/env_escape/client.py,sha256=GsFZqjhGttd4eMU_CTw14sfNBV4vBKUUShp4SR0D8k8,25066
 metaflow/plugins/env_escape/client_modules.py,sha256=hQAcwz41wd6NnHezdI2Tv37ngO3CYuJ2WG8Wak6z59s,9295
 metaflow/plugins/env_escape/consts.py,sha256=jafRUdqZnkeKSgpdmRcTmnsVhEhVAjfQ6TKErbRH7wo,1000
@@ -257,7 +257,7 @@ metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j
 metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
 metaflow/plugins/pypi/bootstrap.py,sha256=nCe8FadqfIM19yj64m4JWdv_QEnQEp01bzQZrxzo5bs,5087
 metaflow/plugins/pypi/conda_decorator.py,sha256=-bPxNtZKjxqOo4sj89uIp8ZVrCIontWhAp7wwRFjYpg,14189
-metaflow/plugins/pypi/conda_environment.py,sha256=UnqWnc7vNPWJCWgnqKDsrIIHg99M1T0SZc6B73Jaqms,19102
+metaflow/plugins/pypi/conda_environment.py,sha256=wruUTU-xyFaEm331_7DcCkRuveAPS-2llZW4TZfGG4M,19236
 metaflow/plugins/pypi/micromamba.py,sha256=wlVN2fm4WXFh3jVNtpDfu4XEz6VJKbmFNp0QvqlMIuI,12179
 metaflow/plugins/pypi/pip.py,sha256=MAgdyP7wK7Cp6iusG6S-jeKKDCxlA9k-jMqIGvyi0Ng,12472
 metaflow/plugins/pypi/pypi_decorator.py,sha256=syWk_oSQhIK9Y7OeOINMG2XVyxh9sj5uJhapwAXRBDw,5583
@@ -298,9 +298,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-metaflow-2.11.10.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.11.10.dist-info/METADATA,sha256=jZ0uyd0DllaYU8fWXFEVX38oIsTJZg6NxSOz7ATZqyU,5908
-metaflow-2.11.10.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-metaflow-2.11.10.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.11.10.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.11.10.dist-info/RECORD,,
+metaflow-2.11.12.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.11.12.dist-info/METADATA,sha256=iWxUdtrUA_QE3eu7Grl1NiOlTjpY4g_5RJgr1hvHajQ,5908
+metaflow-2.11.12.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+metaflow-2.11.12.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.11.12.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.11.12.dist-info/RECORD,,