metaflow 2.11.9__tar.gz → 2.11.11__tar.gz

{metaflow-2.11.9/metaflow.egg-info → metaflow-2.11.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.11.9
+Version: 2.11.11
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.11.9; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.11.11; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/decorators.py

@@ -518,7 +518,8 @@ def _init_flow_decorators(
         else:
             # Each "non-multiple" flow decorator is only allowed to have one set of options
             deco_flow_init_options = {
-                option: deco_options[option] for option in deco.options
+                option: deco_options[option.replace("-", "_")]
+                for option in deco.options
             }
         for deco in decorators:
             deco.flow_init(

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/metaflow_config.py

@@ -312,6 +312,8 @@ KUBERNETES_CONTAINER_REGISTRY = from_conf(
 KUBERNETES_FETCH_EC2_METADATA = from_conf("KUBERNETES_FETCH_EC2_METADATA", False)
 # Shared memory in MB to use for this step
 KUBERNETES_SHARED_MEMORY = from_conf("KUBERNETES_SHARED_MEMORY", None)
+# Default port number to open on the pods
+KUBERNETES_PORT = from_conf("KUBERNETES_PORT", None)
 
 ARGO_WORKFLOWS_KUBERNETES_SECRETS = from_conf("ARGO_WORKFLOWS_KUBERNETES_SECRETS", "")
 ARGO_WORKFLOWS_ENV_VARS_TO_SKIP = from_conf("ARGO_WORKFLOWS_ENV_VARS_TO_SKIP", "")
@@ -361,6 +363,12 @@ CONDA_PACKAGE_GSROOT = from_conf("CONDA_PACKAGE_GSROOT")
 # should result in an appreciable speedup in flow environment initialization.
 CONDA_DEPENDENCY_RESOLVER = from_conf("CONDA_DEPENDENCY_RESOLVER", "conda")
 
+###
+# Escape hatch configuration
+###
+# Print out warning if escape hatch is not used for the target packages
+ESCAPE_HATCH_WARNING = from_conf("ESCAPE_HATCH_WARNING", True)
+
 ###
 # Debug configuration
 ###

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/plugins/argo/argo_workflows.py

@@ -1371,12 +1371,14 @@ class ArgoWorkflows(object):
             # Set shared_memory to 0 if it isn't specified. This results
             # in Kubernetes using it's default value when the pod is created.
             shared_memory = resources.get("shared_memory", 0)
+            port = resources.get("port", None)
+            if port:
+                port = int(port)
 
             tmpfs_enabled = use_tmpfs or (tmpfs_size and not use_tmpfs)
 
             if tmpfs_enabled and tmpfs_tempdir:
                 env["METAFLOW_TEMPDIR"] = tmpfs_path
-
             # Create a ContainerTemplate for this node. Ideally, we would have
             # liked to inline this ContainerTemplate and avoid scanning the workflow
             # twice, but due to issues with variable substitution, we will have to
@@ -1435,6 +1437,9 @@ class ArgoWorkflows(object):
                         kubernetes_sdk.V1Container(
                             name=self._sanitize(node.name),
                             command=cmds,
+                            ports=[kubernetes_sdk.V1ContainerPort(container_port=port)]
+                            if port
+                            else None,
                             env=[
                                 kubernetes_sdk.V1EnvVar(name=k, value=str(v))
                                 for k, v in env.items()
@@ -1983,8 +1988,10 @@ class ArgoWorkflows(object):
                                                 # Technically, we don't need to create
                                                 # a payload carry-on and can stuff
                                                 # everything within the body.
-                                                data_template="{{ .Input.body.payload.%s | toJson }}"
-                                                % v,
+                                                # NOTE: We need the conditional logic in order to successfully fall back to the default value
+                                                # when the event payload does not contain a key for a parameter.
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (.Input.body.payload.%s | toJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                % (v, v),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for
                                                 # the parameters - there doesn't seem

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/plugins/argo/argo_workflows_decorator.py

@@ -72,6 +72,7 @@ class ArgoWorkflowsInternalDecorator(StepDecorator):
         meta["argo-workflow-name"] = os.environ["ARGO_WORKFLOW_NAME"]
         meta["argo-workflow-namespace"] = os.environ["ARGO_WORKFLOW_NAMESPACE"]
         meta["auto-emit-argo-events"] = self.attributes["auto-emit-argo-events"]
+        meta["argo-workflow-template-owner"] = os.environ["METAFLOW_OWNER"]
         entries = [
             MetaDatum(
                 field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/plugins/aws/aws_utils.py

@@ -19,14 +19,30 @@ def get_ec2_instance_metadata():
     # access to this end-point might be blocked on AWS and not available
     # for non-AWS deployments.
     # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html
+    # Set a very aggressive timeout, as the communication is happening in the same subnet,
+    # there should not be any significant delay in the response.
+    # Having a long default timeout here introduces unnecessary delay in launching tasks when the
+    # instance is unreachable.
+    timeout = (1, 10)
+    token = None
     try:
-        # Set a very aggressive timeout, as the communication is happening in the same subnet,
-        # there should not be any significant delay in the response.
-        # Having a long default timeout here introduces unnecessary delay in launching tasks when the
-        # instance is unreachable.
+        # Try to get an IMDSv2 token.
+        token = requests.put(
+            url="http://169.254.169.254/latest/api/token",
+            headers={"X-aws-ec2-metadata-token-ttl-seconds": 100},
+            timeout=timeout,
+        ).text
+    except:
+        pass
+    try:
+        headers = {}
+        # Add IMDSv2 token if available, else fall back to IMDSv1.
+        if token:
+            headers["X-aws-ec2-metadata-token"] = token
         instance_meta = requests.get(
             url="http://169.254.169.254/latest/dynamic/instance-identity/document",
-            timeout=(1, 10),
+            headers=headers,
+            timeout=timeout,
         ).json()
         meta["ec2-instance-id"] = instance_meta.get("instanceId")
         meta["ec2-instance-type"] = instance_meta.get("instanceType")

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/plugins/env_escape/__init__.py

@@ -36,6 +36,7 @@ from metaflow.extension_support import get_modules
 
 from .exception_transferer import RemoteInterpreterException
 from .client_modules import create_modules
+from metaflow.metaflow_config import ESCAPE_HATCH_WARNING
 
 # Determine what is the python executable to use for the environment escape. To do this,
 # we look for ENV_ESCAPE_PY in the environment AND store it. When metaflow
@@ -146,7 +147,7 @@ def load():
             # print("Env escape using executable {python_executable}")
         else:
             # Inverse logic as above here.
-            if sys.executable != "{python_executable}":
+            if sys.executable != "{python_executable}" and ESCAPE_HATCH_WARNING:
                 # We use the package locally and warn user.
                 print("Not using environment escape for '%s' as module present" % prefix)
             # In both cases, we don't load our loader since

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/plugins/kubernetes/kubernetes.py

@@ -175,6 +175,7 @@ class Kubernetes(object):
         tolerations=None,
         labels=None,
         shared_memory=None,
+        port=None,
     ):
         if env is None:
             env = {}
@@ -215,6 +216,7 @@ class Kubernetes(object):
                 tmpfs_path=tmpfs_path,
                 persistent_volume_claims=persistent_volume_claims,
                 shared_memory=shared_memory,
+                port=port,
             )
             .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
             .environment_variable("METAFLOW_CODE_URL", code_package_url)

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -108,6 +108,7 @@ def kubernetes():
     multiple=False,
 )
 @click.option("--shared-memory", default=None, help="Size of shared memory in MiB")
+@click.option("--port", default=None, help="Port number to expose from the container")
 @click.pass_context
 def step(
     ctx,
@@ -134,6 +135,7 @@ def step(
     persistent_volume_claims=None,
     tolerations=None,
     shared_memory=None,
+    port=None,
     **kwargs
 ):
     def echo(msg, stream="stderr", job_id=None, **kwargs):
@@ -248,6 +250,7 @@ def step(
                 persistent_volume_claims=persistent_volume_claims,
                 tolerations=tolerations,
                 shared_memory=shared_memory,
+                port=port,
             )
     except Exception as e:
         traceback.print_exc(chain=False)

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -21,6 +21,7 @@ from metaflow.metaflow_config import (
     KUBERNETES_TOLERATIONS,
     KUBERNETES_SERVICE_ACCOUNT,
     KUBERNETES_SHARED_MEMORY,
+    KUBERNETES_PORT,
 )
 from metaflow.plugins.resources_decorator import ResourcesDecorator
 from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task
@@ -90,6 +91,8 @@ class KubernetesDecorator(StepDecorator):
         volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
     shared_memory: int, optional
         Shared memory size (in MiB) required for this step
+    port: int, optional
+        Port number to specify in the Kubernetes job object
     """
 
     name = "kubernetes"
@@ -113,6 +116,7 @@ class KubernetesDecorator(StepDecorator):
         "tmpfs_path": "/metaflow_temp",
         "persistent_volume_claims": None,  # e.g., {"pvc-name": "/mnt/vol", "another-pvc": "/mnt/vol2"}
         "shared_memory": None,
+        "port": None,
     }
     package_url = None
     package_sha = None
@@ -200,6 +204,8 @@ class KubernetesDecorator(StepDecorator):
                 self.attributes["tmpfs_size"] = int(self.attributes["memory"]) // 2
         if not self.attributes["shared_memory"]:
             self.attributes["shared_memory"] = KUBERNETES_SHARED_MEMORY
+        if not self.attributes["port"]:
+            self.attributes["port"] = KUBERNETES_PORT
 
     # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/plugins/kubernetes/kubernetes_job.py

@@ -118,6 +118,13 @@ class KubernetesJob(object):
                         containers=[
                             client.V1Container(
                                 command=self._kwargs["command"],
+                                ports=[
+                                    client.V1ContainerPort(
+                                        container_port=int(self._kwargs["port"])
+                                    )
+                                ]
+                                if "port" in self._kwargs and self._kwargs["port"]
+                                else None,
                                 env=[
                                     client.V1EnvVar(name=k, value=str(v))
                                     for k, v in self._kwargs.get(

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow/plugins/pypi/conda_environment.py

@@ -266,7 +266,9 @@ class CondaEnvironment(MetaflowEnvironment):
         # Resolve `linux-64` Conda environments if @batch or @kubernetes are in play
         target_platform = conda_platform()
         for decorator in step.decorators:
-            if decorator.name in ["batch", "kubernetes"]:
+            # TODO: rather than relying on decorator names, rely on attributes
+            #       to make them extensible.
+            if decorator.name in ["batch", "kubernetes", "nvidia"]:
                 # TODO: Support arm architectures
                 target_platform = "linux-64"
                 break

metaflow-2.11.11/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.11.11"

{metaflow-2.11.9 → metaflow-2.11.11/metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.11.9
+Version: 2.11.11
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs==2.11.9; extra == "stubs"
+Requires-Dist: metaflow-stubs==2.11.11; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.11.9 → metaflow-2.11.11}/metaflow.egg-info/requires.txt

@@ -2,4 +2,4 @@ requests
 boto3
 
 [stubs]
-metaflow-stubs==2.11.9
+metaflow-stubs==2.11.11

metaflow-2.11.9/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.11.9"