metaflow 2.11.8__py2.py3-none-any.whl → 2.11.10__py2.py3-none-any.whl

metaflow/decorators.py

@@ -518,7 +518,8 @@ def _init_flow_decorators(
         else:
             # Each "non-multiple" flow decorator is only allowed to have one set of options
             deco_flow_init_options = {
-                option: deco_options[option] for option in deco.options
+                option: deco_options[option.replace("-", "_")]
+                for option in deco.options
             }
         for deco in decorators:
             deco.flow_init(

metaflow/metaflow_config.py

@@ -312,6 +312,8 @@ KUBERNETES_CONTAINER_REGISTRY = from_conf(
 KUBERNETES_FETCH_EC2_METADATA = from_conf("KUBERNETES_FETCH_EC2_METADATA", False)
 # Shared memory in MB to use for this step
 KUBERNETES_SHARED_MEMORY = from_conf("KUBERNETES_SHARED_MEMORY", None)
+# Default port number to open on the pods
+KUBERNETES_PORT = from_conf("KUBERNETES_PORT", None)
 
 ARGO_WORKFLOWS_KUBERNETES_SECRETS = from_conf("ARGO_WORKFLOWS_KUBERNETES_SECRETS", "")
 ARGO_WORKFLOWS_ENV_VARS_TO_SKIP = from_conf("ARGO_WORKFLOWS_ENV_VARS_TO_SKIP", "")

metaflow/plugins/argo/argo_workflows.py

@@ -1371,12 +1371,14 @@ class ArgoWorkflows(object):
             # Set shared_memory to 0 if it isn't specified. This results
             # in Kubernetes using it's default value when the pod is created.
             shared_memory = resources.get("shared_memory", 0)
+            port = resources.get("port", None)
+            if port:
+                port = int(port)
 
             tmpfs_enabled = use_tmpfs or (tmpfs_size and not use_tmpfs)
 
             if tmpfs_enabled and tmpfs_tempdir:
                 env["METAFLOW_TEMPDIR"] = tmpfs_path
-
             # Create a ContainerTemplate for this node. Ideally, we would have
             # liked to inline this ContainerTemplate and avoid scanning the workflow
             # twice, but due to issues with variable substitution, we will have to
@@ -1435,6 +1437,9 @@ class ArgoWorkflows(object):
                         kubernetes_sdk.V1Container(
                             name=self._sanitize(node.name),
                             command=cmds,
+                            ports=[kubernetes_sdk.V1ContainerPort(container_port=port)]
+                            if port
+                            else None,
                             env=[
                                 kubernetes_sdk.V1EnvVar(name=k, value=str(v))
                                 for k, v in env.items()
@@ -1983,8 +1988,10 @@ class ArgoWorkflows(object):
                                                 # Technically, we don't need to create
                                                 # a payload carry-on and can stuff
                                                 # everything within the body.
-                                                data_template="{{ .Input.body.payload.%s | toJson }}"
-                                                % v,
+                                                # NOTE: We need the conditional logic in order to successfully fall back to the default value
+                                                # when the event payload does not contain a key for a parameter.
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (.Input.body.payload.%s | toJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                % (v, v),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for
                                                 # the parameters - there doesn't seem

metaflow/plugins/argo/argo_workflows_decorator.py

@@ -72,6 +72,7 @@ class ArgoWorkflowsInternalDecorator(StepDecorator):
         meta["argo-workflow-name"] = os.environ["ARGO_WORKFLOW_NAME"]
         meta["argo-workflow-namespace"] = os.environ["ARGO_WORKFLOW_NAMESPACE"]
         meta["auto-emit-argo-events"] = self.attributes["auto-emit-argo-events"]
+        meta["argo-workflow-template-owner"] = os.environ["METAFLOW_OWNER"]
         entries = [
             MetaDatum(
                 field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]

metaflow/plugins/aws/batch/batch_client.py

@@ -206,13 +206,13 @@ class BatchJob(object):
                 k, v = each_log_option.split(":", 1)
                 log_options_dict[k] = v
 
-        if log_driver is not None or log_options is not None:
+        if log_driver or log_options:
             job_definition["containerProperties"]["logConfiguration"] = {}
-        if log_driver is not None:
+        if log_driver:
             job_definition["containerProperties"]["logConfiguration"][
                 "logDriver"
             ] = log_driver
-        if log_options is not None:
+        if log_options:
             job_definition["containerProperties"]["logConfiguration"][
                 "options"
             ] = log_options_dict

metaflow/plugins/kubernetes/kubernetes.py

@@ -175,6 +175,7 @@ class Kubernetes(object):
         tolerations=None,
         labels=None,
         shared_memory=None,
+        port=None,
     ):
         if env is None:
             env = {}
@@ -215,6 +216,7 @@ class Kubernetes(object):
                 tmpfs_path=tmpfs_path,
                 persistent_volume_claims=persistent_volume_claims,
                 shared_memory=shared_memory,
+                port=port,
             )
             .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
             .environment_variable("METAFLOW_CODE_URL", code_package_url)

metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -108,6 +108,7 @@ def kubernetes():
     multiple=False,
 )
 @click.option("--shared-memory", default=None, help="Size of shared memory in MiB")
+@click.option("--port", default=None, help="Port number to expose from the container")
 @click.pass_context
 def step(
     ctx,
@@ -134,6 +135,7 @@ def step(
     persistent_volume_claims=None,
     tolerations=None,
     shared_memory=None,
+    port=None,
     **kwargs
 ):
     def echo(msg, stream="stderr", job_id=None, **kwargs):
@@ -248,6 +250,7 @@ def step(
                 persistent_volume_claims=persistent_volume_claims,
                 tolerations=tolerations,
                 shared_memory=shared_memory,
+                port=port,
             )
     except Exception as e:
         traceback.print_exc(chain=False)

metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -21,6 +21,7 @@ from metaflow.metaflow_config import (
     KUBERNETES_TOLERATIONS,
     KUBERNETES_SERVICE_ACCOUNT,
     KUBERNETES_SHARED_MEMORY,
+    KUBERNETES_PORT,
 )
 from metaflow.plugins.resources_decorator import ResourcesDecorator
 from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task
@@ -90,6 +91,8 @@ class KubernetesDecorator(StepDecorator):
         volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
     shared_memory: int, optional
         Shared memory size (in MiB) required for this step
+    port: int, optional
+        Port number to specify in the Kubernetes job object
     """
 
     name = "kubernetes"
@@ -113,6 +116,7 @@ class KubernetesDecorator(StepDecorator):
         "tmpfs_path": "/metaflow_temp",
         "persistent_volume_claims": None,  # e.g., {"pvc-name": "/mnt/vol", "another-pvc": "/mnt/vol2"}
         "shared_memory": None,
+        "port": None,
     }
     package_url = None
     package_sha = None
@@ -200,6 +204,8 @@ class KubernetesDecorator(StepDecorator):
                 self.attributes["tmpfs_size"] = int(self.attributes["memory"]) // 2
         if not self.attributes["shared_memory"]:
             self.attributes["shared_memory"] = KUBERNETES_SHARED_MEMORY
+        if not self.attributes["port"]:
+            self.attributes["port"] = KUBERNETES_PORT
 
     # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):

metaflow/plugins/kubernetes/kubernetes_job.py

@@ -118,6 +118,13 @@ class KubernetesJob(object):
                         containers=[
                             client.V1Container(
                                 command=self._kwargs["command"],
+                                ports=[
+                                    client.V1ContainerPort(
+                                        container_port=int(self._kwargs["port"])
+                                    )
+                                ]
+                                if "port" in self._kwargs and self._kwargs["port"]
+                                else None,
                                 env=[
                                     client.V1EnvVar(name=k, value=str(v))
                                     for k, v in self._kwargs.get(

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.11.8"
+metaflow_version = "2.11.10"

{metaflow-2.11.8.dist-info → metaflow-2.11.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: metaflow
-Version: 2.11.8
+Version: 2.11.10
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Metaflow Developers
 Author-email: help@metaflow.org
@@ -26,7 +26,7 @@ License-File: LICENSE
 Requires-Dist: requests
 Requires-Dist: boto3
 Provides-Extra: stubs
-Requires-Dist: metaflow-stubs ==2.11.8 ; extra == 'stubs'
+Requires-Dist: metaflow-stubs ==2.11.10 ; extra == 'stubs'
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{metaflow-2.11.8.dist-info → metaflow-2.11.10.dist-info}/RECORD

@@ -6,7 +6,7 @@ metaflow/cli_args.py,sha256=lcgBGNTvfaiPxiUnejAe60Upt9swG6lRy1_3OqbU6MY,2616
 metaflow/clone_util.py,sha256=ar4jSZt2aTd4monBpkIQmcLcsOd0relAB42qTUGt2j8,1810
 metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
 metaflow/debug.py,sha256=HEmt_16tJtqHXQXsqD9pqOFe3CWR5GZ7VwpaYQgnRdU,1466
-metaflow/decorators.py,sha256=EGL1_nkdxoYG5AZiOQ8sLGA1bprGK8ENwlSIOYQmLhs,21357
+metaflow/decorators.py,sha256=n91bvWA4JJWHwZnI02-rFGBZsWWMtaABYlWYcOvHuRw,21391
 metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
 metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
 metaflow/exception.py,sha256=KC1LHJQzzYkWib0DeQ4l_A2r8VaudywsSqIQuq1RDZU,4954
@@ -15,7 +15,7 @@ metaflow/graph.py,sha256=ZPxyG8uwVMk5YYgX4pQEQaPZtZM5Wy-G4NtJK73IEuA,11818
 metaflow/includefile.py,sha256=yHczcZ_U0SrasxSNhZb3DIBzx8UZnrJCl3FzvpEQLOA,19753
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=_kYAbAtsP7IG1Rd0FqNbo8I8Zs66_0WXbaZJFARO3dE,10394
-metaflow/metaflow_config.py,sha256=dosV_Rg8lfOjUb-vUEH38jw2c068iHrQMbR3Yls1pqc,19775
+metaflow/metaflow_config.py,sha256=7nb97lLxtgynWZNMsmFx43EBy-ngJ4RWKFDlf2cnoQ4,19870
 metaflow/metaflow_config_funcs.py,sha256=pCaiQ2ez9wXixJI3ehmf3QiW9lUqFrZnBZx1my_0wIg,4874
 metaflow/metaflow_current.py,sha256=sCENPBiji3LcPbwgOG0ukGd_yEc5tST8EowES8DzRtA,7430
 metaflow/metaflow_environment.py,sha256=JdsmQsYp1SDQniQ0-q1mKRrmzSFfYuzrf6jLEHmyaiM,7352
@@ -34,7 +34,7 @@ metaflow/task.py,sha256=ecGaULbK8kXPnyWzH1u6wtGclm0qeJm7K95amEL17sQ,25863
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=RrjsvADLKxSqjL76CxKh_J4OJl840B9Ak3V-vXleGas,13429
 metaflow/vendor.py,sha256=LZgXrh7ZSDmD32D1T5jj3OKKpXIqqxKzdMAOc5V0SD4,5162
-metaflow/version.py,sha256=7o_0YNBHyBm4MfovHAyRrE0W6DUzB0TYdptHlJlDxyU,28
+metaflow/version.py,sha256=tdq0nGUyYAyRYN8zn0X6UUGfq_r_tUaaLh61XPNHXyU,29
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/click/__init__.py,sha256=FkyGDQ-cbiQxP_lxgUspyFYS48f2S_pTcfKPz-d_RMo,2463
 metaflow/_vendor/click/_bashcomplete.py,sha256=9J98IHQYmCAr2Jup6TDshUr5FJEen-AoQCZR0K5nKxQ,12309
@@ -149,9 +149,9 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=JUKoGNoTCtrO9MNEneEC7ldRNwg
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=gKGmrcVwEktaqa7dOh9pWwAktGUQwwTACDT5-uUpJ4s,121723
+metaflow/plugins/argo/argo_workflows.py,sha256=ufub5eTXZS9idWbVl37PW2vmbYsNxXYpVTw3hByrzrE,122350
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=sZTpgfmc50eT3e0qIxpVqUgWhTcYlO1HM4gU6Oaya8g,33259
-metaflow/plugins/argo/argo_workflows_decorator.py,sha256=CfKVoHCOsCCQMghhPE30xw15gacwp3hR23HCo9ZZFVg,6580
+metaflow/plugins/argo/argo_workflows_decorator.py,sha256=kCtwB6grJso5UwxKSirJn7L9BWP5rd3arBunAyS-uuU,6656
 metaflow/plugins/argo/process_input_paths.py,sha256=4SiUoxbnTX4rCt0RSLcxG5jysbyd8oU-5JT0UOgy-vk,555
 metaflow/plugins/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/aws/aws_client.py,sha256=mO8UD6pxFaOnxDb3hTP3HB7Gqb_ZxoR-76LT683WHvI,4036
@@ -159,7 +159,7 @@ metaflow/plugins/aws/aws_utils.py,sha256=pkkH8Cy9sF5tp3HoZ84wkN-84NmksgCdNN4cMSd
 metaflow/plugins/aws/batch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/aws/batch/batch.py,sha256=e9ssahWM18GnipPK2sqYB-ztx9w7Eoo7YtWyEtufYxs,17787
 metaflow/plugins/aws/batch/batch_cli.py,sha256=8j5s9RMZu0aJW76GY2lQkJT5tVDzamg9G_iu1AUpW8o,11632
-metaflow/plugins/aws/batch/batch_client.py,sha256=VWprdpI2Gem9GTezqgmYU_6yU4C2ChrFVMxOsQgr5oY,28613
+metaflow/plugins/aws/batch/batch_client.py,sha256=s9ZHhxQPPoBQijLUgn6_16QOaD4-22U_44uJbp-yLkI,28565
 metaflow/plugins/aws/batch/batch_decorator.py,sha256=o2UH_E6k98ZpbWTvMooKv4V7nwpdHmh5SwCGzWubv0c,17313
 metaflow/plugins/aws/secrets_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.py,sha256=JtFUVu00Cg0FzAizgrPLXmrMqsT7YeQMkQlgeivUxcE,7986
@@ -246,11 +246,11 @@ metaflow/plugins/gcp/gs_tail.py,sha256=Jl_wvnzU7dub07A-DOAuP5FeccNIrPM-CeL1xKFs1
 metaflow/plugins/gcp/gs_utils.py,sha256=ZmIGFse1qYyvAVrwga23PQUzF6dXEDLLsZ2F-YRmvow,2030
 metaflow/plugins/gcp/includefile_support.py,sha256=vIDeR-MiJuUh-2S2pV7Z7FBkhIWwtHXaRrj76MWGRiY,3869
 metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/plugins/kubernetes/kubernetes.py,sha256=-t5qKt8HeDyy3Ui9eHfDuttdfsMMcyfcgc7tnC-ShCs,17444
-metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=tyRzzbqIAVaV_Oig_C94SPkGHXN0KprA5Tbc_b6Wr-w,9095
+metaflow/plugins/kubernetes/kubernetes.py,sha256=tbM9lfJcii1iAQ5C4bo8vUvwkBroNRVvgrytpcDQkpo,17490
+metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=cGY-BM5REcVKQCi4JSYSK6PvA9M5TK0ikG09Yfct0jA,9224
 metaflow/plugins/kubernetes/kubernetes_client.py,sha256=irATJpAob4jINkJw0zT_Xoa6JHRtYxx2IOeimlbzvPo,2373
-metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=pwv5eNw_jzCHjDOjO_IPWMvoX0z41g9Y2SWGmQn25wM,21208
-metaflow/plugins/kubernetes/kubernetes_job.py,sha256=j9bH48SKFvHcvYv02aUU9bwskh4rwH9YpK9BJkX1yJc,31880
+metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=StV8TnYtm4hpAGuyC6JQs_LUTHpa5lY-RtVBn9gC8E0,21429
+metaflow/plugins/kubernetes/kubernetes_job.py,sha256=qDsvaOcwcaOOaqLePg0HN19LaeJNBEuik9b8ErF9lhg,32259
 metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7ITX12O8,22428
 metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
@@ -298,9 +298,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-metaflow-2.11.8.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-metaflow-2.11.8.dist-info/METADATA,sha256=dIYhQaapvxdUqjxYGkt-XB6u7pL4885OraaGanBuCUA,5906
-metaflow-2.11.8.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-metaflow-2.11.8.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-metaflow-2.11.8.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-metaflow-2.11.8.dist-info/RECORD,,
+metaflow-2.11.10.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+metaflow-2.11.10.dist-info/METADATA,sha256=jZ0uyd0DllaYU8fWXFEVX38oIsTJZg6NxSOz7ATZqyU,5908
+metaflow-2.11.10.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+metaflow-2.11.10.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+metaflow-2.11.10.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+metaflow-2.11.10.dist-info/RECORD,,