ob-metaflow 2.11.9.1__py2.py3-none-any.whl → 2.11.10.2__py2.py3-none-any.whl

metaflow/decorators.py

@@ -518,7 +518,8 @@ def _init_flow_decorators(
         else:
             # Each "non-multiple" flow decorator is only allowed to have one set of options
             deco_flow_init_options = {
-                option: deco_options[option] for option in deco.options
+                option: deco_options[option.replace("-", "_")]
+                for option in deco.options
             }
         for deco in decorators:
             deco.flow_init(

metaflow/metaflow_config.py

@@ -324,6 +324,8 @@ KUBERNETES_FETCH_EC2_METADATA = from_conf("KUBERNETES_FETCH_EC2_METADATA", False
 KUBERNETES_PORT = from_conf("KUBERNETES_PORT", None)
 # Shared memory in MB to use for this step
 KUBERNETES_SHARED_MEMORY = from_conf("KUBERNETES_SHARED_MEMORY", None)
+# Default port number to open on the pods
+KUBERNETES_PORT = from_conf("KUBERNETES_PORT", None)
 
 
 ARGO_WORKFLOWS_KUBERNETES_SECRETS = from_conf("ARGO_WORKFLOWS_KUBERNETES_SECRETS", "")

metaflow/plugins/argo/argo_workflows.py

@@ -1386,12 +1386,14 @@ class ArgoWorkflows(object):
             # Set shared_memory to 0 if it isn't specified. This results
             # in Kubernetes using it's default value when the pod is created.
             shared_memory = resources.get("shared_memory", 0)
+            port = resources.get("port", None)
+            if port:
+                port = int(port)
 
             tmpfs_enabled = use_tmpfs or (tmpfs_size and not use_tmpfs)
 
             if tmpfs_enabled and tmpfs_tempdir:
                 env["METAFLOW_TEMPDIR"] = tmpfs_path
-
             # Create a ContainerTemplate for this node. Ideally, we would have
             # liked to inline this ContainerTemplate and avoid scanning the workflow
             # twice, but due to issues with variable substitution, we will have to
@@ -1450,6 +1452,9 @@ class ArgoWorkflows(object):
                         kubernetes_sdk.V1Container(
                             name=self._sanitize(node.name),
                             command=cmds,
+                            ports=[kubernetes_sdk.V1ContainerPort(container_port=port)]
+                            if port
+                            else None,
                             env=[
                                 kubernetes_sdk.V1EnvVar(name=k, value=str(v))
                                 for k, v in env.items()
@@ -1998,8 +2003,10 @@ class ArgoWorkflows(object):
                                                 # Technically, we don't need to create
                                                 # a payload carry-on and can stuff
                                                 # everything within the body.
-                                                data_template="{{ .Input.body.payload.%s | toJson }}"
-                                                % v,
+                                                # NOTE: We need the conditional logic in order to successfully fall back to the default value
+                                                # when the event payload does not contain a key for a parameter.
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (.Input.body.payload.%s | toJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                % (v, v),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for
                                                 # the parameters - there doesn't seem

metaflow/plugins/argo/argo_workflows_decorator.py

@@ -72,6 +72,7 @@ class ArgoWorkflowsInternalDecorator(StepDecorator):
         meta["argo-workflow-name"] = os.environ["ARGO_WORKFLOW_NAME"]
         meta["argo-workflow-namespace"] = os.environ["ARGO_WORKFLOW_NAMESPACE"]
         meta["auto-emit-argo-events"] = self.attributes["auto-emit-argo-events"]
+        meta["argo-workflow-template-owner"] = os.environ["METAFLOW_OWNER"]
         entries = [
             MetaDatum(
                 field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]

metaflow/plugins/kubernetes/kubernetes.py

@@ -180,8 +180,8 @@ class Kubernetes(object):
         annotations=None,
         num_parallel=0,
         attrs={},
-        port=None,
         shared_memory=None,
+        port=None,
     ):
         if env is None:
             env = {}
@@ -223,8 +223,8 @@ class Kubernetes(object):
                 persistent_volume_claims=persistent_volume_claims,
                 num_parallel=num_parallel,
                 attrs=attrs,
-                port=port,
                 shared_memory=shared_memory,
+                port=port,
             )
             .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
             .environment_variable("METAFLOW_CODE_URL", code_package_url)

metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -126,8 +126,8 @@ def kubernetes():
     type=int,
     help="Number of parallel nodes to run as a multi-node job.",
 )
-@click.option("--port", default=None, help="port number")
 @click.option("--shared-memory", default=None, help="Size of shared memory in MiB")
+@click.option("--port", default=None, help="Port number to expose from the container")
 @click.pass_context
 def step(
     ctx,
@@ -156,8 +156,8 @@ def step(
     labels=None,
     annotations=None,
     num_parallel=None,
-    port=None,
     shared_memory=None,
+    port=None,
     **kwargs
 ):
     def echo(msg, stream="stderr", job_id=None, **kwargs):
@@ -284,8 +284,8 @@ def step(
                 labels=labels,
                 annotations=annotations,
                 num_parallel=num_parallel,
-                port=port,
                 shared_memory=shared_memory,
+                port=port,
                 attrs=attrs,
             )
     except Exception as e:

metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -23,6 +23,7 @@ from metaflow.metaflow_config import (
     KUBERNETES_SERVICE_ACCOUNT,
     KUBERNETES_PORT,
     KUBERNETES_SHARED_MEMORY,
+    KUBERNETES_PORT,
 )
 from metaflow.plugins.resources_decorator import ResourcesDecorator
 from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task
@@ -93,10 +94,10 @@ class KubernetesDecorator(StepDecorator):
     persistent_volume_claims : Dict[str, str], optional, default None
         A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
         volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
-    port: int, optional
-        Number of the port to specify in the Kubernetes job object
     shared_memory: int, optional
-        Shared memory size (in MiB) required for this steps
+        Shared memory size (in MiB) required for this step
+    port: int, optional
+        Port number to specify in the Kubernetes job object
     """
 
     name = "kubernetes"
@@ -119,8 +120,8 @@ class KubernetesDecorator(StepDecorator):
         "tmpfs_size": None,
         "tmpfs_path": "/metaflow_temp",
         "persistent_volume_claims": None,  # e.g., {"pvc-name": "/mnt/vol", "another-pvc": "/mnt/vol2"}
-        "port": None,
         "shared_memory": None,
+        "port": None,
     }
     package_url = None
     package_sha = None
@@ -206,10 +207,10 @@ class KubernetesDecorator(StepDecorator):
             if not self.attributes["tmpfs_size"]:
                 # default tmpfs behavior - https://man7.org/linux/man-pages/man5/tmpfs.5.html
                 self.attributes["tmpfs_size"] = int(self.attributes["memory"]) // 2
-        if not self.attributes["port"]:
-            self.attributes["port"] = KUBERNETES_PORT
         if not self.attributes["shared_memory"]:
             self.attributes["shared_memory"] = KUBERNETES_SHARED_MEMORY
+        if not self.attributes["port"]:
+            self.attributes["port"] = KUBERNETES_PORT
 
     # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -287,7 +288,7 @@ class KubernetesDecorator(StepDecorator):
         if "memory" in self.attributes and MAX_MEMORY_PER_TASK:
             if float(self.attributes["memory"]) > float(MAX_MEMORY_PER_TASK):
                 raise MetaflowException(
-                    "Step %s requires %s CPU units, but you cannot use more than %s per step in this environment"
+                    "Step %s requires %s memory, but you cannot use more than %s per step in this environment"
                     % (step, self.attributes["memory"], MAX_MEMORY_PER_TASK)
                 )
 

metaflow/plugins/kubernetes/kubernetes_job.py

@@ -423,6 +423,12 @@ class KubernetesJob(object):
                 main_commands[-1] = "/usr/sbin/sshd -D & %s" % main_commands[-1]
                 secondary_commands[-1] = "/usr/sbin/sshd -D & %s" % secondary_commands[-1]
 
+            replicated_jobs = [_get_replicated_job("control", 1, main_commands)]
+            if self._kwargs["num_parallel"] > 1:
+                replicated_jobs.append(
+                    _get_replicated_job("worker", self._kwargs["num_parallel"] - 1, secondary_commands)
+                )
+
             self._jobset = jobset.models.jobset_v1alpha2_job_set.JobsetV1alpha2JobSet(
                 api_version="jobset.x-k8s.io/v1alpha2",
                 kind="JobSet",
@@ -437,14 +443,7 @@ class KubernetesJob(object):
                         enable_dns_hostnames=True if not self._kwargs['attrs']['requires_passwordless_ssh'] else False,
                         subdomain=subdomain
                     ),
-                    replicated_jobs=[
-                        _get_replicated_job("control", 1, main_commands),
-                        _get_replicated_job(
-                            "worker",
-                            self._kwargs["num_parallel"] - 1,
-                            secondary_commands,
-                        ),
-                    ],
+                    replicated_jobs=replicated_jobs
                 ),
             )
             self._passwordless_ssh_service = _get_passwordless_ssh_service()
@@ -484,6 +483,13 @@ class KubernetesJob(object):
                             containers=[
                                 client.V1Container(
                                     command=self._kwargs["command"],
+                                    ports=[
+                                        client.V1ContainerPort(
+                                            container_port=int(self._kwargs["port"])
+                                        )
+                                    ]
+                                    if "port" in self._kwargs and self._kwargs["port"]
+                                    else None,
                                     env=[
                                         client.V1EnvVar(name=k, value=str(v))
                                         for k, v in self._kwargs.get(

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.11.9.1"
+metaflow_version = "2.11.10.2"

{ob_metaflow-2.11.9.1.dist-info → ob_metaflow-2.11.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.11.9.1
+Version: 2.11.10.2
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: ob-metaflow-stubs ==2.11.9.1 ; extra == 'stubs'
+Requires-Dist: ob-metaflow-stubs ==2.11.10.2 ; extra == 'stubs'
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob_metaflow-2.11.9.1.dist-info → ob_metaflow-2.11.10.2.dist-info}/RECORD

@@ -6,7 +6,7 @@ metaflow/cli_args.py,sha256=lcgBGNTvfaiPxiUnejAe60Upt9swG6lRy1_3OqbU6MY,2616
 metaflow/clone_util.py,sha256=ar4jSZt2aTd4monBpkIQmcLcsOd0relAB42qTUGt2j8,1810
 metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
 metaflow/debug.py,sha256=HEmt_16tJtqHXQXsqD9pqOFe3CWR5GZ7VwpaYQgnRdU,1466
-metaflow/decorators.py,sha256=EGL1_nkdxoYG5AZiOQ8sLGA1bprGK8ENwlSIOYQmLhs,21357
+metaflow/decorators.py,sha256=n91bvWA4JJWHwZnI02-rFGBZsWWMtaABYlWYcOvHuRw,21391
 metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
 metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
 metaflow/exception.py,sha256=KC1LHJQzzYkWib0DeQ4l_A2r8VaudywsSqIQuq1RDZU,4954
@@ -15,7 +15,7 @@ metaflow/graph.py,sha256=ZPxyG8uwVMk5YYgX4pQEQaPZtZM5Wy-G4NtJK73IEuA,11818
 metaflow/includefile.py,sha256=yHczcZ_U0SrasxSNhZb3DIBzx8UZnrJCl3FzvpEQLOA,19753
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=_kYAbAtsP7IG1Rd0FqNbo8I8Zs66_0WXbaZJFARO3dE,10394
-metaflow/metaflow_config.py,sha256=WmO8-gQeLlbUJb88REHuR77nll8FF_yQ68pS9bT9Pfs,20624
+metaflow/metaflow_config.py,sha256=mcbOTNN6sFX64oAxGU8i-sgv2Eb9xaK2C31zfWUCzYw,20719
 metaflow/metaflow_config_funcs.py,sha256=pCaiQ2ez9wXixJI3ehmf3QiW9lUqFrZnBZx1my_0wIg,4874
 metaflow/metaflow_current.py,sha256=sCENPBiji3LcPbwgOG0ukGd_yEc5tST8EowES8DzRtA,7430
 metaflow/metaflow_environment.py,sha256=XiMmBZiq3_dwaw0Oi3B8588BahYxzgfqWGMePPZqUUc,7359
@@ -34,7 +34,7 @@ metaflow/task.py,sha256=ecGaULbK8kXPnyWzH1u6wtGclm0qeJm7K95amEL17sQ,25863
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=RrjsvADLKxSqjL76CxKh_J4OJl840B9Ak3V-vXleGas,13429
 metaflow/vendor.py,sha256=LZgXrh7ZSDmD32D1T5jj3OKKpXIqqxKzdMAOc5V0SD4,5162
-metaflow/version.py,sha256=yKncWWFL4OF_XHTiyjxLWbQHk0S4UO7_4qNT0J936-s,30
+metaflow/version.py,sha256=0OX3kt6OXRJDfZgJNAG22iVhu6nDCYv5HFtwAFqZOd8,31
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/click/__init__.py,sha256=FkyGDQ-cbiQxP_lxgUspyFYS48f2S_pTcfKPz-d_RMo,2463
 metaflow/_vendor/click/_bashcomplete.py,sha256=9J98IHQYmCAr2Jup6TDshUr5FJEen-AoQCZR0K5nKxQ,12309
@@ -149,9 +149,9 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=JUKoGNoTCtrO9MNEneEC7ldRNwg
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
 metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
-metaflow/plugins/argo/argo_workflows.py,sha256=lwfCFmgnQiAO5VscqkRWjVtOvcwoZ2gYX0zo3DxtGig,122919
+metaflow/plugins/argo/argo_workflows.py,sha256=uEjOTAOr6y8tExuDjQuSIHZFS_2HsW-tTdRJtTcO4cc,123546
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=sZTpgfmc50eT3e0qIxpVqUgWhTcYlO1HM4gU6Oaya8g,33259
-metaflow/plugins/argo/argo_workflows_decorator.py,sha256=CfKVoHCOsCCQMghhPE30xw15gacwp3hR23HCo9ZZFVg,6580
+metaflow/plugins/argo/argo_workflows_decorator.py,sha256=kCtwB6grJso5UwxKSirJn7L9BWP5rd3arBunAyS-uuU,6656
 metaflow/plugins/argo/process_input_paths.py,sha256=LjUSP8PVU-DRGEPxjas99nzyAO-fI82Bxxbr_QETE88,565
 metaflow/plugins/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/aws/aws_client.py,sha256=mO8UD6pxFaOnxDb3hTP3HB7Gqb_ZxoR-76LT683WHvI,4036
@@ -248,11 +248,11 @@ metaflow/plugins/gcp/gs_tail.py,sha256=Jl_wvnzU7dub07A-DOAuP5FeccNIrPM-CeL1xKFs1
 metaflow/plugins/gcp/gs_utils.py,sha256=ZmIGFse1qYyvAVrwga23PQUzF6dXEDLLsZ2F-YRmvow,2030
 metaflow/plugins/gcp/includefile_support.py,sha256=vIDeR-MiJuUh-2S2pV7Z7FBkhIWwtHXaRrj76MWGRiY,3869
 metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/plugins/kubernetes/kubernetes.py,sha256=Mbah7ALEpExO3NM_Tnrme8DvSLVUpb5tKPAChD84LwQ,19233
-metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=Erx2fKijcpDuotdugak7PzWcp1ZmO-Pr-mP6B7LHV7Q,10245
+metaflow/plugins/kubernetes/kubernetes.py,sha256=5wM8_gRuyyyIv9mbX2lCePsLHoiDgJnbzYwnJy_NEoE,19233
+metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=wSByGQEaoQo1aV9kJoKYmbVVeFQVsMw9RfG4Bw2sMm8,10274
 metaflow/plugins/kubernetes/kubernetes_client.py,sha256=irATJpAob4jINkJw0zT_Xoa6JHRtYxx2IOeimlbzvPo,2373
-metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=obJicDXD5tZP_GO0SpoaTKqjcnR4uSN2q_r6W_vvqWA,24994
-metaflow/plugins/kubernetes/kubernetes_job.py,sha256=X9wgxQsvKlJ_yLIg0jHnIXUELhnNmGRq_8XnDvyn3bk,52259
+metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=3zzCETYE5DtCzcuxQxh60ELTWD6Z4UIJyBqOBSh4rl4,25004
+metaflow/plugins/kubernetes/kubernetes_job.py,sha256=OUAol0FxrXw_g7QSn85f6SHECmtfEGDFLNbEqh4lL4Q,52656
 metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7ITX12O8,22428
 metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
@@ -300,9 +300,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-ob_metaflow-2.11.9.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-ob_metaflow-2.11.9.1.dist-info/METADATA,sha256=64Tiuqy8ZT3w-c-taFJlO9-WwgcjT2LxLfX3t-LBfAI,5146
-ob_metaflow-2.11.9.1.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
-ob_metaflow-2.11.9.1.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-ob_metaflow-2.11.9.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-ob_metaflow-2.11.9.1.dist-info/RECORD,,
+ob_metaflow-2.11.10.2.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+ob_metaflow-2.11.10.2.dist-info/METADATA,sha256=se5ZQsb_11McpFE5wdLOh9KeraVfAIjnYcpav3oLt5E,5148
+ob_metaflow-2.11.10.2.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
+ob_metaflow-2.11.10.2.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+ob_metaflow-2.11.10.2.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+ob_metaflow-2.11.10.2.dist-info/RECORD,,