ob-metaflow 2.11.8.1__tar.gz → 2.11.10.1__tar.gz

{ob-metaflow-2.11.8.1/ob_metaflow.egg-info → ob-metaflow-2.11.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.11.8.1
+Version: 2.11.10.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: ob-metaflow-stubs==2.11.8.1; extra == "stubs"
+Requires-Dist: ob-metaflow-stubs==2.11.10.1; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/metaflow/decorators.py

@@ -518,7 +518,8 @@ def _init_flow_decorators(
         else:
             # Each "non-multiple" flow decorator is only allowed to have one set of options
             deco_flow_init_options = {
-                option: deco_options[option] for option in deco.options
+                option: deco_options[option.replace("-", "_")]
+                for option in deco.options
             }
         for deco in decorators:
             deco.flow_init(

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/metaflow/metaflow_config.py

@@ -324,6 +324,8 @@ KUBERNETES_FETCH_EC2_METADATA = from_conf("KUBERNETES_FETCH_EC2_METADATA", False
 KUBERNETES_PORT = from_conf("KUBERNETES_PORT", None)
 # Shared memory in MB to use for this step
 KUBERNETES_SHARED_MEMORY = from_conf("KUBERNETES_SHARED_MEMORY", None)
+# Default port number to open on the pods
+KUBERNETES_PORT = from_conf("KUBERNETES_PORT", None)
 
 
 ARGO_WORKFLOWS_KUBERNETES_SECRETS = from_conf("ARGO_WORKFLOWS_KUBERNETES_SECRETS", "")

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/metaflow/plugins/argo/argo_workflows.py

@@ -1386,12 +1386,14 @@ class ArgoWorkflows(object):
             # Set shared_memory to 0 if it isn't specified. This results
             # in Kubernetes using it's default value when the pod is created.
             shared_memory = resources.get("shared_memory", 0)
+            port = resources.get("port", None)
+            if port:
+                port = int(port)
 
             tmpfs_enabled = use_tmpfs or (tmpfs_size and not use_tmpfs)
 
             if tmpfs_enabled and tmpfs_tempdir:
                 env["METAFLOW_TEMPDIR"] = tmpfs_path
-
             # Create a ContainerTemplate for this node. Ideally, we would have
             # liked to inline this ContainerTemplate and avoid scanning the workflow
             # twice, but due to issues with variable substitution, we will have to
@@ -1450,6 +1452,9 @@ class ArgoWorkflows(object):
                         kubernetes_sdk.V1Container(
                             name=self._sanitize(node.name),
                             command=cmds,
+                            ports=[kubernetes_sdk.V1ContainerPort(container_port=port)]
+                            if port
+                            else None,
                             env=[
                                 kubernetes_sdk.V1EnvVar(name=k, value=str(v))
                                 for k, v in env.items()
@@ -1998,8 +2003,10 @@ class ArgoWorkflows(object):
                                                 # Technically, we don't need to create
                                                 # a payload carry-on and can stuff
                                                 # everything within the body.
-                                                data_template="{{ .Input.body.payload.%s | toJson }}"
-                                                % v,
+                                                # NOTE: We need the conditional logic in order to successfully fall back to the default value
+                                                # when the event payload does not contain a key for a parameter.
+                                                data_template='{{ if (hasKey $.Input.body.payload "%s") }}{{- (.Input.body.payload.%s | toJson) -}}{{- else -}}{{ (fail "use-default-instead") }}{{- end -}}'
+                                                % (v, v),
                                                 # Unfortunately the sensor needs to
                                                 # record the default values for
                                                 # the parameters - there doesn't seem

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/metaflow/plugins/argo/argo_workflows_decorator.py

@@ -72,6 +72,7 @@ class ArgoWorkflowsInternalDecorator(StepDecorator):
         meta["argo-workflow-name"] = os.environ["ARGO_WORKFLOW_NAME"]
         meta["argo-workflow-namespace"] = os.environ["ARGO_WORKFLOW_NAMESPACE"]
         meta["auto-emit-argo-events"] = self.attributes["auto-emit-argo-events"]
+        meta["argo-workflow-template-owner"] = os.environ["METAFLOW_OWNER"]
         entries = [
             MetaDatum(
                 field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/metaflow/plugins/aws/batch/batch_client.py

@@ -206,13 +206,13 @@ class BatchJob(object):
                 k, v = each_log_option.split(":", 1)
                 log_options_dict[k] = v
 
-        if log_driver is not None or log_options is not None:
+        if log_driver or log_options:
             job_definition["containerProperties"]["logConfiguration"] = {}
-        if log_driver is not None:
+        if log_driver:
             job_definition["containerProperties"]["logConfiguration"][
                 "logDriver"
             ] = log_driver
-        if log_options is not None:
+        if log_options:
             job_definition["containerProperties"]["logConfiguration"][
                 "options"
             ] = log_options_dict

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/metaflow/plugins/kubernetes/kubernetes.py

@@ -182,6 +182,7 @@ class Kubernetes(object):
         attrs={},
         port=None,
         shared_memory=None,
+        port=None,
     ):
         if env is None:
             env = {}
@@ -225,6 +226,7 @@ class Kubernetes(object):
                 attrs=attrs,
                 port=port,
                 shared_memory=shared_memory,
+                port=port,
             )
             .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
             .environment_variable("METAFLOW_CODE_URL", code_package_url)

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -126,8 +126,8 @@ def kubernetes():
     type=int,
     help="Number of parallel nodes to run as a multi-node job.",
 )
-@click.option("--port", default=None, help="port number")
 @click.option("--shared-memory", default=None, help="Size of shared memory in MiB")
+@click.option("--port", default=None, help="Port number to expose from the container")
 @click.pass_context
 def step(
     ctx,
@@ -156,8 +156,8 @@ def step(
     labels=None,
     annotations=None,
     num_parallel=None,
-    port=None,
     shared_memory=None,
+    port=None,
     **kwargs
 ):
     def echo(msg, stream="stderr", job_id=None, **kwargs):
@@ -284,8 +284,8 @@ def step(
                 labels=labels,
                 annotations=annotations,
                 num_parallel=num_parallel,
-                port=port,
                 shared_memory=shared_memory,
+                port=port,
                 attrs=attrs,
             )
     except Exception as e:

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -23,6 +23,7 @@ from metaflow.metaflow_config import (
     KUBERNETES_SERVICE_ACCOUNT,
     KUBERNETES_PORT,
     KUBERNETES_SHARED_MEMORY,
+    KUBERNETES_PORT,
 )
 from metaflow.plugins.resources_decorator import ResourcesDecorator
 from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task
@@ -93,10 +94,10 @@ class KubernetesDecorator(StepDecorator):
     persistent_volume_claims : Dict[str, str], optional, default None
         A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
         volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
-    port: int, optional
-        Number of the port to specify in the Kubernetes job object
     shared_memory: int, optional
-        Shared memory size (in MiB) required for this steps
+        Shared memory size (in MiB) required for this step
+    port: int, optional
+        Port number to specify in the Kubernetes job object
     """
 
     name = "kubernetes"
@@ -119,8 +120,8 @@ class KubernetesDecorator(StepDecorator):
         "tmpfs_size": None,
         "tmpfs_path": "/metaflow_temp",
         "persistent_volume_claims": None,  # e.g., {"pvc-name": "/mnt/vol", "another-pvc": "/mnt/vol2"}
-        "port": None,
         "shared_memory": None,
+        "port": None,
     }
     package_url = None
     package_sha = None
@@ -206,10 +207,10 @@ class KubernetesDecorator(StepDecorator):
             if not self.attributes["tmpfs_size"]:
                 # default tmpfs behavior - https://man7.org/linux/man-pages/man5/tmpfs.5.html
                 self.attributes["tmpfs_size"] = int(self.attributes["memory"]) // 2
-        if not self.attributes["port"]:
-            self.attributes["port"] = KUBERNETES_PORT
         if not self.attributes["shared_memory"]:
             self.attributes["shared_memory"] = KUBERNETES_SHARED_MEMORY
+        if not self.attributes["port"]:
+            self.attributes["port"] = KUBERNETES_PORT
 
     # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -287,7 +288,7 @@ class KubernetesDecorator(StepDecorator):
         if "memory" in self.attributes and MAX_MEMORY_PER_TASK:
             if float(self.attributes["memory"]) > float(MAX_MEMORY_PER_TASK):
                 raise MetaflowException(
-                    "Step %s requires %s CPU units, but you cannot use more than %s per step in this environment"
+                    "Step %s requires %s memory, but you cannot use more than %s per step in this environment"
                     % (step, self.attributes["memory"], MAX_MEMORY_PER_TASK)
                 )
 

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/metaflow/plugins/kubernetes/kubernetes_job.py

@@ -423,6 +423,12 @@ class KubernetesJob(object):
                 main_commands[-1] = "/usr/sbin/sshd -D & %s" % main_commands[-1]
                 secondary_commands[-1] = "/usr/sbin/sshd -D & %s" % secondary_commands[-1]
 
+            replicated_jobs = [_get_replicated_job("control", 1, main_commands)]
+            if self._kwargs["num_parallel"] > 1:
+                replicated_jobs.append(
+                    _get_replicated_job("worker", self._kwargs["num_parallel"] - 1, secondary_commands)
+                )
+
             self._jobset = jobset.models.jobset_v1alpha2_job_set.JobsetV1alpha2JobSet(
                 api_version="jobset.x-k8s.io/v1alpha2",
                 kind="JobSet",
@@ -437,14 +443,7 @@ class KubernetesJob(object):
                         enable_dns_hostnames=True if not self._kwargs['attrs']['requires_passwordless_ssh'] else False,
                         subdomain=subdomain
                     ),
-                    replicated_jobs=[
-                        _get_replicated_job("control", 1, main_commands),
-                        _get_replicated_job(
-                            "worker",
-                            self._kwargs["num_parallel"] - 1,
-                            secondary_commands,
-                        ),
-                    ],
+                    replicated_jobs=replicated_jobs
                 ),
             )
             self._passwordless_ssh_service = _get_passwordless_ssh_service()
@@ -484,6 +483,13 @@ class KubernetesJob(object):
                             containers=[
                                 client.V1Container(
                                     command=self._kwargs["command"],
+                                    ports=[
+                                        client.V1ContainerPort(
+                                            container_port=int(self._kwargs["port"])
+                                        )
+                                    ]
+                                    if "port" in self._kwargs and self._kwargs["port"]
+                                    else None,
                                     env=[
                                         client.V1EnvVar(name=k, value=str(v))
                                         for k, v in self._kwargs.get(

ob-metaflow-2.11.10.1/metaflow/version.py

@@ -0,0 +1 @@
+metaflow_version = "2.11.10.1"

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1/ob_metaflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.11.8.1
+Version: 2.11.10.1
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co
@@ -12,7 +12,7 @@ Requires-Dist: boto3
 Requires-Dist: pylint
 Requires-Dist: kubernetes
 Provides-Extra: stubs
-Requires-Dist: ob-metaflow-stubs==2.11.8.1; extra == "stubs"
+Requires-Dist: ob-metaflow-stubs==2.11.10.1; extra == "stubs"
 
 ![Metaflow_Logo_Horizontal_FullColor_Ribbon_Dark_RGB](https://user-images.githubusercontent.com/763451/89453116-96a57e00-d713-11ea-9fa6-82b29d4d6eff.png)
 

{ob-metaflow-2.11.8.1 → ob-metaflow-2.11.10.1}/ob_metaflow.egg-info/requires.txt

@@ -4,4 +4,4 @@ pylint
 kubernetes
 
 [stubs]
-ob-metaflow-stubs==2.11.8.1
+ob-metaflow-stubs==2.11.10.1

ob-metaflow-2.11.8.1/metaflow/version.py

@@ -1 +0,0 @@
-metaflow_version = "2.11.8.1"