ob-metaflow 2.10.11.1__py2.py3-none-any.whl → 2.11.0.2__py2.py3-none-any.whl

metaflow/metaflow_config.py

@@ -296,6 +296,8 @@ KUBERNETES_CONTAINER_REGISTRY = from_conf(
 )
 # Toggle for trying to fetch EC2 instance metadata
 KUBERNETES_FETCH_EC2_METADATA = from_conf("KUBERNETES_FETCH_EC2_METADATA", False)
+# Default port number to open on the pods
+KUBERNETES_PORT = from_conf("KUBERNETES_PORT", None)
 
 ARGO_WORKFLOWS_KUBERNETES_SECRETS = from_conf("ARGO_WORKFLOWS_KUBERNETES_SECRETS", "")
 ARGO_WORKFLOWS_ENV_VARS_TO_SKIP = from_conf("ARGO_WORKFLOWS_ENV_VARS_TO_SKIP", "")

metaflow/plugins/argo/argo_workflows.py

@@ -836,6 +836,11 @@ class ArgoWorkflows(object):
     # Visit every node and yield the uber DAGTemplate(s).
     def _dag_templates(self):
         def _visit(node, exit_node=None, templates=None, dag_tasks=None):
+            if node.parallel_foreach:
+                raise ArgoWorkflowsException(
+                    "Deploying flows with @parallel decorator(s) "
+                    "as Argo Workflows is not supported currently."
+                )
             # Every for-each node results in a separate subDAG and an equivalent
             # DAGTemplate rooted at the child of the for-each node. Each DAGTemplate
             # has a unique name - the top-level DAGTemplate is named as the name of

metaflow/plugins/kubernetes/kubernetes.py

@@ -4,6 +4,7 @@ import os
 import re
 import shlex
 import time
+import copy
 from typing import Dict, List, Optional
 import uuid
 from uuid import uuid4
@@ -174,6 +175,10 @@ class Kubernetes(object):
         persistent_volume_claims=None,
         tolerations=None,
         labels=None,
+        annotations=None,
+        num_parallel=0,
+        attrs={},
+        port=None,
     ):
         if env is None:
             env = {}
@@ -213,6 +218,9 @@ class Kubernetes(object):
                 tmpfs_size=tmpfs_size,
                 tmpfs_path=tmpfs_path,
                 persistent_volume_claims=persistent_volume_claims,
+                num_parallel=num_parallel,
+                attrs=attrs,
+                port=port,
             )
             .environment_variable("METAFLOW_CODE_SHA", code_package_sha)
             .environment_variable("METAFLOW_CODE_URL", code_package_url)
@@ -266,6 +274,7 @@ class Kubernetes(object):
             # see get_datastore_root_from_config in datastore/local.py).
         )
 
+        self.num_parallel = num_parallel
         # Temporary passing of *some* environment variables. Do not rely on this
         # mechanism as it will be removed in the near future
         for k, v in config_values():
@@ -341,7 +350,7 @@ class Kubernetes(object):
             sigmoid = 1.0 / (1.0 + math.exp(-0.01 * secs_since_start + 9.0))
             return 0.5 + sigmoid * 30.0
 
-        def wait_for_launch(job):
+        def wait_for_launch(job, child_jobs):
             status = job.status
             echo(
                 "Task is starting (%s)..." % status,
@@ -351,11 +360,38 @@ class Kubernetes(object):
             t = time.time()
             start_time = time.time()
             while job.is_waiting:
-                new_status = job.status
-                if status != new_status or (time.time() - t) > 30:
-                    status = new_status
+                # new_status = job.status
+                if status != job.status or (time.time() - t) > 30:
+                    if not child_jobs:
+                        child_statuses = ""
+                    else:
+                        status_keys = set(
+                            [child_job.status for child_job in child_jobs]
+                        )
+                        status_counts = [
+                            (
+                                status,
+                                len(
+                                    [
+                                        child_job.status == status
+                                        for child_job in child_jobs
+                                    ]
+                                ),
+                            )
+                            for status in status_keys
+                        ]
+                        child_statuses = " (parallel node status: [{}])".format(
+                            ", ".join(
+                                [
+                                    "{}:{}".format(status, num)
+                                    for (status, num) in sorted(status_counts)
+                                ]
+                            )
+                        )
+
+                    status = job.status
                     echo(
-                        "Task is starting (%s)..." % status,
+                        "Task is starting (status %s)... %s" % (status, child_statuses),
                         "stderr",
                         job_id=job.id,
                     )
@@ -367,8 +403,9 @@ class Kubernetes(object):
         stdout_tail = get_log_tailer(stdout_location, self._datastore.TYPE)
         stderr_tail = get_log_tailer(stderr_location, self._datastore.TYPE)
 
+        child_jobs = []
         # 1) Loop until the job has started
-        wait_for_launch(self._job)
+        wait_for_launch(self._job, child_jobs)
 
         # 2) Tail logs until the job has finished
         tail_logs(

metaflow/plugins/kubernetes/kubernetes_cli.py

@@ -107,6 +107,26 @@ def kubernetes():
     type=JSONTypeClass(),
     multiple=False,
 )
+@click.option(
+    "--labels",
+    default=None,
+    type=JSONTypeClass(),
+    multiple=False,
+)
+@click.option(
+    "--annotations",
+    default=None,
+    type=JSONTypeClass(),
+    multiple=False,
+)
+@click.option("--ubf-context", default=None, type=click.Choice([None, "ubf_control"]))
+@click.option(
+    "--num-parallel",
+    default=0,
+    type=int,
+    help="Number of parallel nodes to run as a multi-node job.",
+)
+@click.option("--port", default=None, help="port number")
 @click.pass_context
 def step(
     ctx,
@@ -132,6 +152,10 @@ def step(
     run_time_limit=None,
     persistent_volume_claims=None,
     tolerations=None,
+    labels=None,
+    annotations=None,
+    num_parallel=None,
+    port=None,
     **kwargs
 ):
     def echo(msg, stream="stderr", job_id=None, **kwargs):
@@ -177,11 +201,17 @@ def step(
         )
         time.sleep(minutes_between_retries * 60)
 
+    step_args = " ".join(util.dict_to_cli_options(kwargs))
+    num_parallel = num_parallel or 0
+    if num_parallel and num_parallel > 1:
+        # For multinode, we need to add a placeholder that can be mutated by the caller
+        step_args += " [multinode-args]"
+
     step_cli = "{entrypoint} {top_args} step {step} {step_args}".format(
         entrypoint="%s -u %s" % (executable, os.path.basename(sys.argv[0])),
         top_args=" ".join(util.dict_to_cli_options(ctx.parent.parent.params)),
         step=step_name,
-        step_args=" ".join(util.dict_to_cli_options(kwargs)),
+        step_args=step_args,
     )
 
     # Set log tailing.
@@ -207,6 +237,10 @@ def step(
                 ),
             )
 
+    attrs = {
+        "metaflow.task_id": kwargs["task_id"],
+        "requires_passwordless_ssh": any([getattr(deco, "requires_passwordless_ssh", False) for deco in node.decorators]),
+    }
     try:
         kubernetes = Kubernetes(
             datastore=ctx.obj.flow_datastore,
@@ -245,6 +279,11 @@ def step(
                 env=env,
                 persistent_volume_claims=persistent_volume_claims,
                 tolerations=tolerations,
+                labels=labels,
+                annotations=annotations,
+                num_parallel=num_parallel,
+                port=port,
+                attrs=attrs,
             )
     except Exception as e:
         traceback.print_exc(chain=False)

metaflow/plugins/kubernetes/kubernetes_decorator.py

@@ -2,6 +2,7 @@ import json
 import os
 import platform
 import sys
+import time
 
 from metaflow import current
 from metaflow.decorators import StepDecorator
@@ -20,10 +21,12 @@ from metaflow.metaflow_config import (
     KUBERNETES_PERSISTENT_VOLUME_CLAIMS,
     KUBERNETES_TOLERATIONS,
     KUBERNETES_SERVICE_ACCOUNT,
+    KUBERNETES_PORT,
 )
 from metaflow.plugins.resources_decorator import ResourcesDecorator
 from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task
 from metaflow.sidecar import Sidecar
+from metaflow.unbounded_foreach import UBF_CONTROL
 
 from ..aws.aws_utils import get_docker_registry, get_ec2_instance_metadata
 from .kubernetes import KubernetesException, parse_kube_keyvalue_list
@@ -88,6 +91,8 @@ class KubernetesDecorator(StepDecorator):
     persistent_volume_claims: Dict[str, str], optional
         A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
         volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
+    port: int, optional
+        Number of the port to specify in the Kubernetes job object
     """
 
     name = "kubernetes"
@@ -110,6 +115,7 @@ class KubernetesDecorator(StepDecorator):
         "tmpfs_size": None,
         "tmpfs_path": "/metaflow_temp",
         "persistent_volume_claims": None,  # e.g., {"pvc-name": "/mnt/vol", "another-pvc": "/mnt/vol2"}
+        "port": None,
     }
     package_url = None
     package_sha = None
@@ -195,6 +201,8 @@ class KubernetesDecorator(StepDecorator):
             if not self.attributes["tmpfs_size"]:
                 # default tmpfs behavior - https://man7.org/linux/man-pages/man5/tmpfs.5.html
                 self.attributes["tmpfs_size"] = int(self.attributes["memory"]) // 2
+        if not self.attributes["port"]:
+            self.attributes["port"] = KUBERNETES_PORT
 
     # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png
     def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
@@ -216,12 +224,6 @@ class KubernetesDecorator(StepDecorator):
                 "Kubernetes. Please use one or the other.".format(step=step)
             )
 
-        for deco in decos:
-            if getattr(deco, "IS_PARALLEL", False):
-                raise KubernetesException(
-                    "@kubernetes does not support parallel execution currently."
-                )
-
         # Set run time limit for the Kubernetes job.
         self.run_time_limit = get_run_time_limit_for_task(decos)
         if self.run_time_limit < 60:
@@ -432,6 +434,27 @@ class KubernetesDecorator(StepDecorator):
             self._save_logs_sidecar = Sidecar("save_logs_periodically")
             self._save_logs_sidecar.start()
 
+        num_parallel = int(os.environ.get("WORLD_SIZE", 0))
+        if num_parallel >= 1:
+            if ubf_context == UBF_CONTROL:
+                control_task_id = current.task_id
+                top_task_id = control_task_id.replace("control-", "")
+                mapper_task_ids = [control_task_id] + [
+                    "%s-node-%d" % (top_task_id, node_idx)
+                    for node_idx in range(1, num_parallel)
+                ]
+                flow._control_mapper_tasks = [
+                    "%s/%s/%s" % (run_id, step_name, mapper_task_id)
+                    for mapper_task_id in mapper_task_ids
+                ]
+                flow._control_task_is_mapper_zero = True
+            else:
+                worker_job_rank = int(os.environ["RANK"])
+                os.environ["RANK"] = str(worker_job_rank + 1)
+
+        if num_parallel >= 1:
+            _setup_multinode_environment()
+
     def task_finished(
         self, step_name, flow, graph, is_task_ok, retry_count, max_retries
     ):
@@ -459,9 +482,53 @@ class KubernetesDecorator(StepDecorator):
             # Best effort kill
             pass
 
+        if is_task_ok and len(getattr(flow, "_control_mapper_tasks", [])) > 1:
+            self._wait_for_mapper_tasks(flow, step_name)
+
+    def _wait_for_mapper_tasks(self, flow, step_name):
+        """
+        When launching multinode task with UBF, need to wait for the secondary
+        tasks to finish cleanly and produce their output before exiting the
+        main task. Otherwise, the main task finishing will cause secondary nodes
+        to terminate immediately, and possibly prematurely.
+        """
+        from metaflow import Step  # avoid circular dependency
+
+        TIMEOUT = 600
+        last_completion_timeout = time.time() + TIMEOUT
+        print("Waiting for batch secondary tasks to finish")
+        while last_completion_timeout > time.time():
+            time.sleep(2)
+            try:
+                step_path = "%s/%s/%s" % (flow.name, current.run_id, step_name)
+                tasks = [task for task in Step(step_path)]
+                if len(tasks) == len(flow._control_mapper_tasks):
+                    if all(
+                        task.finished_at is not None for task in tasks
+                    ):  # for some reason task.finished fails
+                        return True
+                else:
+                    print(
+                        "Waiting for all parallel tasks to finish. Finished: {}/{}".format(
+                            len(tasks),
+                            len(flow._control_mapper_tasks),
+                        )
+                    )
+            except Exception as e:
+                pass
+        raise Exception(
+            "Batch secondary workers did not finish in %s seconds" % TIMEOUT
+        )
+
     @classmethod
     def _save_package_once(cls, flow_datastore, package):
         if cls.package_url is None:
             cls.package_url, cls.package_sha = flow_datastore.save_data(
                 [package.blob], len_hint=1
             )[0]
+
+def _setup_multinode_environment():
+    import socket
+    os.environ["MF_PARALLEL_MAIN_IP"] = socket.gethostbyname(os.environ["MASTER_ADDR"])
+    os.environ["MF_PARALLEL_NUM_NODES"] = os.environ["WORLD_SIZE"]
+    os.environ["MF_PARALLEL_NODE_INDEX"] = os.environ["RANK"]

metaflow/plugins/kubernetes/kubernetes_job.py

@@ -2,20 +2,18 @@ import json
 import math
 import random
 import time
-
-from metaflow.tracing import inject_tracing_vars
-
+import os
+import socket
+import copy
 
 from metaflow.exception import MetaflowException
 from metaflow.metaflow_config import KUBERNETES_SECRETS
 
 CLIENT_REFRESH_INTERVAL_SECONDS = 300
 
-
 class KubernetesJobException(MetaflowException):
     headline = "Kubernetes job error"
 
-
 # Implements truncated exponential backoff from
 # https://cloud.google.com/storage/docs/retry-strategy#exponential-backoff
 def k8s_retry(deadline_seconds=60, max_backoff=32):
@@ -78,107 +76,260 @@ class KubernetesJob(object):
         tmpfs_size = self._kwargs["tmpfs_size"]
         tmpfs_enabled = use_tmpfs or (tmpfs_size and not use_tmpfs)
 
-        self._job = client.V1Job(
-            api_version="batch/v1",
-            kind="Job",
-            metadata=client.V1ObjectMeta(
-                # Annotations are for humans
-                annotations=self._kwargs.get("annotations", {}),
-                # While labels are for Kubernetes
-                labels=self._kwargs.get("labels", {}),
-                generate_name=self._kwargs["generate_name"],
-                namespace=self._kwargs["namespace"],  # Defaults to `default`
-            ),
-            spec=client.V1JobSpec(
-                # Retries are handled by Metaflow when it is responsible for
-                # executing the flow. The responsibility is moved to Kubernetes
-                # when Argo Workflows is responsible for the execution.
-                backoff_limit=self._kwargs.get("retries", 0),
-                completions=1,  # A single non-indexed pod job
-                ttl_seconds_after_finished=7
-                * 60
-                * 60  # Remove job after a week. TODO: Make this configurable
-                * 24,
-                template=client.V1PodTemplateSpec(
+        jobset_name = "js-%s" % self._kwargs["attrs"]["metaflow.task_id"].split('-')[-1]
+        main_job_name = "control"
+        main_job_index = 0
+        main_pod_index = 0
+        subdomain = jobset_name
+        master_port = int(self._kwargs['port']) if self._kwargs['port'] else None
+
+        passwordless_ssh = self._kwargs["attrs"]["requires_passwordless_ssh"]
+        if passwordless_ssh:
+            passwordless_ssh_service_name = subdomain
+            passwordless_ssh_service_selector = {
+                "passwordless-ssh-jobset": "true"
+            }
+        else:
+            passwordless_ssh_service_name = None
+            passwordless_ssh_service_selector = {}
+
+        fqdn_suffix = "%s.svc.cluster.local" % self._kwargs["namespace"]
+        jobset_main_addr = "%s-%s-%s-%s.%s.%s" % (
+            jobset_name,
+            main_job_name,
+            main_job_index,
+            main_pod_index,
+            subdomain,
+            fqdn_suffix,
+        )
+
+        def _install_jobset(
+            repo_url="https://github.com/kubernetes-sigs/jobset",
+            python_sdk_path="jobset/sdk/python",
+        ):
+
+            # TODO (Eddie): Remove this and suggest to user.
+
+            import subprocess
+            import tempfile
+            import shutil
+            import os
+
+            with open(os.devnull, "wb") as devnull:
+                cwd = os.getcwd()
+                tmp_dir = tempfile.mkdtemp()
+                os.chdir(tmp_dir)
+                subprocess.check_call(
+                    ["git", "clone", repo_url], stdout=devnull, stderr=subprocess.STDOUT
+                )
+                tmp_python_sdk_path = os.path.join(tmp_dir, python_sdk_path)
+                os.chdir(tmp_python_sdk_path)
+                subprocess.check_call(
+                    ["pip", "install", "."], stdout=devnull, stderr=subprocess.STDOUT
+                )
+                os.chdir(cwd)
+                shutil.rmtree(tmp_dir)
+
+        def _get_passwordless_ssh_service():
+
+            return client.V1Service(
+                api_version="v1",
+                kind="Service",
+                metadata=client.V1ObjectMeta(
+                    name=passwordless_ssh_service_name,
+                    namespace=self._kwargs["namespace"]
+                ),
+                spec=client.V1ServiceSpec(
+                    cluster_ip="None",
+                    internal_traffic_policy="Cluster",
+                    ip_families=["IPv4"],
+                    ip_family_policy="SingleStack",
+                    selector=passwordless_ssh_service_selector,
+                    session_affinity="None",
+                    type="ClusterIP",
+                    ports=[
+                        client.V1ServicePort(
+                            name="control",
+                            port=22,
+                            protocol="TCP",
+                            target_port=22
+                        )
+                    ]
+                )
+            )
+
+        def _get_replicated_job(job_name, parallelism, command):
+            return jobset.models.jobset_v1alpha2_replicated_job.JobsetV1alpha2ReplicatedJob(
+                name=job_name,
+                template=client.V1JobTemplateSpec(
                     metadata=client.V1ObjectMeta(
                         annotations=self._kwargs.get("annotations", {}),
                         labels=self._kwargs.get("labels", {}),
                         namespace=self._kwargs["namespace"],
                     ),
-                    spec=client.V1PodSpec(
-                        # Timeout is set on the pod and not the job (important!)
-                        active_deadline_seconds=self._kwargs["timeout_in_seconds"],
-                        # TODO (savin): Enable affinities for GPU scheduling.
-                        # affinity=?,
-                        containers=[
-                            client.V1Container(
-                                command=self._kwargs["command"],
-                                env=[
-                                    client.V1EnvVar(name=k, value=str(v))
-                                    for k, v in self._kwargs.get(
-                                        "environment_variables", {}
-                                    ).items()
-                                ]
-                                # And some downward API magic. Add (key, value)
-                                # pairs below to make pod metadata available
-                                # within Kubernetes container.
-                                + [
-                                    client.V1EnvVar(
-                                        name=k,
-                                        value_from=client.V1EnvVarSource(
-                                            field_ref=client.V1ObjectFieldSelector(
-                                                field_path=str(v)
+                    spec=client.V1JobSpec(
+                        parallelism=parallelism,  # how many jobs can run at once
+                        completions=parallelism,  # how many Pods the JobSet creates in total
+                        backoff_limit=0,
+                        ttl_seconds_after_finished=7
+                        * 60
+                        * 60
+                        * 24,
+                        template=client.V1PodTemplateSpec(
+                            metadata=client.V1ObjectMeta(
+                                annotations=self._kwargs.get("annotations", {}),
+                                labels={
+                                    **self._kwargs.get("labels", {}),
+                                    **passwordless_ssh_service_selector, # TODO: necessary?
+                                    # TODO: cluster-name, app.kubernetes.io/name necessary?
+                                },
+                                namespace=self._kwargs["namespace"],
+                            ),
+                            spec=client.V1PodSpec(
+                                active_deadline_seconds=self._kwargs[
+                                    "timeout_in_seconds"
+                                ],
+                                containers=[
+                                    client.V1Container(
+                                        command=command,
+                                        ports=[client.V1ContainerPort(container_port=master_port)] if master_port and job_name=="control" else [],
+                                        env=[
+                                            client.V1EnvVar(name=k, value=str(v))
+                                            for k, v in self._kwargs.get(
+                                                "environment_variables", {}
+                                            ).items()
+                                        ]
+                                        + [
+                                            client.V1EnvVar(
+                                                name=k,
+                                                value_from=client.V1EnvVarSource(
+                                                    field_ref=client.V1ObjectFieldSelector(
+                                                        field_path=str(v)
+                                                    )
+                                                ),
+                                            )
+                                            for k, v in {
+                                                "METAFLOW_KUBERNETES_POD_NAMESPACE": "metadata.namespace",
+                                                "METAFLOW_KUBERNETES_POD_NAME": "metadata.name",
+                                                "METAFLOW_KUBERNETES_POD_ID": "metadata.uid",
+                                                "METAFLOW_KUBERNETES_SERVICE_ACCOUNT_NAME": "spec.serviceAccountName",
+                                                "METAFLOW_KUBERNETES_NODE_IP": "status.hostIP",
+                                            }.items()
+                                        ]
+                                        # Mimicking the AWS Batch Multinode env vars.
+                                        + [
+                                            client.V1EnvVar(
+                                                name="MASTER_ADDR",
+                                                value=jobset_main_addr,
+                                            ),
+                                            client.V1EnvVar(
+                                                name="MASTER_PORT",
+                                                value=str(master_port),
+                                            ),
+                                            client.V1EnvVar(
+                                                name="RANK",
+                                                value_from=client.V1EnvVarSource(
+                                                    field_ref=client.V1ObjectFieldSelector(
+                                                        field_path="metadata.annotations['batch.kubernetes.io/job-completion-index']"
+                                                    )
+                                                ),
+                                            ),
+                                            client.V1EnvVar(
+                                                name="WORLD_SIZE",
+                                                value=str(self._kwargs["num_parallel"]),
+                                            ),
+                                            client.V1EnvVar(
+                                                name="PYTHONUNBUFFERED",
+                                                value="0",
+                                            ),
+                                        ],
+                                        env_from=[
+                                            client.V1EnvFromSource(
+                                                secret_ref=client.V1SecretEnvSource(
+                                                    name=str(k),
+                                                    # optional=True
+                                                )
                                             )
+                                            for k in list(
+                                                self._kwargs.get("secrets", [])
+                                            )
+                                            + KUBERNETES_SECRETS.split(",")
+                                            if k
+                                        ],
+                                        image=self._kwargs["image"],
+                                        image_pull_policy=self._kwargs[
+                                            "image_pull_policy"
+                                        ],
+                                        name=self._kwargs["step_name"].replace(
+                                            "_", "-"
                                         ),
-                                    )
-                                    for k, v in {
-                                        "METAFLOW_KUBERNETES_POD_NAMESPACE": "metadata.namespace",
-                                        "METAFLOW_KUBERNETES_POD_NAME": "metadata.name",
-                                        "METAFLOW_KUBERNETES_POD_ID": "metadata.uid",
-                                        "METAFLOW_KUBERNETES_SERVICE_ACCOUNT_NAME": "spec.serviceAccountName",
-                                        "METAFLOW_KUBERNETES_NODE_IP": "status.hostIP",
-                                    }.items()
-                                ]
-                                + [
-                                    client.V1EnvVar(name=k, value=str(v))
-                                    for k, v in inject_tracing_vars({}).items()
-                                ],
-                                env_from=[
-                                    client.V1EnvFromSource(
-                                        secret_ref=client.V1SecretEnvSource(
-                                            name=str(k),
-                                            # optional=True
+                                        resources=client.V1ResourceRequirements(
+                                            requests={
+                                                "cpu": str(self._kwargs["cpu"]),
+                                                "memory": "%sM"
+                                                % str(self._kwargs["memory"]),
+                                                "ephemeral-storage": "%sM"
+                                                % str(self._kwargs["disk"]),
+                                            },
+                                            limits={
+                                                "%s.com/gpu".lower()
+                                                % self._kwargs["gpu_vendor"]: str(
+                                                    self._kwargs["gpu"]
+                                                )
+                                                for k in [0]
+                                                # Don't set GPU limits if gpu isn't specified.
+                                                if self._kwargs["gpu"] is not None
+                                            },
+                                        ),
+                                        volume_mounts=(
+                                            [
+                                                client.V1VolumeMount(
+                                                    mount_path=self._kwargs.get(
+                                                        "tmpfs_path"
+                                                    ),
+                                                    name="tmpfs-ephemeral-volume",
+                                                )
+                                            ]
+                                            if tmpfs_enabled
+                                            else []
                                         )
+                                        + (
+                                            [
+                                                client.V1VolumeMount(
+                                                    mount_path=path, name=claim
+                                                )
+                                                for claim, path in self._kwargs[
+                                                    "persistent_volume_claims"
+                                                ].items()
+                                            ]
+                                            if self._kwargs["persistent_volume_claims"]
+                                            is not None
+                                            else []
+                                        ),
                                     )
-                                    for k in list(self._kwargs.get("secrets", []))
-                                    + KUBERNETES_SECRETS.split(",")
-                                    if k
                                 ],
-                                image=self._kwargs["image"],
-                                image_pull_policy=self._kwargs["image_pull_policy"],
-                                name=self._kwargs["step_name"].replace("_", "-"),
-                                resources=client.V1ResourceRequirements(
-                                    requests={
-                                        "cpu": str(self._kwargs["cpu"]),
-                                        "memory": "%sM" % str(self._kwargs["memory"]),
-                                        "ephemeral-storage": "%sM"
-                                        % str(self._kwargs["disk"]),
-                                    },
-                                    limits={
-                                        "%s.com/gpu".lower()
-                                        % self._kwargs["gpu_vendor"]: str(
-                                            self._kwargs["gpu"]
-                                        )
-                                        for k in [0]
-                                        # Don't set GPU limits if gpu isn't specified.
-                                        if self._kwargs["gpu"] is not None
-                                    },
-                                ),
-                                volume_mounts=(
+                                node_selector=self._kwargs.get("node_selector"),
+                                restart_policy="Never",
+
+                                set_hostname_as_fqdn=True, # configure pod hostname as pod's FQDN
+                                share_process_namespace=False, # default
+                                subdomain=subdomain, # FQDN = <hostname>.<subdomain>.<pod namespace>.svc.<cluster domain>
+
+                                service_account_name=self._kwargs["service_account"],
+                                termination_grace_period_seconds=0,
+                                tolerations=[
+                                    client.V1Toleration(**toleration)
+                                    for toleration in self._kwargs.get("tolerations")
+                                    or []
+                                ],
+                                volumes=(
                                     [
-                                        client.V1VolumeMount(
-                                            mount_path=self._kwargs.get("tmpfs_path"),
+                                        client.V1Volume(
                                             name="tmpfs-ephemeral-volume",
+                                            empty_dir=client.V1EmptyDirVolumeSource(
+                                                medium="Memory",
+                                                size_limit="{}Mi".format(tmpfs_size),
+                                            ),
                                         )
                                     ]
                                     if tmpfs_enabled
@@ -186,72 +337,264 @@ class KubernetesJob(object):
                                 )
                                 + (
                                     [
-                                        client.V1VolumeMount(
-                                            mount_path=path, name=claim
+                                        client.V1Volume(
+                                            name=claim,
+                                            persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
+                                                claim_name=claim
+                                            ),
                                         )
-                                        for claim, path in self._kwargs[
+                                        for claim in self._kwargs[
                                             "persistent_volume_claims"
-                                        ].items()
+                                        ].keys()
                                     ]
                                     if self._kwargs["persistent_volume_claims"]
                                     is not None
                                     else []
                                 ),
-                            )
-                        ],
-                        node_selector=self._kwargs.get("node_selector"),
-                        # TODO (savin): Support image_pull_secrets
-                        # image_pull_secrets=?,
-                        # TODO (savin): Support preemption policies
-                        # preemption_policy=?,
-                        #
-                        # A Container in a Pod may fail for a number of
-                        # reasons, such as because the process in it exited
-                        # with a non-zero exit code, or the Container was
-                        # killed due to OOM etc. If this happens, fail the pod
-                        # and let Metaflow handle the retries.
-                        restart_policy="Never",
-                        service_account_name=self._kwargs["service_account"],
-                        # Terminate the container immediately on SIGTERM
-                        termination_grace_period_seconds=0,
-                        tolerations=[
-                            client.V1Toleration(**toleration)
-                            for toleration in self._kwargs.get("tolerations") or []
-                        ],
-                        volumes=(
-                            [
-                                client.V1Volume(
-                                    name="tmpfs-ephemeral-volume",
-                                    empty_dir=client.V1EmptyDirVolumeSource(
-                                        medium="Memory",
-                                        # Add default unit as ours differs from Kubernetes default.
-                                        size_limit="{}Mi".format(tmpfs_size),
+                            ),
+                        ),
+                    ),
+                ),
+            )
+
+        if "num_parallel" in self._kwargs and self._kwargs["num_parallel"] >= 1:
+
+            try:
+                import jobset
+            except ImportError:
+                _install_jobset()
+                import jobset
+
+            main_commands = copy.copy(self._kwargs["command"])
+            main_commands[-1] = main_commands[-1].replace(
+                "[multinode-args]", "--split-index 0"
+            )
+
+            task_id = self._kwargs["attrs"]["metaflow.task_id"]
+            secondary_commands = copy.copy(self._kwargs["command"])
+            # RANK needs +1 because control node is not in the worker index group, yet we want global nodes.
+            # Technically, control and worker could be same replicated job type, but cleaner to separate for future use cases.
+            secondary_commands[-1] = secondary_commands[-1].replace(
+                "[multinode-args]", "--split-index `expr $RANK + 1`"
+            )
+            secondary_commands[-1] = secondary_commands[-1].replace(
+                "ubf_control", "ubf_task"
+            )
+            secondary_commands[-1] = secondary_commands[-1].replace(
+                task_id,
+                task_id.replace("control-", "") + "-node-`expr $RANK + 1`",
+            )
+
+            if passwordless_ssh:
+                if not os.path.exists("/usr/sbin/sshd"):
+                    raise KubernetesJobException(
+                        "This @parallel decorator requires sshd to be installed in the container image."
+                        "Please install OpenSSH."
+                    )
+
+                # run sshd in background
+                main_commands[-1] = "/usr/sbin/sshd -D & %s" % main_commands[-1]
+                secondary_commands[-1] = "/usr/sbin/sshd -D & %s" % secondary_commands[-1]
+
+            self._jobset = jobset.models.jobset_v1alpha2_job_set.JobsetV1alpha2JobSet(
+                api_version="jobset.x-k8s.io/v1alpha2",
+                kind="JobSet",
+                metadata=client.V1ObjectMeta(
+                    annotations=self._kwargs.get("annotations", {}),
+                    labels=self._kwargs.get("labels", {}),
+                    name=jobset_name,
+                    namespace=self._kwargs["namespace"],
+                ),
+                spec=jobset.models.jobset_v1alpha2_job_set_spec.JobsetV1alpha2JobSetSpec(
+                    network=jobset.models.jobset_v1alpha2_network.JobsetV1alpha2Network(
+                        enable_dns_hostnames=True if not self._kwargs['attrs']['requires_passwordless_ssh'] else False,
+                        subdomain=subdomain
+                    ),
+                    replicated_jobs=[
+                        _get_replicated_job("control", 1, main_commands),
+                        _get_replicated_job(
+                            "worker",
+                            self._kwargs["num_parallel"] - 1,
+                            secondary_commands,
+                        ),
+                    ],
+                ),
+            )
+            self._passwordless_ssh_service = _get_passwordless_ssh_service()
+        else:
+            self._job = client.V1Job(
+                api_version="batch/v1",
+                kind="Job",
+                metadata=client.V1ObjectMeta(
+                    # Annotations are for humans
+                    annotations=self._kwargs.get("annotations", {}),
+                    # While labels are for Kubernetes
+                    labels=self._kwargs.get("labels", {}),
+                    generate_name=self._kwargs["generate_name"],
+                    namespace=self._kwargs["namespace"],  # Defaults to `default`
+                ),
+                spec=client.V1JobSpec(
+                    # Retries are handled by Metaflow when it is responsible for
+                    # executing the flow. The responsibility is moved to Kubernetes
+                    # when Argo Workflows is responsible for the execution.
+                    backoff_limit=self._kwargs.get("retries", 0),
+                    completions=1,  # A single non-indexed pod job
+                    ttl_seconds_after_finished=7
+                    * 60
+                    * 60  # Remove job after a week. TODO: Make this configurable
+                    * 24,
+                    template=client.V1PodTemplateSpec(
+                        metadata=client.V1ObjectMeta(
+                            annotations=self._kwargs.get("annotations", {}),
+                            labels=self._kwargs.get("labels", {}),
+                            namespace=self._kwargs["namespace"],
+                        ),
+                        spec=client.V1PodSpec(
+                            # Timeout is set on the pod and not the job (important!)
+                            active_deadline_seconds=self._kwargs["timeout_in_seconds"],
+                            # TODO (savin): Enable affinities for GPU scheduling.
+                            # affinity=?,
+                            containers=[
+                                client.V1Container(
+                                    command=self._kwargs["command"],
+                                    env=[
+                                        client.V1EnvVar(name=k, value=str(v))
+                                        for k, v in self._kwargs.get(
+                                            "environment_variables", {}
+                                        ).items()
+                                    ]
+                                    # And some downward API magic. Add (key, value)
+                                    # pairs below to make pod metadata available
+                                    # within Kubernetes container.
+                                    + [
+                                        client.V1EnvVar(
+                                            name=k,
+                                            value_from=client.V1EnvVarSource(
+                                                field_ref=client.V1ObjectFieldSelector(
+                                                    field_path=str(v)
+                                                )
+                                            ),
+                                        )
+                                        for k, v in {
+                                            "METAFLOW_KUBERNETES_POD_NAMESPACE": "metadata.namespace",
+                                            "METAFLOW_KUBERNETES_POD_NAME": "metadata.name",
+                                            "METAFLOW_KUBERNETES_POD_ID": "metadata.uid",
+                                            "METAFLOW_KUBERNETES_SERVICE_ACCOUNT_NAME": "spec.serviceAccountName",
+                                            "METAFLOW_KUBERNETES_NODE_IP": "status.hostIP",
+                                        }.items()
+                                    ],
+                                    env_from=[
+                                        client.V1EnvFromSource(
+                                            secret_ref=client.V1SecretEnvSource(
+                                                name=str(k),
+                                                # optional=True
+                                            )
+                                        )
+                                        for k in list(self._kwargs.get("secrets", []))
+                                        + KUBERNETES_SECRETS.split(",")
+                                        if k
+                                    ],
+                                    image=self._kwargs["image"],
+                                    image_pull_policy=self._kwargs["image_pull_policy"],
+                                    name=self._kwargs["step_name"].replace("_", "-"),
+                                    resources=client.V1ResourceRequirements(
+                                        requests={
+                                            "cpu": str(self._kwargs["cpu"]),
+                                            "memory": "%sM"
+                                            % str(self._kwargs["memory"]),
+                                            "ephemeral-storage": "%sM"
+                                            % str(self._kwargs["disk"]),
+                                        },
+                                        limits={
+                                            "%s.com/gpu".lower()
+                                            % self._kwargs["gpu_vendor"]: str(
+                                                self._kwargs["gpu"]
+                                            )
+                                            for k in [0]
+                                            # Don't set GPU limits if gpu isn't specified.
+                                            if self._kwargs["gpu"] is not None
+                                        },
                                     ),
-                                )
-                            ]
-                            if tmpfs_enabled
-                            else []
-                        )
-                        + (
-                            [
-                                client.V1Volume(
-                                    name=claim,
-                                    persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
-                                        claim_name=claim
+                                    volume_mounts=(
+                                        [
+                                            client.V1VolumeMount(
+                                                mount_path=self._kwargs.get(
+                                                    "tmpfs_path"
+                                                ),
+                                                name="tmpfs-ephemeral-volume",
+                                            )
+                                        ]
+                                        if tmpfs_enabled
+                                        else []
+                                    )
+                                    + (
+                                        [
+                                            client.V1VolumeMount(
+                                                mount_path=path, name=claim
+                                            )
+                                            for claim, path in self._kwargs[
+                                                "persistent_volume_claims"
+                                            ].items()
+                                        ]
+                                        if self._kwargs["persistent_volume_claims"]
+                                        is not None
+                                        else []
                                     ),
                                 )
-                                for claim in self._kwargs[
-                                    "persistent_volume_claims"
-                                ].keys()
-                            ]
-                            if self._kwargs["persistent_volume_claims"] is not None
-                            else []
+                            ],
+                            node_selector=self._kwargs.get("node_selector"),
+                            # TODO (savin): Support image_pull_secrets
+                            # image_pull_secrets=?,
+                            # TODO (savin): Support preemption policies
+                            # preemption_policy=?,
+                            #
+                            # A Container in a Pod may fail for a number of
+                            # reasons, such as because the process in it exited
+                            # with a non-zero exit code, or the Container was
+                            # killed due to OOM etc. If this happens, fail the pod
+                            # and let Metaflow handle the retries.
+                            restart_policy="Never",
+                            service_account_name=self._kwargs["service_account"],
+                            # Terminate the container immediately on SIGTERM
+                            termination_grace_period_seconds=0,
+                            tolerations=[
+                                client.V1Toleration(**toleration)
+                                for toleration in self._kwargs.get("tolerations") or []
+                            ],
+                            volumes=(
+                                [
+                                    client.V1Volume(
+                                        name="tmpfs-ephemeral-volume",
+                                        empty_dir=client.V1EmptyDirVolumeSource(
+                                            medium="Memory",
+                                            # Add default unit as ours differs from Kubernetes default.
+                                            size_limit="{}Mi".format(tmpfs_size),
+                                        ),
+                                    )
+                                ]
+                                if tmpfs_enabled
+                                else []
+                            )
+                            + (
+                                [
+                                    client.V1Volume(
+                                        name=claim,
+                                        persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
+                                            claim_name=claim
+                                        ),
+                                    )
+                                    for claim in self._kwargs[
+                                        "persistent_volume_claims"
+                                    ].keys()
+                                ]
+                                if self._kwargs["persistent_volume_claims"] is not None
+                                else []
+                            ),
+                            # TODO (savin): Set termination_message_policy
                         ),
-                        # TODO (savin): Set termination_message_policy
                     ),
                 ),
-            ),
-        )
+            )
         return self
 
     def execute(self):
@@ -262,19 +605,53 @@ class KubernetesJob(object):
             #       achieve the guarantees that we are seeking.
             #       https://github.com/kubernetes/enhancements/issues/1040
             #       Hopefully, we will be able to get creative with kube-batch
-            response = (
-                client.BatchV1Api()
-                .create_namespaced_job(
-                    body=self._job, namespace=self._kwargs["namespace"]
+
+            if "num_parallel" in self._kwargs and self._kwargs["num_parallel"] >= 1:
+                # TODO (Eddie): this is kinda gross. fix it.
+                if self._kwargs["attrs"]["requires_passwordless_ssh"]:
+                    api_instance = client.CoreV1Api()
+                    api_response = api_instance.create_namespaced_service(namespace=self._kwargs['namespace'], body=self._passwordless_ssh_service)
+
+                with client.ApiClient() as api_client:
+                    api_instance = client.CustomObjectsApi(api_client)
+
+                response = api_instance.create_namespaced_custom_object(
+                    body=self._jobset,
+                    group="jobset.x-k8s.io",
+                    version="v1alpha2",
+                    namespace=self._kwargs["namespace"],
+                    plural="jobsets",
                 )
-                .to_dict()
-            )
-            return RunningJob(
-                client=self._client,
-                name=response["metadata"]["name"],
-                uid=response["metadata"]["uid"],
-                namespace=response["metadata"]["namespace"],
-            )
+
+                # HACK: Give K8s some time to actually create the job
+                time.sleep(10)
+
+                # TODO (Eddie): Remove hack and make RunningJobSet.
+                # There are many jobs running that should be monitored.
+                job_name = "%s-control-0" % response["metadata"]["name"]
+                fake_id = 123
+                return RunningJob(
+                    client=self._client,
+                    name=job_name,
+                    uid=fake_id,
+                    namespace=response["metadata"]["namespace"],
+                )
+
+            else:
+                response = (
+                    client.BatchV1Api()
+                    .create_namespaced_job(
+                        body=self._job, namespace=self._kwargs["namespace"]
+                    )
+                    .to_dict()
+                )
+                return RunningJob(
+                    client=self._client,
+                    name=response["metadata"]["name"],
+                    uid=response["metadata"]["uid"],
+                    namespace=response["metadata"]["namespace"],
+                )
+
         except client.rest.ApiException as e:
             raise KubernetesJobException(
                 "Unable to launch Kubernetes job.\n %s"
@@ -330,7 +707,6 @@ class KubernetesJob(object):
 
 
 class RunningJob(object):
-
     # State Machine implementation for the lifecycle behavior documented in
     # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
     #
@@ -450,7 +826,6 @@ class RunningJob(object):
         client = self._client.get()
         if not self.is_done:
             if self.is_running:
-
                 # Case 1.
                 from kubernetes.stream import stream
 

metaflow/version.py

@@ -1 +1 @@
-metaflow_version = "2.10.11.1"
+metaflow_version = "2.11.0.2"

{ob_metaflow-2.10.11.1.dist-info → ob_metaflow-2.11.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow
-Version: 2.10.11.1
+Version: 2.11.0.2
 Summary: Metaflow: More Data Science, Less Engineering
 Author: Netflix, Outerbounds & the Metaflow Community
 Author-email: help@outerbounds.co

{ob_metaflow-2.10.11.1.dist-info → ob_metaflow-2.11.0.2.dist-info}/RECORD

@@ -15,7 +15,7 @@ metaflow/graph.py,sha256=ZPxyG8uwVMk5YYgX4pQEQaPZtZM5Wy-G4NtJK73IEuA,11818
 metaflow/includefile.py,sha256=BVQLYTLZN7m3ibFnsTU70dPj9YskxZeQb1FosV3k4-o,19721
 metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
 metaflow/lint.py,sha256=_kYAbAtsP7IG1Rd0FqNbo8I8Zs66_0WXbaZJFARO3dE,10394
-metaflow/metaflow_config.py,sha256=XZMXv79h60-yP1c6GD78tcwRq6FX5yLpt2ALRRtoCj4,18986
+metaflow/metaflow_config.py,sha256=LBEDdQskwtstZxhtSP9ONInccjZAjB7nWBrBce_Fpg0,19081
 metaflow/metaflow_config_funcs.py,sha256=pCaiQ2ez9wXixJI3ehmf3QiW9lUqFrZnBZx1my_0wIg,4874
 metaflow/metaflow_environment.py,sha256=JdsmQsYp1SDQniQ0-q1mKRrmzSFfYuzrf6jLEHmyaiM,7352
 metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
@@ -33,7 +33,7 @@ metaflow/task.py,sha256=yGNU3T3giKiG--vE0DUj_K-8jur2TclCS45XjPVLcq4,25314
 metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
 metaflow/util.py,sha256=jbMJ17rK-dFTBCjimWqxkfcr3v__bHa3tZtX0g8iS2c,13257
 metaflow/vendor.py,sha256=LZgXrh7ZSDmD32D1T5jj3OKKpXIqqxKzdMAOc5V0SD4,5162
-metaflow/version.py,sha256=we5fgY997QHteYdrr1V0ZYnlRhuIe9yUxyP2LejQuv8,31
+metaflow/version.py,sha256=gXS_wIDHs2sEK4Lt7UOfOM6t13X5UUPilPOmvUUcpgA,30
 metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
 metaflow/_vendor/click/__init__.py,sha256=FkyGDQ-cbiQxP_lxgUspyFYS48f2S_pTcfKPz-d_RMo,2463
 metaflow/_vendor/click/_bashcomplete.py,sha256=9J98IHQYmCAr2Jup6TDshUr5FJEen-AoQCZR0K5nKxQ,12309
@@ -145,7 +145,7 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=zym4mUm_f_gBsvHHVqGtX_OOxRj
 metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
 metaflow/plugins/argo/argo_events.py,sha256=TIEOHrvUQ079YzzpzpFFtZjeU2x7hoofyZ6ytCorm2c,5911
-metaflow/plugins/argo/argo_workflows.py,sha256=du8R0vI2R0_Z2c-MOAdLldOCkoHl2c-G2mDbARKysV4,119741
+metaflow/plugins/argo/argo_workflows.py,sha256=h-zXFauJce-44eKA8vh9UaW5kIdaN7irz4QwfrfFCNQ,119978
 metaflow/plugins/argo/argo_workflows_cli.py,sha256=sZTpgfmc50eT3e0qIxpVqUgWhTcYlO1HM4gU6Oaya8g,33259
 metaflow/plugins/argo/argo_workflows_decorator.py,sha256=CfKVoHCOsCCQMghhPE30xw15gacwp3hR23HCo9ZZFVg,6580
 metaflow/plugins/argo/process_input_paths.py,sha256=LjUSP8PVU-DRGEPxjas99nzyAO-fI82Bxxbr_QETE88,565
@@ -243,11 +243,11 @@ metaflow/plugins/gcp/gs_tail.py,sha256=Jl_wvnzU7dub07A-DOAuP5FeccNIrPM-CeL1xKFs1
 metaflow/plugins/gcp/gs_utils.py,sha256=ZmIGFse1qYyvAVrwga23PQUzF6dXEDLLsZ2F-YRmvow,2030
 metaflow/plugins/gcp/includefile_support.py,sha256=vIDeR-MiJuUh-2S2pV7Z7FBkhIWwtHXaRrj76MWGRiY,3869
 metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow/plugins/kubernetes/kubernetes.py,sha256=khwVy1r0bPRjcj5t7S9RwhHm_ACmO5qU0-T_gMxGopw,17371
-metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=Y8XNVTumxGkYLs_pFE04w8X0zCEvCSNaIBQx21zETC8,8942
+metaflow/plugins/kubernetes/kubernetes.py,sha256=ePh4vzHDJFkooJKT75zJgipjQlwslyRs1VFcZVYkabE,18834
+metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=RugVe3UHWFGd03OM76fSzxSt3QYAT8KHQ5-iiKzQrGA,10092
 metaflow/plugins/kubernetes/kubernetes_client.py,sha256=dV3TEGQMBbljmv6Gs1EKfmHTorKt21lhSiYsNx0To08,1901
-metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=HTDM5Z-04nQK75vFXFLdW1a4dWSEHEJerKqKc65khMA,21184
-metaflow/plugins/kubernetes/kubernetes_job.py,sha256=XF8_dXhoEyCys-aIZliCdnnwmkXfXQudFJNAp7NM9Oc,30723
+metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=Rs2KGy0yInQmMq9W2jEockiq2eOrrnd1TAMmpu1Q9pA,24103
+metaflow/plugins/kubernetes/kubernetes_job.py,sha256=8LNMwZSz1afbQXrPNJnDo_nTMIA0SQza6yjxkf2N2_k,50853
 metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7ITX12O8,22428
 metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
@@ -295,9 +295,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
 metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
 metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
 metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
-ob_metaflow-2.10.11.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-ob_metaflow-2.10.11.1.dist-info/METADATA,sha256=aLS4SeD3av7VHhDb1fY4m36TlGjoDqvEY2Bb3TmLqIo,5062
-ob_metaflow-2.10.11.1.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
-ob_metaflow-2.10.11.1.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
-ob_metaflow-2.10.11.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
-ob_metaflow-2.10.11.1.dist-info/RECORD,,
+ob_metaflow-2.11.0.2.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+ob_metaflow-2.11.0.2.dist-info/METADATA,sha256=K4SO4xxndoBOLUCcxWPSyonbeAL6FnLqqTLCOq0CSZU,5061
+ob_metaflow-2.11.0.2.dist-info/WHEEL,sha256=-G_t0oGuE7UD0DrSpVZnq1hHMBV9DD2XkS5v7XpmTnk,110
+ob_metaflow-2.11.0.2.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
+ob_metaflow-2.11.0.2.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
+ob_metaflow-2.11.0.2.dist-info/RECORD,,