ob-metaflow-extensions 1.1.69__py2.py3-none-any.whl → 1.1.70__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/__init__.py

@@ -240,5 +240,6 @@ class ObpGcpAuthProvider(object):
 
 
 GCP_CLIENT_PROVIDERS_DESC = [("obp", ".ObpGcpAuthProvider")]
-
+CLIS_DESC = [("nvcf", ".nvcf.nvcf_cli.cli")]
+STEP_DECORATORS_DESC = [("nvidia", ".nvcf.nvcf_decorator.NvcfDecorator")]
 FLOW_DECORATORS_DESC = [("nim", ".nim.NimDecorator")]

metaflow_extensions/outerbounds/plugins/nvcf/nvcf.py

@@ -0,0 +1,247 @@
+import json
+import os
+import time
+from urllib.request import HTTPError, Request, URLError, urlopen
+
+from metaflow import util
+from metaflow.exception import MetaflowException
+from metaflow.mflog import (
+    BASH_SAVE_LOGS,
+    bash_capture_logs,
+    export_mflog_env_vars,
+    tail_logs,
+    get_log_tailer,
+)
+import requests
+from metaflow.metaflow_config_funcs import init_config
+
+
+class NvcfException(MetaflowException):
+    headline = "Nvidia error"
+
+
+class NvcfKilledException(MetaflowException):
+    headline = "Nvidia job killed"
+
+
+# Redirect structured logs to $PWD/.logs/
+LOGS_DIR = "$PWD/.logs"
+STDOUT_FILE = "mflog_stdout"
+STDERR_FILE = "mflog_stderr"
+STDOUT_PATH = os.path.join(LOGS_DIR, STDOUT_FILE)
+STDERR_PATH = os.path.join(LOGS_DIR, STDERR_FILE)
+
+
+class Nvcf(object):
+    def __init__(self, metadata, datastore, environment):
+        self.metadata = metadata
+        self.datastore = datastore
+        self.environment = environment
+
+    def launch_job(
+        self,
+        step_name,
+        step_cli,
+        task_spec,
+        code_package_sha,
+        code_package_url,
+        code_package_ds,
+        env={},
+    ):
+        mflog_expr = export_mflog_env_vars(
+            datastore_type=code_package_ds,
+            stdout_path=STDOUT_PATH,
+            stderr_path=STDERR_PATH,
+            **task_spec,
+        )
+        init_cmds = self.environment.get_package_commands(
+            code_package_url, code_package_ds
+        )
+        init_expr = " && ".join(init_cmds)
+        step_expr = bash_capture_logs(
+            " && ".join(
+                self.environment.bootstrap_commands(step_name, code_package_ds)
+                + [step_cli]
+            )
+        )
+
+        # construct an entry point that
+        # 1) initializes the mflog environment (mflog_expr)
+        # 2) bootstraps a metaflow environment (init_expr)
+        # 3) executes a task (step_expr)
+
+        cmd_str = "mkdir -p %s && %s && %s && %s; " % (
+            LOGS_DIR,
+            mflog_expr,
+            init_expr,
+            step_expr,
+        )
+        # after the task has finished, we save its exit code (fail/success)
+        # and persist the final logs. The whole entrypoint should exit
+        # with the exit code (c) of the task.
+        #
+        # Note that if step_expr OOMs, this tail expression is never executed.
+        # We lose the last logs in this scenario.
+        cmd_str += "c=$?; %s; exit $c" % BASH_SAVE_LOGS
+        cmd_str = (
+            '${METAFLOW_INIT_SCRIPT:+eval \\"${METAFLOW_INIT_SCRIPT}\\"} && %s'
+            % cmd_str
+        )
+        self.job = Job('bash -c "%s"' % cmd_str, env)
+        self.job.submit()
+
+    def wait(self, stdout_location, stderr_location, echo=None):
+        def wait_for_launch(job):
+            status = job.status
+            echo(
+                "Task status: %s..." % status,
+                "stderr",
+                _id=job.id,
+            )
+
+        prefix = b"[%s] " % util.to_bytes(self.job.id)
+        stdout_tail = get_log_tailer(stdout_location, self.datastore.TYPE)
+        stderr_tail = get_log_tailer(stderr_location, self.datastore.TYPE)
+
+        # 1) Loop until the job has started
+        wait_for_launch(self.job)
+
+        # 2) Tail logs until the job has finished
+        tail_logs(
+            prefix=prefix,
+            stdout_tail=stdout_tail,
+            stderr_tail=stderr_tail,
+            echo=echo,
+            has_log_updates=lambda: self.job.is_running,
+        )
+
+        echo(
+            "Task finished with exit code %s." % self.job.result.get("exit_code"),
+            "stderr",
+            _id=self.job.id,
+        )
+        if self.job.has_failed:
+            raise NvcfException("This could be a transient error. Use @retry to retry.")
+
+
+class JobStatus(object):
+    SUBMITTED = "SUBMITTED"
+    RUNNING = "RUNNING"
+    SUCCESSFUL = "SUCCESSFUL"
+    FAILED = "FAILED"
+
+
+nvcf_url = "https://api.nvcf.nvidia.com"
+submit_endpoint = f"{nvcf_url}/v2/nvcf/pexec/functions"
+result_endpoint = f"{nvcf_url}/v2/nvcf/pexec/status"
+
+
+class Job(object):
+    def __init__(self, command, env):
+
+        self._payload = {
+            "command": command,
+            "env": {k: v for k, v in env.items() if v is not None},
+        }
+        self._result = {}
+
+        conf = init_config()
+        if "OBP_AUTH_SERVER" in conf:
+            auth_host = conf["OBP_AUTH_SERVER"]
+        else:
+            auth_host = "auth." + urlparse(SERVICE_URL).hostname.split(".", 1)[1]
+
+        # NOTE: reusing the same auth_host as the one used in NimMetadata,
+        # however, user should not need to use nim container to use @nvidia.
+        # May want to refactor this to a common endpoint.
+        nim_info_url = "https://" + auth_host + "/generate/nim"
+
+        if "METAFLOW_SERVICE_AUTH_KEY" in conf:
+            headers = {"x-api-key": conf["METAFLOW_SERVICE_AUTH_KEY"]}
+            res = requests.get(nim_info_url, headers=headers)
+        else:
+            headers = json.loads(os.environ.get("METAFLOW_SERVICE_HEADERS"))
+            res = requests.get(nim_info_url, headers=headers)
+
+        res.raise_for_status()
+        self._ngc_api_key = res.json()["nvcf"]["api_key"]
+
+        for f in res.json()["nvcf"]["functions"]:
+            if f["model_key"] == "metaflow_task_executor":
+                self._function_id = f["id"]
+
+    def submit(self):
+        try:
+            headers = {
+                "Authorization": f"Bearer {self._ngc_api_key}",
+                "Content-Type": "application/json",
+            }
+            request_data = json.dumps(self._payload).encode()
+            request = Request(
+                f"{submit_endpoint}/{self._function_id}",
+                data=request_data,
+                headers=headers,
+            )
+            response = urlopen(request)
+            self._invocation_id = response.headers.get("NVCF-REQID")
+            if response.getcode() == 200:
+                data = json.loads(response.read())
+                if data["status"].startswith("Oops"):
+                    self._status = JobStatus.FAILED
+                else:
+                    self._status = JobStatus.SUCCESSFUL
+                self._result = data
+            elif response.getcode() == 202:
+                self._status = JobStatus.SUBMITTED
+            else:
+                self._status = JobStatus.FAILED
+            # TODO: Handle 404s nicely
+        except (HTTPError, URLError) as e:
+            self._state = JobStatus.FAILED
+            raise e
+
+    @property
+    def status(self):
+        if self._status not in [JobStatus.SUCCESSFUL, JobStatus.FAILED]:
+            self._poll()
+        return self._status
+
+    @property
+    def id(self):
+        return self._invocation_id
+
+    @property
+    def is_running(self):
+        return self.status == JobStatus.SUBMITTED
+
+    @property
+    def has_failed(self):
+        return self.status == JobStatus.FAILED
+
+    @property
+    def result(self):
+        return self._result
+
+    def _poll(self):
+        try:
+            invocation_id = self._invocation_id
+            headers = {
+                "Authorization": f"Bearer {self._ngc_api_key}",
+                "Content-Type": "application/json",
+            }
+            request = Request(
+                f"{result_endpoint}/{self._invocation_id}", headers=headers
+            )
+            response = urlopen(request)
+            if response.getcode() == 200:
+                data = json.loads(response.read())
+                if data["status"].startswith("Oops"):
+                    # TODO: Propagate the internal error forward
+                    self._status = JobStatus.FAILED
+                else:
+                    self._status = JobStatus.SUCCESSFUL
+                self._result = data
+            elif response.getcode() in [400, 500]:
+                self._status = JobStatus.FAILED
+        except (HTTPError, URLError) as e:
+            print(f"Error occurred while polling for result: {e}")

metaflow_extensions/outerbounds/plugins/nvcf/nvcf_cli.py

@@ -0,0 +1,202 @@
+import json
+import os
+import sys
+import time
+import traceback
+
+from metaflow import util
+from metaflow._vendor import click
+from metaflow.exception import METAFLOW_EXIT_DISALLOW_RETRY
+from metaflow.metadata.util import sync_local_metadata_from_datastore
+from metaflow.metaflow_config import (
+    CARD_S3ROOT,
+    DATASTORE_LOCAL_DIR,
+    DATASTORE_SYSROOT_S3,
+    DATATOOLS_S3ROOT,
+    DEFAULT_METADATA,
+    SERVICE_HEADERS,
+    SERVICE_URL,
+    DEFAULT_SECRETS_BACKEND_TYPE,
+    DEFAULT_AWS_CLIENT_PROVIDER,
+    AWS_SECRETS_MANAGER_DEFAULT_REGION,
+    S3_ENDPOINT_URL,
+    AZURE_STORAGE_BLOB_SERVICE_ENDPOINT,
+    DATASTORE_SYSROOT_AZURE,
+    CARD_AZUREROOT,
+    DATASTORE_SYSROOT_GS,
+    CARD_GSROOT,
+    KUBERNETES_SANDBOX_INIT_SCRIPT,
+    OTEL_ENDPOINT,
+)
+from metaflow.mflog import TASK_LOG_SOURCE
+
+from .nvcf import Nvcf, NvcfKilledException
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.group(help="Commands related to NVCF.")
+def nvcf():
+    pass
+
+
+@nvcf.command(
+    help="Execute a single task using NVCF. This command calls the "
+    "top-level step command inside a NVCF job with the given options. "
+    "Typically you do not call this command directly; it is used internally by "
+    "Metaflow."
+)
+@click.argument("step-name")
+@click.argument("code-package-sha")
+@click.argument("code-package-url")
+@click.option("--run-id", help="Passed to the top-level 'step'.")
+@click.option("--task-id", help="Passed to the top-level 'step'.")
+@click.option("--input-paths", help="Passed to the top-level 'step'.")
+@click.option("--split-index", help="Passed to the top-level 'step'.")
+@click.option("--clone-path", help="Passed to the top-level 'step'.")
+@click.option("--clone-run-id", help="Passed to the top-level 'step'.")
+@click.option(
+    "--tag", multiple=True, default=None, help="Passed to the top-level 'step'."
+)
+@click.option("--namespace", default=None, help="Passed to the top-level 'step'.")
+@click.option("--retry-count", default=0, help="Passed to the top-level 'step'.")
+@click.option(
+    "--max-user-code-retries", default=0, help="Passed to the top-level 'step'."
+)
+@click.pass_context
+def step(ctx, step_name, code_package_sha, code_package_url, **kwargs):
+    def echo(msg, stream="stderr", _id=None, **kwargs):
+        msg = util.to_unicode(msg)
+        if _id:
+            msg = "[%s] %s" % (_id, msg)
+        ctx.obj.echo_always(msg, err=(stream == sys.stderr), **kwargs)
+
+    executable = ctx.obj.environment.executable(step_name)
+    entrypoint = "%s -u %s" % (executable, os.path.basename(sys.argv[0]))
+
+    top_args = " ".join(util.dict_to_cli_options(ctx.parent.parent.params))
+
+    input_paths = kwargs.get("input_paths")
+    split_vars = None
+    if input_paths:
+        max_size = 30 * 1024
+        split_vars = {
+            "METAFLOW_INPUT_PATHS_%d" % (i // max_size): input_paths[i : i + max_size]
+            for i in range(0, len(input_paths), max_size)
+        }
+        kwargs["input_paths"] = "".join("${%s}" % s for s in split_vars.keys())
+
+    step_args = " ".join(util.dict_to_cli_options(kwargs))
+    step_cli = "{entrypoint} {top_args} step {step} {step_args}".format(
+        entrypoint=entrypoint,
+        top_args=top_args,
+        step=step_name,
+        step_args=step_args,
+    )
+    node = ctx.obj.graph[step_name]
+
+    # Get retry information
+    retry_count = kwargs.get("retry_count", 0)
+    retry_deco = [deco for deco in node.decorators if deco.name == "retry"]
+    minutes_between_retries = None
+    if retry_deco:
+        minutes_between_retries = int(
+            retry_deco[0].attributes.get("minutes_between_retries", 1)
+        )
+
+    task_spec = {
+        "flow_name": ctx.obj.flow.name,
+        "step_name": step_name,
+        "run_id": kwargs["run_id"],
+        "task_id": kwargs["task_id"],
+        "retry_count": str(retry_count),
+    }
+
+    env = {
+        "METAFLOW_CODE_SHA": code_package_sha,
+        "METAFLOW_CODE_URL": code_package_url,
+        "METAFLOW_CODE_DS": ctx.obj.flow_datastore.TYPE,
+        "METAFLOW_SERVICE_URL": SERVICE_URL,
+        "METAFLOW_SERVICE_HEADERS": json.dumps(SERVICE_HEADERS),
+        "METAFLOW_DATASTORE_SYSROOT_S3": DATASTORE_SYSROOT_S3,
+        "METAFLOW_DATATOOLS_S3ROOT": DATATOOLS_S3ROOT,
+        "METAFLOW_DEFAULT_DATASTORE": ctx.obj.flow_datastore.TYPE,
+        "METAFLOW_USER": util.get_username(),
+        "METAFLOW_DEFAULT_METADATA": DEFAULT_METADATA,
+        "METAFLOW_CARD_S3ROOT": CARD_S3ROOT,
+        "METAFLOW_RUNTIME_ENVIRONMENT": "nvcf",
+        "METAFLOW_DEFAULT_SECRETS_BACKEND_TYPE": DEFAULT_SECRETS_BACKEND_TYPE,
+        "METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER": DEFAULT_AWS_CLIENT_PROVIDER,
+        "METAFLOW_AWS_SECRETS_MANAGER_DEFAULT_REGION": AWS_SECRETS_MANAGER_DEFAULT_REGION,
+        "METAFLOW_S3_ENDPOINT_URL": S3_ENDPOINT_URL,
+        "METAFLOW_AZURE_STORAGE_BLOB_SERVICE_ENDPOINT": AZURE_STORAGE_BLOB_SERVICE_ENDPOINT,
+        "METAFLOW_DATASTORE_SYSROOT_AZURE": DATASTORE_SYSROOT_AZURE,
+        "METAFLOW_CARD_AZUREROOT": CARD_AZUREROOT,
+        "METAFLOW_DATASTORE_SYSROOT_GS": DATASTORE_SYSROOT_GS,
+        "METAFLOW_CARD_GSROOT": CARD_GSROOT,
+        "METAFLOW_INIT_SCRIPT": KUBERNETES_SANDBOX_INIT_SCRIPT,
+        "METAFLOW_OTEL_ENDPOINT": OTEL_ENDPOINT,
+    }
+
+    env_deco = [deco for deco in node.decorators if deco.name == "environment"]
+    if env_deco:
+        env.update(env_deco[0].attributes["vars"])
+
+    # Add the environment variables related to the input-paths argument
+    if split_vars:
+        env.update(split_vars)
+
+    if retry_count:
+        ctx.obj.echo_always(
+            "Sleeping %d minutes before the next retry" % minutes_between_retries
+        )
+        time.sleep(minutes_between_retries * 60)
+
+    # this information is needed for log tailing
+    ds = ctx.obj.flow_datastore.get_task_datastore(
+        mode="w",
+        run_id=kwargs["run_id"],
+        step_name=step_name,
+        task_id=kwargs["task_id"],
+        attempt=int(retry_count),
+    )
+    stdout_location = ds.get_log_location(TASK_LOG_SOURCE, "stdout")
+    stderr_location = ds.get_log_location(TASK_LOG_SOURCE, "stderr")
+
+    def _sync_metadata():
+        if ctx.obj.metadata.TYPE == "local":
+            sync_local_metadata_from_datastore(
+                DATASTORE_LOCAL_DIR,
+                ctx.obj.flow_datastore.get_task_datastore(
+                    kwargs["run_id"], step_name, kwargs["task_id"]
+                ),
+            )
+
+    nvcf = Nvcf(ctx.obj.metadata, ctx.obj.flow_datastore, ctx.obj.environment)
+    try:
+        with ctx.obj.monitor.measure("metaflow.nvcf.launch_job"):
+            nvcf.launch_job(
+                step_name,
+                step_cli,
+                task_spec,
+                code_package_sha,
+                code_package_url,
+                ctx.obj.flow_datastore.TYPE,
+                # function_id=function_id,
+                env=env,
+            )
+    except Exception as e:
+        traceback.print_exc()
+        _sync_metadata()
+        sys.exit(METAFLOW_EXIT_DISALLOW_RETRY)
+    try:
+        nvcf.wait(stdout_location, stderr_location, echo=echo)
+    except NvcfKilledException:
+        # don't retry killed tasks
+        traceback.print_exc()
+        sys.exit(METAFLOW_EXIT_DISALLOW_RETRY)
+    finally:
+        _sync_metadata()

metaflow_extensions/outerbounds/plugins/nvcf/nvcf_decorator.py

@@ -0,0 +1,156 @@
+import os
+import sys
+
+from metaflow.decorators import StepDecorator
+from metaflow.metadata.util import sync_local_metadata_to_datastore
+from metaflow.metaflow_config import DATASTORE_LOCAL_DIR
+from metaflow.sidecar import Sidecar
+from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task
+from .nvcf import NvcfException
+
+from metaflow.metadata import MetaDatum
+
+
+class NvcfDecorator(StepDecorator):
+    name = "nvidia"
+    # defaults = {"function_id": "9e5647f2-740f-4101-a129-1c961a075575"}
+    defaults = {}
+    # "0817006f-018b-4590-b2a5-6cf9d64d9d9a"}
+    #
+
+    package_url = None
+    package_sha = None
+
+    # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png
+    # to understand where these functions are invoked in the lifecycle of a
+    # Metaflow flow.
+    def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
+        # Executing NVCF functions requires a non-local datastore.
+        if flow_datastore.TYPE not in ("s3", "azure", "gs"):
+            raise NvcfException(
+                "The *@nvidia* decorator requires --datastore=s3 or --datastore=azure or --datastore=gs at the moment."
+            )
+        # if self.attributes["function_id"] is None:
+        #     raise NvcfException(
+        #         "The *@nvidia* decorator requires a function_id. Please reach out to Outerbounds if you are unsure how to get access to one."
+        #     )
+        # Set internal state.
+        self.logger = logger
+        self.environment = environment
+        self.step = step
+        self.flow_datastore = flow_datastore
+
+        # TODO:
+        # 1. Ensure that @batch and @kubernetes decorators are not applied to this step.
+        # 2. Ensure @parallel is not applied to this step.
+
+        # Set run time limit for the NVCF function.
+        self.run_time_limit = get_run_time_limit_for_task(decos)
+        if self.run_time_limit < 60:
+            raise NvcfException(
+                "The timeout for step *{step}* should be at least 60 seconds for "
+                "execution with @nvidia.".format(step=step)
+            )
+
+    def runtime_init(self, flow, graph, package, run_id):
+        # Set some more internal state.
+        self.flow = flow
+        self.graph = graph
+        self.package = package
+        self.run_id = run_id
+
+    def runtime_task_created(
+        self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context
+    ):
+        if not is_cloned:
+            self._save_package_once(self.flow_datastore, self.package)
+
+    def runtime_step_cli(
+        self, cli_args, retry_count, max_user_code_retries, ubf_context
+    ):
+        if retry_count <= max_user_code_retries:
+            # after all attempts to run the user code have failed, we don't need
+            # to execute on NVCF anymore. We can execute possible fallback
+            # code locally.
+            cli_args.commands = ["nvcf", "step"]
+            cli_args.command_args.append(self.package_sha)
+            cli_args.command_args.append(self.package_url)
+            cli_args.command_options.update(self.attributes)
+            # cli_args.command_options["run-time-limit"] = self.run_time_limit
+            cli_args.entrypoint[0] = sys.executable
+
+    def task_pre_step(
+        self,
+        step_name,
+        task_datastore,
+        metadata,
+        run_id,
+        task_id,
+        flow,
+        graph,
+        retry_count,
+        max_retries,
+        ubf_context,
+        inputs,
+    ):
+        self.metadata = metadata
+        self.task_datastore = task_datastore
+
+        # task_pre_step may run locally if fallback is activated for @catch
+        # decorator.
+
+        if "NVCF_CONTEXT" in os.environ:
+            meta = {}
+
+            meta["nvcf-function-id"] = os.environ.get("NVCF_FUNCTION_ID")
+            meta["nvcf-function-version-id"] = os.environ.get(
+                "NVCF_FUNCTION_VERSION_ID"
+            )
+            meta["nvcf-region"] = os.environ.get("NVCF_REGION")
+            meta["nvcf-ncaid"] = os.environ.get("NVCF_NCAID")
+            meta["nvcf-sub"] = os.environ.get("NVCF_SUB")
+            meta["nvcf-instancetype"] = os.environ.get("NVCF_INSTANCETYPE")
+            meta["nvcf-reqid"] = os.environ.get("NVCF_REQID")
+            meta["nvcf-env"] = os.environ.get("NVCF_ENV")
+            meta["nvcf-backend"] = os.environ.get("NVCF_BACKEND")
+            meta["nvcf-function-name"] = os.environ.get("NVCF_FUNCTION_NAME")
+            meta["nvcf-nspectid"] = os.environ.get("NVCF_NSPECTID")
+
+            entries = [
+                MetaDatum(field=k, value=v, type=k, tags=[])
+                for k, v in meta.items()
+                if v is not None
+            ]
+            # Register book-keeping metadata for debugging.
+            metadata.register_metadata(run_id, step_name, task_id, entries)
+
+            self._save_logs_sidecar = Sidecar("save_logs_periodically")
+            self._save_logs_sidecar.start()
+
+    def task_finished(
+        self, step_name, flow, graph, is_task_ok, retry_count, max_retries
+    ):
+        # task_finished may run locally if fallback is activated for @catch
+        # decorator.
+        if "NVCF_CONTEXT" in os.environ:
+            # If `local` metadata is configured, we would need to copy task
+            # execution metadata from the NVCF container to user's
+            # local file system after the user code has finished execution.
+            # This happens via datastore as a communication bridge.
+            if hasattr(self, "metadata") and self.metadata.TYPE == "local":
+                sync_local_metadata_to_datastore(
+                    DATASTORE_LOCAL_DIR, self.task_datastore
+                )
+
+        try:
+            self._save_logs_sidecar.terminate()
+        except:
+            # Best effort kill
+            pass
+
+    @classmethod
+    def _save_package_once(cls, flow_datastore, package):
+        if cls.package_url is None:
+            cls.package_url, cls.package_sha = flow_datastore.save_data(
+                [package.blob], len_hint=1
+            )[0]

{ob_metaflow_extensions-1.1.69.dist-info → ob_metaflow_extensions-1.1.70.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.69
+Version: 1.1.70
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob_metaflow_extensions-1.1.69.dist-info → ob_metaflow_extensions-1.1.70.dist-info}/RECORD

@@ -1,13 +1,17 @@
 metaflow_extensions/outerbounds/__init__.py,sha256=TRGvIUMjkfneWtYUFSWoubu_Kf2ekAL4WLbV3IxOj9k,499
 metaflow_extensions/outerbounds/remote_config.py,sha256=HPFH4e3ZK3p-wS5HlS75fhR8_2avdD1AHQIZl2KnjeQ,4059
 metaflow_extensions/outerbounds/config/__init__.py,sha256=mYo95obHU1IE1wbPkeVz_pfTzNqlNabp1QBEMTGllbE,112
-metaflow_extensions/outerbounds/plugins/__init__.py,sha256=oR3krG5x3-W4g1sm5ygNPe9KVLBmxg7KEtzJsonQo_4,9398
+metaflow_extensions/outerbounds/plugins/__init__.py,sha256=46NgbJBhVowDR6FyQrZPF2jHHqRTSyCBCYIQAyQ4Ryo,9516
 metaflow_extensions/outerbounds/plugins/auth_server.py,sha256=JhlMFcR7SPSfR1C9w6GlqJq-NYNhOfISmHl2PdkYUok,2212
 metaflow_extensions/outerbounds/plugins/perimeters.py,sha256=z8tSAkWtiITB-JtSQS7fkhlBwvxSxeTgEwFjahAzv-U,2238
 metaflow_extensions/outerbounds/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
 metaflow_extensions/outerbounds/plugins/kubernetes/kubernetes_client.py,sha256=gj6Iaz26bGbZm3aQuNS18Mqh_80iJp5PgFwFSlJRcn8,1968
 metaflow_extensions/outerbounds/plugins/nim/__init__.py,sha256=GVnvSTjqYVj5oG2yh8KJFt7iZ33cEadDD5HbdmC9hJ0,1457
 metaflow_extensions/outerbounds/plugins/nim/nim_manager.py,sha256=l8WDfVtsMt7aZaOaeIPT5ySidxfxXU8gmwLoKUP3f04,7044
+metaflow_extensions/outerbounds/plugins/nvcf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+metaflow_extensions/outerbounds/plugins/nvcf/nvcf.py,sha256=ftxC5SCo64P5Ycpv5vudluTnQi3-VCZW0umdsPP326A,7926
+metaflow_extensions/outerbounds/plugins/nvcf/nvcf_cli.py,sha256=ow3lonclEDoZEUQCDV_L8lEr6HopXqjNXzubRrfdIm4,7219
+metaflow_extensions/outerbounds/plugins/nvcf/nvcf_decorator.py,sha256=0xNA4aRTPJ4SKpRIFKZzlL9a7lf367KGTrVWVXd-uGE,6052
 metaflow_extensions/outerbounds/profilers/__init__.py,sha256=wa_jhnCBr82TBxoS0e8b6_6sLyZX0fdHicuGJZNTqKw,29
 metaflow_extensions/outerbounds/profilers/gpu.py,sha256=a5YZAepujuP0uDqG9UpXBlZS3wjUt4Yv8CjybXqeT2c,24342
 metaflow_extensions/outerbounds/toplevel/__init__.py,sha256=qWUJSv_r5hXJ7jV_On4nEasKIfUCm6_UjkjXWA_A1Ts,90
@@ -15,7 +19,7 @@ metaflow_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.py,
 metaflow_extensions/outerbounds/toplevel/plugins/azure/__init__.py,sha256=WUuhz2YQfI4fz7nIcipwwWq781eaoHEk7n4GAn1npDg,63
 metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3uILlEZ6ntBLKeNyqn3If8nIXZFq_Apd7Dhco,70
 metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
-ob_metaflow_extensions-1.1.69.dist-info/METADATA,sha256=Xw60qLJNjigrBisO_T1GbcUQja8A2r637ntXJy6NpCA,519
-ob_metaflow_extensions-1.1.69.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-ob_metaflow_extensions-1.1.69.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
-ob_metaflow_extensions-1.1.69.dist-info/RECORD,,
+ob_metaflow_extensions-1.1.70.dist-info/METADATA,sha256=oh54d2W1t23zvb-nWBXXp5CiCTn2wRpUKVrtgWERn5c,519
+ob_metaflow_extensions-1.1.70.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+ob_metaflow_extensions-1.1.70.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
+ob_metaflow_extensions-1.1.70.dist-info/RECORD,,