ob-metaflow-extensions 1.1.158__py2.py3-none-any.whl → 1.1.160__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/__init__.py

@@ -347,4 +347,4 @@ SECRETS_PROVIDERS_DESC = [
     ("outerbounds", ".secrets.secrets.OuterboundsSecretsProvider"),
 ]
 # Adding an override here so the library can be imported at the metaflow.plugins level
-__mf_promote_submodules__ = ["snowflake", "ollama"]
+__mf_promote_submodules__ = ["snowflake", "ollama", "torchtune"]

metaflow_extensions/outerbounds/plugins/kubernetes/kubernetes_client.py

@@ -5,6 +5,7 @@ import time
 
 from metaflow.exception import MetaflowException
 from metaflow.metaflow_config import KUBERNETES_NAMESPACE
+from .pod_killer import PodKiller
 
 
 CLIENT_REFRESH_INTERVAL_SECONDS = 300
@@ -105,50 +106,23 @@ class KubernetesClient(object):
         return list(results)
 
     def kill_pods(self, flow_name, run_id, user, echo):
-        from kubernetes.stream import stream
-
-        api_instance = self._client.CoreV1Api()
-        job_api = self._client.BatchV1Api()
-        pods = self._find_active_pods(flow_name, run_id, user)
-
-        def _kill_pod(pod):
-            echo("Killing Kubernetes pod %s\n" % pod.metadata.name)
-            try:
-                stream(
-                    api_instance.connect_get_namespaced_pod_exec,
-                    name=pod.metadata.name,
-                    namespace=pod.metadata.namespace,
-                    command=[
-                        "/bin/sh",
-                        "-c",
-                        "/sbin/killall5",
-                    ],
-                    stderr=True,
-                    stdin=False,
-                    stdout=True,
-                    tty=False,
-                )
-            except Exception:
-                # best effort kill for pod can fail.
-                try:
-                    job_name = pod.metadata.labels.get("job-name", None)
-                    if job_name is None:
-                        raise Exception("Could not determine job name")
-
-                    job_api.patch_namespaced_job(
-                        name=job_name,
-                        namespace=pod.metadata.namespace,
-                        field_manager="metaflow",
-                        body={"spec": {"parallelism": 0}},
-                    )
-                except Exception as e:
-                    echo("failed to kill pod %s - %s" % (pod.metadata.name, str(e)))
-
-        with ThreadPoolExecutor() as executor:
-            operated_pods = list(executor.map(_kill_pod, pods))
-
-            if not operated_pods:
-                echo("No active Kubernetes pods found for run *%s*" % run_id)
+        # Create PodKiller instance
+        killer = PodKiller(self._client, echo, self._namespace)
+
+        # Process all matching jobs and jobsets based on their outcomes
+        (
+            job_jobset_results,
+            num_jobs,
+            num_jobsets,
+        ) = killer.process_matching_jobs_and_jobsets(flow_name, run_id, user)
+
+        if job_jobset_results:
+            successful_operations = sum(1 for result in job_jobset_results if result)
+            echo(
+                f"Found and processed {num_jobs} jobs and {num_jobsets} jobsets, {successful_operations} operations successful\n"
+            )
+        else:
+            echo("No matching jobs or jobsets found for run *%s*" % run_id)
 
     def job(self, **kwargs):
         from metaflow.plugins.kubernetes.kubernetes_job import KubernetesJob

metaflow_extensions/outerbounds/plugins/kubernetes/pod_killer.py

@@ -0,0 +1,296 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from kubernetes.client.models.v1_job import V1Job
+    from kubernetes.client.models.v1_job_status import V1JobStatus
+
+
+def _is_jobset_child(job: "V1Job"):
+    if job.metadata.owner_references:
+        for owner_ref in job.metadata.owner_references:
+            if owner_ref.kind == "JobSet":
+                return owner_ref
+    return None
+
+
+class JobOutcomes:
+    KILL = "kill"
+    DELETE = "delete"
+    LEAVE_UNCHANGED = "leave_unchanged"
+
+
+def derive_jobset_outcome(jobset_status):
+    return (
+        JobOutcomes.LEAVE_UNCHANGED
+        if jobset_status.get("terminalState", None)
+        else JobOutcomes.DELETE
+    )
+
+
+def derive_job_outcome(job_status: "V1JobStatus"):
+    if job_status.start_time is None:
+        # If the job has not started even then just wipe it!
+        return JobOutcomes.DELETE
+    if job_status.succeeded or job_status.failed:
+        return JobOutcomes.LEAVE_UNCHANGED
+
+    if job_status.completion_time is not None:
+        return JobOutcomes.LEAVE_UNCHANGED
+
+    # This means that the job has neither finished or succedded.
+    if job_status.active:
+        return JobOutcomes.KILL
+
+    # This means that the job is not active. Had started. There is not succedded/fail.
+    # This is a weird state. Better to just kill the job
+    return JobOutcomes.DELETE
+
+
+class PodKiller:
+    def __init__(self, kubernetes_client, echo_func, namespace):
+        self.client = kubernetes_client
+        self.echo = echo_func
+        self.api_instance = self.client.CoreV1Api()
+        self.job_api = self.client.BatchV1Api()
+        self._namespace = namespace
+        self.jobset_api = None
+        self.jobset_api = self.client.CustomObjectsApi()
+
+    def _delete_jobset(self, owner_ref, namespace):
+        """Delete a JobSet if it's the owner of a job."""
+        if not self.jobset_api:
+            self.echo("JobSet API not available, cannot delete JobSet\n")
+            return False
+
+        try:
+            jobset_name = owner_ref.name
+            self.echo(f"Deleting JobSet: {jobset_name}\n")
+
+            self.jobset_api.delete_namespaced_custom_object(
+                group="jobset.x-k8s.io",
+                version="v1alpha2",
+                namespace=namespace,
+                plural="jobsets",
+                name=jobset_name,
+            )
+            return True
+        except Exception as e:
+            self.echo(f"Failed to delete JobSet {owner_ref.name}: {str(e)}\n")
+            return False
+
+    def _delete_job(self, job_name, namespace):
+        """Delete a Batch Job and check for JobSet owner reference."""
+        try:
+            # First get the job to check for owner references
+            job = self.job_api.read_namespaced_job(name=job_name, namespace=namespace)
+            # Check for JobSet owner reference
+            jobset_ref = _is_jobset_child(job)
+            if jobset_ref:
+                if self._delete_jobset(jobset_ref, namespace):
+                    return True
+
+            # If no JobSet owner or JobSet deletion failed, delete the job
+            self.echo(f"Deleting Batch Job: {job_name}")
+            self.job_api.delete_namespaced_job(
+                name=job_name, namespace=namespace, propagation_policy="Background"
+            )
+            return True
+
+        except Exception as e:
+            self.echo(f"Failed to delete job {job_name}: {str(e)}")
+            return False
+
+    def _kill_pod_process(self, pod):
+        """Attempt to kill processes inside a pod."""
+        from kubernetes.stream import stream
+
+        try:
+            stream(
+                self.api_instance.connect_get_namespaced_pod_exec,
+                name=pod.metadata.name,
+                namespace=pod.metadata.namespace,
+                command=["/bin/sh", "-c", "/sbin/killall5"],
+                stderr=True,
+                stdin=False,
+                stdout=True,
+                tty=False,
+            )
+            return True
+        except Exception as e:
+            self.echo(
+                f"Failed to kill processes in pod {pod.metadata.name}: {str(e)}\n"
+            )
+            return False
+
+    @staticmethod
+    def _metaflow_matching_spec(run_id, user, flow_name, annotations, labels):
+        # Handle argo prefixes in run_id like in _find_active_pods
+        _argo_run_id = None
+        if run_id is not None:
+            _argo_run_id = run_id[run_id.startswith("argo-") and len("argo-") :]
+        return (
+            annotations
+            and (
+                run_id is None
+                or (annotations.get("metaflow/run_id") == run_id)
+                # we want to also match jobsets launched by argo-workflows
+                # This line has no real value since the We already avoid any
+                # argo-workflows related terminations.
+                or (
+                    labels.get("workflows.argoproj.io/workflow") is not None
+                    and labels.get("workflows.argoproj.io/workflow") == _argo_run_id
+                )
+            )
+            and (user is None or annotations.get("metaflow/user") == user)
+            and (annotations.get("metaflow/flow_name") == flow_name)
+        )
+
+    def _find_matching_jobs(self, flow_name, run_id=None, user=None):
+        """Find jobs that match the flow_name, run_id, and user criteria using similar logic to _find_active_pods"""
+        try:
+            jobs = self.job_api.list_namespaced_job(namespace=self._namespace)
+            matching_jobs = []
+            for _job in jobs.items:
+                job = _job.to_dict()
+                _match = self._metaflow_matching_spec(
+                    run_id=run_id,
+                    user=user,
+                    flow_name=flow_name,
+                    annotations=job.get("metadata", {}).get("annotations", {}),
+                    labels=job.get("metadata", {}).get("labels", {}),
+                )
+                if _match:
+                    matching_jobs.append(_job)
+            return matching_jobs
+        except Exception as e:
+            self.echo(f"Error finding jobs: {str(e)}\n")
+            return []
+
+    def _find_matching_jobsets(self, flow_name, run_id=None, user=None):
+        """Find jobsets that match the flow_name, run_id, and user criteria using similar logic to _find_active_pods"""
+        if not self.jobset_api:
+            return []
+
+        try:
+            jobsets = self.jobset_api.list_namespaced_custom_object(
+                group="jobset.x-k8s.io",
+                version="v1alpha2",
+                namespace=self._namespace,
+                plural="jobsets",
+            )
+            matching_jobsets = []
+
+            for jobset in jobsets.get("items", []):
+                _match = self._metaflow_matching_spec(
+                    run_id=run_id,
+                    user=user,
+                    flow_name=flow_name,
+                    annotations=jobset.get("metadata", {}).get("annotations", {}),
+                    labels=jobset.get("metadata", {}).get("labels", {}),
+                )
+                if _match:
+                    matching_jobsets.append(jobset)
+
+            return matching_jobsets
+        except Exception as e:
+            self.echo(f"Error finding jobsets: {str(e)}\n")
+            return []
+
+    def _kill_pods_for_job(self, job):
+        """Find and kill pods associated with a specific job"""
+        job_name = job.metadata.name
+        namespace = job.metadata.namespace
+
+        try:
+            # Find pods with the job-name label matching this job
+            pods = self.api_instance.list_namespaced_pod(
+                namespace=namespace, label_selector=f"job-name={job_name}"
+            )
+
+            killed_pods = 0
+            for pod in pods.items:
+                if pod.status.phase in ["Running"]:
+                    self.echo(
+                        f"Killing processes in pod {pod.metadata.name} for job {job_name}"
+                    )
+                    if self._kill_pod_process(pod):
+                        killed_pods += 1
+
+            return killed_pods > 0
+        except Exception as e:
+            self.echo(f"Failed to find/kill pods for job {job_name}: {str(e)}")
+            return False
+
+    def _handle_job_outcome(self, job, outcome):
+        """Handle a job based on the derived outcome"""
+        job_name = job.metadata.name
+        namespace = job.metadata.namespace
+
+        if outcome == JobOutcomes.LEAVE_UNCHANGED:
+            # self.echo(f"Job {job_name} is in terminal state, leaving unchanged")
+            return True
+        elif outcome == JobOutcomes.DELETE:
+            self.echo(f"Deleting Job {job_name}")
+            return self._delete_job(job_name, namespace)
+        elif outcome == JobOutcomes.KILL:
+            self.echo(f"Killing Job {job_name}")
+            # First try to kill the pod processes
+            pods_killed = self._kill_pods_for_job(job)
+            if pods_killed > 0:
+                return True
+            # Worst case if we are not able to delete any pod, then delete the Job.
+            return self._delete_job(job_name, namespace)
+        else:
+            self.echo(f"Unknown outcome {outcome} for job {job_name}\n")
+            return False
+
+    def _handle_jobset_outcome(self, jobset, outcome):
+        """Handle a jobset based on the derived outcome"""
+        jobset_name = jobset.get("metadata", {}).get("name", "unknown")
+        namespace = jobset.get("metadata", {}).get("namespace", self._namespace)
+
+        if outcome == JobOutcomes.LEAVE_UNCHANGED:
+            # self.echo(f"JobSet {jobset_name} is in terminal state, leaving unchanged")
+            return True
+        elif outcome == JobOutcomes.DELETE:
+            self.echo(f"Deleting JobSet {jobset_name}")
+            try:
+                self.jobset_api.delete_namespaced_custom_object(
+                    group="jobset.x-k8s.io",
+                    version="v1alpha2",
+                    namespace=namespace,
+                    plural="jobsets",
+                    name=jobset_name,
+                )
+                return True
+            except Exception as e:
+                self.echo(f"Failed to delete JobSet {jobset_name}: {str(e)}")
+                return False
+        else:
+            self.echo(f"Unknown outcome {outcome} for JobSet {jobset_name}")
+            return False
+
+    def process_matching_jobs_and_jobsets(self, flow_name, run_id, user):
+        """Process all matching jobs and jobsets based on their derived outcomes"""
+        results = []
+
+        # Process matching jobs
+        jobs = self._find_matching_jobs(flow_name, run_id, user)
+        for job in jobs:
+            outcome = derive_job_outcome(job.status)
+            result = self._handle_job_outcome(job, outcome)
+            if not result:
+                self.echo("Failed Result for : %s" % job.metadata.name)
+            results.append(result)
+
+        # Process matching jobsets
+        jobsets = self._find_matching_jobsets(flow_name, run_id, user)
+        for jobset in jobsets:
+            jobset_status = jobset.get("status", {})
+            outcome = derive_jobset_outcome(jobset_status)
+            result = self._handle_jobset_outcome(jobset, outcome)
+            if not result:
+                self.echo("Failed Result for : %s" % jobset.get("metadata").get("name"))
+            results.append(result)
+
+        return results, len(jobs), len(jobsets)

metaflow_extensions/outerbounds/plugins/nvcf/constants.py

@@ -1,3 +1,3 @@
-SUPPORTABLE_GPU_TYPES = ["L40", "L40S", "L40G", "H100"]
+SUPPORTABLE_GPU_TYPES = ["L40", "L40S", "L40G", "H100", "NEBIUS_H100"]
 DEFAULT_GPU_TYPE = "H100"
-MAX_N_GPU_BY_TYPE = {"L40": 1, "L40S": 1, "L40G": 1, "H100": 4}
+MAX_N_GPU_BY_TYPE = {"L40": 1, "L40S": 1, "L40G": 1, "H100": 4, "NEBIUS_H100": 8}

metaflow_extensions/outerbounds/plugins/nvct/nvct_decorator.py

@@ -65,6 +65,28 @@ SUPPORTABLE_GPU_TYPES = {
             "backend": "gcp-asia-se-1a",
         },
     ],
+    "NEBIUS_H100": [
+        {
+            "n_gpus": 1,
+            "instance_type": "ON-PREM.GPU.H100_1x",
+            "backend": "default-project-eu-north1",
+        },
+        {
+            "n_gpus": 2,
+            "instance_type": "ON-PREM.GPU.H100_2x",
+            "backend": "default-project-eu-north1",
+        },
+        {
+            "n_gpus": 4,
+            "instance_type": "ON-PREM.GPU.H100_4x",
+            "backend": "default-project-eu-north1",
+        },
+        {
+            "n_gpus": 8,
+            "instance_type": "ON-PREM.GPU.H100_8x",
+            "backend": "default-project-eu-north1",
+        },
+    ],
 }
 
 
@@ -154,6 +176,8 @@ class NvctDecorator(StepDecorator):
 
         self.attributes["instance_type"] = valid_config["instance_type"]
         self.attributes["gpu_type"] = requested_gpu_type
+        if self.attributes["gpu_type"] == "NEBIUS_H100":
+            self.attributes["gpu_type"] = "H100"
         self.attributes["backend"] = valid_config["backend"]
 
     def runtime_init(self, flow, graph, package, run_id):

metaflow_extensions/outerbounds/plugins/torchtune/__init__.py

@@ -0,0 +1,159 @@
+from queue import Queue, Empty
+from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, List, Dict
+import subprocess
+import shutil
+import sys
+from metaflow import current
+
+__mf_promote_submodules__ = ["plugins.torchtune"]
+
+
+class TorchTune:
+    def __init__(
+        self,
+        use_multi_node_config: bool = False,
+    ):
+        """
+        Initialize the Tune launcher.
+
+        :param use_multi_node_config: If True, attempt to build a distributed configuration
+                                      from current.torch.torchrun_args.
+        """
+        self.multi_node_config = {}
+        if use_multi_node_config:
+            if getattr(current, "torch", None):
+                print(
+                    "[Metaflow Tune] Since @torchrun is used, multi-node config can be used to launch the job."
+                )
+                # For distributed torchtune launches, we use similar parameters as torchrun.
+                # (You might need to adjust the keys according to your environment.)
+                self.multi_node_config = {
+                    "nnodes": current.torch.torchrun_args["nnodes"],
+                    "master_addr": current.torch.torchrun_args["master_addr"],
+                    "master_port": int(current.torch.torchrun_args["master_port"]),
+                    "node_rank": current.torch.torchrun_args["node_rank"],
+                    "nproc_per_node": current.torch.torchrun_args["nproc_per_node"],
+                    "num_processes": current.torch.torchrun_args["nproc_per_node"]
+                    * current.torch.torchrun_args["nnodes"],
+                }
+                print(
+                    f"[Metaflow Tune] Discovered multi-node config for torchrun: {self.multi_node_config}"
+                )
+            else:
+                print(
+                    "[Metaflow Tune] Since @torchrun is not used, default multi-node config cannot be used to launch the job."
+                )
+
+    def run(
+        self,
+        recipe: str,
+        config_dict: Dict,
+        additional_cli_options: Optional[List[str]] = None,
+    ):
+        """
+        Launch the torchtune job via its CLI.
+
+        :param recipe: The path to the recipe (or name of the recipe) to run.
+        :param config_dict: Optional dictionary that will be dumped to a YAML file and passed via --config.
+        :param additional_cli_options: Optional list of additional CLI options.
+        :raises: subprocess.CalledProcessError if the subprocess returns a nonzero exit code.
+        """
+        import yaml
+        import tempfile
+        import os
+
+        _temp_dir = tempfile.mkdtemp()
+        try:
+            config_path = os.path.join(_temp_dir, "config.yaml")
+            with open(config_path, "w") as f:
+                yaml.dump(config_dict, f)
+
+            additional_options = (
+                additional_cli_options if additional_cli_options else []
+            )
+
+            # Build the command. Here we use "tune run" as the base command.
+            cmd = ["tune", "run"]
+
+            # If distributed configuration is present, add torchrun–style flags.
+            if self.multi_node_config:
+                cmd.extend(
+                    [
+                        "--nnodes",
+                        str(self.multi_node_config.get("nnodes")),
+                        "--nproc-per-node",
+                        str(self.multi_node_config.get("nproc_per_node")),
+                        # "--rdzv_conf", f"rdzv_endpoint={self.multi_node_config.get('master_addr')}:{self.multi_node_config.get('master_port')}"
+                        "--rdzv-backend",
+                        "c10d",
+                        "--rdzv-endpoint",
+                        f"{self.multi_node_config.get('master_addr')}:{self.multi_node_config.get('master_port')}",
+                        "--rdzv-id",
+                        "1234567890",
+                        "--node-rank",
+                        str(self.multi_node_config.get("node_rank")),
+                        # TODO: should there be a masterip/port here ?
+                    ]
+                )
+
+            cmd.extend(additional_options)
+
+            cmd.append(recipe)
+            # If a recipe configuration was provided, pass it via the --config flag.
+            cmd.extend(["--config", config_path])
+
+            # Append any additional CLI options.
+
+            # Launch the subprocess.
+            print(f"[Metaflow tune] {' '.join(cmd)}")
+            process = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                universal_newlines=True,
+            )
+
+            # Stream the output in real-time.
+            for out_line, err_line in read_popen_pipes(process):
+                print(out_line, end="", flush=True)
+                print(err_line, end="", file=sys.stderr, flush=True)
+
+            process.wait()
+            if process.returncode != 0:
+                raise subprocess.CalledProcessError(process.returncode, cmd)
+        finally:
+            shutil.rmtree(_temp_dir)
+
+
+def enqueue_output(file, queue):
+    for line in iter(file.readline, ""):
+        queue.put(line)
+    file.close()
+
+
+def read_popen_pipes(p):
+
+    with ThreadPoolExecutor(2) as pool:
+        q_stdout, q_stderr = Queue(), Queue()
+
+        pool.submit(enqueue_output, p.stdout, q_stdout)
+        pool.submit(enqueue_output, p.stderr, q_stderr)
+
+        while True:
+
+            if p.poll() is not None and q_stdout.empty() and q_stderr.empty():
+                break
+
+            out_line = err_line = ""
+
+            try:
+                out_line = q_stdout.get_nowait()
+            except Empty:
+                pass
+            try:
+                err_line = q_stderr.get_nowait()
+            except Empty:
+                pass
+
+            yield (out_line, err_line)

metaflow_extensions/outerbounds/toplevel/plugins/torchtune/__init__.py

@@ -0,0 +1 @@
+__mf_promote_submodules__ = ["plugins.torchtune"]

{ob_metaflow_extensions-1.1.158.dist-info → ob_metaflow_extensions-1.1.160.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.158
+Version: 1.1.160
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob_metaflow_extensions-1.1.158.dist-info → ob_metaflow_extensions-1.1.160.dist-info}/RECORD

@@ -1,7 +1,7 @@
 metaflow_extensions/outerbounds/__init__.py,sha256=Gb8u06s9ClQsA_vzxmkCzuMnigPy7kKcDnLfb7eB-64,514
 metaflow_extensions/outerbounds/remote_config.py,sha256=pEFJuKDYs98eoB_-ryPjVi9b_c4gpHMdBHE14ltoxIU,4672
 metaflow_extensions/outerbounds/config/__init__.py,sha256=JsQGRuGFz28fQWjUvxUgR8EKBLGRdLUIk_buPLJplJY,1225
-metaflow_extensions/outerbounds/plugins/__init__.py,sha256=gytuNt3lNabirHLEYzrmHFMp-JWh8dA2AZPK11HmaNw,13242
+metaflow_extensions/outerbounds/plugins/__init__.py,sha256=GxYKjrMJCGVKoxhfdPAlVF9kYrEb3-xn9fgUTb_H9VY,13255
 metaflow_extensions/outerbounds/plugins/auth_server.py,sha256=_Q9_2EL0Xy77bCRphkwT1aSu8gQXRDOH-Z-RxTUO8N4,2202
 metaflow_extensions/outerbounds/plugins/perimeters.py,sha256=QXh3SFP7GQbS-RAIxUOPbhPzQ7KDFVxZkTdKqFKgXjI,2697
 metaflow_extensions/outerbounds/plugins/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -22,13 +22,14 @@ metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery.py,sha256=PE81ZB
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_cli.py,sha256=kqFyu2bJSnc9_9aYfBpz5xK6L6luWFZK_NMuh8f1eVk,1494
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_decorator.py,sha256=MXSIp05-jvt8Q2uGaLKjtuM_ToLeRLxhtMbfHc9Kcko,1515
 metaflow_extensions/outerbounds/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
-metaflow_extensions/outerbounds/plugins/kubernetes/kubernetes_client.py,sha256=fx_XUkgR4r6hF2ilDfT5LubRyVrYMVIv5f6clHkCaEk,5988
+metaflow_extensions/outerbounds/plugins/kubernetes/kubernetes_client.py,sha256=sjBhQ4aa-i1UkKsJyTswdDLYOBAFIvHRco4r7wfs9Tc,5003
+metaflow_extensions/outerbounds/plugins/kubernetes/pod_killer.py,sha256=iZQSXNK14DVrJ6jRfrMfkjoUf4dNKDZA51k6LeVJL6M,11392
 metaflow_extensions/outerbounds/plugins/nim/card.py,sha256=dXOJvsZed5NyYyxYLPDvtwg9z_X4azL9HTJGYaiNriY,4690
 metaflow_extensions/outerbounds/plugins/nim/nim_decorator.py,sha256=50YVvC7mcZYlPluM0Wq1UtufhzlQb-RxzZkTOJJ3LkM,3439
 metaflow_extensions/outerbounds/plugins/nim/nim_manager.py,sha256=y8U71106KJtrC6nlhsNnzX9Xkv3RnyZ1KEpRFwqZZFk,13686
 metaflow_extensions/outerbounds/plugins/nim/utils.py,sha256=nU-v1sheBjmITXfHiJx2ucm_Tq_nGb5BcuAm5c235cQ,1164
 metaflow_extensions/outerbounds/plugins/nvcf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow_extensions/outerbounds/plugins/nvcf/constants.py,sha256=aGHdNw_hqBu8i0zWXcatQM6e769wUXox0l8g0f6fNZ8,146
+metaflow_extensions/outerbounds/plugins/nvcf/constants.py,sha256=8Gcf5nKBVoE2v7_4hTW1EAf-hvXCzQoccaRBeKa_MNk,179
 metaflow_extensions/outerbounds/plugins/nvcf/exceptions.py,sha256=-Pm9cOWUzpv94TvVUeq-FenAWdfLBJd5N7WPqIGZVqU,3671
 metaflow_extensions/outerbounds/plugins/nvcf/heartbeat_store.py,sha256=pOWwm8LFQBbtku0zNBBwCyXxLK8U-hhC4naQcmU69nE,6217
 metaflow_extensions/outerbounds/plugins/nvcf/nvcf.py,sha256=3ZFdYItVpFWnHMOeyV1nslUyelfvX5rknh2d2IWxVws,15591
@@ -39,7 +40,7 @@ metaflow_extensions/outerbounds/plugins/nvct/__init__.py,sha256=47DEQpj8HBSa-_TI
 metaflow_extensions/outerbounds/plugins/nvct/exceptions.py,sha256=1PiV6FdH36CvkmHh5jtsfrsoe3Q_Fo1NomHw5wvgoDM,2886
 metaflow_extensions/outerbounds/plugins/nvct/nvct.py,sha256=Z2ZPWGuHe58au_d6GfHiw6Nl5d8INdLDI5exlsPEOSA,3564
 metaflow_extensions/outerbounds/plugins/nvct/nvct_cli.py,sha256=bB9AURhRep9PV_-b-qLHpgw_GPG_xFoq1PeHEgFP1mQ,10104
-metaflow_extensions/outerbounds/plugins/nvct/nvct_decorator.py,sha256=LaJ_Tk-vNjvrglzSTR-U6pk8f9MtQRKObU9m7vBYtkI,8695
+metaflow_extensions/outerbounds/plugins/nvct/nvct_decorator.py,sha256=HKCvYn1Jh8uwLXeUqPNhxgBatq3mXNG5YIUl-zjNlHE,9429
 metaflow_extensions/outerbounds/plugins/nvct/nvct_runner.py,sha256=8IPkdvuTZNIqgAAt75gVNn-ydr-Zz2sKC8UX_6pNEKI,7091
 metaflow_extensions/outerbounds/plugins/nvct/utils.py,sha256=U4_Fu8H94j_Bbox7mmMhNnlRhlYHqnK28R5w_TMWEFM,1029
 metaflow_extensions/outerbounds/plugins/ollama/__init__.py,sha256=vzh8sQEfwKRdx0fsGFJ-km4mwfi0vm2q1_vsZv-EMcc,3034
@@ -61,6 +62,7 @@ metaflow_extensions/outerbounds/plugins/snowpark/snowpark_exceptions.py,sha256=F
 metaflow_extensions/outerbounds/plugins/snowpark/snowpark_job.py,sha256=aQphxX6jqYgfa83w387pEWl0keuLm38V53I8P8UL2ck,6887
 metaflow_extensions/outerbounds/plugins/snowpark/snowpark_service_spec.py,sha256=AI_kcm1hZV3JRxJkookcH6twiGnAYjk9Dx-MeoYz60Y,8511
 metaflow_extensions/outerbounds/plugins/tensorboard/__init__.py,sha256=9lUM4Cqi5RjrHBRfG6AQMRz8-R96eZC8Ih0KD2lv22Y,1858
+metaflow_extensions/outerbounds/plugins/torchtune/__init__.py,sha256=TOXNeyhcgd8VxplXO_oEuryFEsbk0tikn5GL0-44SU8,5853
 metaflow_extensions/outerbounds/profilers/__init__.py,sha256=wa_jhnCBr82TBxoS0e8b6_6sLyZX0fdHicuGJZNTqKw,29
 metaflow_extensions/outerbounds/profilers/gpu.py,sha256=3Er8uKQzfm_082uadg4yn_D4Y-iSCgzUfFmguYxZsz4,27485
 metaflow_extensions/outerbounds/toplevel/__init__.py,sha256=qWUJSv_r5hXJ7jV_On4nEasKIfUCm6_UjkjXWA_A1Ts,90
@@ -70,7 +72,8 @@ metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3u
 metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
 metaflow_extensions/outerbounds/toplevel/plugins/ollama/__init__.py,sha256=GRSz2zwqkvlmFS6bcfYD_CX6CMko9DHQokMaH1iBshA,47
 metaflow_extensions/outerbounds/toplevel/plugins/snowflake/__init__.py,sha256=LptpH-ziXHrednMYUjIaosS1SXD3sOtF_9_eRqd8SJw,50
-ob_metaflow_extensions-1.1.158.dist-info/METADATA,sha256=0t_P8-Uhi3I39xyeSGv2BpRQO5Upe1eIjs04e6Stjd8,521
-ob_metaflow_extensions-1.1.158.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-ob_metaflow_extensions-1.1.158.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
-ob_metaflow_extensions-1.1.158.dist-info/RECORD,,
+metaflow_extensions/outerbounds/toplevel/plugins/torchtune/__init__.py,sha256=uTVkdSk3xZ7hEKYfdlyVteWj5KeDwaM1hU9WT-_YKfI,50
+ob_metaflow_extensions-1.1.160.dist-info/METADATA,sha256=-Y7aOv3_giiWpXTkncvXFIeHqXqfnWOA0hiHuLqueS8,521
+ob_metaflow_extensions-1.1.160.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+ob_metaflow_extensions-1.1.160.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
+ob_metaflow_extensions-1.1.160.dist-info/RECORD,,