ob-metaflow-extensions 1.1.155rc0__tar.gz → 1.1.157__tar.gz

{ob-metaflow-extensions-1.1.155rc0 → ob-metaflow-extensions-1.1.157}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.155rc0
+Version: 1.1.157
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob-metaflow-extensions-1.1.155rc0 → ob-metaflow-extensions-1.1.157}/metaflow_extensions/outerbounds/plugins/__init__.py

@@ -326,7 +326,7 @@ STEP_DECORATORS_DESC = [
     ("snowpark", ".snowpark.snowpark_decorator.SnowparkDecorator"),
     ("tensorboard", ".tensorboard.TensorboardDecorator"),
     ("gpu_profile", ".profilers.gpu_profile_decorator.GPUProfileDecorator"),
-    ("nim", ".nim.NimDecorator"),
+    ("nim", ".nim.nim_decorator.NimDecorator"),
     ("ollama", ".ollama.OllamaDecorator"),
     ("app_deploy", ".apps.deploy_decorator.WorkstationAppDeployDecorator"),
 ]

{ob-metaflow-extensions-1.1.155rc0 → ob-metaflow-extensions-1.1.157}/metaflow_extensions/outerbounds/plugins/nim/card.py

@@ -1,8 +1,7 @@
-import sqlite3
 from metaflow.cards import Markdown, Table
 from metaflow.metaflow_current import current
 
-from .utilities import get_storage_path
+from .utils import get_storage_path
 from ..card_utilities.async_cards import CardRefresher
 from ..card_utilities.extra_components import BarPlot, ViolinPlot
 
@@ -17,9 +16,7 @@ class NimMetricsRefresher(CardRefresher):
         self._file_name = get_storage_path(current.task_id)
 
     def sqlite_fetch_func(self, conn):
-        cursor = conn.cursor()
         try:
-            conn = sqlite3.connect(self._file_name)
             cursor = conn.cursor()
             cursor.execute(
                 "SELECT error, success, status_code, prompt_tokens, completion_tokens, e2e_time, model FROM metrics"
@@ -85,7 +82,6 @@ class NimMetricsRefresher(CardRefresher):
         current_card.refresh()
 
     def on_error(self, current_card, error_message):
-
         if isinstance(error_message, FileNotFoundError):
             return
 
@@ -99,7 +95,6 @@ class NimMetricsRefresher(CardRefresher):
             current_card.refresh()
 
     def update_only_components(self, current_card, data_object):
-
         # update request success data
         self._metrics_charts["request_success"].spec["data"][0]["values"] = [
             {

ob-metaflow-extensions-1.1.155rc0/metaflow_extensions/outerbounds/plugins/nim/__init__.py → ob-metaflow-extensions-1.1.157/metaflow_extensions/outerbounds/plugins/nim/nim_decorator.py

@@ -1,64 +1,31 @@
-from functools import partial
-from uuid import uuid4
-import os, time
-from metaflow.decorators import StepDecorator
+import os
+import time
 from metaflow import current
-
+from .utils import get_storage_path, NIM_MONITOR_LOCAL_STORAGE_ROOT
 from .nim_manager import NimManager
+from metaflow.decorators import StepDecorator
 from .card import NimMetricsRefresher
-from .utilities import get_storage_path, NIM_MONITOR_LOCAL_STORAGE_ROOT
-from ..card_utilities.async_cards import AsyncPeriodicRefresher
 from ..card_utilities.injector import CardDecoratorInjector
+from ..card_utilities.async_cards import AsyncPeriodicRefresher
 
 
 class NimDecorator(StepDecorator, CardDecoratorInjector):
-    """
-    This decorator is used to run NIM containers in Metaflow tasks as sidecars.
-
-    User code call
-    -----------
-    @nim(
-        models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
-        backend='managed'
-    )
-
-    Valid backend options
-    ---------------------
-    - 'managed': Outerbounds selects a compute provider based on the model.
-
-    Valid model options
-    ----------------
-        - 'meta/llama3-8b-instruct': 8B parameter model
-        - 'meta/llama3-70b-instruct': 70B parameter model
-        - any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
-
-    Parameters
-    ----------
-    models: list[NIM]
-        List of NIM containers running models in sidecars.
-    backend: str
-        Compute provider to run the NIM container.
-    queue_timeout : int
-        Time to keep the job in NVCF's queue.
-    """
-
     name = "nim"
+
     defaults = {
         "models": [],
-        "backend": "managed",
         "monitor": True,
         "persist_db": False,
-        "queue_timeout": 5 * 24 * 3600,  # Default 5 days in seconds
     }
 
-    def step_init(
-        self, flow, graph, step_name, decorators, environment, flow_datastore, logger
-    ):
-
+    # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png
+    # to understand where these functions are invoked in the lifecycle of a
+    # Metaflow flow.
+    def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
         if self.attributes["monitor"]:
             self.attach_card_decorator(
                 flow,
-                step_name,
+                step,
                 NimMetricsRefresher.CARD_ID,
                 "blank",
                 refresh_interval=4.0,
@@ -68,11 +35,9 @@ class NimDecorator(StepDecorator, CardDecoratorInjector):
             {
                 "nim": NimManager(
                     models=self.attributes["models"],
-                    backend=self.attributes["backend"],
                     flow=flow,
-                    step_name=step_name,
+                    step_name=step,
                     monitor=self.attributes["monitor"],
-                    queue_timeout=self.attributes["queue_timeout"],
                 )
             }
         )
@@ -81,15 +46,14 @@ class NimDecorator(StepDecorator, CardDecoratorInjector):
         self, step_func, flow, graph, retry_count, max_user_code_retries, ubf_context
     ):
         if self.attributes["monitor"]:
-
             import sqlite3
-            from metaflow import current
 
             file_path = get_storage_path(current.task_id)
             if os.path.exists(file_path):
                 os.remove(file_path)
             os.makedirs(NIM_MONITOR_LOCAL_STORAGE_ROOT, exist_ok=True)
             conn = sqlite3.connect(file_path)
+
             cursor = conn.cursor()
             cursor.execute(
                 """

ob-metaflow-extensions-1.1.157/metaflow_extensions/outerbounds/plugins/nim/nim_manager.py

@@ -0,0 +1,379 @@
+import sys
+import time
+import requests
+import sqlite3
+from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
+from typing import Dict, Optional, Any
+from .utils import get_ngc_response, get_storage_path
+
+
+def nvcf_submit_helper(
+    url: str,
+    payload: Dict[str, Any],
+    headers: Optional[Dict[str, str]] = None,
+    timeout: int = 30,
+    max_retries: int = 300,
+    backoff_factor: float = 0.3,
+    request_delay: float = 1.1,
+    log_callback: Optional[callable] = None,
+) -> Dict[str, Any]:
+    def _log_error(start_time: float, status_code: int, poll_count: int):
+        if log_callback:
+            end_time = time.time()
+            try:
+                log_callback({}, end_time - start_time, status_code, poll_count)
+            except Exception as log_error:
+                print(f"Warning: Logging callback failed: {log_error}")
+
+    # use default headers
+    if not headers:
+        headers = {"accept": "application/json", "content-type": "application/json"}
+        print(f"Using Default Headers: {headers}")
+
+    # Configure session with retry strategy
+    session = requests.Session()
+    status_forcelist = [429, 500, 502, 503, 504, 404]
+    retry_strategy = Retry(
+        total=max_retries,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+        allowed_methods=["GET", "POST"],
+    )
+    adapter = HTTPAdapter(max_retries=retry_strategy)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+
+    # Add artificial delay if specified
+    time.sleep(request_delay)
+
+    start_time = time.time()
+    poll_count = 0
+    status_code = 0
+    response_data = {}
+
+    try:
+        # Make initial request
+        response = session.post(url, json=payload, headers=headers, timeout=timeout)
+        time.sleep(request_delay)
+
+        # Handle initial response
+        response.raise_for_status()
+        request_id = response.headers.get("NVCF-REQID")
+        polling_url = f"https://api.nvcf.nvidia.com/v2/nvcf/pexec/status/{request_id}"
+
+        print(f"Polling NVCF Request ID: {request_id}")
+
+        # Initial response status
+        status_code = response.status_code
+        print(f"Initial response status: {status_code}")
+
+        # Create a variable to store the final response
+        final_response = response
+
+        # Continue polling while we get 202 (Accepted/Processing)
+        while status_code == 202:
+            poll_count += 1
+            print(f"Polling attempt #{poll_count} to {polling_url}")
+
+            # Wait before next poll
+            time.sleep(request_delay)
+
+            # Make a new poll request
+            poll_response = session.get(polling_url, headers=headers, timeout=timeout)
+            status_code = poll_response.status_code
+            print(f"Poll #{poll_count} status: {status_code}")
+
+            # Check for errors
+            try:
+                poll_response.raise_for_status()
+            except requests.exceptions.HTTPError as e:
+                print(f"Poll request failed: {str(e)}")
+                poll_response.close()
+                # Log the error before re-raising
+                _log_error(start_time, poll_response.status_code, poll_count)
+                raise
+
+            # If status is 200, the job is complete
+            if status_code == 200:
+                print("Polling complete - job finished successfully")
+                # Update our final response to be this poll response
+                final_response = poll_response
+                break
+
+            # Close this poll response if we're going to loop again
+            if status_code == 202:
+                poll_response.close()
+
+        # If we exited the loop without a 200 status, something went wrong
+        if status_code != 200:
+            print(f"Polling ended with unexpected status: {status_code}")
+            # Log the error before raising
+            _log_error(start_time, status_code, poll_count)
+            raise Exception(f"Unexpected status code after polling: {status_code}")
+
+        # Get the response data for logging
+        response_data = final_response.json()
+
+    except requests.exceptions.HTTPError as e:
+        # Handle HTTP errors (4xx, 5xx status codes)
+        status_code = e.response.status_code if e.response else 0
+        print(f"HTTP Error: {str(e)}", file=sys.stderr)
+        # Log the error
+        _log_error(start_time, status_code, poll_count)
+        raise
+
+    except Exception as e:
+        # Handle other errors (connection errors, timeouts, etc.)
+        print(f"Request Error: {str(e)}", file=sys.stderr)
+        # Log the error with status_code 0 to indicate non-HTTP error
+        _log_error(start_time, 0, poll_count)
+        raise
+
+    # Calculate final duration and log successful requests
+    end_time = time.time()
+    duration = end_time - start_time
+
+    # Call the logging callback if provided
+    if log_callback:
+        try:
+            log_callback(response_data, duration, status_code, poll_count)
+        except Exception as e:
+            print(f"Warning: Logging callback failed: {e}")
+
+    # Log metrics
+    print(
+        f"Request completed: duration={duration:.2f}s, polls={poll_count}, "
+        f"status={status_code}, size={len(final_response.content)} bytes"
+    )
+
+    return response_data
+
+
+class NimMetadata(object):
+    def __init__(self):
+        self._nvcf_chat_completion_models = []
+        ngc_response = get_ngc_response()
+
+        self.ngc_api_key = ngc_response["nvcf"]["api_key"]
+
+        for model in ngc_response["nvcf"]["functions"]:
+            self._nvcf_chat_completion_models.append(
+                {
+                    "name": model["model_key"],
+                    "function-id": model["id"],
+                    "version-id": model["version"],
+                }
+            )
+
+    def get_nvcf_chat_completion_models(self):
+        return self._nvcf_chat_completion_models
+
+    def get_headers_for_nvcf_request(self):
+        return {
+            "accept": "application/json",
+            "content-type": "application/json",
+            "Authorization": f"Bearer {self.ngc_api_key}",
+            "NVCF-POLL-SECONDS": "5",
+        }
+
+
+class NimManager(object):
+    def __init__(self, models, flow, step_name, monitor):
+        nim_metadata = NimMetadata()
+        nvcf_models = [
+            m["name"] for m in nim_metadata.get_nvcf_chat_completion_models()
+        ]
+        self.models = {}
+
+        # Convert models to a standard format
+        standardized_models = []
+        # If models is a single string, convert it to a list with a dict
+        if isinstance(models, str):
+            standardized_models = [{"name": models}]
+        # If models is a list, process each item
+        elif isinstance(models, list):
+            for model_item in models:
+                # If the item is a string, convert it to a dict
+                if isinstance(model_item, str):
+                    standardized_models.append({"name": model_item})
+                # If it's already a dict, use it as is
+                elif isinstance(model_item, dict):
+                    standardized_models.append(model_item)
+                else:
+                    raise ValueError(
+                        f"Model specification must be a string or dictionary, got {type(model_item)}"
+                    )
+        else:
+            raise ValueError(
+                f"Models must be a string or a list of strings/dictionaries, got {type(models)}"
+            )
+
+        # Process each standardized model
+        for each_model_dict in standardized_models:
+            model_name = each_model_dict.get("name", "")
+            nvcf_id = each_model_dict.get("nvcf_id", "")
+            nvcf_version = each_model_dict.get("nvcf_version", "")
+
+            if model_name and not (nvcf_id and nvcf_version):
+                if model_name in nvcf_models:
+                    self.models[model_name] = NimChatCompletion(
+                        model=model_name,
+                        nvcf_id=nvcf_id,
+                        nvcf_version=nvcf_version,
+                        nim_metadata=nim_metadata,
+                        monitor=monitor,
+                    )
+                else:
+                    raise ValueError(
+                        f"Model {model_name} not supported by the Outerbounds @nim offering."
+                        f"\nYou can choose from these options: {nvcf_models}\n\n"
+                        "Reach out to Outerbounds if there are other models you'd like supported."
+                    )
+            elif nvcf_id and nvcf_version:
+                self.models[model_name] = NimChatCompletion(
+                    model=model_name,
+                    nvcf_id=nvcf_id,
+                    nvcf_version=nvcf_version,
+                    nim_metadata=nim_metadata,
+                    monitor=monitor,
+                )
+            else:
+                raise ValueError(
+                    "You must provide either a valid 'name' or a custom 'name' along with both 'nvcf_id' and 'nvcf_version'."
+                )
+
+
+class NimChatCompletion(object):
+    def __init__(
+        self,
+        model: str = "meta/llama3-8b-instruct",
+        nvcf_id: str = "",
+        nvcf_version: str = "",
+        nim_metadata: NimMetadata = None,
+        monitor: bool = False,
+        **kwargs,
+    ):
+        if nim_metadata is None:
+            raise ValueError(
+                "NimMetadata object is required to initialize NimChatCompletion object."
+            )
+
+        self.model_name = model
+        self.nim_metadata = nim_metadata
+        self.monitor = monitor
+        all_nvcf_models = self.nim_metadata.get_nvcf_chat_completion_models()
+
+        if nvcf_id and nvcf_version:
+            matching_models = [
+                m
+                for m in all_nvcf_models
+                if m["function-id"] == nvcf_id and m["version-id"] == nvcf_version
+            ]
+            if matching_models:
+                self.model = matching_models[0]
+                self.function_id = self.model["function-id"]
+                self.version_id = self.model["version-id"]
+                self.model_name = self.model["name"]
+            else:
+                raise ValueError(
+                    f"Function {self.function_id} with version {self.version_id} not found on NVCF"
+                )
+        else:
+            all_nvcf_model_names = [m["name"] for m in all_nvcf_models]
+
+            if self.model_name not in all_nvcf_model_names:
+                raise ValueError(
+                    f"Model {self.model_name} not found in available NVCF models"
+                )
+
+            self.model = all_nvcf_models[all_nvcf_model_names.index(self.model_name)]
+            self.function_id = self.model["function-id"]
+            self.version_id = self.model["version-id"]
+
+        self.first_request = True
+
+    def log_stats(self, response_data, duration, status_code, poll_count):
+        if not self.monitor:
+            return
+
+        stats = {
+            "status_code": status_code,
+            "success": 1 if status_code == 200 else 0,
+            "error": 0 if status_code == 200 else 1,
+            "e2e_time": duration,
+            "model": self.model_name,
+            "poll_count": poll_count,
+        }
+
+        if status_code == 200 and response_data:
+            try:
+                stats["prompt_tokens"] = response_data["usage"]["prompt_tokens"]
+            except (KeyError, TypeError):
+                stats["prompt_tokens"] = None
+
+            try:
+                stats["completion_tokens"] = response_data["usage"]["completion_tokens"]
+            except (KeyError, TypeError):
+                stats["completion_tokens"] = None
+        else:
+            stats["prompt_tokens"] = None
+            stats["completion_tokens"] = None
+
+        conn = sqlite3.connect(self.file_name)
+        cursor = conn.cursor()
+
+        try:
+            cursor.execute(
+                """
+                INSERT INTO metrics (error, success, status_code, prompt_tokens, completion_tokens, e2e_time, model)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+                (
+                    stats["error"],
+                    stats["success"],
+                    stats["status_code"],
+                    stats["prompt_tokens"],
+                    stats["completion_tokens"],
+                    stats["e2e_time"],
+                    stats["model"],
+                ),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def __call__(self, **kwargs):
+        if self.first_request:
+            from metaflow import current
+
+            self.file_name = get_storage_path(current.task_id)
+            self.first_request = False
+
+        # Create log callback if monitoring is enabled
+        log_callback = self.log_stats if self.monitor else None
+
+        request_data = {"model": self.model_name, **kwargs}
+        request_url = (
+            f"https://api.nvcf.nvidia.com/v2/nvcf/pexec/functions/{self.function_id}"
+        )
+
+        try:
+            response_data = nvcf_submit_helper(
+                url=request_url,
+                payload=request_data,
+                headers=self.nim_metadata.get_headers_for_nvcf_request(),
+                log_callback=log_callback,
+            )
+
+            return response_data
+
+        except requests.exceptions.HTTPError as e:
+            error_msg = f"[@nim ERROR] NVCF API request failed: {str(e)}"
+            print(error_msg, file=sys.stderr)
+            raise
+
+        except Exception as e:
+            error_msg = f"[@nim ERROR] Unexpected error: {str(e)}"
+            print(error_msg, file=sys.stderr)
+            raise

ob-metaflow-extensions-1.1.157/metaflow_extensions/outerbounds/plugins/nim/utils.py

@@ -0,0 +1,36 @@
+import os
+import json
+import requests
+from urllib.parse import urlparse
+from metaflow.metaflow_config import SERVICE_URL
+from metaflow.metaflow_config_funcs import init_config
+
+
+NIM_MONITOR_LOCAL_STORAGE_ROOT = ".nim-monitor"
+
+
+def get_storage_path(task_id):
+    return f"{NIM_MONITOR_LOCAL_STORAGE_ROOT}/" + task_id + ".sqlite"
+
+
+def get_ngc_response():
+    conf = init_config()
+    if "OBP_AUTH_SERVER" in conf:
+        auth_host = conf["OBP_AUTH_SERVER"]
+    else:
+        auth_host = "auth." + urlparse(SERVICE_URL).hostname.split(".", 1)[1]
+
+    # NOTE: reusing the same auth_host as the one used in NimMetadata,
+    # however, user should not need to use nim container to use @nvct.
+    # May want to refactor this to a common endpoint.
+    nim_info_url = "https://" + auth_host + "/generate/nim"
+
+    if "METAFLOW_SERVICE_AUTH_KEY" in conf:
+        headers = {"x-api-key": conf["METAFLOW_SERVICE_AUTH_KEY"]}
+        res = requests.get(nim_info_url, headers=headers)
+    else:
+        headers = json.loads(os.environ.get("METAFLOW_SERVICE_HEADERS"))
+        res = requests.get(nim_info_url, headers=headers)
+
+    res.raise_for_status()
+    return res.json()

{ob-metaflow-extensions-1.1.155rc0 → ob-metaflow-extensions-1.1.157}/metaflow_extensions/outerbounds/plugins/secrets/secrets.py

@@ -187,11 +187,18 @@ class OuterboundsSecretsProvider(SecretsProvider):
             raise OuterboundsSecretsException(
                 f"Server error: {response.text}. Please reach out to your Outerbounds support team."
             )
-        status_code = response.status_code
+
+        body = response.json()
+        status_code = body.get("error", {}).get("statusCode", response.status_code)
         if status_code == 404:
-            raise OuterboundsSecretsException(f"Secret not found: {response.text}")
+            raise OuterboundsSecretsException(f"Secret not found: {body}")
 
         if status_code >= 400:
-            raise OuterboundsSecretsException(
-                f"status_code={status_code}\t\n\t\t{response.text}"
-            )
+            try:
+                raise OuterboundsSecretsException(
+                    f"status_code={status_code}\t*{body['error']['details']['kind']}*\n{body['error']['details']['message']}"
+                )
+            except KeyError:
+                raise OuterboundsSecretsException(
+                    f"status_code={status_code} Unexpected error: {body}"
+                )

{ob-metaflow-extensions-1.1.155rc0 → ob-metaflow-extensions-1.1.157}/ob_metaflow_extensions.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.155rc0
+Version: 1.1.157
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob-metaflow-extensions-1.1.155rc0 → ob-metaflow-extensions-1.1.157}/ob_metaflow_extensions.egg-info/SOURCES.txt

@@ -25,10 +25,10 @@ metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_cli.py
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery_decorator.py
 metaflow_extensions/outerbounds/plugins/kubernetes/__init__.py
 metaflow_extensions/outerbounds/plugins/kubernetes/kubernetes_client.py
-metaflow_extensions/outerbounds/plugins/nim/__init__.py
 metaflow_extensions/outerbounds/plugins/nim/card.py
+metaflow_extensions/outerbounds/plugins/nim/nim_decorator.py
 metaflow_extensions/outerbounds/plugins/nim/nim_manager.py
-metaflow_extensions/outerbounds/plugins/nim/utilities.py
+metaflow_extensions/outerbounds/plugins/nim/utils.py
 metaflow_extensions/outerbounds/plugins/nvcf/__init__.py
 metaflow_extensions/outerbounds/plugins/nvcf/constants.py
 metaflow_extensions/outerbounds/plugins/nvcf/exceptions.py

ob-metaflow-extensions-1.1.157/ob_metaflow_extensions.egg-info/requires.txt

@@ -0,0 +1,3 @@
+boto3
+kubernetes
+ob-metaflow==2.15.14.1

{ob-metaflow-extensions-1.1.155rc0 → ob-metaflow-extensions-1.1.157}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
 from pathlib import Path
 
 
-version = "1.1.155rc0"
+version = "1.1.157"
 this_directory = Path(__file__).parent
 long_description = (this_directory / "README.md").read_text()
 
@@ -18,6 +18,6 @@ setup(
     install_requires=[
         "boto3",
         "kubernetes",
-        "ob-metaflow == 2.15.11.2",
+        "ob-metaflow == 2.15.14.1",
     ],
 )

ob-metaflow-extensions-1.1.155rc0/metaflow_extensions/outerbounds/plugins/nim/nim_manager.py

@@ -1,318 +0,0 @@
-import os, sys, time, json, random, requests, sqlite3
-from urllib.parse import urlparse
-from metaflow.metaflow_config import SERVICE_URL
-from metaflow.metaflow_config_funcs import init_config
-from .utilities import get_storage_path
-from ..nvcf.nvcf import retry_on_status
-
-
-NVCF_URL = "https://api.nvcf.nvidia.com"
-NVCF_SUBMIT_ENDPOINT = f"{NVCF_URL}/v2/nvcf/pexec/functions"
-NVCF_RESULT_ENDPOINT = f"{NVCF_URL}/v2/nvcf/pexec/status"
-NVCF_POLL_INTERVAL_SECONDS = 1
-COMMON_HEADERS = {
-    "accept": "application/json",
-    "Content-Type": "application/json",
-    "nvcf-feature-enable-gateway-timeout": "true",
-}
-
-
-class NimMetadata(object):
-    def __init__(self):
-        self._nvcf_chat_completion_models = []
-        self._coreweave_chat_completion_models = []
-
-        conf = init_config()
-
-        if "OBP_AUTH_SERVER" in conf:
-            auth_host = conf["OBP_AUTH_SERVER"]
-        else:
-            auth_host = "auth." + urlparse(SERVICE_URL).hostname.split(".", 1)[1]
-
-        nim_info_url = "https://" + auth_host + "/generate/nim"
-
-        if "METAFLOW_SERVICE_AUTH_KEY" in conf:
-            headers = {"x-api-key": conf["METAFLOW_SERVICE_AUTH_KEY"]}
-            res = requests.get(nim_info_url, headers=headers)
-        else:
-            headers = json.loads(os.environ.get("METAFLOW_SERVICE_HEADERS"))
-            res = requests.get(nim_info_url, headers=headers)
-
-        res.raise_for_status()
-        self._ngc_api_key = res.json()["nvcf"]["api_key"]
-
-        for model in res.json()["nvcf"]["functions"]:
-            self._nvcf_chat_completion_models.append(
-                {
-                    "name": model["model_key"],
-                    "function-id": model["id"],
-                    "version-id": model["version"],
-                }
-            )
-        for model in res.json()["coreweave"]["containers"]:
-            self._coreweave_chat_completion_models.append(
-                {"name": model["nim_name"], "ip-address": model["ip_addr"]}
-            )
-
-    def get_nvcf_chat_completion_models(self):
-        return self._nvcf_chat_completion_models
-
-    def get_headers_for_nvcf_request(self):
-        return {**COMMON_HEADERS, "Authorization": f"Bearer {self._ngc_api_key}"}
-
-
-class NimManager(object):
-    def __init__(self, models, backend, flow, step_name, monitor, queue_timeout):
-
-        nim_metadata = NimMetadata()
-        if backend == "managed":
-            nvcf_models = [
-                m["name"] for m in nim_metadata.get_nvcf_chat_completion_models()
-            ]
-
-            self.models = {}
-            for m in models:
-                if m in nvcf_models:
-                    self.models[m] = NimChatCompletion(
-                        model=m,
-                        provider="NVCF",
-                        nim_metadata=nim_metadata,
-                        monitor=monitor,
-                        queue_timeout=queue_timeout,
-                    )
-                else:
-                    raise ValueError(
-                        f"Model {m} not supported by the Outerbounds @nim offering."
-                        f"\nYou can choose from these options: {nvcf_models}\n\n"
-                        "Reach out to Outerbounds if there are other models you'd like supported."
-                    )
-        else:
-            raise ValueError(
-                f"Backend {backend} not supported by the Outerbounds @nim offering. Please reach out to Outerbounds."
-            )
-
-
-class JobStatus(object):
-    SUBMITTED = "SUBMITTED"
-    RUNNING = "RUNNING"
-    SUCCESSFUL = "SUCCESSFUL"
-    FAILED = "FAILED"
-
-
-class NimChatCompletion(object):
-    def __init__(
-        self,
-        model="meta/llama3-8b-instruct",
-        provider="NVCF",
-        nim_metadata=None,
-        monitor=False,
-        queue_timeout=None,
-        **kwargs,
-    ):
-        if nim_metadata is None:
-            raise ValueError(
-                "NimMetadata object is required to initialize NimChatCompletion object."
-            )
-
-        self._nim_metadata = nim_metadata
-        self.compute_provider = provider
-        self.invocations = []
-        self.max_request_retries = int(
-            os.environ.get("METAFLOW_EXT_HTTP_MAX_RETRIES", "10")
-        )
-        self.monitor = monitor
-
-        if self.compute_provider == "NVCF":
-            nvcf_model_names = [
-                m["name"] for m in self._nim_metadata.get_nvcf_chat_completion_models()
-            ]
-            self.model = model
-            self.function_id = self._nim_metadata.get_nvcf_chat_completion_models()[
-                nvcf_model_names.index(model)
-            ]["function-id"]
-            self.version_id = self._nim_metadata.get_nvcf_chat_completion_models()[
-                nvcf_model_names.index(model)
-            ]["version-id"]
-        else:
-            raise ValueError(
-                f"Backend compute provider {self.compute_provider} not yet supported for @nim."
-            )
-
-        # to know whether to set file_name
-        self.first_request = True
-
-        # TODO (Eddie) - this may make more sense in a base class.
-        # @nim arch needs redesign if customers start using it in more creative ways.
-        self._poll_seconds = "3600"
-        self._queue_timeout = queue_timeout
-        self._status = None
-        self._result = {}
-
-    @property
-    def status(self):
-        return self._status
-
-    @property
-    def has_failed(self):
-        return self._status == JobStatus.FAILED
-
-    @property
-    def is_running(self):
-        return self._status == JobStatus.SUBMITTED
-
-    @property
-    def result(self):
-        return self._result
-
-    def _log_stats(self, response, e2e_time):
-        stats = {}
-        if response.status_code == 200:
-            stats["success"] = 1
-            stats["error"] = 0
-        else:
-            stats["success"] = 0
-            stats["error"] = 1
-        stats["status_code"] = response.status_code
-        try:
-            stats["prompt_tokens"] = response.json()["usage"]["prompt_tokens"]
-        except KeyError:
-            stats["prompt_tokens"] = None
-        try:
-            stats["completion_tokens"] = response.json()["usage"]["completion_tokens"]
-        except KeyError:
-            stats["completion_tokens"] = None
-        stats["e2e_time"] = e2e_time
-        stats["provider"] = self.compute_provider
-        stats["model"] = self.model
-
-        conn = sqlite3.connect(self.file_name)
-        cursor = conn.cursor()
-        try:
-            cursor.execute(
-                """
-                INSERT INTO metrics (error, success, status_code, prompt_tokens, completion_tokens, e2e_time, model)
-                VALUES (?, ?, ?, ?, ?, ?, ?)
-            """,
-                (
-                    stats["error"],
-                    stats["success"],
-                    stats["status_code"],
-                    stats["prompt_tokens"],
-                    stats["completion_tokens"],
-                    stats["e2e_time"],
-                    stats["model"],
-                ),
-            )
-            conn.commit()
-        finally:
-            conn.close()
-
-    @retry_on_status(status_codes=[500], max_retries=3, delay=5)
-    @retry_on_status(status_codes=[504])
-    def __call__(self, **kwargs):
-
-        if self.first_request:
-            # Put here to guarantee self.file_name is set after task_id exists.
-            from metaflow import current
-
-            self.file_name = get_storage_path(current.task_id)
-
-        request_data = {"model": self.model, **kwargs}
-        request_url = f"{NVCF_SUBMIT_ENDPOINT}/{self.function_id}"
-        retry_delay = 1
-        attempts = 0
-        t0 = time.time()
-        while attempts < self.max_request_retries:
-            try:
-                attempts += 1
-                response = requests.post(
-                    request_url,
-                    headers=self._nim_metadata.get_headers_for_nvcf_request(),
-                    json=request_data,
-                )
-                if response.status_code == 202:
-                    invocation_id = response.headers.get("NVCF-REQID")
-                    self.invocations.append(invocation_id)
-                    self._status = JobStatus.SUBMITTED
-                elif response.status_code == 200:
-                    tf = time.time()
-                    if self.monitor:
-                        self._log_stats(response, tf - t0)
-                    self._status = JobStatus.SUCCESSFUL
-                    self._result = response.json()
-                    return self._result
-                elif response.status_code == 400:
-                    self._status = JobStatus.FAILED
-                    msg = (
-                        "[@nim ERROR] The OpenAI-compatible returned a 400 status code. "
-                        + "Known causes include improper requests or prompts with too many tokens for the selected model. "
-                        + "Please contact Outerbounds if you need assistance resolving the issue."
-                    )
-                    print(msg, file=sys.stderr)
-                    self._result = {"ERROR": msg}
-                    return self._result
-            except (
-                requests.exceptions.ConnectionError,
-                requests.exceptions.ReadTimeout,
-            ) as e:
-                # ConnectionErrors are generally temporary errors like DNS resolution failures,
-                # timeouts etc.
-                print(
-                    "received error of type {}. Retrying...".format(type(e)),
-                    e,
-                    file=sys.stderr,
-                )
-                time.sleep(retry_delay)
-                retry_delay *= 2  # Double the delay for the next attempt
-                retry_delay += random.uniform(0, 1)  # Add jitter
-                retry_delay = min(retry_delay, 10)
-
-        def _poll():
-            poll_request_url = f"{NVCF_RESULT_ENDPOINT}/{invocation_id}"
-            attempts = 0
-            retry_delay = 1
-            while attempts < self.max_request_retries:
-                try:
-                    attempts += 1
-                    poll_response = requests.get(
-                        poll_request_url,
-                        headers=self._nim_metadata.get_headers_for_nvcf_request(),
-                    )
-                    if poll_response.status_code == 200:
-                        tf = time.time()
-                        self._log_stats(response, tf - t0)
-                        self._status = JobStatus.SUCCESSFUL
-                        self._result = poll_response.json()
-                        return self._result
-                    elif poll_response.status_code == 202:
-                        self._status = JobStatus.SUBMITTED
-                        return 202
-                    elif poll_response.status_code == 400:
-                        self._status = JobStatus.FAILED
-                        msg = (
-                            "[@nim ERROR] The OpenAI-compatible API returned a 400 status code. "
-                            + "Known causes include improper requests or prompts with too many tokens for the selected model. "
-                            + "Please contact Outerbounds if you need assistance resolving the issue."
-                        )
-                        print(msg, file=sys.stderr)
-                        self._result = {"@nim ERROR": msg}
-                        return self._result
-                except (
-                    requests.exceptions.ConnectionError,
-                    requests.exceptions.ReadTimeout,
-                ) as e:
-                    print(
-                        "received error of type {}. Retrying...".format(type(e)),
-                        e,
-                        file=sys.stderr,
-                    )
-                    time.sleep(retry_delay)
-                    retry_delay *= 2  # Double the delay for the next attempt
-                    retry_delay += random.uniform(0, 1)  # Add jitter
-                    retry_delay = min(retry_delay, 10)
-
-        while True:
-            data = _poll()
-            if data and data != 202:
-                return data
-            time.sleep(NVCF_POLL_INTERVAL_SECONDS)

ob-metaflow-extensions-1.1.155rc0/metaflow_extensions/outerbounds/plugins/nim/utilities.py

@@ -1,5 +0,0 @@
-NIM_MONITOR_LOCAL_STORAGE_ROOT = ".nim-monitor"
-
-
-def get_storage_path(task_id):
-    return f"{NIM_MONITOR_LOCAL_STORAGE_ROOT}/" + task_id + ".sqlite"

ob-metaflow-extensions-1.1.155rc0/ob_metaflow_extensions.egg-info/requires.txt

@@ -1,3 +0,0 @@
-boto3
-kubernetes
-ob-metaflow==2.15.11.2