ob-metaflow-extensions 1.1.123rc3__tar.gz → 1.1.125__tar.gz

{ob-metaflow-extensions-1.1.123rc3 → ob-metaflow-extensions-1.1.125}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.123rc3
+Version: 1.1.125
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob-metaflow-extensions-1.1.123rc3 → ob-metaflow-extensions-1.1.125}/metaflow_extensions/outerbounds/plugins/__init__.py

@@ -319,7 +319,6 @@ STEP_DECORATORS_DESC = [
     ("snowpark", ".snowpark.snowpark_decorator.SnowparkDecorator"),
     ("tensorboard", ".tensorboard.TensorboardDecorator"),
     ("gpu_profile", ".profilers.gpu_profile_decorator.GPUProfileDecorator"),
-    ("app_deploy", ".apps.deploy_decorator.WorkstationAppDeployDecorator"),
 ]
 FLOW_DECORATORS_DESC = [("nim", ".nim.NimDecorator")]
 TOGGLE_STEP_DECORATOR = [

{ob-metaflow-extensions-1.1.123rc3 → ob-metaflow-extensions-1.1.125}/metaflow_extensions/outerbounds/plugins/nvcf/exceptions.py

@@ -79,5 +79,16 @@ class NvcfTimeoutTooShortException(MetaflowException):
         super(NvcfTimeoutTooShortException, self).__init__(msg)
 
 
+class NvcfQueueTimeoutTooShortException(MetaflowException):
+    headline = "[@nvidia NvcfQueueTimeoutTooShortException] Queue Timeout too short"
+
+    def __init__(self, step):
+        msg = (
+            "The queue timeout for step *{step}* should be at least 60 seconds for "
+            "execution with @nvidia".format(step=step)
+        )
+        super(NvcfQueueTimeoutTooShortException, self).__init__(msg)
+
+
 class NvcfKilledException(MetaflowException):
     headline = "Nvidia job killed"

{ob-metaflow-extensions-1.1.123rc3 → ob-metaflow-extensions-1.1.125}/metaflow_extensions/outerbounds/plugins/nvcf/nvcf.py

@@ -23,29 +23,53 @@ STDOUT_PATH = os.path.join(LOGS_DIR, STDOUT_FILE)
 STDERR_PATH = os.path.join(LOGS_DIR, STDERR_FILE)
 
 
-RETRIABLE_STATUS_CODES = [500]
-
-
-def retry_on_status(status_codes=RETRIABLE_STATUS_CODES, max_retries=3, delay=1):
+def retry_on_status(status_codes=[500], max_retries=3, delay=1):
     def decorator(func):
         @wraps(func)
-        def wrapper(*args, **kwargs):
+        def wrapper(instance, *args, **kwargs):
             retries = 0
-            while retries <= max_retries:
+
+            # Determine retry limit upfront
+            use_queue_timeout = 504 in status_codes
+            if use_queue_timeout:
+                poll_seconds = int(instance._poll_seconds)
+                retry_limit = (
+                    instance._queue_timeout + (poll_seconds - 1)
+                ) // poll_seconds
+                remainder = instance._queue_timeout % poll_seconds
+                last_timeout = remainder if remainder != 0 else poll_seconds
+            else:
+                retry_limit = max_retries
+
+            while retries < retry_limit:
                 try:
-                    return func(*args, **kwargs)
+                    return func(instance, *args, **kwargs)
                 except HTTPError as e:
-                    if e.code in status_codes and retries < max_retries:
-                        retries += 1
-                        print(
-                            f"[@nvidia] Received {e.code} error, retrying ({retries}/{max_retries})..."
-                        )
+                    if e.code not in status_codes or retries >= retry_limit:
+                        instance._status = JobStatus.FAILED
+                        if e.code == 504 and retries >= retry_limit:
+                            raise NvcfPollingConnectionError(
+                                "Request timed out after all retries"
+                            )
+                        raise
+
+                    if e.code == 504 and retries == retry_limit - 1:
+                        instance._poll_seconds = str(last_timeout)
+
+                    print(
+                        f"[@nvidia] {'Queue timeout' if e.code == 504 else f'Received {e.code}'}, "
+                        f"retrying ({retries + 1}/{retry_limit})... with poll seconds as {instance._poll_seconds}"
+                    )
+
+                    if e.code != 504:
                         time.sleep(delay)
-                        continue
-                    raise
-                except Exception:
+
+                    retries += 1
+                except URLError as e:
+                    instance._status = JobStatus.FAILED
                     raise
-            return func(*args, **kwargs)
+            # final attempt
+            return func(instance, *args, **kwargs)
 
         return wrapper
 
@@ -53,12 +77,15 @@ def retry_on_status(status_codes=RETRIABLE_STATUS_CODES, max_retries=3, delay=1)
 
 
 class Nvcf(object):
-    def __init__(self, metadata, datastore, environment, function_id, ngc_api_key):
+    def __init__(
+        self, metadata, datastore, environment, function_id, ngc_api_key, queue_timeout
+    ):
         self.metadata = metadata
         self.datastore = datastore
         self.environment = environment
         self._function_id = function_id
         self._ngc_api_key = ngc_api_key
+        self._queue_timeout = queue_timeout
 
     def launch_job(
         self,
@@ -120,6 +147,7 @@ class Nvcf(object):
             self.datastore._storage_impl,
             self._function_id,
             self._ngc_api_key,
+            self._queue_timeout,
         )
         self.job.submit()
 
@@ -172,7 +200,9 @@ result_endpoint = f"{nvcf_url}/v2/nvcf/pexec/status"
 
 
 class Job(object):
-    def __init__(self, command, env, task_spec, backend, function_id, ngc_api_key):
+    def __init__(
+        self, command, env, task_spec, backend, function_id, ngc_api_key, queue_timeout
+    ):
         self._payload = {
             "command": command,
             "env": {k: v for k, v in env.items() if v is not None},
@@ -180,6 +210,8 @@ class Job(object):
         self._result = {}
         self._function_id = function_id
         self._ngc_api_key = ngc_api_key
+        self._queue_timeout = queue_timeout
+        self._poll_seconds = "3600"
 
         flow_name = task_spec.get("flow_name")
         run_id = task_spec.get("run_id")
@@ -214,11 +246,14 @@ class Job(object):
         )
         self.heartbeat_thread.start()
 
+    @retry_on_status(status_codes=[504])
     def submit(self):
         try:
             headers = {
                 "Authorization": f"Bearer {self._ngc_api_key}",
                 "Content-Type": "application/json",
+                "nvcf-feature-enable-gateway-timeout": "true",
+                "NVCF-POLL-SECONDS": self._poll_seconds,
             }
             request_data = json.dumps(self._payload).encode()
             request = Request(
@@ -239,12 +274,9 @@ class Job(object):
                 self._status = JobStatus.SUBMITTED
             else:
                 self._status = JobStatus.FAILED
-            # TODO: Handle 404s nicely
-        except (HTTPError, URLError) as e:
-            # TODO: If queue is full, wait in line and retry?
-            # without that, branching over concurrent requests causes error.
-            self._state = JobStatus.FAILED
-            raise e
+        except URLError:
+            self._status = JobStatus.FAILED
+            raise
 
     @property
     def status(self):
@@ -272,12 +304,15 @@ class Job(object):
     def result(self):
         return self._result
 
-    @retry_on_status(status_codes=RETRIABLE_STATUS_CODES, max_retries=3, delay=5)
+    @retry_on_status(status_codes=[500], max_retries=3, delay=5)
+    @retry_on_status(status_codes=[504])
     def _poll(self):
         try:
             headers = {
                 "Authorization": f"Bearer {self._ngc_api_key}",
                 "Content-Type": "application/json",
+                "nvcf-feature-enable-gateway-timeout": "true",
+                "NVCF-POLL-SECONDS": self._poll_seconds,
             }
             request = Request(
                 f"{result_endpoint}/{self._invocation_id}", headers=headers
@@ -296,11 +331,6 @@ class Job(object):
                     f"[@nvidia] Unexpected response code: {response.getcode()}. Please notify an Outerbounds support engineer if this error persists."
                 )
                 self._status = JobStatus.FAILED
-        # 4xx and 5xx responses go in 'except' block
-        except HTTPError as e:
-            if e.code not in RETRIABLE_STATUS_CODES:
-                self._status = JobStatus.FAILED
-            raise
         except URLError:
             self._status = JobStatus.FAILED
             raise

{ob-metaflow-extensions-1.1.123rc3 → ob-metaflow-extensions-1.1.125}/metaflow_extensions/outerbounds/plugins/nvcf/nvcf_cli.py

@@ -111,6 +111,9 @@ def kill(ctx, run_id):
 @click.argument("code-package-url")
 @click.option("--function-id", help="NVCF function id.")
 @click.option("--ngc-api-key", help="NGC API key.")
+@click.option(
+    "--queue-timeout", default=5 * 24 * 3600, help="Queue timeout in seconds."
+)
 @click.option("--run-id", help="Passed to the top-level 'step'.")
 @click.option("--task-id", help="Passed to the top-level 'step'.")
 @click.option("--input-paths", help="Passed to the top-level 'step'.")
@@ -133,6 +136,7 @@ def step(
     code_package_url,
     function_id,
     ngc_api_key,
+    queue_timeout,
     **kwargs,
 ):
     def echo(msg, stream="stderr", _id=None, **kwargs):
@@ -249,6 +253,7 @@ def step(
         ctx.obj.environment,
         function_id,
         ngc_api_key,
+        queue_timeout,
     )
     try:
         with ctx.obj.monitor.measure("metaflow.nvcf.launch_job"):

{ob-metaflow-extensions-1.1.123rc3 → ob-metaflow-extensions-1.1.125}/metaflow_extensions/outerbounds/plugins/nvcf/nvcf_decorator.py

@@ -20,6 +20,7 @@ from .exceptions import (
     UnsupportedNvcfConfigurationException,
     UnsupportedNvcfDatastoreException,
     NvcfTimeoutTooShortException,
+    NvcfQueueTimeoutTooShortException,
 )
 
 from metaflow.metaflow_config import SERVICE_URL
@@ -36,10 +37,16 @@ class NvcfDecorator(StepDecorator):
         Number of GPUs to use.
     gpu_type : str
         Type of Nvidia GPU to use.
+    queue_timeout : int
+        Time to keep the job in NVCF's queue.
     """
 
     name = "nvidia"
-    defaults = {"gpu": 1, "gpu_type": None}
+    defaults = {
+        "gpu": 1,
+        "gpu_type": None,
+        "queue_timeout": 5 * 24 * 3600,  # Default 5 days in seconds
+    }
 
     package_url = None
     package_sha = None
@@ -118,6 +125,10 @@ class NvcfDecorator(StepDecorator):
             )
         self.attributes["function_id"] = available_configurations[desired_configuration]
 
+        queue_timeout = self.attributes["queue_timeout"]
+        if not isinstance(queue_timeout, int) or queue_timeout < 60:
+            raise NvcfQueueTimeoutTooShortException(step)
+
     def runtime_init(self, flow, graph, package, run_id):
         # Set some more internal state.
         self.flow = flow
@@ -144,6 +155,7 @@ class NvcfDecorator(StepDecorator):
             cli_options = {
                 "function_id": self.attributes["function_id"],
                 "ngc_api_key": self.attributes["ngc_api_key"],
+                "queue_timeout": self.attributes["queue_timeout"],
             }
             cli_args.command_options.update(cli_options)
             cli_args.entrypoint[0] = sys.executable

{ob-metaflow-extensions-1.1.123rc3 → ob-metaflow-extensions-1.1.125}/ob_metaflow_extensions.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.123rc3
+Version: 1.1.125
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial

{ob-metaflow-extensions-1.1.123rc3 → ob-metaflow-extensions-1.1.125}/ob_metaflow_extensions.egg-info/SOURCES.txt

@@ -6,10 +6,6 @@ metaflow_extensions/outerbounds/config/__init__.py
 metaflow_extensions/outerbounds/plugins/__init__.py
 metaflow_extensions/outerbounds/plugins/auth_server.py
 metaflow_extensions/outerbounds/plugins/perimeters.py
-metaflow_extensions/outerbounds/plugins/apps/__init__.py
-metaflow_extensions/outerbounds/plugins/apps/app_utils.py
-metaflow_extensions/outerbounds/plugins/apps/deploy_decorator.py
-metaflow_extensions/outerbounds/plugins/apps/supervisord_utils.py
 metaflow_extensions/outerbounds/plugins/fast_bakery/__init__.py
 metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery.py

ob-metaflow-extensions-1.1.125/ob_metaflow_extensions.egg-info/requires.txt

@@ -0,0 +1,3 @@
+boto3
+kubernetes
+ob-metaflow==2.13.9.1

{ob-metaflow-extensions-1.1.123rc3 → ob-metaflow-extensions-1.1.125}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
 from pathlib import Path
 
 
-version = "1.1.123rc3"
+version = "1.1.125"
 this_directory = Path(__file__).parent
 long_description = (this_directory / "README.md").read_text()
 
@@ -18,6 +18,6 @@ setup(
     install_requires=[
         "boto3",
         "kubernetes",
-        "ob-metaflow == 2.13.8.1",
+        "ob-metaflow == 2.13.9.1",
     ],
 )

ob-metaflow-extensions-1.1.123rc3/metaflow_extensions/outerbounds/plugins/apps/app_utils.py

@@ -1,170 +0,0 @@
-from metaflow.exception import MetaflowException
-import os
-from metaflow.metaflow_config_funcs import init_config
-import requests
-import time
-import random
-
-# IMPORTANT: Currently contents of this file are mostly duplicated from the outerbounds package.
-# This is purely due to the time rush of having to deliver this feature. As a fast forward, we
-# will reorganize things in a way that the amount of duplication in minimum.
-
-
-APP_READY_POLL_TIMEOUT_SECONDS = 300
-# Even after our backend validates that the app routes are ready, it takes a few seconds for
-# the app to be accessible via the browser. Till we hunt down this delay, add an extra buffer.
-APP_READY_EXTRA_BUFFER_SECONDS = 30
-
-
-def start_app(port=-1, name=""):
-    if len(name) == 0 or len(name) >= 20:
-        raise MetaflowException("App name should not be more than 20 characters long.")
-    elif not name.isalnum() or not name.islower():
-        raise MetaflowException(
-            "App name can only contain lowercase alphanumeric characters."
-        )
-
-    if "WORKSTATION_ID" not in os.environ:
-        raise MetaflowException(
-            "All outerbounds app commands can only be run from a workstation."
-        )
-
-    workstation_id = os.environ["WORKSTATION_ID"]
-
-    try:
-        try:
-            conf = init_config()
-            metaflow_token = conf["METAFLOW_SERVICE_AUTH_KEY"]
-            api_url = conf["OBP_API_SERVER"]
-
-            workstations_response = requests.get(
-                f"https://{api_url}/v1/workstations",
-                headers={"x-api-key": metaflow_token},
-            )
-            workstations_response.raise_for_status()
-        except:
-            raise MetaflowException("Failed to list workstations!")
-
-        workstations_json = workstations_response.json()["workstations"]
-        for workstation in workstations_json:
-            if workstation["instance_id"] == os.environ["WORKSTATION_ID"]:
-                if "named_ports" in workstation["spec"]:
-                    try:
-                        ensure_app_start_request_is_valid(
-                            workstation["spec"]["named_ports"], port, name
-                        )
-                    except ValueError as e:
-                        raise MetaflowException(str(e))
-
-                    for named_port in workstation["spec"]["named_ports"]:
-                        if int(named_port["port"]) == port:
-                            if named_port["enabled"] and named_port["name"] == name:
-                                print(f"App {name} started on port {port}!")
-                                print(
-                                    f"Browser URL: https://{api_url.replace('api', 'ui')}/apps/{os.environ['WORKSTATION_ID']}/{name}/"
-                                )
-                                print(
-                                    f"API URL: https://{api_url}/apps/{os.environ['WORKSTATION_ID']}/{name}/"
-                                )
-                                return
-                            else:
-                                try:
-                                    response = requests.put(
-                                        f"https://{api_url}/v1/workstations/update/{workstation_id}/namedports",
-                                        headers={"x-api-key": metaflow_token},
-                                        json={
-                                            "port": port,
-                                            "name": name,
-                                            "enabled": True,
-                                        },
-                                    )
-
-                                    response.raise_for_status()
-                                    poll_success = wait_for_app_port_to_be_accessible(
-                                        api_url,
-                                        metaflow_token,
-                                        workstation_id,
-                                        name,
-                                        APP_READY_POLL_TIMEOUT_SECONDS,
-                                    )
-                                    if poll_success:
-                                        print(f"App {name} started on port {port}!")
-                                        print(
-                                            f"Browser URL: https://{api_url.replace('api', 'ui')}/apps/{os.environ['WORKSTATION_ID']}/{name}/"
-                                        )
-                                        print(
-                                            f"API URL: https://{api_url}/apps/{os.environ['WORKSTATION_ID']}/{name}/"
-                                        )
-                                    else:
-                                        raise MetaflowException(
-                                            f"The app could not be deployed in {APP_READY_POLL_TIMEOUT_SECONDS / 60} minutes. Please try again later."
-                                        )
-                                except Exception:
-                                    raise MetaflowException(
-                                        f"Failed to start app {name} on port {port}!"
-                                    )
-    except Exception as e:
-        raise MetaflowException(f"Failed to start app {name} on port {port}!")
-
-
-def ensure_app_start_request_is_valid(existing_named_ports, port: int, name: str):
-    existing_apps_by_port = {np["port"]: np for np in existing_named_ports}
-
-    if port not in existing_apps_by_port:
-        raise MetaflowException(f"Port {port} not found on workstation")
-
-    for existing_named_port in existing_named_ports:
-        if (
-            name == existing_named_port["name"]
-            and existing_named_port["port"] != port
-            and existing_named_port["enabled"]
-        ):
-            raise MetaflowException(
-                f"App with name '{name}' is already deployed on port {existing_named_port['port']}"
-            )
-
-
-def wait_for_app_port_to_be_accessible(
-    api_url, metaflow_token, workstation_id, app_name, poll_timeout_seconds
-) -> bool:
-    num_retries_per_request = 3
-    start_time = time.time()
-    retry_delay = 1.0
-    poll_interval = 10
-    wait_message = f"App {app_name} is currently being deployed..."
-    while time.time() - start_time < poll_timeout_seconds:
-        for _ in range(num_retries_per_request):
-            try:
-                workstations_response = requests.get(
-                    f"https://{api_url}/v1/workstations",
-                    headers={"x-api-key": metaflow_token},
-                )
-                workstations_response.raise_for_status()
-                if is_app_ready(workstations_response.json(), workstation_id, app_name):
-                    print(wait_message)
-                    time.sleep(APP_READY_EXTRA_BUFFER_SECONDS)
-                    return True
-                else:
-                    print(wait_message)
-                    time.sleep(poll_interval)
-            except (
-                requests.exceptions.ConnectionError,
-                requests.exceptions.ReadTimeout,
-            ):
-                time.sleep(retry_delay)
-                retry_delay *= 2  # Double the delay for the next attempt
-                retry_delay += random.uniform(0, 1)  # Add jitter
-                retry_delay = min(retry_delay, 10)
-    return False
-
-
-def is_app_ready(response_json: dict, workstation_id: str, app_name: str) -> bool:
-    """Checks if the app is ready in the given workstation's response."""
-    workstations = response_json.get("workstations", [])
-    for workstation in workstations:
-        if workstation.get("instance_id") == workstation_id:
-            hosted_apps = workstation.get("status", {}).get("hosted_apps", [])
-            for hosted_app in hosted_apps:
-                if hosted_app.get("name") == app_name:
-                    return bool(hosted_app.get("ready"))
-    return False

ob-metaflow-extensions-1.1.123rc3/metaflow_extensions/outerbounds/plugins/apps/deploy_decorator.py

@@ -1,154 +0,0 @@
-from metaflow.exception import MetaflowException
-from metaflow.decorators import StepDecorator
-from metaflow import current
-from .app_utils import start_app
-from .supervisord_utils import SupervisorClient, SupervisorClientException
-import os
-import random
-import string
-import tempfile
-import sys
-
-DEFAULT_WAIT_TIME_SECONDS_FOR_PROCESS_TO_START = 10
-BASE_DIR_FOR_APP_ASSETS = "/home/ob-workspace/.appdaemon/apps/"
-
-
-class WorkstationAppDeployDecorator(StepDecorator):
-    """
-    Specifies that this step is used to deploy an instance of the app.
-    Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
-
-    Parameters
-    ----------
-    app_port : int
-        Number of GPUs to use.
-    app_name : str
-        Name of the app to deploy.
-    """
-
-    name = "app_deploy"
-    defaults = {"app_port": 8080, "app_name": "app"}
-
-    def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
-        if any([deco.name == "kubernetes" for deco in decos]):
-            raise MetaflowException(
-                "@app_deploy decorator is only supported locally and does not work with remote execution environments like @kubernetes, @nvidia."
-            )
-
-        # We always need to have some environment defined through the flow to deploy and app.
-        # Which means either step decorators like @pypi / @conda must be defined.
-        # or flow level decorators like @conda_base / @pypi_base.
-        if not any([deco.name == "pypi" or deco.name == "conda" for deco in decos]):
-            flow_decorators = flow._flow_decorators.keys()
-            if (
-                "conda_base" not in flow_decorators
-                and "pypi_base" not in flow_decorators
-            ):
-                raise MetaflowException(
-                    "@app_deploy requires either step decorators like @pypi / @conda or flow level decorators like @conda_base / @pypi_base to be defined."
-                )
-
-        app_port = self.attributes["app_port"]
-        app_name = self.attributes["app_name"]
-
-        # Currently this decorator is expected to only execute on workstation.
-        if app_port is None or app_port < 6000 or app_port > 6002:
-            raise MetaflowException(
-                "AppDeployDecorator requires app_port to be between 6000 and 6002."
-            )
-
-        if app_name is None:
-            raise MetaflowException("AppDeployDecorator requires app_name to be set.")
-
-    def task_pre_step(
-        self,
-        step_name,
-        task_datastore,
-        metadata,
-        run_id,
-        task_id,
-        flow,
-        graph,
-        retry_count,
-        max_user_code_retries,
-        ubf_context,
-        inputs,
-    ):
-        os.makedirs(BASE_DIR_FOR_APP_ASSETS, exist_ok=True)
-        # First we want to create a directory where the user's app directory and artifacts can be stored.
-        with tempfile.TemporaryDirectory(
-            prefix=BASE_DIR_FOR_APP_ASSETS, delete=False
-        ) as temp_dir:
-            launch_temp_dir = temp_dir
-
-        # Expose this to the user, so that they can use it write their artifacts.
-        setattr(flow, "deploy_dir", launch_temp_dir)
-
-        # Make sure to record deploy_dir so that the user cannot accidentally override it.
-        self._deploy_dir = launch_temp_dir
-
-    def task_post_step(
-        self, step_name, flow, graph, retry_count, max_user_code_retries
-    ):
-        deploy_dir = self._deploy_dir
-
-        # By default we assume that the user has a __main__.py file in their app directory.
-        # They can always override this behavior.
-        user_provided_entrypoint = getattr(flow, "entrypoint", None)
-
-        if user_provided_entrypoint is not None and not isinstance(
-            user_provided_entrypoint, str
-        ):
-            raise MetaflowException(
-                f"@app_deploy requires entrypoint to be set to a string. The current value of entrypoint {user_provided_entrypoint} is not valid."
-            )
-
-        flow_directory = os.path.dirname(os.path.abspath(sys.argv[0]))
-
-        # By default, we assume that the layout of the flow directory is:
-        # flow_dir/
-        # - deployer_flow.py
-        # - my_custom_app/
-        #   - __main__.py
-        #   - other_files
-        #   - other_dirs/
-        # This can be overridden by the user by setting the app_dir attribute.
-        # None of this matters if the user provides a custom entrypoint, since in that case we don't copy
-        # anything anywhere.
-        app_location = getattr(
-            flow, "app_dir", os.path.join(flow_directory, self.attributes["app_name"])
-        )
-
-        if user_provided_entrypoint is None and not os.path.exists(app_location):
-            raise MetaflowException(f"App directory {app_location} does not exist.")
-
-        wait_time_for_app_start = getattr(
-            flow,
-            "wait_time_for_app_start",
-            DEFAULT_WAIT_TIME_SECONDS_FOR_PROCESS_TO_START,
-        )
-
-        try:
-            supervisor_client = SupervisorClient(
-                wait_time_seconds_for_app_start=wait_time_for_app_start
-            )
-
-            # First, let's deploy the app.
-            start_app(
-                port=self.attributes["app_port"], name=self.attributes["app_name"]
-            )
-
-            # Now, let's add the app to supervisor.
-            supervisor_client.start_process_with_supervisord(
-                self.attributes["app_name"],
-                self.attributes["app_port"],
-                user_provided_entrypoint,
-                deploy_dir,
-                app_location,
-            )
-        except SupervisorClientException as e:
-            raise MetaflowException(str(e))
-        except Exception as e:
-            raise MetaflowException(
-                f"Failed to start {self.attributes['app_name']}! Cause: {str(e)}"
-            ) from e

ob-metaflow-extensions-1.1.123rc3/metaflow_extensions/outerbounds/plugins/apps/supervisord_utils.py

@@ -1,239 +0,0 @@
-import argparse
-import os
-import configparser
-import tempfile
-import sys
-import subprocess
-from pathlib import Path
-import shutil
-from enum import Enum
-import time
-
-
-class SupervisorClientException(Exception):
-    pass
-
-
-class SupervisorClient:
-    class SupervisodProcessCodes(Enum):
-        STOPPED = 0
-        STARTING = 10
-        RUNNING = 20
-        BACKOFF = 30
-        STOPPING = 40
-        EXITED = 100
-        FATAL = 200
-        UNKNOWN = 1000
-
-    def __init__(self, wait_time_seconds_for_app_start: int):
-        self.supervisor_conf_loc = os.environ.get("SUPERVISOR_CONF_PATH")
-
-        self.wait_time_seconds_for_app_start = wait_time_seconds_for_app_start
-        if self.supervisor_conf_loc is None or not os.path.exists(
-            self.supervisor_conf_loc
-        ):
-            raise SupervisorClientException(
-                "This workstation does not support deploying apps! Please reach out to Outerbounds for support."
-            )
-
-        self.metaflow_envs_persistent_path = os.environ.get(
-            "SUPERVISOR_PYTHON_ENVS_PATH"
-        )
-        if self.metaflow_envs_persistent_path is None:
-            raise SupervisorClientException(
-                "This workstation does not support deploying apps! Please reach out to Outerbounds for support."
-            )
-
-        # Check if supervisorctl is installed
-        if not shutil.which("supervisorctl"):
-            raise SupervisorClientException(
-                "This workstation does not support deploying apps! Please reach out to Outerbounds for support."
-            )
-
-    def _stop_existing_app_at_port(self, app_port):
-        supervisor_config = configparser.ConfigParser()
-        supervisor_config.read(self.supervisor_conf_loc)
-
-        for program in supervisor_config.sections():
-            if "obp_app_port" in supervisor_config[program]:
-                if supervisor_config[program]["obp_app_port"].strip() == str(app_port):
-                    res = subprocess.run(
-                        ["supervisorctl", "stop", program],
-                        stdout=subprocess.DEVNULL,
-                        stderr=subprocess.DEVNULL,
-                    )
-
-                    del supervisor_config[program]
-
-        with tempfile.NamedTemporaryFile(
-            "w", dir=os.path.dirname(self.supervisor_conf_loc), delete=False
-        ) as f:
-            supervisor_config.write(f)
-            tmp_file = f.name
-
-        os.rename(tmp_file, self.supervisor_conf_loc)
-
-    def start_process_with_supervisord(
-        self,
-        app_name,
-        app_port,
-        user_provided_entrypoint,
-        deploy_dir=None,
-        app_dir=None,
-    ):
-        """
-        Add a new program entry to supervisor configuration.
-
-        Args:
-            app_name: The name of the app to start.
-            entrypoint: The entrypoint to start the app with.
-            directory: The directory to run the app in.
-            deploy_dir: The directory to copy the app to and deploy from.
-            app_dir: The directory to copy the app from.
-        """
-
-        entrypoint = user_provided_entrypoint
-        deploy_dir_for_port = "/home/ob-workspace/.appdaemon/apps/6000"
-        launch_directory = (
-            "/home/ob-workspace/.appdaemon/apps"
-            if entrypoint is None
-            else "/home/ob-workspace/.appdaemon"
-        )
-
-        # Step 1: Stop any existing apps that are running on the same port.
-        self._stop_existing_app_at_port(app_port)
-
-        if user_provided_entrypoint is None:
-            # Step 2: Copy the app_dir to the deploy_dir.
-            recursive_copy(app_dir, deploy_dir)
-
-            # Step 3: Copy the entire deploy_dir to the port specific directory.
-            if os.path.exists(deploy_dir_for_port):
-                shutil.rmtree(deploy_dir_for_port)
-
-            os.makedirs(deploy_dir_for_port)
-            recursive_copy(deploy_dir, deploy_dir_for_port)
-
-            # Apply default value
-            entrypoint = f"-m {str(app_port)}"
-
-        shutil.rmtree(deploy_dir)
-
-        persistent_path_for_executable = (
-            self.persist_metaflow_generated_python_environment()
-        )
-
-        command = f"{persistent_path_for_executable} {entrypoint}"
-
-        entry = {
-            "command": command,
-            "directory": launch_directory,
-            "autostart": "true",
-            "autorestart": "true",
-            "obp_app_port": app_port,  # Record the app port for internal reference. This is not used by supervisor.
-        }
-
-        supervisor_config = configparser.ConfigParser()
-        supervisor_config.read(self.supervisor_conf_loc)
-
-        supervisor_config[f"program:{app_name}"] = entry
-
-        with tempfile.NamedTemporaryFile(
-            "w", dir=os.path.dirname(self.supervisor_conf_loc), delete=False
-        ) as f:
-            supervisor_config.write(f)
-            tmp_file = f.name
-
-        os.rename(tmp_file, self.supervisor_conf_loc)
-
-        # Execute supervisorctl reload
-        # Capture the exit code
-        exit_code = subprocess.run(
-            ["supervisorctl", "reload"],
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-        ).returncode
-        if exit_code != 0:
-            print("Failed to reload supervisor configuration!", file=sys.stderr)
-            return
-
-        print(
-            f"Waiting for {self.wait_time_seconds_for_app_start} seconds for {app_name} to start..."
-        )
-        time.sleep(self.wait_time_seconds_for_app_start)
-        status = self._get_launched_prcoess_status(app_name)
-
-        if status not in [
-            self.SupervisodProcessCodes.RUNNING,
-            self.SupervisodProcessCodes.STARTING,
-        ]:
-            raise SupervisorClientException(
-                f"Failed to start {app_name}! Try running {command} manually to debug."
-            )
-
-    def _get_launched_prcoess_status(self, app_name):
-        status_cmd_output = subprocess.run(
-            ["supervisorctl", "status", app_name],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        ).stdout.decode("utf-8")
-
-        status_cmd_output_parts = [
-            x.strip() for x in status_cmd_output.split(" ") if x.strip()
-        ]
-
-        status_str = status_cmd_output_parts[1]
-
-        if status_str == "RUNNING":
-            return self.SupervisodProcessCodes.RUNNING
-        elif status_str == "STOPPED":
-            return self.SupervisodProcessCodes.STOPPED
-        elif status_str == "STARTING":
-            return self.SupervisodProcessCodes.STARTING
-        elif status_str == "BACKOFF":
-            return self.SupervisodProcessCodes.BACKOFF
-        elif status_str == "STOPPING":
-            return self.SupervisodProcessCodes.STOPPING
-        elif status_str == "EXITED":
-            return self.SupervisodProcessCodes.EXITED
-        elif status_str == "FATAL":
-            return self.SupervisodProcessCodes.FATAL
-        else:
-            return self.SupervisodProcessCodes.UNKNOWN
-
-    # By default, an environment generated by metaflow will end up in a path like: /root/micromamba/envs/metaflow/linux-64/02699a4d2d50cfc/bin/python
-    # However, on a workstation these environments are not persisted, so we need to copy them over to /home/ob-workspace
-    def persist_metaflow_generated_python_environment(self):
-        current_executable = sys.executable
-        environment_path = Path(current_executable).parent.parent
-
-        persistent_path_for_this_environment = os.path.join(
-            self.metaflow_envs_persistent_path,
-            environment_path.parent.name,
-            environment_path.name,
-        )
-
-        final_executable_path = os.path.join(
-            persistent_path_for_this_environment,
-            Path(current_executable).parent.name,
-            Path(current_executable).name,
-        )
-
-        if os.path.exists(final_executable_path):
-            return final_executable_path
-
-        os.makedirs(persistent_path_for_this_environment, exist_ok=True)
-
-        recursive_copy(environment_path, persistent_path_for_this_environment)
-
-        return final_executable_path
-
-
-def recursive_copy(src, dst):
-    for item in os.listdir(src):
-        s = os.path.join(src, item)
-        d = os.path.join(dst, item)
-        if os.path.isdir(s):
-            shutil.copytree(s, d, dirs_exist_ok=True)
-        else:
-            shutil.copy2(s, d)

ob-metaflow-extensions-1.1.123rc3/ob_metaflow_extensions.egg-info/requires.txt

@@ -1,3 +0,0 @@
-boto3
-kubernetes
-ob-metaflow==2.13.8.1