ob-metaflow-extensions 1.1.123rc0__py2.py3-none-any.whl → 1.1.123rc2__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/deploy_decorator.py

@@ -4,8 +4,13 @@ from metaflow import current
 from .app_utils import start_app
 from .supervisord_utils import SupervisorClient, SupervisorClientException
 import os
+import random
+import string
+import tempfile
+import sys
 
 DEFAULT_WAIT_TIME_SECONDS_FOR_PROCESS_TO_START = 10
+BASE_DIR_FOR_APP_ASSETS = "/home/ob-workspace/.appdaemon/apps/"
 
 
 class WorkstationAppDeployDecorator(StepDecorator):
@@ -31,6 +36,19 @@ class WorkstationAppDeployDecorator(StepDecorator):
                 "Nvidia. Please use one or the other.".format(step=step)
             )
 
+        # We always need to have some environment defined through the flow to deploy and app.
+        # Which means either step decorators like @pypi / @conda must be defined.
+        # or flow level decorators like @conda_base / @pypi_base.
+        if not any([deco.name == "pypi" or deco.name == "conda" for deco in decos]):
+            flow_decorators = flow._flow_decorators.keys()
+            if (
+                "conda_base" not in flow_decorators
+                and "pypi_base" not in flow_decorators
+            ):
+                raise MetaflowException(
+                    "@app_deploy requires either step decorators like @pypi / @conda or flow level decorators like @conda_base / @pypi_base to be defined."
+                )
+
         app_port = self.attributes["app_port"]
         app_name = self.attributes["app_name"]
 
@@ -43,29 +61,74 @@ class WorkstationAppDeployDecorator(StepDecorator):
         if app_name is None:
             raise MetaflowException("AppDeployDecorator requires app_name to be set.")
 
+    def task_pre_step(
+        self,
+        step_name,
+        task_datastore,
+        metadata,
+        run_id,
+        task_id,
+        flow,
+        graph,
+        retry_count,
+        max_user_code_retries,
+        ubf_context,
+        inputs,
+    ):
+        os.makedirs(BASE_DIR_FOR_APP_ASSETS, exist_ok=True)
+        # First we want to create a directory where the user's app directory and artifacts can be stored.
+        with tempfile.TemporaryDirectory(
+            prefix=BASE_DIR_FOR_APP_ASSETS, delete=False
+        ) as temp_dir:
+            launch_temp_dir = temp_dir
+
+        # Expose this to the user, so that they can use it write their artifacts.
+        setattr(flow, "deploy_dir", launch_temp_dir)
+
+        # Make sure to record deploy_dir so that the user cannot accidentally override it.
+        self._deploy_dir = launch_temp_dir
+
     def task_post_step(
         self, step_name, flow, graph, retry_count, max_user_code_retries
     ):
-        entrypoint = getattr(flow, "entrypoint", None)
+        deploy_dir = self._deploy_dir
+
+        # By default we assume that the user has a __main__.py file in their app directory.
+        # They can always override this behavior.
+        user_provided_entrypoint = getattr(flow, "entrypoint", None)
+
+        if user_provided_entrypoint is not None and not isinstance(
+            user_provided_entrypoint, str
+        ):
+            raise MetaflowException(
+                f"@app_deploy requires entrypoint to be set to a string. The current value of entrypoint {user_provided_entrypoint} is not valid."
+            )
+
+        flow_directory = os.path.dirname(os.path.abspath(sys.argv[0]))
+
+        # By default, we assume that the layout of the flow directory is:
+        # flow_dir/
+        # - deployer_flow.py
+        # - my_custom_app/
+        #   - __main__.py
+        #   - other_files
+        #   - other_dirs/
+        # This can be overridden by the user by setting the app_dir attribute.
+        # None of this matters if the user provides a custom entrypoint, since in that case we don't copy
+        # anything anywhere.
+        app_location = getattr(
+            flow, "app_dir", os.path.join(flow_directory, self.attributes["app_name"])
+        )
+
+        if user_provided_entrypoint is None and not os.path.exists(app_location):
+            raise MetaflowException(f"App directory {app_location} does not exist.")
+
         wait_time_for_app_start = getattr(
             flow,
             "wait_time_for_app_start",
             DEFAULT_WAIT_TIME_SECONDS_FOR_PROCESS_TO_START,
         )
 
-        if entrypoint is None or not isinstance(entrypoint, str):
-            raise MetaflowException(
-                f"@app_deploy requires entrypoint to be set to a string. The current value of entrypoint {entrypoint} is not valid."
-            )
-
-        launch_dir = getattr(flow, "launch_dir", None)
-        if launch_dir is None or launch_dir == "":
-            raise MetaflowException("@app_deploy requires launch_dir to be set.")
-        elif not isinstance(launch_dir, str) or not os.path.exists(launch_dir):
-            raise MetaflowException(
-                f"@app_deploy requires launch_dir to be set to a valid directory. The current value of launch_dir {launch_dir} is not valid."
-            )
-
         try:
             supervisor_client = SupervisorClient(
                 wait_time_seconds_for_app_start=wait_time_for_app_start
@@ -79,8 +142,10 @@ class WorkstationAppDeployDecorator(StepDecorator):
             # Now, let's add the app to supervisor.
             supervisor_client.start_process_with_supervisord(
                 self.attributes["app_name"],
-                entrypoint,
-                launch_dir,
+                self.attributes["app_port"],
+                user_provided_entrypoint,
+                deploy_dir,
+                app_location,
             )
         except SupervisorClientException as e:
             raise MetaflowException(str(e))

metaflow_extensions/outerbounds/plugins/apps/supervisord_utils.py

@@ -50,8 +50,74 @@ class SupervisorClient:
                 "This workstation does not support deploying apps! Please reach out to Outerbounds for support."
             )
 
-    def start_process_with_supervisord(self, app_name, entrypoint, directory=None):
-        """Add a new program entry to supervisor configuration."""
+    def _stop_existing_app_at_port(self, app_port):
+        supervisor_config = configparser.ConfigParser()
+        supervisor_config.read(self.supervisor_conf_loc)
+
+        for program in supervisor_config.sections():
+            if "obp_app_port" in supervisor_config[program]:
+                if supervisor_config[program]["obp_app_port"].strip() == str(app_port):
+                    res = subprocess.run(
+                        ["supervisorctl", "stop", program],
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL,
+                    )
+
+                    del supervisor_config[program]
+
+        with tempfile.NamedTemporaryFile(
+            "w", dir=os.path.dirname(self.supervisor_conf_loc), delete=False
+        ) as f:
+            supervisor_config.write(f)
+            tmp_file = f.name
+
+        os.rename(tmp_file, self.supervisor_conf_loc)
+
+    def start_process_with_supervisord(
+        self,
+        app_name,
+        app_port,
+        user_provided_entrypoint,
+        deploy_dir=None,
+        app_dir=None,
+    ):
+        """
+        Add a new program entry to supervisor configuration.
+
+        Args:
+            app_name: The name of the app to start.
+            entrypoint: The entrypoint to start the app with.
+            directory: The directory to run the app in.
+            deploy_dir: The directory to copy the app to and deploy from.
+            app_dir: The directory to copy the app from.
+        """
+
+        entrypoint = user_provided_entrypoint
+        deploy_dir_for_port = "/home/ob-workspace/.appdaemon/apps/6000"
+        launch_directory = (
+            "/home/ob-workspace/.appdaemon/apps"
+            if entrypoint is None
+            else "/home/ob-workspace/.appdaemon"
+        )
+
+        # Step 1: Stop any existing apps that are running on the same port.
+        self._stop_existing_app_at_port(app_port)
+
+        if user_provided_entrypoint is None:
+            # Step 2: Copy the app_dir to the deploy_dir.
+            recursive_copy(app_dir, deploy_dir)
+
+            # Step 3: Copy the entire deploy_dir to the port specific directory.
+            if os.path.exists(deploy_dir_for_port):
+                shutil.rmtree(deploy_dir_for_port)
+
+            os.makedirs(deploy_dir_for_port)
+            recursive_copy(deploy_dir, deploy_dir_for_port)
+
+            # Apply default value
+            entrypoint = f"-m {str(app_port)}"
+
+        shutil.rmtree(deploy_dir)
 
         persistent_path_for_executable = (
             self.persist_metaflow_generated_python_environment()
@@ -61,9 +127,10 @@ class SupervisorClient:
 
         entry = {
             "command": command,
-            "directory": directory or os.getcwd(),
+            "directory": launch_directory,
             "autostart": "true",
             "autorestart": "true",
+            "obp_app_port": app_port,  # Record the app port for internal reference. This is not used by supervisor.
         }
 
         supervisor_config = configparser.ConfigParser()
@@ -76,6 +143,7 @@ class SupervisorClient:
         ) as f:
             supervisor_config.write(f)
             tmp_file = f.name
+
         os.rename(tmp_file, self.supervisor_conf_loc)
 
         # Execute supervisorctl reload
@@ -156,12 +224,16 @@ class SupervisorClient:
 
         os.makedirs(persistent_path_for_this_environment, exist_ok=True)
 
-        for item in os.listdir(environment_path):
-            src_path = os.path.join(environment_path, item)
-            dst_path = os.path.join(persistent_path_for_this_environment, item)
-            if os.path.isdir(src_path):
-                shutil.copytree(src_path, dst_path, dirs_exist_ok=True)
-            else:
-                shutil.copy2(src_path, dst_path)
+        recursive_copy(environment_path, persistent_path_for_this_environment)
 
         return final_executable_path
+
+
+def recursive_copy(src, dst):
+    for item in os.listdir(src):
+        s = os.path.join(src, item)
+        d = os.path.join(dst, item)
+        if os.path.isdir(s):
+            shutil.copytree(s, d, dirs_exist_ok=True)
+        else:
+            shutil.copy2(s, d)

metaflow_extensions/outerbounds/plugins/nvcf/nvcf.py

@@ -1,8 +1,9 @@
 import json
 import os
+import time
 import threading
-from urllib.parse import urlparse
 from urllib.request import HTTPError, Request, URLError, urlopen
+from functools import wraps
 
 from metaflow import util
 from metaflow.mflog import (
@@ -12,7 +13,6 @@ from metaflow.mflog import (
     tail_logs,
     get_log_tailer,
 )
-import requests
 from .exceptions import NvcfJobFailedException, NvcfPollingConnectionError
 
 # Redirect structured logs to $PWD/.logs/
@@ -23,6 +23,35 @@ STDOUT_PATH = os.path.join(LOGS_DIR, STDOUT_FILE)
 STDERR_PATH = os.path.join(LOGS_DIR, STDERR_FILE)
 
 
+RETRIABLE_STATUS_CODES = [500]
+
+
+def retry_on_status(status_codes=RETRIABLE_STATUS_CODES, max_retries=3, delay=1):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            retries = 0
+            while retries <= max_retries:
+                try:
+                    return func(*args, **kwargs)
+                except HTTPError as e:
+                    if e.code in status_codes and retries < max_retries:
+                        retries += 1
+                        print(
+                            f"[@nvidia] Received {e.code} error, retrying ({retries}/{max_retries})..."
+                        )
+                        time.sleep(delay)
+                        continue
+                    raise
+                except Exception:
+                    raise
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
 class Nvcf(object):
     def __init__(self, metadata, datastore, environment, function_id, ngc_api_key):
         self.metadata = metadata
@@ -220,7 +249,11 @@ class Job(object):
     @property
     def status(self):
         if self._status not in [JobStatus.SUCCESSFUL, JobStatus.FAILED]:
-            self._poll()
+            try:
+                self._poll()
+            except (HTTPError, URLError) as e:
+                self._status = JobStatus.FAILED
+                raise NvcfPollingConnectionError(e)
         return self._status
 
     @property
@@ -239,6 +272,7 @@ class Job(object):
     def result(self):
         return self._result
 
+    @retry_on_status(status_codes=RETRIABLE_STATUS_CODES, max_retries=3, delay=5)
     def _poll(self):
         try:
             headers = {
@@ -257,11 +291,16 @@ class Job(object):
                 else:
                     self._status = JobStatus.FAILED
                 self._result = data
-            elif response.getcode() in [400, 500]:
-                self._status = JobStatus.FAILED
             elif response.getcode() != 202:
                 print(
                     f"[@nvidia] Unexpected response code: {response.getcode()}. Please notify an Outerbounds support engineer if this error persists."
                 )
-        except (HTTPError, URLError) as e:
-            raise NvcfPollingConnectionError(e)
+                self._status = JobStatus.FAILED
+        # 4xx and 5xx responses go in 'except' block
+        except HTTPError as e:
+            if e.code not in RETRIABLE_STATUS_CODES:
+                self._status = JobStatus.FAILED
+            raise
+        except URLError:
+            self._status = JobStatus.FAILED
+            raise

{ob_metaflow_extensions-1.1.123rc0.dist-info → ob_metaflow_extensions-1.1.123rc2.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.123rc0
+Version: 1.1.123rc2
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial
 Description-Content-Type: text/markdown
 Requires-Dist: boto3
 Requires-Dist: kubernetes
-Requires-Dist: ob-metaflow (==2.13.7.1)
+Requires-Dist: ob-metaflow (==2.13.8.1)
 
 # Outerbounds platform package
 

{ob_metaflow_extensions-1.1.123rc0.dist-info → ob_metaflow_extensions-1.1.123rc2.dist-info}/RECORD

@@ -6,8 +6,8 @@ metaflow_extensions/outerbounds/plugins/auth_server.py,sha256=_Q9_2EL0Xy77bCRphk
 metaflow_extensions/outerbounds/plugins/perimeters.py,sha256=QXh3SFP7GQbS-RAIxUOPbhPzQ7KDFVxZkTdKqFKgXjI,2697
 metaflow_extensions/outerbounds/plugins/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow_extensions/outerbounds/plugins/apps/app_utils.py,sha256=JrVKlbRx8-nSmI4cRrB7F8BQGDHleABIYZudK4P-XFE,7905
-metaflow_extensions/outerbounds/plugins/apps/deploy_decorator.py,sha256=goD50XtFOrTl8jsHazdYbu6owVzReOL_1YeukOgX75E,3439
-metaflow_extensions/outerbounds/plugins/apps/supervisord_utils.py,sha256=1ImWHiQ4GNIyognIg-IT-cpInS1-CKWCEy2h9upZufo,5975
+metaflow_extensions/outerbounds/plugins/apps/deploy_decorator.py,sha256=oHCkcXHYIoCi9LujhBsJsktZM44Zkf4d_g4RHLsiW18,5858
+metaflow_extensions/outerbounds/plugins/apps/supervisord_utils.py,sha256=QDM7s-iVKnnmE7fM8K-nFoLojQvL_cT8hUj1LF1JOBs,8372
 metaflow_extensions/outerbounds/plugins/fast_bakery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 metaflow_extensions/outerbounds/plugins/fast_bakery/docker_environment.py,sha256=i6F3FXwvEhkmUCTHDJ4VmSoL6vKyQhC_YRCtY6F4EkA,14209
 metaflow_extensions/outerbounds/plugins/fast_bakery/fast_bakery.py,sha256=cj63FrdioggipQFP8GwgxU3FYe6IyzjGSUGYxLQZ4nQ,5189
@@ -21,7 +21,7 @@ metaflow_extensions/outerbounds/plugins/nvcf/__init__.py,sha256=47DEQpj8HBSa-_TI
 metaflow_extensions/outerbounds/plugins/nvcf/constants.py,sha256=aGHdNw_hqBu8i0zWXcatQM6e769wUXox0l8g0f6fNZ8,146
 metaflow_extensions/outerbounds/plugins/nvcf/exceptions.py,sha256=Wn5WvE_sY-L2jEz-iObMLii5Ds_HQJuE437ufadPFLk,3258
 metaflow_extensions/outerbounds/plugins/nvcf/heartbeat_store.py,sha256=pOWwm8LFQBbtku0zNBBwCyXxLK8U-hhC4naQcmU69nE,6217
-metaflow_extensions/outerbounds/plugins/nvcf/nvcf.py,sha256=zOIDbN4PhRM2VMHczfoAHUeo1df2UrqWMgTwcppsTwc,8990
+metaflow_extensions/outerbounds/plugins/nvcf/nvcf.py,sha256=sfWaKZyKuM02v5DujPdfLbm-WoecxHfGn8g432Roct4,10273
 metaflow_extensions/outerbounds/plugins/nvcf/nvcf_cli.py,sha256=5pLEekiw3krlwpcjfjjfUL-URep6soZgmfTqtzLz4Vo,9362
 metaflow_extensions/outerbounds/plugins/nvcf/nvcf_decorator.py,sha256=yGv_6EmrBZNiQQP0rEWWE3akAL-KfI3Wd4ZFrcgl3VQ,8663
 metaflow_extensions/outerbounds/plugins/nvcf/utils.py,sha256=DxWSCayfa95e0HJkWacey1s1nxoTpaunGhrb_0Ayv28,133
@@ -48,7 +48,7 @@ metaflow_extensions/outerbounds/toplevel/plugins/azure/__init__.py,sha256=WUuhz2
 metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3uILlEZ6ntBLKeNyqn3If8nIXZFq_Apd7Dhco,70
 metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
 metaflow_extensions/outerbounds/toplevel/plugins/snowflake/__init__.py,sha256=LptpH-ziXHrednMYUjIaosS1SXD3sOtF_9_eRqd8SJw,50
-ob_metaflow_extensions-1.1.123rc0.dist-info/METADATA,sha256=jVJeDkUJvaWZ3B2Xkor72wjgNKwd1u3LThYzdotjQYs,523
-ob_metaflow_extensions-1.1.123rc0.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-ob_metaflow_extensions-1.1.123rc0.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
-ob_metaflow_extensions-1.1.123rc0.dist-info/RECORD,,
+ob_metaflow_extensions-1.1.123rc2.dist-info/METADATA,sha256=JYsmhOhSHRHULTOGp9b7wA3UE-KKMg6b4x-9BMcfHGc,523
+ob_metaflow_extensions-1.1.123rc2.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+ob_metaflow_extensions-1.1.123rc2.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
+ob_metaflow_extensions-1.1.123rc2.dist-info/RECORD,,