ob-metaflow-extensions 1.1.148__py2.py3-none-any.whl → 1.1.150__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/secrets/secrets.py

@@ -4,13 +4,48 @@ from typing import Dict
 import base64
 import json
 import requests
+import random
 import time
+import sys
 
 
 class OuterboundsSecretsException(Exception):
     pass
 
 
+def _api_server_get(*args, conn_error_retries=2, **kwargs):
+    """
+    There are two categories of errors that we need to handle when dealing with any API server.
+    1. HTTP errors. These are are errors that are returned from the API server.
+        - How to handle retries for this case will be application specific.
+    2. Errors when the API server may not be reachable (DNS resolution / network issues)
+        - In this scenario, we know that something external to the API server is going wrong causing the issue.
+        - Failing pre-maturely in the case might not be the best course of action since critical user jobs might crash on intermittent issues.
+        - So in this case, we can just planely retry the request.
+
+    This function handles the second case. It's a simple wrapper to handle the retry logic for connection errors.
+    If this function is provided a `conn_error_retries` of 5, then the last retry will have waited 32 seconds.
+    Generally this is a safe enough number of retries after which we can assume that something is really broken. Until then,
+    there can be intermittent issues that would resolve themselves if we retry gracefully.
+    """
+    _num_retries = 0
+    noise = random.uniform(-0.5, 0.5)
+    while _num_retries < conn_error_retries:
+        try:
+            return requests.get(*args, **kwargs)
+        except requests.exceptions.ConnectionError:
+            if _num_retries <= conn_error_retries - 1:
+                # Exponential backoff with 2^(_num_retries+1) seconds
+                time.sleep((2 ** (_num_retries + 1)) + noise)
+                _num_retries += 1
+            else:
+                print(
+                    "[@secrets] Failed to connect to the API server. ",
+                    file=sys.stderr,
+                )
+                raise
+
+
 class OuterboundsSecretsApiResponse:
     def __init__(self, response):
         self.response = response
@@ -133,7 +168,9 @@ class OuterboundsSecretsProvider(SecretsProvider):
         retryable_status_codes = [409]
         json_payload = json.dumps(payload)
         for attempt in range(2):  # 0 = initial attempt, 1-2 = retries
-            response = requests.get(url, data=json_payload, headers=request_headers)
+            response = _api_server_get(
+                url, data=json_payload, headers=request_headers, conn_error_retries=5
+            )
             if response.status_code not in retryable_status_codes:
                 break
 
@@ -141,7 +178,6 @@ class OuterboundsSecretsProvider(SecretsProvider):
                 sleep_time = 0.5 * (attempt + 1)
                 time.sleep(sleep_time)
 
-        response = requests.get(url, data=json_payload, headers=request_headers)
         self._handle_error_response(response)
         return OuterboundsSecretsApiResponse(response.json())
 

{ob_metaflow_extensions-1.1.148.dist-info → ob_metaflow_extensions-1.1.150.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: ob-metaflow-extensions
-Version: 1.1.148
+Version: 1.1.150
 Summary: Outerbounds Platform Extensions for Metaflow
 Author: Outerbounds, Inc.
 License: Commercial
 Description-Content-Type: text/markdown
 Requires-Dist: boto3
 Requires-Dist: kubernetes
-Requires-Dist: ob-metaflow (==2.15.10.1)
+Requires-Dist: ob-metaflow (==2.15.11.1)
 
 # Outerbounds platform package
 

{ob_metaflow_extensions-1.1.148.dist-info → ob_metaflow_extensions-1.1.150.dist-info}/RECORD

@@ -40,7 +40,7 @@ metaflow_extensions/outerbounds/plugins/ollama/ollama.py,sha256=KlP8_EmnUoi8-Pid
 metaflow_extensions/outerbounds/plugins/profilers/deco_injector.py,sha256=oI_C3c64XBm7n88FILqHwn-Nnc5DeT_68I67lM9rXaI,2434
 metaflow_extensions/outerbounds/plugins/profilers/gpu_profile_decorator.py,sha256=gDHQ2sMIp4NuZSzUspbSd8RGdFAoO5mgZAyFcZ2a51Y,2619
 metaflow_extensions/outerbounds/plugins/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-metaflow_extensions/outerbounds/plugins/secrets/secrets.py,sha256=2bLrrzcZdIX8RYTnO6f0GszQS36Wc1IvtpenX-K3qls,6623
+metaflow_extensions/outerbounds/plugins/secrets/secrets.py,sha256=3s98hO_twKkM22tKyDdcUjGQNfYpSXW_jLKISV9ju_U,8433
 metaflow_extensions/outerbounds/plugins/snowflake/__init__.py,sha256=RG4ixt3jwqcK1_tt0QxLcUbNmf7wWAMnZhBx-ZMGgLk,114
 metaflow_extensions/outerbounds/plugins/snowflake/snowflake.py,sha256=zoWSHM4CJSfUmJSP-_i4zREWyQOW4USBlgjhQnEhlTE,13669
 metaflow_extensions/outerbounds/plugins/snowpark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -61,7 +61,7 @@ metaflow_extensions/outerbounds/toplevel/plugins/gcp/__init__.py,sha256=BbZiaH3u
 metaflow_extensions/outerbounds/toplevel/plugins/kubernetes/__init__.py,sha256=5zG8gShSj8m7rgF4xgWBZFuY3GDP5n1T0ktjRpGJLHA,69
 metaflow_extensions/outerbounds/toplevel/plugins/ollama/__init__.py,sha256=GRSz2zwqkvlmFS6bcfYD_CX6CMko9DHQokMaH1iBshA,47
 metaflow_extensions/outerbounds/toplevel/plugins/snowflake/__init__.py,sha256=LptpH-ziXHrednMYUjIaosS1SXD3sOtF_9_eRqd8SJw,50
-ob_metaflow_extensions-1.1.148.dist-info/METADATA,sha256=JMOBwxmg167Pp0EsDI3KVAfgDFfNwxTvtZtYaWuqG0Q,521
-ob_metaflow_extensions-1.1.148.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
-ob_metaflow_extensions-1.1.148.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
-ob_metaflow_extensions-1.1.148.dist-info/RECORD,,
+ob_metaflow_extensions-1.1.150.dist-info/METADATA,sha256=0stXBYhOaQQvZRBO8sAKGNW5E0VgODLNT-LaWtqyxZI,521
+ob_metaflow_extensions-1.1.150.dist-info/WHEEL,sha256=bb2Ot9scclHKMOLDEHY6B2sicWOgugjFKaJsT7vwMQo,110
+ob_metaflow_extensions-1.1.150.dist-info/top_level.txt,sha256=NwG0ukwjygtanDETyp_BUdtYtqIA_lOjzFFh1TsnxvI,20
+ob_metaflow_extensions-1.1.150.dist-info/RECORD,,