sovereign 0.23.0__tar.gz → 0.24.1__tar.gz

{sovereign-0.23.0 → sovereign-0.24.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sovereign
-Version: 0.23.0
+Version: 0.24.1
 Summary: Envoy Proxy control-plane written in Python
 Home-page: https://pypi.org/project/sovereign/
 License: Apache-2.0
@@ -29,23 +29,23 @@ Provides-Extra: statsd
 Provides-Extra: ujson
 Requires-Dist: Jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: aiofiles (>=23.1.0,<24.0.0)
-Requires-Dist: boto3 (>=1.26.136,<2.0.0) ; extra == "boto"
+Requires-Dist: aiofiles (>=23.2.1,<24.0.0)
+Requires-Dist: boto3 (>=1.28.62,<2.0.0) ; extra == "boto"
 Requires-Dist: cachelib (>=0.10.2,<0.11.0)
-Requires-Dist: cashews[redis] (>=6.1.0,<7.0.0) ; extra == "caching"
-Requires-Dist: croniter (>=1.3.14,<2.0.0)
-Requires-Dist: cryptography (>=41.0.2,<42.0.0)
-Requires-Dist: datadog (>=0.45.0,<0.46.0) ; extra == "statsd"
-Requires-Dist: fastapi (>=0.95.2,<0.96.0)
+Requires-Dist: cashews[redis] (>=6.3.0,<7.0.0) ; extra == "caching"
+Requires-Dist: croniter (>=1.4.1,<2.0.0)
+Requires-Dist: cryptography (>=41.0.4,<42.0.0)
+Requires-Dist: datadog (>=0.47.0,<0.48.0) ; extra == "statsd"
+Requires-Dist: fastapi (>=0.99.1,<0.100.0)
 Requires-Dist: glom (>=23.3.0,<24.0.0)
-Requires-Dist: gunicorn (>=20.1.0,<21.0.0)
-Requires-Dist: httptools (>=0.5.0,<0.6.0) ; extra == "httptools"
-Requires-Dist: orjson (>=3.8.12,<4.0.0) ; extra == "orjson"
-Requires-Dist: requests (>=2.30.0,<3.0.0)
+Requires-Dist: gunicorn (>=21.2.0,<22.0.0)
+Requires-Dist: httptools (>=0.6.0,<0.7.0) ; extra == "httptools"
+Requires-Dist: orjson (>=3.9.7,<4.0.0) ; extra == "orjson"
+Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: sentry-sdk (>=1.23.1,<2.0.0) ; extra == "sentry"
 Requires-Dist: structlog (>=23.1.0,<24.0.0)
-Requires-Dist: ujson (>=5.7.0,<6.0.0) ; extra == "ujson"
-Requires-Dist: uvicorn (>=0.22.0,<0.23.0)
+Requires-Dist: ujson (>=5.8.0,<6.0.0) ; extra == "ujson"
+Requires-Dist: uvicorn (>=0.23.2,<0.24.0)
 Requires-Dist: uvloop (>=0.17.0,<0.18.0)
 Project-URL: Documentation, https://vsyrakis.bitbucket.io/sovereign/docs/
 Project-URL: Repository, https://bitbucket.org/atlassian/sovereign/src/master/

{sovereign-0.23.0 → sovereign-0.24.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sovereign"
-version = "0.23.0"
+version = "0.24.1"
 description = "Envoy Proxy control-plane written in Python"
 license = "Apache-2.0"
 packages = [
@@ -38,26 +38,26 @@ sovereign = 'sovereign.server:main'
 
 [tool.poetry.dependencies]
 python = "^3.11"
-uvicorn = "^0.22.0"
-gunicorn = "^20.1.0"
-aiofiles = "^23.1.0"
-requests = "^2.30.0"
+uvicorn = "^0.23.2"
+gunicorn = "^21.2.0"
+aiofiles = "^23.2.1"
+requests = "^2.31.0"
 PyYAML = "^6.0.1"
 Jinja2 = "^3.1.2"
 structlog = "^23.1.0"
 cachelib = "^0.10.2"
 glom = "^23.3.0"
-cryptography = "^41.0.2"
-fastapi = "^0.95.2"
+cryptography = "^41.0.4"
+fastapi = "^0.99.1"
 uvloop = "^0.17.0"
 sentry-sdk = "^1.23.1"
-boto3 = {version = "^1.26.136", optional = true}
-datadog = {version = "^0.45.0", optional = true}
-ujson = {version = "^5.7.0", optional = true}
-orjson = {version = "^3.8.12", optional = true}
-croniter = "^1.3.14"
-cashews = {extras = ["redis"], version = "^6.1.0", optional = true}
-httptools = {version = "^0.5.0", optional = true}
+boto3 = {version = "^1.28.62", optional = true}
+datadog = {version = "^0.47.0", optional = true}
+ujson = {version = "^5.8.0", optional = true}
+orjson = {version = "^3.9.7", optional = true}
+croniter = "^1.4.1"
+cashews = {extras = ["redis"], version = "^6.3.0", optional = true}
+httptools = {version = "^0.6.0", optional = true}
 
 [tool.poetry.extras]
 sentry = ["sentry-sdk"]
@@ -70,29 +70,29 @@ httptools = ["httptools"]
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^6.2.5"
-"ruamel.yaml" = "^0.17.26"
+"ruamel.yaml" = "^0.17.32"
 pytest-asyncio = "^0.20.0"
 pytest-mock = "^3.10.0"
 pytest-spec = "^3.2.0"
 pytest-timeout = "^2.1.0"
-coverage = "^7.2.5"
-invoke = "^2.1.2"
-pylint = "^2.17.4"
+coverage = "^7.3.1"
+invoke = "^2.2.0"
+pylint = "^2.17.6"
 tavern = "^1.24.1"
 twine = "^4.0.2"
-poethepoet = "^0.20.0"
-mypy = "^1.3.0"
-black = "^23.3.0"
+poethepoet = "^0.23.0"
+mypy = "^1.5.1"
+black = "^23.9.1"
 freezegun = "^1.2.2"
-moto = "^4.1.10"
-httpx = "^0.24.1"
-types-croniter = "^1.3.2.9"
-types-requests = "^2.30.0.0"
-types-setuptools = "^67.8.0.0"
-types-ujson = "^5.7.0.5"
-types-PyYAML = "^6.0.12.9"
+moto = "^4.2.4"
+httpx = "^0.25.0"
+types-croniter = "^1.4.0.1"
+types-requests = "^2.31.0.8"
+types-setuptools = "^68.2.0.0"
+types-ujson = "^5.8.0.1"
+types-PyYAML = "^6.0.12.12"
 pylama = "^8.4.1"
-prospector = "^1.10.1"
+prospector = "^1.10.2"
 
 [tool.poe.tasks]
 types       = { cmd = "mypy src/sovereign --ignore-missing-imports", help = "Check types with mypy" }

{sovereign-0.23.0 → sovereign-0.24.1}/src/sovereign/__init__.py

@@ -92,6 +92,8 @@ cipher_suite = CipherContainer(
 template_context = TemplateContext(
     refresh_rate=config.template_context.refresh_rate,
     refresh_cron=config.template_context.refresh_cron,
+    refresh_num_retries=config.template_context.refresh_num_retries,
+    refresh_retry_interval_secs=config.template_context.refresh_retry_interval_secs,
     configured_context=config.template_context.context,
     poller=poller,
     encryption_suite=cipher_suite,

{sovereign-0.23.0 → sovereign-0.24.1}/src/sovereign/configuration.py

@@ -61,6 +61,8 @@ POLLER = SourcePoller(
 TEMPLATE_CONTEXT = TemplateContext(
     refresh_rate=CONFIG.template_context.refresh_rate,
     refresh_cron=CONFIG.template_context.refresh_cron,
+    refresh_num_retries=CONFIG.template_context.refresh_num_retries,
+    refresh_retry_interval_secs=CONFIG.template_context.refresh_retry_interval_secs,
     configured_context=CONFIG.template_context.context,
     poller=POLLER,
     encryption_suite=CIPHER_SUITE,

{sovereign-0.23.0 → sovereign-0.24.1}/src/sovereign/context.py

@@ -1,13 +1,30 @@
+import asyncio
 import traceback
-from typing import Dict, Any, Generator, Iterable, NoReturn, Optional
 from copy import deepcopy
+from typing import (
+    Any,
+    Awaitable,
+    Dict,
+    Generator,
+    Iterable,
+    NamedTuple,
+    NoReturn,
+    Optional,
+)
+
 from fastapi import HTTPException
+from structlog.stdlib import BoundLogger
+
 from sovereign.config_loader import Loadable
 from sovereign.schemas import DiscoveryRequest, XdsTemplate
 from sovereign.sources import SourcePoller
-from sovereign.utils.crypto import CipherSuite, CipherContainer
+from sovereign.utils.crypto import CipherContainer, CipherSuite
 from sovereign.utils.timer import poll_forever, poll_forever_cron
-from structlog.stdlib import BoundLogger
+
+
+class LoadContextResponse(NamedTuple):
+    context_name: str
+    context: Dict[str, Any]
 
 
 class TemplateContext:
@@ -15,9 +32,11 @@ class TemplateContext:
         self,
         refresh_rate: Optional[int],
         refresh_cron: Optional[str],
+        refresh_num_retries: int,
+        refresh_retry_interval_secs: int,
         configured_context: Dict[str, Loadable],
         poller: SourcePoller,
-        encryption_suite: CipherContainer,
+        encryption_suite: Optional[CipherContainer],
         disabled_suite: CipherSuite,
         logger: BoundLogger,
         stats: Any,
@@ -25,13 +44,15 @@ class TemplateContext:
         self.poller = poller
         self.refresh_rate = refresh_rate
         self.refresh_cron = refresh_cron
+        self.refresh_num_retries = refresh_num_retries
+        self.refresh_retry_interval_secs = refresh_retry_interval_secs
         self.configured_context = configured_context
         self.crypto = encryption_suite
         self.disabled_suite = disabled_suite
         self.logger = logger
         self.stats = stats
         # initial load
-        self.context = self.load_context_variables()
+        self.context = asyncio.run(self.load_context_variables())
 
     async def start_refresh_context(self) -> NoReturn:
         if self.refresh_cron is not None:
@@ -42,30 +63,66 @@ class TemplateContext:
         raise RuntimeError("Failed to start refresh_context, this should never happen")
 
     async def refresh_context(self) -> None:
-        self.context = self.load_context_variables()
+        self.context = await self.load_context_variables()
 
-    def load_context_variables(self) -> Dict[str, Any]:
-        ret = dict()
-        for k, v in self.configured_context.items():
+    async def _load_context(
+        self,
+        context_name: str,
+        context_config: Loadable | str,
+        refresh_num_retries: int,
+        refresh_retry_interval_secs: int,
+    ) -> LoadContextResponse:
+        retries_left = refresh_num_retries
+        context_response = {}
+
+        while True:
             try:
-                if isinstance(v, Loadable):
-                    ret[k] = v.load()
-                elif isinstance(v, str):
-                    ret[k] = Loadable.from_legacy_fmt(v).load()
+                if isinstance(context_config, Loadable):
+                    context_response = context_config.load()
+                elif isinstance(context_config, str):
+                    context_response = Loadable.from_legacy_fmt(context_config).load()
                 self.stats.increment(
                     "context.refresh.success",
-                    tags=[f"context:{k}"],
+                    tags=[f"context:{context_name}"],
                 )
-            except Exception as e:  # pylint: disable=broad-exception-caught
-                tb = [line for line in traceback.format_exc().split("\n")]
-                self.logger.error(str(e), traceback=tb)
-                self.stats.increment(
-                    "context.refresh.error",
-                    tags=[f"context:{k}"],
+                return LoadContextResponse(context_name, context_response)
+            # pylint: disable=broad-except
+            except Exception as e:
+                retries_left -= 1
+                if retries_left < 0:
+                    tb = [line for line in traceback.format_exc().split("\n")]
+                    self.logger.error(str(e), traceback=tb)
+                    self.stats.increment(
+                        "context.refresh.error",
+                        tags=[f"context:{context_name}"],
+                    )
+                    return LoadContextResponse(context_name, context_response)
+                else:
+                    await asyncio.sleep(refresh_retry_interval_secs)
+
+    async def load_context_variables(self) -> Dict[str, Any]:
+        context_response: Dict[str, Any] = dict()
+
+        context_coroutines: list[Awaitable[LoadContextResponse]] = []
+        for context_name, context_config in self.configured_context.items():
+            context_coroutines.append(
+                self._load_context(
+                    context_name,
+                    context_config,
+                    self.refresh_num_retries,
+                    self.refresh_retry_interval_secs,
                 )
-        if "crypto" not in ret:
-            ret["crypto"] = self.crypto
-        return ret
+            )
+
+        context_results: list[LoadContextResponse] = await asyncio.gather(
+            *context_coroutines
+        )
+        for context_result in context_results:
+            context_response[context_result.context_name] = context_result.context
+
+        if "crypto" not in context_response and self.crypto:
+            context_response["crypto"] = self.crypto
+        return context_response
 
     def build_new_context_from_instances(self, node_value: str) -> Dict[str, Any]:
         matches = self.poller.match_node(node_value=node_value)

{sovereign-0.23.0 → sovereign-0.24.1}/src/sovereign/schemas.py

@@ -542,6 +542,8 @@ class ContextConfiguration(BaseSettings):
     refresh: bool = False
     refresh_rate: Optional[int] = None
     refresh_cron: Optional[str] = None
+    refresh_num_retries: int = 3
+    refresh_retry_interval_secs: int = 10
 
     @staticmethod
     def context_from_legacy(context: Dict[str, str]) -> Dict[str, Loadable]:
@@ -585,6 +587,10 @@ class ContextConfiguration(BaseSettings):
             "refresh": {"env": "SOVEREIGN_REFRESH_CONTEXT"},
             "refresh_rate": {"env": "SOVEREIGN_CONTEXT_REFRESH_RATE"},
             "refresh_cron": {"env": "SOVEREIGN_CONTEXT_REFRESH_CRON"},
+            "refresh_num_retries": {"env": "SOVEREIGN_CONTEXT_REFRESH_NUM_RETRIES"},
+            "refresh_retry_interval_secs": {
+                "env": "SOVEREIGN_CONTEXT_REFRESH_RETRY_INTERVAL_SECS"
+            },
         }