sovereign 0.22.0__tar.gz → 0.24.0__tar.gz

{sovereign-0.22.0 → sovereign-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sovereign
-Version: 0.22.0
+Version: 0.24.0
 Summary: Envoy Proxy control-plane written in Python
 Home-page: https://pypi.org/project/sovereign/
 License: Apache-2.0

{sovereign-0.22.0 → sovereign-0.24.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sovereign"
-version = "0.22.0"
+version = "0.24.0"
 description = "Envoy Proxy control-plane written in Python"
 license = "Apache-2.0"
 packages = [

sovereign-0.24.0/src/sovereign/__init__.py

@@ -0,0 +1,105 @@
+import os
+from contextvars import ContextVar
+from typing import Type, Any, Mapping
+from importlib.metadata import version
+
+from fastapi.responses import JSONResponse
+from starlette.templating import Jinja2Templates
+from pydantic.error_wrappers import ValidationError
+
+from sovereign.schemas import (
+    SovereignAsgiConfig,
+    SovereignConfig,
+    SovereignConfigv2,
+)
+from sovereign import config_loader
+from sovereign.logging.bootstrapper import LoggerBootstrapper
+from sovereign.statistics import configure_statsd
+from sovereign.utils.dictupdate import merge  # type: ignore
+from sovereign.sources import SourcePoller
+from sovereign.context import TemplateContext
+from sovereign.utils.crypto import CipherContainer, create_cipher_suite
+from sovereign.utils.resources import get_package_file
+
+
+json_response_class: Type[JSONResponse] = JSONResponse
+try:
+    import orjson
+    from fastapi.responses import ORJSONResponse
+
+    json_response_class = ORJSONResponse
+except ImportError:
+    try:
+        import ujson
+        from fastapi.responses import UJSONResponse
+
+        json_response_class = UJSONResponse
+    except ImportError:
+        pass
+
+
+def parse_raw_configuration(path: str) -> Mapping[Any, Any]:
+    ret: Mapping[Any, Any] = dict()
+    for p in path.split(","):
+        spec = config_loader.Loadable.from_legacy_fmt(p)
+        ret = merge(obj_a=ret, obj_b=spec.load(), merge_lists=True)
+    return ret
+
+
+_request_id_ctx_var: ContextVar[str] = ContextVar("request_id", default="")
+
+
+def get_request_id() -> str:
+    return _request_id_ctx_var.get()
+
+
+DIST_NAME = "sovereign"
+
+__version__ = version(DIST_NAME)
+config_path = os.getenv("SOVEREIGN_CONFIG", "file:///etc/sovereign.yaml")
+
+html_templates = Jinja2Templates(get_package_file(DIST_NAME, "templates"))  # type: ignore[arg-type]
+
+try:
+    config = SovereignConfigv2(**parse_raw_configuration(config_path))
+except ValidationError:
+    old_config = SovereignConfig(**parse_raw_configuration(config_path))
+    config = SovereignConfigv2.from_legacy_config(old_config)
+asgi_config = SovereignAsgiConfig()
+XDS_TEMPLATES = config.xds_templates()
+
+logs = LoggerBootstrapper(config)
+stats = configure_statsd(config=config.statsd)
+poller = SourcePoller(
+    sources=config.sources,
+    matching_enabled=config.matching.enabled,
+    node_match_key=config.matching.node_key,
+    source_match_key=config.matching.source_key,
+    source_refresh_rate=config.source_config.refresh_rate,
+    logger=logs.application_logger.logger,
+    stats=stats,
+)
+
+fernet_keys = config.authentication.encryption_key
+encryption_keys = fernet_keys.get_secret_value().encode().split()
+cipher_suite = CipherContainer(
+    [
+        create_cipher_suite(key=key, logger=logs.application_logger.logger)
+        for key in encryption_keys
+    ]
+)
+
+template_context = TemplateContext(
+    refresh_rate=config.template_context.refresh_rate,
+    refresh_cron=config.template_context.refresh_cron,
+    refresh_num_retries=config.template_context.refresh_num_retries,
+    refresh_retry_interval_secs=config.template_context.refresh_retry_interval_secs,
+    configured_context=config.template_context.context,
+    poller=poller,
+    encryption_suite=cipher_suite,
+    disabled_suite=create_cipher_suite(b"", logs.application_logger.logger),
+    logger=logs.application_logger.logger,
+    stats=stats,
+)
+poller.lazy_load_modifiers(config.modifiers)
+poller.lazy_load_global_modifiers(config.global_modifiers)

sovereign-0.24.0/src/sovereign/app.py

@@ -0,0 +1,126 @@
+import asyncio
+import traceback
+import uvicorn
+from collections import namedtuple
+from fastapi import FastAPI, Request
+from fastapi.responses import RedirectResponse, FileResponse, Response, JSONResponse
+from sovereign import (
+    __version__,
+    config,
+    asgi_config,
+    json_response_class,
+    poller,
+    template_context,
+    logs,
+)
+from sovereign.error_info import ErrorInfo
+from sovereign.views import crypto, discovery, healthchecks, admin, interface
+from sovereign.middlewares import (
+    RequestContextLogMiddleware,
+    LoggingMiddleware,
+)
+from sovereign.utils.resources import get_package_file
+
+Router = namedtuple("Router", "module tags prefix")
+
+DEBUG = config.debug
+SENTRY_DSN = config.sentry_dsn.get_secret_value()
+
+try:
+    import sentry_sdk
+    from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
+
+    SENTRY_INSTALLED = True
+except ImportError:  # pragma: no cover
+    SENTRY_INSTALLED = False
+
+
+def generic_error_response(e: Exception) -> JSONResponse:
+    """
+    Responds with a JSON object containing basic context
+    about the exception passed in to this function.
+
+    If the server is in debug mode, it will include a traceback in the response.
+
+    The traceback is **always** emitted in logs.
+    """
+    tb = [line for line in traceback.format_exc().split("\n")]
+    info = ErrorInfo.from_exception(e)
+    logs.access_logger.queue_log_fields(
+        ERROR=info.error,
+        ERROR_DETAIL=info.detail,
+        TRACEBACK=tb,
+    )
+    # Don't expose tracebacks in responses, but add it to the logs
+    if DEBUG:
+        info.traceback = tb
+    return json_response_class(
+        content=info.response, status_code=getattr(e, "status_code", 500)
+    )
+
+
+def init_app() -> FastAPI:
+    application = FastAPI(
+        title="Sovereign",
+        version=__version__,
+        debug=DEBUG,
+        default_response_class=json_response_class,
+    )
+
+    routers = (
+        Router(discovery.router, ["Configuration Discovery"], ""),
+        Router(crypto.router, ["Cryptographic Utilities"], "/crypto"),
+        Router(admin.router, ["Debugging Endpoints"], "/admin"),
+        Router(interface.router, ["User Interface"], "/ui"),
+        Router(healthchecks.router, ["Healthchecks"], ""),
+    )
+    for router in routers:
+        application.include_router(
+            router.module, tags=router.tags, prefix=router.prefix
+        )
+
+    application.add_middleware(RequestContextLogMiddleware)
+    application.add_middleware(LoggingMiddleware)
+
+    if SENTRY_INSTALLED and SENTRY_DSN:
+        sentry_sdk.init(SENTRY_DSN)
+        application.add_middleware(SentryAsgiMiddleware)
+        logs.application_logger.logger.info("Sentry middleware enabled")
+
+    @application.exception_handler(500)
+    async def exception_handler(_: Request, exc: Exception) -> JSONResponse:
+        """
+        We cannot incur the execution of this function from unit tests
+        because the starlette test client simply returns exceptions and does
+        not run them through the exception handler.
+        Ergo, this is a facade function for `generic_error_response`
+        """
+        return generic_error_response(exc)  # pragma: no cover
+
+    @application.get("/")
+    async def redirect_to_docs() -> Response:
+        return RedirectResponse("/ui")
+
+    @application.get("/static/{filename}", summary="Return a static asset")
+    async def static(filename: str) -> Response:
+        return FileResponse(get_package_file("sovereign", f"static/{filename}"))  # type: ignore[arg-type]
+
+    @application.on_event("startup")
+    async def keep_sources_uptodate() -> None:
+        asyncio.create_task(poller.poll_forever())
+
+    @application.on_event("startup")
+    async def refresh_template_context() -> None:
+        asyncio.create_task(template_context.start_refresh_context())
+
+    return application
+
+
+app = init_app()
+logs.application_logger.logger.info(
+    f"Sovereign started and listening on {asgi_config.host}:{asgi_config.port}"
+)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    uvicorn.run(app, host="0.0.0.0", port=8000, access_log=False)

{sovereign-0.22.0 → sovereign-0.24.0}/src/sovereign/config_loader.py

@@ -7,7 +7,6 @@ import yaml
 import jinja2
 import requests
 import importlib
-from importlib.util import find_spec
 from importlib.machinery import SourceFileLoader
 from pathlib import Path
 from pydantic import BaseModel
@@ -57,13 +56,20 @@ serializers: Dict[Serialization, Callable[[Any], Any]] = {
     Serialization.raw: passthrough,
 }
 
-if find_spec("ujson"):
-    ujson = importlib.import_module("ujson")
+try:
+    import ujson
+
     serializers[Serialization.ujson] = ujson.loads
     jinja_env.policies["json.dumps_function"] = ujson.dumps
+except ImportError:
+    # This lambda will raise an exception when the serializer is used; otherwise we should not crash
+    serializers[Serialization.ujson] = lambda *a, **kw: raise_(
+        ImportError("ujson must be installed to use in config_loaders")
+    )
+
+try:
+    import orjson
 
-if find_spec("orjson"):
-    orjson = importlib.import_module("orjson")
     serializers[Serialization.orjson] = orjson.loads
 
     # orjson.dumps returns bytes, so we have to wrap & decode it
@@ -81,6 +87,11 @@ if find_spec("orjson"):
 
     jinja_env.policies["json.dumps_function"] = orjson_dumps
     jinja_env.policies["json.dumps_kwargs"] = {"option": orjson.OPT_SORT_KEYS}
+except ImportError:
+    # This lambda will raise an exception when the serializer is used; otherwise we should not crash
+    serializers[Serialization.orjson] = lambda *a, **kw: raise_(
+        ImportError("orjson must be installed to use in config_loaders")
+    )
 
 try:
     import boto3

{sovereign-0.22.0 → sovereign-0.24.0}/src/sovereign/configuration.py

@@ -61,6 +61,8 @@ POLLER = SourcePoller(
 TEMPLATE_CONTEXT = TemplateContext(
     refresh_rate=CONFIG.template_context.refresh_rate,
     refresh_cron=CONFIG.template_context.refresh_cron,
+    refresh_num_retries=CONFIG.template_context.refresh_num_retries,
+    refresh_retry_interval_secs=CONFIG.template_context.refresh_retry_interval_secs,
     configured_context=CONFIG.template_context.context,
     poller=POLLER,
     encryption_suite=CIPHER_SUITE,

sovereign-0.24.0/src/sovereign/context.py

@@ -0,0 +1,179 @@
+import asyncio
+import traceback
+from copy import deepcopy
+from typing import (
+    Any,
+    Awaitable,
+    Dict,
+    Generator,
+    Iterable,
+    NamedTuple,
+    NoReturn,
+    Optional,
+)
+
+from fastapi import HTTPException
+from structlog.stdlib import BoundLogger
+
+from sovereign.config_loader import Loadable
+from sovereign.schemas import DiscoveryRequest, XdsTemplate
+from sovereign.sources import SourcePoller
+from sovereign.utils.crypto import CipherContainer, CipherSuite
+from sovereign.utils.timer import poll_forever, poll_forever_cron
+
+
+class LoadContextResponse(NamedTuple):
+    context_name: str
+    context: Dict[str, Any]
+
+
+class TemplateContext:
+    def __init__(
+        self,
+        refresh_rate: Optional[int],
+        refresh_cron: Optional[str],
+        refresh_num_retries: int,
+        refresh_retry_interval_secs: int,
+        configured_context: Dict[str, Loadable],
+        poller: SourcePoller,
+        encryption_suite: Optional[CipherContainer],
+        disabled_suite: CipherSuite,
+        logger: BoundLogger,
+        stats: Any,
+    ) -> None:
+        self.poller = poller
+        self.refresh_rate = refresh_rate
+        self.refresh_cron = refresh_cron
+        self.refresh_num_retries = refresh_num_retries
+        self.refresh_retry_interval_secs = refresh_retry_interval_secs
+        self.configured_context = configured_context
+        self.crypto = encryption_suite
+        self.disabled_suite = disabled_suite
+        self.logger = logger
+        self.stats = stats
+        # initial load
+        self.context = asyncio.run(self.load_context_variables())
+
+    async def start_refresh_context(self) -> NoReturn:
+        if self.refresh_cron is not None:
+            await poll_forever_cron(self.refresh_cron, self.refresh_context)
+        elif self.refresh_rate is not None:
+            await poll_forever(self.refresh_rate, self.refresh_context)
+
+        raise RuntimeError("Failed to start refresh_context, this should never happen")
+
+    async def refresh_context(self) -> None:
+        self.context = await self.load_context_variables()
+
+    async def _load_context(
+        self,
+        context_name: str,
+        context_config: Loadable | str,
+        refresh_num_retries: int,
+        refresh_retry_interval_secs: int,
+    ) -> LoadContextResponse:
+        retries_left = refresh_num_retries
+        context_response = {}
+
+        while True:
+            try:
+                if isinstance(context_config, Loadable):
+                    context_response = context_config.load()
+                elif isinstance(context_config, str):
+                    context_response = Loadable.from_legacy_fmt(context_config).load()
+                self.stats.increment(
+                    "context.refresh.success",
+                    tags=[f"context:{context_name}"],
+                )
+                return LoadContextResponse(context_name, context_response)
+            # pylint: disable=broad-except
+            except Exception as e:
+                retries_left -= 1
+                if retries_left < 0:
+                    tb = [line for line in traceback.format_exc().split("\n")]
+                    self.logger.error(str(e), traceback=tb)
+                    self.stats.increment(
+                        "context.refresh.error",
+                        tags=[f"context:{context_name}"],
+                    )
+                    return LoadContextResponse(context_name, context_response)
+                else:
+                    await asyncio.sleep(refresh_retry_interval_secs)
+
+    async def load_context_variables(self) -> Dict[str, Any]:
+        context_response: Dict[str, Any] = dict()
+
+        context_coroutines: list[Awaitable[LoadContextResponse]] = []
+        for context_name, context_config in self.configured_context.items():
+            context_coroutines.append(
+                self._load_context(
+                    context_name,
+                    context_config,
+                    self.refresh_num_retries,
+                    self.refresh_retry_interval_secs,
+                )
+            )
+
+        context_results: list[LoadContextResponse] = await asyncio.gather(
+            *context_coroutines
+        )
+        for context_result in context_results:
+            context_response[context_result.context_name] = context_result.context
+
+        if "crypto" not in context_response and self.crypto:
+            context_response["crypto"] = self.crypto
+        return context_response
+
+    def build_new_context_from_instances(self, node_value: str) -> Dict[str, Any]:
+        matches = self.poller.match_node(node_value=node_value)
+        ret = dict()
+        for key, value in self.context.items():
+            try:
+                ret[key] = deepcopy(value)
+            except TypeError:
+                ret[key] = value
+
+        to_add = dict()
+        for scope, instances in matches.scopes.items():
+            if scope in ("default", None):
+                to_add["instances"] = instances
+            else:
+                to_add[scope] = instances
+        if to_add == {}:
+            raise HTTPException(
+                detail=(
+                    "This node does not match any instances! ",
+                    "If node matching is enabled, check that the node "
+                    "match key aligns with the source match key. "
+                    "If you don't know what any of this is, disable "
+                    "node matching via the config",
+                ),
+                status_code=400,
+            )
+        ret.update(to_add)
+        return ret
+
+    def get_context(
+        self, request: DiscoveryRequest, template: XdsTemplate
+    ) -> Dict[str, Any]:
+        ret = self.build_new_context_from_instances(
+            node_value=self.poller.extract_node_key(request.node),
+        )
+        if request.hide_private_keys:
+            ret["crypto"] = self.disabled_suite
+        if not template.is_python_source:
+            keys_to_remove = self.unused_variables(list(ret), template.jinja_variables)
+            for key in keys_to_remove:
+                ret.pop(key, None)
+        return ret
+
+    @staticmethod
+    def unused_variables(
+        keys: Iterable[str], variables: Iterable[str]
+    ) -> Generator[str, None, None]:
+        for key in keys:
+            if key not in variables:
+                yield key
+
+    def get(self, *args: Any, **kwargs: Any) -> Any:
+        return self.context.get(*args, **kwargs)

{sovereign-0.22.0 → sovereign-0.24.0}/src/sovereign/discovery.py

@@ -20,9 +20,9 @@ try:
 except ImportError:
     SENTRY_INSTALLED = False
 
+from sovereign import XDS_TEMPLATES, config, logs, template_context
 from sovereign.utils.version_info import compute_hash
-from sovereign.schemas import XdsTemplate, DiscoveryRequest
-from sovereign.configuration import XDS_TEMPLATES, CONFIG, LOGS, TEMPLATE_CONTEXT
+from sovereign.schemas import XdsTemplate, DiscoveryRequest, ProcessedTemplate
 
 
 try:
@@ -33,7 +33,7 @@ except KeyError:
         "https://vsyrakis.bitbucket.io/sovereign/docs/html/guides/tutorial.html#create-templates "
     )
 
-cache_strategy = CONFIG.source_config.cache_strategy
+cache_strategy = config.source_config.cache_strategy
 
 # Create an enum that bases all the available discovery types off what has been configured
 discovery_types = (_type for _type in sorted(XDS_TEMPLATES["__any__"].keys()))
@@ -65,7 +65,7 @@ def select_template(
         )
 
 
-def response(request: DiscoveryRequest, xds_type: str) -> Dict[str, Any]:
+def response(request: DiscoveryRequest, xds_type: str) -> ProcessedTemplate:
     """
     A Discovery **Request** typically looks something like:
 
@@ -103,7 +103,7 @@ def response(request: DiscoveryRequest, xds_type: str) -> Dict[str, Any]:
         discovery_request=request,
         host_header=request.desired_controlplane,
         resource_names=request.resources,
-        **TEMPLATE_CONTEXT.get_context(request.node),
+        **template_context.get_context(request, template),
     )
     content = template(**context)
 
@@ -117,20 +117,20 @@ def response(request: DiscoveryRequest, xds_type: str) -> Dict[str, Any]:
 
     # Early return if the template is identical
     config_version = compute_hash(content)
-    if config_version == request.version_info and not CONFIG.discovery_cache.enabled:
-        return dict(version_info=config_version, resources=[])
+    if config_version == request.version_info and not config.discovery_cache.enabled:
+        return ProcessedTemplate(version_info=config_version, resources=[])
 
     if not isinstance(content, dict):
         raise RuntimeError(f"Attempting to filter unstructured data: {content}")
     resources = filter_resources(content["resources"], request.resources)
-    return dict(resources=resources, version_info=config_version)
+    return ProcessedTemplate(resources=resources, version_info=config_version)
 
 
 def deserialize_config(content: str) -> Dict[str, Any]:
     try:
         envoy_configuration = yaml.safe_load(content)
     except (ParserError, ScannerError) as e:
-        LOGS.access_logger.queue_log_fields(
+        logs.access_logger.queue_log_fields(
             error=repr(e),
             YAML_CONTEXT=e.context,
             YAML_CONTEXT_MARK=e.context_mark,
@@ -139,7 +139,7 @@ def deserialize_config(content: str) -> Dict[str, Any]:
             YAML_PROBLEM_MARK=e.problem_mark,
         )
 
-        if SENTRY_INSTALLED and CONFIG.sentry_dsn:
+        if SENTRY_INSTALLED and config.sentry_dsn:
             sentry_sdk.capture_exception(e)
 
         raise HTTPException(

{sovereign-0.22.0 → sovereign-0.24.0}/src/sovereign/middlewares.py

@@ -4,8 +4,7 @@ from uuid import uuid4
 from fastapi.requests import Request
 from fastapi.responses import Response
 from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
-from sovereign import get_request_id, _request_id_ctx_var
-from sovereign.configuration import CONFIG, LOGS, STATS
+from sovereign import config, logs, get_request_id, _request_id_ctx_var, stats
 
 
 class RequestContextLogMiddleware(BaseHTTPMiddleware):
@@ -19,7 +18,7 @@ class RequestContextLogMiddleware(BaseHTTPMiddleware):
         finally:
             req_id = get_request_id()
             response.headers["X-Request-ID"] = req_id
-            LOGS.access_logger.queue_log_fields(REQUEST_ID=req_id)
+            logs.access_logger.queue_log_fields(REQUEST_ID=req_id)
             _request_id_ctx_var.reset(token)
         return response
 
@@ -37,9 +36,9 @@ class LoggingMiddleware(BaseHTTPMiddleware):
             source_port = addr.port
         if xff := request.headers.get("X-Forwarded-For"):
             source_ip = xff.split(",")[0]  # leftmost address
-        LOGS.access_logger.clear_log_fields()
-        LOGS.access_logger.queue_log_fields(
-            ENVIRONMENT=CONFIG.legacy_fields.environment,
+        logs.access_logger.clear_log_fields()
+        logs.access_logger.queue_log_fields(
+            ENVIRONMENT=config.legacy_fields.environment,
             HOST=request.headers.get("host", "-"),
             METHOD=request.method,
             PATH=request.url.path,
@@ -54,7 +53,7 @@ class LoggingMiddleware(BaseHTTPMiddleware):
             response = await call_next(request)
         finally:
             duration = time.time() - start_time
-            LOGS.access_logger.queue_log_fields(
+            logs.access_logger.queue_log_fields(
                 BYTES_TX=response.headers.get("content-length", "-"),
                 STATUS_CODE=response.status_code,
                 DURATION=duration,
@@ -71,7 +70,7 @@ class LoggingMiddleware(BaseHTTPMiddleware):
                     for k, v in request_info.items()
                     if v is not None
                 ]
-                STATS.increment("discovery.rq_total", tags=tags)
-                STATS.timing("discovery.rq_ms", value=duration * 1000, tags=tags)
-            LOGS.access_logger.logger.info("request")
+                stats.increment("discovery.rq_total", tags=tags)
+                stats.timing("discovery.rq_ms", value=duration * 1000, tags=tags)
+            logs.access_logger.logger.info("request")
         return response

{sovereign-0.22.0 → sovereign-0.24.0}/src/sovereign/modifiers/test.py

@@ -1,6 +1,6 @@
+from sovereign import template_context
 from sovereign.modifiers.lib import Modifier
 from sovereign.utils import eds, templates
-from sovereign.configuration import TEMPLATE_CONTEXT
 
 
 class Test(Modifier):
@@ -8,7 +8,7 @@ class Test(Modifier):
         return True
 
     def apply(self) -> None:
-        assert TEMPLATE_CONTEXT
+        assert template_context
         assert eds
         assert templates
         self.instance["modifier_test_executed"] = True