sovereign 0.19.3__py3-none-any.whl → 1.0.0a4__py3-none-any.whl

sovereign/dynamic_config/loaders.py

@@ -0,0 +1,120 @@
+import importlib
+import os
+from importlib.machinery import SourceFileLoader
+from pathlib import Path
+from typing import Any, Protocol
+
+import requests
+
+from sovereign.utils.resources import get_package_file_bytes
+
+try:
+    import boto3
+
+    BOTO_IS_AVAILABLE = True
+except ImportError:
+    BOTO_IS_AVAILABLE = False
+
+
+class CustomLoader(Protocol):
+    """
+    Custom loaders can be added to sovereign by creating a subclass
+    and then in config:
+
+    template_context:
+      context:
+          ...:
+            protocol: <loader name>
+            serialization: ...
+            path: <path argument>
+    """
+
+    default_deser: str = "yaml"
+
+    def load(self, path: str) -> Any: ...
+
+
+class File(CustomLoader):
+    default_deser = "passthrough"
+
+    def load(self, path: str) -> Any:
+        with open(path) as f:
+            contents = f.read()
+            try:
+                return contents
+            except FileNotFoundError:
+                raise FileNotFoundError(f"Unable to load {path}")
+
+
+class PackageData(CustomLoader):
+    default_deser = "string"
+
+    def load(self, path: str) -> Any:
+        pkg, pkg_file = path.split(":")
+        data = get_package_file_bytes(pkg, pkg_file)
+        return data
+
+
+class Web(CustomLoader):
+    default_deser = "json"
+
+    def load(self, path: str) -> Any:
+        response = requests.get(path)
+        response.raise_for_status()
+        data = response.text
+        return data
+
+
+class EnvironmentVariable(CustomLoader):
+    default_deser = "raw"
+
+    def load(self, path: str) -> Any:
+        data = os.getenv(path)
+        if data is None:
+            raise AttributeError(f"Unable to read environment variable {path}")
+        return data
+
+
+class PythonModule(CustomLoader):
+    default_deser = "passthrough"
+
+    def load(self, path: str) -> Any:
+        if ":" in path:
+            mod, fn = path.rsplit(":", maxsplit=1)
+        else:
+            mod, fn = path, ""
+        imported = importlib.import_module(mod)
+        if fn != "":
+            return getattr(imported, fn)
+        return imported
+
+
+class S3Bucket(CustomLoader):
+    default_deser = "raw"
+
+    def load(self, path: str) -> Any:
+        if not BOTO_IS_AVAILABLE:
+            raise ImportError(
+                "boto3 must be installed to load S3 paths. Use ``pip install sovereign[boto]``"
+            )
+        bucket, key = path.split("/", maxsplit=1)
+        s3 = boto3.client("s3")
+        response = s3.get_object(Bucket=bucket, Key=key)
+        data = "".join([chunk.decode() for chunk in response["Body"]])
+        return data
+
+
+class PythonInlineCode(CustomLoader):
+    default_deser = "passthrough"
+
+    def load(self, path: str) -> Any:
+        p = str(Path(path).absolute())
+        loader = SourceFileLoader(p, path=p)
+        return loader.load_module(p)
+
+
+class Inline(CustomLoader):
+    default_deser = "string"
+
+    def load(self, path: str) -> Any:
+        return path

sovereign/error_info.py

@@ -1,8 +1,7 @@
 import json
-
-from typing import Union, Optional, Any
-from dataclasses import dataclass, field, asdict
+from dataclasses import asdict, dataclass, field
 from functools import singledispatchmethod
+from typing import Any, Optional, Union
 
 from sovereign import get_request_id
 

sovereign/events.py

@@ -0,0 +1,49 @@
+from asyncio import Queue, gather
+from collections import defaultdict
+from enum import IntEnum
+from typing import Sequence, final
+
+import pydantic
+
+Primitives = str | int | float | bool | Sequence[str]
+
+
+class Topic(IntEnum):
+    CONTEXT = 1
+
+
+class Event(pydantic.BaseModel):
+    message: str
+    metadata: dict[str, Primitives] = pydantic.Field(default_factory=dict)
+
+
+@final
+class EventBus:
+    def __init__(self, maxsize: int = 0):
+        self._topics: dict[Topic, list[Queue[Event]]] = defaultdict(list)
+        self._maxsize = maxsize
+
+    def subscribe(self, topic: Topic) -> Queue[Event]:
+        q: Queue[Event] = Queue(self._maxsize)
+        self._topics[topic].append(q)
+        return q
+
+    def unsubscribe(self, topic: Topic, q: Queue[Event]) -> None:
+        qs = self._topics.get(topic)
+        if not qs:
+            return
+        try:
+            qs.remove(q)
+        except ValueError:
+            pass
+        if not qs:
+            _ = self._topics.pop(topic, None)
+
+    async def publish(self, topic: Topic, msg: Event) -> None:
+        qs = self._topics.get(topic, [])
+        if not qs:
+            return
+        _ = await gather(*(q.put(msg) for q in qs))
+
+
+bus = EventBus()

sovereign/logging/access_logger.py

@@ -0,0 +1,85 @@
+from copy import deepcopy
+from functools import cached_property
+from typing import Any, Dict
+
+import structlog
+from starlette_context import context
+from structlog.stdlib import BoundLogger
+
+from sovereign.configuration import SovereignConfigv2
+from sovereign.logging.base_logger import BaseLogger
+from sovereign.logging.types import EventDict, LoggingType, ProcessedMessage
+
+
+class AccessLogger(BaseLogger):
+    def __init__(self, root_logger: BoundLogger, config: SovereignConfigv2):
+        self._access_logs_enabled = config.logging.access_logs.enabled
+        self._ignore_empty = config.logging.access_logs.ignore_empty_fields
+        self._user_log_fmt = config.logging.access_logs.log_fmt
+
+        self.logger: BoundLogger = structlog.wrap_logger(
+            root_logger,
+            wrapper_class=structlog.BoundLogger,
+            processors=[
+                self.is_enabled_processor,
+                self.merge_starlette_contextvars,
+                self.format_access_log_fields,
+            ],
+            type=LoggingType.ACCESS,
+        )
+
+    @cached_property
+    def is_enabled(self) -> bool:
+        return self._access_logs_enabled
+
+    @cached_property
+    def _default_log_fmt(self) -> Dict[str, str]:
+        return {
+            "type": "{type}",
+            "event": "{event}",
+            "env": "{ENVIRONMENT}",
+            "site": "{HOST}",
+            "method": "{METHOD}",
+            "uri_path": "{PATH}",
+            "uri_query": "{QUERY}",
+            "src_ip": "{SOURCE_IP}",
+            "src_port": "{SOURCE_PORT}",
+            "pid": "{PID}",
+            "user_agent": "{USER_AGENT}",
+            "bytes_in": "{BYTES_RX}",
+            "bytes_out": "{BYTES_TX}",
+            "status": "{STATUS_CODE}",
+            "duration": "{DURATION}",
+            "request_id": "{REQUEST_ID}",
+            "resource_version": "{XDS_CLIENT_VERSION} -> {XDS_SERVER_VERSION}",
+            "resource_names": "{XDS_RESOURCES}",
+            "envoy_ver": "{XDS_ENVOY_VERSION}",
+            "traceback": "{TRACEBACK}",
+            "error": "{ERROR}",
+            "detail": "{ERROR_DETAIL}",
+        }
+
+    def format_access_log_fields(
+        self, logger: BoundLogger, method_name: str, event_dict: EventDict
+    ) -> ProcessedMessage:
+        formatted_dict: Dict[str, Any] = dict()
+        for k, v in self.get_configured_log_format.items():
+            try:
+                value: str = v.format(**event_dict)
+            except KeyError:
+                value = "-"
+            if value in (None, "-") and self._ignore_empty:
+                continue
+            formatted_dict[k] = value
+        return formatted_dict
+
+    def merge_starlette_contextvars(
+        self, _, __, event_dict: EventDict
+    ) -> ProcessedMessage:
+        merged_context = deepcopy(event_dict)
+        for k, v in context.data.items():
+            merged_context[k] = v
+        return merged_context
+
+    def queue_log_fields(self, **kwargs: Any) -> None:
+        context.update(kwargs)

sovereign/logging/application_logger.py

@@ -0,0 +1,54 @@
+from functools import cached_property
+from typing import Any, Dict
+
+import structlog
+from structlog.stdlib import BoundLogger
+
+from sovereign.configuration import SovereignConfigv2
+from sovereign.logging.base_logger import BaseLogger
+from sovereign.logging.types import EventDict, LoggingType, ProcessedMessage
+
+
+class ApplicationLogger(BaseLogger):
+    def __init__(self, root_logger: BoundLogger, config: SovereignConfigv2):
+        self._application_logs_enabled = config.logging.application_logs.enabled
+        self._user_log_fmt = config.logging.application_logs.log_fmt
+
+        self.logger: BoundLogger = structlog.wrap_logger(
+            root_logger,
+            wrapper_class=structlog.BoundLogger,
+            processors=[
+                self.is_enabled_processor,
+                self.format_application_log_fields,
+            ],
+            type=LoggingType.APPLICATION,
+        )
+
+    @cached_property
+    def is_enabled(self) -> bool:
+        return self._application_logs_enabled
+
+    @cached_property
+    def _default_log_fmt(self) -> Dict[str, str]:
+        return {
+            "type": "{type}",
+            "event": "{event}",
+            "error": "{error}",
+            "traceback": "{traceback}",
+            "last_update": "{last_update}",
+            "instance_count": "{instance_count}",
+        }
+
+    def format_application_log_fields(
+        self, logger: BoundLogger, method_name: str, event_dict: EventDict
+    ) -> ProcessedMessage:
+        formatted_dict: Dict[str, Any] = {
+            "level": method_name,
+        }
+        for k, v in self.get_configured_log_format.items():
+            try:
+                value: str = v.format(**event_dict)
+            except KeyError:
+                continue
+            formatted_dict[k] = value
+        return formatted_dict

sovereign/logging/base_logger.py

@@ -0,0 +1,41 @@
+import json
+from abc import ABC, abstractmethod
+from functools import cached_property
+from typing import Dict, Optional
+
+from structlog.exceptions import DropEvent
+from structlog.stdlib import BoundLogger
+
+from sovereign.logging.types import EventDict, ProcessedMessage
+
+
+class BaseLogger(ABC):
+    _user_log_fmt: Optional[str]
+
+    @property
+    @abstractmethod
+    def is_enabled(self) -> bool: ...
+
+    @property
+    @abstractmethod
+    def _default_log_fmt(self) -> Dict[str, str]: ...
+
+    def is_enabled_processor(
+        self, logger: BoundLogger, method_name: str, event_dict: EventDict
+    ) -> ProcessedMessage:
+        if not self.is_enabled:
+            raise DropEvent
+        return event_dict
+
+    @cached_property
+    def get_configured_log_format(self) -> Dict[str, str]:
+        if isinstance(self._user_log_fmt, str) and self._user_log_fmt != "":
+            format = json.loads(self._user_log_fmt)
+            if not isinstance(format, dict):
+                raise RuntimeError(
+                    f"Failed to parse log format as JSON: {self._user_log_fmt}"
+                )
+            if "event" not in format:
+                format["event"] = "{event}"
+            return format
+        return self._default_log_fmt

sovereign/logging/bootstrapper.py

@@ -0,0 +1,36 @@
+import structlog
+from structlog.exceptions import DropEvent
+from structlog.stdlib import BoundLogger
+
+from sovereign.configuration import SovereignConfigv2
+from sovereign.logging.access_logger import AccessLogger
+from sovereign.logging.application_logger import ApplicationLogger
+from sovereign.logging.types import EventDict, ProcessedMessage
+
+
+class LoggerBootstrapper:
+    def __init__(self, config: SovereignConfigv2) -> None:
+        self.show_debug: bool = config.debug
+
+        structlog.configure(
+            processors=[
+                self.debug_logs_processor,
+                structlog.processors.JSONRenderer(),
+            ]
+        )
+        root_logger: BoundLogger = structlog.get_logger()
+        self.logger = root_logger
+
+        self.access_logger = AccessLogger(root_logger=root_logger, config=config)
+        self.application_logger = ApplicationLogger(
+            root_logger=root_logger, config=config
+        )
+
+    def debug_logs_processor(
+        self, logger: BoundLogger, method_name: str, event_dict: EventDict
+    ) -> ProcessedMessage:
+        if not self.show_debug and event_dict.get("level", "").lower() == "debug":
+            raise DropEvent
+        if method_name == "debug" and self.show_debug is False:
+            raise DropEvent
+        return event_dict

sovereign/logging/types.py

@@ -0,0 +1,10 @@
+from enum import StrEnum
+from typing import Any, Mapping, MutableMapping, Tuple, Union
+
+EventDict = MutableMapping[str, Any]
+ProcessedMessage = Union[Mapping[str, Any], str, bytes, Tuple[Any, ...]]
+
+
+class LoggingType(StrEnum):
+    ACCESS = "access"
+    APPLICATION = "application"

sovereign/middlewares.py

@@ -1,10 +1,12 @@
 import os
 import time
 from uuid import uuid4
+
 from fastapi.requests import Request
 from fastapi.responses import Response
 from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
-from sovereign import config, logs, get_request_id, _request_id_ctx_var, stats
+
+from sovereign import _request_id_ctx_var, config, get_request_id, logs, stats
 
 
 class RequestContextLogMiddleware(BaseHTTPMiddleware):
@@ -17,8 +19,8 @@ class RequestContextLogMiddleware(BaseHTTPMiddleware):
             response = await call_next(request)
         finally:
             req_id = get_request_id()
-            response.headers["X-Request-ID"] = req_id
-            logs.queue_log_fields(REQUEST_ID=req_id)
+            req_id = response.headers.setdefault("X-Request-Id", get_request_id())
+            logs.access_logger.queue_log_fields(REQUEST_ID=req_id)
             _request_id_ctx_var.reset(token)
         return response
 
@@ -36,8 +38,7 @@ class LoggingMiddleware(BaseHTTPMiddleware):
             source_port = addr.port
         if xff := request.headers.get("X-Forwarded-For"):
             source_ip = xff.split(",")[0]  # leftmost address
-        logs.clear_log_fields()
-        logs.queue_log_fields(
+        logs.access_logger.queue_log_fields(
             ENVIRONMENT=config.legacy_fields.environment,
             HOST=request.headers.get("host", "-"),
             METHOD=request.method,
@@ -53,7 +54,7 @@ class LoggingMiddleware(BaseHTTPMiddleware):
             response = await call_next(request)
         finally:
             duration = time.time() - start_time
-            logs.queue_log_fields(
+            logs.access_logger.queue_log_fields(
                 BYTES_TX=response.headers.get("content-length", "-"),
                 STATUS_CODE=response.status_code,
                 DURATION=duration,
@@ -72,5 +73,5 @@ class LoggingMiddleware(BaseHTTPMiddleware):
                 ]
                 stats.increment("discovery.rq_total", tags=tags)
                 stats.timing("discovery.rq_ms", value=duration * 1000, tags=tags)
-            logs.logger.msg("request")
+            logs.access_logger.logger.info("request")
         return response

sovereign/modifiers/lib.py

@@ -6,8 +6,9 @@ used via configuration.
 
 `todo entry point install guide`
 """
+
 import abc
-from typing import List, Any, Dict
+from typing import Any, Dict, List
 
 
 class Modifier(metaclass=abc.ABCMeta):

sovereign/rendering.py

@@ -0,0 +1,124 @@
+"""
+Discovery
+---------
+
+Functions used to render and return discovery responses to Envoy proxies.
+
+The templates are configurable. `todo See ref:Configuration#Templates`
+"""
+
+import importlib
+import os
+import traceback
+from concurrent.futures import ThreadPoolExecutor
+from multiprocessing import Pipe, Process, cpu_count
+
+# noinspection PyProtectedMember
+from multiprocessing.connection import Connection
+from typing import Any
+
+import pydantic
+
+from sovereign import application_logger as log
+from sovereign import cache, stats
+from sovereign.cache.types import Entry
+from sovereign.configuration import config
+from sovereign.rendering_common import (
+    add_type_urls,
+    deserialize_config,
+    filter_resources,
+)
+from sovereign.types import DiscoveryRequest, ProcessedTemplate
+from sovereign.utils import templates
+
+writer = cache.CacheWriter()
+# limit render jobs to number of cores
+POOL = ThreadPoolExecutor(max_workers=cpu_count())
+
+
+class RenderJob(pydantic.BaseModel):
+    id: str
+    request: DiscoveryRequest
+    context: dict[str, Any]
+
+    def submit(self):
+        return POOL.submit(self._run)
+
+    def _run(self):
+        rx, tx = Pipe()
+        proc = Process(target=generate, args=[self, tx])
+        proc.start()
+        log.info(
+            (
+                f"Spawning process for id={self.id} "
+                f"max_workers={POOL._max_workers} "
+                f"threads={len(POOL._threads)} "
+                f"shutdown={POOL._shutdown} "
+                f"queue_size={POOL._work_queue.qsize()}"
+            )
+        )
+        proc.join(timeout=60)  # TODO: render timeout configurable
+        if proc.is_alive():
+            log.warning(f"Render job for {self.id} has been running longer than 60s")
+        while rx.poll(timeout=10):
+            level, message = rx.recv()
+            logger = getattr(log, level)
+            logger(message)
+
+
+# noinspection DuplicatedCode
+def generate(job: RenderJob, tx: Connection) -> None:
+    request = job.request
+    tags = [f"type:{request.resource_type}"]
+    try:
+        with stats.timed("template.render_ms", tags=tags):
+            content = request.template.generate(
+                discovery_request=request,
+                host_header=request.desired_controlplane,
+                resource_names=request.resources,
+                utils=templates,
+                **job.context,
+            )
+            if not request.template.is_python_source:
+                assert isinstance(content, str)
+                content = deserialize_config(content)
+            assert isinstance(content, dict)
+            resources = filter_resources(content["resources"], request.resources)
+            add_type_urls(request.api_version, request.resource_type, resources)
+            response = ProcessedTemplate(resources=resources)
+            tx.send(
+                (
+                    "info",
+                    f"Completed rendering of {request}: client_id={job.id} version={response.version_info} "
+                    f"resources={len(response.resources)} pid={os.getpid()}",
+                )
+            )
+            cached, cache_result = writer.set(
+                job.id,
+                Entry(
+                    text=response.model_dump_json(indent=None),
+                    len=len(response.resources),
+                    version=response.version_info,
+                    node=request.node,
+                ),
+            )
+            tx.send(cache_result)
+            if cached:
+                tags.append("result:ok")
+            else:
+                tags.append("result:cache_failed")
+    except Exception as e:
+        tx.send(
+            (
+                "error",
+                f"Failed to render job for {job.id}: " + str(traceback.format_exc()),
+            )
+        )
+        tags.append("result:err")
+        tags.append(f"error:{e.__class__.__name__.lower()}")
+        if config.sentry_dsn.get_secret_value():
+            mod = importlib.import_module("sentry_sdk")
+            mod.capture_exception(e)
+    finally:
+        stats.increment("template.render", tags=tags)
+        tx.close()

sovereign/rendering_common.py

@@ -0,0 +1,91 @@
+import importlib
+from typing import Any
+
+import yaml
+from starlette.exceptions import HTTPException
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
+
+from sovereign import config, logs
+
+type_urls = {
+    "v2": {
+        "listeners": "type.googleapis.com/envoy.api.v2.Listener",
+        "clusters": "type.googleapis.com/envoy.api.v2.Cluster",
+        "endpoints": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+        "secrets": "type.googleapis.com/envoy.api.v2.auth.Secret",
+        "routes": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
+        "scoped-routes": "type.googleapis.com/envoy.api.v2.ScopedRouteConfiguration",
+    },
+    "v3": {
+        "listeners": "type.googleapis.com/envoy.config.listener.v3.Listener",
+        "clusters": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+        "endpoints": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+        "secrets": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
+        "routes": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
+        "scoped-routes": "type.googleapis.com/envoy.config.route.v3.ScopedRouteConfiguration",
+        "runtime": "type.googleapis.com/envoy.service.runtime.v3.Runtime",
+    },
+}
+
+
+def add_type_urls(api_version, resource_type, resources):
+    type_url = type_urls.get(api_version, {}).get(resource_type)
+    if type_url is not None:
+        for resource in resources:
+            if not resource.get("@type"):
+                resource["@type"] = type_url
+
+
+def deserialize_config(content: str) -> dict[str, Any]:
+    try:
+        envoy_configuration = yaml.safe_load(content)
+    except (ParserError, ScannerError) as e:
+        logs.access_logger.queue_log_fields(
+            error=repr(e),
+            YAML_CONTEXT=e.context,
+            YAML_CONTEXT_MARK=e.context_mark,
+            YAML_NOTE=e.note,
+            YAML_PROBLEM=e.problem,
+            YAML_PROBLEM_MARK=e.problem_mark,
+        )
+
+        if config.sentry_dsn:
+            mod = importlib.import_module("sentry_sdk")
+            mod.capture_exception(e)
+
+        raise HTTPException(
+            status_code=500,
+            detail=(
+                "Failed to load configuration, there may be "
+                "a syntax error in the configured templates. "
+                "Please check Sentry if you have configured Sentry DSN"
+            ),
+        )
+    if not isinstance(envoy_configuration, dict):
+        raise RuntimeError(
+            f"Deserialized configuration is of unexpected format: {envoy_configuration}"
+        )
+    return envoy_configuration
+
+
+def filter_resources(
+    generated: list[dict[str, Any]], requested: list[str]
+) -> list[dict[str, Any]]:
+    """
+    If Envoy specifically requested a resource, this removes everything
+    that does not match the name of the resource.
+    If Envoy did not specifically request anything, every resource is retained.
+    """
+    if len(requested) == 0:
+        return generated
+    return [resource for resource in generated if resource_name(resource) in requested]
+
+
+def resource_name(resource: dict[str, Any]) -> str:
+    name = resource.get("name") or resource.get("cluster_name")
+    if isinstance(name, str):
+        return name
+    raise KeyError(
+        f"Failed to determine the name or cluster_name of the following resource: {resource}"
+    )