sovereign 1.0.0a4__py3-none-any.whl → 1.0.0b148__py3-none-any.whl

sovereign/rendering.py

@@ -7,34 +7,48 @@ Functions used to render and return discovery responses to Envoy proxies.
 The templates are configurable. `todo See ref:Configuration#Templates`
 """
 
-import importlib
-import os
 import traceback
+import importlib
 from concurrent.futures import ThreadPoolExecutor
-from multiprocessing import Pipe, Process, cpu_count
-
-# noinspection PyProtectedMember
+from multiprocessing import Process, Pipe, cpu_count
 from multiprocessing.connection import Connection
 from typing import Any
 
+import yaml
 import pydantic
+from starlette.exceptions import HTTPException
+from yaml.parser import ParserError, ScannerError  # type: ignore
 
-from sovereign import application_logger as log
-from sovereign import cache, stats
+from sovereign import logs, cache, stats, application_logger as log
 from sovereign.cache.types import Entry
 from sovereign.configuration import config
-from sovereign.rendering_common import (
-    add_type_urls,
-    deserialize_config,
-    filter_resources,
-)
 from sovereign.types import DiscoveryRequest, ProcessedTemplate
-from sovereign.utils import templates
+
 
 writer = cache.CacheWriter()
 # limit render jobs to number of cores
 POOL = ThreadPoolExecutor(max_workers=cpu_count())
 
+type_urls = {
+    "v2": {
+        "listeners": "type.googleapis.com/envoy.api.v2.Listener",
+        "clusters": "type.googleapis.com/envoy.api.v2.Cluster",
+        "endpoints": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+        "secrets": "type.googleapis.com/envoy.api.v2.auth.Secret",
+        "routes": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
+        "scoped-routes": "type.googleapis.com/envoy.api.v2.ScopedRouteConfiguration",
+    },
+    "v3": {
+        "listeners": "type.googleapis.com/envoy.config.listener.v3.Listener",
+        "clusters": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+        "endpoints": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+        "secrets": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
+        "routes": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
+        "scoped-routes": "type.googleapis.com/envoy.config.route.v3.ScopedRouteConfiguration",
+        "runtime": "type.googleapis.com/envoy.service.runtime.v3.Runtime",
+    },
+}
+
 
 class RenderJob(pydantic.BaseModel):
     id: str
@@ -66,7 +80,6 @@ class RenderJob(pydantic.BaseModel):
             logger(message)
 
 
-# noinspection DuplicatedCode
 def generate(job: RenderJob, tx: Connection) -> None:
     request = job.request
     tags = [f"type:{request.resource_type}"]
@@ -76,7 +89,6 @@ def generate(job: RenderJob, tx: Connection) -> None:
                 discovery_request=request,
                 host_header=request.desired_controlplane,
                 resource_names=request.resources,
-                utils=templates,
                 **job.context,
             )
             if not request.template.is_python_source:
@@ -86,13 +98,7 @@ def generate(job: RenderJob, tx: Connection) -> None:
             resources = filter_resources(content["resources"], request.resources)
             add_type_urls(request.api_version, request.resource_type, resources)
             response = ProcessedTemplate(resources=resources)
-            tx.send(
-                (
-                    "info",
-                    f"Completed rendering of {request}: client_id={job.id} version={response.version_info} "
-                    f"resources={len(response.resources)} pid={os.getpid()}",
-                )
-            )
+            tx.send(("info", f"Completed rendering of {request} for {job.id}"))
             cached, cache_result = writer.set(
                 job.id,
                 Entry(
@@ -122,3 +128,65 @@ def generate(job: RenderJob, tx: Connection) -> None:
     finally:
         stats.increment("template.render", tags=tags)
         tx.close()
+
+
+def deserialize_config(content: str) -> dict[str, Any]:
+    try:
+        envoy_configuration = yaml.safe_load(content)
+    except (ParserError, ScannerError) as e:
+        logs.access_logger.queue_log_fields(
+            error=repr(e),
+            YAML_CONTEXT=e.context,
+            YAML_CONTEXT_MARK=e.context_mark,
+            YAML_NOTE=e.note,
+            YAML_PROBLEM=e.problem,
+            YAML_PROBLEM_MARK=e.problem_mark,
+        )
+
+        if config.sentry_dsn:
+            mod = importlib.import_module("sentry_sdk")
+            mod.capture_exception(e)
+
+        raise HTTPException(
+            status_code=500,
+            detail=(
+                "Failed to load configuration, there may be "
+                "a syntax error in the configured templates. "
+                "Please check Sentry if you have configured Sentry DSN"
+            ),
+        )
+    if not isinstance(envoy_configuration, dict):
+        raise RuntimeError(
+            f"Deserialized configuration is of unexpected format: {envoy_configuration}"
+        )
+    return envoy_configuration
+
+
+def filter_resources(
+    generated: list[dict[str, Any]], requested: list[str]
+) -> list[dict[str, Any]]:
+    """
+    If Envoy specifically requested a resource, this removes everything
+    that does not match the name of the resource.
+    If Envoy did not specifically request anything, every resource is retained.
+    """
+    if len(requested) == 0:
+        return generated
+    return [resource for resource in generated if resource_name(resource) in requested]
+
+
+def resource_name(resource: dict[str, Any]) -> str:
+    name = resource.get("name") or resource.get("cluster_name")
+    if isinstance(name, str):
+        return name
+    raise KeyError(
+        f"Failed to determine the name or cluster_name of the following resource: {resource}"
+    )
+
+
+def add_type_urls(api_version, resource_type, resources):
+    type_url = type_urls.get(api_version, {}).get(resource_type)
+    if type_url is not None:
+        for resource in resources:
+            if not resource.get("@type"):
+                resource["@type"] = type_url

sovereign/response_class.py

@@ -1,8 +1,8 @@
-from importlib.util import find_spec
 from typing import Type
-
+from importlib.util import find_spec
 from fastapi.responses import JSONResponse
 
+
 json_response_class: Type[JSONResponse] = JSONResponse
 if find_spec("orjson"):
     from fastapi.responses import ORJSONResponse

sovereign/server.py

@@ -1,65 +1,46 @@
-import configparser
-import tempfile
 import warnings
+import tempfile
+import configparser
 from pathlib import Path
 
 import uvicorn
 
 from sovereign import application_logger as log
-from sovereign.configuration import SovereignAsgiConfig, SupervisordConfig, config
-from sovereign.v2.worker import Worker
+from sovereign.configuration import SovereignAsgiConfig, SupervisordConfig
 
-# noinspection PyArgumentList
 asgi_config = SovereignAsgiConfig()
-# noinspection PyArgumentList
 supervisord_config = SupervisordConfig()
 
 
-def web(supervisor_enabled=True) -> None:
+def web() -> None:
     from sovereign.app import app
 
     log.debug("Starting web server")
-
-    if not supervisor_enabled:
-        uvicorn.run(
-            app,
-            log_level=asgi_config.log_level,
-            access_log=False,
-            timeout_keep_alive=asgi_config.keepalive,
-            host=asgi_config.host,
-            port=asgi_config.port,
-            workers=1,  # per managed supervisor proc
-        )
-    else:
-        uvicorn.run(
-            app,
-            fd=0,
-            log_level=asgi_config.log_level,
-            access_log=False,
-            timeout_keep_alive=asgi_config.keepalive,
-            host=asgi_config.host,
-            port=asgi_config.port,
-            workers=1,  # per managed supervisor proc
-        )
+    uvicorn.run(
+        app,
+        fd=0,
+        log_level=asgi_config.log_level,
+        access_log=False,
+        timeout_keep_alive=asgi_config.keepalive,
+        host=asgi_config.host,
+        port=asgi_config.port,
+        workers=1,  # per managed supervisor proc
+    )
 
 
 def worker():
-    if config.worker_v2_enabled:
-        log.debug("Starting worker v2")
-        Worker().start()
-    else:
-        from sovereign.worker import worker as worker_app
-
-        log.debug("Starting worker")
-        uvicorn.run(
-            worker_app,
-            log_level=asgi_config.log_level,
-            access_log=False,
-            timeout_keep_alive=asgi_config.keepalive,
-            host="127.0.0.1",
-            port=9080,
-            workers=1,  # per managed supervisor proc
-        )
+    from sovereign.worker import worker as worker_app
+
+    log.debug("Starting worker")
+    uvicorn.run(
+        worker_app,
+        log_level=asgi_config.log_level,
+        access_log=False,
+        timeout_keep_alive=asgi_config.keepalive,
+        host="127.0.0.1",
+        port=9080,
+        workers=1,  # per managed supervisor proc
+    )
 
 
 def write_supervisor_conf() -> Path:

sovereign/sources/__init__.py

@@ -0,0 +1,3 @@
+from sovereign.sources.poller import SourcePoller
+
+__all__ = ["SourcePoller"]

sovereign/sources/file.py

@@ -0,0 +1,21 @@
+from typing import Any, Dict
+from sovereign.sources.lib import Source
+from sovereign.dynamic_config import Loadable
+
+
+class File(Source):
+    def __init__(self, config: Dict[str, Any], scope: str = "default"):
+        super(File, self).__init__(config, scope)
+        try:
+            self.path = Loadable.from_legacy_fmt(config["path"])
+        except KeyError:
+            try:
+                self.path = Loadable(**config["spec"])
+            except KeyError:
+                raise KeyError('File source needs to specify "spec" within config')
+
+    def get(self) -> Any:
+        """
+        Uses the file config loader to load the given path
+        """
+        return self.path.load()

sovereign/sources/inline.py

@@ -0,0 +1,39 @@
+"""
+Inline Source
+-------------
+
+Example configuration (YAML):
+
+.. code-block:: yaml
+
+   sources:
+     - type: inline
+       config:
+         instances:
+           - instance_id: my_service
+             service_clusters:
+               - P2
+             parameters:
+               upstream_address:
+                 - address: service.domain.com
+                   region: us-east-1
+             plan_id: 7d57270a-0348-58d3-829d-447a98fe98d5
+"""
+
+from typing import Dict, Any, List
+from sovereign.sources.lib import Source
+
+
+class Inline(Source):
+    def __init__(
+        self, config: Dict[str, List[Dict[str, Any]]], scope: str = "default"
+    ) -> None:
+        super(Inline, self).__init__(config, scope)
+        try:
+            self.instances = config["instances"]
+        except KeyError:
+            raise KeyError('Inline source config must contain "instances"')
+
+    def get(self) -> List[Dict[str, Any]]:
+        """Returns the data passed via configuration"""
+        return self.instances

sovereign/sources/lib.py

@@ -0,0 +1,41 @@
+"""
+Source abstract class
+---------------------
+This class can be subclassed, installed as an entry point, and then
+used via configuration.
+
+`todo entry point install guide`
+"""
+
+import abc
+from typing import Any, Dict, List
+
+
+class Source(abc.ABC):
+    def __init__(self, config: Dict[str, Any], scope: str) -> None:
+        """
+        :param config: arbitrary configuration which can be used by the subclass
+        """
+        self.config = config
+        self.scope = scope
+
+    def setup(self) -> None:
+        """
+        Optional method which is invoked prior to the Source running self.get()
+        """
+        return None
+
+    @abc.abstractmethod
+    def get(self) -> List[Dict[str, Any]]:
+        """
+        Required method to retrieve data from an arbitrary source
+        """
+        raise NotImplementedError
+
+
+class SourceImplementation(Source):
+    def __call__(self, config: Dict[str, Any], scope: str) -> "SourceImplementation":
+        return self
+
+    def get(self) -> List[Dict[str, Any]]:
+        return []