sovereign 0.14.2__py3-none-any.whl → 1.0.0a4__py3-none-any.whl

sovereign/views/discovery.py

@@ -1,44 +1,21 @@
-from typing import Dict
-
 from fastapi import Body, Header
-from fastapi.routing import APIRouter
 from fastapi.responses import Response
+from fastapi.routing import APIRouter
 
-from sovereign import discovery, logs
-from sovereign.utils.auth import authenticate
-from sovereign.schemas import (
+from sovereign import cache, config, logs
+from sovereign.cache.types import Entry
+from sovereign.types import (
     DiscoveryRequest,
     DiscoveryResponse,
-    ProcessedTemplate,
 )
-
-
-router = APIRouter()
-
-type_urls = {
-    "v2": {
-        "listeners": "type.googleapis.com/envoy.api.v2.Listener",
-        "clusters": "type.googleapis.com/envoy.api.v2.Cluster",
-        "endpoints": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
-        "secrets": "type.googleapis.com/envoy.api.v2.auth.Secret",
-        "routes": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
-        "scoped-routes": "type.googleapis.com/envoy.api.v2.ScopedRouteConfiguration",
-    },
-    "v3": {
-        "listeners": "type.googleapis.com/envoy.config.listener.v3.Listener",
-        "clusters": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
-        "endpoints": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
-        "secrets": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
-        "routes": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
-        "scoped-routes": "type.googleapis.com/envoy.config.route.v3.ScopedRouteConfiguration",
-        "runtime": "type.googleapis.com/envoy.service.runtime.v3.Runtime",
-    },
-}
+from sovereign.utils.auth import authenticate
+from sovereign.v2.web import wait_for_discovery_response
+from sovereign.views import reader
 
 
 def response_headers(
-    discovery_request: DiscoveryRequest, response: ProcessedTemplate, xds: str
-) -> Dict[str, str]:
+    discovery_request: DiscoveryRequest, response: Entry, xds: str
+) -> dict[str, str]:
     return {
         "X-Sovereign-Client-Build": discovery_request.envoy_version,
         "X-Sovereign-Client-Version": discovery_request.version_info,
@@ -49,6 +26,9 @@ def response_headers(
     }
 
 
+router = APIRouter()
+
+
 @router.post(
     "/{version}/discovery:{xds_type}",
     summary="Envoy Discovery Service Endpoint",
@@ -62,46 +42,51 @@ def response_headers(
 async def discovery_response(
     version: str,
     xds_type: str,
-    discovery_request: DiscoveryRequest = Body(...),
+    xds_req: DiscoveryRequest = Body(...),
     host: str = Header("no_host_provided"),
 ) -> Response:
-    discovery_request.desired_controlplane = host
-    response = perform_discovery(discovery_request, version, xds_type, skip_auth=False)
-    logs.queue_log_fields(
-        XDS_RESOURCES=discovery_request.resource_names,
-        XDS_ENVOY_VERSION=discovery_request.envoy_version,
-        XDS_CLIENT_VERSION=discovery_request.version_info,
-        XDS_SERVER_VERSION=response.version,
-    )
-    headers = response_headers(discovery_request, response, xds_type)
+    authenticate(xds_req)
 
-    if response.version == discovery_request.version_info:
-        return not_modified(headers)
-    elif getattr(response, "resources", None) == []:
-        return Response(status_code=404, headers=headers)
-    elif response.version != discovery_request.version_info:
-        return Response(
-            response.rendered, headers=headers, media_type="application/json"
+    # Pack additional info into the request
+    xds_req.desired_controlplane = host
+    xds_req.resource_type = xds_type
+    xds_req.api_version = version
+    if xds_req.error_detail:
+        logs.access_logger.queue_log_fields(
+            XDS_ERROR_DETAIL=xds_req.error_detail.message
         )
-    return Response(content="Resources could not be determined", status_code=500)
+    logs.access_logger.queue_log_fields(
+        XDS_RESOURCES=xds_req.resource_names,
+        XDS_ENVOY_VERSION=xds_req.envoy_version,
+        XDS_CLIENT_VERSION=xds_req.version_info,
+    )
 
+    def handle_response(entry: cache.Entry):
+        logs.access_logger.queue_log_fields(
+            XDS_SERVER_VERSION=entry.version,
+        )
+        headers = response_headers(xds_req, entry, xds_type)
+        if entry.len == 0:
+            return Response(status_code=404, headers=headers)
+        if entry.version == xds_req.version_info:
+            return Response(status_code=304, headers=headers)
+        return Response(entry.text, media_type="application/json", headers=headers)
 
-def perform_discovery(
-    req: DiscoveryRequest,
-    api_version: str,
-    resource_type: str,
-    skip_auth: bool = False,
-) -> ProcessedTemplate:
-    if not skip_auth:
-        authenticate(req)
-    template = discovery.response(req, resource_type)
-    type_url = type_urls.get(api_version, {}).get(resource_type)
-    if type_url is not None:
-        for resource in template.resources:
-            if not resource.get("@type"):
-                resource["@type"] = type_url
-    return template
+    if config.worker_v2_enabled:
+        # we're set up to use v2 of the worker
+        response = await wait_for_discovery_response(xds_req)
+        if response is not None:
+            entry = Entry(
+                text=response.model_dump_json(indent=None),
+                len=len(response.resources),
+                version=response.version_info,
+                node=xds_req.node,
+            )
+            return handle_response(entry)
 
+    else:
+        entry: Entry | None
+        if entry := await reader.blocking_read(xds_req):  # ty: ignore[possibly-missing-attribute]
+            return handle_response(entry)
 
-def not_modified(headers: Dict[str, str]) -> Response:
-    return Response(status_code=304, headers=headers)
+    return Response(content="Something went wrong", status_code=500)

sovereign/views/healthchecks.py

@@ -1,38 +1,125 @@
-from typing import List
-from fastapi import Response
+import asyncio
+
+import pydantic
+import requests
+from fastapi import Query, Request, Response
+from fastapi.responses import JSONResponse, PlainTextResponse
 from fastapi.routing import APIRouter
-from fastapi.responses import PlainTextResponse
-from sovereign import XDS_TEMPLATES, __version__, json_response_class
-from sovereign.utils.mock import mock_discovery_request
-from sovereign.views.discovery import perform_discovery
+from typing_extensions import Annotated, Literal
 
+from sovereign import __version__
+from sovereign.configuration import XDS_TEMPLATES, config
+from sovereign.response_class import json_response_class
+from sovereign.utils.mock import mock_discovery_request
+from sovereign.v2.web import wait_for_discovery_response
+from sovereign.views import reader
 
 router = APIRouter()
 
+State = Literal["FAIL"] | Literal["OK"]
+Message = str
+CheckResult = tuple[State, Message] | State
+
+
+class DeepCheckResult(pydantic.BaseModel):
+    templates: dict[str, CheckResult] = pydantic.Field(default_factory=dict)
+    worker: CheckResult = pydantic.Field(default=("FAIL", "Worker unavailable"))
+
+    def response(self) -> PlainTextResponse:
+        return PlainTextResponse(content=self.message, status_code=self.status)
+
+    def json_response(self) -> JSONResponse:
+        return json_response_class(content=self.model_dump(), status_code=self.status)
+
+    @property
+    def message(self) -> str:
+        msg = "Templates:\n"
+        for template, result in sorted(self.templates.items()):
+            msg += f"* {template} {result}\n"
+        msg += f"Worker: {self.worker}\n"
+        return msg
+
+    @property
+    def status(self) -> int:
+        if self.is_err():
+            return 500
+        return 200
+
+    def is_err(self):
+        for result in self.templates.values():
+            if result[0] == "FAIL":
+                return True
+        if self.worker[0] == "FAIL":
+            return True
+        return False
+
 
 @router.get("/healthcheck", summary="Healthcheck (Does the server respond to HTTP?)")
 async def health_check() -> Response:
-    return PlainTextResponse("OK")
+    return PlainTextResponse("OK", status_code=200)
 
 
 @router.get(
     "/deepcheck",
-    summary="Deepcheck (Can the server render all configured templates?)",
-    response_class=json_response_class,
+    summary="Deepcheck (Can the server render all default templates?)",
 )
-async def deep_check(response: Response) -> List[str]:
-    response.status_code = 200
-    ret = list()
+async def deep_check(
+    request: Request,
+    worker_attempts: Annotated[
+        int,
+        Query(
+            description="How many times to try to contact the worker before giving up",
+        ),
+    ] = 5,
+    envoy_service_cluster: Annotated[
+        str,
+        Query(
+            description="Which service cluster to use when checking if a template can be rendered",
+        ),
+    ] = "*",
+) -> Response:
+    result = DeepCheckResult()
     for template in list(XDS_TEMPLATES["default"].keys()):
-        try:
-            req = mock_discovery_request(service_cluster="*")
-            perform_discovery(req, "v3", resource_type=template, skip_auth=True)
-        # pylint: disable=broad-except
-        except Exception as e:
-            ret.append(f"Failed {template}: {str(e)}")
+        discovery_request = mock_discovery_request(
+            "v3",
+            template,
+            expressions=[f"cluster={envoy_service_cluster}"],
+        )
+
+        if config.worker_v2_enabled:
+            # we're set up to use v2 of the worker
+            response = await wait_for_discovery_response(discovery_request)
+            if response:
+                result.templates[template] = "OK"
+            else:
+                result.templates[template] = (
+                    "FAIL",
+                    f"Failed to render {template}",
+                )
+
+            result.worker = "OK"
         else:
-            ret.append(f"Rendered {template} OK")
-    return ret
+            try:
+                _ = await reader.blocking_read(discovery_request)  # ty: ignore[possibly-missing-attribute]
+                result.templates[template] = "OK"
+            except Exception as e:
+                result.templates[template] = ("FAIL", f"Failed {template}: {str(e)}")
+
+    if not config.worker_v2_enabled:
+        for attempt in range(worker_attempts):
+            try:
+                worker_health = requests.get("http://localhost:9080/health")
+                if worker_health.ok:
+                    result.worker = "OK"
+                    break
+            except Exception as e:
+                result.worker = ("FAIL", str(e))
+                await asyncio.sleep(attempt)
+
+    if "json" in request.headers.get("Accept", ""):
+        return result.json_response()
+
+    return result.response()
 
 
 @router.get("/version", summary="Display the current version of Sovereign")

sovereign/views/interface.py

@@ -1,162 +1,211 @@
-from typing import List, Dict, Any
+import json
+import logging
 from collections import defaultdict
-from fastapi import APIRouter, Query, Path, Cookie
+from typing import Any, Dict, List
+
+from fastapi import APIRouter, Cookie, Path, Query
 from fastapi.encoders import jsonable_encoder
 from fastapi.requests import Request
-from fastapi.responses import RedirectResponse, JSONResponse, Response
-from sovereign import html_templates, XDS_TEMPLATES
-from sovereign.discovery import DiscoveryTypes
-from sovereign import poller, json_response_class
-from sovereign.utils.mock import mock_discovery_request
-from sovereign.views.discovery import perform_discovery
+from fastapi.responses import HTMLResponse, JSONResponse, Response
+from starlette.templating import Jinja2Templates
+from structlog.typing import FilteringBoundLogger
+
+from sovereign import __version__
+from sovereign.cache import Entry
+from sovereign.configuration import XDS_TEMPLATES, ConfiguredResourceTypes, config
+from sovereign.response_class import json_response_class
+from sovereign.utils.mock import NodeExpressionError, mock_discovery_request
+from sovereign.utils.resources import get_package_file
+from sovereign.v2.logging import get_named_logger
+from sovereign.v2.web import wait_for_discovery_response
+from sovereign.views import reader
 
 router = APIRouter()
 
-all_types = [t.value for t in DiscoveryTypes]
+all_types = [t.value for t in ConfiguredResourceTypes]
+
+html_templates = Jinja2Templates(
+    directory=str(get_package_file("sovereign", "templates"))
+)
 
 
 @router.get("/")
-async def ui_main(request: Request) -> Response:
+@router.get("/resources")
+async def ui_main(request: Request) -> HTMLResponse:
     try:
         return html_templates.TemplateResponse(
+            request=request,
             name="base.html",
             media_type="text/html",
-            context={
-                "request": request,
-                "all_types": all_types,
-                "last_update": str(poller.last_updated),
-            },
+            context={"all_types": all_types, "sovereign_version": __version__},
         )
     except IndexError:
         return html_templates.TemplateResponse(
+            request=request,
             name="err.html",
             media_type="text/html",
             context={
-                "request": request,
                 "title": "No resource types configured",
-                "message": "A template should be defined for every resource "
-                "type that you want your envoy proxies to discover.",
-                "doc_link": "https://vsyrakis.bitbucket.io/sovereign/docs/tutorial/templates/",
+                "message": (
+                    "A template should be defined for every resource "
+                    "type that you want your envoy proxies to discover."
+                ),
+                "doc_link": "https://developer.atlassian.com/platform/sovereign/tutorial/templates/#templates",
+                "sovereign_version": __version__,
             },
         )
 
 
-@router.get(
-    "/set-version", summary="Filter the UI by a certain Envoy Version (stores a Cookie)"
-)
-async def set_envoy_version(
-    request: Request,
-    version: str = Query(
-        "__any__", title="The clients envoy version to emulate in this XDS request"
-    ),
-) -> Response:
-    url = request.headers.get("Referer", "/ui")
-    response = RedirectResponse(url=url)
-    response.set_cookie(key="envoy_version", value=version)
-    return response
-
-
-@router.get(
-    "/set-service-cluster",
-    summary="Filter the UI by a certain service cluster (stores a Cookie)",
-)
-async def set_service_cluster(
-    request: Request,
-    service_cluster: str = Query(
-        "__any__", title="The clients envoy version to emulate in this XDS request"
-    ),
-) -> Response:
-    url = request.headers.get("Referer", "/ui")
-    response = RedirectResponse(url=url)
-    response.set_cookie(key="service_cluster", value=service_cluster)
-    return response
-
-
+# noinspection DuplicatedCode
 @router.get(
     "/resources/{xds_type}", summary="List available resources for a given xDS type"
 )
 async def resources(
     request: Request,
-    xds_type: str = Path(
-        "clusters", title="xDS type", description="The type of request"
-    ),
+    xds_type: str = Path(title="xDS type", description="The type of request"),
     region: str = Query(
         None, title="The clients region to emulate in this XDS request"
     ),
     api_version: str = Query("v2", title="The desired Envoy API version"),
-    service_cluster: str = Cookie(
-        "*", title="The clients service cluster to emulate in this XDS request"
+    node_expression: str = Cookie(
+        "cluster=*", title="Node expression to filter resources with"
     ),
     envoy_version: str = Cookie(
         "__any__", title="The clients envoy version to emulate in this XDS request"
     ),
-) -> Response:
+    debug: int = Query(0, title="Show debug information on errors"),
+) -> HTMLResponse:
+    logger: FilteringBoundLogger = get_named_logger(
+        f"{__name__}.{resources.__qualname__} ({__file__})",
+        level=logging.DEBUG,
+    )
+
     ret: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
     try:
-        response = perform_discovery(
-            req=mock_discovery_request(
-                service_cluster=service_cluster,
-                resource_names=[],
-                version=envoy_version,
-                region=region,
-            ),
-            api_version=api_version,
-            resource_type=xds_type,
-            skip_auth=True,
+        mock_request = mock_discovery_request(
+            api_version,
+            xds_type,
+            version=envoy_version,
+            region=region,
+            expressions=node_expression.split(),
         )
-    except KeyError:
-        ret["resources"] = []
+        clear_cookie = False
+        error = None
+    except NodeExpressionError as e:
+        mock_request = mock_discovery_request(
+            api_version,
+            xds_type,
+            version=envoy_version,
+            region=region,
+        )
+        clear_cookie = True
+        error = str(e)
+
+    logger.debug("Making mock request", mock_request=mock_request)
+
+    entry: Entry | None = None
+
+    if config.worker_v2_enabled:
+        # we're set up to use v2 of the worker
+        discovery_response = await wait_for_discovery_response(mock_request)
+        if discovery_response is not None:
+            entry = Entry(
+                text=discovery_response.model_dump_json(indent=None),
+                len=len(discovery_response.resources),
+                version=discovery_response.version_info,
+                node=mock_request.node,
+            )
+
     else:
-        ret["resources"] += response.deserialize_resources()
-    return html_templates.TemplateResponse(
+        entry = await reader.blocking_read(mock_request)  # ty: ignore[possibly-missing-attribute]
+
+    if entry:
+        ret["resources"] = json.loads(entry.text).get("resources", [])
+
+    resp = html_templates.TemplateResponse(
+        request=request,
         name="resources.html",
         media_type="text/html",
         context={
+            "show_debuginfo": True if debug else False,
+            "discovery_response": entry,
+            "discovery_request": mock_request,
             "resources": ret["resources"],
-            "request": request,
             "resource_type": xds_type,
             "all_types": all_types,
             "version": envoy_version,
             "available_versions": list(XDS_TEMPLATES.keys()),
-            "service_cluster": service_cluster,
-            "available_service_clusters": poller.match_keys,
-            "last_update": str(poller.last_updated),
+            "error": error,
+            "sovereign_version": __version__,
         },
     )
+    if clear_cookie:
+        resp.delete_cookie("node_expression", path="/ui/resources/")
+    return resp
 
 
+# noinspection DuplicatedCode
 @router.get(
     "/resources/{xds_type}/{resource_name}",
     summary="Return JSON representation of a resource",
 )
 async def resource(
-    xds_type: str = Path(
-        "clusters", title="xDS type", description="The type of request"
-    ),
+    xds_type: str = Path(title="xDS type", description="The type of request"),
     resource_name: str = Path(..., title="Name of the resource to view"),
     region: str = Query(
         None, title="The clients region to emulate in this XDS request"
     ),
     api_version: str = Query("v2", title="The desired Envoy API version"),
-    service_cluster: str = Cookie(
-        "*", title="The clients service cluster to emulate in this XDS request"
+    node_expression: str = Cookie(
+        "cluster=*", title="Node expression to filter resources with"
     ),
     envoy_version: str = Cookie(
         "__any__", title="The clients envoy version to emulate in this XDS request"
     ),
 ) -> Response:
-    response = perform_discovery(
-        req=mock_discovery_request(
-            service_cluster=service_cluster,
-            resource_names=[resource_name],
-            version=envoy_version,
-            region=region,
-        ),
-        api_version=api_version,
-        resource_type=xds_type,
-        skip_auth=True,
+    logger: FilteringBoundLogger = get_named_logger(
+        f"{__name__}.{resources.__qualname__} ({__file__})",
+        level=logging.DEBUG,
+    )
+
+    mock_request = mock_discovery_request(
+        api_version,
+        xds_type,
+        version=envoy_version,
+        region=region,
+        expressions=node_expression.split(),
+    )
+
+    logger.debug("Making mock request", mock_request=mock_request)
+
+    entry: Entry | None = None
+
+    if config.worker_v2_enabled:
+        # we're set up to use v2 of the worker
+        discovery_response = await wait_for_discovery_response(mock_request)
+        if discovery_response is not None:
+            entry = Entry(
+                text=discovery_response.model_dump_json(indent=None),
+                len=len(discovery_response.resources),
+                version=discovery_response.version_info,
+                node=mock_request.node,
+            )
+
+    else:
+        entry = await reader.blocking_read(mock_request)  # ty: ignore[possibly-missing-attribute]
+
+    if entry:
+        for res in json.loads(entry.text).get("resources", []):
+            if res.get("name", res.get("cluster_name")) == resource_name:
+                safe_response = jsonable_encoder(res)
+                try:
+                    return json_response_class(content=safe_response)
+                except TypeError:
+                    return JSONResponse(content=safe_response)
+    return Response(
+        json.dumps({"title": "No resources found", "status": 404}),
+        media_type="application/json+problem",
     )
-    return Response(response.rendered, media_type="application/json")
 
 
 @router.get(
@@ -170,36 +219,43 @@ async def virtual_hosts(
         None, title="The clients region to emulate in this XDS request"
     ),
     api_version: str = Query("v2", title="The desired Envoy API version"),
-    service_cluster: str = Cookie(
-        "*", title="The clients service cluster to emulate in this XDS request"
+    node_expression: str = Cookie(
+        "cluster=*", title="Node expression to filter resources with"
     ),
     envoy_version: str = Cookie(
         "__any__", title="The clients envoy version to emulate in this XDS request"
     ),
 ) -> Response:
-    response = perform_discovery(
-        req=mock_discovery_request(
-            service_cluster=service_cluster,
-            resource_names=[route_configuration],
-            version=envoy_version,
-            region=region,
-        ),
-        api_version=api_version,
-        resource_type="routes",
-        skip_auth=True,
+    logger: FilteringBoundLogger = get_named_logger(
+        f"{__name__}.{virtual_hosts.__qualname__} ({__file__})",
+        level=logging.DEBUG,
+    )
+
+    mock_request = mock_discovery_request(
+        api_version,
+        "routes",
+        version=envoy_version,
+        region=region,
+        expressions=node_expression.split(),
+    )
+
+    logger.debug("Making mock request", mock_request=mock_request)
+
+    if response := await reader.blocking_read(mock_request):  # ty: ignore[possibly-missing-attribute]
+        route_configs = [
+            resource_
+            for resource_ in json.loads(response.text).get("resources", [])
+            if resource_["name"] == route_configuration
+        ]
+        for route_config in route_configs:
+            for vhost in route_config["virtual_hosts"]:
+                if vhost["name"] == virtual_host:
+                    safe_response = jsonable_encoder(vhost)
+                    try:
+                        return json_response_class(content=safe_response)
+                    except TypeError:
+                        return JSONResponse(content=safe_response)
+    return Response(
+        json.dumps({"title": "No resources found", "status": 404}),
+        media_type="application/json+problem",
     )
-    route_configs = [
-        resource_
-        for resource_ in response.deserialize_resources()
-        if resource_["name"] == route_configuration
-    ]
-    for route_config in route_configs:
-        for vhost in route_config["virtual_hosts"]:
-            if vhost["name"] == virtual_host:
-                safe_response = jsonable_encoder(vhost)
-                try:
-                    return json_response_class(content=safe_response)
-                except TypeError:
-                    return JSONResponse(content=safe_response)
-        break
-    return JSONResponse(content={})