sovereign 0.19.3__py3-none-any.whl → 1.0.0b148__py3-none-any.whl

sovereign/utils/crypto/suites/disabled_cipher.py

@@ -0,0 +1,25 @@
+from typing import Any
+
+from .base_cipher import CipherSuite
+
+
+class DisabledCipher(CipherSuite):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        pass
+
+    def __str__(self) -> str:
+        return "disabled"
+
+    def encrypt(self, _: str) -> str:
+        return "Unavailable (No Secret Key)"
+
+    def decrypt(self, _: str) -> str:
+        return "Unavailable (No Secret Key)"
+
+    @property
+    def key_available(self) -> bool:
+        return False
+
+    @classmethod
+    def generate_key(cls) -> bytes:
+        return b"Unavailable (No key to generate)"

sovereign/utils/crypto/suites/fernet_cipher.py

@@ -0,0 +1,29 @@
+import base64
+import os
+
+from cryptography.fernet import Fernet
+
+from .base_cipher import CipherSuite
+
+
+class FernetCipher(CipherSuite):
+    def __str__(self) -> str:
+        return "fernet"
+
+    def __init__(self, secret_key: str):
+        self.fernet = Fernet(secret_key)
+
+    def encrypt(self, data: str) -> str:
+        return self.fernet.encrypt(data.encode()).decode("utf-8")
+
+    def decrypt(self, data: str) -> str:
+        return self.fernet.decrypt(data).decode("utf-8")
+
+    @property
+    def key_available(self) -> bool:
+        return True
+
+    @classmethod
+    def generate_key(cls) -> bytes:
+        # Generate 256 bit length key
+        return base64.urlsafe_b64encode(os.urandom(32))

sovereign/utils/dictupdate.py

@@ -1,6 +1,7 @@
 # type: ignore
 # pylint: disable=no-name-in-module,too-many-branches
-""" Stolen from the saltstack library """
+"""Stolen from the saltstack library"""
+
 from collections.abc import Mapping
 import copy
 

sovereign/utils/eds.py

@@ -2,8 +2,8 @@ import random
 from typing import Dict, Any, Optional, List
 from copy import deepcopy
 from starlette.exceptions import HTTPException
-from sovereign import config
-from sovereign.schemas import DiscoveryRequest
+from sovereign.configuration import config
+from sovereign.types import DiscoveryRequest
 from sovereign.utils.templates import resolve
 
 HARD_FAIL_ON_DNS_FAILURE = config.legacy_fields.dns_hard_fail
@@ -26,12 +26,16 @@ def _upstream_kwargs(
         if hard_fail:
             raise
         ip_addresses = [upstream["address"]]
-    return {
+    ret = {
         "addrs": ip_addresses,
         "port": upstream["port"],
         "region": default_region or upstream.get("region", "unknown"),
         "zone": proxy_region,
     }
+    if "health_check_config" in upstream:
+        ret["health_check_config"] = upstream["health_check_config"]
+
+    return ret
 
 
 def total_zones(endpoints: List[Dict[str, Dict[str, Any]]]) -> int:
@@ -84,14 +88,21 @@ def locality_lb_endpoints(
     return ret
 
 
-def lb_endpoints(addrs: List[str], port: int, region: str, zone: str) -> Dict[str, Any]:
+def lb_endpoints(
+    addrs: List[str],
+    port: int,
+    region: str,
+    zone: str,
+    health_check_config: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
     """
     Creates an envoy endpoint.LbEndpoints proto
 
-    :param addrs:  The IP addresses or hostname(s) of the upstream.
-    :param port:   The port that the upstream should be accessed on.
-    :param region: The region of the upstream.
-    :param zone:   The region of the proxy asking for the endpoint configuration.
+    :param addrs:                   The IP addresses or hostname(s) of the upstream.
+    :param port:                    The port that the upstream should be accessed on.
+    :param region:                  The region of the upstream.
+    :param zone:                    The region of the proxy asking for the endpoint configuration.
+    :param health_check_config:     Optional health check config for the upstream.
     """
     if PRIORITY_MAPPING is None:
         raise RuntimeError(
@@ -100,20 +111,25 @@ def lb_endpoints(addrs: List[str], port: int, region: str, zone: str) -> Dict[st
         )
     node_priorities = PRIORITY_MAPPING.get(zone, {})
     priority = node_priorities.get(region, 10)
+
+    endpoints = []
+    for addr in addrs:
+        endpoint = {
+            "endpoint": {
+                "address": {
+                    "socket_address": {
+                        "address": addr,
+                        "port_value": port,
+                    }
+                },
+            }
+        }
+        if health_check_config:
+            endpoint["endpoint"]["health_check_config"] = health_check_config
+        endpoints.append(endpoint)
+
     return {
         "priority": priority,
         "locality": {"zone": region},
-        "lb_endpoints": [
-            {
-                "endpoint": {
-                    "address": {
-                        "socket_address": {
-                            "address": addr,
-                            "port_value": port,
-                        }
-                    }
-                }
-            }
-            for addr in addrs
-        ],
+        "lb_endpoints": endpoints,
     }

sovereign/utils/mock.py

@@ -1,36 +1,74 @@
+import re
+import ast
 from typing import Optional, Dict, List
 from random import randint
-from sovereign.schemas import DiscoveryRequest, Node, Locality, Resources, Status
+from sovereign.types import DiscoveryRequest, Node, Locality, Status
+
+scrub = re.compile(r"[^a-zA-Z_\.]")
+
+
+class NodeExpressionError(Exception):
+    pass
 
 
 def mock_discovery_request(
-    service_cluster: Optional[str] = None,
-    resource_names: Optional[List[str]] = None,
-    region: str = "none",
-    version: str = "1.11.1",
+    api_version: Optional[str] = "V3",
+    resource_type: Optional[str] = None,
+    resource_names: Optional[List[str] | str] = None,
+    region: Optional[str] = "none",
+    version: Optional[str] = "<envoy_version>",
     metadata: Optional[Dict[str, str]] = None,
     error_message: Optional[str] = None,
+    expressions: Optional[list[str]] = None,
 ) -> DiscoveryRequest:
     if resource_names is None:
-        resource_names = Resources()
-    else:
-        resource_names = Resources(resource_names)
+        resource_names = []
+    if isinstance(resource_names, str):
+        resource_names = [resource_names]
+    if expressions is None:
+        expressions = []
+    base_node = Node(
+        id="sovereign-interface",
+        cluster="*",
+        build_version=f"<randomHash>/{version}/Clean/RELEASE",
+        locality=Locality(zone=region),
+    ).model_dump()
+    set_node_expressions(base_node, expressions)
     request = DiscoveryRequest(
         type_url=None,
-        node=Node(
-            id="mock",
-            cluster=service_cluster or "",
-            build_version=f"e5f864a82d4f27110359daa2fbdcb12d99e415b9/{version}/Clean/RELEASE",
-            locality=Locality(zone=region, region="example", sub_zone="a"),
-        ),
+        node=Node.model_validate(base_node),
         version_info=str(randint(100000, 1000000000)),
         resource_names=resource_names,
-        hide_private_keys=True,
-        desired_controlplane="controlplane",
+        is_internal_request=True,
+        desired_controlplane="__sovereign__",
         error_detail=Status(code=200, message="None", details=["None"]),
+        api_version=api_version,
+        resource_type=resource_type,
     )
     if isinstance(metadata, dict):
         request.node.metadata = metadata
     if error_message:
         request.error_detail = Status(code=666, message=error_message, details=["foo"])
     return request
+
+
+def set_node_expressions(node, expressions):
+    for expr in expressions:
+        try:
+            field, value = re.split(r"\s*=\s*", expr, maxsplit=1)
+            value = f'"{value}"'
+        except ValueError:
+            raise NodeExpressionError(f"Invalid node filter format: {expr}")
+
+        field = scrub.sub("", field)
+        parts = field.split(".")
+
+        try:
+            value = ast.literal_eval(value)
+        except Exception as e:
+            raise NodeExpressionError(f"Invalid node filter value: {value}") from e
+
+        current = node
+        for part in parts[:-1]:
+            current = current.setdefault(part, {})
+        current[parts[-1]] = value

sovereign/utils/resources.py

@@ -0,0 +1,17 @@
+from functools import cache
+import importlib.resources as res
+from importlib.resources.abc import Traversable
+
+
+@cache
+def get_package(name: str) -> Traversable:
+    return res.files(name)
+
+
+def get_package_file(package_name: str, filename: str) -> Traversable:
+    return get_package(package_name).joinpath(filename)
+
+
+def get_package_file_bytes(package_name: str, filename: str) -> bytes:
+    file = get_package_file(package_name, filename)
+    return file.read_bytes()

sovereign/utils/version_info.py

@@ -11,3 +11,11 @@ def compute_hash(*args: Any) -> str:
         zlib.crc32(data) & 0xFFFFFFFF
     )  # same numeric value across all py versions & platforms
     return str(version_info)
+
+
+def compute_hash_int(*args: Any) -> int:
+    data: bytes = repr(args).encode()
+    version_info = (
+        zlib.crc32(data) & 0xFFFFFFFF
+    )  # same numeric value across all py versions & platforms
+    return version_info

sovereign/views/__init__.py

@@ -0,0 +1,4 @@
+from sovereign import cache
+
+
+reader = cache.CacheReader()

sovereign/views/api.py

@@ -0,0 +1,61 @@
+import json
+from typing import Optional
+
+from fastapi import APIRouter, Query, Path
+from fastapi.responses import Response
+
+from sovereign.views import reader
+from sovereign.configuration import ConfiguredResourceTypes
+from sovereign.utils.mock import mock_discovery_request
+
+router = APIRouter()
+
+
+def _traverse(data, prefix, expressions):
+    for key, value in data.items():
+        path = f"{prefix}.{key}" if prefix else key
+        if isinstance(value, dict):
+            yield from _traverse(value, path, expressions)
+        else:
+            yield f"{path}={value}"
+
+
+def expand_metadata_to_expr(m):
+    exprs = []
+    yield from _traverse(m, "", exprs)
+
+
+@router.get("/resources/{resource_type}", summary="Get resources for a given type")
+async def resource(
+    resource_type: ConfiguredResourceTypes = Path(title="xDS Resource type"),
+    resource_name: Optional[str] = Query(None, title="Resource name"),
+    api_version: Optional[str] = Query("v3", title="Envoy API version"),
+    service_cluster: Optional[str] = Query("*", title="Envoy Service cluster"),
+    region: Optional[str] = Query(None, title="Locality Zone"),
+    version: Optional[str] = Query(None, title="Envoy Semantic Version"),
+    metadata: Optional[str] = Query(None, title="Envoy node metadata to filter by"),
+) -> Response:
+    expressions = [f"cluster={service_cluster}"]
+    try:
+        data = {"metadata": json.loads(metadata or "{}")}
+        for expr in expand_metadata_to_expr(data):
+            expressions.append(expr)
+    except Exception:
+        pass
+    kwargs = {
+        "api_version": api_version,
+        "resource_type": ConfiguredResourceTypes(resource_type).value,
+        "resource_names": resource_name,
+        "version": version,
+        "region": region,
+        "expressions": expressions,
+    }
+    req = mock_discovery_request(**{k: v for k, v in kwargs.items() if v is not None})  # type: ignore
+    response = await reader.blocking_read(req)
+    if content := getattr(response, "text", None):
+        return Response(content, media_type="application/json")
+    else:
+        return Response(
+            json.dumps({"title": "No resources found", "status": 404}),
+            media_type="application/json+problem",
+        )

sovereign/views/crypto.py

@@ -1,31 +1,38 @@
-from typing import Dict
-from pydantic import BaseModel, Field
+from typing import Any, Dict, Optional
+
 from fastapi import APIRouter, Body
 from fastapi.responses import JSONResponse
-from sovereign import json_response_class, cipher_suite
-from sovereign.utils.crypto import generate_key
+from pydantic import BaseModel, Field
+
+from sovereign import logs, server_cipher_container
+from sovereign.response_class import json_response_class
+from sovereign.configuration import EncryptionConfig
+from sovereign.utils.crypto.crypto import CipherContainer
+from sovereign.utils.crypto.suites import EncryptionType
 
 router = APIRouter()
 
 
 class EncryptionRequest(BaseModel):
     data: str = Field(..., title="Text to be encrypted", min_length=1, max_length=65535)
-    key: str = Field(
+    key: Optional[str] = Field(
         None,
-        title="Optional Fernet encryption key to use to encrypt",
+        title="Optional encryption key to use to encrypt",
         min_length=44,
         max_length=44,
     )
+    encryption_type: str = Field(default="fernet", title="Encryption type to be used")
 
 
 class DecryptionRequest(BaseModel):
     data: str = Field(..., title="Text to be decrypted", min_length=1, max_length=65535)
     key: str = Field(
         ...,
-        title="Fernet encryption key to use to decrypt",
+        title="Encryption key to use to decrypt",
         min_length=44,
         max_length=44,
     )
+    encryption_type: str = Field(default="fernet", title="Encryption type to be used")
 
 
 class DecryptableRequest(BaseModel):
@@ -37,17 +44,37 @@ class DecryptableRequest(BaseModel):
     summary="Decrypt provided encrypted data using a provided key",
     response_class=json_response_class,
 )
-async def _decrypt(request: DecryptionRequest = Body(None)) -> Dict[str, str]:
-    return {"result": cipher_suite.decrypt(request.data, request.key)}
+async def _decrypt(request: DecryptionRequest = Body(None)) -> dict[str, Any]:
+    user_cipher_container = CipherContainer.from_encryption_configs(
+        encryption_configs=[
+            EncryptionConfig(
+                encryption_key=request.key,
+                encryption_type=EncryptionType(request.encryption_type),
+            )
+        ],
+        logger=logs.application_logger.logger,
+    )
+    return {**user_cipher_container.decrypt_with_type(request.data)}
 
 
 @router.post(
     "/encrypt",
-    summary="Encrypt provided data using this servers key",
+    summary="Encrypt provided data using this servers key or provided key",
     response_class=json_response_class,
 )
-async def _encrypt(request: EncryptionRequest = Body(None)) -> Dict[str, str]:
-    return {"result": cipher_suite.encrypt(data=request.data, key=request.key)}
+async def _encrypt(request: EncryptionRequest = Body(None)) -> dict[str, Any]:
+    if request.key:
+        user_cipher_container = CipherContainer.from_encryption_configs(
+            encryption_configs=[
+                EncryptionConfig(
+                    encryption_key=request.key,
+                    encryption_type=EncryptionType(request.encryption_type),
+                )
+            ],
+            logger=logs.application_logger.logger,
+        )
+        return {**user_cipher_container.encrypt(request.data)}
+    return {**server_cipher_container.encrypt(request.data)}
 
 
 @router.post(
@@ -56,7 +83,7 @@ async def _encrypt(request: EncryptionRequest = Body(None)) -> Dict[str, str]:
     response_class=json_response_class,
 )
 async def _decryptable(request: DecryptableRequest = Body(None)) -> JSONResponse:
-    cipher_suite.decrypt(request.data)
+    server_cipher_container.decrypt(request.data)
     return json_response_class({})
 
 
@@ -65,5 +92,9 @@ async def _decryptable(request: DecryptableRequest = Body(None)) -> JSONResponse
     summary="Generate a new asymmetric encryption key",
     response_class=json_response_class,
 )
-def _generate_key() -> Dict[str, str]:
-    return {"result": generate_key()}
+def _generate_key(encryption_type: str = "fernet") -> Dict[str, str]:
+    cipher_suite = CipherContainer.get_cipher_suite(EncryptionType(encryption_type))
+    return {
+        "key": cipher_suite.generate_key().decode(),
+        "encryption_type": encryption_type,
+    }

sovereign/views/discovery.py

@@ -1,63 +1,20 @@
-from typing import Dict
-
 from fastapi import Body, Header
-from fastapi.routing import APIRouter
 from fastapi.responses import Response
+from fastapi.routing import APIRouter
 
-from sovereign import discovery, logs, config
+from sovereign import cache, logs
+from sovereign.views import reader
+from sovereign.cache.types import Entry
 from sovereign.utils.auth import authenticate
-from sovereign.utils.version_info import compute_hash
-from sovereign.schemas import (
+from sovereign.types import (
     DiscoveryRequest,
     DiscoveryResponse,
-    ProcessedTemplate,
 )
 
-discovery_cache = config.discovery_cache
-
-if discovery_cache.enabled:
-    from cashews import cache
-
-    cache.setup(
-        f"{discovery_cache.protocol}{discovery_cache.host}:{discovery_cache.port}",
-        password=discovery_cache.password.get_secret_value(),
-        client_side=discovery_cache.client_side,
-        wait_for_connection_timeout=discovery_cache.wait_for_connection_timeout,
-        socket_connect_timeout=discovery_cache.socket_connect_timeout,
-        socket_timeout=discovery_cache.socket_timeout,
-        max_connections=discovery_cache.max_connections,
-        retry_on_timeout=discovery_cache.retry_on_timeout,
-        suppress=discovery_cache.suppress,
-        socket_keepalive=discovery_cache.socket_keepalive,
-        enable=discovery_cache.enabled,
-    )
-
-router = APIRouter()
-
-type_urls = {
-    "v2": {
-        "listeners": "type.googleapis.com/envoy.api.v2.Listener",
-        "clusters": "type.googleapis.com/envoy.api.v2.Cluster",
-        "endpoints": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
-        "secrets": "type.googleapis.com/envoy.api.v2.auth.Secret",
-        "routes": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
-        "scoped-routes": "type.googleapis.com/envoy.api.v2.ScopedRouteConfiguration",
-    },
-    "v3": {
-        "listeners": "type.googleapis.com/envoy.config.listener.v3.Listener",
-        "clusters": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
-        "endpoints": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
-        "secrets": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
-        "routes": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
-        "scoped-routes": "type.googleapis.com/envoy.config.route.v3.ScopedRouteConfiguration",
-        "runtime": "type.googleapis.com/envoy.service.runtime.v3.Runtime",
-    },
-}
-
 
 def response_headers(
-    discovery_request: DiscoveryRequest, response: ProcessedTemplate, xds: str
-) -> Dict[str, str]:
+    discovery_request: DiscoveryRequest, response: Entry, xds: str
+) -> dict[str, str]:
     return {
         "X-Sovereign-Client-Build": discovery_request.envoy_version,
         "X-Sovereign-Client-Version": discovery_request.version_info,
@@ -68,6 +25,9 @@ def response_headers(
     }
 
 
+router = APIRouter()
+
+
 @router.post(
     "/{version}/discovery:{xds_type}",
     summary="Envoy Discovery Service Endpoint",
@@ -81,76 +41,37 @@ def response_headers(
 async def discovery_response(
     version: str,
     xds_type: str,
-    discovery_request: DiscoveryRequest = Body(...),
+    xds_req: DiscoveryRequest = Body(...),
     host: str = Header("no_host_provided"),
 ) -> Response:
-    discovery_request.desired_controlplane = host
-    response = await perform_discovery(
-        discovery_request, version, xds_type, skip_auth=False
-    )
-    logs.queue_log_fields(
-        XDS_RESOURCES=discovery_request.resource_names,
-        XDS_ENVOY_VERSION=discovery_request.envoy_version,
-        XDS_CLIENT_VERSION=discovery_request.version_info,
-        XDS_SERVER_VERSION=response.version,
-    )
-    if discovery_request.error_detail:
-        logs.queue_log_fields(XDS_ERROR_DETAIL=discovery_request.error_detail.message)
-    headers = response_headers(discovery_request, response, xds_type)
-
-    if response.version == discovery_request.version_info:
-        return not_modified(headers)
-    elif getattr(response, "resources", None) == []:
-        return Response(status_code=404, headers=headers)
-    elif response.version != discovery_request.version_info:
-        return Response(
-            response.rendered, headers=headers, media_type="application/json"
+    authenticate(xds_req)
+
+    # Pack additional info into the request
+    xds_req.desired_controlplane = host
+    xds_req.resource_type = xds_type
+    xds_req.api_version = version
+    if xds_req.error_detail:
+        logs.access_logger.queue_log_fields(
+            XDS_ERROR_DETAIL=xds_req.error_detail.message
         )
-    return Response(content="Resources could not be determined", status_code=500)
-
+    logs.access_logger.queue_log_fields(
+        XDS_RESOURCES=xds_req.resource_names,
+        XDS_ENVOY_VERSION=xds_req.envoy_version,
+        XDS_CLIENT_VERSION=xds_req.version_info,
+    )
 
-async def perform_discovery(
-    req: DiscoveryRequest,
-    api_version: str,
-    resource_type: str,
-    skip_auth: bool = False,
-) -> ProcessedTemplate:
-    if not skip_auth:
-        authenticate(req)
-    if discovery_cache.enabled:
-        logs.queue_log_fields(CACHE_XDS_HIT=False)
-        cache_key = compute_hash(
-            [
-                api_version,
-                resource_type,
-                req.envoy_version,
-                req.resource_names,
-                req.desired_controlplane,
-                req.hide_private_keys,
-                req.type_url,
-                req.node.cluster,
-                req.node.locality,
-                req.node.metadata.get("auth", None),
-                req.node.metadata.get("num_cpus", None),
-            ]
-        )
-        if template := await cache.get(key=cache_key, default=None):
-            logs.queue_log_fields(CACHE_XDS_HIT=True)
-            return template  # type: ignore[no-any-return]
-    template = discovery.response(req, resource_type)
-    type_url = type_urls.get(api_version, {}).get(resource_type)
-    if type_url is not None:
-        for resource in template.resources:
-            if not resource.get("@type"):
-                resource["@type"] = type_url
-    if discovery_cache.enabled:
-        await cache.set(
-            key=cache_key,
-            value=template,
-            expire=discovery_cache.ttl,
+    def handle_response(entry: cache.Entry):
+        logs.access_logger.queue_log_fields(
+            XDS_SERVER_VERSION=entry.version,
         )
-    return template  # type: ignore[no-any-return]
+        headers = response_headers(xds_req, entry, xds_type)
+        if entry.len == 0:
+            return Response(status_code=404, headers=headers)
+        if entry.version == xds_req.version_info:
+            return Response(status_code=304, headers=headers)
+        return Response(entry.text, media_type="application/json", headers=headers)
 
+    if entry := await reader.blocking_read(xds_req):
+        return handle_response(entry)
 
-def not_modified(headers: Dict[str, str]) -> Response:
-    return Response(status_code=304, headers=headers)
+    return Response(content="Something went wrong", status_code=500)