sovereign 0.19.3__py3-none-any.whl → 1.0.0a4__py3-none-any.whl

sovereign/types.py

@@ -0,0 +1,304 @@
+import hashlib
+import importlib
+from functools import cached_property
+from types import ModuleType
+
+import jmespath
+from jinja2 import Template
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    computed_field,
+)
+from typing_extensions import Any, cast
+
+from sovereign.dynamic_config import Loadable
+from sovereign.utils.version_info import compute_hash
+
+missing_arguments = {"missing", "positional", "arguments:"}
+
+
+class Resources(list[str]):
+    """
+    Acts like a regular list except it returns True
+    for all membership tests when empty.
+    """
+
+    def __contains__(self, item: object) -> bool:
+        if len(self) == 0:
+            return True
+        return super().__contains__(item)
+
+
+class Locality(BaseModel):
+    region: str | None = Field(None)
+    zone: str | None = Field(None)
+    sub_zone: str | None = Field(None)
+
+    def __str__(self) -> str:
+        return f"{self.region}::{self.zone}::{self.sub_zone}"
+
+
+class SemanticVersion(BaseModel):
+    major_number: int = 0
+    minor_number: int = 0
+    patch: int = 0
+
+    def __str__(self) -> str:
+        return f"{self.major_number}.{self.minor_number}.{self.patch}"
+
+
+class BuildVersion(BaseModel):
+    version: SemanticVersion = SemanticVersion()
+    metadata: dict[str, Any] = {}
+
+
+class Extension(BaseModel):
+    name: str | None = None
+    category: str | None = None
+    version: BuildVersion | None = None
+    disabled: bool | None = None
+
+
+class Node(BaseModel):
+    id: str = Field("-", title="Hostname")
+    cluster: str = Field(
+        ...,
+        title="Envoy service-cluster",
+        description="The ``--service-cluster`` configured by the Envoy client",
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict, title="Key:value metadata")
+    # noinspection PyArgumentList
+    locality: Locality = Field(Locality(), title="Locality")
+    build_version: str | None = Field(
+        None,  # Optional in the v3 Envoy API
+        title="Envoy build/release version string",
+        description="Used to identify what version of Envoy the "
+        "client is running, and what config to provide in response",
+    )
+    user_agent_name: str = "envoy"
+    user_agent_version: str = ""
+    user_agent_build_version: BuildVersion = BuildVersion()
+    extensions: list[Extension] = []
+    client_features: list[str] = []
+
+    @property
+    def common(self) -> tuple[str, str | None, str, BuildVersion, Locality]:
+        """
+        Returns fields that are the same in adjacent proxies
+        ie. proxies that are part of the same logical group
+        """
+        return (
+            self.cluster,
+            self.build_version,
+            self.user_agent_version,
+            self.user_agent_build_version,
+            self.locality,
+        )
+
+
+class Status(BaseModel):
+    code: int
+    message: str
+    details: list[Any]
+
+
+class XdsTemplate(BaseModel):
+    path: str | Loadable
+    resource_type: str
+    depends_on: list[str] = Field(default_factory=list)
+
+    @property
+    def loadable(self):
+        if isinstance(self.path, str):
+            return Loadable.from_legacy_fmt(self.path)
+        elif isinstance(self.path, Loadable):
+            return self.path
+        raise TypeError(
+            "Template path must be a loadable format. "
+            "e.g. file+yaml:///etc/templates/clusters.yaml"
+        )
+
+    @property
+    def is_python_source(self):
+        return self.loadable.protocol == "python"
+
+    @property
+    def code(self) -> ModuleType | Template:
+        return self.loadable.load()
+
+    def generate(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str | None:
+        if isinstance(self.code, ModuleType):
+            try:
+                template_fn = self.code.call  # type: ignore
+                return {"resources": list(template_fn(*args, **kwargs))}
+            except TypeError as e:
+                if not set(str(e).split()).issuperset(missing_arguments):
+                    raise ValueError(
+                        f"Tried to render template '{self.resource_type}'. "
+                        f"Error calling function: {str(e)}"
+                    )
+                message_start = str(e).find(":")
+                missing_args = str(e)[message_start + 2 :]
+                supplied_args = list(kwargs.keys())
+                raise TypeError(
+                    f"Tried to render template '{self.resource_type}' using partial arguments. "
+                    f"Missing args: {missing_args}. Supplied args: {args} "
+                    f"Supplied keyword args: {supplied_args}. "
+                    f"Add to `depends_on` to ensure required context is provided."
+                )
+        else:
+            return self.code.render(*args, **kwargs)
+
+    @property
+    def source(self) -> str:
+        old_serialization = self.loadable.serialization
+        if self.loadable.serialization in ("jinja", "jinja2"):
+            # The Jinja2 template serializer does not properly set a name
+            # for the loaded template.
+            # The repr for the template prints out as the memory address
+            # This makes it really hard to generate a consistent version_info string
+            # in rendered configuration.
+            # For this reason, we re-load the template as a string instead, and create a checksum.
+            self.loadable.serialization = "string"
+            ret = self.loadable.load()
+            self.loadable.serialization = old_serialization
+            return str(ret)
+        elif self.is_python_source:
+            # If the template specified is a python source file,
+            # we can simply read and return the source of it.
+            old_protocol = self.loadable.protocol
+            self.loadable.protocol = "inline"
+            self.loadable.serialization = "string"
+            ret = self.loadable.load()
+            self.loadable.protocol = old_protocol
+            self.loadable.serialization = old_serialization
+            return str(ret)
+        ret = self.loadable.load()
+        return str(ret)
+
+    def __repr__(self) -> str:
+        return f"XdsTemplate({self.loadable}, {hash(self)})"
+
+    @computed_field  # type: ignore[prop-decorator]
+    @cached_property
+    def version(self) -> str:
+        return compute_hash(self.source)
+
+    def __hash__(self) -> int:
+        return int(self.version)
+
+    __str__ = __repr__
+
+
+class DiscoveryRequest(BaseModel):
+    # Actual envoy fields
+    node: Node = Field(..., title="Node information about the envoy proxy")
+    version_info: str = Field(
+        "0", title="The version of the envoy clients current configuration"
+    )
+    resource_names: list[str] = Field(
+        default_factory=list, title="list of requested resource names"
+    )
+    error_detail: Status | None = Field(
+        None, title="Error details from the previous xDS request"
+    )
+    # Internal fields for sovereign
+    is_internal_request: bool = False
+    type_url: str | None = Field(
+        None, title="The corresponding type_url for the requested resource"
+    )
+    resource_type: str | None = Field(None, title="Resource type requested")
+    api_version: str | None = Field(None, title="Envoy API version (v2/v3/etc)")
+    desired_controlplane: str | None = Field(
+        None, title="The host header provided in the Discovery Request"
+    )
+    # Pydantic
+    model_config = ConfigDict(extra="ignore")
+
+    @computed_field  # type: ignore[prop-decorator]
+    @cached_property
+    def envoy_version(self) -> str:
+        try:
+            version = str(self.node.user_agent_build_version.version)
+            assert version != "0.0.0"
+        except AssertionError:
+            try:
+                build_version = self.node.build_version
+                if build_version is None:
+                    return "default"
+                _, version, *_ = build_version.split("/")
+            except (AttributeError, ValueError):
+                # TODO: log/metric this?
+                return "default"
+        return version
+
+    @property
+    def resources(self) -> Resources:
+        return Resources(self.resource_names)
+
+    # noinspection PyShadowingBuiltins
+    def cache_key(self, rules: list[str]) -> str:
+        map = self.model_dump()
+        hash = hashlib.sha256()
+        for expr in sorted(rules):
+            value = cast(str, jmespath.search(expr, map))
+            val_str = f"{expr}={repr(value)}"
+            hash.update(val_str.encode())
+        return hash.hexdigest()
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def template(self) -> XdsTemplate:
+        # lazy load configured templates
+        mod = importlib.import_module("sovereign.configuration")
+        templates = mod.XDS_TEMPLATES
+
+        version = self.envoy_version
+        selection = "default"
+        for v in templates.keys():
+            if version.startswith(v):
+                selection = v
+        selected_version = templates[selection]
+        try:
+            assert self.resource_type
+            return selected_version[self.resource_type]
+        except AssertionError:
+            raise RuntimeError(
+                "DiscoveryRequest has no resource type set, cannot find template"
+            )
+        except KeyError:
+            raise KeyError(
+                (
+                    f"Unable to get {self.resource_type} for template "
+                    f'version "{selection}". Envoy client version: {version}'
+                )
+            )
+
+    def debug(self):
+        return f"version={self.envoy_version}, cluster={self.node.cluster}, resource={self.resource_type}, names={self.resources}"
+
+    def __str__(self) -> str:
+        return f"DiscoveryRequest({self.debug()})"
+
+
+class DiscoveryResponse(BaseModel):
+    version_info: str = Field(
+        ..., title="The version of the configuration in the response"
+    )
+    resources: list[Any] = Field(..., title="The requested configuration resources")
+
+
+class ProcessedTemplate(BaseModel):
+    resources: list[dict[str, Any]]
+    metadata: list[str] = Field(default_factory=list, exclude=True)
+
+    @computed_field  # type: ignore[prop-decorator]
+    @cached_property
+    def version_info(self) -> str:
+        return compute_hash(self.resources)
+
+
+class RegisterClientRequest(BaseModel):
+    request: DiscoveryRequest

sovereign/utils/auth.py

@@ -1,18 +1,21 @@
-from fastapi.exceptions import HTTPException
 from cryptography.fernet import InvalidToken
-from sovereign import config, stats, cipher_suite
-from sovereign.schemas import DiscoveryRequest
+from fastapi.exceptions import HTTPException
+
+from sovereign import application_logger as log
+from sovereign import server_cipher_container, stats
+from sovereign.configuration import config
+from sovereign.types import DiscoveryRequest
 
 AUTH_ENABLED = config.authentication.enabled
 
 
+@stats.timed("discovery.auth.ms")
 def validate_authentication_string(s: str) -> bool:
     try:
-        password = cipher_suite.decrypt(s)
+        password = server_cipher_container.decrypt(s)
     except Exception:
         stats.increment("discovery.auth.failed")
         raise
-
     if password in config.passwords:
         stats.increment("discovery.auth.success")
         return True
@@ -23,28 +26,39 @@ def validate_authentication_string(s: str) -> bool:
 def authenticate(request: DiscoveryRequest) -> None:
     if not AUTH_ENABLED:
         return
-    if not cipher_suite.key_available:
+    if not server_cipher_container.key_available:
         raise RuntimeError(
-            "No Fernet key loaded, and auth is enabled. "
-            "A fernet key must be provided via SOVEREIGN_ENCRYPTION_KEY. "
-            "See https://vsyrakis.bitbucket.io/sovereign/docs/html/guides/encryption.html "
-            "for more details"
+            "No encryption key loaded, and auth is enabled. "
+            "An encryption key must be provided via SOVEREIGN_ENCRYPTION_KEY. "
         )
     try:
         encrypted_auth = request.node.metadata["auth"]
-        with stats.timed("discovery.auth.ms"):
-            assert validate_authentication_string(encrypted_auth)
     except KeyError:
         raise HTTPException(
             status_code=401,
             detail=f"Discovery request from {request.node.id} is missing auth field",
         )
+    except Exception as e:
+        description = getattr(e, "detail", "unknown")
+        raise HTTPException(
+            status_code=400,
+            detail=f"The authentication provided was malformed [Reason: {description}]",
+        )
+
+    try:
+        assert isinstance(encrypted_auth, str)
+        assert validate_authentication_string(encrypted_auth)
     except (InvalidToken, AssertionError):
         raise HTTPException(
             status_code=401, detail="The authentication provided was invalid"
         )
     except Exception as e:
-        description = getattr(e, "detail", "Unknown")
+        alt_desc = repr(e)
+        alt_desc = alt_desc.replace(encrypted_auth, "********")
+        for password in config.passwords:
+            alt_desc = alt_desc.replace(password, "********")
+        description = getattr(e, "detail", alt_desc)
+        log.exception(f"Failed to auth client: {description}")
         raise HTTPException(
             status_code=400,
             detail=f"The authentication provided was malformed [Reason: {description}]",

sovereign/utils/crypto/crypto.py

@@ -0,0 +1,135 @@
+from typing import Literal, Self, Sequence
+
+from cachetools import TTLCache, cached
+from fastapi.exceptions import HTTPException
+from structlog.stdlib import BoundLogger
+from typing_extensions import TypedDict
+
+from sovereign.configuration import EncryptionConfig
+from sovereign.utils.crypto.suites import (
+    AESGCMCipher,
+    CipherSuite,
+    DisabledCipher,
+    EncryptionType,
+    FernetCipher,
+)
+
+
+class EncryptOutput(TypedDict):
+    encrypted_data: str
+    encryption_type: str
+
+
+class DecryptOutput(TypedDict):
+    decrypted_data: str
+    encryption_type: str
+
+
+class CipherContainer:
+    """
+    Object which intercepts encrypt/decryptions
+    Tries to decrypt data using the ciphers provided in order
+    Encrypts with the first suite available.
+    """
+
+    def __init__(self, suites: Sequence[CipherSuite], logger: BoundLogger) -> None:
+        self.suites: Sequence[CipherSuite] = suites
+        self.logger = logger
+
+    def encrypt(self, data: str) -> EncryptOutput:
+        try:
+            # Use the first cipher suite to encrypt the data
+            encrypted_data = self.suites[0].encrypt(data)
+            return {
+                "encrypted_data": encrypted_data,
+                "encryption_type": str(self.suites[0]),
+            }
+        # pylint: disable=broad-except
+        except Exception as e:
+            self.logger.exception(str(e))
+            # TODO: defer this http error to later, return a normal error here
+            raise HTTPException(status_code=400, detail="Encryption failed")
+
+    def decrypt_with_type(self, data: str) -> DecryptOutput:
+        return self._decrypt(data)
+
+    def decrypt(self, data: str) -> str:
+        return self._decrypt(data)["decrypted_data"]
+
+    @cached(cache=TTLCache(maxsize=128, ttl=600))
+    def _decrypt(self, data: str) -> DecryptOutput:
+        try:
+            for suite in self.suites:
+                try:
+                    decrypted_data = suite.decrypt(data)
+                    return {
+                        "decrypted_data": decrypted_data,
+                        "encryption_type": str(suite),
+                    }
+                # pylint: disable=broad-except
+                except Exception as e:
+                    self.logger.exception(str(e))
+                    self.logger.debug(f"Failed to decrypt with suite {suite}")
+            else:
+                raise ValueError("Could not decrypt with any suite")
+        except Exception as e:
+            self.logger.exception(str(e))
+            # TODO: defer this http error to later, return a normal error here
+            raise HTTPException(status_code=400, detail="Decryption failed")
+
+    @property
+    def key_available(self) -> bool:
+        if not self.suites:
+            return False
+        return self.suites[0].key_available
+
+    AVAILABLE_CIPHERS: dict[EncryptionType | Literal["default"], type[CipherSuite]] = {
+        EncryptionType.DISABLED: DisabledCipher,
+        EncryptionType.AESGCM: AESGCMCipher,
+        EncryptionType.FERNET: FernetCipher,
+        "default": FernetCipher,
+    }
+
+    @classmethod
+    def get_cipher_suite(cls, encryption_type: EncryptionType) -> type[CipherSuite]:
+        SelectedCipher = cls.AVAILABLE_CIPHERS.get(
+            encryption_type, cls.AVAILABLE_CIPHERS["default"]
+        )
+        return SelectedCipher
+
+    @classmethod
+    def create_cipher_suite(
+        cls,
+        encryption_type: EncryptionType,
+        key: str,
+        logger: BoundLogger,
+    ) -> CipherSuite:
+        kwargs = {
+            "secret_key": key,
+        }
+        try:
+            SelectedCipher = cls.get_cipher_suite(encryption_type)
+            return SelectedCipher(**kwargs)
+        except TypeError:
+            pass
+        except ValueError as e:
+            if key not in (b"", ""):
+                logger.error(
+                    f"Encryption key was provided, but appears to be invalid: {repr(e)}"
+                )
+        return DisabledCipher(**kwargs)
+
+    @classmethod
+    def from_encryption_configs(
+        cls, encryption_configs: Sequence[EncryptionConfig], logger: BoundLogger
+    ) -> Self:
+        cipher_suites: list[CipherSuite] = []
+        for encryption_config in encryption_configs:
+            cipher_suites.append(
+                cls.create_cipher_suite(
+                    key=encryption_config.encryption_key,
+                    encryption_type=encryption_config.encryption_type,
+                    logger=logger,
+                )
+            )
+        return cls(suites=cipher_suites, logger=logger)

sovereign/utils/crypto/suites/__init__.py

@@ -0,0 +1,21 @@
+from enum import StrEnum
+
+from sovereign.utils.crypto.suites.aes_gcm_cipher import AESGCMCipher
+from sovereign.utils.crypto.suites.base_cipher import CipherSuite
+from sovereign.utils.crypto.suites.disabled_cipher import DisabledCipher
+from sovereign.utils.crypto.suites.fernet_cipher import FernetCipher
+
+
+class EncryptionType(StrEnum):
+    FERNET = "fernet"
+    AESGCM = "aesgcm"
+    DISABLED = "disabled"
+
+
+__all__ = [
+    "AESGCMCipher",
+    "CipherSuite",
+    "DisabledCipher",
+    "FernetCipher",
+    "EncryptionType",
+]

sovereign/utils/crypto/suites/aes_gcm_cipher.py

@@ -0,0 +1,42 @@
+import base64
+import os
+
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+from .base_cipher import CipherSuite
+
+
+class AESGCMCipher(CipherSuite):
+    AUTHENTICATE_DATA = "sovereign".encode()
+    NONCE_LEN = 12
+
+    def __str__(self) -> str:
+        return "aesgcm"
+
+    def __init__(self, secret_key: str):
+        self.secret_key = base64.urlsafe_b64decode(secret_key)
+
+    def encrypt(self, data: str) -> str:
+        aesgcm = AESGCM(self.secret_key)
+        nonce = os.urandom(self.NONCE_LEN)
+        ct = aesgcm.encrypt(nonce, data.encode(), self.AUTHENTICATE_DATA)
+        return base64.b64encode(nonce + ct).decode("utf-8")
+
+    def decrypt(self, data: str) -> str:
+        decoded_data = base64.b64decode(data)
+        nonce, ct = (
+            decoded_data[: self.NONCE_LEN],
+            decoded_data[self.NONCE_LEN :],
+        )
+        aesgcm = AESGCM(self.secret_key)
+        decrypted_output = aesgcm.decrypt(nonce, ct, self.AUTHENTICATE_DATA)
+        return decrypted_output.decode("utf-8")
+
+    @property
+    def key_available(self) -> bool:
+        return True
+
+    @classmethod
+    def generate_key(cls) -> bytes:
+        # Generate 256 bit length key
+        return base64.urlsafe_b64encode(os.urandom(32))

sovereign/utils/crypto/suites/base_cipher.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class CipherSuite(ABC):
+    @abstractmethod
+    def __init__(self, *args: Any, **kwargs: Any) -> None: ...
+
+    @abstractmethod
+    def encrypt(self, data: str) -> str: ...
+
+    @abstractmethod
+    def decrypt(self, data: str) -> str: ...
+
+    @property
+    @abstractmethod
+    def key_available(self) -> bool: ...
+
+    @classmethod
+    @abstractmethod
+    def generate_key(cls) -> bytes: ...

sovereign/utils/crypto/suites/disabled_cipher.py

@@ -0,0 +1,25 @@
+from typing import Any
+
+from .base_cipher import CipherSuite
+
+
+class DisabledCipher(CipherSuite):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        pass
+
+    def __str__(self) -> str:
+        return "disabled"
+
+    def encrypt(self, data: str) -> str:
+        return "Unavailable (No Secret Key)"
+
+    def decrypt(self, data: str) -> str:
+        return "Unavailable (No Secret Key)"
+
+    @property
+    def key_available(self) -> bool:
+        return False
+
+    @classmethod
+    def generate_key(cls) -> bytes:
+        return b"Unavailable (No key to generate)"

sovereign/utils/crypto/suites/fernet_cipher.py

@@ -0,0 +1,29 @@
+import base64
+import os
+
+from cryptography.fernet import Fernet
+
+from .base_cipher import CipherSuite
+
+
+class FernetCipher(CipherSuite):
+    def __str__(self) -> str:
+        return "fernet"
+
+    def __init__(self, secret_key: str):
+        self.fernet = Fernet(secret_key)
+
+    def encrypt(self, data: str) -> str:
+        return self.fernet.encrypt(data.encode()).decode("utf-8")
+
+    def decrypt(self, data: str) -> str:
+        return self.fernet.decrypt(data).decode("utf-8")
+
+    @property
+    def key_available(self) -> bool:
+        return True
+
+    @classmethod
+    def generate_key(cls) -> bytes:
+        # Generate 256 bit length key
+        return base64.urlsafe_b64encode(os.urandom(32))

sovereign/utils/dictupdate.py

@@ -1,8 +1,9 @@
 # type: ignore
 # pylint: disable=no-name-in-module,too-many-branches
-""" Stolen from the saltstack library """
-from collections.abc import Mapping
+"""Stolen from the saltstack library"""
+
 import copy
+from collections.abc import Mapping
 
 
 def update(dest, upd, recursive_update=True, merge_lists=False):