sovereign 0.19.3__py3-none-any.whl → 1.0.0b148__py3-none-any.whl

sovereign/__init__.py

@@ -1,50 +1,11 @@
-import os
 from contextvars import ContextVar
-from typing import Type, Any, Mapping
 from importlib.metadata import version
-from pkg_resources import resource_filename
 
-from fastapi.responses import JSONResponse
-from starlette.templating import Jinja2Templates
-from pydantic.error_wrappers import ValidationError
-
-from sovereign.schemas import (
-    SovereignAsgiConfig,
-    SovereignConfig,
-    SovereignConfigv2,
-)
-from sovereign import config_loader
-from sovereign.logs import LoggerBootstrapper
+from sovereign.utils.crypto.suites import EncryptionType
+from sovereign.logging.bootstrapper import LoggerBootstrapper
+from sovereign.configuration import config, EncryptionConfig
 from sovereign.statistics import configure_statsd
-from sovereign.utils.dictupdate import merge  # type: ignore
-from sovereign.sources import SourcePoller
-from sovereign.context import TemplateContext
-from sovereign.utils.crypto import CipherContainer, create_cipher_suite
-
-
-json_response_class: Type[JSONResponse] = JSONResponse
-try:
-    import orjson
-    from fastapi.responses import ORJSONResponse
-
-    json_response_class = ORJSONResponse
-except ImportError:
-    try:
-        import ujson
-        from fastapi.responses import UJSONResponse
-
-        json_response_class = UJSONResponse
-    except ImportError:
-        pass
-
-
-def parse_raw_configuration(path: str) -> Mapping[Any, Any]:
-    ret: Mapping[Any, Any] = dict()
-    for p in path.split(","):
-        spec = config_loader.Loadable.from_legacy_fmt(p)
-        ret = merge(obj_a=ret, obj_b=spec.load(), merge_lists=True)
-    return ret
-
+from sovereign.utils.crypto.crypto import CipherContainer
 
 _request_id_ctx_var: ContextVar[str] = ContextVar("request_id", default="")
 
@@ -53,48 +14,19 @@ def get_request_id() -> str:
     return _request_id_ctx_var.get()
 
 
+WORKER_URL = "http://localhost:9080"
 DIST_NAME = "sovereign"
-
 __version__ = version(DIST_NAME)
-config_path = os.getenv("SOVEREIGN_CONFIG", "file:///etc/sovereign.yaml")
-
-html_templates = Jinja2Templates(resource_filename(DIST_NAME, "templates"))
-
-try:
-    config = SovereignConfigv2(**parse_raw_configuration(config_path))
-except ValidationError:
-    old_config = SovereignConfig(**parse_raw_configuration(config_path))
-    config = SovereignConfigv2.from_legacy_config(old_config)
-asgi_config = SovereignAsgiConfig()
-XDS_TEMPLATES = config.xds_templates()
 
+stats = configure_statsd()
 logs = LoggerBootstrapper(config)
-stats = configure_statsd(config=config.statsd)
-poller = SourcePoller(
-    sources=config.sources,
-    matching_enabled=config.matching.enabled,
-    node_match_key=config.matching.node_key,
-    source_match_key=config.matching.source_key,
-    source_refresh_rate=config.source_config.refresh_rate,
-    logger=logs.application_log,
-    stats=stats,
-)
+application_logger = logs.application_logger.logger
 
-fernet_keys = config.authentication.encryption_key
-encryption_keys = fernet_keys.get_secret_value().encode().split()
-cipher_suite = CipherContainer(
-    [create_cipher_suite(key=key, logger=logs) for key in encryption_keys]
+encryption_configs = config.authentication.encryption_configs
+server_cipher_container = CipherContainer.from_encryption_configs(
+    encryption_configs, logger=application_logger
 )
-
-template_context = TemplateContext(
-    refresh_rate=config.template_context.refresh_rate,
-    refresh_cron=config.template_context.refresh_cron,
-    configured_context=config.template_context.context,
-    poller=poller,
-    encryption_suite=cipher_suite,
-    disabled_suite=create_cipher_suite(b"", logs),
-    logger=logs.application_log,
-    stats=stats,
+disabled_ciphersuite = CipherContainer.from_encryption_configs(
+    encryption_configs=[EncryptionConfig("", EncryptionType.DISABLED)],
+    logger=application_logger,
 )
-poller.lazy_load_modifiers(config.modifiers)
-poller.lazy_load_global_modifiers(config.global_modifiers)

sovereign/app.py

@@ -1,38 +1,22 @@
-import asyncio
 import traceback
-import uvicorn
 from collections import namedtuple
+
+import uvicorn
 from fastapi import FastAPI, Request
-from fastapi.responses import RedirectResponse, FileResponse, Response, JSONResponse
-from pkg_resources import resource_filename
-from sovereign import (
-    __version__,
-    config,
-    asgi_config,
-    json_response_class,
-    poller,
-    template_context,
-    logs,
-)
+from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, Response
+from starlette_context.middleware import RawContextMiddleware
+
+from sovereign import __version__, logs
+from sovereign.configuration import config, ConfiguredResourceTypes
+from sovereign.response_class import json_response_class
 from sovereign.error_info import ErrorInfo
-from sovereign.views import crypto, discovery, healthchecks, admin, interface
-from sovereign.middlewares import (
-    RequestContextLogMiddleware,
-    LoggingMiddleware,
-)
+from sovereign.middlewares import LoggingMiddleware, RequestContextLogMiddleware
+from sovereign.utils.resources import get_package_file
+from sovereign.views import crypto, discovery, healthchecks, interface, api
 
 Router = namedtuple("Router", "module tags prefix")
 
 DEBUG = config.debug
-SENTRY_DSN = config.sentry_dsn.get_secret_value()
-
-try:
-    import sentry_sdk
-    from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
-
-    SENTRY_INSTALLED = True
-except ImportError:  # pragma: no cover
-    SENTRY_INSTALLED = False
 
 
 def generic_error_response(e: Exception) -> JSONResponse:
@@ -46,7 +30,7 @@ def generic_error_response(e: Exception) -> JSONResponse:
     """
     tb = [line for line in traceback.format_exc().split("\n")]
     info = ErrorInfo.from_exception(e)
-    logs.queue_log_fields(
+    logs.access_logger.queue_log_fields(
         ERROR=info.error,
         ERROR_DETAIL=info.detail,
         TRACEBACK=tb,
@@ -69,23 +53,32 @@ def init_app() -> FastAPI:
 
     routers = (
         Router(discovery.router, ["Configuration Discovery"], ""),
-        Router(crypto.router, ["Cryptographic Utilities"], "/crypto"),
-        Router(admin.router, ["Debugging Endpoints"], "/admin"),
-        Router(interface.router, ["User Interface"], "/ui"),
+        Router(api.router, ["API"], "/api"),
         Router(healthchecks.router, ["Healthchecks"], ""),
+        Router(interface.router, ["User Interface"], "/ui"),
+        Router(crypto.router, ["Cryptographic Utilities"], "/crypto"),
     )
     for router in routers:
         application.include_router(
             router.module, tags=router.tags, prefix=router.prefix
         )
 
-    application.add_middleware(RequestContextLogMiddleware)
-    application.add_middleware(LoggingMiddleware)
+    application.add_middleware(RequestContextLogMiddleware)  # type: ignore
+    application.add_middleware(LoggingMiddleware)  # type: ignore
+
+    if dsn := config.sentry_dsn.get_secret_value():
+        try:
+            import sentry_sdk
+            from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
 
-    if SENTRY_INSTALLED and SENTRY_DSN:
-        sentry_sdk.init(SENTRY_DSN)
-        application.add_middleware(SentryAsgiMiddleware)
-        logs.application_log(event="Sentry middleware enabled")
+            sentry_sdk.init(dsn)
+            application.add_middleware(SentryAsgiMiddleware)  # type: ignore
+        except ImportError:
+            logs.application_logger.logger.error(
+                "Sentry DSN configured but failed to attach to webserver"
+            )
+
+    application.add_middleware(RawContextMiddleware)  # type: ignore
 
     @application.exception_handler(500)
     async def exception_handler(_: Request, exc: Exception) -> JSONResponse:
@@ -103,23 +96,41 @@ def init_app() -> FastAPI:
 
     @application.get("/static/{filename}", summary="Return a static asset")
     async def static(filename: str) -> Response:
-        return FileResponse(resource_filename("sovereign", f"static/{filename}"))
-
-    @application.on_event("startup")
-    async def keep_sources_uptodate() -> None:
-        asyncio.create_task(poller.poll_forever())
+        return FileResponse(get_package_file("sovereign_files", f"static/{filename}"))  # type: ignore[arg-type]
 
-    @application.on_event("startup")
-    async def refresh_template_context() -> None:
-        asyncio.create_task(template_context.start_refresh_context())
+    @application.get(
+        "/admin/xds_dump",
+        summary="Deprecated API, please use /api/resources/{resource_type}",
+    )
+    async def dump_resources(request: Request) -> Response:
+        resource_type = ConfiguredResourceTypes(
+            request.query_params.get("xds_type", "cluster")
+        )
+        resource_name = request.query_params.get("name")
+        api_version = request.query_params.get("api_version", "v3")
+        service_cluster = request.query_params.get("service_cluster", "*")
+        region = request.query_params.get("region")
+        version = request.query_params.get("version")
+        response = await api.resource(
+            resource_type=resource_type,
+            resource_name=resource_name,
+            api_version=api_version,
+            service_cluster=service_cluster,
+            region=region,
+            version=version,
+        )
+        response.headers["Deprecation"] = "true"
+        response.headers["Link"] = f'</api/resources/{resource_type}>; rel="alternate"'
+        response.headers["Warning"] = (
+            f'299 - "Deprecated API: please use /api/resources/{resource_type}"'
+        )
+        response.status_code = 299
+        return response
 
     return application
 
 
 app = init_app()
-logs.application_log(
-    event=f"Sovereign started and listening on {asgi_config.host}:{asgi_config.port}"
-)
 
 
 if __name__ == "__main__":  # pragma: no cover

sovereign/cache/__init__.py

@@ -0,0 +1,172 @@
+"""
+Sovereign Cache Module
+
+This module provides an extensible cache backend system that allows clients
+to configure their own remote cache backends through entry points.
+"""
+
+import asyncio
+from typing import Any
+from typing_extensions import final
+
+import requests
+
+from sovereign import WORKER_URL, stats, application_logger as log
+from sovereign.types import DiscoveryRequest, RegisterClientRequest
+from sovereign.configuration import config
+from sovereign.cache.types import Entry, CacheResult
+from sovereign.cache.backends import CacheBackend, get_backend
+from sovereign.cache.filesystem import FilesystemCache
+
+
+CACHE_READ_TIMEOUT = config.cache.read_timeout
+
+
+class CacheManagerBase:
+    def __init__(self) -> None:
+        self.local: FilesystemCache = FilesystemCache()
+        self.remote: CacheBackend | None = get_backend()
+        if self.remote is None:
+            log.info("Cache initialized with filesystem backend only")
+        else:
+            log.info("Cache initialized with filesystem and remote backends")
+
+    # Client Id registration
+
+    def register(self, req: DiscoveryRequest):
+        id = client_id(req)
+        log.debug(f"Registering client {id}")
+        self.local.register(id, req)
+        stats.increment("client.registration", tags=["status:registered"])
+        return id, req
+
+    def registered(self, req: DiscoveryRequest) -> bool:
+        ret = False
+        id = client_id(req)
+        if value := self.local.registered(id):
+            ret = value
+        log.debug(f"Client {id} registered={ret}")
+        return ret
+
+    def get_registered_clients(self) -> list[tuple[str, DiscoveryRequest]]:
+        if value := self.local.get_registered_clients():
+            return value
+        return []
+
+
+@final
+class CacheReader(CacheManagerBase):
+    def try_read(self, key: str) -> CacheResult | None:
+        # Try filesystem first
+        if value := self.local.get(key):
+            stats.increment("cache.fs.hit")
+            return CacheResult(value=value, from_remote=False)
+        stats.increment("cache.fs.miss")
+
+        # Fallback to remote cache if available
+        if self.remote:
+            try:
+                if value := self.remote.get(key):
+                    ret = CacheResult(value=value, from_remote=True)
+                    stats.increment("cache.remote.hit")
+                    return ret
+            except Exception as e:
+                log.warning(f"Failed to read from remote cache: {e}")
+                stats.increment("cache.remote.error")
+            stats.increment("cache.remote.miss")
+            log.warning(f"Failed to read from either cache for {key}")
+        return None
+
+    def get(self, req: DiscoveryRequest) -> Entry | None:
+        id = client_id(req)
+        if result := self.try_read(id):
+            if result.from_remote:
+                self.register(req)
+                # Write back to filesystem
+                self.local.set(id, result.value)
+            return result.value
+        return None
+
+    @stats.timed("cache.read_ms")
+    async def blocking_read(
+        self, req: DiscoveryRequest, timeout_s=CACHE_READ_TIMEOUT, poll_interval_s=0.5
+    ) -> Entry | None:
+        cid = client_id(req)
+        metric = "client.registration"
+        if entry := self.get(req):
+            return entry
+
+        log.info(f"Cache entry not found for {cid}, registering and waiting")
+        registered = False
+        start = asyncio.get_event_loop().time()
+        attempt = 1
+        while (asyncio.get_event_loop().time() - start) < timeout_s:
+            if not registered:
+                try:
+                    if self.register_over_http(req):
+                        stats.increment(metric, tags=["status:registered"])
+                        registered = True
+                        log.info(f"Client {cid} registered")
+                    else:
+                        stats.increment(metric, tags=["status:ratelimited"])
+                        await asyncio.sleep(min(attempt, CACHE_READ_TIMEOUT))
+                        attempt *= 2
+                except Exception as e:
+                    stats.increment(metric, tags=["status:failed"])
+                    log.exception(f"Tried to register client but failed: {e}")
+            if entry := self.get(req):
+                log.info(f"Entry has been populated for {cid}")
+                return entry
+            await asyncio.sleep(poll_interval_s)
+
+        return None
+
+    def register_over_http(self, req: DiscoveryRequest) -> bool:
+        registration = RegisterClientRequest(request=req)
+        log.debug(f"Sending registration to worker for {req}")
+        try:
+            response = requests.put(
+                f"{WORKER_URL}/client",
+                json=registration.model_dump(),
+                timeout=3,
+            )
+            match response.status_code:
+                case 200 | 202:
+                    log.debug("Worker responded OK to registration")
+                    return True
+                case code:
+                    log.debug(f"Worker responded with {code} to registration")
+        except Exception as e:
+            log.exception(f"Error while registering client: {e}")
+        return False
+
+
+@final
+class CacheWriter(CacheManagerBase):
+    def set(
+        self, key: str, value: Entry, timeout: int | None = None
+    ) -> tuple[bool, list[tuple[str, str]]]:
+        msg = []
+        cached = False
+        try:
+            self.local.set(key, value, timeout)
+            stats.increment("cache.fs.write.success")
+            cached = True
+        except Exception as e:
+            log.warning(f"Failed to write to filesystem cache: {e}")
+            msg.append(("warning", f"Failed to write to filesystem cache: {e}"))
+            stats.increment("cache.fs.write.error")
+        if self.remote:
+            try:
+                self.remote.set(key, value, timeout)
+                stats.increment("cache.remote.write.success")
+                cached = True
+            except Exception as e:
+                log.warning(f"Failed to write to remote cache: {e}")
+                msg.append(("warning", f"Failed to write to remote cache: {e}"))
+                stats.increment("cache.remote.write.error")
+        return cached, msg
+
+
+def client_id(req: DiscoveryRequest) -> str:
+    return req.cache_key(config.cache.hash_rules)

sovereign/cache/backends/__init__.py

@@ -0,0 +1,110 @@
+"""
+Cache backends module
+
+This module provides the protocol definition for cache backends and
+the loading mechanism for extensible cache backends via entry points.
+"""
+
+from collections.abc import Sequence
+from importlib.metadata import EntryPoints
+from typing import Protocol, Any, runtime_checkable
+
+from sovereign import application_logger as log
+from sovereign.utils.entry_point_loader import EntryPointLoader
+
+
+@runtime_checkable
+class CacheBackend(Protocol):
+    def __init__(self, config: dict[str, Any]) -> None:
+        """Initialize the cache backend with generic configuration
+
+        Args:
+            config: Dictionary containing backend-specific configuration
+        """
+        ...
+
+    def get(self, key: str) -> Any | None:
+        """Get a value from the cache
+
+        Args:
+            key: The cache key
+
+        Returns:
+            The cached value or None if not found
+        """
+        ...
+
+    def set(self, key: str, value: Any, timeout: int | None = None) -> None:
+        """Set a value in the cache
+
+        Args:
+            key: The cache key
+            value: The value to cache
+            timeout: Optional timeout in seconds
+        """
+        ...
+
+
+def get_backend() -> CacheBackend | None:
+    from sovereign import config
+
+    cache_config = config.cache.remote_backend
+    if not cache_config:
+        log.info("No remote cache backend configured, using filesystem only")
+        return None
+
+    backend_type = cache_config.type
+
+    loader = EntryPointLoader("cache.backends")
+    entry_points: EntryPoints | Sequence[Any] = loader.groups.get("cache.backends", [])
+
+    backend = None
+    for ep in entry_points:
+        if ep.name == backend_type:
+            backend = ep.load()
+            break
+
+    if not backend:
+        raise KeyError(
+            (
+                f"Cache backend '{backend_type}' not found. "
+                f"Available backends: {[ep.name for ep in entry_points]}"
+            )
+        )
+
+    backend_config = _process_loadable_config(cache_config.config)
+    instance = backend(backend_config)
+
+    if not isinstance(instance, CacheBackend):
+        raise TypeError(
+            (f"Cache backend '{backend_type}' does not implement CacheBackend protocol")
+        )
+
+    log.info(f"Successfully initialized cache backend: {backend_type}")
+    return instance
+
+
+def _process_loadable_config(config: dict[str, Any]) -> dict[str, Any]:
+    from sovereign.dynamic_config import Loadable
+
+    processed = {}
+    for key, value in config.items():
+        try:
+            if isinstance(value, str):
+                loadable = Loadable.from_legacy_fmt(value)
+                processed[key] = loadable.load()
+            elif isinstance(value, dict):
+                loadable = Loadable(**value)
+                processed[key] = loadable.load()
+            else:
+                processed[key] = value
+            continue
+        except Exception as e:
+            log.warning(f"Failed to load value for {key}: {e}")
+
+        if isinstance(value, dict):
+            processed[key] = _process_loadable_config(value)
+        else:
+            processed[key] = value
+
+    return processed

sovereign/cache/backends/s3.py

@@ -0,0 +1,143 @@
+import pickle
+from datetime import datetime, timezone, timedelta
+from typing import Any
+from typing_extensions import override
+from urllib.parse import quote
+from importlib.util import find_spec
+
+from sovereign import application_logger as log
+from sovereign.cache.backends import CacheBackend
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError
+except ImportError:
+    pass
+
+BOTO_AVAILABLE = find_spec("boto3") is not None
+
+
+class S3Client:
+    def __init__(self, role_arn: str | None, client_args: dict[str, Any]):
+        self.role_arn = role_arn
+        self.client_args = client_args
+        self._client = None
+        self._credentials_expiry = None
+        self._base_session = boto3.Session()
+        self._make_client()
+
+    def _make_client(self) -> None:
+        if self.role_arn:
+            log.debug(f"Refreshing credentials for role: {self.role_arn}")
+            sts = self._base_session.client("sts")
+            duration_seconds = 3600  # 4 hours
+            response = sts.assume_role(
+                RoleArn=self.role_arn,
+                RoleSessionName="sovereign-s3-cache",
+                DurationSeconds=duration_seconds,
+            )
+            credentials = response["Credentials"]
+            session = boto3.Session(
+                aws_access_key_id=credentials["AccessKeyId"],
+                aws_secret_access_key=credentials["SecretAccessKey"],
+                aws_session_token=credentials["SessionToken"],
+            )
+            self._credentials_expiry = credentials["Expiration"]
+        else:
+            session = self._base_session
+            self._credentials_expiry = None
+        self._client = session.client("s3", **self.client_args)
+
+    def _session_expiring_soon(self) -> bool:
+        if not self.role_arn or self._credentials_expiry is None:
+            return False
+        refresh_threshold = timedelta(minutes=30).seconds
+        time_until_expiry = (
+            self._credentials_expiry - datetime.now(timezone.utc)
+        ).total_seconds()
+        return time_until_expiry <= refresh_threshold
+
+    def __getattr__(self, name):
+        if self._session_expiring_soon():
+            self._make_client()
+        return getattr(self._client, name)
+
+
+class S3Backend(CacheBackend):
+    """S3 cache backend implementation"""
+
+    @override
+    def __init__(self, config: dict[str, Any]) -> None:  # pyright: ignore[reportMissingSuperCall]
+        """Initialize S3 backend
+
+        Args:
+            config: Configuration dictionary containing S3 connection parameters
+                   Expected keys: bucket_name, prefix
+                   Optional keys: assume_role, endpoint_url
+        """
+        if not BOTO_AVAILABLE:
+            raise ImportError("boto3 not installed")
+
+        self.bucket_name = config.get("bucket_name")
+        if not self.bucket_name:
+            raise ValueError("bucket_name is required for S3 cache backend")
+
+        self.prefix = config.get("prefix", "sovereign-cache")
+        self.registration_prefix = config.get("registration_prefix", "registrations-")
+        self.role = config.get("assume_role")
+
+        client_args: dict[str, Any] = {}
+        if endpoint_url := config.get("endpoint_url"):
+            client_args["endpoint_url"] = endpoint_url
+
+        self.client_args = client_args
+        self.s3 = S3Client(self.role, self.client_args)
+
+        try:
+            self.s3.head_bucket(Bucket=self.bucket_name)
+            log.info(f"S3 cache backend connected to bucket '{self.bucket_name}'")
+        except Exception as e:
+            log.error(
+                f"Failed to access S3 bucket '{self.bucket_name}' with current credentials: {e}"
+            )
+            raise
+
+    def _make_key(self, key: str) -> str:
+        encoded_key = quote(key, safe="")
+        return f"{self.prefix}/{encoded_key}"
+
+    def get(self, key: str) -> Any | None:
+        try:
+            log.debug(f"Retrieving object {key} from bucket")
+            response = self.s3.get_object(
+                Bucket=self.bucket_name, Key=self._make_key(key)
+            )
+            data = response["Body"].read()
+            unpickled = pickle.loads(data)
+            log.debug(f"Successfully obtained object {key} from bucket")
+            return unpickled
+        except self.s3.exceptions.NoSuchKey:
+            log.debug(f"{key} not in bucket")
+            return None
+        except ClientError as e:
+            if e.response["Error"]["Code"] == "404":
+                log.debug(f"{key} not in bucket")
+                return None
+            log.warning(f"Failed to get key '{key}' from S3: {e}")
+            return None
+        except Exception as e:
+            log.warning(f"Failed to get key '{key}' from S3: {e}")
+            return None
+
+    @override
+    def set(self, key: str, value: Any, timeout: int | None = None) -> None:
+        try:
+            log.debug(f"Putting new object {key} into bucket")
+            self.s3.put_object(
+                Bucket=self.bucket_name,
+                Key=self._make_key(key),
+                Body=pickle.dumps(value),
+            )
+        except Exception as e:
+            log.warning(f"Failed to set key '{key}' in S3: {e}")
+            raise