sovereign 0.19.3__py3-none-any.whl → 1.0.0a4__py3-none-any.whl

sovereign/__init__.py

@@ -1,50 +1,11 @@
-import os
 from contextvars import ContextVar
-from typing import Type, Any, Mapping
 from importlib.metadata import version
-from pkg_resources import resource_filename
 
-from fastapi.responses import JSONResponse
-from starlette.templating import Jinja2Templates
-from pydantic.error_wrappers import ValidationError
-
-from sovereign.schemas import (
-    SovereignAsgiConfig,
-    SovereignConfig,
-    SovereignConfigv2,
-)
-from sovereign import config_loader
-from sovereign.logs import LoggerBootstrapper
+from sovereign.configuration import EncryptionConfig, config
+from sovereign.logging.bootstrapper import LoggerBootstrapper
 from sovereign.statistics import configure_statsd
-from sovereign.utils.dictupdate import merge  # type: ignore
-from sovereign.sources import SourcePoller
-from sovereign.context import TemplateContext
-from sovereign.utils.crypto import CipherContainer, create_cipher_suite
-
-
-json_response_class: Type[JSONResponse] = JSONResponse
-try:
-    import orjson
-    from fastapi.responses import ORJSONResponse
-
-    json_response_class = ORJSONResponse
-except ImportError:
-    try:
-        import ujson
-        from fastapi.responses import UJSONResponse
-
-        json_response_class = UJSONResponse
-    except ImportError:
-        pass
-
-
-def parse_raw_configuration(path: str) -> Mapping[Any, Any]:
-    ret: Mapping[Any, Any] = dict()
-    for p in path.split(","):
-        spec = config_loader.Loadable.from_legacy_fmt(p)
-        ret = merge(obj_a=ret, obj_b=spec.load(), merge_lists=True)
-    return ret
-
+from sovereign.utils.crypto.crypto import CipherContainer
+from sovereign.utils.crypto.suites import EncryptionType
 
 _request_id_ctx_var: ContextVar[str] = ContextVar("request_id", default="")
 
@@ -53,48 +14,19 @@ def get_request_id() -> str:
     return _request_id_ctx_var.get()
 
 
+WORKER_URL = "http://localhost:9080"
 DIST_NAME = "sovereign"
-
 __version__ = version(DIST_NAME)
-config_path = os.getenv("SOVEREIGN_CONFIG", "file:///etc/sovereign.yaml")
-
-html_templates = Jinja2Templates(resource_filename(DIST_NAME, "templates"))
-
-try:
-    config = SovereignConfigv2(**parse_raw_configuration(config_path))
-except ValidationError:
-    old_config = SovereignConfig(**parse_raw_configuration(config_path))
-    config = SovereignConfigv2.from_legacy_config(old_config)
-asgi_config = SovereignAsgiConfig()
-XDS_TEMPLATES = config.xds_templates()
 
+stats = configure_statsd()
 logs = LoggerBootstrapper(config)
-stats = configure_statsd(config=config.statsd)
-poller = SourcePoller(
-    sources=config.sources,
-    matching_enabled=config.matching.enabled,
-    node_match_key=config.matching.node_key,
-    source_match_key=config.matching.source_key,
-    source_refresh_rate=config.source_config.refresh_rate,
-    logger=logs.application_log,
-    stats=stats,
-)
+application_logger = logs.application_logger.logger
 
-fernet_keys = config.authentication.encryption_key
-encryption_keys = fernet_keys.get_secret_value().encode().split()
-cipher_suite = CipherContainer(
-    [create_cipher_suite(key=key, logger=logs) for key in encryption_keys]
+encryption_configs = config.authentication.encryption_configs
+server_cipher_container = CipherContainer.from_encryption_configs(
+    encryption_configs, logger=application_logger
 )
-
-template_context = TemplateContext(
-    refresh_rate=config.template_context.refresh_rate,
-    refresh_cron=config.template_context.refresh_cron,
-    configured_context=config.template_context.context,
-    poller=poller,
-    encryption_suite=cipher_suite,
-    disabled_suite=create_cipher_suite(b"", logs),
-    logger=logs.application_log,
-    stats=stats,
+disabled_ciphersuite = CipherContainer.from_encryption_configs(
+    encryption_configs=[EncryptionConfig("", EncryptionType.DISABLED)],
+    logger=application_logger,
 )
-poller.lazy_load_modifiers(config.modifiers)
-poller.lazy_load_global_modifiers(config.global_modifiers)

sovereign/app.py

@@ -1,38 +1,22 @@
-import asyncio
 import traceback
-import uvicorn
 from collections import namedtuple
+
+import uvicorn
 from fastapi import FastAPI, Request
-from fastapi.responses import RedirectResponse, FileResponse, Response, JSONResponse
-from pkg_resources import resource_filename
-from sovereign import (
-    __version__,
-    config,
-    asgi_config,
-    json_response_class,
-    poller,
-    template_context,
-    logs,
-)
+from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, Response
+from starlette_context.middleware import RawContextMiddleware
+
+from sovereign import __version__, logs
+from sovereign.configuration import ConfiguredResourceTypes, config
 from sovereign.error_info import ErrorInfo
-from sovereign.views import crypto, discovery, healthchecks, admin, interface
-from sovereign.middlewares import (
-    RequestContextLogMiddleware,
-    LoggingMiddleware,
-)
+from sovereign.middlewares import LoggingMiddleware, RequestContextLogMiddleware
+from sovereign.response_class import json_response_class
+from sovereign.utils.resources import get_package_file
+from sovereign.views import api, crypto, discovery, healthchecks, interface
 
 Router = namedtuple("Router", "module tags prefix")
 
 DEBUG = config.debug
-SENTRY_DSN = config.sentry_dsn.get_secret_value()
-
-try:
-    import sentry_sdk
-    from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
-
-    SENTRY_INSTALLED = True
-except ImportError:  # pragma: no cover
-    SENTRY_INSTALLED = False
 
 
 def generic_error_response(e: Exception) -> JSONResponse:
@@ -46,7 +30,7 @@ def generic_error_response(e: Exception) -> JSONResponse:
     """
     tb = [line for line in traceback.format_exc().split("\n")]
     info = ErrorInfo.from_exception(e)
-    logs.queue_log_fields(
+    logs.access_logger.queue_log_fields(
         ERROR=info.error,
         ERROR_DETAIL=info.detail,
         TRACEBACK=tb,
@@ -69,23 +53,35 @@ def init_app() -> FastAPI:
 
     routers = (
         Router(discovery.router, ["Configuration Discovery"], ""),
-        Router(crypto.router, ["Cryptographic Utilities"], "/crypto"),
-        Router(admin.router, ["Debugging Endpoints"], "/admin"),
-        Router(interface.router, ["User Interface"], "/ui"),
+        Router(api.router, ["API"], "/api"),
         Router(healthchecks.router, ["Healthchecks"], ""),
+        Router(interface.router, ["User Interface"], "/ui"),
+        Router(crypto.router, ["Cryptographic Utilities"], "/crypto"),
     )
     for router in routers:
         application.include_router(
             router.module, tags=router.tags, prefix=router.prefix
         )
 
-    application.add_middleware(RequestContextLogMiddleware)
-    application.add_middleware(LoggingMiddleware)
+    application.add_middleware(RequestContextLogMiddleware)  # type: ignore
+    application.add_middleware(LoggingMiddleware)  # type: ignore
+
+    if dsn := config.sentry_dsn.get_secret_value():
+        try:
+            # noinspection PyUnusedImports
+            import sentry_sdk
+
+            # noinspection PyUnusedImports
+            from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
 
-    if SENTRY_INSTALLED and SENTRY_DSN:
-        sentry_sdk.init(SENTRY_DSN)
-        application.add_middleware(SentryAsgiMiddleware)
-        logs.application_log(event="Sentry middleware enabled")
+            sentry_sdk.init(dsn)
+            application.add_middleware(SentryAsgiMiddleware)  # type: ignore
+        except ImportError:
+            logs.application_logger.logger.error(
+                "Sentry DSN configured but failed to attach to webserver"
+            )
+
+    application.add_middleware(RawContextMiddleware)  # type: ignore
 
     @application.exception_handler(500)
     async def exception_handler(_: Request, exc: Exception) -> JSONResponse:
@@ -103,23 +99,41 @@ def init_app() -> FastAPI:
 
     @application.get("/static/{filename}", summary="Return a static asset")
     async def static(filename: str) -> Response:
-        return FileResponse(resource_filename("sovereign", f"static/{filename}"))
-
-    @application.on_event("startup")
-    async def keep_sources_uptodate() -> None:
-        asyncio.create_task(poller.poll_forever())
+        return FileResponse(get_package_file("sovereign_files", f"static/{filename}"))  # type: ignore[arg-type]
 
-    @application.on_event("startup")
-    async def refresh_template_context() -> None:
-        asyncio.create_task(template_context.start_refresh_context())
+    @application.get(
+        "/admin/xds_dump",
+        summary="Deprecated API, please use /api/resources/{resource_type}",
+    )
+    async def dump_resources(request: Request) -> Response:
+        resource_type = ConfiguredResourceTypes(
+            request.query_params.get("xds_type", "cluster")
+        )
+        resource_name = request.query_params.get("name")
+        api_version = request.query_params.get("api_version", "v3")
+        service_cluster = request.query_params.get("service_cluster", "*")
+        region = request.query_params.get("region")
+        version = request.query_params.get("version")
+        response = await api.resource(
+            resource_type=resource_type,
+            resource_name=resource_name,
+            api_version=api_version,
+            service_cluster=service_cluster,
+            region=region,
+            version=version,
+        )
+        response.headers["Deprecation"] = "true"
+        response.headers["Link"] = f'</api/resources/{resource_type}>; rel="alternate"'
+        response.headers["Warning"] = (
+            f'299 - "Deprecated API: please use /api/resources/{resource_type}"'
+        )
+        response.status_code = 299
+        return response
 
     return application
 
 
 app = init_app()
-logs.application_log(
-    event=f"Sovereign started and listening on {asgi_config.host}:{asgi_config.port}"
-)
 
 
 if __name__ == "__main__":  # pragma: no cover

sovereign/cache/__init__.py

@@ -0,0 +1,245 @@
+"""
+Sovereign Cache Module
+
+This module provides an extensible cache backend system that allows clients
+to configure their own remote cache backends through entry points.
+"""
+
+import asyncio
+import os
+import threading
+import time
+
+import requests
+from typing_extensions import final
+
+from sovereign import WORKER_URL, stats
+from sovereign import application_logger as log
+from sovereign.cache.backends import CacheBackend, get_backend
+from sovereign.cache.filesystem import FilesystemCache
+from sovereign.cache.types import CacheResult, Entry
+from sovereign.configuration import config
+from sovereign.types import DiscoveryRequest, RegisterClientRequest
+
+CACHE_READ_TIMEOUT = config.cache.read_timeout
+REMOTE_TTL = 300  # 5 minutes - TTL for entries read from remote cache
+
+
+class CacheManagerBase:
+    def __init__(self) -> None:
+        self.local: FilesystemCache = FilesystemCache()
+        self.remote: CacheBackend | None = get_backend()
+        if self.remote is None:
+            log.info("Cache initialized with filesystem backend only")
+        else:
+            log.info("Cache initialized with filesystem and remote backends")
+
+    # Client Id registration
+
+    def register(self, req: DiscoveryRequest):
+        id = client_id(req)
+        log.debug(f"Registering client {id}")
+        self.local.register(id, req)
+        stats.increment("client.registration", tags=["status:registered"])
+        return id, req
+
+    def registered(self, req: DiscoveryRequest) -> bool:
+        ret = False
+        id = client_id(req)
+        if value := self.local.registered(id):
+            ret = value
+        log.debug(f"Client {id} registered={ret}")
+        return ret
+
+    def get_registered_clients(self) -> list[tuple[str, DiscoveryRequest]]:
+        if value := self.local.get_registered_clients():
+            return value
+        return []
+
+
+@final
+class CacheReader(CacheManagerBase):
+    def try_read(self, key: str) -> CacheResult | None:
+        # Try filesystem first
+        if value := self.local.get(key):
+            stats.increment("cache.fs.hit")
+            return CacheResult(value=value, from_remote=False)
+        stats.increment("cache.fs.miss")
+
+        # Fallback to remote cache if available
+        if self.remote:
+            try:
+                if value := self.remote.get(key):
+                    ret = CacheResult(value=value, from_remote=True)
+                    stats.increment("cache.remote.hit")
+                    return ret
+            except Exception as e:
+                log.warning(f"Failed to read from remote cache: {e}")
+                stats.increment("cache.remote.error")
+            stats.increment("cache.remote.miss")
+            log.warning(f"Failed to read from either cache for {key}")
+        return None
+
+    def get(self, req: DiscoveryRequest) -> Entry | None:
+        """Read from cache, writing back from remote with short TTL if needed.
+
+        Flow:
+          1. Entry read from remote → cached with 300s TTL
+          2. Background registration triggers worker to generate fresh config
+          3. Remote entry expires after 300s
+          4. Next request gets worker-generated config (cached infinitely)
+        """
+        id = client_id(req)
+        if result := self.try_read(id):
+            if result.from_remote:
+                # Write immediately with short TTL to prevent empty cache window
+                self.local.set(id, result.value, timeout=REMOTE_TTL)
+                log.info(
+                    f"Cache writeback from remote: client_id={id} version={result.value.version} "
+                    f"ttl={REMOTE_TTL} type=remote pid={os.getpid()}"
+                )
+                stats.increment("cache.fs.writeback", tags=["type:remote"])
+
+                # Background thread triggers worker to generate fresh config
+                self.register_async(req)
+            return result.value
+        return None
+
+    @stats.timed("cache.read_ms")
+    async def blocking_read(
+        self, req: DiscoveryRequest, timeout_s=CACHE_READ_TIMEOUT, poll_interval_s=0.5
+    ) -> Entry | None:
+        cid = client_id(req)
+        metric = "client.registration"
+        if entry := self.get(req):
+            return entry
+
+        log.info(f"Cache entry not found for {cid}, registering and waiting")
+        registered = False
+        start = asyncio.get_event_loop().time()
+        attempt = 1
+        while (asyncio.get_event_loop().time() - start) < timeout_s:
+            if not registered:
+                try:
+                    if self.register_over_http(req):
+                        stats.increment(metric, tags=["status:registered"])
+                        registered = True
+                        log.info(f"Client {cid} registered")
+                    else:
+                        stats.increment(metric, tags=["status:ratelimited"])
+                        await asyncio.sleep(min(attempt, CACHE_READ_TIMEOUT))
+                        attempt *= 2
+                except Exception as e:
+                    stats.increment(metric, tags=["status:failed"])
+                    log.exception(f"Tried to register client but failed: {e}")
+            if entry := self.get(req):
+                log.info(f"Entry has been populated for {cid}")
+                return entry
+            await asyncio.sleep(poll_interval_s)
+
+        return None
+
+    def register_over_http(self, req: DiscoveryRequest) -> bool:
+        registration = RegisterClientRequest(request=req)
+        log.debug(f"Sending registration to worker for {req}")
+        try:
+            response = requests.put(
+                f"{WORKER_URL}/client",
+                json=registration.model_dump(),
+                timeout=3,
+            )
+            match response.status_code:
+                case 200 | 202:
+                    log.debug("Worker responded OK to registration")
+                    return True
+                case code:
+                    log.debug(f"Worker responded with {code} to registration")
+        except Exception as e:
+            log.exception(f"Error while registering client: {e}")
+        return False
+
+    def register_async(self, req: DiscoveryRequest):
+        """Register client async to trigger worker to generate fresh config.
+
+        Registration tells the worker about this client so it generates fresh config.
+        """
+
+        def job():
+            start_time = time.time()
+            attempts = 5
+            backoff = 1.0
+            attempt_num = 0
+
+            while attempts:
+                attempt_num += 1
+                if self.register_over_http(req):
+                    duration_ms = (time.time() - start_time) * 1000
+                    stats.increment(
+                        "client.registration.async",
+                        tags=["status:success", f"attempts:{attempt_num}"],
+                    )
+                    stats.timing("client.registration.async.duration_ms", duration_ms)
+                    log.debug(f"Async registration succeeded: attempts={attempt_num}")
+                    return
+                attempts -= 1
+                if attempts:
+                    log.debug(
+                        f"Async registration failed: retrying_in={backoff}s remaining={attempts}"
+                    )
+                    time.sleep(backoff)
+                    backoff *= 2
+
+            # Registration failed - entry stays at REMOTE_TTL, will expire and retry
+            duration_ms = (time.time() - start_time) * 1000
+            stats.increment("client.registration.async", tags=["status:exhausted"])
+            stats.timing("client.registration.async.duration_ms", duration_ms)
+            log.warning(
+                f"Async registration exhausted for {req}: remote entry will expire "
+                f"in {REMOTE_TTL}s and retry"
+            )
+
+        t = threading.Thread(target=job)
+        t.start()
+
+
+@final
+class CacheWriter(CacheManagerBase):
+    def set(
+        self, key: str, value: Entry, timeout: int | None = None
+    ) -> tuple[bool, list[tuple[str, str]]]:
+        msg = []
+        cached = False
+        try:
+            self.local.set(key, value, timeout)
+            log.info(
+                f"Cache write to filesystem: client_id={key} version={value.version} "
+                f"ttl={timeout} pid={os.getpid()} thread_id={threading.get_ident()}"
+            )
+            stats.increment("cache.fs.write.success")
+            cached = True
+        except Exception as e:
+            log.warning(
+                f"Failed to write to filesystem cache: client_id={key} error={e}"
+            )
+            msg.append(("warning", f"Failed to write to filesystem cache: {e}"))
+            stats.increment("cache.fs.write.error")
+        if self.remote:
+            try:
+                self.remote.set(key, value, timeout)
+                log.info(
+                    f"Cache write to remote: client_id={key} version={value.version} "
+                    f"ttl={timeout} pid={os.getpid()} thread_id={threading.get_ident()}"
+                )
+                stats.increment("cache.remote.write.success")
+                cached = True
+            except Exception as e:
+                log.warning(
+                    f"Failed to write to remote cache: client_id={key} error={e}"
+                )
+                msg.append(("warning", f"Failed to write to remote cache: {e}"))
+                stats.increment("cache.remote.write.error")
+        return cached, msg
+
+
+def client_id(req: DiscoveryRequest) -> str:
+    return req.cache_key(config.cache.hash_rules)

sovereign/cache/backends/__init__.py

@@ -0,0 +1,110 @@
+"""
+Cache backends module
+
+This module provides the protocol definition for cache backends and
+the loading mechanism for extensible cache backends via entry points.
+"""
+
+from collections.abc import Sequence
+from importlib.metadata import EntryPoints
+from typing import Any, Protocol, runtime_checkable
+
+from sovereign import application_logger as log
+from sovereign.utils.entry_point_loader import EntryPointLoader
+
+
+@runtime_checkable
+class CacheBackend(Protocol):
+    def __init__(self, config: dict[str, Any]) -> None:
+        """Initialize the cache backend with generic configuration
+
+        Args:
+            config: Dictionary containing backend-specific configuration
+        """
+        ...
+
+    def get(self, key: str) -> Any | None:
+        """Get a value from the cache
+
+        Args:
+            key: The cache key
+
+        Returns:
+            The cached value or None if not found
+        """
+        ...
+
+    def set(self, key: str, value: Any, timeout: int | None = None) -> None:
+        """Set a value in the cache
+
+        Args:
+            key: The cache key
+            value: The value to cache
+            timeout: Optional timeout in seconds
+        """
+        ...
+
+
+def get_backend() -> CacheBackend | None:
+    from sovereign import config
+
+    cache_config = config.cache.remote_backend
+    if not cache_config:
+        log.info("No remote cache backend configured, using filesystem only")
+        return None
+
+    backend_type = cache_config.type
+
+    loader = EntryPointLoader("cache.backends")
+    entry_points: EntryPoints | Sequence[Any] = loader.groups.get("cache.backends", [])
+
+    backend = None
+    for ep in entry_points:
+        if ep.name == backend_type:
+            backend = ep.load()
+            break
+
+    if not backend:
+        raise KeyError(
+            (
+                f"Cache backend '{backend_type}' not found. "
+                f"Available backends: {[ep.name for ep in entry_points]}"
+            )
+        )
+
+    backend_config = _process_loadable_config(cache_config.config)
+    instance = backend(backend_config)
+
+    if not isinstance(instance, CacheBackend):
+        raise TypeError(
+            (f"Cache backend '{backend_type}' does not implement CacheBackend protocol")
+        )
+
+    log.info(f"Successfully initialized cache backend: {backend_type}")
+    return instance
+
+
+def _process_loadable_config(config: dict[str, Any]) -> dict[str, Any]:
+    from sovereign.dynamic_config import Loadable
+
+    processed = {}
+    for key, value in config.items():
+        try:
+            if isinstance(value, str):
+                loadable = Loadable.from_legacy_fmt(value)
+                processed[key] = loadable.load()
+            elif isinstance(value, dict):
+                loadable = Loadable(**value)
+                processed[key] = loadable.load()
+            else:
+                processed[key] = value
+            continue
+        except Exception as e:
+            log.warning(f"Failed to load value for {key}: {e}")
+
+        if isinstance(value, dict):
+            processed[key] = _process_loadable_config(value)
+        else:
+            processed[key] = value
+
+    return processed