sovereign 1.0.0b121__tar.gz → 1.0.0b148__tar.gz

{sovereign-1.0.0b121 → sovereign-1.0.0b148}/PKG-INFO

@@ -1,13 +1,13 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: sovereign
-Version: 1.0.0b121
+Version: 1.0.0b148
 Summary: Envoy Proxy control-plane written in Python
-Home-page: https://pypi.org/project/sovereign/
-License: Apache-2.0
-Keywords: envoy,envoyproxy,control-plane,management,server
-Author: Vasili Syrakis
-Author-email: vsyrakis@atlassian.com
-Requires-Python: >=3.11,<4.0
+Project-URL: Homepage, https://pypi.org/project/sovereign/
+Project-URL: Repository, https://bitbucket.org/atlassian/sovereign/src/master/
+Project-URL: Documentation, https://developer.atlassian.com/platform/sovereign/
+Author-email: Vasili Syrakis <vsyrakis@atlassian.com>
+License-Expression: Apache-2.0
+Keywords: control-plane,envoy,envoyproxy,management,server
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: No Input/Output (Daemon)
 Classifier: Intended Audience :: Developers
@@ -16,51 +16,45 @@ Classifier: Intended Audience :: System Administrators
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Natural Language :: English
 Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Internet :: Proxy Servers
+Requires-Python: <4.0,>=3.11.0
+Requires-Dist: aiofiles<24,>=23.2.1
+Requires-Dist: cachelib<0.11,>=0.10.2
+Requires-Dist: cachetools<6,>=5.3.2
+Requires-Dist: croniter<2,>=1.4.1
+Requires-Dist: cryptography>=45.0.2
+Requires-Dist: fastapi<0.117,>=0.116.0
+Requires-Dist: glom<24,>=23.3.0
+Requires-Dist: h11<0.17,>=0.16.0
+Requires-Dist: jinja2<4,>=3.1.2
+Requires-Dist: jmespath<2,>=1.0.1
+Requires-Dist: pydantic-settings<2.6.0
+Requires-Dist: pydantic<3,>=2.7.2
+Requires-Dist: pyyaml<7,>=6.0.1
+Requires-Dist: requests<3,>=2.32.4
+Requires-Dist: starlette-context<0.4,>=0.3.6
+Requires-Dist: starlette<0.48,>=0.47.2
+Requires-Dist: structlog<24,>=23.1.0
+Requires-Dist: supervisor<5,>=4.2.5
+Requires-Dist: uvicorn<0.24,>=0.23.2
+Requires-Dist: uvloop<1.0,>0.19.0
 Provides-Extra: boto
+Requires-Dist: boto3<2,>=1.28.62; extra == 'boto'
 Provides-Extra: caching
 Provides-Extra: httptools
+Requires-Dist: httptools<0.7,>=0.6.0; extra == 'httptools'
 Provides-Extra: orjson
+Requires-Dist: orjson<4,>=3.9.15; extra == 'orjson'
 Provides-Extra: sentry
+Requires-Dist: sentry-sdk<3,>=2.14.0; extra == 'sentry'
 Provides-Extra: statsd
+Requires-Dist: datadog>=0.50.1; extra == 'statsd'
 Provides-Extra: ujson
-Requires-Dist: Jinja2 (>=3.1.2,<4.0.0)
-Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
-Requires-Dist: aiofiles (>=23.2.1,<24.0.0)
-Requires-Dist: boto3 (>=1.28.62,<2.0.0) ; extra == "boto"
-Requires-Dist: cachelib (>=0.10.2,<0.11.0)
-Requires-Dist: cachetools (>=5.3.2,<6.0.0)
-Requires-Dist: cashews[redis] (>=6.3.0,<7.0.0) ; extra == "caching"
-Requires-Dist: croniter (>=1.4.1,<2.0.0)
-Requires-Dist: cryptography (>=45.0.2)
-Requires-Dist: datadog (>=0.50.1) ; extra == "statsd"
-Requires-Dist: fastapi (>=0.116.0,<0.117.0)
-Requires-Dist: glom (>=23.3.0,<24.0.0)
-Requires-Dist: h11 (>=0.16.0,<0.17.0)
-Requires-Dist: httptools (>=0.6.0,<0.7.0) ; extra == "httptools"
-Requires-Dist: jmespath (>=1.0.1,<2.0.0)
-Requires-Dist: orjson (>=3.9.15,<4.0.0) ; extra == "orjson"
-Requires-Dist: pydantic (>=2.7.2,<3.0.0)
-Requires-Dist: pydantic-settings (<2.6.0)
-Requires-Dist: redis (<=5.0.0)
-Requires-Dist: requests (>=2.32.4,<3.0.0)
-Requires-Dist: sentry-sdk (>=2.14.0,<3.0.0) ; extra == "sentry"
-Requires-Dist: starlette (>=0.47.2,<0.48.0)
-Requires-Dist: starlette-context (>=0.3.6,<0.4.0)
-Requires-Dist: structlog (>=23.1.0,<24.0.0)
-Requires-Dist: supervisor (>=4.2.5,<5.0.0)
-Requires-Dist: ujson (>=5.8.0,<6.0.0) ; extra == "ujson"
-Requires-Dist: uvicorn (>=0.23.2,<0.24.0)
-Requires-Dist: uvloop (>=0.19.0,<0.20.0)
-Project-URL: Documentation, https://developer.atlassian.com/platform/sovereign/
-Project-URL: Repository, https://bitbucket.org/atlassian/sovereign/src/master/
+Requires-Dist: ujson<6,>=5.8.0; extra == 'ujson'
 Description-Content-Type: text/markdown
 
 sovereign
@@ -136,17 +130,18 @@ Local development
 
 Requirements
 ------------
-* Poetry
+* uv
 * Docker
 * Docker-compose
 
 
 Installing dependencies for dev
 -------------------------------
-Dependencies and creation of virtualenv is handled by poetry
+Dependencies and creation of virtualenv is handled by uv
 ```
-poetry install
-poetry shell
+uv sync
+uv venv activate
+uv run <command>
 ```
 
 Running locally
@@ -211,4 +206,3 @@ Copyright (c) 2018 Atlassian and others.
 Apache 2.0 licensed, see [LICENSE.txt](LICENSE.txt) file.
 
 
-

{sovereign-1.0.0b121 → sovereign-1.0.0b148}/README.md

@@ -71,17 +71,18 @@ Local development
 
 Requirements
 ------------
-* Poetry
+* uv
 * Docker
 * Docker-compose
 
 
 Installing dependencies for dev
 -------------------------------
-Dependencies and creation of virtualenv is handled by poetry
+Dependencies and creation of virtualenv is handled by uv
 ```
-poetry install
-poetry shell
+uv sync
+uv venv activate
+uv run <command>
 ```
 
 Running locally

sovereign-1.0.0b148/pyproject.toml

@@ -0,0 +1,146 @@
+[project]
+name = "sovereign"
+version = "1.0.0b148"
+description = "Envoy Proxy control-plane written in Python"
+authors = [{ name = "Vasili Syrakis", email = "vsyrakis@atlassian.com" }]
+requires-python = ">=3.11.0, <4.0"
+readme = "README.md"
+license = "Apache-2.0"
+keywords = [
+    "envoy",
+    "envoyproxy",
+    "control-plane",
+    "management",
+    "server",
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: No Input/Output (Daemon)",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: Apache Software License",
+    "Natural Language :: English",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Internet :: Proxy Servers",
+]
+dependencies = [
+    "uvicorn>=0.23.2,<0.24",
+    "aiofiles>=23.2.1,<24",
+    "requests>=2.32.4,<3",
+    "PyYAML>=6.0.1,<7",
+    "Jinja2>=3.1.2,<4",
+    "structlog>=23.1.0,<24",
+    "cachelib>=0.10.2,<0.11",
+    "glom>=23.3.0,<24",
+    "cryptography>=45.0.2",
+    "fastapi>=0.116.0,<0.117",
+    "uvloop>0.19.0,<1.0",
+    "croniter>=1.4.1,<2",
+    "cachetools>=5.3.2,<6",
+    "pydantic>=2.7.2,<3",
+    "pydantic-settings<2.6.0",
+    "starlette-context>=0.3.6,<0.4",
+    "starlette>=0.47.2,<0.48",
+    "h11>=0.16.0,<0.17",
+    "supervisor>=4.2.5,<5",
+    "jmespath>=1.0.1,<2",
+]
+
+[project.optional-dependencies]
+sentry = ["sentry-sdk>=2.14.0,<3"]
+boto = ["boto3>=1.28.62,<2"]
+statsd = ["datadog>=0.50.1"]
+ujson = ["ujson>=5.8.0,<6"]
+orjson = ["orjson>=3.9.15,<4"]
+caching = []
+httptools = ["httptools>=0.6.0,<0.7"]
+
+[project.urls]
+Homepage = "https://pypi.org/project/sovereign/"
+Repository = "https://bitbucket.org/atlassian/sovereign/src/master/"
+Documentation = "https://developer.atlassian.com/platform/sovereign/"
+
+[project.scripts]
+sovereign = "sovereign.server:main"
+sovereign-web = "sovereign.server:web"
+sovereign-worker = "sovereign.server:worker"
+
+[project.entry-points."sovereign.sources"]
+file = "sovereign.sources.file:File"
+inline = "sovereign.sources.inline:Inline"
+
+[project.entry-points."sovereign.modifiers"]
+sovereign_3rd_party_test = "sovereign.testing.modifiers:Test"
+
+[project.entry-points."sovereign.loaders"]
+example = "sovereign.testing.loaders:Multiply"
+file = "sovereign.dynamic_config.loaders:File"
+pkgdata = "sovereign.dynamic_config.loaders:PackageData"
+http = "sovereign.dynamic_config.loaders:Web"
+https = "sovereign.dynamic_config.loaders:Web"
+env = "sovereign.dynamic_config.loaders:EnvironmentVariable"
+module = "sovereign.dynamic_config.loaders:PythonModule"
+s3 = "sovereign.dynamic_config.loaders:S3Bucket"
+python = "sovereign.dynamic_config.loaders:PythonInlineCode"
+inline = "sovereign.dynamic_config.loaders:Inline"
+
+[project.entry-points."sovereign.deserializers"]
+yaml = "sovereign.dynamic_config.deser:YamlDeserializer"
+json = "sovereign.dynamic_config.deser:JsonDeserializer"
+jinja = "sovereign.dynamic_config.deser:JinjaDeserializer"
+jinja2 = "sovereign.dynamic_config.deser:JinjaDeserializer"
+string = "sovereign.dynamic_config.deser:StringDeserializer"
+raw = "sovereign.dynamic_config.deser:PassthroughDeserializer"
+none = "sovereign.dynamic_config.deser:PassthroughDeserializer"
+passthrough = "sovereign.dynamic_config.deser:PassthroughDeserializer"
+ujson = "sovereign.dynamic_config.deser:UjsonDeserializer"
+orjson = "sovereign.dynamic_config.deser:OrjsonDeserializer"
+
+[project.entry-points."sovereign.cache.backends"]
+s3 = "sovereign.cache.backends.s3:S3Backend"
+
+[dependency-groups]
+dev = [
+    "tavern",
+    "moto",
+    "freezegun",
+    "pytest-mock",
+    "pytest-asyncio",
+    "ruff",
+    "toml",
+    "httpx",
+    "types-croniter",
+    "types-requests",
+    "types-setuptools",
+    "types-ujson",
+    "types-PyYAML",
+    "types-cachetools",
+    "ty",
+]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/sovereign",
+    "src/sovereign_files",
+    "src/sovereign_files/**/*",
+]
+
+[tool.hatch.build.targets.wheel]
+include = [
+    "src/sovereign",
+    "src/sovereign_files",
+    "src/sovereign_files/**/*",
+]
+
+[tool.hatch.build.targets.wheel.sources]
+"src/sovereign" = "sovereign"
+"src/sovereign_files" = "sovereign_files"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

{sovereign-1.0.0b121 → sovereign-1.0.0b148}/src/sovereign/__init__.py

@@ -1,20 +1,11 @@
-import sys
 from contextvars import ContextVar
 from importlib.metadata import version
-from sovereign.context import TemplateContext
 
 from sovereign.utils.crypto.suites import EncryptionType
-from starlette.templating import Jinja2Templates
-
 from sovereign.logging.bootstrapper import LoggerBootstrapper
-from sovereign.schemas import (
-    config,
-    EncryptionConfig,
-    migrate_configs,
-)
+from sovereign.configuration import config, EncryptionConfig
 from sovereign.statistics import configure_statsd
 from sovereign.utils.crypto.crypto import CipherContainer
-from sovereign.utils.resources import get_package_file
 
 _request_id_ctx_var: ContextVar[str] = ContextVar("request_id", default="")
 
@@ -23,23 +14,14 @@ def get_request_id() -> str:
     return _request_id_ctx_var.get()
 
 
+WORKER_URL = "http://localhost:9080"
 DIST_NAME = "sovereign"
-
 __version__ = version(DIST_NAME)
 
-html_templates = Jinja2Templates(
-    directory=str(get_package_file(DIST_NAME, "templates"))
-)
-
-if sys.argv[0].endswith("sovereign"):
-    migrate_configs()
-
 stats = configure_statsd()
 logs = LoggerBootstrapper(config)
 application_logger = logs.application_logger.logger
 
-template_context = TemplateContext.from_config()
-
 encryption_configs = config.authentication.encryption_configs
 server_cipher_container = CipherContainer.from_encryption_configs(
     encryption_configs, logger=application_logger

{sovereign-1.0.0b121 → sovereign-1.0.0b148}/src/sovereign/app.py

@@ -6,12 +6,8 @@ from fastapi import FastAPI, Request
 from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, Response
 from starlette_context.middleware import RawContextMiddleware
 
-from sovereign import (
-    __version__,
-    config,
-    logs,
-)
-from sovereign.schemas import DiscoveryTypes
+from sovereign import __version__, logs
+from sovereign.configuration import config, ConfiguredResourceTypes
 from sovereign.response_class import json_response_class
 from sovereign.error_info import ErrorInfo
 from sovereign.middlewares import LoggingMiddleware, RequestContextLogMiddleware
@@ -21,15 +17,6 @@ from sovereign.views import crypto, discovery, healthchecks, interface, api
 Router = namedtuple("Router", "module tags prefix")
 
 DEBUG = config.debug
-SENTRY_DSN = config.sentry_dsn.get_secret_value()
-
-try:
-    import sentry_sdk
-    from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
-
-    SENTRY_INSTALLED = True
-except ImportError:  # pragma: no cover
-    SENTRY_INSTALLED = False
 
 
 def generic_error_response(e: Exception) -> JSONResponse:
@@ -76,14 +63,22 @@ def init_app() -> FastAPI:
             router.module, tags=router.tags, prefix=router.prefix
         )
 
-    application.add_middleware(RequestContextLogMiddleware)
-    application.add_middleware(LoggingMiddleware)
+    application.add_middleware(RequestContextLogMiddleware)  # type: ignore
+    application.add_middleware(LoggingMiddleware)  # type: ignore
 
-    if SENTRY_INSTALLED and SENTRY_DSN:
-        sentry_sdk.init(SENTRY_DSN)
-        application.add_middleware(SentryAsgiMiddleware)
+    if dsn := config.sentry_dsn.get_secret_value():
+        try:
+            import sentry_sdk
+            from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
 
-    application.add_middleware(RawContextMiddleware)
+            sentry_sdk.init(dsn)
+            application.add_middleware(SentryAsgiMiddleware)  # type: ignore
+        except ImportError:
+            logs.application_logger.logger.error(
+                "Sentry DSN configured but failed to attach to webserver"
+            )
+
+    application.add_middleware(RawContextMiddleware)  # type: ignore
 
     @application.exception_handler(500)
     async def exception_handler(_: Request, exc: Exception) -> JSONResponse:
@@ -101,14 +96,16 @@ def init_app() -> FastAPI:
 
     @application.get("/static/{filename}", summary="Return a static asset")
     async def static(filename: str) -> Response:
-        return FileResponse(get_package_file("sovereign", f"static/{filename}"))  # type: ignore[arg-type]
+        return FileResponse(get_package_file("sovereign_files", f"static/{filename}"))  # type: ignore[arg-type]
 
     @application.get(
         "/admin/xds_dump",
         summary="Deprecated API, please use /api/resources/{resource_type}",
     )
     async def dump_resources(request: Request) -> Response:
-        resource_type = DiscoveryTypes(request.query_params.get("xds_type", "cluster"))
+        resource_type = ConfiguredResourceTypes(
+            request.query_params.get("xds_type", "cluster")
+        )
         resource_name = request.query_params.get("name")
         api_version = request.query_params.get("api_version", "v3")
         service_cluster = request.query_params.get("service_cluster", "*")

sovereign-1.0.0b148/src/sovereign/cache/__init__.py

@@ -0,0 +1,172 @@
+"""
+Sovereign Cache Module
+
+This module provides an extensible cache backend system that allows clients
+to configure their own remote cache backends through entry points.
+"""
+
+import asyncio
+from typing import Any
+from typing_extensions import final
+
+import requests
+
+from sovereign import WORKER_URL, stats, application_logger as log
+from sovereign.types import DiscoveryRequest, RegisterClientRequest
+from sovereign.configuration import config
+from sovereign.cache.types import Entry, CacheResult
+from sovereign.cache.backends import CacheBackend, get_backend
+from sovereign.cache.filesystem import FilesystemCache
+
+
+CACHE_READ_TIMEOUT = config.cache.read_timeout
+
+
+class CacheManagerBase:
+    def __init__(self) -> None:
+        self.local: FilesystemCache = FilesystemCache()
+        self.remote: CacheBackend | None = get_backend()
+        if self.remote is None:
+            log.info("Cache initialized with filesystem backend only")
+        else:
+            log.info("Cache initialized with filesystem and remote backends")
+
+    # Client Id registration
+
+    def register(self, req: DiscoveryRequest):
+        id = client_id(req)
+        log.debug(f"Registering client {id}")
+        self.local.register(id, req)
+        stats.increment("client.registration", tags=["status:registered"])
+        return id, req
+
+    def registered(self, req: DiscoveryRequest) -> bool:
+        ret = False
+        id = client_id(req)
+        if value := self.local.registered(id):
+            ret = value
+        log.debug(f"Client {id} registered={ret}")
+        return ret
+
+    def get_registered_clients(self) -> list[tuple[str, DiscoveryRequest]]:
+        if value := self.local.get_registered_clients():
+            return value
+        return []
+
+
+@final
+class CacheReader(CacheManagerBase):
+    def try_read(self, key: str) -> CacheResult | None:
+        # Try filesystem first
+        if value := self.local.get(key):
+            stats.increment("cache.fs.hit")
+            return CacheResult(value=value, from_remote=False)
+        stats.increment("cache.fs.miss")
+
+        # Fallback to remote cache if available
+        if self.remote:
+            try:
+                if value := self.remote.get(key):
+                    ret = CacheResult(value=value, from_remote=True)
+                    stats.increment("cache.remote.hit")
+                    return ret
+            except Exception as e:
+                log.warning(f"Failed to read from remote cache: {e}")
+                stats.increment("cache.remote.error")
+            stats.increment("cache.remote.miss")
+            log.warning(f"Failed to read from either cache for {key}")
+        return None
+
+    def get(self, req: DiscoveryRequest) -> Entry | None:
+        id = client_id(req)
+        if result := self.try_read(id):
+            if result.from_remote:
+                self.register(req)
+                # Write back to filesystem
+                self.local.set(id, result.value)
+            return result.value
+        return None
+
+    @stats.timed("cache.read_ms")
+    async def blocking_read(
+        self, req: DiscoveryRequest, timeout_s=CACHE_READ_TIMEOUT, poll_interval_s=0.5
+    ) -> Entry | None:
+        cid = client_id(req)
+        metric = "client.registration"
+        if entry := self.get(req):
+            return entry
+
+        log.info(f"Cache entry not found for {cid}, registering and waiting")
+        registered = False
+        start = asyncio.get_event_loop().time()
+        attempt = 1
+        while (asyncio.get_event_loop().time() - start) < timeout_s:
+            if not registered:
+                try:
+                    if self.register_over_http(req):
+                        stats.increment(metric, tags=["status:registered"])
+                        registered = True
+                        log.info(f"Client {cid} registered")
+                    else:
+                        stats.increment(metric, tags=["status:ratelimited"])
+                        await asyncio.sleep(min(attempt, CACHE_READ_TIMEOUT))
+                        attempt *= 2
+                except Exception as e:
+                    stats.increment(metric, tags=["status:failed"])
+                    log.exception(f"Tried to register client but failed: {e}")
+            if entry := self.get(req):
+                log.info(f"Entry has been populated for {cid}")
+                return entry
+            await asyncio.sleep(poll_interval_s)
+
+        return None
+
+    def register_over_http(self, req: DiscoveryRequest) -> bool:
+        registration = RegisterClientRequest(request=req)
+        log.debug(f"Sending registration to worker for {req}")
+        try:
+            response = requests.put(
+                f"{WORKER_URL}/client",
+                json=registration.model_dump(),
+                timeout=3,
+            )
+            match response.status_code:
+                case 200 | 202:
+                    log.debug("Worker responded OK to registration")
+                    return True
+                case code:
+                    log.debug(f"Worker responded with {code} to registration")
+        except Exception as e:
+            log.exception(f"Error while registering client: {e}")
+        return False
+
+
+@final
+class CacheWriter(CacheManagerBase):
+    def set(
+        self, key: str, value: Entry, timeout: int | None = None
+    ) -> tuple[bool, list[tuple[str, str]]]:
+        msg = []
+        cached = False
+        try:
+            self.local.set(key, value, timeout)
+            stats.increment("cache.fs.write.success")
+            cached = True
+        except Exception as e:
+            log.warning(f"Failed to write to filesystem cache: {e}")
+            msg.append(("warning", f"Failed to write to filesystem cache: {e}"))
+            stats.increment("cache.fs.write.error")
+        if self.remote:
+            try:
+                self.remote.set(key, value, timeout)
+                stats.increment("cache.remote.write.success")
+                cached = True
+            except Exception as e:
+                log.warning(f"Failed to write to remote cache: {e}")
+                msg.append(("warning", f"Failed to write to remote cache: {e}"))
+                stats.increment("cache.remote.write.error")
+        return cached, msg
+
+
+def client_id(req: DiscoveryRequest) -> str:
+    return req.cache_key(config.cache.hash_rules)