sovereign 0.29.4__tar.gz → 0.31.0__tar.gz

{sovereign-0.29.4 → sovereign-0.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sovereign
-Version: 0.29.4
+Version: 0.31.0
 Summary: Envoy Proxy control-plane written in Python
 Home-page: https://pypi.org/project/sovereign/
 License: Apache-2.0
@@ -43,7 +43,7 @@ Requires-Dist: cryptography (>=42.0.0,<43.0.0)
 Requires-Dist: datadog (>=0.47.0,<0.48.0) ; extra == "statsd"
 Requires-Dist: fastapi (>=0.115.2,<0.116.0)
 Requires-Dist: glom (>=23.3.0,<24.0.0)
-Requires-Dist: gunicorn (>=22.0.0,<23.0.0)
+Requires-Dist: gunicorn (>=23.0.0,<24.0.0)
 Requires-Dist: httptools (>=0.6.0,<0.7.0) ; extra == "httptools"
 Requires-Dist: orjson (>=3.9.15,<4.0.0) ; extra == "orjson"
 Requires-Dist: pydantic (>=2.7.2,<3.0.0)
@@ -51,60 +51,70 @@ Requires-Dist: pydantic-settings (<2.6.0)
 Requires-Dist: redis (<=5.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: sentry-sdk (>=2.14.0,<3.0.0) ; extra == "sentry"
+Requires-Dist: starlette-context (>=0.3.6,<0.4.0)
 Requires-Dist: structlog (>=23.1.0,<24.0.0)
 Requires-Dist: ujson (>=5.8.0,<6.0.0) ; extra == "ujson"
 Requires-Dist: uvicorn (>=0.23.2,<0.24.0)
 Requires-Dist: uvloop (>=0.19.0,<0.20.0)
-Project-URL: Documentation, https://vsyrakis.bitbucket.io/sovereign/docs/
+Project-URL: Documentation, https://developer.atlassian.com/platform/sovereign/
 Project-URL: Repository, https://bitbucket.org/atlassian/sovereign/src/master/
 Description-Content-Type: text/markdown
 
 sovereign
 =========
 
+
 Mission statement
 -----------------
 This project implements a JSON control-plane based on the [envoy](https://envoyproxy.io) [data-plane-api](https://github.com/envoyproxy/data-plane-api)
 
-The purpose of `sovereign` is to supply downstream envoy proxies with 
-configuration in near-realtime by responding to discovery requests.
+The purpose of `sovereign` is to supply downstream envoy proxies with dynamic configuration.
+
 
 Mechanism of Operation
 ----------------------
-tl;dr version:
-```
-* Polls HTTP/File/Other for data
-* (optional) Applies transforms to the data
-* Uses the data to generate Envoy configuration from templates
-```
+Sovereign allows you to define templates that represent each resource type
+provided by Envoy. For example, clusters, routes, listeners, secrets,
+extension_configs, etc.
+
+In order to enrich the templates with data, Sovereign has ways of polling data
+out-of-band which it then includes as variables that can be accessed within the
+templates.
+
+This allows Sovereign to provide configuration to Envoy that changes over time
+depending on the data sources, without needing to redeploy the control-plane.
 
-In a nutshell, Sovereign 
-gathers contextual data (*"sources"* and *"template context"*), 
-optionally applies transforms to that data (using *"modifiers"*) and finally 
-uses the data to generate envoy configuration from either python code, or jinja2 templates.
+Sovereign provides some built-in ways of polling data (such as over HTTP, or
+on-disk) but also exposes extension points, allowing you to write your own
+plugins in Python.
 
-This is performed in a semi-stateless way, where the only state is data cached in memory.
 
-Template context is intended to be statically configured, whereas *Sources* 
-are meant to be dynamic - for example, fetching from an API, an S3 bucket, 
-or a file that receives updates.
+Support
+------------
+[Submit new issues here](https://bitbucket.org/atlassian/sovereign/issues/new)
+
+If you're unable to submit an issue on Bitbucket, send an email to [vsyrakis@atlassian.com](mailto:vsyrakis@atlassian.com)
+
 
-*Modifiers* can mutate the data retrieved from sources, just in case the data 
-is in a less than favorable structure.
+Release
+------------
+See [RELEASE.md]
 
-Both modifiers and sources are pluggable, i.e. it's easy to write your own and 
-plug them into Sovereign for your use-case.
 
-Currently, Sovereign supports only providing configuration to Envoy as JSON. 
-That is to say, gRPC is not supported yet. Contributions in this area are highly
-appreciated!
+Roadmap
+------------
+* Performance improvements
+* Data persistence
+* Push API (versus polling)
+* Client for Sovereign
+* gRPC
 
-The JSON configuration can be viewed in real-time with Sovereign's read-only web interface.
 
 Requirements
 ------------
 * Python 3.8+
 
+
 Installation
 ------------
 ```
@@ -113,62 +123,52 @@ pip install sovereign
 
 Documentation
 -------------
-[Read the docs here!](https://vsyrakis.bitbucket.io/sovereign/docs/)
+[Read the docs here!](https://developer.atlassian.com/platform/sovereign/)
 
-:new: Read-only user interface
-------------------------
-Added in `v0.5.3`!
 
-This interface allows you to browse the resources currently returned by Sovereign.
-
-![Sovereign User Interface Screenshot](https://bitbucket.org/atlassian/sovereign/src/master/assets/sovereign_ui.png)
 
 Local development
 =================
 
+
 Requirements
 ------------
+* Poetry
 * Docker
 * Docker-compose
 
+
 Installing dependencies for dev
 -------------------------------
-I recommend creating a virtualenv before doing any dev work
-
+Dependencies and creation of virtualenv is handled by poetry
 ```
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements-dev.txt
+poetry install
+poetry shell
 ```
 
 Running locally
 ---------------
 Running the test env
-
 ```
 make run
 ```
     
 Running the test env daemonized
-
 ```
 make run-daemon
 ```
 
 Pylint
-
 ```
 make lint
 ```
 
 Unit tests
-
 ```
 make unit
 ```
 
 Acceptance tests
-
 ```
 make run-daemon acceptance
 ```
@@ -176,7 +176,6 @@ make run-daemon acceptance
 
 Contributors
 ============
-
 Pull requests, issues and comments welcome. For pull requests:
 
 * Add tests for new features and bug fixes
@@ -205,7 +204,6 @@ those contributing as an individual.
 
 License
 ========
-
 Copyright (c) 2018 Atlassian and others.
 Apache 2.0 licensed, see [LICENSE.txt](LICENSE.txt) file.
 

{sovereign-0.29.4 → sovereign-0.31.0}/README.md

@@ -1,49 +1,58 @@
 sovereign
 =========
 
+
 Mission statement
 -----------------
 This project implements a JSON control-plane based on the [envoy](https://envoyproxy.io) [data-plane-api](https://github.com/envoyproxy/data-plane-api)
 
-The purpose of `sovereign` is to supply downstream envoy proxies with 
-configuration in near-realtime by responding to discovery requests.
+The purpose of `sovereign` is to supply downstream envoy proxies with dynamic configuration.
+
 
 Mechanism of Operation
 ----------------------
-tl;dr version:
-```
-* Polls HTTP/File/Other for data
-* (optional) Applies transforms to the data
-* Uses the data to generate Envoy configuration from templates
-```
+Sovereign allows you to define templates that represent each resource type
+provided by Envoy. For example, clusters, routes, listeners, secrets,
+extension_configs, etc.
+
+In order to enrich the templates with data, Sovereign has ways of polling data
+out-of-band which it then includes as variables that can be accessed within the
+templates.
+
+This allows Sovereign to provide configuration to Envoy that changes over time
+depending on the data sources, without needing to redeploy the control-plane.
 
-In a nutshell, Sovereign 
-gathers contextual data (*"sources"* and *"template context"*), 
-optionally applies transforms to that data (using *"modifiers"*) and finally 
-uses the data to generate envoy configuration from either python code, or jinja2 templates.
+Sovereign provides some built-in ways of polling data (such as over HTTP, or
+on-disk) but also exposes extension points, allowing you to write your own
+plugins in Python.
 
-This is performed in a semi-stateless way, where the only state is data cached in memory.
 
-Template context is intended to be statically configured, whereas *Sources* 
-are meant to be dynamic - for example, fetching from an API, an S3 bucket, 
-or a file that receives updates.
+Support
+------------
+[Submit new issues here](https://bitbucket.org/atlassian/sovereign/issues/new)
+
+If you're unable to submit an issue on Bitbucket, send an email to [vsyrakis@atlassian.com](mailto:vsyrakis@atlassian.com)
+
 
-*Modifiers* can mutate the data retrieved from sources, just in case the data 
-is in a less than favorable structure.
+Release
+------------
+See [RELEASE.md]
 
-Both modifiers and sources are pluggable, i.e. it's easy to write your own and 
-plug them into Sovereign for your use-case.
 
-Currently, Sovereign supports only providing configuration to Envoy as JSON. 
-That is to say, gRPC is not supported yet. Contributions in this area are highly
-appreciated!
+Roadmap
+------------
+* Performance improvements
+* Data persistence
+* Push API (versus polling)
+* Client for Sovereign
+* gRPC
 
-The JSON configuration can be viewed in real-time with Sovereign's read-only web interface.
 
 Requirements
 ------------
 * Python 3.8+
 
+
 Installation
 ------------
 ```
@@ -52,62 +61,52 @@ pip install sovereign
 
 Documentation
 -------------
-[Read the docs here!](https://vsyrakis.bitbucket.io/sovereign/docs/)
+[Read the docs here!](https://developer.atlassian.com/platform/sovereign/)
 
-:new: Read-only user interface
-------------------------
-Added in `v0.5.3`!
 
-This interface allows you to browse the resources currently returned by Sovereign.
-
-![Sovereign User Interface Screenshot](https://bitbucket.org/atlassian/sovereign/src/master/assets/sovereign_ui.png)
 
 Local development
 =================
 
+
 Requirements
 ------------
+* Poetry
 * Docker
 * Docker-compose
 
+
 Installing dependencies for dev
 -------------------------------
-I recommend creating a virtualenv before doing any dev work
-
+Dependencies and creation of virtualenv is handled by poetry
 ```
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements-dev.txt
+poetry install
+poetry shell
 ```
 
 Running locally
 ---------------
 Running the test env
-
 ```
 make run
 ```
     
 Running the test env daemonized
-
 ```
 make run-daemon
 ```
 
 Pylint
-
 ```
 make lint
 ```
 
 Unit tests
-
 ```
 make unit
 ```
 
 Acceptance tests
-
 ```
 make run-daemon acceptance
 ```
@@ -115,7 +114,6 @@ make run-daemon acceptance
 
 Contributors
 ============
-
 Pull requests, issues and comments welcome. For pull requests:
 
 * Add tests for new features and bug fixes
@@ -144,7 +142,6 @@ those contributing as an individual.
 
 License
 ========
-
 Copyright (c) 2018 Atlassian and others.
 Apache 2.0 licensed, see [LICENSE.txt](LICENSE.txt) file.
 

{sovereign-0.29.4 → sovereign-0.31.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sovereign"
-version = "0.29.4"
+version = "0.31.0"
 description = "Envoy Proxy control-plane written in Python"
 license = "Apache-2.0"
 packages = [
@@ -8,7 +8,7 @@ packages = [
 ]
 readme = "README.md"
 #include = ["CHANGELOG.md", "CODE_OF_CONDUCT.md"]
-documentation = "https://vsyrakis.bitbucket.io/sovereign/docs/"
+documentation = "https://developer.atlassian.com/platform/sovereign/"
 repository = "https://bitbucket.org/atlassian/sovereign/src/master/"
 homepage = "https://pypi.org/project/sovereign/"
 authors = ["Vasili Syrakis <vsyrakis@atlassian.com>"]
@@ -58,6 +58,8 @@ sovereign = 'sovereign.server:main'
 "jinja2" = "sovereign.dynamic_config.deser:JinjaDeserializer"
 "string" = "sovereign.dynamic_config.deser:StringDeserializer"
 "raw" = "sovereign.dynamic_config.deser:PassthroughDeserializer"
+"none" = "sovereign.dynamic_config.deser:PassthroughDeserializer"
+"passthrough" = "sovereign.dynamic_config.deser:PassthroughDeserializer"
 "ujson" = "sovereign.dynamic_config.deser:UjsonDeserializer"
 "orjson" = "sovereign.dynamic_config.deser:OrjsonDeserializer"
 
@@ -65,7 +67,7 @@ sovereign = 'sovereign.server:main'
 [tool.poetry.dependencies]
 python = "^3.11"
 uvicorn = "^0.23.2"
-gunicorn = "^22.0.0"
+gunicorn = "^23.0.0"
 aiofiles = "^23.2.1"
 requests = "^2.31.0"
 PyYAML = "^6.0.1"
@@ -88,6 +90,7 @@ httptools = {version = "^0.6.0", optional = true}
 cachetools = "^5.3.2"
 pydantic = "^2.7.2"
 pydantic-settings = "<2.6.0"
+starlette-context = "^0.3.6"
 
 [tool.poetry.extras]
 sentry = ["sentry-sdk"]
@@ -99,40 +102,28 @@ caching = ["cashews"]
 httptools = ["httptools"]
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.0.0"
-"ruamel.yaml" = "^0.17.32"
-pytest-asyncio = "^0.20.0"
-pytest-mock = "^3.10.0"
-pytest-spec = "^3.2.0"
-pytest-timeout = "^2.1.0"
-coverage = "^7.3.1"
-invoke = "^2.2.0"
-pylint = "^2.17.6"
-tavern = "^2.9.2"
-twine = "^4.0.2"
-poethepoet = "^0.23.0"
-mypy = "^1.8.0"
-black = "^23.9.1"
-freezegun = "^1.2.2"
+# http integration test
+tavern = "^2.11.0"
+# mocking
 moto = "^4.2.4"
-httpx = "^0.25.0"
+freezegun = "^1.2.2"
+pytest-mock = "^3.14.0"
+pytest-asyncio = "^0.23"
+# type and lint checks
+mypy = "^1.8.0"
+ruff = "^0.8.4"
+# used in release checking script
+toml = "^0.10.2"
+# needed for starlette testclient
+httpx = "^0.28.1"
+# mypy types
 types-croniter = "^1.4.0.1"
 types-requests = "^2.31.0.8"
 types-setuptools = "^68.2.0.0"
 types-ujson = "^5.8.0.1"
 types-PyYAML = "^6.0.12.12"
-pylama = "^8.4.1"
-prospector = "^1.10.2"
-toml = "^0.10.2"
 types-cachetools = "^5.3.0.7"
 
-[tool.poe.tasks]
-types       = { cmd = "mypy src/sovereign --ignore-missing-imports", help = "Check types with mypy" }
-format      = { cmd = "black . --exclude test", help = "Apply black formatting to source code" }
-clean       = { cmd = "rm -rf site test-reports logs .coverage .mypy_cache .pytest_cache dist sovereign.egg-info **/__pycache__", help = "Clean out generated files from the workspace" }
-check-style = { cmd = "black . --check --diff --exclude test", help = "Check if code style is correct" }
-lint        = { cmd = "pylint src/sovereign", help = "Run linter checks" }
-
 [tool.black]
 target-version = ['py311']
 
@@ -142,9 +133,6 @@ plugins = [
 ]
 ignore_missing_imports = true
 
-[tool.coverage.run]
-omit = ["test/*"]
-
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"

{sovereign-0.29.4 → sovereign-0.31.0}/src/sovereign/__init__.py

@@ -1,46 +1,26 @@
 import os
+import sys
 from contextvars import ContextVar
 from importlib.metadata import version
-from typing import Any, Mapping, Type
+from typing import Optional
 
-from fastapi.responses import JSONResponse
 from pydantic import ValidationError
 from starlette.templating import Jinja2Templates
 
-from sovereign import dynamic_config
 from sovereign.context import TemplateContext
 from sovereign.logging.bootstrapper import LoggerBootstrapper
-from sovereign.schemas import SovereignAsgiConfig, SovereignConfig, SovereignConfigv2
+from sovereign.schemas import (
+    SovereignAsgiConfig,
+    SovereignConfig,
+    SovereignConfigv2,
+    migrate_configs,
+    parse_raw_configuration,
+)
 from sovereign.sources import SourcePoller
 from sovereign.statistics import configure_statsd
 from sovereign.utils.crypto.crypto import CipherContainer
-from sovereign.utils.dictupdate import merge  # type: ignore
 from sovereign.utils.resources import get_package_file
 
-json_response_class: Type[JSONResponse] = JSONResponse
-try:
-    import orjson
-    from fastapi.responses import ORJSONResponse
-
-    json_response_class = ORJSONResponse
-except ImportError:
-    try:
-        import ujson
-        from fastapi.responses import UJSONResponse
-
-        json_response_class = UJSONResponse
-    except ImportError:
-        pass
-
-
-def parse_raw_configuration(path: str) -> Mapping[Any, Any]:
-    ret: Mapping[Any, Any] = dict()
-    for p in path.split(","):
-        spec = dynamic_config.Loadable.from_legacy_fmt(p)
-        ret = merge(obj_a=ret, obj_b=spec.load(), merge_lists=True)
-    return ret
-
-
 _request_id_ctx_var: ContextVar[str] = ContextVar("request_id", default="")
 
 
@@ -57,6 +37,9 @@ html_templates = Jinja2Templates(
     directory=str(get_package_file(DIST_NAME, "templates"))
 )
 
+if sys.argv[0].endswith("sovereign"):
+    migrate_configs()
+
 try:
     config = SovereignConfigv2(**parse_raw_configuration(config_path))
 except ValidationError:
@@ -69,15 +52,25 @@ logs = LoggerBootstrapper(config)
 application_logger = logs.application_logger.logger
 
 stats = configure_statsd(config=config.statsd)
-poller = SourcePoller(
-    sources=config.sources,
-    matching_enabled=config.matching.enabled,
-    node_match_key=config.matching.node_key,
-    source_match_key=config.matching.source_key,
-    source_refresh_rate=config.source_config.refresh_rate,
-    logger=application_logger,
-    stats=stats,
-)
+poller = None
+if config.sources is not None:
+    if config.matching is not None:
+        matching_enabled = config.matching.enabled
+        node_key: Optional[str] = config.matching.node_key
+        source_key: Optional[str] = config.matching.source_key
+    else:
+        matching_enabled = False
+        node_key = None
+        source_key = None
+    poller = SourcePoller(
+        sources=config.sources,
+        matching_enabled=matching_enabled,
+        node_match_key=node_key,
+        source_match_key=source_key,
+        source_refresh_rate=config.source_config.refresh_rate,
+        logger=application_logger,
+        stats=stats,
+    )
 
 encryption_configs = config.authentication.encryption_configs
 server_cipher_container = CipherContainer.from_encryption_configs(
@@ -95,5 +88,6 @@ template_context = TemplateContext(
     logger=application_logger,
     stats=stats,
 )
-poller.lazy_load_modifiers(config.modifiers)
-poller.lazy_load_global_modifiers(config.global_modifiers)
+if poller is not None:
+    poller.lazy_load_modifiers(config.modifiers)
+    poller.lazy_load_global_modifiers(config.global_modifiers)

{sovereign-0.29.4 → sovereign-0.31.0}/src/sovereign/app.py

@@ -4,24 +4,22 @@ from collections import namedtuple
 
 import uvicorn
 from fastapi import FastAPI, Request
-from fastapi.responses import RedirectResponse, FileResponse, Response, JSONResponse
+from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, Response
+from starlette_context.middleware import RawContextMiddleware
 
 from sovereign import (
     __version__,
-    config,
     asgi_config,
-    json_response_class,
+    config,
+    logs,
     poller,
     template_context,
-    logs,
 )
+from sovereign.response_class import json_response_class
 from sovereign.error_info import ErrorInfo
+from sovereign.middlewares import LoggingMiddleware, RequestContextLogMiddleware
 from sovereign.utils.resources import get_package_file
-from sovereign.views import crypto, discovery, healthchecks, admin, interface
-from sovereign.middlewares import (
-    RequestContextLogMiddleware,
-    LoggingMiddleware,
-)
+from sovereign.views import admin, crypto, discovery, healthchecks, interface
 
 Router = namedtuple("Router", "module tags prefix")
 
@@ -89,6 +87,8 @@ def init_app() -> FastAPI:
         application.add_middleware(SentryAsgiMiddleware)
         logs.application_logger.logger.info("Sentry middleware enabled")
 
+    application.add_middleware(RawContextMiddleware)
+
     @application.exception_handler(500)
     async def exception_handler(_: Request, exc: Exception) -> JSONResponse:
         """
@@ -109,7 +109,8 @@ def init_app() -> FastAPI:
 
     @application.on_event("startup")
     async def keep_sources_uptodate() -> None:
-        asyncio.create_task(poller.poll_forever())
+        if poller is not None:
+            asyncio.create_task(poller.poll_forever())
 
     @application.on_event("startup")
     async def refresh_template_context() -> None: