sovereign 0.14.2__py3-none-any.whl → 1.0.0a4__py3-none-any.whl

sovereign/rendering_common.py

@@ -0,0 +1,91 @@
+import importlib
+from typing import Any
+
+import yaml
+from starlette.exceptions import HTTPException
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
+
+from sovereign import config, logs
+
+type_urls = {
+    "v2": {
+        "listeners": "type.googleapis.com/envoy.api.v2.Listener",
+        "clusters": "type.googleapis.com/envoy.api.v2.Cluster",
+        "endpoints": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+        "secrets": "type.googleapis.com/envoy.api.v2.auth.Secret",
+        "routes": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
+        "scoped-routes": "type.googleapis.com/envoy.api.v2.ScopedRouteConfiguration",
+    },
+    "v3": {
+        "listeners": "type.googleapis.com/envoy.config.listener.v3.Listener",
+        "clusters": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
+        "endpoints": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
+        "secrets": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret",
+        "routes": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
+        "scoped-routes": "type.googleapis.com/envoy.config.route.v3.ScopedRouteConfiguration",
+        "runtime": "type.googleapis.com/envoy.service.runtime.v3.Runtime",
+    },
+}
+
+
+def add_type_urls(api_version, resource_type, resources):
+    type_url = type_urls.get(api_version, {}).get(resource_type)
+    if type_url is not None:
+        for resource in resources:
+            if not resource.get("@type"):
+                resource["@type"] = type_url
+
+
+def deserialize_config(content: str) -> dict[str, Any]:
+    try:
+        envoy_configuration = yaml.safe_load(content)
+    except (ParserError, ScannerError) as e:
+        logs.access_logger.queue_log_fields(
+            error=repr(e),
+            YAML_CONTEXT=e.context,
+            YAML_CONTEXT_MARK=e.context_mark,
+            YAML_NOTE=e.note,
+            YAML_PROBLEM=e.problem,
+            YAML_PROBLEM_MARK=e.problem_mark,
+        )
+
+        if config.sentry_dsn:
+            mod = importlib.import_module("sentry_sdk")
+            mod.capture_exception(e)
+
+        raise HTTPException(
+            status_code=500,
+            detail=(
+                "Failed to load configuration, there may be "
+                "a syntax error in the configured templates. "
+                "Please check Sentry if you have configured Sentry DSN"
+            ),
+        )
+    if not isinstance(envoy_configuration, dict):
+        raise RuntimeError(
+            f"Deserialized configuration is of unexpected format: {envoy_configuration}"
+        )
+    return envoy_configuration
+
+
+def filter_resources(
+    generated: list[dict[str, Any]], requested: list[str]
+) -> list[dict[str, Any]]:
+    """
+    If Envoy specifically requested a resource, this removes everything
+    that does not match the name of the resource.
+    If Envoy did not specifically request anything, every resource is retained.
+    """
+    if len(requested) == 0:
+        return generated
+    return [resource for resource in generated if resource_name(resource) in requested]
+
+
+def resource_name(resource: dict[str, Any]) -> str:
+    name = resource.get("name") or resource.get("cluster_name")
+    if isinstance(name, str):
+        return name
+    raise KeyError(
+        f"Failed to determine the name or cluster_name of the following resource: {resource}"
+    )

sovereign/response_class.py

@@ -0,0 +1,18 @@
+from importlib.util import find_spec
+from typing import Type
+
+from fastapi.responses import JSONResponse
+
+json_response_class: Type[JSONResponse] = JSONResponse
+if find_spec("orjson"):
+    from fastapi.responses import ORJSONResponse
+
+    json_response_class = ORJSONResponse
+
+elif find_spec("ujson"):
+    from fastapi.responses import UJSONResponse
+
+    json_response_class = UJSONResponse
+
+
+__all__ = ["json_response_class"]

sovereign/server.py

@@ -1,31 +1,126 @@
-import gunicorn.app.base
-from fastapi import FastAPI
-from typing import Optional, Dict, Any
-from sovereign import asgi_config
-from sovereign.app import app
-
-
-class StandaloneApplication(gunicorn.app.base.BaseApplication):  # type: ignore
-    def __init__(
-        self, application: FastAPI, options: Optional[Dict[str, Any]] = None
-    ) -> None:
-        self.options = options or {}
-        self.application = application
-        super().__init__()
-
-    def load_config(self) -> None:
-        for key, value in self.options.items():
-            self.cfg.set(key.lower(), value)
-
-    def load(self) -> FastAPI:
-        return self.application
-
-
-def main() -> None:
-    asgi = StandaloneApplication(
-        application=app, options=asgi_config.as_gunicorn_conf()
-    )
-    asgi.run()
+import configparser
+import tempfile
+import warnings
+from pathlib import Path
+
+import uvicorn
+
+from sovereign import application_logger as log
+from sovereign.configuration import SovereignAsgiConfig, SupervisordConfig, config
+from sovereign.v2.worker import Worker
+
+# noinspection PyArgumentList
+asgi_config = SovereignAsgiConfig()
+# noinspection PyArgumentList
+supervisord_config = SupervisordConfig()
+
+
+def web(supervisor_enabled=True) -> None:
+    from sovereign.app import app
+
+    log.debug("Starting web server")
+
+    if not supervisor_enabled:
+        uvicorn.run(
+            app,
+            log_level=asgi_config.log_level,
+            access_log=False,
+            timeout_keep_alive=asgi_config.keepalive,
+            host=asgi_config.host,
+            port=asgi_config.port,
+            workers=1,  # per managed supervisor proc
+        )
+    else:
+        uvicorn.run(
+            app,
+            fd=0,
+            log_level=asgi_config.log_level,
+            access_log=False,
+            timeout_keep_alive=asgi_config.keepalive,
+            host=asgi_config.host,
+            port=asgi_config.port,
+            workers=1,  # per managed supervisor proc
+        )
+
+
+def worker():
+    if config.worker_v2_enabled:
+        log.debug("Starting worker v2")
+        Worker().start()
+    else:
+        from sovereign.worker import worker as worker_app
+
+        log.debug("Starting worker")
+        uvicorn.run(
+            worker_app,
+            log_level=asgi_config.log_level,
+            access_log=False,
+            timeout_keep_alive=asgi_config.keepalive,
+            host="127.0.0.1",
+            port=9080,
+            workers=1,  # per managed supervisor proc
+        )
+
+
+def write_supervisor_conf() -> Path:
+    proc_env = {
+        "LANG": "en_US.UTF-8",
+        "LC_ALL": "en_US.UTF-8",
+    }
+    base = {
+        "autostart": "true",
+        "autorestart": "true",
+        "stdout_logfile": "/dev/stdout",
+        "stdout_logfile_maxbytes": "0",
+        "stderr_logfile": "/dev/stderr",
+        "stderr_logfile_maxbytes": "0",
+        "stopsignal": "QUIT",
+        "environment": ",".join(["=".join((k, v)) for k, v in proc_env.items()]),
+    }
+
+    conf = configparser.RawConfigParser()
+    conf["supervisord"] = supervisord = {
+        "nodaemon": str(supervisord_config.nodaemon).lower(),
+        "loglevel": supervisord_config.loglevel,
+        "pidfile": supervisord_config.pidfile,
+        "logfile": supervisord_config.logfile,
+        "directory": supervisord_config.directory,
+    }
+
+    conf["fcgi-program:web"] = web = {
+        **base,
+        "socket": f"tcp://{asgi_config.host}:{asgi_config.port}",
+        "numprocs": str(asgi_config.workers),
+        "process_name": "%(program_name)s-%(process_num)02d",
+        "command": "sovereign-web",  # default niceness, higher CPU priority
+    }
+
+    conf["program:data"] = worker = {
+        **base,
+        "numprocs": "1",
+        "command": "nice -n 2 sovereign-worker",  # run worker with reduced CPU priority (higher niceness value)
+    }
+
+    if user := asgi_config.user:
+        supervisord["user"] = user
+        web["user"] = user
+        worker["user"] = user
+
+    log.debug("Writing supervisor config")
+    with tempfile.NamedTemporaryFile("w", delete=False) as f:
+        conf.write(f)
+        log.debug("Supervisor config written out")
+        return Path(f.name)
+
+
+def main():
+    path = write_supervisor_conf()
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        from supervisor import supervisord
+
+        log.debug("Starting processes")
+        supervisord.main(["-c", path])
 
 
 if __name__ == "__main__":

sovereign/statistics.py

@@ -1,15 +1,15 @@
 import logging
-from typing import Optional, Any, Callable, Dict
 from functools import wraps
-from sovereign.schemas import StatsdConfig
+from typing import Any, Callable, Dict, Optional
 
-emitted: Dict[str, Any] = dict()
+from sovereign.configuration import config as sovereign_config
+
+STATSD: Dict[str, Optional["StatsDProxy"]] = {"instance": None}
 
 
 class StatsDProxy:
     def __init__(self, statsd_instance: Optional[Any] = None) -> None:
         self.statsd = statsd_instance
-        self.emitted = emitted
 
     def __getattr__(self, item: str) -> Any:
         if self.statsd is not None:
@@ -20,14 +20,12 @@ class StatsDProxy:
             return self.do_nothing
 
     def do_nothing(self, *args: Any, **kwargs: Any) -> None:
-        k = args[0]
-        emitted[k] = emitted.setdefault(k, 0) + 1
+        _ = args[0]
 
 
 class StatsdNoop:
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        k = args[0]
-        emitted[k] = emitted.setdefault(k, 0) + 1
+    def __init__(self, *args, **kwargs):
+        pass
 
     def __enter__(self):  # type: ignore
         return self
@@ -43,21 +41,18 @@ class StatsdNoop:
         return wrapped
 
 
-def configure_statsd(config: StatsdConfig) -> StatsDProxy:
+def configure_statsd() -> StatsDProxy:
+    if STATSD["instance"] is not None:
+        return STATSD["instance"]
+    config = sovereign_config.statsd
     try:
         from datadog import DogStatsd
 
-        class CustomStatsd(DogStatsd):  # type: ignore
-            def _report(self, metric, metric_type, value, tags, sample_rate) -> None:  # type: ignore
-                super()._report(metric, metric_type, value, tags, sample_rate)
-                self.emitted: Dict[str, Any] = dict()
-                self.emitted[metric] = self.emitted.setdefault(metric, 0) + 1
-
-        module: Optional[CustomStatsd]
-        module = CustomStatsd()
-        if config.enabled:
+        module: Optional[DogStatsd]
+        module = DogStatsd()
+        if config.enabled and module:
             module.host = config.host
-            module.port = config.port
+            module.port = int(config.port)
             module.namespace = config.namespace
             module.use_ms = config.use_ms
             for tag, value in config.tags.items():
@@ -71,4 +66,7 @@ def configure_statsd(config: StatsdConfig) -> StatsDProxy:
             raise
         module = None
 
-    return StatsDProxy(module)
+    ret = StatsDProxy(module)
+    if STATSD["instance"] is None:
+        STATSD["instance"] = ret
+    return ret

sovereign/templates/base.html

@@ -1,64 +1,77 @@
 <!DOCTYPE html>
-<html lang="en" class="has-navbar-fixed-top">
+<html lang="en">
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
     <link rel="stylesheet" type="text/css" href="/static/style.css">
     <title>{% block title %}{% endblock %} - Sovereign</title>
+    <script src="/static/darkmode.js"></script>
     {%- block head %}{% endblock %}
 </head>
 <body>
-<div class="columns">
-    <div class="column"></div>
     {%- block nav %}
-        <div class="column is-two-thirds">
-        {%- set active_page = resource_type|default('redirect_to_docs') -%}
-        <nav class="navbar is-primary is-fixed-top" role="navigation" aria-label="main navigation">
-            <div class="navbar-brand">
-                <a class="navbar-item" href="/ui">
-                    <h4 class="title is-4" style="color: white">
-                        sovereign
-                    </h4>
-                </a>
-            </div>
-            <div id="navbar" class="navbar-menu">
-                <div class="navbar-start">
-                    <div class="navbar-item has-dropdown is-hoverable">
-                        <a class="navbar-link">
-                            Resources
-                        </a>
-                        <div class="navbar-dropdown">
-                            {%- for type in all_types %}
-                                <a class="navbar-item{% if type == active_page %} is-active{% endif %}"
-                                   href="/ui/resources/{{ type }}">
-                                    {{ type }}
+    <div class="hero">
+        <div class="hero-body p-4">
+            <a href="#">
+                <h3 class="title is-3" style="margin-bottom: 0px">sovereign</h3>
+                <h7 class="title is-7">version {{ sovereign_version }}</h7>
+            </a>
+        </div>
+    </div>
+    
+    <div class="columns is-gapless" style="min-height: calc(100vh - 120px);">
+        <div class="column is-narrow" style="width: 250px;">
+            <aside class="menu p-4" style="height: 100%;">
+                <div class="mb-4">
+                    <p class="menu-label">RESOURCES</p>
+                    <ul class="menu-list">
+                        {%- set active_page = resource_type|default('redirect_to_docs') -%}
+                        {%- for type in all_types %}
+                            <li style="padding: 2px">
+                                <a href="/ui/resources/{{ type }}" 
+                                   style="padding: 3px; border-radius: 4px"
+                                   {% if type == active_page %}class="is-active has-background-primary has-text-white"{% endif %}>
+                                    {{ type|capitalize }}
                                 </a>
-                            {%- endfor %}
-                        </div>
-                    </div>
+                            </li>
+                        {%- endfor %}
+                    </ul>
                 </div>
-                <div class="navbar-end">
-                    <div class="navbar-item">
-                        <a class="button is-dark" href="/docs">
-                            <span>OpenAPI</span>
-                        </a>
-                    </div>
+                
+                <div class="mb-4">
+                    <p class="menu-label">LINKS</p>
+                    <ul class="menu-list">
+                        <li>
+                            <a href="/docs">OpenAPI Spec</a>
+                        </li>
+                        <li>
+                            <a href="https://developer.atlassian.com/platform/sovereign/">Documentation</a>
+                        </li>
+                        <li>
+                            <a href="https://bitbucket.org/atlassian/sovereign">Repository</a>
+                        </li>
+                    </ul>
                 </div>
+                
+                <p class="menu-label">THEME</p>
+                <button id="dark-mode-toggle" class="button is-small">
+                    <span>🌘</span>
+                </button>
+            </aside>
+        </div>
+        
+        <div class="column">
+            <div class="p-4" style="max-width: 1000px">
+                {%- endblock %}
+                {%- block body %}
+                {% endblock -%}
             </div>
-        </nav>
-    {%- endblock %}
-
-    {%- block subnav %}
-    {%- endblock %}
-
-    {%- block body %}
-    {% endblock -%}
+        </div>
     </div>
-    <div class="column"></div>
-</div>
-<footer class="footer">
-{% block footer %}{% endblock %}
-</footer>
+    
+    <footer class="footer">
+        {% block footer %}{% endblock %}
+    </footer>
 </body>
 </html>