sovereign 0.29.4__py3-none-any.whl → 0.30.0__py3-none-any.whl

sovereign/schemas.py

@@ -1,48 +1,35 @@
-import multiprocessing
 import warnings
+import multiprocessing
 from collections import defaultdict
 from dataclasses import dataclass
 from enum import Enum
 from os import getenv
 from types import ModuleType
-from typing import Any, Dict, List, Optional, Self, Tuple, Type, Union
+from typing import Any, Dict, List, Mapping, Optional, Self, Tuple, Union
 
+import yaml
 from croniter import CroniterBadCronError, croniter
-from fastapi.responses import JSONResponse
 from jinja2 import Template, meta
 from pydantic import (
     BaseModel,
     ConfigDict,
     Field,
     SecretStr,
+    ValidationError,
     model_validator,
     field_validator,
 )
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+from sovereign.response_class import json_response_class
 from sovereign.dynamic_config import Loadable
 from sovereign.dynamic_config.deser import jinja_env
 from sovereign.utils.crypto.suites import EncryptionType
+from sovereign.utils import dictupdate
 from sovereign.utils.version_info import compute_hash
 
 missing_arguments = {"missing", "positional", "arguments:"}
 
-JsonResponseClass: Type[JSONResponse] = JSONResponse
-# pylint: disable=unused-import
-try:
-    import orjson
-    from fastapi.responses import ORJSONResponse
-
-    JsonResponseClass = ORJSONResponse
-except ImportError:
-    try:
-        import ujson
-        from fastapi.responses import UJSONResponse
-
-        JsonResponseClass = UJSONResponse
-    except ImportError:
-        pass
-
 
 class CacheStrategy(str, Enum):
     context = "context"
@@ -206,7 +193,7 @@ class ProcessedTemplate:
         self.resources = resources
         self.version_info = version_info
         self._rendered: Optional[bytes] = None
-        if metadata == None:
+        if metadata is None:
             metadata = []
         self.metadata = metadata
 
@@ -217,7 +204,7 @@ class ProcessedTemplate:
     @property
     def rendered(self) -> bytes:
         if self._rendered is None:
-            result = JsonResponseClass(content="").render(
+            result = json_response_class(content="").render(
                 content={
                     "version_info": self.version,
                     "resources": self.resources,
@@ -381,7 +368,7 @@ class SovereignAsgiConfig(BaseSettings):
     log_level: str = "warning"
     worker_class: str = "uvicorn.workers.UvicornWorker"
     worker_timeout: int = Field(30, alias="SOVEREIGN_WORKER_TIMEOUT")
-    worker_tmp_dir: str = "/dev/shm"
+    worker_tmp_dir: Optional[str] = Field(None, alias="SOVEREIGN_WORKER_TMP_DIR")
     graceful_timeout: Optional[int] = Field(None)
     max_requests: int = Field(0, alias="SOVEREIGN_MAX_REQUESTS")
     max_requests_jitter: int = Field(0, alias="SOVEREIGN_MAX_REQUESTS_JITTER")
@@ -399,7 +386,7 @@ class SovereignAsgiConfig(BaseSettings):
         return self
 
     def as_gunicorn_conf(self) -> Dict[str, Any]:
-        return {
+        ret = {
             "bind": ":".join(map(str, [self.host, self.port])),
             "keepalive": self.keepalive,
             "reuse_port": self.reuse_port,
@@ -409,11 +396,13 @@ class SovereignAsgiConfig(BaseSettings):
             "threads": self.threads,
             "workers": self.workers,
             "worker_class": self.worker_class,
-            "worker_tmp_dir": self.worker_tmp_dir,
             "graceful_timeout": self.graceful_timeout,
             "max_requests": self.max_requests,
             "max_requests_jitter": self.max_requests_jitter,
         }
+        if self.worker_tmp_dir is not None:
+            ret["worker_tmp_dir"] = self.worker_tmp_dir
+        return ret
 
 
 class SovereignConfig(BaseSettings):
@@ -437,7 +426,7 @@ class SovereignConfig(BaseSettings):
         "service_clusters", alias="SOVEREIGN_SOURCE_MATCH_KEY"
     )
     sources_refresh_rate: int = Field(30, alias="SOVEREIGN_SOURCES_REFRESH_RATE")
-    cache_strategy: str = Field("context", alias="SOVEREIGN_CACHE_STRATEGY")
+    cache_strategy: Optional[Any] = Field(None, alias="SOVEREIGN_CACHE_STRATEGY")
     refresh_context: bool = Field(False, alias="SOVEREIGN_REFRESH_CONTEXT")
     context_refresh_rate: Optional[int] = Field(
         None, alias="SOVEREIGN_CONTEXT_REFRESH_RATE"
@@ -461,6 +450,17 @@ class SovereignConfig(BaseSettings):
         populate_by_name=True,
     )
 
+    def __init__(self, *args, **kwargs):
+        warnings.warn(
+            (
+                "This version of Sovereign config is deprecated and will be removed in a future release. "
+                "To migrate, use `sovereign config migrate file:///etc/sovereign.yaml,file://./config/example.yaml`"
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(*args, **kwargs)
+
     @property
     def passwords(self) -> List[str]:
         return self.auth_passwords.split(",") or []
@@ -629,9 +629,7 @@ class ContextConfiguration(BaseSettings):
 
 class SourcesConfiguration(BaseSettings):
     refresh_rate: int = Field(30, alias="SOVEREIGN_SOURCES_REFRESH_RATE")
-    cache_strategy: CacheStrategy = Field(
-        CacheStrategy.context, alias="SOVEREIGN_CACHE_STRATEGY"
-    )
+    cache_strategy: Optional[Any] = None
     model_config = SettingsConfigDict(
         env_file=".env",
         extra="ignore",
@@ -753,21 +751,30 @@ class LegacyConfig(BaseSettings):
 
 
 class SovereignConfigv2(BaseSettings):
-    sources: List[ConfiguredSource]
     templates: Dict[str, List[TemplateSpecification]]
-    source_config: SourcesConfiguration = SourcesConfiguration()
-    modifiers: List[str] = []
-    global_modifiers: List[str] = []
     template_context: ContextConfiguration = ContextConfiguration()
-    matching: NodeMatching = NodeMatching()
     authentication: AuthConfiguration = AuthConfiguration()
     logging: LoggingConfiguration = LoggingConfiguration()
     statsd: StatsdConfig = StatsdConfig()
     sentry_dsn: SecretStr = Field(SecretStr(""), alias="SOVEREIGN_SENTRY_DSN")
-    debug: bool = Field(False, alias="SOVEREIGN_DEBUG")
-    legacy_fields: LegacyConfig = LegacyConfig()
     discovery_cache: DiscoveryCacheConfig = DiscoveryCacheConfig()
-    tracing: TracingConfig = TracingConfig()
+    tracing: Optional[TracingConfig] = Field(default_factory=TracingConfig)
+    debug: bool = Field(False, alias="SOVEREIGN_DEBUG")
+
+    # Deprecated in 0.30
+    sources: Optional[List[ConfiguredSource]] = Field(None, deprecated=True)
+    source_config: SourcesConfiguration = Field(
+        default_factory=SourcesConfiguration, deprecated=True
+    )
+    matching: Optional[NodeMatching] = Field(
+        default_factory=NodeMatching, deprecated=True
+    )
+    modifiers: List[str] = Field(default_factory=list, deprecated=True)
+    global_modifiers: List[str] = Field(default_factory=list, deprecated=True)
+    legacy_fields: LegacyConfig = Field(default_factory=LegacyConfig, deprecated=True)
+
+    expected_service_clusters: List[str] = Field(default_factory=list, deprecated=True)
+
     model_config = SettingsConfigDict(
         env_file=".env",
         extra="ignore",
@@ -823,7 +830,7 @@ class SovereignConfigv2(BaseSettings):
             templates=new_templates,
             source_config=SourcesConfiguration(
                 refresh_rate=other.sources_refresh_rate,
-                cache_strategy=CacheStrategy(other.cache_strategy),
+                cache_strategy=None,
             ),
             modifiers=other.modifiers,
             global_modifiers=other.global_modifiers,
@@ -867,3 +874,45 @@ class SovereignConfigv2(BaseSettings):
             ),
             discovery_cache=other.discovery_cache,
         )
+
+
+def migrate_configs():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Tool to manage configurations.")
+    subparsers = parser.add_subparsers(dest="command", help="Main commands")
+    config_parser = subparsers.add_parser("config", help="Configuration commands")
+    config_subparsers = config_parser.add_subparsers(
+        dest="subcommand", help="Config subcommands"
+    )
+    migrate_parser = config_subparsers.add_parser(
+        "migrate", help="Migrate configuration files"
+    )
+    migrate_parser.add_argument("files", help="Files to migrate")
+    args = parser.parse_args()
+
+    if args.command == "config" and args.subcommand == "migrate":
+
+        def secret_str_representer(dumper, data):
+            return dumper.represent_scalar(
+                "tag:yaml.org,2002:str", data.get_secret_value()
+            )
+
+        yaml.SafeDumper.add_representer(SecretStr, secret_str_representer)
+        try:
+            SovereignConfigv2(**parse_raw_configuration(args.files))
+        except ValidationError:
+            print("Already v2")
+        old_config = SovereignConfig(**parse_raw_configuration(args.files))
+        config = SovereignConfigv2.from_legacy_config(old_config)
+        print(yaml.safe_dump(config.model_dump()))
+        exit(0)
+
+
+def parse_raw_configuration(path: str) -> Mapping[Any, Any]:
+    ret: Mapping[Any, Any] = dict()
+    for p in path.split(","):
+        spec = Loadable.from_legacy_fmt(p)
+        # For some reason mypy is broken here
+        ret = dictupdate.merge(obj_a=ret, obj_b=spec.load(), merge_lists=True)  # type: ignore
+    return ret

sovereign/sources/inline.py

@@ -19,6 +19,7 @@ Example configuration (YAML):
                    region: us-east-1
              plan_id: 7d57270a-0348-58d3-829d-447a98fe98d5
 """
+
 from typing import Dict, Any, List
 from sovereign.sources.lib import Source
 

sovereign/sources/lib.py

@@ -6,6 +6,7 @@ used via configuration.
 
 `todo entry point install guide`
 """
+
 import abc
 from typing import Any, Dict, List
 

sovereign/sources/poller.py

@@ -40,8 +40,8 @@ class SourcePoller:
         self,
         sources: List[ConfiguredSource],
         matching_enabled: bool,
-        node_match_key: str,
-        source_match_key: str,
+        node_match_key: Optional[str],
+        source_match_key: Optional[str],
         source_refresh_rate: int,
         logger: BoundLogger,
         stats: Any,
@@ -181,6 +181,8 @@ class SourcePoller:
             return new
 
     def extract_node_key(self, node: Union[Node, Dict[Any, Any]]) -> Any:
+        if self.node_match_key is None:
+            return
         if "." not in self.node_match_key:
             # key is not nested, don't need glom
             node_value = getattr(node, self.node_match_key)
@@ -189,13 +191,13 @@ class SourcePoller:
                 node_value = glom(node, self.node_match_key)
             except PathAccessError:
                 raise RuntimeError(
-                    f'Failed to find key "{self.node_match_key}" in discoveryRequest({node}).\n'
-                    f"See the docs for more info: "
-                    f"https://vsyrakis.bitbucket.io/sovereign/docs/html/guides/node_matching.html"
+                    f'Failed to find key "{self.node_match_key}" in discoveryRequest({node})'
                 )
         return node_value
 
     def extract_source_key(self, source: Dict[Any, Any]) -> Any:
+        if self.source_match_key is None:
+            return
         if "." not in self.source_match_key:
             # key is not nested, don't need glom
             source_value = source[self.source_match_key]
@@ -204,9 +206,7 @@ class SourcePoller:
                 source_value = glom(source, self.source_match_key)
             except PathAccessError:
                 raise RuntimeError(
-                    f'Failed to find key "{self.source_match_key}" in instance({source}).\n'
-                    f"See the docs for more info: "
-                    f"https://vsyrakis.bitbucket.io/sovereign/docs/html/guides/node_matching.html"
+                    f'Failed to find key "{self.source_match_key}" in instance({source})'
                 )
         return source_value
 

sovereign/static/panel.js

@@ -0,0 +1,56 @@
+document.addEventListener('DOMContentLoaded', function() {
+    function clearSearch() {
+        const searchInput = document.getElementById('searchInput');
+        searchInput.placeholder = '';
+    }
+
+    // Function to hide all panels except active
+    function updateVisibility() {
+        const panelBlocks = document.querySelectorAll('.virtualhost');
+        panelBlocks.forEach(block => {
+            if (!block.classList.contains('is-active')) {
+                block.classList.add('filtered');
+            } else {
+                block.classList.remove('filtered');
+            }
+        });
+    }
+    updateVisibility();
+
+    window.filterTabs = function(element, filter) {
+        const tabs = document.querySelectorAll('.panel-tabs a');
+        tabs.forEach(tab => tab.classList.remove('is-active'));
+        element.classList.add('is-active');
+
+        const virtualHosts = document.querySelectorAll('.virtualhost');
+        if (filter === "all") {
+            virtualHosts.forEach(vh => vh.classList.remove('filtered'));
+        } else {
+            virtualHosts.forEach(vh => {
+                if (vh.getAttribute("data-category") == filter) {
+                    vh.classList.remove('filtered');
+                } else {
+                    vh.classList.add('filtered');
+                }
+            });
+        }
+        clearSearch();
+    };
+
+    const searchInput = document.getElementById('searchInput');
+    searchInput.addEventListener('input', function() {
+        const query = searchInput.value.toLowerCase();
+        const panelBlocks = document.querySelectorAll('.virtualhost');
+        panelBlocks.forEach(block => {
+            const text = block.textContent.toLowerCase();
+            if (text.includes(query)) {
+                block.classList.remove('filtered');
+            } else {
+                block.classList.add('filtered');
+            }
+        });
+    });
+
+    const allTab = document.querySelector('.panel-tabs a.is-active');
+    filterTabs(allTab, 'all');
+});

sovereign/static/search_filter.js

@@ -0,0 +1,20 @@
+function filter_results(id, list) {
+    // Declare variables
+    let input, filter, container, iterable, resource, i, txtValue;
+    input = document.getElementById(id);
+    filter = input.value.toUpperCase();
+
+    container = document.getElementById(list);
+    iterable = container.getElementsByTagName("a");
+
+    // Loop through all list items, and hide those who don't match the search query
+    for (i = 0; i < iterable.length; i++) {
+        resource = iterable[i];
+        txtValue = resource.textContent;
+        if (txtValue.toUpperCase().indexOf(filter) > -1) {
+            iterable[i].style.display = "";
+        } else {
+            iterable[i].style.display = "none";
+        }
+    }
+}

sovereign/templates/resources.html

@@ -1,10 +1,15 @@
 {%- extends 'base.html' %}
-{%- import 'ul_filter.html' as filter %}
 {% block title %}{{ resource_type|capitalize }}{% endblock %}
 
 {% block head %}
     <meta name="last_update" content="{{ last_update }}">
-    {{ filter }}
+    <script src="/static/search_filter.js"></script>
+    <script src="/static/panel.js"></script>
+    <style>
+        .filtered {
+            display: none;
+        }
+    </style>
 {% endblock %}
 
 
@@ -74,7 +79,7 @@
                             type="text"
                             id="search_filter"
                             onkeyup="filter_results('search_filter', 'resources')"
-                            placeholder="Filter resources by any string"
+                            placeholder="Filter {{ resource_type }} by any string"
                     >
                 </label>
                 <span class="icon is-left">
@@ -118,39 +123,37 @@
         </nav>
 
         {% if resource_type == 'routes' %}
-            {% for resource in res %}
-                <nav class="panel" id="virtual_hosts">
-                    <p class="panel-heading">
-                    Virtual Hosts (Route Configuration: {{ resource["name"] }})
-                    </p>
-                    <div class="panel-block">
-                        <p class="control has-icons-left">
-                            <label for="search_filter_virtual_hosts">
-                                <input
-                                        class="input"
-                                        type="text"
-                                        id="search_filter_virtual_hosts"
-                                        onkeyup="filter_results('search_filter_virtual_hosts', 'virtual_hosts')"
-                                        placeholder="Filter resources by any string"
-                                >
-                            </label>
-                            <span class="icon is-left">
-                        <i class="fas fa-search" aria-hidden="true"></i>
-                    </span>
-                        </p>
-                    </div>
-
-                    {% for virtualhost in resource['virtual_hosts'] %}
-                        <a class="panel-block"
-                           href="/ui/resources/routes/{{ resource['name'] }}/{{ virtualhost['name'] }}">
-                        <span class="panel-icon">
-                            <i class="fas fa-arrow-right" aria-hidden="true"></i>
-                        </span>
-                            {{ virtualhost['name'] }}
-                        </a>
-                    {% endfor %}
-                </nav>
-            {% endfor %}
+            <nav class="panel">
+              <p class="panel-heading">Virtual Hosts</p>
+              <div class="panel-block">
+                <p class="control has-icons-left">
+                  <input id="searchInput" class="input" type="text" placeholder="Filter virtual-hosts by any string" />
+                  <span class="icon is-left">
+                    <i class="fas fa-search" aria-hidden="true"></i>
+                  </span>
+                </p>
+              </div>
+              <p class="panel-tabs">
+                <a class="is-active" onclick="filterTabs(this, 'all')">All</a>
+                {% for resource in res %}
+                    <a onclick="filterTabs(this, '{{ resource["name"] }}')">
+                        {{ resource['name'] }}
+                    </a>
+                {% endfor %}
+              </p>
+              {% for resource in res %}
+                  {% for virtualhost in resource['virtual_hosts'] %}
+                      <a class="panel-block virtualhost"
+                         data-category="{{ resource['name'] }}"
+                         href="/ui/resources/routes/{{ resource['name'] }}/{{ virtualhost['name'] }}">
+                      <span class="panel-icon">
+                          <i class="fas fa-arrow-right" aria-hidden="true"></i>
+                      </span>
+                          {{ virtualhost['name'] }}
+                      </a>
+                  {% endfor %}
+              {% endfor %}
+            </nav>
         {% endif %}
     {% else %}
         <span class="panel-icon">

sovereign/tracing.py

@@ -31,55 +31,55 @@ def timestamp():
 
 
 TRACING = config.tracing
-TRACING_DISABLED = not TRACING.enabled
-
-
-class Tracer:
-    def gen_id(self):
-        if TRACING.trace_id_128bit:
-            trace_id = generate_128bit()
-        else:
-            trace_id = generate_64bit()
-        _trace_id_ctx_var.set(trace_id)
-        return trace_id
-
-    def __init__(self, span_name):
-        if TRACING_DISABLED:
-            return
-        span_id = get_span_id()
-        self.parent_span_id = None
-        if span_id != "":
-            # We are already inside a trace context
-            self.parent_span_id = span_id
-        self.trace_id = get_trace_id()
-        self.span_id = self.gen_id()
-        self.span_name = span_name
-
-    def __enter__(self):
-        if TRACING_DISABLED:
-            return nullcontext()
-        self.trace = {
-            "traceId": self.trace_id,
-            "id": self.span_id,
-            "name": self.span_name,
-            "timestamp": time.time(),
-            "tags": TRACING.tags,
-        }
-        if self.parent_span_id:
-            self.trace["parent_span_id"] = self.parent_span_id
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        if TRACING_DISABLED:
-            return
-        self.trace["duration"] = time.time() - self.trace["timestamp"]
-        self.submit()
-
-    def submit(self):
-        print(f"{self.span_name}: {self.trace['duration']}")
-        try:
-            url = f"{TRACING.collector}{TRACING.endpoint}"
-            requests.post(url, json=self.trace)
-        # pylint: disable=broad-except
-        except Exception as e:
-            print(f"Failed to submit trace: {self.trace}, Error:{e}")
+if TRACING is not None:
+    TRACING_DISABLED = not TRACING.enabled
+
+    class Tracer:
+        def gen_id(self):
+            if TRACING.trace_id_128bit:
+                trace_id = generate_128bit()
+            else:
+                trace_id = generate_64bit()
+            _trace_id_ctx_var.set(trace_id)
+            return trace_id
+
+        def __init__(self, span_name):
+            if TRACING_DISABLED:
+                return
+            span_id = get_span_id()
+            self.parent_span_id = None
+            if span_id != "":
+                # We are already inside a trace context
+                self.parent_span_id = span_id
+            self.trace_id = get_trace_id()
+            self.span_id = self.gen_id()
+            self.span_name = span_name
+
+        def __enter__(self):
+            if TRACING_DISABLED:
+                return nullcontext()
+            self.trace = {
+                "traceId": self.trace_id,
+                "id": self.span_id,
+                "name": self.span_name,
+                "timestamp": time.time(),
+                "tags": TRACING.tags,
+            }
+            if self.parent_span_id:
+                self.trace["parent_span_id"] = self.parent_span_id
+            return self
+
+        def __exit__(self, exc_type, exc_value, traceback):
+            if TRACING_DISABLED:
+                return
+            self.trace["duration"] = time.time() - self.trace["timestamp"]
+            self.submit()
+
+        def submit(self):
+            print(f"{self.span_name}: {self.trace['duration']}")
+            try:
+                url = f"{TRACING.collector}{TRACING.endpoint}"
+                requests.post(url, json=self.trace)
+            # pylint: disable=broad-except
+            except Exception as e:
+                print(f"Failed to submit trace: {self.trace}, Error:{e}")

sovereign/utils/crypto/suites/base_cipher.py

@@ -4,23 +4,18 @@ from typing import Any
 
 class CipherSuite(ABC):
     @abstractmethod
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        ...
+    def __init__(self, *args: Any, **kwargs: Any) -> None: ...
 
     @abstractmethod
-    def encrypt(self, data: str) -> str:
-        ...
+    def encrypt(self, data: str) -> str: ...
 
     @abstractmethod
-    def decrypt(self, data: str) -> str:
-        ...
+    def decrypt(self, data: str) -> str: ...
 
     @property
     @abstractmethod
-    def key_available(self) -> bool:
-        ...
+    def key_available(self) -> bool: ...
 
     @classmethod
     @abstractmethod
-    def generate_key(cls) -> bytes:
-        ...
+    def generate_key(cls) -> bytes: ...

sovereign/utils/dictupdate.py

@@ -1,6 +1,7 @@
 # type: ignore
 # pylint: disable=no-name-in-module,too-many-branches
-""" Stolen from the saltstack library """
+"""Stolen from the saltstack library"""
+
 from collections.abc import Mapping
 import copy
 

sovereign/views/admin.py

@@ -92,6 +92,7 @@ def instances(
         title="Whether the sources should run Modifiers/Global Modifiers prior to being returned",
     ),
 ) -> JSONResponse:
+    assert poller is not None  # how else would there be sources
     node = mock_discovery_request(service_cluster=service_cluster).node
     args = {
         "modify": yaml.safe_load(modified),

sovereign/views/crypto.py

@@ -4,7 +4,8 @@ from fastapi import APIRouter, Body
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel, Field
 
-from sovereign import json_response_class, logs, server_cipher_container
+from sovereign import logs, server_cipher_container
+from sovereign.response_class import json_response_class
 from sovereign.schemas import EncryptionConfig
 from sovereign.utils.crypto.crypto import CipherContainer
 from sovereign.utils.crypto.suites import EncryptionType

sovereign/views/healthchecks.py

@@ -2,7 +2,8 @@ from typing import List
 from fastapi import Response
 from fastapi.routing import APIRouter
 from fastapi.responses import PlainTextResponse
-from sovereign import XDS_TEMPLATES, __version__, json_response_class
+from sovereign import XDS_TEMPLATES, __version__
+from sovereign.response_class import json_response_class
 from sovereign.utils.mock import mock_discovery_request
 from sovereign.views.discovery import perform_discovery