sovereign 1.0.0b123__py3-none-any.whl → 1.0.0b134__py3-none-any.whl

sovereign/rendering.py

@@ -7,7 +7,11 @@ Functions used to render and return discovery responses to Envoy proxies.
 The templates are configurable. `todo See ref:Configuration#Templates`
 """
 
-from typing import Any, Dict, List
+import traceback
+from concurrent.futures import ThreadPoolExecutor
+from multiprocessing import Process, Pipe, cpu_count
+from multiprocessing.connection import Connection
+from typing import Any
 
 import yaml
 import pydantic
@@ -21,12 +25,14 @@ try:
 except ImportError:
     SENTRY_INSTALLED = False
 
-from sovereign import config, logs, cache, stats
+from sovereign import config, logs, cache, stats, application_logger as log
 from sovereign.schemas import (
     DiscoveryRequest,
     ProcessedTemplate,
 )
 
+# limit render jobs to number of cores
+POOL = ThreadPoolExecutor(max_workers=cpu_count())
 
 type_urls = {
     "v2": {
@@ -54,42 +60,75 @@ class RenderJob(pydantic.BaseModel):
     request: DiscoveryRequest
     context: dict[str, Any]
 
+    def submit(self):
+        _ = POOL.submit(self._run)
+
+    def _run(self):
+        rx, tx = Pipe()
+        proc = Process(target=generate, args=[self, tx])
+        proc.start()
+        log.debug(
+            (
+                f"Spawning process for id={self.id} "
+                f"max_workers={POOL._max_workers} "
+                f"threads={len(POOL._threads)} "
+                f"shutdown={POOL._shutdown} "
+                f"queue_size={POOL._work_queue.qsize()}"
+            )
+        )
+        proc.join(timeout=30)  # TODO: render timeout configurable
+        while rx.poll(timeout=30):
+            level, message = rx.recv()
+            logger = getattr(log, level)
+            logger(message)
+
 
-def generate(job: RenderJob) -> None:
+def generate(job: RenderJob, tx: Connection) -> None:
     request = job.request
     tags = [f"type:{request.resource_type}"]
-    stats.increment("template.render", tags=tags)
-    with stats.timed("template.render_ms", tags=tags):
-        content = request.template(
-            discovery_request=request,
-            host_header=request.desired_controlplane,
-            resource_names=request.resources,
-            **job.context,
-        )
-        if not request.template.is_python_source:
-            assert isinstance(content, str)
-            content = deserialize_config(content)
-        assert isinstance(content, dict)
-        resources = filter_resources(content["resources"], request.resources)
-        add_type_urls(request.api_version, request.resource_type, resources)
-        response = ProcessedTemplate(resources=resources)
-        cache.write(
-            job.id,
-            cache.Entry(
-                text=response.model_dump_json(indent=None),
-                len=len(response.resources),
-                version=response.version_info,
-                node=request.node,
-            ),
+    try:
+        with stats.timed("template.render_ms", tags=tags):
+            content = request.template(
+                discovery_request=request,
+                host_header=request.desired_controlplane,
+                resource_names=request.resources,
+                **job.context,
+            )
+            if not request.template.is_python_source:
+                assert isinstance(content, str)
+                content = deserialize_config(content)
+            assert isinstance(content, dict)
+            resources = filter_resources(content["resources"], request.resources)
+            add_type_urls(request.api_version, request.resource_type, resources)
+            response = ProcessedTemplate(resources=resources)
+            cache.write(
+                job.id,
+                cache.Entry(
+                    text=response.model_dump_json(indent=None),
+                    len=len(response.resources),
+                    version=response.version_info,
+                    node=request.node,
+                ),
+            )
+        tags.append("result:ok")
+        tx.send(("debug", f"Completed rendering of {request} for {job.id}"))
+    except Exception as e:
+        tx.send(
+            (
+                "error",
+                f"Failed to render job for {job.id}: " + str(traceback.format_exc()),
+            )
         )
+        tags.append("result:err")
+        tags.append(f"error:{e.__class__.__name__.lower()}")
+        if SENTRY_INSTALLED and config.sentry_dsn:
+            sentry_sdk.capture_exception(e)
+    finally:
+        stats.increment("template.render", tags=tags)
+        tx.close()
 
 
-def batch_generate(jobs: list[RenderJob]) -> None:
-    for job in jobs:
-        generate(job)
-
-
-def deserialize_config(content: str) -> Dict[str, Any]:
+def deserialize_config(content: str) -> dict[str, Any]:
     try:
         envoy_configuration = yaml.safe_load(content)
     except (ParserError, ScannerError) as e:
@@ -119,8 +158,8 @@ def deserialize_config(content: str) -> Dict[str, Any]:
 
 
 def filter_resources(
-    generated: List[Dict[str, Any]], requested: List[str]
-) -> List[Dict[str, Any]]:
+    generated: list[dict[str, Any]], requested: list[str]
+) -> list[dict[str, Any]]:
     """
     If Envoy specifically requested a resource, this removes everything
     that does not match the name of the resource.
@@ -131,7 +170,7 @@ def filter_resources(
     return [resource for resource in generated if resource_name(resource) in requested]
 
 
-def resource_name(resource: Dict[str, Any]) -> str:
+def resource_name(resource: dict[str, Any]) -> str:
     name = resource.get("name") or resource.get("cluster_name")
     if isinstance(name, str):
         return name

sovereign/schemas.py

@@ -1,6 +1,8 @@
+from collections import defaultdict
 import os
 import warnings
 import importlib
+import hashlib
 import multiprocessing
 from pathlib import Path
 from enum import Enum
@@ -8,7 +10,18 @@ from os import getenv
 from types import ModuleType
 from dataclasses import dataclass
 from functools import cached_property
-from typing import Any, Dict, List, Mapping, Optional, Self, Tuple, Union, Callable
+from typing import (
+    Any,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Self,
+    Tuple,
+    Union,
+    Callable,
+    cast,
+)
 
 import yaml
 import jmespath
@@ -32,9 +45,6 @@ from sovereign.utils import dictupdate
 from sovereign.utils.version_info import compute_hash
 
 missing_arguments = {"missing", "positional", "arguments:"}
-BASIS = 2166136261
-PRIME = 16777619
-OVERFLOW = 0xFFFFFFFF
 
 
 class CacheStrategy(str, Enum):
@@ -89,44 +99,10 @@ class StatsdConfig(BaseModel):
         return ret
 
 
-class DiscoveryCacheConfig(BaseModel):
-    enabled: bool = False
-    host: str = "localhost"
-    port: int = 6379
-    secure: bool = False
-    protocol: str = "redis://"
-    password: SecretStr = SecretStr("")
-    client_side: bool = True  # True = Try in-memory cache before hitting redis
-    wait_for_connection_timeout: int = 5
-    socket_connect_timeout: int = 5
-    socket_timeout: int = 5
-    max_connections: int = 100
-    retry_on_timeout: bool = True  # Retry connections if they timeout.
-    suppress: bool = False  # False = Don't suppress connection errors. True = suppress connection errors
-    socket_keepalive: bool = True  # Try to keep connections to redis around.
-    ttl: int = 60
-    extra_keys: Dict[str, Any] = {}
-
-    @model_validator(mode="after")
-    def set_default_protocol(self) -> Self:
-        if self.secure:
-            self.protocol = "rediss://"
-        return self
-
-    @model_validator(mode="before")
-    @classmethod
-    def set_environmental_variables(cls, values: Dict[str, Any]) -> Dict[str, Any]:
-        if host := getenv("SOVEREIGN_DISCOVERY_CACHE_REDIS_HOST"):
-            values["host"] = host
-        if port := getenv("SOVEREIGN_DISCOVERY_CACHE_REDIS_PORT"):
-            values["port"] = int(port)
-        if password := getenv("SOVEREIGN_DISCOVERY_CACHE_REDIS_PASSWORD"):
-            values["password"] = SecretStr(password)
-        return values
-
-
 class XdsTemplate(BaseModel):
     path: Union[str, Loadable]
+    resource_type: str
+    depends_on: list[str] = Field(default_factory=list)
 
     @property
     def loadable(self):
@@ -348,37 +324,14 @@ class DiscoveryRequest(BaseModel):
     def resources(self) -> Resources:
         return Resources(self.resource_names)
 
-    @property
-    def default_cache_rules(self):
-        return [
-            # Sovereign internal fields
-            "template.version",
-            "is_internal_request",
-            "desired_controlplane",
-            "resource_type",
-            "api_version",
-            "envoy_version",
-            # Envoy fields from the real Discovery Request
-            "resource_names",
-            "node.cluster",
-            "node.locality",
-        ]
-
-    def cache_key(self, rules: Optional[list[str]] = None):
-        if rules is None:
-            rules = self.default_cache_rules
-        combined = 0
+    def cache_key(self, rules: list[str]) -> str:
         map = self.model_dump()
+        hash = hashlib.sha256()
         for expr in sorted(rules):
-            value = jmespath.search(expr, map)
+            value = cast(str, jmespath.search(expr, map))
             val_str = f"{expr}={repr(value)}"
-            # 32bit FNV hash
-            h = BASIS
-            for c in val_str:
-                h = (h ^ ord(c)) * PRIME
-                h &= OVERFLOW
-            combined ^= h
-        return combined
+            hash.update(val_str.encode())
+        return hash.hexdigest()
 
     @computed_field  # type: ignore[prop-decorator]
     @cached_property
@@ -511,7 +464,6 @@ class SovereignConfig(BaseSettings):
     enable_access_logs: bool = Field(True, alias="SOVEREIGN_ENABLE_ACCESS_LOGS")
     log_fmt: Optional[str] = Field("", alias="SOVEREIGN_LOG_FORMAT")
     ignore_empty_log_fields: bool = Field(False, alias="SOVEREIGN_LOG_IGNORE_EMPTY")
-    discovery_cache: DiscoveryCacheConfig = DiscoveryCacheConfig()
     tracing: Optional["TracingConfig"] = None
     model_config = SettingsConfigDict(
         env_file=".env",
@@ -541,7 +493,8 @@ class SovereignConfig(BaseSettings):
         }  # Special key to hold templates from all versions
         for version, templates in self.templates.items():
             loaded_templates = {
-                _type: XdsTemplate(path=path) for _type, path in templates.items()
+                _type: XdsTemplate(path=path, resource_type=_type)
+                for _type, path in templates.items()
             }
             ret[str(version)] = loaded_templates
             ret["__any__"].update(loaded_templates)
@@ -566,6 +519,7 @@ class SovereignConfig(BaseSettings):
 class TemplateSpecification(BaseModel):
     type: str
     spec: Loadable
+    depends_on: list[str] = Field(default_factory=list)
 
 
 class NodeMatching(BaseSettings):
@@ -671,6 +625,7 @@ class ContextConfiguration(BaseSettings):
     refresh_retry_interval_secs: int = Field(
         10, alias="SOVEREIGN_CONTEXT_REFRESH_RETRY_INTERVAL_SECS"
     )
+    cooldown: int = Field(15, alias="SOVEREIGN_CONTEXT_REFRESH_COOLDOWN")
     model_config = SettingsConfigDict(
         env_file=".env",
         extra="ignore",
@@ -717,7 +672,6 @@ class SourcesConfiguration(BaseSettings):
     refresh_rate: int = Field(30, alias="SOVEREIGN_SOURCES_REFRESH_RATE")
     max_retries: int = Field(3, alias="SOVEREIGN_SOURCES_MAX_RETRIES")
     retry_delay: int = Field(1, alias="SOVEREIGN_SOURCES_RETRY_DELAY")
-    cache_strategy: Optional[Any] = None
     model_config = SettingsConfigDict(
         env_file=".env",
         extra="ignore",
@@ -852,29 +806,78 @@ class LegacyConfig(BaseSettings):
             return None
 
 
+class TemplateConfiguration(BaseModel):
+    default: list[TemplateSpecification]
+    versions: dict[str, list[TemplateSpecification]] = Field(default_factory=dict)
+
+
+def default_hash_rules():
+    return [
+        # Sovereign internal fields
+        "template.version",
+        "is_internal_request",
+        "desired_controlplane",
+        "resource_type",
+        "api_version",
+        "envoy_version",
+        # Envoy fields from the real Discovery Request
+        "resource_names",
+        "node.cluster",
+        "node.locality",
+    ]
+
+
+class CacheBackendConfig(BaseModel):
+    type: str = Field(..., description="Cache backend type (e.g., 'redis', 's3')")
+    config: dict[str, Any] = Field(
+        default_factory=dict, description="Backend-specific configuration"
+    )
+
+
+class CacheConfiguration(BaseModel):
+    hash_rules: list[str] = Field(
+        default_factory=default_hash_rules,
+        description="The set of JMES expressions against incoming Discovery Requests used to form a cache key.",
+    )
+    read_timeout: float = Field(
+        5.0,
+        description="How long to block when trying to read from the cache before giving up",
+    )
+    local_fs_path: str = Field(
+        "/var/run/sovereign_cache",
+        description="Local filesystem cache path. Used to provide fast responses to clients and reduce hits against remote cache backend.",
+    )
+    remote_backend: CacheBackendConfig | None = Field(
+        None, description="Remote cache backend configuration"
+    )
+
+
 class SovereignConfigv2(BaseSettings):
-    templates: Dict[str, List[TemplateSpecification]]
+    # Config generation
+    templates: TemplateConfiguration
     template_context: ContextConfiguration = ContextConfiguration()
+
+    # Web/Discovery
     authentication: AuthConfiguration = AuthConfiguration()
-    logging: LoggingConfiguration = LoggingConfiguration()
-    statsd: StatsdConfig = StatsdConfig()
-    sentry_dsn: SecretStr = Field(SecretStr(""), alias="SOVEREIGN_SENTRY_DSN")
-    discovery_cache: DiscoveryCacheConfig = DiscoveryCacheConfig()
 
-    # Worker stuff
-    caching_rules: Optional[list[str]] = None
-    cache_path: str = Field("/var/run/sovereign_cache", alias="SOVEREIGN_CACHE_PATH")
-    cache_timeout: float = Field(5.0, alias="SOVEREIGN_CACHE_READ_TIMEOUT")
+    # Cache
+    cache: CacheConfiguration = CacheConfiguration()
+
+    # Worker
     worker_host: Optional[str] = Field("localhost", alias="SOVEREIGN_WORKER_HOST")
     worker_port: Optional[int] = Field(9080, alias="SOVEREIGN_WORKER_PORT")
 
-    tracing: Optional[TracingConfig] = Field(default_factory=TracingConfig)
-    debug: bool = Field(False, alias="SOVEREIGN_DEBUG")
-
     # Supervisord settings
     supervisord: SupervisordConfig = SupervisordConfig()
 
-    # Deprecated in 0.30
+    # Misc
+    tracing: Optional[TracingConfig] = Field(default_factory=TracingConfig)
+    debug: bool = Field(False, alias="SOVEREIGN_DEBUG")
+    logging: LoggingConfiguration = LoggingConfiguration()
+    statsd: StatsdConfig = StatsdConfig()
+    sentry_dsn: SecretStr = Field(SecretStr(""), alias="SOVEREIGN_SENTRY_DSN")
+
+    # Planned for removal/deprecated/blocked by circular context usage internally
     sources: Optional[List[ConfiguredSource]] = Field(None, deprecated=True)
     source_config: SourcesConfiguration = Field(
         default_factory=SourcesConfiguration, deprecated=True
@@ -884,6 +887,8 @@ class SovereignConfigv2(BaseSettings):
     )
     modifiers: List[str] = Field(default_factory=list, deprecated=True)
     global_modifiers: List[str] = Field(default_factory=list, deprecated=True)
+
+    # Deprecated, need to migrate off internally
     legacy_fields: LegacyConfig = Field(default_factory=LegacyConfig, deprecated=True)
 
     model_config = SettingsConfigDict(
@@ -898,16 +903,23 @@ class SovereignConfigv2(BaseSettings):
         return self.authentication.auth_passwords.get_secret_value().split(",") or []
 
     def xds_templates(self) -> Dict[str, Dict[str, XdsTemplate]]:
-        ret: Dict[str, Dict[str, XdsTemplate]] = {
-            "__any__": {}
-        }  # Special key to hold templates from all versions
-        for version, template_specs in self.templates.items():
-            loaded_templates = {
-                template.type: XdsTemplate(path=template.spec)
-                for template in template_specs
-            }
-            ret[str(version)] = loaded_templates
-            ret["__any__"].update(loaded_templates)
+        ret: Dict[str, Dict[str, XdsTemplate]] = defaultdict(dict)
+        for template in self.templates.default:
+            ret["default"][template.type] = XdsTemplate(
+                path=template.spec,
+                resource_type=template.type,
+                depends_on=template.depends_on,
+            )
+        for version, templates in self.templates.versions.items():
+            for template in templates:
+                loaded = XdsTemplate(
+                    path=template.spec,
+                    resource_type=template.type,
+                    depends_on=template.depends_on,
+                )
+                ret[version][template.type] = loaded
+                ret["__any__"][template.type] = loaded
+        ret["__any__"].update(ret["default"])
         return ret
 
     def __str__(self) -> str:
@@ -921,7 +933,7 @@ class SovereignConfigv2(BaseSettings):
 
     @staticmethod
     def from_legacy_config(other: SovereignConfig) -> "SovereignConfigv2":
-        new_templates = dict()
+        new_templates = TemplateConfiguration(default=list())
         for version, templates in other.templates.items():
             specs = list()
             for type, path in templates.items():
@@ -934,14 +946,16 @@ class SovereignConfigv2(BaseSettings):
                 else:
                     # Just in case? Although this shouldn't happen
                     specs.append(TemplateSpecification(type=type, spec=path))
-            new_templates[str(version)] = specs
+            if version == "default":
+                new_templates.default = specs
+            else:
+                new_templates.versions[str(version)] = specs
 
         return SovereignConfigv2(
             sources=other.sources,
             templates=new_templates,
             source_config=SourcesConfiguration(
                 refresh_rate=other.sources_refresh_rate,
-                cache_strategy=None,
             ),
             modifiers=other.modifiers,
             global_modifiers=other.global_modifiers,
@@ -983,7 +997,6 @@ class SovereignConfigv2(BaseSettings):
                 dns_hard_fail=other.dns_hard_fail,
                 environment=other.environment,
             ),
-            discovery_cache=other.discovery_cache,
         )
 
 
@@ -1030,20 +1043,9 @@ def parse_raw_configuration(path: str) -> Mapping[Any, Any]:
 
 
 config_path = os.getenv("SOVEREIGN_CONFIG", "file:///etc/sovereign.yaml")
-try:
-    config = SovereignConfigv2(**parse_raw_configuration(config_path))
-except ValidationError:
-    old_config = SovereignConfig(**parse_raw_configuration(config_path))
-    config = SovereignConfigv2.from_legacy_config(old_config)
+config = SovereignConfigv2(**parse_raw_configuration(config_path))
 
 XDS_TEMPLATES = config.xds_templates()
-try:
-    default_templates = XDS_TEMPLATES["default"]
-except KeyError:
-    warnings.warn(
-        "Your configuration should contain default templates. For more details, see "
-        "https://developer.atlassian.com/platform/sovereign/tutorial/templates/#versioning-templates"
-    )
 
 # Create an enum that bases all the available discovery types off what has been configured
 discovery_types = (_type for _type in sorted(XDS_TEMPLATES["__any__"].keys()))

sovereign/server.py

@@ -6,16 +6,15 @@ from pathlib import Path
 import uvicorn
 
 from sovereign import application_logger as log
-from sovereign.app import app
-from sovereign.worker import worker as worker_app
 from sovereign.schemas import SovereignAsgiConfig, SupervisordConfig
 
-
 asgi_config = SovereignAsgiConfig()
 supervisord_config = SupervisordConfig()
 
 
 def web() -> None:
+    from sovereign.app import app
+
     log.debug("Starting web server")
     uvicorn.run(
         app,
@@ -30,6 +29,8 @@ def web() -> None:
 
 
 def worker():
+    from sovereign.worker import worker as worker_app
+
     log.debug("Starting worker")
     uvicorn.run(
         worker_app,

sovereign/sources/poller.py

@@ -13,7 +13,7 @@ from sovereign.schemas import ConfiguredSource, SourceData, Node, config
 from sovereign.utils.entry_point_loader import EntryPointLoader
 from sovereign.sources.lib import Source
 from sovereign.modifiers.lib import Modifier, GlobalModifier
-from sovereign.context import NEW_CONTEXT
+from sovereign.events import bus, Topic, Event
 
 from structlog.stdlib import BoundLogger
 
@@ -206,6 +206,7 @@ class SourcePoller:
         self.source_data_modified: SourceData = SourceData()
         self.last_updated = datetime.now()
         self.instance_count = 0
+        self.initialized = False
 
         self.cache: dict[str, dict[str, list[dict[str, Any]]]] = {}
         self.registry: set[Any] = set()
@@ -483,17 +484,32 @@ class SourcePoller:
         self.cache[node_value] = result
         return result
 
-    def poll(self) -> None:
+    async def poll(self) -> None:
         updated = self.refresh()
         self.source_data_modified = self.apply_modifications(self.source_data)
+        if not self.initialized:
+            await bus.publish(
+                Topic.CONTEXT,
+                Event(
+                    message="Sources initialized",
+                    metadata={"name": "sources"},
+                ),
+            )
+            self.initialized = True
         if updated:
             self.cache.clear()
-            NEW_CONTEXT.set()
+            await bus.publish(
+                Topic.CONTEXT,
+                Event(
+                    message="Sources refreshed",
+                    metadata={"name": "sources"},
+                ),
+            )
 
     async def poll_forever(self) -> None:
         while True:
             try:
-                self.poll()
+                await self.poll()
 
                 # If we have retry count, use exponential backoff for next attempt
                 if self.retry_count > 0:

sovereign/statistics.py

@@ -51,7 +51,7 @@ def configure_statsd() -> StatsDProxy:
         module = DogStatsd()
         if config.enabled and module:
             module.host = config.host
-            module.port = config.port
+            module.port = int(config.port)
             module.namespace = config.namespace
             module.use_ms = config.use_ms
             for tag, value in config.tags.items():