sovereign 0.19.3__py3-none-any.whl → 1.0.0b148__py3-none-any.whl

sovereign/server.py

@@ -1,49 +1,107 @@
-import gunicorn.app.base
-from fastapi import FastAPI
-from typing import Optional, Dict, Any, Callable
-from sovereign import asgi_config
-from sovereign.app import app
-from sovereign.utils.entry_point_loader import EntryPointLoader
+import warnings
+import tempfile
+import configparser
+from pathlib import Path
 
+import uvicorn
 
-class StandaloneApplication(gunicorn.app.base.BaseApplication):  # type: ignore
-    _HOOKS = ["pre_fork", "post_fork"]
+from sovereign import application_logger as log
+from sovereign.configuration import SovereignAsgiConfig, SupervisordConfig
 
-    def __init__(
-        self, application: FastAPI, options: Optional[Dict[str, Any]] = None
-    ) -> None:
-        self.loader = EntryPointLoader(*self._HOOKS)
-        self.options = options or {}
-        self.application = application
-        super().__init__()
+asgi_config = SovereignAsgiConfig()
+supervisord_config = SupervisordConfig()
 
-    def load_config(self) -> None:
-        for key, value in self.options.items():
-            self.cfg.set(key.lower(), value)
 
-        for hook in self._HOOKS:
-            self._install_hooks(hook)
+def web() -> None:
+    from sovereign.app import app
 
-    def _install_hooks(self, name: str) -> None:
-        hooks: list[Callable[[Any, Any], None]] = [
-            ep.load() for ep in self.loader.groups[name]
-        ]
+    log.debug("Starting web server")
+    uvicorn.run(
+        app,
+        fd=0,
+        log_level=asgi_config.log_level,
+        access_log=False,
+        timeout_keep_alive=asgi_config.keepalive,
+        host=asgi_config.host,
+        port=asgi_config.port,
+        workers=1,  # per managed supervisor proc
+    )
 
-        def master_hook(server: Any, worker: Any) -> None:
-            for hook in hooks:
-                hook(server, worker)
 
-        self.cfg.set(name, master_hook)
+def worker():
+    from sovereign.worker import worker as worker_app
 
-    def load(self) -> FastAPI:
-        return self.application
+    log.debug("Starting worker")
+    uvicorn.run(
+        worker_app,
+        log_level=asgi_config.log_level,
+        access_log=False,
+        timeout_keep_alive=asgi_config.keepalive,
+        host="127.0.0.1",
+        port=9080,
+        workers=1,  # per managed supervisor proc
+    )
 
 
-def main() -> None:
-    asgi = StandaloneApplication(
-        application=app, options=asgi_config.as_gunicorn_conf()
-    )
-    asgi.run()
+def write_supervisor_conf() -> Path:
+    proc_env = {
+        "LANG": "en_US.UTF-8",
+        "LC_ALL": "en_US.UTF-8",
+    }
+    base = {
+        "autostart": "true",
+        "autorestart": "true",
+        "stdout_logfile": "/dev/stdout",
+        "stdout_logfile_maxbytes": "0",
+        "stderr_logfile": "/dev/stderr",
+        "stderr_logfile_maxbytes": "0",
+        "stopsignal": "QUIT",
+        "environment": ",".join(["=".join((k, v)) for k, v in proc_env.items()]),
+    }
+
+    conf = configparser.RawConfigParser()
+    conf["supervisord"] = supervisord = {
+        "nodaemon": str(supervisord_config.nodaemon).lower(),
+        "loglevel": supervisord_config.loglevel,
+        "pidfile": supervisord_config.pidfile,
+        "logfile": supervisord_config.logfile,
+        "directory": supervisord_config.directory,
+    }
+
+    conf["fcgi-program:web"] = web = {
+        **base,
+        "socket": f"tcp://{asgi_config.host}:{asgi_config.port}",
+        "numprocs": str(asgi_config.workers),
+        "process_name": "%(program_name)s-%(process_num)02d",
+        "command": "sovereign-web",  # default niceness, higher CPU priority
+    }
+
+    conf["program:data"] = worker = {
+        **base,
+        "numprocs": "1",
+        "command": "nice -n 2 sovereign-worker",  # run worker with reduced CPU priority (higher niceness value)
+    }
+
+    if user := asgi_config.user:
+        supervisord["user"] = user
+        web["user"] = user
+        worker["user"] = user
+
+    log.debug("Writing supervisor config")
+    with tempfile.NamedTemporaryFile("w", delete=False) as f:
+        conf.write(f)
+        log.debug("Supervisor config written out")
+        return Path(f.name)
+
+
+def main():
+    path = write_supervisor_conf()
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        from supervisor import supervisord
+
+        log.debug("Starting processes")
+        supervisord.main(["-c", path])
 
 
 if __name__ == "__main__":

sovereign/sources/file.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict
 from sovereign.sources.lib import Source
-from sovereign.config_loader import Loadable
+from sovereign.dynamic_config import Loadable
 
 
 class File(Source):

sovereign/sources/inline.py

@@ -19,6 +19,7 @@ Example configuration (YAML):
                    region: us-east-1
              plan_id: 7d57270a-0348-58d3-829d-447a98fe98d5
 """
+
 from typing import Dict, Any, List
 from sovereign.sources.lib import Source
 

sovereign/sources/lib.py

@@ -6,6 +6,7 @@ used via configuration.
 
 `todo entry point install guide`
 """
+
 import abc
 from typing import Any, Dict, List
 

sovereign/sources/poller.py

@@ -1,3 +1,5 @@
+import json
+import uuid
 import asyncio
 import traceback
 from copy import deepcopy
@@ -6,15 +8,16 @@ from datetime import timedelta, datetime
 from typing import Iterable, Any, Dict, List, Union, Type, Optional
 
 from glom import glom, PathAccessError
+from sovereign.statistics import StatsDProxy
 
+from sovereign.types import Node
+from sovereign.configuration import ConfiguredSource, SourceData, config
 from sovereign.utils.entry_point_loader import EntryPointLoader
 from sovereign.sources.lib import Source
 from sovereign.modifiers.lib import Modifier, GlobalModifier
-from sovereign.schemas import (
-    ConfiguredSource,
-    SourceData,
-    Node,
-)
+from sovereign.events import bus, Topic, Event
+
+from structlog.stdlib import BoundLogger
 
 
 def is_debug_request(v: str, debug: bool = False) -> bool:
@@ -33,15 +36,153 @@ Mods = Dict[str, Type[Modifier]]
 GMods = Dict[str, Type[GlobalModifier]]
 
 
+def _deep_diff(old, new, path="") -> list[dict[str, Any]]:
+    changes: list[dict[str, Any]] = []
+
+    # handle add/remove
+    if (old, new) == (None, None):
+        return changes
+    elif old is None:
+        changes.append({"op": "add", "path": path, "value": new})
+        return changes
+    elif new is None:
+        changes.append({"op": "remove", "path": path, "old_value": old})
+        return changes
+
+    # handle completely different types
+    if type(old) is not type(new):
+        changes.append(
+            {"op": "change", "path": path, "old_value": old, "new_value": new}
+        )
+        return changes
+
+    # handle fields recursively
+    if isinstance(old, dict) and isinstance(new, dict):
+        all_keys = set(old.keys()) | set(new.keys())
+
+        for key in sorted(all_keys):
+            old_val = old.get(key)
+            new_val = new.get(key)
+
+            current_path = f"{path}.{key}" if path else key
+
+            if key not in old:
+                changes.append({"op": "add", "path": current_path, "value": new_val})
+            elif key not in new:
+                changes.append(
+                    {"op": "remove", "path": current_path, "old_value": old_val}
+                )
+            elif old_val != new_val:
+                nested_changes = _deep_diff(old_val, new_val, current_path)
+                changes.extend(nested_changes)
+
+    # handle items recursively
+    elif isinstance(old, list) and isinstance(new, list):
+        max_len = max(len(old), len(new))
+
+        for i in range(max_len):
+            current_path = f"{path}[{i}]" if path else f"[{i}]"
+
+            if i >= len(old):
+                changes.append({"op": "add", "path": current_path, "value": new[i]})
+            elif i >= len(new):
+                changes.append(
+                    {"op": "remove", "path": current_path, "old_value": old[i]}
+                )
+            elif old[i] != new[i]:
+                nested_changes = _deep_diff(old[i], new[i], current_path)
+                changes.extend(nested_changes)
+
+    # handle primitives
+    else:
+        if old != new:
+            changes.append(
+                {"op": "change", "path": path, "old_value": old, "new_value": new}
+            )
+
+    return changes
+
+
+def per_field_diff(old, new) -> list[dict[str, Any]]:
+    changes = []
+    max_len = max(len(old), len(new))
+
+    for i in range(max_len):
+        old_inst = old[i] if i < len(old) else None
+        new_inst = new[i] if i < len(new) else None
+
+        if old_inst is None:
+            changes.append({"op": "add", "path": f"[{i}]", "value": new_inst})
+        elif new_inst is None:
+            changes.append({"op": "remove", "path": f"[{i}]", "old_value": old_inst})
+        elif old_inst != new_inst:
+            # Use the deep diff with index prefix
+            field_changes = _deep_diff(old_inst, new_inst, f"[{i}]")
+            changes.extend(field_changes)
+
+    return changes
+
+
+def _gen_uuid(diff_summary: dict[str, Any]) -> str:
+    blob = json.dumps(diff_summary, sort_keys=True, separators=("", ""))
+    return str(uuid.uuid5(uuid.NAMESPACE_DNS, blob))
+
+
+def source_diff_summary(prev, curr) -> dict[str, Any]:
+    if prev is None:
+        summary = {
+            "type": "initial_load",
+            "scopes": {
+                scope: {"added": len(instances)}
+                for scope, instances in curr.scopes.items()
+                if instances
+            },
+        }
+    else:
+        summary = {"type": "update", "scopes": {}}
+
+        all_scopes = set(prev.scopes.keys()) | set(curr.scopes.keys())
+
+        for scope in sorted(all_scopes):
+            old = prev.scopes.get(scope, [])
+            new = curr.scopes.get(scope, [])
+
+            n_old = len(old)
+            n_new = len(new)
+
+            scope_changes: dict[str, Any] = {}
+
+            if n_old == 0 and n_new > 0:
+                scope_changes["added"] = n_new
+            elif n_old > 0 and n_new == 0:
+                scope_changes["removed"] = n_old
+            elif old != new:
+                detailed_changes = per_field_diff(old, new)
+                if detailed_changes:
+                    scope_changes["field_changes"] = detailed_changes
+                    scope_changes["count_change"] = n_new - n_old
+
+            if scope_changes:
+                summary["scopes"][scope] = scope_changes  # type: ignore
+
+        if not summary["scopes"]:
+            summary = {"type": "no_changes"}
+
+    summary["uuid"] = _gen_uuid(summary)
+    return summary
+
+
 class SourcePoller:
+    stats: StatsDProxy
+
     def __init__(
         self,
         sources: List[ConfiguredSource],
         matching_enabled: bool,
-        node_match_key: str,
-        source_match_key: str,
+        node_match_key: Optional[str],
+        source_match_key: Optional[str],
         source_refresh_rate: int,
-        logger: Any,
+        logger: BoundLogger,
         stats: Any,
     ):
         self.matching_enabled = matching_enabled
@@ -65,9 +206,17 @@ class SourcePoller:
         self.global_modifiers: GMods = dict()
 
         # initially set data and modify
-        self.source_data = self.refresh()
+        self.source_data: SourceData = SourceData()
+        self.source_data_modified: SourceData = SourceData()
         self.last_updated = datetime.now()
         self.instance_count = 0
+        self.initialized = False
+
+        self.cache: dict[str, dict[str, list[dict[str, Any]]]] = {}
+        self.registry: set[Any] = set()
+
+        # Retry state
+        self.retry_count = 0
 
     @property
     def data_is_stale(self) -> bool:
@@ -102,7 +251,7 @@ class SourcePoller:
         ret = dict()
         for entry_point in entry_points:
             if entry_point.name in configured_modifiers:
-                self.logger(event=f"Loading modifier {entry_point.name}")
+                self.logger.debug(f"Loading modifier {entry_point.name}")
                 ret[entry_point.name] = entry_point.load()
         loaded = len(ret)
         configured = len(configured_modifiers)
@@ -118,7 +267,7 @@ class SourcePoller:
         ret = dict()
         for entry_point in entry_points:
             if entry_point.name in configured_modifiers:
-                self.logger(event=f"Loading global modifier {entry_point.name}")
+                self.logger.debug(f"Loading global modifier {entry_point.name}")
                 ret[entry_point.name] = entry_point.load()
 
         loaded = len(ret)
@@ -130,55 +279,98 @@ class SourcePoller:
         return ret
 
     def apply_modifications(self, data: Optional[SourceData]) -> SourceData:
-        with self.stats.timed("modifiers.apply_ms"):
-            if data is None:
-                data = self.source_data
-
-            source_data = deepcopy(data)
-            for scope, instances in source_data.scopes.items():
-                for g in self.global_modifiers.values():
-                    global_modifier = g(instances)
-                    global_modifier.apply()
-                    source_data.scopes[scope] = global_modifier.join()
-
-                for instance in source_data.scopes[scope]:
-                    for m in self.modifiers.values():
-                        modifier = m(instance)
-                        if modifier.match():
-                            # Modifies the instance in-place
-                            modifier.apply()
-            return source_data
-
-    def refresh(self) -> SourceData:
+        if data is None:
+            data = self.source_data
+        if len(self.modifiers) or len(self.global_modifiers):
+            try:
+                with self.stats.timed("modifiers.apply_ms"):
+                    data = deepcopy(data)
+                    for scope, instances in data.scopes.items():
+                        for g in self.global_modifiers.values():
+                            global_modifier = g(instances)
+                            global_modifier.apply()
+                            data.scopes[scope] = global_modifier.join()
+
+                        for instance in data.scopes[scope]:
+                            for m in self.modifiers.values():
+                                modifier = m(instance)
+                                if modifier.match():
+                                    # Modifies the instance in-place
+                                    modifier.apply()
+                self.stats.increment("modifiers.apply.success")
+
+            except Exception:
+                self.stats.increment("modifiers.apply.failure")
+                raise
+
+        return data
+
+    def refresh(self) -> bool:
         self.stats.increment("sources.attempt")
+
+        # Get retry config from global source config
+        max_retries = config.source_config.max_retries
+
         try:
             new = SourceData()
             for source in self.sources:
-                new.scopes[source.scope].extend(source.get())
+                scope = source.scope
+                if scope not in new.scopes:
+                    new.scopes[scope] = []
+                new.scopes[scope].extend(source.get())
         except Exception as e:
-            self.logger(
-                event="Error while refreshing sources",
+            self.retry_count += 1
+            self.logger.error(
+                event=f"Error while refreshing sources (attempt {self.retry_count}/{max_retries})",
                 traceback=[line for line in traceback.format_exc().split("\n")],
                 error=e.__class__.__name__,
                 detail=getattr(e, "detail", "-"),
+                retry_count=self.retry_count,
             )
             self.stats.increment("sources.error")
-            raise
+
+            if self.retry_count >= max_retries:
+                # Reset retry count for next cycle
+                self.retry_count = 0
+                self.stats.increment("sources.error.final")
+            return False
+
+        # Success - reset retry count
+        self.retry_count = 0
 
         # Is the new data the same as what we currently have
         if new == getattr(self, "source_data", None):
             self.stats.increment("sources.unchanged")
             self.last_updated = datetime.now()
-            return self.source_data
+            return False
         else:
             self.stats.increment("sources.refreshed")
             self.last_updated = datetime.now()
+            old_data = getattr(self, "source_data", None)
             self.instance_count = len(
                 [instance for scope in new.scopes.values() for instance in scope]
             )
-            return new
+
+            if config.logging.log_source_diffs:
+                diff_summary = source_diff_summary(old_data, new)
+                # printing json directly because the logger is fucking stupid
+                print(
+                    json.dumps(
+                        dict(
+                            event="Sources refreshed with changes",
+                            level="info",
+                            diff=diff_summary,
+                            total_instances=self.instance_count,
+                        )
+                    )
+                )
+
+            self.source_data = new
+            return True
 
     def extract_node_key(self, node: Union[Node, Dict[Any, Any]]) -> Any:
+        if self.node_match_key is None:
+            return
         if "." not in self.node_match_key:
             # key is not nested, don't need glom
             node_value = getattr(node, self.node_match_key)
@@ -187,13 +379,13 @@ class SourcePoller:
                 node_value = glom(node, self.node_match_key)
             except PathAccessError:
                 raise RuntimeError(
-                    f'Failed to find key "{self.node_match_key}" in discoveryRequest({node}).\n'
-                    f"See the docs for more info: "
-                    f"https://vsyrakis.bitbucket.io/sovereign/docs/html/guides/node_matching.html"
+                    f'Failed to find key "{self.node_match_key}" in discoveryRequest({node})'
                 )
         return node_value
 
     def extract_source_key(self, source: Dict[Any, Any]) -> Any:
+        if self.source_match_key is None:
+            return
         if "." not in self.source_match_key:
             # key is not nested, don't need glom
             source_value = source[self.source_match_key]
@@ -202,9 +394,7 @@ class SourcePoller:
                 source_value = glom(source, self.source_match_key)
             except PathAccessError:
                 raise RuntimeError(
-                    f'Failed to find key "{self.source_match_key}" in instance({source}).\n'
-                    f"See the docs for more info: "
-                    f"https://vsyrakis.bitbucket.io/sovereign/docs/html/guides/node_matching.html"
+                    f'Failed to find key "{self.source_match_key}" in instance({source})'
                 )
         return source_value
 
@@ -221,18 +411,14 @@ class SourcePoller:
         if self.data_is_stale:
             # Log/emit metric and manually refresh sources.
             self.stats.increment("sources.stale")
-            self.logger(
-                event="Sources have not been refreshed in 2 minutes",
+            self.logger.debug(
+                "Sources have not been refreshed in 2 minutes",
                 last_update=self.last_updated,
                 instance_count=self.instance_count,
             )
-            self.poll()
 
         ret = SourceData()
-
         if modify:
-            if not hasattr(self, "source_data_modified"):
-                self.poll()
             data = self.source_data_modified
         else:
             data = self.source_data
@@ -257,6 +443,8 @@ class SourcePoller:
                     or is_debug_request(node_value)
                 )
                 if match:
+                    if scope not in ret.scopes:
+                        ret.scopes[scope] = []
                     ret.scopes[scope].append(instance)
         return ret
 
@@ -284,11 +472,66 @@ class SourcePoller:
                 ret[source_value] = None
         return list(ret.keys())
 
-    def poll(self) -> None:
-        self.source_data = self.refresh()
+    def add_to_context(self, request, output):
+        """middleware for adding matched instances to context"""
+        node_value = self.extract_node_key(request.node)
+        self.registry.add(node_value)
+
+        if instances := self.cache.get(node_value, None):
+            output.update(instances)
+            return
+
+        result = self.get_filtered_instances(node_value)
+        output.update(result)
+
+    def get_filtered_instances(self, node_value):
+        matches = self.match_node(node_value=node_value)
+        result = {}
+        for scope, instances in matches.scopes.items():
+            if scope in ("default", None):
+                result["instances"] = instances
+            else:
+                result[scope] = instances
+        self.cache[node_value] = result
+        return result
+
+    async def poll(self) -> None:
+        updated = self.refresh()
         self.source_data_modified = self.apply_modifications(self.source_data)
+        if not self.initialized:
+            await bus.publish(
+                Topic.CONTEXT,
+                Event(
+                    message="Sources initialized",
+                    metadata={"name": "sources"},
+                ),
+            )
+            self.initialized = True
+        if updated:
+            self.cache.clear()
+            await bus.publish(
+                Topic.CONTEXT,
+                Event(
+                    message="Sources refreshed",
+                    metadata={"name": "sources"},
+                ),
+            )
 
     async def poll_forever(self) -> None:
         while True:
-            self.poll()
-            await asyncio.sleep(self.source_refresh_rate)
+            try:
+                await self.poll()
+
+                # If we have retry count, use exponential backoff for next attempt
+                if self.retry_count > 0:
+                    retry_delay = config.source_config.retry_delay
+                    delay = min(
+                        retry_delay * (2 ** (self.retry_count - 1)),
+                        self.source_refresh_rate,  # Cap at normal refresh rate
+                    )
+                    await asyncio.sleep(delay)
+                else:
+                    await asyncio.sleep(self.source_refresh_rate)
+            except Exception as e:
+                self.logger.error(f"Unexpected error in poll loop: {e}")
+                await asyncio.sleep(self.source_refresh_rate)