sovereign 1.0.0b101__py3-none-any.whl → 1.0.0b102__py3-none-any.whl

sovereign/context.py

@@ -75,6 +75,7 @@ class TemplateContext:
         new = hash(result)
 
         if old != new:
+            stats.increment("context.updated", tags=[f"context:{name}"])
             self.hashes[name] = new
             # Debounced event notification to the worker
             if self.notify_consumers:

sovereign/schemas.py

@@ -410,7 +410,7 @@ class DiscoveryRequest(BaseModel):
         return f"version={self.envoy_version}, cluster={self.node.cluster}, resource={self.resource_type}, names={self.resources}"
 
     def __str__(self) -> str:
-        return f"DiscoverRequest({self.debug()})"
+        return f"DiscoveryRequest({self.debug()})"
 
 
 class DiscoveryResponse(BaseModel):
@@ -645,6 +645,7 @@ class AccessLogConfiguration(BaseSettings):
 class LoggingConfiguration(BaseSettings):
     application_logs: ApplicationLogConfiguration = ApplicationLogConfiguration()
     access_logs: AccessLogConfiguration = AccessLogConfiguration()
+    log_source_diffs: bool = False
 
 
 class ContextFileCache(BaseSettings):

sovereign/sources/poller.py

@@ -1,3 +1,4 @@
+import json
 import asyncio
 import traceback
 from copy import deepcopy
@@ -7,11 +8,7 @@ from typing import Iterable, Any, Dict, List, Union, Type, Optional
 
 from glom import glom, PathAccessError
 
-from sovereign.schemas import (
-    ConfiguredSource,
-    SourceData,
-    Node,
-)
+from sovereign.schemas import ConfiguredSource, SourceData, Node, config
 from sovereign.utils.entry_point_loader import EntryPointLoader
 from sovereign.sources.lib import Source
 from sovereign.modifiers.lib import Modifier, GlobalModifier
@@ -36,6 +33,78 @@ Mods = Dict[str, Type[Modifier]]
 GMods = Dict[str, Type[GlobalModifier]]
 
 
+def item_comparison(old, new):
+    if not isinstance(old, dict) or not isinstance(new, dict):
+        return ["content_changed"]
+
+    changes = []
+    all_keys = set(old.keys()) | set(new.keys())
+
+    for key in sorted(all_keys):
+        old_val = old.get(key)
+        new_val = new.get(key)
+
+        if old_val != new_val:
+            if key not in old:
+                changes.append(f"+{key}:{new_val}")
+            elif key not in new:
+                changes.append(f"-{key}:{old_val}")
+            else:
+                changes.append(f"~{key}:{old_val}→{new_val}")
+
+    return changes
+
+
+def per_field_diff(old, new):
+    changes = []
+    max_len = max(len(old), len(new))
+
+    for i in range(max_len):
+        old_inst = old[i] if i < len(old) else None
+        new_inst = new[i] if i < len(new) else None
+
+        if old_inst is None:
+            changes.append(f"added [index:{i}] {new_inst}")
+        elif new_inst is None:
+            changes.append(f"removed [index:{i}] {old_inst}")
+        elif old_inst != new_inst:
+            field_changes = item_comparison(old_inst, new_inst)
+            if field_changes:
+                changes.append(f"modified [index:{i}]: {', '.join(field_changes)}")
+
+    return changes
+
+
+def source_diff_summary(prev, curr):
+    if prev is None:
+        return [
+            f"scope:{scope} added:{len(instances)} instances"
+            for scope, instances in curr.scopes.items()
+            if instances
+        ]
+
+    summary = []
+    all_scopes = set(prev.scopes.keys()) | set(curr.scopes.keys())
+
+    for scope in sorted(all_scopes):
+        old = prev.scopes.get(scope, [])
+        new = curr.scopes.get(scope, [])
+
+        n_old = len(old)
+        n_new = len(new)
+
+        if n_old == 0 and n_new > 0:
+            summary.append(f"scope:{scope} added:{n_new} instances")
+        elif n_old > 0 and n_new == 0:
+            summary.append(f"scope:{scope} removed:{n_old} instances")
+        elif old != new:
+            detailed_changes = per_field_diff(old, new)
+            if detailed_changes:
+                summary.append(f"scope:{scope} changes: {'; '.join(detailed_changes)}")
+
+    return summary if summary else ["no changes detected"]
+
+
 class SourcePoller:
     def __init__(
         self,
@@ -179,9 +248,25 @@ class SourcePoller:
         else:
             self.stats.increment("sources.refreshed")
             self.last_updated = datetime.now()
+            old_data = getattr(self, "source_data", None)
             self.instance_count = len(
                 [instance for scope in new.scopes.values() for instance in scope]
             )
+
+            if config.logging.log_source_diffs:
+                diff_summary = source_diff_summary(old_data, new)
+                # printing json directly because the logger is fucking stupid
+                print(
+                    json.dumps(
+                        dict(
+                            event="Sources refreshed with changes",
+                            level="info",
+                            diff=diff_summary,
+                            total_instances=self.instance_count,
+                        )
+                    )
+                )
+
             self.source_data = new
             return True
 

sovereign/templates/resources.html

@@ -14,6 +14,12 @@
       ];
       const resources = resourceNames.map((name, index) => ({ name: name, index: index }));
       let filteredResources = [...resources];
+
+      // Function to set envoy_version cookie and reload page
+      function setEnvoyVersion(version) {
+        document.cookie = `envoy_version=${version}; path=/ui/resources/; max-age=31536000`;
+        window.location.reload();
+      }
     </script>
     <style>
         #filterInput {
@@ -284,7 +290,7 @@
                             </div>
                             {% for v in available_versions %}
                                 <a class="dropdown-item{% if v == version %} is-active{% endif %}"
-                                   href="/ui/set-version?version={{ v }}">
+                                   href="#" onclick="setEnvoyVersion('{{ v }}'); return false;">
                                     {{ v.replace('_', '') }}
                                 </a>
                             {% endfor %}

sovereign/views/api.py

@@ -11,6 +11,20 @@ from sovereign.utils.mock import mock_discovery_request
 router = APIRouter()
 
 
+def _traverse(data, prefix, expressions):
+    for key, value in data.items():
+        path = f"{prefix}.{key}" if prefix else key
+        if isinstance(value, dict):
+            yield from _traverse(value, path, expressions)
+        else:
+            yield f"{path}={value}"
+
+
+def expand_metadata_to_expr(m):
+    exprs = []
+    yield from _traverse(m, "", exprs)
+
+
 @router.get("/resources/{resource_type}", summary="Get resources for a given type")
 async def resource(
     resource_type: DiscoveryTypes = Path(title="xDS Resource type"),
@@ -19,16 +33,25 @@ async def resource(
     service_cluster: Optional[str] = Query("*", title="Envoy Service cluster"),
     region: Optional[str] = Query(None, title="Locality Zone"),
     version: Optional[str] = Query(None, title="Envoy Semantic Version"),
+    metadata: Optional[str] = Query(None, title="Envoy node metadata to filter by"),
 ) -> Response:
+    expressions = [f"cluster={service_cluster}"]
+    try:
+        metadata = json.loads(metadata or "{}")
+        for expr in expand_metadata_to_expr(metadata):
+            expressions.append(expr)
+    except Exception:
+        pass
     kwargs = dict(
         api_version=api_version,
         resource_type=DiscoveryTypes(resource_type).value,
         resource_names=resource_name,
         version=version,
         region=region,
-        expressions=[f"cluster={service_cluster}"],
+        expressions=expressions,
     )
     req = mock_discovery_request(**{k: v for k, v in kwargs.items() if v is not None})
+    print(req)
     response = await cache.blocking_read(req)
     if content := getattr(response, "text", None):
         return Response(content, media_type="application/json")

sovereign/views/interface.py

@@ -5,7 +5,7 @@ from typing import Any, Dict, List
 from fastapi import APIRouter, Cookie, Path, Query
 from fastapi.encoders import jsonable_encoder
 from fastapi.requests import Request
-from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response
+from fastapi.responses import HTMLResponse, JSONResponse, Response
 
 from sovereign import html_templates, cache
 from sovereign.schemas import DiscoveryTypes, XDS_TEMPLATES
@@ -18,6 +18,7 @@ all_types = [t.value for t in DiscoveryTypes]
 
 
 @router.get("/")
+@router.get("/resources")
 async def ui_main(request: Request) -> HTMLResponse:
     try:
         return html_templates.TemplateResponse(
@@ -42,21 +43,6 @@ async def ui_main(request: Request) -> HTMLResponse:
         )
 
 
-@router.get(
-    "/set-version", summary="Filter the UI by a certain Envoy Version (stores a Cookie)"
-)
-async def set_envoy_version(
-    request: Request,
-    version: str = Query(
-        "__any__", title="The clients envoy version to emulate in this XDS request"
-    ),
-) -> Response:
-    url = request.headers.get("Referer", "/ui")
-    response = RedirectResponse(url=url)
-    response.set_cookie(key="envoy_version", value=version)
-    return response
-
-
 @router.get(
     "/resources/{xds_type}", summary="List available resources for a given xDS type"
 )

sovereign/worker.py

@@ -1,8 +1,7 @@
 import asyncio
 from typing import Optional
-from multiprocessing import Process
+from multiprocessing import Process, cpu_count
 from contextlib import asynccontextmanager
-from concurrent.futures import ThreadPoolExecutor
 
 from fastapi import FastAPI, Body
 
@@ -23,10 +22,9 @@ from sovereign.context import NEW_CONTEXT
 
 ClientId = str
 ONDEMAND: asyncio.Queue[tuple[ClientId, DiscoveryRequest]] = asyncio.Queue(100)
-executor = ThreadPoolExecutor(4)
+RENDER_SEMAPHORE = asyncio.Semaphore(cpu_count())
 
 
-# TODO: do something about this ---------------------------------------
 def hidden_field(*args, **kwargs):
     return "(value hidden)"
 
@@ -62,43 +60,37 @@ if config.sources is not None:
     context_middleware.append(poller.add_to_context)
 
 
-if poller is not None:
-    poller.lazy_load_modifiers(config.modifiers)
-    poller.lazy_load_global_modifiers(config.global_modifiers)
-
-template_context.middleware = context_middleware
-# ---------------------------------------------------------------------
-
-
 def render(job: rendering.RenderJob):
     log.debug(f"Spawning render process for {job.id}")
     Process(target=rendering.generate, args=[job]).start()
 
 
-def batch_render(jobs: list[rendering.RenderJob]):
-    log.debug("Spawning batch render process for all clients")
-    Process(target=rendering.batch_generate, args=[jobs]).start()
+async def submit_render(job: rendering.RenderJob):
+    async with RENDER_SEMAPHORE:
+        render(job)
 
 
 async def render_on_event():
     while True:
         # block forever until new context arrives
         await NEW_CONTEXT.wait()
-        stats.increment("template.render_on_event")
         log.debug("New context detected, re-rendering templates")
         try:
             if registered := cache.clients():
                 log.debug("New context detected, re-rendering templates")
-                batch_render(
-                    [
-                        rendering.RenderJob(
-                            id=client,
-                            request=request,
-                            context=template_context.get_context(request),
-                        )
-                        for client, request in registered
-                    ]
-                )
+                jobs = [
+                    rendering.RenderJob(
+                        id=client,
+                        request=request,
+                        context=template_context.get_context(request),
+                    )
+                    for client, request in registered
+                ]
+                tasks = [submit_render(job) for job in jobs]
+                size = len(tasks)
+                stats.increment("template.render_on_event", tags=[f"batch_size:{size}"])
+                await asyncio.gather(*tasks)
+                log.debug(f"Completed rendering {size} jobs")
         finally:
             NEW_CONTEXT.clear()
 
@@ -111,7 +103,7 @@ async def render_on_demand():
         job = rendering.RenderJob(
             id=id, request=request, context=template_context.get_context(request)
         )
-        await asyncio.get_event_loop().run_in_executor(executor, render, job)
+        await submit_render(job)
         ONDEMAND.task_done()
 
 
@@ -124,25 +116,19 @@ async def lifespan(_: FastAPI):
 
     # Template context
     log.debug("Starting context loop")
+    template_context.middleware = context_middleware
     asyncio.create_task(template_context.start())
     await NEW_CONTEXT.wait()  # first refresh finished
 
     # Source polling
     if poller is not None:
         log.debug("Starting source poller")
-        import threading
-
-        threading.Thread(target=poller_thread, args=[poller], daemon=True).start()
+        poller.lazy_load_modifiers(config.modifiers)
+        poller.lazy_load_global_modifiers(config.global_modifiers)
+        asyncio.create_task(poller.poll_forever())
     yield
 
 
-def poller_thread(poller):
-    log.debug("Starting source poller")
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-    loop.run_until_complete(poller.poll_forever())
-
-
 worker = FastAPI(lifespan=lifespan)
 try:
     import sentry_sdk

{sovereign-1.0.0b101.dist-info → sovereign-1.0.0b102.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sovereign
-Version: 1.0.0b101
+Version: 1.0.0b102
 Summary: Envoy Proxy control-plane written in Python
 Home-page: https://pypi.org/project/sovereign/
 License: Apache-2.0

{sovereign-1.0.0b101.dist-info → sovereign-1.0.0b102.dist-info}/RECORD

@@ -2,7 +2,7 @@ sovereign/__init__.py,sha256=m8MVzaMSW4AvAqHvUAsXFdp8Oas5oQ8X7BcVt7Hfcik,1431
 sovereign/app.py,sha256=fsf4Jgni2G4EYJO0oQSWfGRZVDBvE2Yfick4n2YR6K4,4876
 sovereign/cache.py,sha256=P2LAskGxZSFgQ392aKz7auHty57HJNepQ7xl2-iWbBA,3842
 sovereign/constants.py,sha256=qdWD1lTvkaW5JGF7TmZhfksQHlRAJFVqbG7v6JQA9k8,46
-sovereign/context.py,sha256=4rFAocaKquMIGzu72RwEi72AVUJ6Hhtvkp2Nzj9k_YQ,8839
+sovereign/context.py,sha256=aoGJ5k1n8ytCk7jggQ6XTx6Hx_ocy7cEkvGOFS3gzzc,8912
 sovereign/dynamic_config/__init__.py,sha256=0hrI9Y-FzDywEM9Lu6i2mPFhs1c47C096R1B_-E3sKA,3161
 sovereign/dynamic_config/deser.py,sha256=N3iUvDpuNHWjxUbGFydMVKicx4o8DyfvNukorqnQdt8,1834
 sovereign/dynamic_config/loaders.py,sha256=gPkxTL7gep20HIMRvjgOqAdUWqtb3970VBCAcUrIM4c,2915
@@ -17,13 +17,13 @@ sovereign/modifiers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 sovereign/modifiers/lib.py,sha256=Cx0VrpTKbSjb3YmHyG4Jy6YEaPlrwpeqNaom3zu1_hw,2885
 sovereign/rendering.py,sha256=MIA7se7-C4WTWf7xZSgqpf7NvhDT7NkZbR3_G9N1dHI,5015
 sovereign/response_class.py,sha256=beMAFV-4L6DwyWzJzy71GkEW4gb7fzH1jd8-Tul13cU,427
-sovereign/schemas.py,sha256=wCovDUrK4Tsu-eFktfy9Fo13zVmpiaOMxiL-GQBzNUE,37225
+sovereign/schemas.py,sha256=nuiLKq26Fd5MGDMPcTEzyb0QuVuWKaXZRtIdlQ3BzkA,37261
 sovereign/server.py,sha256=3qfcUGaRrTF2GTfGEJgGnPXBiZGis1qxOlpIKvoCyFA,2596
 sovereign/sources/__init__.py,sha256=g9hEpFk8j5i1ApHQpbc9giTyJW41Ppgsqv5P9zGxOJk,78
 sovereign/sources/file.py,sha256=prUThsDCSPNwZaZpkKXhAm-GVRZWbBoGKGU0It4HHXs,690
 sovereign/sources/inline.py,sha256=pP77m7bHjqE3sSoqZthcuw1ARVMf9gooVwbz4B8OAek,1003
 sovereign/sources/lib.py,sha256=0hk_G6mKJrB65WokVZnqF5kdJ3vsQZMNPuJqJO0mBsI,1031
-sovereign/sources/poller.py,sha256=U_5oEnui36ItU7fvIa2JYTNASKRYsRWFpC3P3MmXYfs,11880
+sovereign/sources/poller.py,sha256=sJ4bi6pCPX_cw1RGAUTQfM11bsWLeohEkpzSfFYwUeI,14706
 sovereign/static/node_expression.js,sha256=dL9QLM49jorqavf3Qtye6E1QTWYDT1rFI0tQR1HsiLQ,504
 sovereign/static/panel.js,sha256=i5mGExjv-I4Gtt9dQiTyFwPZa8pg5rXeuTeidXNUiTE,2695
 sovereign/static/sass/style.scss,sha256=tPHPEm3sZeBFGDyyn3pHcA-nbaKT-h-UsSTsf6dHNDU,1158
@@ -31,7 +31,7 @@ sovereign/static/style.css,sha256=vG8HPsbCbPIZfHgy7gSeof97Pnp0okkyaXyJzIEEW-8,44
 sovereign/statistics.py,sha256=gXpQgQOTKqU68loQ_NU1OmxNvsRpAp38RpunjbecIRo,2587
 sovereign/templates/base.html,sha256=5vw3-NmN291pXRdArpCwhSce9bAYBWCJVRhvO5EmE9g,2296
 sovereign/templates/err.html,sha256=a3cEzOqyqWOIe3YxfTEjkxbTfxBxq1knD6GwzEFljfs,603
-sovereign/templates/resources.html,sha256=NYWrfI19HzIVDe9A-DtZkBr5h4o4HCIyXRGZ_D_HwSE,39955
+sovereign/templates/resources.html,sha256=QaZ1S38JhAZg3-PfQS1cAKhCczVLXw9e4pztBrqr4qs,40217
 sovereign/testing/loaders.py,sha256=mcmErhI9ZkJUBZl8jv2qP-PCBRFeAIgyBFlfCgU4Vvk,199
 sovereign/testing/modifiers.py,sha256=7_c2hWXn_sYJ6997N1_uSWtClOikcOzu1yRCY56-l-4,361
 sovereign/tracing.py,sha256=Xo3npgh6yesACSlynv9j6qnXxvYEBzXv5LL4Zkc1QDw,2446
@@ -54,14 +54,14 @@ sovereign/utils/timer.py,sha256=_dUtEasj0BKbWYuQ_T3HFIyjurXXj-La-dNSMAwKMSo,795
 sovereign/utils/version_info.py,sha256=adBfu0z6jsg8E5-BIUjZyBwZvfLASj7fpCpYeIvBeMY,576
 sovereign/utils/weighted_clusters.py,sha256=bPzuRE7Qgvv04HcR2AhMDvBrFlZ8AfteweLKhY9SvWg,1166
 sovereign/views/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sovereign/views/api.py,sha256=xiTk8QWvk8PyUQyYBX4Ufcvo5zBtLvNULDd_ePvt3xI,1498
+sovereign/views/api.py,sha256=T_le6Cfw03KsAc1nx93vZw97uUIFtF86vz9daXR_yUM,2177
 sovereign/views/crypto.py,sha256=7y0eHWtt-bbr2CwHEkH7odPaJ1IEviU-71U-MYJD0Kc,3360
 sovereign/views/discovery.py,sha256=4sVRPWpH8nYwY26GWK9N8maOYUNbIJTrUckZ3MahqAU,1876
 sovereign/views/healthchecks.py,sha256=TaXbxkX679jyQ8v5FxtBa2Qa0Z7KuqQ10WgAqfuVGUc,1743
-sovereign/views/interface.py,sha256=BQ-lBWC0Ob1I40W7XKqPjLwVxRC6pOkZfgl2QsaKZmQ,6796
-sovereign/worker.py,sha256=NveJbhO01W44wCBVVTKw-Em54Fp8hbrHt8dQY8tTPFI,5365
-sovereign-1.0.0b101.dist-info/LICENSE.txt,sha256=2X125zvAb9AYLjCgdMDQZuufhm0kwcg31A8pGKj_-VY,560
-sovereign-1.0.0b101.dist-info/METADATA,sha256=9IQsOnZQ8DZtFtKN4IOafbp16i4f4T9bx783shS_dMY,6268
-sovereign-1.0.0b101.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sovereign-1.0.0b101.dist-info/entry_points.txt,sha256=VKJdnnN_HNL8xYQMXsFXfFmN6QkdXMEk5S964avxQJI,1404
-sovereign-1.0.0b101.dist-info/RECORD,,
+sovereign/views/interface.py,sha256=FmQ7LiUPLSvkEDOKCncrnKMD9g1lJKu-DQNbbyi8mqk,6346
+sovereign/worker.py,sha256=NqXlfi9QRlXcWeUPqHWvnYx2CPldY44iLb_fBJAiZ-4,4983
+sovereign-1.0.0b102.dist-info/LICENSE.txt,sha256=2X125zvAb9AYLjCgdMDQZuufhm0kwcg31A8pGKj_-VY,560
+sovereign-1.0.0b102.dist-info/METADATA,sha256=3vjnWeVBnS-UJ9buN9wk8n9PEgdeCi2hY5r2RFZ4fhc,6268
+sovereign-1.0.0b102.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sovereign-1.0.0b102.dist-info/entry_points.txt,sha256=VKJdnnN_HNL8xYQMXsFXfFmN6QkdXMEk5S964avxQJI,1404
+sovereign-1.0.0b102.dist-info/RECORD,,