sovereign 1.0.0b102__tar.gz → 1.0.0b104__tar.gz

{sovereign-1.0.0b102 → sovereign-1.0.0b104}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sovereign
-Version: 1.0.0b102
+Version: 1.0.0b104
 Summary: Envoy Proxy control-plane written in Python
 Home-page: https://pypi.org/project/sovereign/
 License: Apache-2.0

{sovereign-1.0.0b102 → sovereign-1.0.0b104}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sovereign"
-version = "1.0.0b102"
+version = "1.0.0b104"
 description = "Envoy Proxy control-plane written in Python"
 license = "Apache-2.0"
 packages = [

{sovereign-1.0.0b102 → sovereign-1.0.0b104}/src/sovereign/schemas.py

@@ -44,7 +44,7 @@ class CacheStrategy(str, Enum):
 
 
 class SourceData(BaseModel):
-    scopes: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
+    scopes: Dict[str, List[Dict[str, Any]]] = Field(default_factory=dict)
 
 
 class ConfiguredSource(BaseModel):
@@ -716,6 +716,8 @@ class ContextConfiguration(BaseSettings):
 
 class SourcesConfiguration(BaseSettings):
     refresh_rate: int = Field(30, alias="SOVEREIGN_SOURCES_REFRESH_RATE")
+    max_retries: int = Field(3, alias="SOVEREIGN_SOURCES_MAX_RETRIES")
+    retry_delay: int = Field(1, alias="SOVEREIGN_SOURCES_RETRY_DELAY")
     cache_strategy: Optional[Any] = None
     model_config = SettingsConfigDict(
         env_file=".env",

{sovereign-1.0.0b102 → sovereign-1.0.0b104}/src/sovereign/sources/poller.py

@@ -1,4 +1,5 @@
 import json
+import uuid
 import asyncio
 import traceback
 from copy import deepcopy
@@ -33,29 +34,74 @@ Mods = Dict[str, Type[Modifier]]
 GMods = Dict[str, Type[GlobalModifier]]
 
 
-def item_comparison(old, new):
-    if not isinstance(old, dict) or not isinstance(new, dict):
-        return ["content_changed"]
-
+def _deep_diff(old, new, path="") -> list[dict[str, Any]]:
     changes = []
-    all_keys = set(old.keys()) | set(new.keys())
 
-    for key in sorted(all_keys):
-        old_val = old.get(key)
-        new_val = new.get(key)
+    # handle add/remove
+    if (old, new) == (None, None):
+        return changes
+    elif old is None:
+        changes.append({"op": "add", "path": path, "value": new})
+        return changes
+    elif new is None:
+        changes.append({"op": "remove", "path": path, "old_value": old})
+        return changes
+
+    # handle completely different types
+    if type(old) is not type(new):
+        changes.append(
+            {"op": "change", "path": path, "old_value": old, "new_value": new}
+        )
+        return changes
+
+    # handle fields recursively
+    if isinstance(old, dict) and isinstance(new, dict):
+        all_keys = set(old.keys()) | set(new.keys())
+
+        for key in sorted(all_keys):
+            old_val = old.get(key)
+            new_val = new.get(key)
+
+            current_path = f"{path}.{key}" if path else key
 
-        if old_val != new_val:
             if key not in old:
-                changes.append(f"+{key}:{new_val}")
+                changes.append({"op": "add", "path": current_path, "value": new_val})
             elif key not in new:
-                changes.append(f"-{key}:{old_val}")
-            else:
-                changes.append(f"~{key}:{old_val}→{new_val}")
+                changes.append(
+                    {"op": "remove", "path": current_path, "old_value": old_val}
+                )
+            elif old_val != new_val:
+                nested_changes = _deep_diff(old_val, new_val, current_path)
+                changes.extend(nested_changes)
+
+    # handle items recursively
+    elif isinstance(old, list) and isinstance(new, list):
+        max_len = max(len(old), len(new))
+
+        for i in range(max_len):
+            current_path = f"{path}[{i}]" if path else f"[{i}]"
+
+            if i >= len(old):
+                changes.append({"op": "add", "path": current_path, "value": new[i]})
+            elif i >= len(new):
+                changes.append(
+                    {"op": "remove", "path": current_path, "old_value": old[i]}
+                )
+            elif old[i] != new[i]:
+                nested_changes = _deep_diff(old[i], new[i], current_path)
+                changes.extend(nested_changes)
+
+    # handle primitives
+    else:
+        if old != new:
+            changes.append(
+                {"op": "change", "path": path, "old_value": old, "new_value": new}
+            )
 
     return changes
 
 
-def per_field_diff(old, new):
+def per_field_diff(old, new) -> list[dict[str, Any]]:
     changes = []
     max_len = max(len(old), len(new))
 
@@ -64,45 +110,64 @@ def per_field_diff(old, new):
         new_inst = new[i] if i < len(new) else None
 
         if old_inst is None:
-            changes.append(f"added [index:{i}] {new_inst}")
+            changes.append({"op": "add", "path": f"[{i}]", "value": new_inst})
         elif new_inst is None:
-            changes.append(f"removed [index:{i}] {old_inst}")
+            changes.append({"op": "remove", "path": f"[{i}]", "old_value": old_inst})
         elif old_inst != new_inst:
-            field_changes = item_comparison(old_inst, new_inst)
-            if field_changes:
-                changes.append(f"modified [index:{i}]: {', '.join(field_changes)}")
+            # Use the deep diff with index prefix
+            field_changes = _deep_diff(old_inst, new_inst, f"[{i}]")
+            changes.extend(field_changes)
 
     return changes
 
 
-def source_diff_summary(prev, curr):
+def _gen_uuid(diff_summary: dict[str, Any]) -> str:
+    blob = json.dumps(diff_summary, sort_keys=True, separators=("", ""))
+    return str(uuid.uuid5(uuid.NAMESPACE_DNS, blob))
+
+
+def source_diff_summary(prev, curr) -> dict[str, Any]:
     if prev is None:
-        return [
-            f"scope:{scope} added:{len(instances)} instances"
-            for scope, instances in curr.scopes.items()
-            if instances
-        ]
+        summary = {
+            "type": "initial_load",
+            "scopes": {
+                scope: {"added": len(instances)}
+                for scope, instances in curr.scopes.items()
+                if instances
+            },
+        }
+    else:
+        summary = {"type": "update", "scopes": {}}
+
+        all_scopes = set(prev.scopes.keys()) | set(curr.scopes.keys())
+
+        for scope in sorted(all_scopes):
+            old = prev.scopes.get(scope, [])
+            new = curr.scopes.get(scope, [])
 
-    summary = []
-    all_scopes = set(prev.scopes.keys()) | set(curr.scopes.keys())
+            n_old = len(old)
+            n_new = len(new)
 
-    for scope in sorted(all_scopes):
-        old = prev.scopes.get(scope, [])
-        new = curr.scopes.get(scope, [])
+            scope_changes = {}
 
-        n_old = len(old)
-        n_new = len(new)
+            if n_old == 0 and n_new > 0:
+                scope_changes["added"] = n_new
+            elif n_old > 0 and n_new == 0:
+                scope_changes["removed"] = n_old
+            elif old != new:
+                detailed_changes = per_field_diff(old, new)
+                if detailed_changes:
+                    scope_changes["field_changes"] = detailed_changes
+                    scope_changes["count_change"] = n_new - n_old
 
-        if n_old == 0 and n_new > 0:
-            summary.append(f"scope:{scope} added:{n_new} instances")
-        elif n_old > 0 and n_new == 0:
-            summary.append(f"scope:{scope} removed:{n_old} instances")
-        elif old != new:
-            detailed_changes = per_field_diff(old, new)
-            if detailed_changes:
-                summary.append(f"scope:{scope} changes: {'; '.join(detailed_changes)}")
+            if scope_changes:
+                summary["scopes"][scope] = scope_changes
 
-    return summary if summary else ["no changes detected"]
+        if not summary["scopes"]:
+            summary = {"type": "no_changes"}
+
+    summary["uuid"] = _gen_uuid(summary)
+    return summary
 
 
 class SourcePoller:
@@ -137,13 +202,17 @@ class SourcePoller:
         self.global_modifiers: GMods = dict()
 
         # initially set data and modify
-        self.source_data: SourceData
+        self.source_data: SourceData = SourceData()
+        self.source_data_modified: SourceData = SourceData()
         self.last_updated = datetime.now()
         self.instance_count = 0
 
         self.cache: dict[str, dict[str, list[dict[str, Any]]]] = {}
         self.registry: set[Any] = set()
 
+        # Retry state
+        self.retry_count = 0
+
     @property
     def data_is_stale(self) -> bool:
         return self.last_updated < datetime.now() - timedelta(minutes=2)
@@ -226,20 +295,37 @@ class SourcePoller:
 
     def refresh(self) -> bool:
         self.stats.increment("sources.attempt")
+
+        # Get retry config from global source config
+        max_retries = config.source_config.max_retries
+
         try:
             new = SourceData()
             for source in self.sources:
-                new.scopes[source.scope].extend(source.get())
+                scope = source.scope
+                if scope not in new.scopes:
+                    new.scopes[scope] = []
+                new.scopes[scope].extend(source.get())
         except Exception as e:
+            self.retry_count += 1
             self.logger.error(
-                event="Error while refreshing sources",
+                event=f"Error while refreshing sources (attempt {self.retry_count}/{max_retries})",
                 traceback=[line for line in traceback.format_exc().split("\n")],
                 error=e.__class__.__name__,
                 detail=getattr(e, "detail", "-"),
+                retry_count=self.retry_count,
             )
             self.stats.increment("sources.error")
+
+            if self.retry_count >= max_retries:
+                # Reset retry count for next cycle
+                self.retry_count = 0
+                self.stats.increment("sources.error.final")
             return False
 
+        # Success - reset retry count
+        self.retry_count = 0
+
         # Is the new data the same as what we currently have
         if new == getattr(self, "source_data", None):
             self.stats.increment("sources.unchanged")
@@ -345,6 +431,8 @@ class SourcePoller:
                     or is_debug_request(node_value)
                 )
                 if match:
+                    if scope not in ret.scopes:
+                        ret.scopes[scope] = []
                     ret.scopes[scope].append(instance)
         return ret
 
@@ -406,5 +494,17 @@ class SourcePoller:
         while True:
             try:
                 self.poll()
-            finally:
+
+                # If we have retry count, use exponential backoff for next attempt
+                if self.retry_count > 0:
+                    retry_delay = config.source_config.retry_delay
+                    delay = min(
+                        retry_delay * (2 ** (self.retry_count - 1)),
+                        self.source_refresh_rate,  # Cap at normal refresh rate
+                    )
+                    await asyncio.sleep(delay)
+                else:
+                    await asyncio.sleep(self.source_refresh_rate)
+            except Exception as e:
+                self.logger.error(f"Unexpected error in poll loop: {e}")
                 await asyncio.sleep(self.source_refresh_rate)

{sovereign-1.0.0b102 → sovereign-1.0.0b104}/src/sovereign/statistics.py

@@ -53,10 +53,12 @@ def configure_statsd() -> StatsDProxy:
         from datadog import DogStatsd
 
         class CustomStatsd(DogStatsd):  # type: ignore
-            def _report(self, metric, metric_type, value, tags, sample_rate) -> None:  # type: ignore
-                super()._report(metric, metric_type, value, tags, sample_rate)
-                self.emitted: Dict[str, Any] = dict()
-                self.emitted[metric] = self.emitted.setdefault(metric, 0) + 1
+            def _report(self, *args, **kwargs) -> None:  # type: ignore
+                super()._report(*args, **kwargs)
+                # Capture the metric name and increment its count for debugging
+                if metric := kwargs.get("metric"):
+                    self.emitted: Dict[str, Any] = dict()
+                    self.emitted[metric] = self.emitted.setdefault(metric, 0) + 1
 
         module: Optional[CustomStatsd]
         module = CustomStatsd()

{sovereign-1.0.0b102 → sovereign-1.0.0b104}/src/sovereign/views/api.py

@@ -37,8 +37,8 @@ async def resource(
 ) -> Response:
     expressions = [f"cluster={service_cluster}"]
     try:
-        metadata = json.loads(metadata or "{}")
-        for expr in expand_metadata_to_expr(metadata):
+        data = {"metadata": json.loads(metadata or "{}")}
+        for expr in expand_metadata_to_expr(data):
             expressions.append(expr)
     except Exception:
         pass
@@ -51,7 +51,6 @@ async def resource(
         expressions=expressions,
     )
     req = mock_discovery_request(**{k: v for k, v in kwargs.items() if v is not None})
-    print(req)
     response = await cache.blocking_read(req)
     if content := getattr(response, "text", None):
         return Response(content, media_type="application/json")