sovereign 0.28.0__tar.gz → 0.29.0a3__tar.gz

{sovereign-0.28.0 → sovereign-0.29.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sovereign
-Version: 0.28.0
+Version: 0.29.0a3
 Summary: Envoy Proxy control-plane written in Python
 Home-page: https://pypi.org/project/sovereign/
 License: Apache-2.0

{sovereign-0.28.0 → sovereign-0.29.0a3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sovereign"
-version = "0.28.0"
+version = "0.29.0a3"
 description = "Envoy Proxy control-plane written in Python"
 license = "Apache-2.0"
 packages = [
@@ -41,6 +41,26 @@ sovereign = 'sovereign.server:main'
 
 [tool.poetry.plugins."sovereign.loaders"]
 "example" = "sovereign.testing.loaders:Multiply"
+"file" = "sovereign.dynamic_config.loaders:File"
+"pkgdata" = "sovereign.dynamic_config.loaders:PackageData"
+"http" = "sovereign.dynamic_config.loaders:Web"
+"https" = "sovereign.dynamic_config.loaders:Web"
+"env" = "sovereign.dynamic_config.loaders:EnvironmentVariable"
+"module" = "sovereign.dynamic_config.loaders:PythonModule"
+"s3" = "sovereign.dynamic_config.loaders:S3Bucket"
+"python" = "sovereign.dynamic_config.loaders:PythonInlineCode"
+"inline" = "sovereign.dynamic_config.loaders:Inline"
+
+[tool.poetry.plugins."sovereign.deserializers"]
+"yaml" = "sovereign.dynamic_config.deser:YamlDeserializer"
+"json" = "sovereign.dynamic_config.deser:JsonDeserializer"
+"jinja" = "sovereign.dynamic_config.deser:JinjaDeserializer"
+"jinja2" = "sovereign.dynamic_config.deser:JinjaDeserializer"
+"string" = "sovereign.dynamic_config.deser:StringDeserializer"
+"raw" = "sovereign.dynamic_config.deser:PassthroughDeserializer"
+"ujson" = "sovereign.dynamic_config.deser:UjsonDeserializer"
+"orjson" = "sovereign.dynamic_config.deser:OrjsonDeserializer"
+
 
 [tool.poetry.dependencies]
 python = "^3.11"

{sovereign-0.28.0 → sovereign-0.29.0a3}/src/sovereign/__init__.py

@@ -7,7 +7,7 @@ from fastapi.responses import JSONResponse
 from pydantic import ValidationError
 from starlette.templating import Jinja2Templates
 
-from sovereign import config_loader
+from sovereign import dynamic_config
 from sovereign.context import TemplateContext
 from sovereign.logging.bootstrapper import LoggerBootstrapper
 from sovereign.schemas import SovereignAsgiConfig, SovereignConfig, SovereignConfigv2
@@ -36,7 +36,7 @@ except ImportError:
 def parse_raw_configuration(path: str) -> Mapping[Any, Any]:
     ret: Mapping[Any, Any] = dict()
     for p in path.split(","):
-        spec = config_loader.Loadable.from_legacy_fmt(p)
+        spec = dynamic_config.Loadable.from_legacy_fmt(p)
         ret = merge(obj_a=ret, obj_b=spec.load(), merge_lists=True)
     return ret
 

{sovereign-0.28.0 → sovereign-0.29.0a3}/src/sovereign/configuration.py

@@ -3,7 +3,7 @@ from typing import Any, Mapping
 
 from pydantic import ValidationError
 
-from sovereign import config_loader
+from sovereign import dynamic_config
 from sovereign.context import TemplateContext
 from sovereign.logging.bootstrapper import LoggerBootstrapper
 from sovereign.schemas import SovereignAsgiConfig, SovereignConfig, SovereignConfigv2
@@ -16,7 +16,7 @@ from sovereign.utils.dictupdate import merge  # type: ignore
 def parse_raw_configuration(path: str) -> Mapping[Any, Any]:
     ret: Mapping[Any, Any] = dict()
     for p in path.split(","):
-        spec = config_loader.Loadable.from_legacy_fmt(p)
+        spec = dynamic_config.Loadable.from_legacy_fmt(p)
         ret = merge(obj_a=ret, obj_b=spec.load(), merge_lists=True)
     return ret
 

{sovereign-0.28.0 → sovereign-0.29.0a3}/src/sovereign/context.py

@@ -13,11 +13,9 @@ from typing import (
 )
 
 from fastapi import HTTPException
-from sovereign import config_loader
-from sovereign.utils.entry_point_loader import EntryPointLoader
 from structlog.stdlib import BoundLogger
 
-from sovereign.config_loader import Loadable
+from sovereign.dynamic_config import Loadable
 from sovereign.schemas import DiscoveryRequest, EncryptionConfig, XdsTemplate
 from sovereign.sources import SourcePoller
 from sovereign.utils.crypto.crypto import CipherContainer
@@ -55,14 +53,6 @@ class TemplateContext:
         self.stats = stats
         # initial load
         self.context: Dict[str, Any] = {}
-        entry_points = EntryPointLoader("loaders")
-        for entry_point in entry_points.groups["loaders"]:
-            custom_loader = entry_point.load()
-            try:
-                func = custom_loader.load
-            except AttributeError:
-                raise AttributeError("Custom loader does not implement .load()")
-            config_loader.loaders[entry_point.name] = func
         asyncio.run(self.load_context_variables())
 
     async def start_refresh_context(self) -> NoReturn:

{sovereign-0.28.0 → sovereign-0.29.0a3}/src/sovereign/discovery.py

@@ -23,6 +23,7 @@ except ImportError:
 from sovereign import XDS_TEMPLATES, config, logs, template_context
 from sovereign.utils.version_info import compute_hash
 from sovereign.schemas import XdsTemplate, DiscoveryRequest, ProcessedTemplate
+from sovereign.tracing import Tracer
 
 
 try:
@@ -105,7 +106,8 @@ def response(request: DiscoveryRequest, xds_type: str) -> ProcessedTemplate:
         resource_names=request.resources,
         **template_context.get_context(request, template),
     )
-    content = template(**context)
+    with Tracer("template rendering"):
+        content = template(**context)
 
     # Deserialize YAML output from Jinja2
     if not template.is_python_source:
@@ -116,13 +118,19 @@ def response(request: DiscoveryRequest, xds_type: str) -> ProcessedTemplate:
         content = deserialize_config(content)
 
     # Early return if the template is identical
-    config_version = compute_hash(content)
-    if config_version == request.version_info and not config.discovery_cache.enabled:
-        return ProcessedTemplate(version_info=config_version, resources=[])
-
-    if not isinstance(content, dict):
-        raise RuntimeError(f"Attempting to filter unstructured data: {content}")
-    resources = filter_resources(content["resources"], request.resources)
+    with Tracer("nested test"):
+        with Tracer("hashing"):
+            config_version = compute_hash(content)
+        if (
+            config_version == request.version_info
+            and not config.discovery_cache.enabled
+        ):
+            return ProcessedTemplate(version_info=config_version, resources=[])
+
+        if not isinstance(content, dict):
+            raise RuntimeError(f"Attempting to filter unstructured data: {content}")
+        with Tracer("filtering"):
+            resources = filter_resources(content["resources"], request.resources)
     return ProcessedTemplate(resources=resources, version_info=config_version)
 
 

sovereign-0.29.0a3/src/sovereign/dynamic_config/__init__.py

@@ -0,0 +1,85 @@
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel
+
+from sovereign.utils.entry_point_loader import EntryPointLoader
+from sovereign.dynamic_config.loaders import CustomLoader
+from sovereign.dynamic_config.deser import ConfigDeserializer
+
+
+class Loadable(BaseModel):
+    path: str
+    protocol: str
+    serialization: Optional[str] = None
+
+    def load(self, default: Any = None) -> Any:
+        if self.protocol not in custom_loaders:
+            raise KeyError(
+                f"Could not find CustomLoader {self.protocol}. Available: {custom_loaders}"
+            )
+        loader_ = custom_loaders[self.protocol]
+
+        ser = self.serialization
+        if ser is None:
+            ser = loader_.default_deser
+        if ser not in deserializers:
+            raise KeyError(
+                f"Could not find Deserializer {ser}. Available: {deserializers}"
+            )
+        deserializer = deserializers[ser]
+
+        try:
+            data = loader_.load(self.path)
+            return deserializer.deserialize(data)
+        except Exception as original_error:
+            if default is not None:
+                return default
+            raise Exception(
+                f"{self.protocol=}, {self.path=}, {self.serialization=}, {original_error=}"
+            )
+
+    @staticmethod
+    def from_legacy_fmt(fmt_string: str) -> "Loadable":
+        if "://" not in fmt_string:
+            return Loadable(protocol="inline", serialization="string", path=fmt_string)
+        try:
+            scheme, path = fmt_string.split("://")
+        except ValueError:
+            raise ValueError(fmt_string)
+        try:
+            proto, ser = scheme.split("+")
+        except ValueError:
+            proto, ser = scheme, "yaml"
+
+        if proto in ("python", "module"):
+            ser = "raw"
+        if proto in ("http", "https"):
+            path = "://".join([proto, path])
+
+        return Loadable(
+            protocol=proto,
+            serialization=ser,
+            path=path,
+        )
+
+
+custom_loaders: Dict[str, CustomLoader] = {}
+loader_entry_point = EntryPointLoader("loaders")
+for entry_point in loader_entry_point.groups["loaders"]:
+    custom_loader = entry_point.load()
+    try:
+        func = custom_loader()
+    except AttributeError:
+        raise AttributeError("CustomLoader does not implement .load()")
+    custom_loaders[entry_point.name] = func
+
+
+deserializers: Dict[str, ConfigDeserializer] = {}
+deser_entry_point = EntryPointLoader("deserializers")
+for entry_point in deser_entry_point.groups["deserializers"]:
+    deserializer = entry_point.load()
+    try:
+        func = deserializer()
+    except AttributeError:
+        raise AttributeError("Deserializer does not implement .deserialize()")
+    deserializers[entry_point.name] = func

sovereign-0.29.0a3/src/sovereign/dynamic_config/deser.py

@@ -0,0 +1,79 @@
+import json
+from typing import Any, Protocol
+
+import yaml
+import jinja2
+
+try:
+    import ujson
+
+    UJSON_AVAILABLE = True
+except ImportError:
+    UJSON_AVAILABLE = False
+
+try:
+    import orjson
+
+    ORJSON_AVAILABLE = True
+except ImportError:
+    ORJSON_AVAILABLE = False
+
+jinja_env = jinja2.Environment(autoescape=True)
+
+
+class ConfigDeserializer(Protocol):
+    """
+    Deserializers can be added to sovereign by creating a subclass
+    and then specified in config:
+
+    template_context:
+      context:
+          ...:
+            protocol: ...
+            serialization: <serializer name>
+            path: ...
+    """
+
+    def deserialize(self, input: Any) -> Any:
+        ...
+
+
+class YamlDeserializer(ConfigDeserializer):
+    def deserialize(self, input: Any) -> Any:
+        return yaml.safe_load(input)
+
+
+class JsonDeserializer(ConfigDeserializer):
+    def deserialize(self, input: Any) -> Any:
+        return json.loads(input)
+
+
+class JinjaDeserializer(ConfigDeserializer):
+    def deserialize(self, input: Any) -> Any:
+        return jinja_env.from_string(input)
+
+
+class StringDeserializer(ConfigDeserializer):
+    def deserialize(self, input: Any) -> Any:
+        return str(input)
+
+
+class PassthroughDeserializer(ConfigDeserializer):
+    def deserialize(self, input: Any) -> Any:
+        return input
+
+
+class UjsonDeserializer(ConfigDeserializer):
+    def deserialize(self, input: Any) -> Any:
+        if not UJSON_AVAILABLE:
+            raise ImportError("Configured a UJSON deserializer but it's not installed")
+        return ujson.loads(input)
+
+
+class OrjsonDeserializer(ConfigDeserializer):
+    def deserialize(self, input: Any) -> Any:
+        if not ORJSON_AVAILABLE:
+            raise ImportError(
+                "Configured an ORJSON deserializer but it's not installed"
+            )
+        return orjson.loads(input)

sovereign-0.29.0a3/src/sovereign/dynamic_config/loaders.py

@@ -0,0 +1,121 @@
+import os
+import importlib
+from typing import Any, Protocol
+from pathlib import Path
+from importlib.machinery import SourceFileLoader
+
+import requests
+
+from sovereign.utils.resources import get_package_file_bytes
+
+try:
+    import boto3
+
+    BOTO_IS_AVAILABLE = True
+except ImportError:
+    BOTO_IS_AVAILABLE = False
+
+
+class CustomLoader(Protocol):
+    """
+    Custom loaders can be added to sovereign by creating a subclass
+    and then in config:
+
+    template_context:
+      context:
+          ...:
+            protocol: <loader name>
+            serialization: ...
+            path: <path argument>
+    """
+
+    default_deser: str = "yaml"
+
+    def load(self, path: str) -> Any:
+        ...
+
+
+class File(CustomLoader):
+    default_deser = "passthrough"
+
+    def load(self, path: str) -> Any:
+        with open(path) as f:
+            contents = f.read()
+            try:
+                return contents
+            except FileNotFoundError:
+                raise FileNotFoundError(f"Unable to load {path}")
+
+
+class PackageData(CustomLoader):
+    default_deser = "string"
+
+    def load(self, path: str) -> Any:
+        pkg, pkg_file = path.split(":")
+        data = get_package_file_bytes(pkg, pkg_file)
+        return data
+
+
+class Web(CustomLoader):
+    default_deser = "json"
+
+    def load(self, path: str) -> Any:
+        response = requests.get(path)
+        response.raise_for_status()
+        data = response.text
+        return data
+
+
+class EnvironmentVariable(CustomLoader):
+    default_deser = "raw"
+
+    def load(self, path: str) -> Any:
+        data = os.getenv(path)
+        if data is None:
+            raise AttributeError(f"Unable to read environment variable {path}")
+        return data
+
+
+class PythonModule(CustomLoader):
+    default_deser = "passthrough"
+
+    def load(self, path: str) -> Any:
+        if ":" in path:
+            mod, fn = path.rsplit(":", maxsplit=1)
+        else:
+            mod, fn = path, ""
+        imported = importlib.import_module(mod)
+        if fn != "":
+            return getattr(imported, fn)
+        return imported
+
+
+class S3Bucket(CustomLoader):
+    default_deser = "raw"
+
+    def load(self, path: str) -> Any:
+        if not BOTO_IS_AVAILABLE:
+            raise ImportError(
+                "boto3 must be installed to load S3 paths. Use ``pip install sovereign[boto]``"
+            )
+        bucket, key = path.split("/", maxsplit=1)
+        s3 = boto3.client("s3")
+        response = s3.get_object(Bucket=bucket, Key=key)
+        data = "".join([chunk.decode() for chunk in response["Body"]])
+        return data
+
+
+class PythonInlineCode(CustomLoader):
+    default_deser = "passthrough"
+
+    def load(self, path: str) -> Any:
+        p = str(Path(path).absolute())
+        loader = SourceFileLoader(p, path=p)
+        return loader.load_module(p)
+
+
+class Inline(CustomLoader):
+    default_deser = "string"
+
+    def load(self, path: str) -> Any:
+        return path

{sovereign-0.28.0 → sovereign-0.29.0a3}/src/sovereign/schemas.py

@@ -20,7 +20,8 @@ from pydantic import (
 )
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-from sovereign.config_loader import Loadable, Serialization, jinja_env
+from sovereign.dynamic_config import Loadable
+from sovereign.dynamic_config.deser import jinja_env
 from sovereign.utils.crypto.suites import EncryptionType
 from sovereign.utils.version_info import compute_hash
 
@@ -164,15 +165,15 @@ class XdsTemplate:
             return self.code.render(*args, **kwargs)
 
     def load_source(self) -> str:
-        if self.loadable.serialization in (Serialization.jinja, Serialization.jinja2):
+        old_serialization = self.loadable.serialization
+        if self.loadable.serialization in ("jinja", "jinja2"):
             # The Jinja2 template serializer does not properly set a name
             # for the loaded template.
             # The repr for the template prints out as the memory address
             # This makes it really hard to generate a consistent version_info string
             # in rendered configuration.
             # For this reason, we re-load the template as a string instead, and create a checksum.
-            old_serialization = self.loadable.serialization
-            self.loadable.serialization = Serialization("string")
+            self.loadable.serialization = "string"
             ret = self.loadable.load()
             self.loadable.serialization = old_serialization
             return str(ret)
@@ -180,9 +181,8 @@ class XdsTemplate:
             # If the template specified is a python source file,
             # we can simply read and return the source of it.
             old_protocol = self.loadable.protocol
-            old_serialization = self.loadable.serialization
             self.loadable.protocol = "inline"
-            self.loadable.serialization = Serialization("string")
+            self.loadable.serialization = "string"
             ret = self.loadable.load()
             self.loadable.protocol = old_protocol
             self.loadable.serialization = old_serialization
@@ -458,6 +458,7 @@ class SovereignConfig(BaseSettings):
     log_fmt: Optional[str] = Field("", alias="SOVEREIGN_LOG_FORMAT")
     ignore_empty_log_fields: bool = Field(False, alias="SOVEREIGN_LOG_IGNORE_EMPTY")
     discovery_cache: DiscoveryCacheConfig = DiscoveryCacheConfig()
+    tracing: Optional["TracingConfig"] = None
     model_config = SettingsConfigDict(
         env_file=".env",
         extra="ignore",
@@ -644,6 +645,46 @@ class SourcesConfiguration(BaseSettings):
     )
 
 
+class TracingConfig(BaseSettings):
+    enabled: bool = Field(False)
+    collector: str = Field("notset")
+    endpoint: str = Field("/v2/api/spans")
+    trace_id_128bit: bool = Field(True)
+    tags: Dict[str, Union[Loadable, str]] = dict()
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        extra="ignore",
+        env_file_encoding="utf-8",
+        populate_by_name=True,
+    )
+
+    @field_validator("tags", mode="before")
+    @classmethod
+    def load_tags(cls, v: Dict[str, Union[Loadable, str]]) -> Dict[str, Any]:
+        ret = dict()
+        for key, value in v.items():
+            if isinstance(value, dict):
+                ret[key] = Loadable(**value).load()
+            elif isinstance(value, str):
+                ret[key] = Loadable.from_legacy_fmt(value).load()
+            else:
+                raise ValueError(f"Received an invalid tag for tracing: {value}")
+        return ret
+
+    @model_validator(mode="before")
+    @classmethod
+    def set_environmental_variables(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if enabled := getenv("SOVEREIGN_TRACING_ENABLED"):
+            values["enabled"] = enabled
+        if collector := getenv("SOVEREIGN_TRACING_COLLECTOR"):
+            values["collector"] = collector
+        if endpoint := getenv("SOVEREIGN_TRACING_ENDPOINT"):
+            values["endpoint"] = endpoint
+        if trace_id_128bit := getenv("SOVEREIGN_TRACING_TRACE_ID_128BIT"):
+            values["trace_id_128bit"] = trace_id_128bit
+        return values
+
+
 class LegacyConfig(BaseSettings):
     regions: Optional[List[str]] = None
     eds_priority_matrix: Optional[Dict[str, Dict[str, int]]] = None
@@ -731,6 +772,7 @@ class SovereignConfigv2(BaseSettings):
     debug: bool = Field(False, alias="SOVEREIGN_DEBUG")
     legacy_fields: LegacyConfig = LegacyConfig()
     discovery_cache: DiscoveryCacheConfig = DiscoveryCacheConfig()
+    tracing: TracingConfig = TracingConfig()
     model_config = SettingsConfigDict(
         env_file=".env",
         extra="ignore",
@@ -821,6 +863,7 @@ class SovereignConfigv2(BaseSettings):
             statsd=other.statsd,
             sentry_dsn=SecretStr(other.sentry_dsn),
             debug=other.debug_enabled,
+            tracing=other.tracing,
             legacy_fields=LegacyConfig(
                 regions=other.regions,
                 eds_priority_matrix=other.eds_priority_matrix,

{sovereign-0.28.0 → sovereign-0.29.0a3}/src/sovereign/sources/file.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict
 from sovereign.sources.lib import Source
-from sovereign.config_loader import Loadable
+from sovereign.dynamic_config import Loadable
 
 
 class File(Source):

sovereign-0.29.0a3/src/sovereign/testing/loaders.py

@@ -0,0 +1,8 @@
+from typing import Any
+from sovereign.dynamic_config.loaders import CustomLoader
+
+
+class Multiply(CustomLoader):
+    def load(self, path: str) -> Any:
+        result = path * 2
+        return result

sovereign-0.29.0a3/src/sovereign/tracing.py

@@ -0,0 +1,85 @@
+import time
+import uuid
+import requests
+from contextvars import ContextVar
+from contextlib import nullcontext
+
+from sovereign import config
+
+_trace_id_ctx_var: ContextVar[str] = ContextVar("trace_id", default="")
+_span_id_ctx_var: ContextVar[str] = ContextVar("span_id", default="")
+
+
+def get_trace_id() -> str:
+    return _trace_id_ctx_var.get()
+
+
+def get_span_id() -> str:
+    return _span_id_ctx_var.get()
+
+
+def generate_128bit():
+    return str(uuid.uuid4()).replace("-", "")
+
+
+def generate_64bit():
+    return generate_128bit()[:32]
+
+
+def timestamp():
+    return str(time.time()).replace(".", "")
+
+
+TRACING = config.tracing
+TRACING_DISABLED = not TRACING.enabled
+
+
+class Tracer:
+    def gen_id(self):
+        if TRACING.trace_id_128bit:
+            trace_id = generate_128bit()
+        else:
+            trace_id = generate_64bit()
+        _trace_id_ctx_var.set(trace_id)
+        return trace_id
+
+    def __init__(self, span_name):
+        if TRACING_DISABLED:
+            return
+        span_id = get_span_id()
+        self.parent_span_id = None
+        if span_id != "":
+            # We are already inside a trace context
+            self.parent_span_id = span_id
+        self.trace_id = get_trace_id()
+        self.span_id = self.gen_id()
+        self.span_name = span_name
+
+    def __enter__(self):
+        if TRACING_DISABLED:
+            return nullcontext()
+        self.trace = {
+            "traceId": self.trace_id,
+            "id": self.span_id,
+            "name": self.span_name,
+            "timestamp": time.time(),
+            "tags": TRACING.tags,
+        }
+        if self.parent_span_id:
+            self.trace["parent_span_id"] = self.parent_span_id
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if TRACING_DISABLED:
+            return
+        self.trace["duration"] = time.time() - self.trace["timestamp"]
+        self.submit()
+
+    def submit(self):
+        print(f"{self.span_name}: {self.trace['duration']}")
+        try:
+            url = f"{TRACING.collector}{TRACING.endpoint}"
+            requests.post(url, json=self.trace)
+        # pylint: disable=broad-except
+        except Exception as e:
+            print(f"Failed to submit trace: {self.trace}, Error:{e}")

sovereign-0.28.0/src/sovereign/config_loader.py

@@ -1,222 +0,0 @@
-import os
-import json
-from enum import Enum
-from typing import Any, Dict, Callable, Union, Protocol
-from types import ModuleType
-import yaml
-import jinja2
-import requests
-import importlib
-from importlib.machinery import SourceFileLoader
-from pathlib import Path
-from pydantic import BaseModel
-from sovereign.utils.resources import get_package_file_bytes
-
-
-class Serialization(Enum):
-    """
-    Types of deserialization available in Sovereign
-    for loading configuration field values.
-    """
-
-    yaml = "yaml"
-    json = "json"
-    orjson = "orjson"
-    ujson = "ujson"
-    jinja = "jinja"
-    jinja2 = "jinja2"
-    string = "string"
-    raw = "raw"
-    skip = "skip"
-
-
-jinja_env = jinja2.Environment(autoescape=True)
-
-
-def passthrough(item: Any) -> Any:
-    return item
-
-
-def string(item: Any) -> Any:
-    return str(item)
-
-
-serializers: Dict[Serialization, Callable[[Any], Any]] = {
-    Serialization.yaml: yaml.safe_load,
-    Serialization.json: json.loads,
-    Serialization.jinja: jinja_env.from_string,
-    Serialization.jinja2: jinja_env.from_string,
-    Serialization.string: string,
-    Serialization.raw: passthrough,
-}
-
-
-try:
-    import ujson
-
-    serializers[Serialization.ujson] = ujson.loads
-    jinja_env.policies["json.dumps_function"] = ujson.dumps
-except ImportError:
-    # This lambda will raise an exception when the serializer is used; otherwise we should not crash
-    serializers[Serialization.ujson] = lambda *a, **kw: raise_(
-        ImportError("ujson must be installed to use in config_loaders")
-    )
-
-try:
-    import orjson
-
-    serializers[Serialization.orjson] = orjson.loads
-
-    # orjson.dumps returns bytes, so we have to wrap & decode it
-    def orjson_dumps(*args: Any, **kwargs: Any) -> Any:
-        try:
-            representation = orjson.dumps(*args, **kwargs)
-        except TypeError:
-            raise TypeError(f"Unable to dump objects using ORJSON: {args}, {kwargs}")
-        try:
-            return representation.decode()
-        except Exception as e:
-            raise e.__class__(
-                f"Unable to decode ORJSON: {representation!r}. Original exception: {e}"
-            )
-
-    jinja_env.policies["json.dumps_function"] = orjson_dumps
-    jinja_env.policies["json.dumps_kwargs"] = {"option": orjson.OPT_SORT_KEYS}
-except ImportError:
-    # This lambda will raise an exception when the serializer is used; otherwise we should not crash
-    serializers[Serialization.orjson] = lambda *a, **kw: raise_(
-        ImportError("orjson must be installed to use in config_loaders")
-    )
-
-try:
-    import boto3
-except ImportError:
-    boto3 = None
-
-
-class CustomLoader(Protocol):
-    def load(self, path: str, ser: Serialization) -> Any:
-        ...
-
-
-class Loadable(BaseModel):
-    protocol: str = "http"
-    serialization: Serialization = Serialization.yaml
-    path: str
-
-    def load(self, default: Any = None) -> Any:
-        try:
-            return loaders[self.protocol](self.path, self.serialization)
-        except Exception:
-            if default is not None:
-                return default
-            raise
-
-    @staticmethod
-    def from_legacy_fmt(fmt_string: str) -> "Loadable":
-        if "://" not in fmt_string:
-            return Loadable(
-                protocol="inline", serialization=Serialization.string, path=fmt_string
-            )
-        try:
-            scheme, path = fmt_string.split("://")
-        except ValueError:
-            raise ValueError(fmt_string)
-        try:
-            proto, ser = scheme.split("+")
-        except ValueError:
-            proto, ser = scheme, "yaml"
-
-        serialization: Serialization = Serialization(ser)
-        if proto in ("python", "module"):
-            serialization = Serialization.raw
-        if proto in ("http", "https"):
-            path = "://".join([proto, path])
-
-        return Loadable(
-            protocol=proto,
-            serialization=serialization,
-            path=path,
-        )
-
-
-def raise_(e: Exception) -> Exception:
-    raise e
-
-
-def load_file(path: str, ser: Serialization) -> Any:
-    with open(path) as f:
-        contents = f.read()
-        try:
-            return serializers[ser](contents)
-        except FileNotFoundError:
-            raise FileNotFoundError(f"Unable to load {path}")
-
-
-def load_package_data(path: str, ser: Serialization) -> Any:
-    pkg, pkg_file = path.split(":")
-    data = get_package_file_bytes(pkg, pkg_file)
-    return serializers[ser](data)
-
-
-def load_http(path: str, ser: Serialization) -> Any:
-    response = requests.get(path)
-    response.raise_for_status()
-    data = response.text
-    return serializers[ser](data)
-
-
-def load_env(variable: str, ser: Serialization = Serialization.raw) -> Any:
-    data = os.getenv(variable)
-    try:
-        return serializers[ser](data)
-    except AttributeError as e:
-        raise AttributeError(
-            f"Unable to read environment variable {variable}: {repr(e)}"
-        )
-
-
-def load_module(name: str, _: Serialization = Serialization.raw) -> Any:
-    if ":" in name:
-        mod, fn = name.rsplit(":", maxsplit=1)
-    else:
-        mod, fn = name, ""
-    imported = importlib.import_module(mod)
-    if fn != "":
-        return getattr(imported, fn)
-    return imported
-
-
-def load_s3(path: str, ser: Serialization = Serialization.raw) -> Any:
-    if isinstance(boto3, type(None)):
-        raise ImportError(
-            "boto3 must be installed to load S3 paths. Use ``pip install sovereign[boto]``"
-        )
-    bucket, key = path.split("/", maxsplit=1)
-    s3 = boto3.client("s3")
-    response = s3.get_object(Bucket=bucket, Key=key)
-    data = "".join([chunk.decode() for chunk in response["Body"]])
-    return serializers[ser](data)
-
-
-def load_python(path: str, _: Serialization = Serialization.raw) -> ModuleType:
-    p = str(Path(path).absolute())
-    loader = SourceFileLoader(p, path=p)
-    return loader.load_module(p)
-
-
-def load_inline(path: str, _: Serialization = Serialization.raw) -> Any:
-    return str(path)
-
-
-loaders: Dict[str, Callable[[str, Serialization], Union[str, Any]]] = {
-    "file": load_file,
-    "pkgdata": load_package_data,
-    "http": load_http,
-    "https": load_http,
-    "env": load_env,
-    "module": load_module,
-    "s3": load_s3,
-    "python": load_python,
-    "inline": load_inline,
-}

sovereign-0.28.0/src/sovereign/testing/loaders.py

@@ -1,9 +0,0 @@
-from typing import Any
-from sovereign.config_loader import CustomLoader, Serialization
-
-
-class Multiply(CustomLoader):
-    @staticmethod
-    def load(path: str, ser: Serialization) -> Any:
-        result = path * 2
-        return f"{ser}:{result}"