psyplus 0.9.2__py3-none-any.whl

psyplus/__init__.py

@@ -0,0 +1,2 @@
+from psyplus.psyplus import YamlSettingsPlus
+from psyplus.env_var_handler import EnvVarHandlerExtended

psyplus/env_var_handler.py

@@ -0,0 +1,245 @@
+from typing import Tuple, List, Dict, Any, get_args, get_origin
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings
+from pydantic_core import PydanticUndefined
+import os
+from inspect import isclass
+from psyplus.field_container import FieldInfoContainer
+from psyplus.utils import (
+    clean_annotation,
+    get_dict_val_key_insensitive,
+    env_to_nested_dict,
+)
+import logging
+
+log = logging.getLogger(__name__)
+######
+# WIP - needs total rewrite
+######
+# This is gargabe atm.
+# pydantic-settings 2.x does most the thing now i tried to achieve here.
+# todo: only thing left is indexes env var e.g. `MY_LIST__0=1`,`MY_DICT__KEY1=2`, `MY_OBJ_LIST__0__OBJ0_ATTR=Hello`
+#
+
+
+class ListitemPlaceholder:
+    pass
+
+
+class EnvVarHandler:
+    def __init__(self, settings: BaseSettings | BaseModel):
+        self.settings: BaseSettings | BaseModel = settings
+        self.settings_as_dict: BaseSettings | BaseModel = settings.model_dump()
+        self.env_var_delimiter, self.env_var_prefix = self._get_env_var_seps()
+        self.env_vars: Dict[str, str] = self._get_env_vars()
+        for key in self.env_vars.keys():
+            val_dic = self.get_value_dict_by_env_var_key(key)
+            if val_dic:
+                self._deep_merge(self.settings_as_dict, val_dic)
+        self.settings = settings.__class__.model_validate(self.settings_as_dict)
+
+    def get_value_dict_by_env_var_key(
+        self, env_var_key: str, default: Any = PydanticUndefined
+    ) -> Dict | None:
+        try:
+            val = self.env_vars[env_var_key]
+        except KeyError:
+            prefix_hint = ""
+            if self.env_var_prefix and env_var_key.startswith(self.env_var_prefix):
+                prefix_hint = f" Did you exclude the prefix? If no, try it whithout the prefix `{self.env_var_prefix}`."
+            raise ValueError(
+                f"No environment var with key `{env_var_key}` found.{prefix_hint}"
+            )
+        env_var_splitted = self._split_env_var(env_var_key)
+        try:
+            result = self._get_value_dict_by_env_var_key(
+                env_var_key_splitted=env_var_splitted,
+                settings=self.settings,
+                value=val,
+            )
+        except KeyError:
+            return None
+        return result
+
+    def _get_env_vars(self) -> Dict[str, str]:
+        """Collect env vars. if applicable filter by prefix and remove prefix for internal use."""
+        result = {}
+        for key, val in os.environ.items():
+            if key.startswith(self.env_var_prefix):
+                non_prefixed_key = key.replace(self.env_var_prefix, "", 1)
+                result[non_prefixed_key] = val
+        return result
+
+    def _get_env_var_seps(self) -> Tuple[str, str]:
+        env_var_delimiter: str = (
+            self.settings.model_config["env_nested_delimiter"]
+            if self.settings.model_config["env_nested_delimiter"]
+            else "__"
+        )
+        env_prefix: str = (
+            self.settings.model_config["env_prefix"]
+            if self.settings.model_config["env_prefix"]
+            else ""
+        )
+        return env_var_delimiter, env_prefix
+
+    def _get_value_dict_by_env_var_key(
+        self,
+        env_var_key_splitted: List[str],
+        settings: BaseSettings | BaseModel | Dict | List = None,
+        annotation: Any = None,
+        value: Any = None,
+        parent_keys: List[str] = None,
+    ) -> Dict:
+        if len(env_var_key_splitted) == 0:
+            return value
+
+        env_var_fragment = env_var_key_splitted[0]
+
+        annotation = clean_annotation(annotation)
+        next_annotation = None
+        if annotation is None or (
+            isclass(annotation) and issubclass(annotation, (BaseSettings, BaseModel))
+        ):
+            if annotation is None:
+                annotation = settings
+            try:
+                key = next(
+                    k
+                    for k in annotation.model_fields.keys()
+                    if k.upper() == env_var_fragment.upper()
+                )
+            except StopIteration:
+                # we found no setting key that matches the env var key.
+                # this env var propably has nothing to do with the settings here
+                raise KeyError(
+                    f"Attr/Key `{env_var_fragment}` not found in `{annotation.__class__}`"
+                )
+            next_annotation = annotation.model_fields[key].annotation
+            next_settings_instance = getattr(settings, key, None)
+            result = {}
+            result[key] = self._get_value_dict_by_env_var_key(
+                env_var_key_splitted=env_var_key_splitted[1:],
+                settings=next_settings_instance,
+                annotation=next_annotation,
+                value=value,
+                parent_keys=env_var_key_splitted,
+            )
+        elif get_origin(annotation) == dict:
+            next_annotation = get_args(annotation)[1]
+
+            next_settings_instance = get_dict_val_key_insensitive(
+                settings, env_var_fragment, None
+            )
+            result = {}
+            result[env_var_fragment.lower()] = self._get_value_dict_by_env_var_key(
+                env_var_key_splitted=env_var_key_splitted[1:],
+                settings=next_settings_instance,
+                annotation=next_annotation,
+                value=value,
+                parent_keys=env_var_key_splitted,
+            )
+        elif annotation == dict:
+            # omg, we are in the wildlands. any dict is allowed.
+            # we just make our best guess by creating a nested dict based on the path fragments
+            if not os.getenv("PSYPLUS_SUPRESS_MISSING_TYPE_WARNING", None) in [
+                "yes",
+                "true",
+                "1",
+            ]:
+                log.warning(
+                    f"Env var key `{self.env_var_delimiter.join(parent_keys)}` is mapped to a simple `dict` annotation (subkey: '{env_var_key_splitted[0]}', value: `{value}`) in the config model `{self.settings.__class__.__name__}`. This is not recommended. "
+                    + "Please use a `Dict[<type>]` annotation."
+                    + "PsYplus is now creating a nested dict based on the path fragments, which maybe is not what you expected."
+                    + "Set env var 'PSYPLUS_SUPRESS_MISSING_TYPE_WARNING=true' to supress this warning."
+                )
+
+            result = env_to_nested_dict(env_var_key_splitted, value)
+
+        elif get_origin(annotation) == list:
+            next_annotation = get_args(annotation)[0]
+            # fill up list with placeholder to respect the env vars given index
+            result = [ListitemPlaceholder] * int(env_var_fragment)
+            try:
+                next_settings_instance = (
+                    settings[int(env_var_fragment)] if settings is not None else None
+                )
+            except IndexError:
+                next_settings_instance = None
+
+            result.append(
+                self._get_value_dict_by_env_var_key(
+                    env_var_key_splitted=env_var_key_splitted[1:],
+                    settings=next_settings_instance,
+                    annotation=next_annotation,
+                    value=value,
+                    parent_keys=env_var_key_splitted,
+                )
+            )
+        elif annotation == list:
+            # omg, any list is allowed.
+            # this is stupid. lets output a warning and just make a simple list the value
+            if not os.getenv("PSYPLUS_SUPRESS_MISSING_TYPE_WARNING", None) in [
+                "yes",
+                "true",
+                "1",
+            ]:
+                log.warning(
+                    f"Env var key `{self.env_var_delimiter.join(parent_keys)}` is mapped to a simple `list` annotation. This is not recommended. "
+                    + "Please use a `List[<type>]` annotation. "
+                    + "We are just creating a list with the value and ignoring following env path fragments, which is possibly not what you expected."
+                    + "Set env var 'PSYPLUS_SUPRESS_MISSING_TYPE_WARNING=true' to supress this warning."
+                )
+            result = [value]
+
+        return result
+
+    def _get_setting_dict_by_env_var(self, env_var_key: str) -> Dict:
+        self._get_value_dict_by_env_var_key(env_var_key)
+
+    def _split_env_var(self, env_var_key: str) -> List[str]:
+        return env_var_key.split(self.env_var_delimiter)
+
+    def _deep_merge(self, base: Dict | List, update: Dict | List):
+        """Deep merge a complex nested dict/list object. Lists can have `ListitemPlaceholder` which will be recessive while merging
+
+        Args:
+            base (Dict | List): _description_
+            update (Dict | List): _description_
+
+        Raises:
+            ValueError: _description_
+        """
+        if (isinstance(base, dict) or base in (None, PydanticUndefined)) and isinstance(
+            update, dict
+        ):
+            if base in (None, PydanticUndefined):
+                base = {}
+            for key in set(base.keys()).union(update.keys()):
+                if key in base and key in update:
+                    # we have both keys. we need to go deeper
+                    base[key] = self._deep_merge(base[key], update[key])
+                elif key in update:
+                    base[key] = update[key]
+        elif (
+            isinstance(base, list) or base in (None, PydanticUndefined)
+        ) and isinstance(update, list):
+            if base in (None, PydanticUndefined):
+                base = []
+            length = max(len(base), len(update))
+            # Pad the base list, if its shorter
+            base.extend([ListitemPlaceholder] * (length - len(base)))
+            # iter through items and merge them
+            for index, item in enumerate(update):
+                if item != ListitemPlaceholder:
+                    if isinstance(item, (dict, list)):
+                        base[index] = self._deep_merge(base[index], update[index])
+                    else:
+                        base[index] = update[index]
+        elif update is None:
+            pass
+        else:
+            raise ValueError(
+                "Source and update obj are too divergent in structure to be merged."
+            )
+        return base

psyplus/field_container.py

@@ -0,0 +1,173 @@
+from pydantic import fields, BaseModel
+from pydantic_settings import BaseSettings
+from typing import List, Any
+from typing_extensions import Self
+import yaml
+
+from dataclasses import dataclass
+from pydantic_core import PydanticUndefined
+import json
+
+
+from psyplus.utils import (
+    nested_pydantic_to_dict,
+    get_str_dict_as_table,
+    python_annotation_to_generic_readable,
+    has_literal,
+    get_literal_list,
+    indent_multilines,
+)
+
+ENV_VAR_LISTINDEX_PLACEHOLDER: str = "<list-index>"
+ENV_VAR_DICTKEY_PLACEHOLDER: str = "<dict-key>"
+
+
+@dataclass
+class ListIndex:
+    index: int
+
+    def __str__(self):
+        return f"[{self.index}]"
+
+
+@dataclass
+class DictKey:
+    key: int
+
+    def __str__(self):
+        return f"['{self.key}']"
+
+
+@dataclass
+class FieldInfoContainer:
+    """A wrapper class to store and simplify access to certain (meta-)informations in/of a `pydantic.fields.FieldInfo` instance.
+    Intended for internal use only"""
+
+    path: List[str | ListIndex | DictKey]
+    field_name: str
+    field_info: fields.FieldInfo | None = None
+    annotation: Any = None
+
+    def get_annotation(self) -> Any:
+        if self.field_info:
+            return self.field_info.annotation
+        return self.annotation
+
+    def get_env_var_scheme(
+        self, env_var_delimiter: str = "__", prefix: str = ""
+    ) -> str:
+        result = []
+        for key in self.path:
+            if isinstance(key, str):
+                result.append(key.upper())
+            elif isinstance(key, ListIndex):
+                result.append(ENV_VAR_LISTINDEX_PLACEHOLDER)
+            elif isinstance(key, DictKey):
+                result.append(ENV_VAR_DICTKEY_PLACEHOLDER)
+            else:
+                # unsupported type; we can not generate a env var
+                return None
+        return prefix + env_var_delimiter.join(result)
+
+    def get_path_str(self) -> str:
+        return ".".join(str(p) for p in self.path)
+
+    def get_type_annotation_string(self):
+        return python_annotation_to_generic_readable(self.get_annotation())
+
+    def get_enum_vals(
+        self,
+    ) -> List[Any] | None:
+        """If the value has a fixed list of allowed values, this return the list of these values
+
+        Returns:
+            List[Any]: List of allowed values for the field
+        """
+        annot = self.get_annotation()
+        if has_literal(annot):
+            return get_literal_list(annot)
+        return None
+
+    def get_entry_comment_header(self):
+        """Generates a more simple header comment of keys that do not have its in pydantic.fields.FieldInfo instance"""
+        comment: List[str] = []
+        data_header = {}
+        # Title
+        key = self.field_name
+        path = self.get_path_str()
+        header_line = f"## {key}"
+        data_header["YAML-path: "] = f"{path}"
+        data_header["Env-var: "] = f"'{self.get_env_var_scheme()}'"
+        comment.append(header_line)
+        comment.extend(get_str_dict_as_table(data_header).rstrip().split("\n"))
+        return comment
+
+    def get_field_comment_header(self):
+        comment: List[str] = []
+        data_header = {}
+        # Title
+        key = self.field_name
+        path = self.get_path_str()
+        header_line = f"## {key}"
+        field_info: fields.FieldInfo = (
+            self.field_info if self.field_info else fields.FieldInfo()
+        )
+
+        if field_info.title:
+            header_line += f" - {field_info.title}"
+        header_line += f" ###"
+        comment.append(header_line)
+        # Data fields
+        if key != path:
+            data_header["YAML-path: "] = f"{path}"
+
+        if self.get_type_annotation_string():
+            data_header["Type: "] = f"{self.get_type_annotation_string()}"
+        data_header["Required: "] = f"{field_info.is_required()}"
+        if hasattr(field_info, "default") and field_info.default != PydanticUndefined:
+            if field_info.default is not None:
+                def_val = f"'{json.dumps(nested_pydantic_to_dict(field_info.default))}'"
+            else:
+                def_val = "null/None"
+            data_header["Default: "] = def_val
+        if self.get_enum_vals():
+            data_header["Allowed vals: "] = f"{self.get_enum_vals()}"
+
+        if field_info.metadata:
+            data_header["Constraints: "] = f"{field_info.metadata}"
+
+        data_header["Env-var: "] = f"'{self.get_env_var_scheme()}'"
+        if field_info.description:
+            data_header["Description: "] = f"{field_info.description}"
+
+        comment.extend(get_str_dict_as_table(data_header).rstrip().split("\n"))
+
+        if field_info.examples:
+            comment.extend(self._generate_examples_comment_text(key))
+
+        return comment
+
+    def _generate_examples_comment_text(
+        self, key: str, indent_depth: int = 0
+    ) -> List[str] | None:
+        if not self.field_info.examples:
+            return None
+        text_lines = []
+        for index, example in enumerate(self.field_info.examples):
+            text_lines.append(
+                f"Example No. {index+1}:"
+                if len(self.field_info.examples) > 1
+                else "Example:"
+            )
+
+            example_as_yaml = yaml.dump(nested_pydantic_to_dict({key: example}))
+            text_lines.extend(
+                indent_multilines(
+                    text=example_as_yaml.split("\n"),
+                    indent_depth=0,
+                    line_prefix=">",
+                    extra_indent_depth_after_prefix=indent_depth,
+                    add_extra_indent_for_subsequent_lines_after_line_prefix=False,
+                )
+            )
+        return text_lines[:-1]

psyplus/psyplus.py

@@ -0,0 +1,223 @@
+import typing
+
+from typing import (
+    List,
+    Any,
+    Dict,
+    Union,
+    Type,
+)
+import inspect
+
+from pydantic import BaseModel, fields
+
+
+from pydantic_settings import BaseSettings
+from pydantic_core import PydanticUndefined
+from pathlib import Path
+
+import yaml
+
+
+from psyplus.yaml_pydantic_metadata_comment_injector import YamlFileGenerator
+from psyplus.env_var_handler import EnvVarHandlerExtended
+
+
+class YamlSettingsPlus:
+    def __init__(
+        self,
+        model: Type[BaseSettings],
+        file_path: Union[str, Path] = None,
+        parse_finde_grained_env_var: bool = True,
+    ):
+        self.parse_finde_grained_env_var = parse_finde_grained_env_var
+        self.config_file: Path = (
+            file_path if isinstance(file_path, Path) else Path(file_path)
+        )
+        self.model: Type[BaseSettings] = model
+        self._settings_cache: BaseSettings = None
+
+    def get_config(self) -> BaseSettings:
+        with open(self.config_file) as file:
+            raw_yaml_object = file.read()
+        obj: Dict = yaml.safe_load(raw_yaml_object)
+        if self.parse_finde_grained_env_var:
+            env_var_handler = EnvVarHandlerExtended(self.model)
+            obj = env_var_handler.settings_as_dict
+        self._settings_cache = self.model.model_validate(obj)
+        return
+
+    def generate_config_file(self, overwrite_existing: bool = False, exists_ok=True):
+        null_placeholder = "NULL_PLACEHOLDER_328472384623746012386389621948"
+        dummy_values = self._get_fields_filler(
+            required_only=False,
+            use_example_values_if_exists=False,
+            fallback_fill_value=null_placeholder,
+        )
+        # print("dummy_values", dummy_values)
+        config = self.model.model_validate(dummy_values)
+        self._generate_file(
+            config,
+            overwrite_existing=overwrite_existing,
+            generate_with_example_values=True,
+            exists_ok=exists_ok,
+            replace_pattern={null_placeholder: "null"},
+        )
+
+    def generate_config_file_with_examples_values(
+        self, overwrite_existing: bool = False
+    ):
+        dummy_values = self._get_fields_filler(
+            required_only=True, use_example_values_if_exists=True
+        )
+        print("dummy_values", dummy_values)
+        config = self.model.model_validate(dummy_values)
+        return config
+        filegen = YamlFileGenerator(config)
+        filegen.parse_pydantic_model()
+        print(filegen.get_yaml())
+        exit()
+        self._generate_file(
+            config,
+            generate_with_example_values=True,
+            overwrite_existing=overwrite_existing,
+        )
+
+    def generate_config_file_from_config_object(self, config: BaseSettings):
+        self._generate_file(config)
+
+    def generate_config_file_with_only_required_keys(self):
+        config = self.model.model_validate(
+            self._get_fields_filler(
+                required_only=True,
+                use_example_values_if_exists=True,
+            )
+        )
+        self._generate_file(config, generate_with_optional_fields=False)
+
+    def generate_markdown_doc(self):
+        raise NotImplementedError()
+
+    def _generate_file(
+        self,
+        config: BaseSettings,
+        overwrite_existing: bool = False,
+        exists_ok: bool = False,
+        generate_with_optional_fields: bool = True,
+        comment_out_optional_fields: bool = True,
+        generate_with_comment_desc_header: bool = True,
+        generate_with_example_values: bool = False,
+        replace_pattern: Dict = None,
+    ):
+        self.config_file.parent.mkdir(exist_ok=True, parents=True)
+        if self.config_file.is_file() and not overwrite_existing:
+            if exists_ok:
+                return
+            else:
+                raise FileExistsError(
+                    f"Can not generate config file at {self.config_file}. File allready exists."
+                )
+        if replace_pattern is None:
+            replace_pattern = {}
+        yaml_content: str = yaml.dump(config.model_dump(), sort_keys=False)
+        from psyplus.yaml_pydantic_metadata_comment_injector import YamlFileGenerator
+
+        y = YamlFileGenerator(settings_instance=config)
+        y.parse_pydantic_model()
+        yaml_content = y.get_yaml()
+        for key, val in replace_pattern.items():
+            yaml_content = yaml_content.replace(key, val)
+
+        with open(self.config_file, "w") as file:
+            file.write(yaml_content)
+
+    def _get_fields_filler(
+        self,
+        required_only: bool = True,
+        use_example_values_if_exists: bool = False,
+        fallback_fill_value: Any = "",
+    ) -> Dict:
+        """Needed for creating dummy values for non nullable values. Otherwise we are not able to initialize a living config from the model
+
+        Args:
+            required_only (bool, optional): _description_. Defaults to True.
+            use_example_values_if_exists (bool, optional): _description_. Defaults to False.
+            fallback_fill_value (Any, optional): _description_. Defaults to None.
+
+        Returns:
+            Dict: _description_
+        """
+
+        def parse_model_class(m_cls: Type[BaseSettings | BaseModel]) -> Dict:
+            result: Dict = {}
+            for key, field in m_cls.model_fields.items():
+                if key == "only_for_groupnames_starting_with":
+                    print(
+                        "only_for_groupnames_starting_with.is_required()",
+                        field.is_required(),
+                    )
+                # if not required_only or field.is_required():
+                if True:
+                    if use_example_values_if_exists and field.examples:
+                        example = field.examples[0]
+                        # We want to generate a example models and there are examples in the annotation
+                        # if it is a real config object we pass it to as a values else we try to create a json compatible string
+
+                        result[key] = example
+                        """
+                        if inspect.isclass(field.annotation) and issubclass(
+                            field.annotation, BaseModel
+                        ):
+                            result[key] = example
+                        else:
+                            result[key] = self.jsonfy_example(example)
+                        """
+                    elif inspect.isclass(field.annotation) and issubclass(
+                        field.annotation, BaseModel | BaseSettings
+                    ):
+                        if field.default is not PydanticUndefined:
+                            result[key] = field.default
+                        elif field.default_factory is not None:
+                            print("field.default_factory", field.default_factory)
+                            result[key] = field.default_factory()
+                        else:
+                            result[key] = parse_model_class(field.annotation)
+                    elif field.annotation == Any:
+                        result[key] = ""
+                    elif type(field.annotation) in (typing._GenericAlias, type):
+                        # This is a basic type. we can provide some reasonable sane default values like 0 for int or "" for str
+                        if hasattr(field.annotation, "__origin__"):
+                            result[key] = field.annotation.__origin__()
+                        elif type(field.annotation) == type:
+                            # we have a basic type
+                            result[key] = field.annotation()
+
+                    elif (
+                        isinstance(field, fields.FieldInfo)
+                        and field.default_factory is not None
+                    ):
+                        result[key] = self.jsonfy_example(field.default_factory())
+                    else:
+                        result[key] = (
+                            fallback_fill_value
+                            if field.is_required()
+                            else self.jsonfy_example(field.default)
+                        )
+            return result
+
+        return parse_model_class(self.model)
+
+    def jsonfy_example(self, val: Any) -> List | Dict:
+        if isinstance(val, dict):
+            result: Dict = {}
+            for k, v in val.items():
+                result[k] = self.jsonfy_example(v)
+            return result
+        elif isinstance(val, (list, set, tuple)):
+            return [self.jsonfy_example(i) for i in val]
+        elif isinstance(val, BaseModel):
+            return val.model_dump_json()
+        elif val is not None:
+            return str(val)
+        else:
+            return None

psyplus/utils.py

@@ -0,0 +1,205 @@
+from typing import (
+    Any,
+    Union,
+    Dict,
+    List,
+    get_origin,
+    Literal,
+    get_args,
+    Generator,
+)
+from pydantic_core import PydanticUndefined
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings
+
+
+import datetime
+from pydantic import (
+    PastDate,
+    FutureDate,
+    PastDatetime,
+    FutureDatetime,
+    AwareDatetime,
+    NaiveDatetime,
+)
+
+
+def python_annotation_to_generic_readable(
+    annotation,
+) -> str:
+    # https://docs.pydantic.dev/2.5/api/json_schema/#pydantic.json_schema.GenerateJsonSchema
+    JSON_BASIC_TYPES = Literal["boolean", "number", "string", "array", "object"]
+
+    # https://datatracker.ietf.org/doc/html/draft-bhutton-json-schema-00#section-10.2.1
+    JSON_SUBSCHEMAS = Literal["allOf", "anyOf", "oneOf", "not"]
+
+    PYTHON_SCALAR_TYPES = [
+        int,
+        float,
+        str,
+        bool,
+        datetime.time,
+        datetime.date,
+        datetime.datetime,
+        PastDate,
+        FutureDate,
+        PastDatetime,
+        FutureDatetime,
+        AwareDatetime,
+        NaiveDatetime,
+    ]
+
+    def stringifiy_annotation(annot) -> str:
+        if type(annot) == tuple:
+            res = []
+            for item in annot:
+                res.append(stringifiy_annotation(item))
+            if res:
+                return ",".join(res)
+        if is_typingOptional(annot):
+            return stringifiy_annotation(get_typingOptionalArg(annot))
+        elif annot == Any:
+            return None
+        elif annot in PYTHON_SCALAR_TYPES:
+            return annot.__name__
+        elif get_origin(annot) == list or annot == list:
+            list_annotation_args = get_args(annot)
+            if list_annotation_args:
+                return "List of " + stringifiy_annotation(list_annotation_args)
+            else:
+                return "List"
+        elif get_origin(annot) == dict or annot == dict:
+            dict_annotation_args = get_args(annot)
+            if dict_annotation_args:
+                return f"Dictionary of ({stringifiy_annotation(get_args(annot))})"
+            else:
+                return "Dictionary"
+        elif get_origin(annot) == Literal:
+            return "Enum"
+        else:
+            return "Object"
+
+    return stringifiy_annotation(annotation)
+
+
+def is_typingOptional(annotation: Any) -> bool:
+    return (
+        hasattr(annotation, "__origin__")
+        and annotation.__origin__ is Union
+        and annotation.__args__[1] is type(None)
+    )
+
+
+def get_typingOptionalArg(annotation) -> Any:
+    if is_typingOptional(annotation):
+        return annotation.__args__[0]
+    return annotation
+
+
+def clean_annotation(annotation) -> Any:
+    """extract actual type annotation from field annotations like typing.Optional,...
+
+    Args:
+        annotation (_type_): _description_
+
+    Returns:
+        Any: _description_
+    """
+    if is_typingOptional(annotation=annotation):
+        annotation = get_typingOptionalArg(annotation=annotation)
+        return clean_annotation(annotation=annotation)
+    return annotation
+
+
+def has_literal(annotation: Any) -> bool:
+    clean_annot = clean_annotation(annotation=annotation)
+    if hasattr(clean_annot, "__origin__"):
+        return clean_annot.__origin__ == Literal
+
+
+def get_literal_list(annotation: Any) -> List[Any] | None:
+    clean_annot = clean_annotation(annotation=annotation)
+    if hasattr(clean_annot, "__origin__") and clean_annot.__origin__ == Literal:
+        return list(clean_annot.__args__)
+    return None
+
+
+def nested_pydantic_to_dict(obj: Any) -> Any:
+    if isinstance(obj, (BaseModel, BaseSettings)):
+        return nested_pydantic_to_dict(obj.model_dump())
+    elif isinstance(obj, dict):
+        return {k: nested_pydantic_to_dict(v) for k, v in obj.items()}
+    elif isinstance(obj, (list, tuple)):
+        return [nested_pydantic_to_dict(item) for item in obj]
+    else:
+        return obj
+
+
+def get_dict_val_key_insensitive(
+    dictionary: Dict, key: str, default: Any = PydanticUndefined
+):
+    for k in dictionary.keys():
+        if k.upper() == key.upper():
+            return dictionary[k]
+    if default != PydanticUndefined:
+        raise KeyError(f"Can not find key '{key}' in {dictionary.keys()}")
+    return default
+
+
+def env_to_nested_dict(key: str | List[str], val: Any, delimiter: str = "__") -> Dict:
+    """Convert a env var key to a nestesd dict.
+    e.g.
+    `MYDICT__NEXTKEY__WHATEVER=val111` -> `{'MYDICT': {'NEXTKEY': {'WHATEVER': 'val111'}}}`
+
+    Returns:
+        Dict:
+    """
+
+    if isinstance(key, str):
+        fragments = key.split(delimiter)
+    elif isinstance(key, list):
+        fragments = key
+    else:
+        raise ValueError(f"key must be str or list but is {type(key)}")
+    result = {}
+    temp = result
+
+    for frag in fragments[:-1]:
+        temp = temp.setdefault(frag, {})
+    temp[fragments[-1]] = val
+
+    return result
+
+
+def get_str_dict_as_table(
+    d: Dict, vertical_seperator: str = "", respect_line_breaks_in_val: bool = True
+) -> str:
+    length_key_column = (max(len(string) for string in d.keys()) if d.keys() else 0) + 1
+    result = ""
+    for key, val in d.items():
+        if respect_line_breaks_in_val:
+            val = str(val).split("\n")
+            result += f"{key.ljust(length_key_column)}{vertical_seperator}{val[0]}\n"
+            for v in val[1:]:
+                result += f"{''.ljust(length_key_column)}{vertical_seperator}{v}\n"
+        else:
+            result += f"{key.ljust(length_key_column)}{vertical_seperator}{val}\n"
+    return result
+
+
+def indent_multilines(
+    text: List[str],
+    indent_depth: int = 0,
+    line_prefix: str = "",
+    line_suffix: str = "",
+    extra_indent_depth_after_prefix: int = 0,
+    add_extra_indent_for_subsequent_lines_after_line_prefix: bool = False,
+    indent: str = "  ",
+) -> Generator[str, None, None]:
+    indent = f"{indent_depth*indent}"
+    inner_indent = f"{extra_indent_depth_after_prefix*indent}"
+    for index, line in enumerate(text):
+        line_prefix_real = f"{line_prefix}{inner_indent}"
+        if index != 0 and add_extra_indent_for_subsequent_lines_after_line_prefix:
+            line_prefix_real = f"{line_prefix_real}{indent}"
+        yield f"{indent}{line_prefix_real}{line}{line_suffix}"

psyplus/yaml_pydantic_metadata_comment_injector.py

@@ -0,0 +1,137 @@
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings
+from typing import List, get_args, Any
+
+
+from psyplus.utils import clean_annotation
+
+
+from ruamel.yaml import YAML, CommentedMap, CommentedSeq
+from io import StringIO
+
+from psyplus.field_container import FieldInfoContainer, ListIndex, DictKey
+
+
+class YamlFileGenerator:
+    def __init__(
+        self, settings_instance: BaseSettings | BaseModel, indent_size: int = 2
+    ):
+        self.indent_size = indent_size
+        self.yaml = YAML()
+
+        # https://yaml.readthedocs.io/en/latest/detail/#indentation-of-block-sequences
+        # > It is best to always have sequence >= offset + 2 but this is not enforced. Depending on your structure, not following this advice might lead to invalid output.
+        self.yaml.indent(sequence=indent_size + 2, offset=indent_size)
+        self.settings = settings_instance
+
+    def parse_pydantic_model(self):
+        self.model: CommentedMap = self._parse_pydantic_model(
+            self.settings, parent_path=[]
+        )
+
+    def get_yaml(self) -> str:
+        stream = StringIO()
+        self.yaml.dump(self.model, stream)
+
+        return stream.getvalue()
+
+    def _parse_pydantic_model(
+        self,
+        setting_inst: BaseSettings | BaseModel,
+        parent_path: List[ListIndex | DictKey | str],
+    ) -> CommentedMap:
+        level = len(parent_path)
+        result: CommentedMap = CommentedMap()
+        for key, field_info in setting_inst.model_fields.items():
+            path = parent_path.copy() + [key]
+            field_value = getattr(setting_inst, key)
+            if isinstance(field_value, (BaseSettings, BaseModel)):
+                result[key] = self._parse_pydantic_model(field_value, path)
+            elif isinstance(field_value, dict):
+                result[key] = self._parse_dict(field_value, field_info.annotation, path)
+            elif isinstance(field_value, list):
+                result[key] = self._parse_list(field_value, field_info.annotation, path)
+            else:
+                result[key] = field_value
+            header_comment = FieldInfoContainer(
+                path=path, field_name=key, field_info=field_info
+            ).get_field_comment_header()
+            result.yaml_set_comment_before_after_key(
+                key=key,
+                before="\n" + "\n".join(header_comment),
+                indent=level * self.indent_size,
+            )
+        return result
+
+    def _parse_dict(
+        self,
+        value,
+        field_annotation: Any,
+        parent_path: List[ListIndex | DictKey | str],
+    ) -> CommentedMap:
+        level = len(parent_path)
+        result: CommentedMap = CommentedMap()
+        if isinstance(value, dict):
+            if field_annotation is not None:
+                dict_annotation_args = get_args(clean_annotation(field_annotation))
+                if dict_annotation_args:
+                    dict_key_annotation, dict_val_annotation = dict_annotation_args
+            for key, val in value.items():
+                path = parent_path.copy() + [DictKey(key=key)]
+                if isinstance(val, (BaseSettings, BaseModel)):
+                    result[key] = self._parse_pydantic_model(val, parent_path=path)
+                elif isinstance(val, dict):
+                    result[key] = self._parse_dict(
+                        val, dict_val_annotation, parent_path=path
+                    )
+                elif isinstance(val, list):
+                    result[key] = self._parse_list(
+                        val, dict_val_annotation, parent_path=path
+                    )
+                else:
+                    result[key] = val
+                header_comment = FieldInfoContainer(
+                    path=path, field_name=key, annotation=field_annotation
+                ).get_field_comment_header()
+                result.yaml_set_comment_before_after_key(
+                    key=key,
+                    before="\n" + "\n".join(header_comment),
+                    indent=level * self.indent_size,
+                )
+        return result
+
+    def _parse_list(
+        self,
+        value,
+        field_annotation: Any,
+        parent_path: List[ListIndex | DictKey | str],
+    ) -> CommentedSeq:
+        level = len(parent_path)
+        result: CommentedSeq = CommentedSeq()
+        if isinstance(value, list):
+            list_val_annotation = get_args(clean_annotation(field_annotation))
+            for index, item in enumerate(value):
+                path = parent_path.copy() + [ListIndex(index=index)]
+                if isinstance(item, (BaseSettings, BaseModel)):
+                    result.append(self._parse_pydantic_model(item, parent_path=path))
+                elif isinstance(item, dict):
+                    result.append(
+                        self._parse_dict(item, list_val_annotation, parent_path=path)
+                    )
+                elif isinstance(item, list):
+                    result.append(
+                        self._parse_list(item, list_val_annotation, parent_path=path)
+                    )
+                else:
+                    result.append(item)
+                header_comment = FieldInfoContainer(
+                    path=path,
+                    field_name=f"List[{index}]",
+                    annotation=list_val_annotation,
+                ).get_field_comment_header()
+                result.yaml_set_comment_before_after_key(
+                    key=index,
+                    before="\n" + "\n".join(header_comment),
+                    indent=level * self.indent_size,
+                )
+        return result

psyplus-0.9.2.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Deutsche Zentrum für Diabetesforschung e.V.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

psyplus-0.9.2.dist-info/METADATA

@@ -0,0 +1,241 @@
+Metadata-Version: 2.1
+Name: psyplus
+Version: 0.9.2
+Summary: A helper module that builds upon pydantic-settings to generate, read and comment a config file in yaml
+Author-email: Tim Bleimehl <bleimehl@helmholtz-munich.de>
+License: MIT
+Project-URL: Source, https://github.com/DZD-eV-Diabetes-Research/pydantic-settings-yaml-plus
+Project-URL: Issues, https://github.com/DZD-eV-Diabetes-Research/pydantic-settings-yaml-plus/issues
+Keywords: DZD,pydantic,pydantic-settings
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic
+Requires-Dist: pydantic-setting
+Requires-Dist: ruamel.yaml
+Requires-Dist: pyaml
+
+# pydantic-settings-yaml-plus
+A helper module that builds upon [pydantic-settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) to generate, read and comment a config file in yaml ~~and improve ENV var capabilities~~  
+ 
+> [!WARNING]  
+> work in progress. please ignore this repo for now
+
+
+- [pydantic-settings-yaml-plus](#pydantic-settings-yaml-plus)
+  - [Target use cases](#target-use-cases)
+  - [Features](#features)
+  - [Known Issues](#known-issues)
+  - [Goals](#goals)
+    - [ToDo / ToInvestigate](#todo--toinvestigate)
+    - [Ideas/Roadmap](#ideasroadmap)
+  - [How to use](#how-to-use)
+
+
+## Target use cases
+
+* **Config File Provisioning** The idea is to use this module during the build/release- (deploy on pypi.org) or init(First start)-process of your python module to generate a documented yaml file the user can understand.
+* ~~**Extended Nested Env Var Parsing** Provide complex configs via env var without the need to write json~~ Needs a rewrite with pydantic-settings 2.x
+
+
+
+## Features
+
+* Generate a commented/documented yaml file based on your `pydantic-settings`.`BaseSettings` class
+* ~~Support for nested listed and dict and var. E.g. `MYLIST__0__MYKEY=val` -> `config`.`MYLIST``[0]`.`MYKEY`=`val`~~ Needs a rewrite with pydantic-settings 2.x
+
+## Known Issues
+* Multiline values are crippled
+
+## Goals
+
+* Have a single source of truth for config and all its meta-data/documentation (type, descpription, examples)
+* ~~All keys/values are addressable by env vars~~
+
+### ToDo / ToInvestigate
+* What about date(times) support?
+* Make indexed env vars work again!
+* Remove debug prints
+* Write some more test
+* Make pypi package
+
+### Ideas/Roadmap
+* Generate a mark down doc of all settings
+* Generate template (minimal with required values only or maximum with all values listed) and example config files (all YAML only!)
+* generate diff betwen current config and config model (when config model changed after update)
+* update existing config files metadata
+  * Update info, descs
+  * Add missing/new required values
+  
+
+
+
+## How to use
+
+Lets have the following examplary pydantic-settings config.py file:
+  
+```python
+from typing import List, Dict, Optional, Literal, Annotated
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+from pathlib import Path, PurePath
+
+
+class DatabaseServerSettings(BaseSettings):
+    host: Optional[str] = Field(
+        default="localhost",
+        description="The Hostname the database will be available at",
+    )
+    port: Optional[int] = Field(
+        default=5678, description="The port to connect to the database"
+    )
+    database_names: List[str] = Field(
+        description="The names of the databases to use",
+        examples=[["mydb", "theotherdb"]],
+    )
+
+
+class MyAppConfig(BaseSettings):
+    log_level: Optional[Literal["INFO", "DEBUG"]] = "INFO"
+    app_name: Optional[str] = Field(
+        default="THE APP",
+        description="The display name of the app",
+        examples=["THAT APP", "THIS APP"],
+    )
+    storage_dir: Optional[str] = Field(
+        description="A directory to store the file of the apps.",
+        default_factory=lambda: str(Path(PurePath(Path().home(), ".config/myapp/"))),
+    )
+    admin_pw: Annotated[str, Field(description="The init password the admin account")]
+    database_server: DatabaseServerSettings = Field(
+        description="The settings for the database server",
+        examples=[
+            DatabaseServerSettings(
+                host="db.company.org", port=1234, database_names=["db1", "db2"]
+            )
+        ],
+    )
+    init_values: Dict[str, str]
+```
+  
+With the help of psyplus we can generate a fully documented  
+  
+
+```python
+from psyplus import YamlSettingsPlus
+from config import MyAppConfig
+
+yaml_handler = YamlSettingsPlus(MyAppConfig, "test.config.yaml")
+yaml_handler.generate_config_file(overwrite_existing=True)
+```
+  
+which will generate a yaml file `./test.config.yaml` that looks like the following:
+  
+```yaml
+# ## log_level ###
+# Type:          Enum
+# Required:      False
+# Default:       '"INFO"'
+# Allowed vals:  ['INFO', 'DEBUG']
+# Env-var:       'LOG_LEVEL'
+log_level: INFO
+
+# ## app_name ###
+# Type:         str
+# Required:     False
+# Default:      '"THE APP"'
+# Env-var:      'APP_NAME'
+# Description:  The display name of the app
+# Example No. 1:
+#  >app_name: THAT APP
+#  >
+# Example No. 2:
+#  >app_name: THIS APP
+app_name: THE APP
+
+# ## storage_dir ###
+# Type:         str
+# Required:     False
+# Env-var:      'STORAGE_DIR'
+# Description:  A directory to store the file of the apps.
+storage_dir: /home/tim/.config/myapp
+
+# ## admin_pw ###
+# Type:         str
+# Required:     True
+# Env-var:      'ADMIN_PW'
+# Description:  The init password the admin account
+admin_pw: ''
+
+# ## database_server ###
+# Type:         Object
+# Required:     True
+# Env-var:      'DATABASE_SERVER'
+# Description:  The settings for the database server
+# Example:
+#  >database_server:
+#  >  database_names:
+#  >  - db1
+#  >  - db2
+#  >  host: db.company.org
+#  >  port: 1234
+database_server:
+
+  # ## host ###
+  # YAML-path:    database_server.host
+  # Type:         str
+  # Required:     False
+  # Default:      '"localhost"'
+  # Env-var:      'DATABASE_SERVER__HOST'
+  # Description:  The Hostname the database will be available at
+  host: localhost
+
+  # ## port ###
+  # YAML-path:    database_server.port
+  # Type:         int
+  # Required:     False
+  # Default:      '5678'
+  # Env-var:      'DATABASE_SERVER__PORT'
+  # Description:  The port to connect to the database
+  port: 5678
+
+  # ## database_names ###
+  # YAML-path:    database_server.database_names
+  # Type:         List of str
+  # Required:     True
+  # Env-var:      'DATABASE_SERVER__DATABASE_NAMES'
+  # Description:  The names of the databases to use
+  # Example:
+  #  >database_names:
+  #  >- mydb
+  #  >- theotherdb
+  database_names: []
+```
+  
+To use this yaml file you just psyplus: need to parse it and validate on your pydantic-setting model.
+  
+```python
+from psyplus import YamlSettingsPlus
+
+yaml_handler = YamlSettingsPlus(MyAppConfig, "test.config.yaml")
+config: MyAppConfig = yaml_handler.get_config()
+print(config.database_server.host)
+```
+  
+Alternativly you can parse and validate the pydantic-settings model yourself:
+  
+```python
+import yaml  # pip install PyYAML
+
+with open("test.config.yaml") as file:
+    raw_yaml_str = file.read()
+obj: Dict = yaml.safe_load(raw_yaml_str)
+config: MyAppConfig = MyAppConfig.model_validate(obj)
+
+```

psyplus-0.9.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+psyplus/__init__.py,sha256=bPyOH6PATVFpL8jGAfknuREUfiBMlYqfCN1z_IiIoLo,103
+psyplus/env_var_handler.py,sha256=dT8O-4JhfWIjQYJX6fzdEvDds_wMo7760gjcTCHMBXU,10288
+psyplus/field_container.py,sha256=dD3lAjv8tmWfwWks_hcb5th_srfMGuvMqQq17yezl6E,5628
+psyplus/psyplus.py,sha256=BNs2nxeFXkbpQC3wXvBWNpWx5eqgwdQbF-BiqsCeCMo,8710
+psyplus/utils.py,sha256=aWPm4UnKjAjzBPbkbHqZRBifLK4JLGaFmTLu248zJHU,6227
+psyplus/yaml_pydantic_metadata_comment_injector.py,sha256=H87eG4PrrQ95yncLtCBryiOb6uWsdgQjFU_SnnY8J8g,5602
+psyplus-0.9.2.dist-info/LICENSE,sha256=TuZlsZ-MBR0SxDZR0I8eTSMK2oI-UPKSs8Du8cSpxU0,1101
+psyplus-0.9.2.dist-info/METADATA,sha256=sKjkXtglqFZr9vZXg3MWhTTJWeXDQzZQDWahI9rSojA,7349
+psyplus-0.9.2.dist-info/WHEEL,sha256=nCVcAvsfA9TDtwGwhYaRrlPhTLV9m-Ga6mdyDtuwK18,91
+psyplus-0.9.2.dist-info/top_level.txt,sha256=WbyNW_Spt0-iBhS-I_NwcM_TrFqpnsRlBaPMOgdL5iw,8
+psyplus-0.9.2.dist-info/RECORD,,

psyplus-0.9.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (73.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

psyplus-0.9.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+psyplus