ominfra 0.0.0.dev193__py3-none-any.whl → 0.0.0.dev195__py3-none-any.whl

ominfra/clouds/aws/journald2aws/main.py

@@ -5,9 +5,9 @@ import dataclasses as dc
 import os.path
 import sys
 
+from omlish.lite.configs import load_config_file_obj
 from omlish.logs.standard import configure_standard_logging
 
-from ....configs import read_config_file
 from .driver import JournalctlToAwsDriver
 
 
@@ -36,7 +36,10 @@ def _main() -> None:
 
     config: JournalctlToAwsDriver.Config
     if args.config_file:
-        config = read_config_file(os.path.expanduser(args.config_file), JournalctlToAwsDriver.Config)
+        config = load_config_file_obj(
+            os.path.expanduser(args.config_file),
+            JournalctlToAwsDriver.Config,
+        )
     else:
         config = JournalctlToAwsDriver.Config()
 

ominfra/manage/deploy/conf/manager.py

@@ -26,8 +26,8 @@ from omlish.lite.json import json_dumps_pretty
 from omlish.lite.strings import strip_with_newline
 from omlish.os.paths import is_path_in_dir
 from omlish.os.paths import relative_symlink
-from omserv.nginx.configs import NginxConfigItems
-from omserv.nginx.configs import render_nginx_config_str
+from omlish.configs.nginx import NginxConfigItems
+from omlish.configs.nginx import render_nginx_config_str
 
 from omlish.formats.ini.sections import render_ini_sections
 from ..paths.paths import DeployPath

ominfra/manage/main.py

@@ -17,12 +17,12 @@ from omlish.argparse.cli import argparse_arg
 from omlish.argparse.cli import argparse_cmd
 from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
+from omlish.lite.configs import load_config_file_obj
 from omlish.lite.logs import log  # noqa
 from omlish.lite.marshal import ObjMarshalerManager
 from omlish.lite.marshal import ObjMarshalOptions
 from omlish.lite.pycharm import PycharmRemoteDebug
 
-from ..configs import read_config_file
 from .bootstrap import MainBootstrap
 from .bootstrap_ import main_bootstrap
 from .commands.base import Command
@@ -51,7 +51,7 @@ class MainCli(ArgparseCli):
         if cf is None:
             return ManageConfig()
         else:
-            return read_config_file(cf, ManageConfig)
+            return load_config_file_obj(cf, ManageConfig)
 
     #
 
@@ -122,7 +122,7 @@ class MainCli(ArgparseCli):
             cmds.append(cmd)
 
         for cf in self.args.command_file or []:
-            cmd = read_config_file(cf, Command, msh=msh)
+            cmd = load_config_file_obj(cf, Command, msh=msh)
             cmds.append(cmd)
 
         #

ominfra/scripts/journald2aws.py

@@ -9,6 +9,7 @@ import argparse
 import base64
 import collections
 import collections.abc
+import configparser
 import contextlib
 import dataclasses as dc
 import datetime
@@ -52,6 +53,12 @@ if sys.version_info < (3, 8):
 ########################################
 
 
+# ../../../../omlish/configs/types.py
+ConfigMap = ta.Mapping[str, ta.Any]
+
+# ../../../../omlish/formats/ini/sections.py
+IniSectionSettingsMap = ta.Mapping[str, ta.Mapping[str, ta.Union[str, ta.Sequence[str]]]]  # ta.TypeAlias
+
 # ../../../../omlish/formats/toml/parser.py
 TomlParseFloat = ta.Callable[[str], ta.Any]
 TomlKey = ta.Tuple[str, ...]
@@ -69,12 +76,12 @@ CheckOnRaiseFn = ta.Callable[[Exception], None]  # ta.TypeAlias
 CheckExceptionFactory = ta.Callable[..., Exception]  # ta.TypeAlias
 CheckArgsRenderer = ta.Callable[..., ta.Optional[str]]  # ta.TypeAlias
 
+# ../../../../omlish/configs/formats.py
+ConfigDataT = ta.TypeVar('ConfigDataT', bound='ConfigData')
+
 # ../../../../omlish/lite/contextmanagers.py
 ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
 
-# ../../../configs.py
-ConfigMapping = ta.Mapping[str, ta.Any]
-
 # ../../../threadworkers.py
 ThreadWorkerT = ta.TypeVar('ThreadWorkerT', bound='ThreadWorker')
 
@@ -82,6 +89,54 @@ ThreadWorkerT = ta.TypeVar('ThreadWorkerT', bound='ThreadWorker')
 SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']  # ta.TypeAlias
 
 
+########################################
+# ../../../../../omlish/configs/types.py
+
+
+#
+
+
+########################################
+# ../../../../../omlish/formats/ini/sections.py
+
+
+##
+
+
+def extract_ini_sections(cp: configparser.ConfigParser) -> IniSectionSettingsMap:
+    config_dct: ta.Dict[str, ta.Any] = {}
+    for sec in cp.sections():
+        cd = config_dct
+        for k in sec.split('.'):
+            cd = cd.setdefault(k, {})
+        cd.update(cp.items(sec))
+    return config_dct
+
+
+##
+
+
+def render_ini_sections(
+        settings_by_section: IniSectionSettingsMap,
+) -> str:
+    out = io.StringIO()
+
+    for i, (section, settings) in enumerate(settings_by_section.items()):
+        if i:
+            out.write('\n')
+
+        out.write(f'[{section}]\n')
+
+        for k, v in settings.items():
+            if isinstance(v, str):
+                out.write(f'{k}={v}\n')
+            else:
+                for vv in v:
+                    out.write(f'{k}={vv}\n')
+
+    return out.getvalue()
+
+
 ########################################
 # ../../../../../omlish/formats/toml/parser.py
 # SPDX-License-Identifier: MIT
@@ -900,6 +955,129 @@ def toml_make_safe_parse_float(parse_float: TomlParseFloat) -> TomlParseFloat:
     return safe_parse_float
 
 
+########################################
+# ../../../../../omlish/formats/toml/writer.py
+
+
+class TomlWriter:
+    @dc.dataclass(frozen=True)
+    class Literal:
+        s: str
+
+    def __init__(self, out: ta.TextIO) -> None:
+        super().__init__()
+        self._out = out
+
+        self._indent = 0
+        self._wrote_indent = False
+
+    #
+
+    def _w(self, s: str) -> None:
+        if not self._wrote_indent:
+            self._out.write('    ' * self._indent)
+            self._wrote_indent = True
+        self._out.write(s)
+
+    def _nl(self) -> None:
+        self._out.write('\n')
+        self._wrote_indent = False
+
+    def _needs_quote(self, s: str) -> bool:
+        return (
+            not s or
+            any(c in s for c in '\'"\n') or
+            s[0] not in string.ascii_letters
+        )
+
+    def _maybe_quote(self, s: str) -> str:
+        if self._needs_quote(s):
+            return repr(s)
+        else:
+            return s
+
+    #
+
+    def write_root(self, obj: ta.Mapping) -> None:
+        for i, (k, v) in enumerate(obj.items()):
+            if i:
+                self._nl()
+            self._w('[')
+            self._w(self._maybe_quote(k))
+            self._w(']')
+            self._nl()
+            self.write_table_contents(v)
+
+    def write_table_contents(self, obj: ta.Mapping) -> None:
+        for k, v in obj.items():
+            self.write_key(k)
+            self._w(' = ')
+            self.write_value(v)
+            self._nl()
+
+    def write_array(self, obj: ta.Sequence) -> None:
+        self._w('[')
+        self._nl()
+        self._indent += 1
+        for e in obj:
+            self.write_value(e)
+            self._w(',')
+            self._nl()
+        self._indent -= 1
+        self._w(']')
+
+    def write_inline_table(self, obj: ta.Mapping) -> None:
+        self._w('{')
+        for i, (k, v) in enumerate(obj.items()):
+            if i:
+                self._w(', ')
+            self.write_key(k)
+            self._w(' = ')
+            self.write_value(v)
+        self._w('}')
+
+    def write_inline_array(self, obj: ta.Sequence) -> None:
+        self._w('[')
+        for i, e in enumerate(obj):
+            if i:
+                self._w(', ')
+            self.write_value(e)
+        self._w(']')
+
+    def write_key(self, obj: ta.Any) -> None:
+        if isinstance(obj, TomlWriter.Literal):
+            self._w(obj.s)
+        elif isinstance(obj, str):
+            self._w(self._maybe_quote(obj.replace('_', '-')))
+        elif isinstance(obj, int):
+            self._w(repr(str(obj)))
+        else:
+            raise TypeError(obj)
+
+    def write_value(self, obj: ta.Any) -> None:
+        if isinstance(obj, bool):
+            self._w(str(obj).lower())
+        elif isinstance(obj, (str, int, float)):
+            self._w(repr(obj))
+        elif isinstance(obj, ta.Mapping):
+            self.write_inline_table(obj)
+        elif isinstance(obj, ta.Sequence):
+            if not obj:
+                self.write_inline_array(obj)
+            else:
+                self.write_array(obj)
+        else:
+            raise TypeError(obj)
+
+    #
+
+    @classmethod
+    def write_str(cls, obj: ta.Any) -> str:
+        out = io.StringIO()
+        cls(out).write_value(obj)
+        return out.getvalue()
+
+
 ########################################
 # ../../../../../omlish/lite/cached.py
 
@@ -2217,6 +2395,235 @@ class JournalctlToAwsCursor:
         os.rename(ncf, cf)
 
 
+########################################
+# ../../../../../omlish/configs/formats.py
+"""
+Notes:
+ - necessarily string-oriented
+ - single file, as this is intended to be amalg'd and thus all included anyway
+
+TODO:
+ - ConfigDataMapper? to_map -> ConfigMap?
+ - nginx ?
+ - raw ?
+"""
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class ConfigData(abc.ABC):  # noqa
+    @abc.abstractmethod
+    def as_map(self) -> ConfigMap:
+        raise NotImplementedError
+
+
+#
+
+
+class ConfigLoader(abc.ABC, ta.Generic[ConfigDataT]):
+    @property
+    def file_exts(self) -> ta.Sequence[str]:
+        return ()
+
+    def match_file(self, n: str) -> bool:
+        return '.' in n and n.split('.')[-1] in check.not_isinstance(self.file_exts, str)
+
+    #
+
+    def load_file(self, p: str) -> ConfigDataT:
+        with open(p) as f:
+            return self.load_str(f.read())
+
+    @abc.abstractmethod
+    def load_str(self, s: str) -> ConfigDataT:
+        raise NotImplementedError
+
+
+#
+
+
+class ConfigRenderer(abc.ABC, ta.Generic[ConfigDataT]):
+    @property
+    @abc.abstractmethod
+    def data_cls(self) -> ta.Type[ConfigDataT]:
+        raise NotImplementedError
+
+    def match_data(self, d: ConfigDataT) -> bool:
+        return isinstance(d, self.data_cls)
+
+    #
+
+    @abc.abstractmethod
+    def render(self, d: ConfigDataT) -> str:
+        raise NotImplementedError
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class ObjConfigData(ConfigData, abc.ABC):
+    obj: ta.Any
+
+    def as_map(self) -> ConfigMap:
+        return check.isinstance(self.obj, collections.abc.Mapping)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class JsonConfigData(ObjConfigData):
+    pass
+
+
+class JsonConfigLoader(ConfigLoader[JsonConfigData]):
+    file_exts = ('json',)
+
+    def load_str(self, s: str) -> JsonConfigData:
+        return JsonConfigData(json.loads(s))
+
+
+class JsonConfigRenderer(ConfigRenderer[JsonConfigData]):
+    data_cls = JsonConfigData
+
+    def render(self, d: JsonConfigData) -> str:
+        return json_dumps_pretty(d.obj)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class TomlConfigData(ObjConfigData):
+    pass
+
+
+class TomlConfigLoader(ConfigLoader[TomlConfigData]):
+    file_exts = ('toml',)
+
+    def load_str(self, s: str) -> TomlConfigData:
+        return TomlConfigData(toml_loads(s))
+
+
+class TomlConfigRenderer(ConfigRenderer[TomlConfigData]):
+    data_cls = TomlConfigData
+
+    def render(self, d: TomlConfigData) -> str:
+        return TomlWriter.write_str(d.obj)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class YamlConfigData(ObjConfigData):
+    pass
+
+
+class YamlConfigLoader(ConfigLoader[YamlConfigData]):
+    file_exts = ('yaml', 'yml')
+
+    def load_str(self, s: str) -> YamlConfigData:
+        return YamlConfigData(__import__('yaml').safe_load(s))
+
+
+class YamlConfigRenderer(ConfigRenderer[YamlConfigData]):
+    data_cls = YamlConfigData
+
+    def render(self, d: YamlConfigData) -> str:
+        return __import__('yaml').safe_dump(d.obj)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class IniConfigData(ConfigData):
+    sections: IniSectionSettingsMap
+
+    def as_map(self) -> ConfigMap:
+        return self.sections
+
+
+class IniConfigLoader(ConfigLoader[IniConfigData]):
+    file_exts = ('ini',)
+
+    def load_str(self, s: str) -> IniConfigData:
+        cp = configparser.ConfigParser()
+        cp.read_string(s)
+        return IniConfigData(extract_ini_sections(cp))
+
+
+class IniConfigRenderer(ConfigRenderer[IniConfigData]):
+    data_cls = IniConfigData
+
+    def render(self, d: IniConfigData) -> str:
+        return render_ini_sections(d.sections)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SwitchedConfigFileLoader:
+    loaders: ta.Sequence[ConfigLoader]
+    default: ta.Optional[ConfigLoader] = None
+
+    def load_file(self, p: str) -> ConfigData:
+        n = os.path.basename(p)
+
+        for l in self.loaders:
+            if l.match_file(n):
+                return l.load_file(p)
+
+        if (d := self.default) is not None:
+            return d.load_file(p)
+
+        raise NameError(n)
+
+
+DEFAULT_CONFIG_LOADERS: ta.Sequence[ConfigLoader] = [
+    JsonConfigLoader(),
+    TomlConfigLoader(),
+    YamlConfigLoader(),
+    IniConfigLoader(),
+]
+
+DEFAULT_CONFIG_LOADER: ConfigLoader = JsonConfigLoader()
+
+DEFAULT_CONFIG_FILE_LOADER = SwitchedConfigFileLoader(
+    loaders=DEFAULT_CONFIG_LOADERS,
+    default=DEFAULT_CONFIG_LOADER,
+)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class SwitchedConfigRenderer:
+    renderers: ta.Sequence[ConfigRenderer]
+
+    def render(self, d: ConfigData) -> str:
+        for r in self.renderers:
+            if r.match_data(d):
+                return r.render(d)
+        raise TypeError(d)
+
+
+DEFAULT_CONFIG_RENDERERS: ta.Sequence[ConfigRenderer] = [
+    JsonConfigRenderer(),
+    TomlConfigRenderer(),
+    YamlConfigRenderer(),
+    IniConfigRenderer(),
+]
+
+DEFAULT_CONFIG_RENDERER = SwitchedConfigRenderer(DEFAULT_CONFIG_RENDERERS)
+
+
 ########################################
 # ../../../../../omlish/io/buffers.py
 
@@ -3279,97 +3686,6 @@ class AwsLogMessageBuilder:
         return [post]
 
 
-########################################
-# ../../../../configs.py
-
-
-##
-
-
-def parse_config_file(
-        name: str,
-        f: ta.TextIO,
-) -> ConfigMapping:
-    if name.endswith('.toml'):
-        return toml_loads(f.read())
-
-    elif any(name.endswith(e) for e in ('.yml', '.yaml')):
-        yaml = __import__('yaml')
-        return yaml.safe_load(f)
-
-    elif name.endswith('.ini'):
-        import configparser
-        cp = configparser.ConfigParser()
-        cp.read_file(f)
-        config_dct: ta.Dict[str, ta.Any] = {}
-        for sec in cp.sections():
-            cd = config_dct
-            for k in sec.split('.'):
-                cd = cd.setdefault(k, {})
-            cd.update(cp.items(sec))
-        return config_dct
-
-    else:
-        return json.loads(f.read())
-
-
-def read_config_file(
-        path: str,
-        cls: ta.Type[T],
-        *,
-        prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
-        msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
-) -> T:
-    with open(path) as cf:
-        config_dct = parse_config_file(os.path.basename(path), cf)
-
-    if prepare is not None:
-        config_dct = prepare(config_dct)
-
-    return msh.unmarshal_obj(config_dct, cls)
-
-
-##
-
-
-def build_config_named_children(
-        o: ta.Union[
-            ta.Sequence[ConfigMapping],
-            ta.Mapping[str, ConfigMapping],
-            None,
-        ],
-        *,
-        name_key: str = 'name',
-) -> ta.Optional[ta.Sequence[ConfigMapping]]:
-    if o is None:
-        return None
-
-    lst: ta.List[ConfigMapping] = []
-    if isinstance(o, ta.Mapping):
-        for k, v in o.items():
-            check.isinstance(v, ta.Mapping)
-            if name_key in v:
-                n = v[name_key]
-                if k != n:
-                    raise KeyError(f'Given names do not match: {n} != {k}')
-                lst.append(v)
-            else:
-                lst.append({name_key: k, **v})
-
-    else:
-        check.not_isinstance(o, str)
-        lst.extend(o)
-
-    seen = set()
-    for d in lst:
-        n = d['name']
-        if n in d:
-            raise KeyError(f'Duplicate name: {n}')
-        seen.add(n)
-
-    return lst
-
-
 ########################################
 # ../../../../journald/messages.py
 
@@ -3629,6 +3945,38 @@ class ThreadWorkerGroup:
         return dct
 
 
+########################################
+# ../../../../../omlish/lite/configs.py
+
+
+##
+
+
+def load_config_file_obj(
+        f: str,
+        cls: ta.Type[T],
+        *,
+        prepare: ta.Union[
+            ta.Callable[[ConfigMap], ConfigMap],
+            ta.Iterable[ta.Callable[[ConfigMap], ConfigMap]],
+        ] = (),
+        msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
+) -> T:
+    config_data = DEFAULT_CONFIG_FILE_LOADER.load_file(f)
+
+    config_dct = config_data.as_map()
+
+    if prepare is not None:
+        if isinstance(prepare, ta.Iterable):
+            pfs = list(prepare)
+        else:
+            pfs = [prepare]
+        for pf in pfs:
+            config_dct = pf(config_dct)
+
+    return msh.unmarshal_obj(config_dct, cls)
+
+
 ########################################
 # ../../../../../omlish/logs/standard.py
 """
@@ -4909,7 +5257,10 @@ def _main() -> None:
 
     config: JournalctlToAwsDriver.Config
     if args.config_file:
-        config = read_config_file(os.path.expanduser(args.config_file), JournalctlToAwsDriver.Config)
+        config = load_config_file_obj(
+            os.path.expanduser(args.config_file),
+            JournalctlToAwsDriver.Config,
+        )
     else:
         config = JournalctlToAwsDriver.Config()