omdev 0.0.0.dev392__py3-none-any.whl → 0.0.0.dev394__py3-none-any.whl

omdev/manifests/{build.py → building.py}

@@ -1,4 +1,3 @@
-# ruff: noqa: UP006 UP007 UP045
 """
 TODO:
  - verify classes instantiate
@@ -6,6 +5,10 @@ TODO:
  - roundtrip flexibility regarding json-ness - tuples vs lists vs sets vs frozensets etc
  - kill _MANIFEST_GLOBAL_PATS lol, ast walk
   - garbage skip_pat doesn't handle multiline decos, etc
+ - relative paths
+ - is this lite? or not?
+ - can this run externally? or not? what does it have to import?
+  - has to import manifest classes, but not modules with manifest magics
 
 See (entry_points):
  - https://github.com/pytest-dev/pluggy/blob/main/src/pluggy/_manager.py#L405
@@ -34,16 +37,17 @@ from omlish.lite.imports import import_attr
 from omlish.lite.json import json_dumps_pretty
 from omlish.lite.logs import log
 from omlish.manifests.base import ModAttrManifest
-from omlish.manifests.load import MANIFEST_LOADER
+from omlish.manifests.globals import MANIFEST_LOADER
 from omlish.manifests.types import Manifest
 from omlish.manifests.types import ManifestOrigin
 
 from .. import magic
-from .dumping import _ModuleManifestDumper
 
 
 T = ta.TypeVar('T')
 
+ManifestDumperTarget = ta.Union['InlineManifestDumperTarget', 'AttrManifestDumperTarget']  # ta.TypeAlias
+
 
 ##
 
@@ -58,7 +62,7 @@ _NAME_PAT_PART = rf'(?P<name>{_IDENT_PAT_PART})'
 @dc.dataclass(frozen=True)
 class _ManifestGlobalPat:
     name_pat: re.Pattern
-    skip_pat: ta.Optional[re.Pattern] = None
+    skip_pat: re.Pattern | None = None
 
 
 _MANIFEST_GLOBAL_PATS: ta.Sequence[_ManifestGlobalPat] = [
@@ -87,45 +91,57 @@ _INLINE_MANIFEST_CLS_NAME_PAT = re.compile(r'^(?P<cls_name>[_a-zA-Z][_a-zA-Z0-9.
 ##
 
 
+class InlineManifestDumperTarget(ta.TypedDict):
+    origin: ta.Mapping[str, ta.Any]
+    kind: ta.Literal['inline']
+    cls_mod_name: str
+    cls_qualname: str
+    init_src: str
+    kwargs: ta.Mapping[str, ta.Any]
+
+
+class AttrManifestDumperTarget(ta.TypedDict):
+    origin: ta.Mapping[str, ta.Any]
+    kind: ta.Literal['attr']
+    attr: str
+
+
+##
+
+
 @cached_nullary
-def _payload_src() -> str:
-    return inspect.getsource(_ModuleManifestDumper)
+def _module_manifest_dumper_payload_src() -> str:
+    from . import _dumping
+    return inspect.getsource(_dumping)
 
 
 class ManifestBuilder:
     def __init__(
             self,
-            base: str,
+            base_dir: str,
             concurrency: int = 8,
             *,
-            write: bool = False,
+            subprocess_kwargs: ta.Mapping[str, ta.Any] | None = None,
+            module_dumper_payload_src: str | None = None,
     ) -> None:
         super().__init__()
 
-        self._base = base
+        self._base_dir = base_dir
+        self._subprocess_kwargs = subprocess_kwargs
+        self._module_dumper_payload_src = module_dumper_payload_src
+
         self._sem = asyncio.Semaphore(concurrency)
-        self._write = write
 
-    async def _spawn(self, fn: ta.Callable[..., ta.Awaitable[T]], *args: ta.Any, **kwargs: ta.Any) -> T:
-        await self._sem.acquire()
-        try:
-            try:
-                return await fn(*args, **kwargs)
-            except Exception:  # noqa
-                log.exception('Exception in task: %s, %r, %r', fn, args, kwargs)
-                raise
-        finally:
-            self._sem.release()
+    #
 
-    async def build_module_manifests(
-            self,
-            file: str,
-            *,
-            shell_wrap: bool = True,
-            warn_threshold_s: ta.Optional[float] = 1.,
-    ) -> ta.Sequence[Manifest]:
-        log.info('Extracting manifests from file %s', file)
+    @dc.dataclass(frozen=True)
+    class FileModule:
+        file: str
+
+        mod_name: str
+        mod_base: str
 
+    def build_file_module(self, file: str) -> FileModule:
         if not file.endswith('.py'):
             raise Exception(file)
 
@@ -134,31 +150,41 @@ class ManifestBuilder:
         if mod_base != (first_dir := file.split(os.path.sep)[0]):
             raise Exception(f'Unexpected module base: {mod_base=} != {first_dir=}')
 
-        with open(os.path.join(self._base, file)) as f:  # noqa
+        return ManifestBuilder.FileModule(
+            file=file,
+
+            mod_name=mod_name,
+            mod_base=mod_base,
+        )
+
+    #
+
+    def collect_module_manifest_targets(self, fm: FileModule) -> list[ManifestDumperTarget]:
+        with open(os.path.join(self._base_dir, fm.file)) as f:  # noqa
             src = f.read()
 
         lines = src.splitlines(keepends=True)
 
         def prepare(s: str) -> ta.Any:
             if s.startswith('$.'):
-                s = f'{mod_base}.{s[2:]}'
+                s = f'{fm.mod_base}.{s[2:]}'
             return magic.py_compile_magic_preparer(s)
 
         magics = magic.find_magic(
             magic.PY_MAGIC_STYLE,
             lines,
-            file=file,
+            file=fm.file,
             keys={MANIFEST_MAGIC_KEY},
             preparer=prepare,
         )
 
-        origins: ta.List[ManifestOrigin] = []
-        targets: ta.List[dict] = []
+        origins: list[ManifestOrigin] = []
+        targets: list[ManifestDumperTarget] = []
         for m in magics:
             if m.body:
                 body = m.body
                 if body.startswith('$.'):
-                    body = f'{mod_base}.{body[2:]}'
+                    body = f'{fm.mod_base}.{body[2:]}'
 
                 pat_match = check.not_none(_INLINE_MANIFEST_CLS_NAME_PAT.match(body))
                 cls_name = check.non_empty_str(pat_match.groupdict()['cls_name'])
@@ -184,15 +210,15 @@ class ManifestBuilder:
                 if issubclass(cls, ModAttrManifest):
                     attr_name = extract_manifest_target_name(lines, m.end_line)
                     inl_kw.update({
-                        'mod_name': mod_name,
+                        'mod_name': fm.mod_name,
                         'attr_name': attr_name,
                     })
 
                 origin = ManifestOrigin(
-                    module='.'.join(['', *mod_name.split('.')[1:]]),
+                    module='.'.join(['', *fm.mod_name.split('.')[1:]]),
                     attr=None,
 
-                    file=file,
+                    file=fm.file,
                     line=m.start_line,
                 )
 
@@ -210,10 +236,10 @@ class ManifestBuilder:
                 attr_name = extract_manifest_target_name(lines, m.end_line)
 
                 origin = ManifestOrigin(
-                    module='.'.join(['', *mod_name.split('.')[1:]]),
+                    module='.'.join(['', *fm.mod_name.split('.')[1:]]),
                     attr=attr_name,
 
-                    file=file,
+                    file=fm.file,
                     line=m.start_line,
                 )
 
@@ -230,9 +256,27 @@ class ManifestBuilder:
         if (dups := [k for k, v in collections.Counter(o.attr for o in origins if o.attr is not None).items() if v > 1]):  # noqa
             raise Exception(f'Duplicate attrs: {dups}')
 
+        return targets
+
+    #
+
+    async def _dump_module_manifests(
+            self,
+            fm: FileModule,
+            targets: ta.Sequence[ManifestDumperTarget],
+            *,
+            shell_wrap: bool = True,
+            warn_threshold_s: float | None = 1.,
+    ):
+        dumper_payload_src: str
+        if self._module_dumper_payload_src is not None:
+            dumper_payload_src = self._module_dumper_payload_src
+        else:
+            dumper_payload_src = _module_manifest_dumper_payload_src()
+
         subproc_src = '\n\n'.join([
-            _payload_src(),
-            f'_ModuleManifestDumper({mod_name!r})({", ".join(repr(tgt) for tgt in targets)})\n',
+            dumper_payload_src,
+            f'_ModuleManifestDumper({fm.mod_name!r})({", ".join(repr(tgt) for tgt in targets)})\n',
         ])
 
         args = [
@@ -246,7 +290,12 @@ class ManifestBuilder:
 
         start_time = time.time()
 
-        proc = await asyncio.create_subprocess_exec(*args, stdout=subprocess.PIPE)
+        proc = await asyncio.create_subprocess_exec(
+            *args,
+            stdout=subprocess.PIPE,
+            **(self._subprocess_kwargs or {}),
+        )
+
         subproc_out, _ = await proc.communicate()
         if proc.returncode:
             raise Exception('Subprocess failed')
@@ -254,18 +303,21 @@ class ManifestBuilder:
         end_time = time.time()
 
         if warn_threshold_s is not None and (elapsed_time := (end_time - start_time)) >= warn_threshold_s:
-            log.warning('Manifest extraction took a long time: %s, %.2f s', file, elapsed_time)
+            log.warning('Manifest extraction took a long time: %s, %.2f s', fm.file, elapsed_time)
 
         sp_lines = subproc_out.decode().strip().splitlines()
         if len(sp_lines) != 1:
             raise Exception('Unexpected subprocess output')
 
         sp_outs = json.loads(sp_lines[0])
-        # FIXME:
-        # if set(dct) != set(attrs):
-        #     raise Exception('Unexpected subprocess output keys')
+        return sp_outs
 
-        out: ta.List[Manifest] = []
+    def _process_module_dump_output(
+            self,
+            fm: FileModule,
+            sp_outs: ta.Sequence[ta.Mapping[str, ta.Any]],
+    ) -> list[Manifest]:
+        out: list[Manifest] = []
         for sp_out in sp_outs:
             value = sp_out['value']
 
@@ -278,7 +330,7 @@ class ManifestBuilder:
 
             [(key, value_dct)] = value.items()
             kb, _, kr = key[1:].partition('.')  # noqa
-            if kb == mod_base:  # noqa
+            if kb == fm.mod_base:  # noqa
                 key = f'$.{kr}'
                 value = {key: value_dct}
 
@@ -289,11 +341,45 @@ class ManifestBuilder:
 
         return out
 
+    #
+
+    async def build_module_manifests(self, file: str) -> ta.Sequence[Manifest]:
+        log.info('Extracting manifests from file %s', file)
+
+        fm = self.build_file_module(file)
+
+        targets = self.collect_module_manifest_targets(fm)
+
+        sp_outs = await self._dump_module_manifests(fm, targets)
+
+        # FIXME:
+        # if set(dct) != set(attrs):
+        #     raise Exception('Unexpected subprocess output keys')
+
+        out = self._process_module_dump_output(fm, sp_outs)
+
+        return out
+
+    #
+
+    async def _spawn(self, fn: ta.Callable[..., ta.Awaitable[T]], *args: ta.Any, **kwargs: ta.Any) -> T:
+        await self._sem.acquire()
+        try:
+            try:
+                return await fn(*args, **kwargs)
+            except Exception:  # noqa
+                log.exception('Exception in task: %s, %r, %r', fn, args, kwargs)
+                raise
+        finally:
+            self._sem.release()
+
     async def build_package_manifests(
             self,
             name: str,
-    ) -> ta.List[Manifest]:
-        pkg_dir = os.path.join(self._base, name)
+            *,
+            write: bool = False,
+    ) -> list[Manifest]:
+        pkg_dir = os.path.join(self._base_dir, name)
         if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
             raise Exception(pkg_dir)
 
@@ -302,15 +388,15 @@ class ManifestBuilder:
             [pkg_dir],
             keys=[MANIFEST_MAGIC_KEY],
         ))
-        manifests: ta.List[Manifest] = list(itertools.chain.from_iterable(await asyncio.gather(*[
+        manifests: list[Manifest] = list(itertools.chain.from_iterable(await asyncio.gather(*[
             self._spawn(
                 self.build_module_manifests,
-                os.path.relpath(file, self._base),
+                os.path.relpath(file, self._base_dir),
             )
             for file in files
         ])))
 
-        if self._write:
+        if write:
             with open(os.path.join(pkg_dir, '.manifests.json'), 'w') as f:  # noqa
                 f.write(json_dumps_pretty([dc.asdict(m) for m in manifests]))
                 f.write('\n')
@@ -323,9 +409,9 @@ class ManifestBuilder:
 
 def check_package_manifests(
         name: str,
-        base: str,
+        base_dir: str,
 ) -> None:
-    pkg_dir = os.path.join(base, name)
+    pkg_dir = os.path.join(base_dir, name)
     if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
         raise Exception(pkg_dir)
 
@@ -342,4 +428,4 @@ def check_package_manifests(
         if key.startswith('$.'):
             key = f'${name}{key[1:]}'
         cls = MANIFEST_LOADER.load_cls(key)
-        value = cls(**value_dct)  # noqa
+        value = MANIFEST_LOADER.instantiate_cls(cls, **value_dct)  # noqa

omdev/manifests/dumping.py

@@ -1,30 +1,99 @@
 # ruff: noqa: UP006 UP007 UP037 UP045
+# @omlish-lite
+# @omlish-amalg _dumping.py
+import collections.abc
+import dataclasses as dc
+import functools
+import importlib
+import json
 import typing as ta
 
+from omlish.lite.cached import cached_nullary
+from omlish.lite.marshal import marshal_obj
+from omlish.lite.marshal import unmarshal_obj
+
 
 ##
 
 
 class _ModuleManifestDumper:
-    def __init__(self, spec: str) -> None:
+    def __init__(
+            self,
+            spec: str,
+            *,
+            output: ta.Optional[ta.Callable[[str], None]] = None,
+    ) -> None:
         super().__init__()
 
         self._spec = spec
-        self._imported_mod: 'ta.Optional[ta.Any]' = None
+        if output is None:
+            output = print
+        self._output = output
+
+    #
+
+    @cached_nullary
+    def _mod(self) -> ta.Any:
+        return importlib.import_module(self._spec)
+
+    #
+
+    def _build_manifest_dct(self, manifest: ta.Any) -> ta.Mapping[str, ta.Any]:
+        manifest_json = json.dumps(marshal_obj(manifest))
+        manifest_dct = json.loads(manifest_json)
+
+        rt_manifest: ta.Any = unmarshal_obj(manifest_dct, type(manifest))
+        rt_manifest_json: ta.Any = json.dumps(marshal_obj(rt_manifest))
+        rt_manifest_dct: ta.Any = json.loads(rt_manifest_json)
+        if rt_manifest_dct != manifest_dct:
+            raise Exception(
+                f'Manifest failed to roundtrip: '
+                f'{manifest} => {manifest_dct} != {rt_manifest} => {rt_manifest_dct}',
+            )
+
+        return manifest_dct
+
+    #
+
+    def _load_attr_manifest(self, target: dict) -> dict:
+        attr = target['attr']
+        manifest = getattr(self._mod(), attr)
+
+        if dc.is_dataclass(manifest):
+            # Support static dataclasses
+            if isinstance(manifest, type):
+                manifest = manifest()
+
+            manifest_dct = self._build_manifest_dct(manifest)
+
+            cls = type(manifest)
+            key = f'${cls.__module__}.{cls.__qualname__}'
+
+            return {key: manifest_dct}
+
+        elif isinstance(manifest, collections.abc.Mapping):
+            [(key, manifest_dct)] = manifest.items()
+            if not key.startswith('$'):  # noqa
+                raise Exception(f'Bad key: {key}')
 
-    def _mod(self) -> 'ta.Any':
-        if (mod := self._imported_mod) is not None:
-            return mod
+            if not isinstance(manifest_dct, collections.abc.Mapping):
+                raise Exception(f'Bad value: {manifest_dct}')
 
-        import importlib
+            manifest_json = json.dumps(manifest_dct)
 
-        mod = importlib.import_module(self._spec)
+            rt_manifest_dct = json.loads(manifest_json)
+            if rt_manifest_dct != manifest_dct:
+                raise Exception(f'Manifest failed to roundtrip: {manifest_dct} != {rt_manifest_dct}')
 
-        self._imported_mod = mod
-        return mod
+            return {key: manifest_dct}
+
+        else:
+            raise TypeError(f'Manifest must be dataclass or mapping: {manifest!r}')
+
+    #
 
     class _LazyGlobals(dict):
-        def __init__(self, get_missing: 'ta.Callable[[str], ta.Any]') -> None:
+        def __init__(self, get_missing: ta.Callable[[str], ta.Any]) -> None:
             super().__init__()
 
             self.__get_missing = get_missing
@@ -32,79 +101,46 @@ class _ModuleManifestDumper:
         def __missing__(self, key):
             return self.__get_missing(key)
 
-    def __call__(self, *targets: dict) -> None:
-        import collections.abc
-        import dataclasses as dc  # noqa
-        import functools
-        import importlib
-        import json
+    def _load_inline_manifest(self, target: dict) -> dict:
+        cls: ta.Any = importlib.import_module(target['cls_mod_name'])
+        for p in target['cls_qualname'].split('.'):
+            cls = getattr(cls, p)
+        if not isinstance(cls, type) or not dc.is_dataclass(cls):
+            raise TypeError(cls)
 
-        cls: 'ta.Any'
+        cls_fac = functools.partial(cls, **target['kwargs'])
+        eval_attr_name = '__manifest_factory__'
 
-        out = []
-        for target in targets:
-            origin = target['origin']
-
-            if target['kind'] == 'attr':
-                attr = target['attr']
-                manifest = getattr(self._mod(), attr)
+        inl_glo = self._LazyGlobals(lambda k: getattr(self._mod(), k))
+        inl_glo.update({
+            eval_attr_name: cls_fac,
+        })
 
-                if dc.is_dataclass(manifest):
-                    # Support static dataclasses
-                    if isinstance(manifest, type):
-                        manifest = manifest()
+        inl_src = eval_attr_name + target['init_src']
+        inl_code = compile(inl_src, '<magic>', 'eval')
 
-                    cls = type(manifest)
-                    manifest_json = json.dumps(dc.asdict(manifest))
-                    manifest_dct = json.loads(manifest_json)
+        manifest = eval(inl_code, inl_glo)  # noqa
 
-                    rt_manifest = cls(**manifest_dct)
-                    if rt_manifest != manifest:
-                        raise Exception(f'Manifest failed to roundtrip: {manifest} => {manifest_dct} != {rt_manifest}')
+        manifest_dct = self._build_manifest_dct(manifest)
 
-                    key = f'${cls.__module__}.{cls.__qualname__}'
-                    out_value = {key: manifest_dct}
+        key = f'${cls.__module__}.{cls.__qualname__}'
+        return {key: manifest_dct}
 
-                elif isinstance(manifest, collections.abc.Mapping):
-                    [(key, manifest_dct)] = manifest.items()
-                    if not key.startswith('$'):  # noqa
-                        raise Exception(f'Bad key: {key}')
+    #
 
-                    if not isinstance(manifest_dct, collections.abc.Mapping):
-                        raise Exception(f'Bad value: {manifest_dct}')
-
-                    manifest_json = json.dumps(manifest_dct)
-                    rt_manifest_dct = json.loads(manifest_json)
-                    if manifest_dct != rt_manifest_dct:
-                        raise Exception(f'Manifest failed to roundtrip: {manifest_dct} != {rt_manifest_dct}')
-
-                    out_value = {key: manifest_dct}
+    def __call__(
+            self,
+            *targets: dict,  # .build.ManifestDumperTarget
+    ) -> None:
+        out = []
+        for target in targets:
+            origin = target['origin']
 
-                else:
-                    raise TypeError(f'Manifest must be dataclass or mapping: {manifest!r}')
+            if target['kind'] == 'attr':
+                out_value = self._load_attr_manifest(target)
 
             elif target['kind'] == 'inline':
-                cls = importlib.import_module(target['cls_mod_name'])
-                for p in target['cls_qualname'].split('.'):
-                    cls = getattr(cls, p)
-                if not isinstance(cls, type) or not dc.is_dataclass(cls):
-                    raise TypeError(cls)
-
-                cls_fac = functools.partial(cls, **target['kwargs'])
-                eval_attr_name = '__manifest_factory__'
-
-                inl_glo = self._LazyGlobals(lambda k: getattr(self._mod(), k))
-                inl_glo.update({
-                    eval_attr_name: cls_fac,
-                })
-
-                inl_src = eval_attr_name + target['init_src']
-                inl_code = compile(inl_src, '<magic>', 'eval')
-                manifest = eval(inl_code, inl_glo)  # noqa
-                manifest_json = json.dumps(dc.asdict(manifest))
-                manifest_dct = json.loads(manifest_json)
-                key = f'${cls.__module__}.{cls.__qualname__}'
-                out_value = {key: manifest_dct}
+                out_value = self._load_inline_manifest(target)
 
             else:
                 raise ValueError(target)
@@ -115,4 +151,4 @@ class _ModuleManifestDumper:
             })
 
         out_json = json.dumps(out, indent=None, separators=(',', ':'))
-        print(out_json)
+        self._output(out_json)

omdev/manifests/main.py

@@ -1,4 +1,3 @@
-# ruff: noqa: UP006 UP007 UP045
 import argparse
 import asyncio
 import dataclasses as dc
@@ -8,14 +7,14 @@ import os.path
 from omlish.lite.json import json_dumps_pretty
 from omlish.logs.standard import configure_standard_logging
 
-from .build import ManifestBuilder
-from .build import check_package_manifests
+from .building import ManifestBuilder
+from .building import check_package_manifests
 
 
 ##
 
 
-def _get_base(args) -> str:
+def _get_base_dir(args) -> str:
     if args.base is not None:
         base = args.base
     else:
@@ -27,18 +26,20 @@ def _get_base(args) -> str:
 
 
 def _gen_cmd(args) -> None:
-    base = _get_base(args)
+    base_dir = _get_base_dir(args)
 
     jobs = args.jobs or int(max(mp.cpu_count() // 1.5, 1))
     builder = ManifestBuilder(
-        base,
+        base_dir,
         jobs,
-        write=args.write or False,
     )
 
     async def do():
         return await asyncio.gather(*[
-            builder.build_package_manifests(pkg)
+            builder.build_package_manifests(
+                pkg,
+                write=bool(args.write),
+            )
             for pkg in args.package
         ])
 
@@ -49,12 +50,12 @@ def _gen_cmd(args) -> None:
 
 
 def _check_cmd(args) -> None:
-    base = _get_base(args)
+    base_dir = _get_base_dir(args)
 
     for pkg in args.package:
         check_package_manifests(
             pkg,
-            base,
+            base_dir,
         )
 
 

omdev/precheck/manifests.py

@@ -1,7 +1,7 @@
 import dataclasses as dc
 import typing as ta
 
-from omlish.manifests.load import MANIFEST_LOADER
+from omlish.manifests.globals import MANIFEST_LOADER
 
 from .base import Precheck
 from .base import PrecheckContext

omdev/tools/git/messages.py

@@ -6,7 +6,7 @@ from omlish import cached
 from omlish import check
 from omlish import dataclasses as dc
 from omlish import lang
-from omlish.manifests import load as manifest_load
+from omlish.manifests import globals as manifest_globals
 from omlish.manifests.base import ModAttrManifest
 from omlish.manifests.base import NameAliasesManifest
 from omlish.manifests.static import StaticModAttrManifest
@@ -54,7 +54,7 @@ class StaticGitMessageGeneratorManifest(StaticModAttrManifest, GitMessageGenerat
 
 @cached.function
 def load_message_generator_manifests() -> ta.Sequence[GitMessageGeneratorManifest]:
-    ldr = manifest_load.MANIFEST_LOADER
+    ldr = manifest_globals.MANIFEST_LOADER
     pkgs = ldr.scan_or_discover_pkgs(fallback_root=os.getcwd())
     mfs = ldr.load(*pkgs, only=[GitMessageGeneratorManifest])
     return [mf.value for mf in mfs]