omdev 0.0.0.dev392__py3-none-any.whl → 0.0.0.dev393__py3-none-any.whl

omdev/.manifests.json

@@ -122,6 +122,17 @@
       }
     }
   },
+  {
+    "module": ".manifests.tests.packages.foo.fargles",
+    "attr": "_SIMPLE_THINGY",
+    "file": "omdev/manifests/tests/packages/foo/fargles.py",
+    "line": 4,
+    "value": {
+      "$.manifests.tests.packages.foo.thingies.manifests.SimpleThingyManifest": {
+        "what": "fargles"
+      }
+    }
+  },
   {
     "module": ".precheck.__main__",
     "attr": "_CLI_MODULE",

omdev/manifests/__init__.py

@@ -1 +0,0 @@
-# @omlish-lite

omdev/manifests/{build.py → building.py}

@@ -1,4 +1,5 @@
 # ruff: noqa: UP006 UP007 UP045
+# @omlish-lite
 """
 TODO:
  - verify classes instantiate
@@ -6,6 +7,10 @@ TODO:
  - roundtrip flexibility regarding json-ness - tuples vs lists vs sets vs frozensets etc
  - kill _MANIFEST_GLOBAL_PATS lol, ast walk
   - garbage skip_pat doesn't handle multiline decos, etc
+ - relative paths
+ - is this lite? or not?
+ - can this run externally? or not? what does it have to import?
+  - has to import manifest classes, but not modules with manifest magics
 
 See (entry_points):
  - https://github.com/pytest-dev/pluggy/blob/main/src/pluggy/_manager.py#L405
@@ -44,6 +49,8 @@ from .dumping import _ModuleManifestDumper
 
 T = ta.TypeVar('T')
 
+ManifestDumperTarget = ta.Union['InlineManifestDumperTarget', 'AttrManifestDumperTarget']  # ta.TypeAlias
+
 
 ##
 
@@ -87,45 +94,56 @@ _INLINE_MANIFEST_CLS_NAME_PAT = re.compile(r'^(?P<cls_name>[_a-zA-Z][_a-zA-Z0-9.
 ##
 
 
+class InlineManifestDumperTarget(ta.TypedDict):
+    origin: ta.Mapping[str, ta.Any]
+    kind: ta.Literal['inline']
+    cls_mod_name: str
+    cls_qualname: str
+    init_src: str
+    kwargs: ta.Mapping[str, ta.Any]
+
+
+class AttrManifestDumperTarget(ta.TypedDict):
+    origin: ta.Mapping[str, ta.Any]
+    kind: ta.Literal['attr']
+    attr: str
+
+
+##
+
+
 @cached_nullary
-def _payload_src() -> str:
+def _module_manifest_dumper_payload_src() -> str:
     return inspect.getsource(_ModuleManifestDumper)
 
 
 class ManifestBuilder:
     def __init__(
             self,
-            base: str,
+            base_dir: str,
             concurrency: int = 8,
             *,
-            write: bool = False,
+            subprocess_kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None,
+            module_dumper_payload_src: ta.Optional[str] = None,
     ) -> None:
         super().__init__()
 
-        self._base = base
+        self._base_dir = base_dir
+        self._subprocess_kwargs = subprocess_kwargs
+        self._module_dumper_payload_src = module_dumper_payload_src
+
         self._sem = asyncio.Semaphore(concurrency)
-        self._write = write
 
-    async def _spawn(self, fn: ta.Callable[..., ta.Awaitable[T]], *args: ta.Any, **kwargs: ta.Any) -> T:
-        await self._sem.acquire()
-        try:
-            try:
-                return await fn(*args, **kwargs)
-            except Exception:  # noqa
-                log.exception('Exception in task: %s, %r, %r', fn, args, kwargs)
-                raise
-        finally:
-            self._sem.release()
+    #
 
-    async def build_module_manifests(
-            self,
-            file: str,
-            *,
-            shell_wrap: bool = True,
-            warn_threshold_s: ta.Optional[float] = 1.,
-    ) -> ta.Sequence[Manifest]:
-        log.info('Extracting manifests from file %s', file)
+    @dc.dataclass(frozen=True)
+    class FileModule:
+        file: str
+
+        mod_name: str
+        mod_base: str
 
+    def build_file_module(self, file: str) -> FileModule:
         if not file.endswith('.py'):
             raise Exception(file)
 
@@ -134,31 +152,41 @@ class ManifestBuilder:
         if mod_base != (first_dir := file.split(os.path.sep)[0]):
             raise Exception(f'Unexpected module base: {mod_base=} != {first_dir=}')
 
-        with open(os.path.join(self._base, file)) as f:  # noqa
+        return ManifestBuilder.FileModule(
+            file=file,
+
+            mod_name=mod_name,
+            mod_base=mod_base,
+        )
+
+    #
+
+    def collect_module_manifest_targets(self, fm: FileModule) -> ta.List[ManifestDumperTarget]:
+        with open(os.path.join(self._base_dir, fm.file)) as f:  # noqa
             src = f.read()
 
         lines = src.splitlines(keepends=True)
 
         def prepare(s: str) -> ta.Any:
             if s.startswith('$.'):
-                s = f'{mod_base}.{s[2:]}'
+                s = f'{fm.mod_base}.{s[2:]}'
             return magic.py_compile_magic_preparer(s)
 
         magics = magic.find_magic(
             magic.PY_MAGIC_STYLE,
             lines,
-            file=file,
+            file=fm.file,
             keys={MANIFEST_MAGIC_KEY},
             preparer=prepare,
         )
 
         origins: ta.List[ManifestOrigin] = []
-        targets: ta.List[dict] = []
+        targets: ta.List[ManifestDumperTarget] = []
         for m in magics:
             if m.body:
                 body = m.body
                 if body.startswith('$.'):
-                    body = f'{mod_base}.{body[2:]}'
+                    body = f'{fm.mod_base}.{body[2:]}'
 
                 pat_match = check.not_none(_INLINE_MANIFEST_CLS_NAME_PAT.match(body))
                 cls_name = check.non_empty_str(pat_match.groupdict()['cls_name'])
@@ -184,15 +212,15 @@ class ManifestBuilder:
                 if issubclass(cls, ModAttrManifest):
                     attr_name = extract_manifest_target_name(lines, m.end_line)
                     inl_kw.update({
-                        'mod_name': mod_name,
+                        'mod_name': fm.mod_name,
                         'attr_name': attr_name,
                     })
 
                 origin = ManifestOrigin(
-                    module='.'.join(['', *mod_name.split('.')[1:]]),
+                    module='.'.join(['', *fm.mod_name.split('.')[1:]]),
                     attr=None,
 
-                    file=file,
+                    file=fm.file,
                     line=m.start_line,
                 )
 
@@ -210,10 +238,10 @@ class ManifestBuilder:
                 attr_name = extract_manifest_target_name(lines, m.end_line)
 
                 origin = ManifestOrigin(
-                    module='.'.join(['', *mod_name.split('.')[1:]]),
+                    module='.'.join(['', *fm.mod_name.split('.')[1:]]),
                     attr=attr_name,
 
-                    file=file,
+                    file=fm.file,
                     line=m.start_line,
                 )
 
@@ -230,9 +258,27 @@ class ManifestBuilder:
         if (dups := [k for k, v in collections.Counter(o.attr for o in origins if o.attr is not None).items() if v > 1]):  # noqa
             raise Exception(f'Duplicate attrs: {dups}')
 
+        return targets
+
+    #
+
+    async def _dump_module_manifests(
+            self,
+            fm: FileModule,
+            targets: ta.Sequence[ManifestDumperTarget],
+            *,
+            shell_wrap: bool = True,
+            warn_threshold_s: ta.Optional[float] = 1.,
+    ):
+        dumper_payload_src: str
+        if self._module_dumper_payload_src is not None:
+            dumper_payload_src = self._module_dumper_payload_src
+        else:
+            dumper_payload_src = _module_manifest_dumper_payload_src()
+
         subproc_src = '\n\n'.join([
-            _payload_src(),
-            f'_ModuleManifestDumper({mod_name!r})({", ".join(repr(tgt) for tgt in targets)})\n',
+            dumper_payload_src,
+            f'_ModuleManifestDumper({fm.mod_name!r})({", ".join(repr(tgt) for tgt in targets)})\n',
         ])
 
         args = [
@@ -246,7 +292,12 @@ class ManifestBuilder:
 
         start_time = time.time()
 
-        proc = await asyncio.create_subprocess_exec(*args, stdout=subprocess.PIPE)
+        proc = await asyncio.create_subprocess_exec(
+            *args,
+            stdout=subprocess.PIPE,
+            **(self._subprocess_kwargs or {}),
+        )
+
         subproc_out, _ = await proc.communicate()
         if proc.returncode:
             raise Exception('Subprocess failed')
@@ -254,17 +305,20 @@ class ManifestBuilder:
         end_time = time.time()
 
         if warn_threshold_s is not None and (elapsed_time := (end_time - start_time)) >= warn_threshold_s:
-            log.warning('Manifest extraction took a long time: %s, %.2f s', file, elapsed_time)
+            log.warning('Manifest extraction took a long time: %s, %.2f s', fm.file, elapsed_time)
 
         sp_lines = subproc_out.decode().strip().splitlines()
         if len(sp_lines) != 1:
             raise Exception('Unexpected subprocess output')
 
         sp_outs = json.loads(sp_lines[0])
-        # FIXME:
-        # if set(dct) != set(attrs):
-        #     raise Exception('Unexpected subprocess output keys')
+        return sp_outs
 
+    def _process_module_dump_output(
+            self,
+            fm: FileModule,
+            sp_outs: ta.Sequence[ta.Mapping[str, ta.Any]],
+    ) -> ta.List[Manifest]:
         out: ta.List[Manifest] = []
         for sp_out in sp_outs:
             value = sp_out['value']
@@ -278,7 +332,7 @@ class ManifestBuilder:
 
             [(key, value_dct)] = value.items()
             kb, _, kr = key[1:].partition('.')  # noqa
-            if kb == mod_base:  # noqa
+            if kb == fm.mod_base:  # noqa
                 key = f'$.{kr}'
                 value = {key: value_dct}
 
@@ -289,11 +343,45 @@ class ManifestBuilder:
 
         return out
 
+    #
+
+    async def build_module_manifests(self, file: str) -> ta.Sequence[Manifest]:
+        log.info('Extracting manifests from file %s', file)
+
+        fm = self.build_file_module(file)
+
+        targets = self.collect_module_manifest_targets(fm)
+
+        sp_outs = await self._dump_module_manifests(fm, targets)
+
+        # FIXME:
+        # if set(dct) != set(attrs):
+        #     raise Exception('Unexpected subprocess output keys')
+
+        out = self._process_module_dump_output(fm, sp_outs)
+
+        return out
+
+    #
+
+    async def _spawn(self, fn: ta.Callable[..., ta.Awaitable[T]], *args: ta.Any, **kwargs: ta.Any) -> T:
+        await self._sem.acquire()
+        try:
+            try:
+                return await fn(*args, **kwargs)
+            except Exception:  # noqa
+                log.exception('Exception in task: %s, %r, %r', fn, args, kwargs)
+                raise
+        finally:
+            self._sem.release()
+
     async def build_package_manifests(
             self,
             name: str,
+            *,
+            write: bool = False,
     ) -> ta.List[Manifest]:
-        pkg_dir = os.path.join(self._base, name)
+        pkg_dir = os.path.join(self._base_dir, name)
         if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
             raise Exception(pkg_dir)
 
@@ -305,12 +393,12 @@ class ManifestBuilder:
         manifests: ta.List[Manifest] = list(itertools.chain.from_iterable(await asyncio.gather(*[
             self._spawn(
                 self.build_module_manifests,
-                os.path.relpath(file, self._base),
+                os.path.relpath(file, self._base_dir),
             )
             for file in files
         ])))
 
-        if self._write:
+        if write:
             with open(os.path.join(pkg_dir, '.manifests.json'), 'w') as f:  # noqa
                 f.write(json_dumps_pretty([dc.asdict(m) for m in manifests]))
                 f.write('\n')
@@ -323,9 +411,9 @@ class ManifestBuilder:
 
 def check_package_manifests(
         name: str,
-        base: str,
+        base_dir: str,
 ) -> None:
-    pkg_dir = os.path.join(base, name)
+    pkg_dir = os.path.join(base_dir, name)
     if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
         raise Exception(pkg_dir)
 

omdev/manifests/dumping.py

@@ -1,4 +1,5 @@
 # ruff: noqa: UP006 UP007 UP037 UP045
+# @omlish-lite
 import typing as ta
 
 
@@ -6,10 +7,19 @@ import typing as ta
 
 
 class _ModuleManifestDumper:
-    def __init__(self, spec: str) -> None:
+    def __init__(
+            self,
+            spec: str,
+            *,
+            output: 'ta.Optional[ta.Callable[[str], None]]' = None,
+    ) -> None:
         super().__init__()
 
         self._spec = spec
+        if output is None:
+            output = print
+        self._output = output
+
         self._imported_mod: 'ta.Optional[ta.Any]' = None
 
     def _mod(self) -> 'ta.Any':
@@ -32,7 +42,10 @@ class _ModuleManifestDumper:
         def __missing__(self, key):
             return self.__get_missing(key)
 
-    def __call__(self, *targets: dict) -> None:
+    def __call__(
+            self,
+            *targets: dict,  # .build.ManifestDumperTarget
+    ) -> None:
         import collections.abc
         import dataclasses as dc  # noqa
         import functools
@@ -115,4 +128,4 @@ class _ModuleManifestDumper:
             })
 
         out_json = json.dumps(out, indent=None, separators=(',', ':'))
-        print(out_json)
+        self._output(out_json)

omdev/manifests/main.py

@@ -1,4 +1,5 @@
 # ruff: noqa: UP006 UP007 UP045
+# @omlish-lite
 import argparse
 import asyncio
 import dataclasses as dc
@@ -8,14 +9,14 @@ import os.path
 from omlish.lite.json import json_dumps_pretty
 from omlish.logs.standard import configure_standard_logging
 
-from .build import ManifestBuilder
-from .build import check_package_manifests
+from .building import ManifestBuilder
+from .building import check_package_manifests
 
 
 ##
 
 
-def _get_base(args) -> str:
+def _get_base_dir(args) -> str:
     if args.base is not None:
         base = args.base
     else:
@@ -27,18 +28,20 @@ def _get_base(args) -> str:
 
 
 def _gen_cmd(args) -> None:
-    base = _get_base(args)
+    base_dir = _get_base_dir(args)
 
     jobs = args.jobs or int(max(mp.cpu_count() // 1.5, 1))
     builder = ManifestBuilder(
-        base,
+        base_dir,
         jobs,
-        write=args.write or False,
     )
 
     async def do():
         return await asyncio.gather(*[
-            builder.build_package_manifests(pkg)
+            builder.build_package_manifests(
+                pkg,
+                write=bool(args.write),
+            )
             for pkg in args.package
         ])
 
@@ -49,12 +52,12 @@ def _gen_cmd(args) -> None:
 
 
 def _check_cmd(args) -> None:
-    base = _get_base(args)
+    base_dir = _get_base_dir(args)
 
     for pkg in args.package:
         check_package_manifests(
             pkg,
-            base,
+            base_dir,
         )
 
 

{omdev-0.0.0.dev392.dist-info → omdev-0.0.0.dev393.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omdev
-Version: 0.0.0.dev392
+Version: 0.0.0.dev393
 Summary: omdev
 Author: wrmsr
 License-Expression: BSD-3-Clause
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev392
+Requires-Dist: omlish==0.0.0.dev393
 Provides-Extra: all
 Requires-Dist: black~=25.1; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev392.dist-info → omdev-0.0.0.dev393.dist-info}/RECORD

@@ -1,4 +1,4 @@
-omdev/.manifests.json,sha256=gnB4QF-VVal4TXg8B48JefTAsjmkXGk7imY8vHYyejc,11863
+omdev/.manifests.json,sha256=IuYiaGqGlhwqLm9NpW2koVLjr5dmDxt-BztIpn3E9YQ,12170
 omdev/__about__.py,sha256=fQNmzSa1MntcPSrzg_Vpo6JRU2RbXik2NqRz0oQCApE,1202
 omdev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/cmake.py,sha256=9rfSvFHPmKDj9ngvfDB2vK8O-xO_ZwUm7hMKLWA-yOw,4578
@@ -175,11 +175,11 @@ omdev/magic/find.py,sha256=n9v4XzkYAyh9MVAXRBSmiOlLgZSVbwLYrSJdQBx0G8s,6223
 omdev/magic/magic.py,sha256=M1aOpp1eimbSxZSlKLbobXRZ15MoRrTtLWK-FQrxtGI,235
 omdev/magic/prepare.py,sha256=SEOK-bl4zDxq0aphYXsEI-hCjbkV908VNnJt-dk0kL4,594
 omdev/magic/styles.py,sha256=6LAL7XR3fkkH2rh-8nwUvdCYVHBkQxCfP0oEuPuw1Bg,670
-omdev/manifests/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
+omdev/manifests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/manifests/__main__.py,sha256=JqyVDyV7_jo-NZ3wSs5clDU_xCMlxzJv-XFohoZWQ7E,174
-omdev/manifests/build.py,sha256=HirTi1z0hC_XzUNmvLWsCapr6OrywYk7s1ZQmd-islA,11017
-omdev/manifests/dumping.py,sha256=Pldka4gNa87w1OOKfcP2IvmyNW_hZ5G_xsOVPD2yR0Q,4059
-omdev/manifests/main.py,sha256=g2ju2dZtUuZxDC1yL9RvnqzA7dWmvqvbWfTQG54bEh0,2137
+omdev/manifests/building.py,sha256=VMTSHQgIYv7ysm-ne2aIjj1RDeyh57rgtV0RiarK1RI,13372
+omdev/manifests/dumping.py,sha256=CKUCHeO-7cLwCDmzc75FX7fvqIRmOx4s10Vx7nB3C6w,4342
+omdev/manifests/main.py,sha256=jh7YN2D8YCDkT1ptDIv-HK0aMJJzuTw6CZXIM98mTgs,2222
 omdev/mypy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/mypy/debug.py,sha256=VskRcr9trNhyPG2ErZZ7IX_v1DLKTLBOjp34o-fEWaM,3294
 omdev/mypy/report.py,sha256=faX1GmbX_irmZptsOZN2HJnmHxxnrqxScmPdl7B3BI4,1570
@@ -316,9 +316,9 @@ omdev/tools/jsonview/resources/jsonview.js,sha256=faDvXDOXKvEvjOuIlz4D3F2ReQXb_b
 omdev/tools/pawk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/tools/pawk/__main__.py,sha256=VCqeRVnqT1RPEoIrqHFSu4PXVMg4YEgF4qCQm90-eRI,66
 omdev/tools/pawk/pawk.py,sha256=ao5mdrpiSU4AZ8mBozoEaV3UVlmVTnRG9wD9XP70MZE,11429
-omdev-0.0.0.dev392.dist-info/licenses/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-omdev-0.0.0.dev392.dist-info/METADATA,sha256=YGB9TXCuzKQ-Qy2_P2jEdUGVFOttzZCAYK-4GvAoltY,5094
-omdev-0.0.0.dev392.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-omdev-0.0.0.dev392.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
-omdev-0.0.0.dev392.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
-omdev-0.0.0.dev392.dist-info/RECORD,,
+omdev-0.0.0.dev393.dist-info/licenses/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+omdev-0.0.0.dev393.dist-info/METADATA,sha256=eQk1MmelIztqeIJPUYpqr501nWXSQSIBkD5ZzgXHh4M,5094
+omdev-0.0.0.dev393.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+omdev-0.0.0.dev393.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
+omdev-0.0.0.dev393.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
+omdev-0.0.0.dev393.dist-info/RECORD,,