omdev 0.0.0.dev52__py3-none-any.whl → 0.0.0.dev54__py3-none-any.whl

omdev/cli/main.py

@@ -64,9 +64,10 @@ def _main() -> None:
 
     dct: dict[str, CliCmd] = {}
     for cc in ccs:
-        if cc.cmd_name in dct:
-            raise NameError(cc)
-        dct[cc.cmd_name] = cc
+        for cn in [cc.cmd_name] if isinstance(cc.cmd_name, str) else cc.cmd_name:
+            if cn in dct:
+                raise NameError(cc)
+            dct[cn] = cc
 
     #
 
@@ -76,7 +77,26 @@ def _main() -> None:
 
     args = parser.parse_args()
     if not args.cmd:
-        parser.print_help()
+        mdct: dict = {}
+        for cc in ccs:
+            if isinstance(cc.cmd_name, str) and cc.cmd_name[0] == '_':
+                continue
+            if isinstance(cc, CliFunc):
+                mdct.setdefault('-', []).append(cc)
+            elif isinstance(cc, CliModule):
+                mdct.setdefault(cc.mod_name.partition('.')[0], []).append(cc)
+            else:
+                raise TypeError(cc)
+
+        print('Subcommands:\n')
+        for m, l in sorted(mdct.items(), key=lambda t: (t[0] == '-', t[0])):
+            print(f'  {m}')
+            for cc in sorted(l, key=lambda c: c.primary_name):
+                if isinstance(cc.cmd_name, str):
+                    print(f'    {cc.cmd_name}')
+                else:
+                    print(f'    {cc.cmd_name[0]} ({", ".join(cc.cmd_name[1:])})')
+            print()
         return
 
     #
@@ -84,7 +104,7 @@ def _main() -> None:
     cc = dct[args.cmd]
 
     if isinstance(cc, CliModule):
-        sys.argv = [cc.cmd_name, *(args.args or ())]
+        sys.argv = [args.cmd, *(args.args or ())]
         runpy._run_module_as_main(cc.mod_name)  # type: ignore  # noqa
 
     elif isinstance(cc, CliFunc):

omdev/cli/types.py

@@ -1,10 +1,18 @@
+# ruff: noqa: UP007
 import dataclasses as dc
 import typing as ta
 
 
 @dc.dataclass(frozen=True)
 class CliCmd:
-    cmd_name: str
+    cmd_name: ta.Union[str, ta.Sequence[str]]
+
+    @property
+    def primary_name(self) -> str:
+        if isinstance(self.cmd_name, str):
+            return self.cmd_name
+        else:
+            return self.cmd_name[0]
 
 
 @dc.dataclass(frozen=True)

omdev/manifests/build.py

@@ -12,10 +12,9 @@ See (entry_points):
 """
 # ruff: noqa: UP006 UP007
 import argparse
+import asyncio
 import collections
-import concurrent.futures as cf
 import dataclasses as dc
-import functools
 import inspect
 import itertools
 import json
@@ -39,6 +38,9 @@ from .types import Manifest
 from .types import ManifestOrigin
 
 
+T = ta.TypeVar('T')
+
+
 ##
 
 
@@ -98,138 +100,160 @@ def _payload_src() -> str:
     return inspect.getsource(_dump_module_manifests)
 
 
-def build_module_manifests(
-        file: str,
-        base: str,
-        *,
-        shell_wrap: bool = True,
-        warn_threshold_s: ta.Optional[float] = 1.,
-) -> ta.Sequence[Manifest]:
-    log.info('Extracting manifests from file %s', file)
-
-    if not file.endswith('.py'):
-        raise Exception(file)
-
-    mod_name = file.rpartition('.')[0].replace(os.sep, '.')
-    mod_base = mod_name.split('.')[0]
-    if mod_base != (first_dir := file.split(os.path.sep)[0]):
-        raise Exception(f'Unexpected module base: {mod_base=} != {first_dir=}')
-
-    with open(os.path.join(base, file)) as f:
-        src = f.read()
-
-    origins: ta.List[ManifestOrigin] = []
-    lines = src.splitlines(keepends=True)
-    for i, l in enumerate(lines):
-        if l.startswith(MANIFEST_MAGIC):
-            if (m := _MANIFEST_GLOBAL_PAT.match(nl := lines[i + 1])) is None:
-                raise Exception(nl)
-
-            origins.append(ManifestOrigin(
-                module='.'.join(['', *mod_name.split('.')[1:]]),
-                attr=m.groupdict()['name'],
-
-                file=file,
-                line=i + 1,
-            ))
+class ManifestBuilder:
+    def __init__(
+            self,
+            base: str,
+            concurrency: int = 8,
+            *,
+            write: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._base = base
+        self._sem = asyncio.Semaphore(concurrency)
+        self._write = write
+
+    async def _spawn(self, fn: ta.Callable[..., ta.Awaitable[T]], *args: ta.Any, **kwargs: ta.Any) -> T:
+        await self._sem.acquire()
+        try:
+            try:
+                return await fn(*args, **kwargs)
+            except Exception:  # noqa
+                log.exception('Exception in task: %s, %r, %r', fn, args, kwargs)
+                raise
+        finally:
+            self._sem.release()
+
+    async def build_module_manifests(
+            self,
+            file: str,
+            *,
+            shell_wrap: bool = True,
+            warn_threshold_s: ta.Optional[float] = 1.,
+    ) -> ta.Sequence[Manifest]:
+        log.info('Extracting manifests from file %s', file)
+
+        if not file.endswith('.py'):
+            raise Exception(file)
+
+        mod_name = file.rpartition('.')[0].replace(os.sep, '.')
+        mod_base = mod_name.split('.')[0]
+        if mod_base != (first_dir := file.split(os.path.sep)[0]):
+            raise Exception(f'Unexpected module base: {mod_base=} != {first_dir=}')
+
+        with open(os.path.join(self._base, file)) as f:  # noqa
+            src = f.read()
+
+        origins: ta.List[ManifestOrigin] = []
+        lines = src.splitlines(keepends=True)
+        for i, l in enumerate(lines):
+            if l.startswith(MANIFEST_MAGIC):
+                if (m := _MANIFEST_GLOBAL_PAT.match(nl := lines[i + 1])) is None:
+                    raise Exception(nl)
+
+                origins.append(ManifestOrigin(
+                    module='.'.join(['', *mod_name.split('.')[1:]]),
+                    attr=m.groupdict()['name'],
+
+                    file=file,
+                    line=i + 1,
+                ))
 
-    if not origins:
-        raise Exception('no manifests found')
+        if not origins:
+            raise Exception('no manifests found')
 
-    if (dups := [k for k, v in collections.Counter(o.attr for o in origins).items() if v > 1]):
-        raise Exception(f'Duplicate attrs: {dups}')
+        if (dups := [k for k, v in collections.Counter(o.attr for o in origins).items() if v > 1]):
+            raise Exception(f'Duplicate attrs: {dups}')
 
-    attrs = [o.attr for o in origins]
+        attrs = [o.attr for o in origins]
 
-    subproc_src = '\n\n'.join([
-        _payload_src(),
-        f'_dump_module_manifests({mod_name!r}, {", ".join(repr(a) for a in attrs)})\n',
-    ])
+        subproc_src = '\n\n'.join([
+            _payload_src(),
+            f'_dump_module_manifests({mod_name!r}, {", ".join(repr(a) for a in attrs)})\n',
+        ])
 
-    args = [
-        sys.executable,
-        '-c',
-        subproc_src,
-    ]
+        args = [
+            sys.executable,
+            '-c',
+            subproc_src,
+        ]
 
-    if shell_wrap:
-        args = ['sh', '-c', ' '.join(map(shlex.quote, args))]
+        if shell_wrap:
+            args = ['sh', '-c', ' '.join(map(shlex.quote, args))]
 
-    start_time = time.time()
+        start_time = time.time()
 
-    subproc_out = subprocess.check_output(args)
+        proc = await asyncio.create_subprocess_exec(*args, stdout=subprocess.PIPE)
+        subproc_out, _ = await proc.communicate()
+        if proc.returncode:
+            raise Exception('Subprocess failed')
 
-    end_time = time.time()
+        end_time = time.time()
 
-    if warn_threshold_s is not None and (elapsed_time := (end_time - start_time)) >= warn_threshold_s:
-        log.warning('Manifest extraction took a long time: %s, %.2f s', file, elapsed_time)
+        if warn_threshold_s is not None and (elapsed_time := (end_time - start_time)) >= warn_threshold_s:
+            log.warning('Manifest extraction took a long time: %s, %.2f s', file, elapsed_time)
 
-    sp_lines = subproc_out.decode().strip().splitlines()
-    if len(sp_lines) != 1:
-        raise Exception('Unexpected subprocess output')
+        sp_lines = subproc_out.decode().strip().splitlines()
+        if len(sp_lines) != 1:
+            raise Exception('Unexpected subprocess output')
 
-    dct = json.loads(sp_lines[0])
-    if set(dct) != set(attrs):
-        raise Exception('Unexpected subprocess output keys')
+        dct = json.loads(sp_lines[0])
+        if set(dct) != set(attrs):
+            raise Exception('Unexpected subprocess output keys')
 
-    out: ta.List[Manifest] = []
+        out: ta.List[Manifest] = []
 
-    for o in origins:
-        value = dct[o.attr]
+        for o in origins:
+            value = dct[o.attr]
 
-        if not (
-                isinstance(value, ta.Mapping) and
-                len(value) == 1 and
-                all(isinstance(k, str) and k.startswith('$') and len(k) > 1 for k in value)
-        ):
-            raise TypeError(f'Manifests must be mappings of strings starting with $: {value!r}')
+            if not (
+                    isinstance(value, ta.Mapping) and
+                    len(value) == 1 and
+                    all(isinstance(k, str) and k.startswith('$') and len(k) > 1 for k in value)
+            ):
+                raise TypeError(f'Manifests must be mappings of strings starting with $: {value!r}')
 
-        [(key, value_dct)] = value.items()
-        kb, _, kr = key[1:].partition('.')  # noqa
-        if kb == mod_base:  # noqa
-            key = f'$.{kr}'
-            value = {key: value_dct}
+            [(key, value_dct)] = value.items()
+            kb, _, kr = key[1:].partition('.')  # noqa
+            if kb == mod_base:  # noqa
+                key = f'$.{kr}'
+                value = {key: value_dct}
 
-        out.append(Manifest(
-            **dc.asdict(o),
-            value=value,
-        ))
+            out.append(Manifest(
+                **dc.asdict(o),
+                value=value,
+            ))
 
-    return out
+        return out
 
+    async def build_package_manifests(
+            self,
+            name: str,
+    ) -> ta.List[Manifest]:
+        pkg_dir = os.path.join(self._base, name)
+        if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
+            raise Exception(pkg_dir)
 
-def build_package_manifests(
-        ex: cf.Executor,
-        name: str,
-        base: str,
-        *,
-        write: bool = False,
-) -> ta.List[Manifest]:
-    pkg_dir = os.path.join(base, name)
-    if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
-        raise Exception(pkg_dir)
-
-    files = sorted(findmagic.find_magic(
-        [pkg_dir],
-        [MANIFEST_MAGIC],
-        ['py'],
-    ))
-    futs = [
-        ex.submit(functools.partial(
-            build_module_manifests,
-            os.path.relpath(file, base),
-            base,
+        files = sorted(findmagic.find_magic(
+            [pkg_dir],
+            [MANIFEST_MAGIC],
+            ['py'],
         ))
-        for file in files
-    ]
-    manifests: ta.List[Manifest] = list(itertools.chain.from_iterable(fut.result() for fut in futs))
+        manifests: ta.List[Manifest] = list(itertools.chain.from_iterable(await asyncio.gather(*[
+            self._spawn(
+                self.build_module_manifests,
+                os.path.relpath(file, self._base),
+            )
+            for file in files
+        ])))
 
-    if write:
-        with open(os.path.join(pkg_dir, '.manifests.json'), 'w') as f:
-            f.write(json_dumps_pretty([dc.asdict(m) for m in manifests]))
-            f.write('\n')
+        if self._write:
+            with open(os.path.join(pkg_dir, '.manifests.json'), 'w') as f:  # noqa
+                f.write(json_dumps_pretty([dc.asdict(m) for m in manifests]))
+                f.write('\n')
 
-    return manifests
+        return manifests
 
 
 ##
@@ -277,22 +301,23 @@ if __name__ == '__main__':
     def _gen_cmd(args) -> None:
         base = _get_base(args)
 
-        num_threads = args.jobs or max(mp.cpu_count() // 2, 1)
-        with cf.ThreadPoolExecutor(num_threads) as ex:
-            futs = [
-                ex.submit(functools.partial(
-                    build_package_manifests,
-                    ex,
-                    pkg,
-                    base,
-                    write=args.write or False,
-                ))
+        jobs = args.jobs or int(max(mp.cpu_count() // 1.5, 1))
+        builder = ManifestBuilder(
+            base,
+            jobs,
+            write=args.write or False,
+        )
+
+        async def do():
+            return await asyncio.gather(*[
+                builder.build_package_manifests(pkg)
                 for pkg in args.package
-            ]
-            mss = [fut.result() for fut in futs]
-            if not args.quiet:
-                for ms in mss:
-                    print(json_dumps_pretty([dc.asdict(m) for m in ms]))
+            ])
+
+        mss = asyncio.run(do())
+        if not args.quiet:
+            for ms in mss:
+                print(json_dumps_pretty([dc.asdict(m) for m in ms]))
 
     def _check_cmd(args) -> None:
         base = _get_base(args)

omdev/pyproject/cli.py

@@ -348,7 +348,7 @@ def _pkg_cmd(args) -> None:
         ]
         pgs = list(itertools.chain.from_iterable([pg, *pg.children()] for pg in pgs))
 
-        num_threads = args.jobs or max(mp.cpu_count() // 2, 1)
+        num_threads = args.jobs or int(max(mp.cpu_count() // 1.5, 1))
         futs: ta.List[cf.Future]
         with cf.ThreadPoolExecutor(num_threads) as ex:
             futs = [ex.submit(pg.gen) for pg in pgs]

omdev/scripts/pyproject.py

@@ -5287,7 +5287,7 @@ def _pkg_cmd(args) -> None:
         ]
         pgs = list(itertools.chain.from_iterable([pg, *pg.children()] for pg in pgs))
 
-        num_threads = args.jobs or max(mp.cpu_count() // 2, 1)
+        num_threads = args.jobs or int(max(mp.cpu_count() // 1.5, 1))
         futs: ta.List[cf.Future]
         with cf.ThreadPoolExecutor(num_threads) as ex:
             futs = [ex.submit(pg.gen) for pg in pgs]

{omdev-0.0.0.dev52.dist-info → omdev-0.0.0.dev54.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev52
+Version: 0.0.0.dev54
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omlish ==0.0.0.dev52
+Requires-Dist: omlish ==0.0.0.dev54
 Provides-Extra: all
 Requires-Dist: pycparser ~=2.22 ; extra == 'all'
 Requires-Dist: cffi ~=1.17 ; extra == 'all'

{omdev-0.0.0.dev52.dist-info → omdev-0.0.0.dev54.dist-info}/RECORD

@@ -57,9 +57,9 @@ omdev/cli/__init__.py,sha256=V_l6VP1SZMlJbO-8CJwSuO9TThOy2S_oaPepNYgIrbE,37
 omdev/cli/__main__.py,sha256=5IeIERm-371fSI5ZvPv8eldAJBwgKwpR0R49pTsILNM,76
 omdev/cli/clicli.py,sha256=rQ9-F9XI3BevnfIVALHVKrrHNQPdYhkQ-IKESTNB5RA,2447
 omdev/cli/install.py,sha256=C-W171YlIHt4Cfok-nWSMbHwWhqF_PFqq2HixFttYx8,4460
-omdev/cli/main.py,sha256=wVhrzwEfUFKjlfSeJK5CM23yId4v3-36IuY4VbrOSQA,1873
+omdev/cli/main.py,sha256=uyrpniI2nz84EV5-mFbcKJXf0sROJXSomEsangsRI_w,2751
 omdev/cli/managers.py,sha256=zqhMgNnv2VbXoTP6SgZeSuWfAVo1Ge-HJnQ1ftL-Mq8,2590
-omdev/cli/types.py,sha256=7_Owg0P8C8oOObSuOp6aEYSjkEukVFxTT00SRy1bLHM,250
+omdev/cli/types.py,sha256=bqKw9SbtBtAip2vF9v4khh0CqKG6LBr6n9VzWBz7AJE,474
 omdev/interp/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/interp/__main__.py,sha256=GMCqeGYltgt5dlJzHxY9gqisa8cRkrPfmZYuZnjg4WI,162
 omdev/interp/cli.py,sha256=sh7PZQoLletUViw1Y9OXNr9ekyNZ6YyxYuOQ_n9hyqU,2072
@@ -71,7 +71,7 @@ omdev/interp/standalone.py,sha256=XcltiL7ypcfV89C82_3knQ3Kx7aW4wnnxf2056ZXC3A,77
 omdev/interp/system.py,sha256=bI-JhX4GVJqW7wMxnIa-DGJWnCLmFcIsnl9pc1RGY2g,3513
 omdev/interp/types.py,sha256=EMN3StEMkFoQAMUIZd7JYL4uUWqzFGY-2hTL8EBznYM,2437
 omdev/manifests/__init__.py,sha256=P2B0dpT8D7l5lJwRGPA92IcQj6oeXfd90X5-q9BJrKg,51
-omdev/manifests/build.py,sha256=UxmuJhnWG16nwyFU9bTsv_FRzQ3lw1nZvAFI_XI3wuY,9727
+omdev/manifests/build.py,sha256=65Bw8gc8avQdXveKqO6hG25LQjdkXknY3cypojmw7Js,10823
 omdev/manifests/load.py,sha256=LtEsluDdd8CkNGj0QGBxee3twBn095Fru0xz2mtr7uk,4788
 omdev/manifests/types.py,sha256=Jv6PAdVLPb9Hh4y6vDhPlWuMNBBViin1bC_u83jfsH4,234
 omdev/mypy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -91,7 +91,7 @@ omdev/precheck/scripts.py,sha256=qq6MXkxgrYngPg5pWnXH4uRSuRkP3mFqbeml1UmvGBc,126
 omdev/pyproject/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/pyproject/__main__.py,sha256=gn3Rl1aYPYdiTtEqa9ifi0t-e4ZwPY0vhJ4UXvYdJDY,165
 omdev/pyproject/cexts.py,sha256=x13piOOnNrYbA17qZLDVuR0p1sqhgEwpk4FtImX-klM,4281
-omdev/pyproject/cli.py,sha256=QnlptoCJQ0yvJ1C2mXEYfOMS6agRQMn2nGS9KgWoJ5U,11358
+omdev/pyproject/cli.py,sha256=kw0Jyzywwt6QOc6QHDevWW9p85aXT7jvVFFb8Sb_EAU,11365
 omdev/pyproject/configs.py,sha256=K9H5cGwVLgHi8wKwtYvlXHZ9ThtmnI4jo8JAb-t1-70,2859
 omdev/pyproject/pkg.py,sha256=rNKzJOIgPDrtT2i14Pebldoboz45w00sKb5l_kYFaRI,14562
 omdev/pyproject/reqs.py,sha256=8feZ71YnGzwKbLK4zO28CDQeNcZIIuq6cnkBhs6M-7E,2406
@@ -101,7 +101,7 @@ omdev/scripts/execrss.py,sha256=d6purJqU99OkMcNxCS1kG2CMfSsw7wjnvBQW7SjyJ70,448
 omdev/scripts/exectime.py,sha256=LRVIJsnvGYUqH-zfCdFnrQCZv1KLXJPtWBWplXMDwhI,407
 omdev/scripts/importtrace.py,sha256=4ozdphT4VuP8L0kE0HmGMyWaWHnXg1KEukbET4pFgJU,14064
 omdev/scripts/interp.py,sha256=7xeihsrt3DI_s1fZqNweivBUUwuGkS5C5cWw4QY6q30,70999
-omdev/scripts/pyproject.py,sha256=ETGBPUdMeifuDdG7lcf8HUBVRwLPdx3VE_GfvhulTrY,157486
+omdev/scripts/pyproject.py,sha256=fuCXOk44Ui5cangCFGi44ckPezawW2N9aJ8bdOAxL50,157493
 omdev/toml/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/toml/parser.py,sha256=84bn09uhYHwQGyfww6Rw6y1RxPAE_HDltODOSakcqDM,29186
 omdev/toml/writer.py,sha256=lk3on3YXVbWuLJa-xsOzOhs1bBAT1vXqw4mBbluZl_w,3040
@@ -114,9 +114,9 @@ omdev/tools/piptools.py,sha256=-jR5q3w4sHqntxCLExFCBNIARB788FUsAbJ62PK2sBU,2774
 omdev/tools/proftools.py,sha256=xKSm_yPoCnfsvS3iT9MblDqFMuZmGfI3_koGj8amMyU,145
 omdev/tools/rst.py,sha256=6dWk8QZHoGiLSuBw3TKsXZjjFK6wWBEtPi9krdCLKKg,977
 omdev/tools/sqlrepl.py,sha256=tmFZh80-xsGM62dyQ7_UGLebChrj7IHbIPYBWDJMgVk,5741
-omdev-0.0.0.dev52.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-omdev-0.0.0.dev52.dist-info/METADATA,sha256=i-4H3TQh4h9ahk0VXKuSobql_G4gPEjzk_PVRfqjF3A,1252
-omdev-0.0.0.dev52.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-omdev-0.0.0.dev52.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
-omdev-0.0.0.dev52.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
-omdev-0.0.0.dev52.dist-info/RECORD,,
+omdev-0.0.0.dev54.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+omdev-0.0.0.dev54.dist-info/METADATA,sha256=C1LUlE7rmomgvnwcZ8ChfuCsoULQWW-wuRrFMpZe-vQ,1252
+omdev-0.0.0.dev54.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+omdev-0.0.0.dev54.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
+omdev-0.0.0.dev54.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
+omdev-0.0.0.dev54.dist-info/RECORD,,