omdev 0.0.0.dev40__py3-none-any.whl → 0.0.0.dev43__py3-none-any.whl

omdev/.manifests.json

@@ -35,6 +35,18 @@
       }
     }
   },
+  {
+    "module": ".cli.clicmds",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/cli/clicmds.py",
+    "line": 57,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": "cli",
+        "mod_name": "omdev.cli.clicmds"
+      }
+    }
+  },
   {
     "module": ".interp.__main__",
     "attr": "_CLI_MODULE",
@@ -71,6 +83,42 @@
       }
     }
   },
+  {
+    "module": ".scripts.execrss",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/scripts/execrss.py",
+    "line": 11,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": "execrss",
+        "mod_name": "omdev.scripts.execrss"
+      }
+    }
+  },
+  {
+    "module": ".scripts.exectime",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/scripts/exectime.py",
+    "line": 7,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": "exectime",
+        "mod_name": "omdev.scripts.exectime"
+      }
+    }
+  },
+  {
+    "module": ".scripts.importtrace",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/scripts/importtrace.py",
+    "line": 481,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": "importtrace",
+        "mod_name": "omdev.scripts.importtrace"
+      }
+    }
+  },
   {
     "module": ".tools.dockertools",
     "attr": "_CLI_MODULE",
@@ -95,6 +143,18 @@
       }
     }
   },
+  {
+    "module": ".tools.importscan",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/tools/importscan.py",
+    "line": 165,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": "importscan",
+        "mod_name": "omdev.tools.importscan"
+      }
+    }
+  },
   {
     "module": ".tools.piptools",
     "attr": "_CLI_MODULE",

omdev/cache/data/cache.py

@@ -10,11 +10,14 @@ TODO:
  - chaining? or is this compcache..
  - download resume ala hf_hub
 """
+import contextlib
 import logging
 import os.path
 import shutil
 import subprocess
 import tempfile
+import typing as ta
+import urllib.error
 import urllib.parse
 import urllib.request
 
@@ -40,6 +43,57 @@ log = logging.getLogger(__name__)
 ##
 
 
+def _url_retrieve(
+        req: urllib.request.Request,
+        out_file: str | None = None,
+) -> tuple[str, ta.Any]:
+    p = urllib.parse.urlparse(req.full_url)
+
+    with contextlib.ExitStack() as es:
+        fp = es.enter_context(contextlib.closing(urllib.request.urlopen(req)))  # noqa
+
+        headers = fp.info()
+
+        # Just return the local path and the "headers" for file:// URLs. No sense in performing a copy unless requested.
+        if p.scheme == 'file' and not out_file:
+            return os.path.normpath(p.path), headers
+
+        success = False
+
+        tfp: ta.Any
+        if out_file:
+            tfp = es.enter_context(open(out_file, 'wb'))
+
+        else:
+            tfp = es.enter_context(tempfile.NamedTemporaryFile(delete=False))
+            out_file = tfp.name
+
+            def _cleanup():
+                if not success and out_file:
+                    os.unlink(out_file)
+
+            es.enter_context(lang.defer(_cleanup))  # noqa
+
+        result = out_file, headers
+        size = -1
+        read = 0
+        if 'content-length' in headers:
+            size = int(headers['Content-Length'])
+
+        while block := fp.read(1024 * 8):
+            read += len(block)
+            tfp.write(block)
+
+    if size >= 0 and read < size:
+        raise urllib.error.ContentTooShortError(
+            f'retrieval incomplete: got only {read} out of {size} bytes',
+            result,  # type: ignore
+        )
+
+    success = True
+    return result  # type: ignore
+
+
 class Cache:
     def __init__(self, base_dir: str) -> None:
         super().__init__()
@@ -49,16 +103,32 @@ class Cache:
 
     #
 
-    def _fetch_url(self, url: str, out_file: str) -> None:
+    def _fetch_url(
+            self,
+            url: str,
+            out_file: str,
+            *,
+            headers: ta.Mapping[str, str] | None = None,
+    ) -> None:
         log.info('Fetching url: %s -> %s', url, out_file)
 
-        urllib.request.urlretrieve(url, out_file)  # noqa
+        _url_retrieve(
+            urllib.request.Request(  # noqa
+                url,
+                **(dict(headers=headers) if headers is not None else {}),  # type: ignore
+            ),
+            out_file,
+        )
 
     def _fetch_into(self, spec: Spec, data_dir: str) -> None:
         log.info('Fetching spec: %s %r -> %s', spec.digest, spec, data_dir)
 
         if isinstance(spec, UrlSpec):
-            self._fetch_url(spec.url, os.path.join(data_dir, spec.file_name_or_default))
+            self._fetch_url(
+                spec.url,
+                os.path.join(data_dir, spec.file_name_or_default),
+                headers=spec.headers,
+            )
 
         elif isinstance(spec, GithubContentSpec):
             for repo_file in spec.files:
@@ -104,7 +174,7 @@ class Cache:
 
     def _perform_action(self, action: Action, data_dir: str) -> None:
         if isinstance(action, ExtractAction):
-            for f in action.files:
+            for f in [action.files] if isinstance(action.files, str) else action.files:
                 file = os.path.join(data_dir, f)
                 if not os.path.isfile(file):
                     raise Exception(f'Not file: {file}')

omdev/cache/data/specs.py

@@ -1,4 +1,5 @@
 import hashlib
+import operator
 import typing as ta
 import urllib.parse
 import urllib.request
@@ -59,6 +60,10 @@ class UrlSpec(Spec):
     url: str = dc.xfield(validate=lambda u: bool(urllib.parse.urlparse(u)))
     file_name: str | None = None
 
+    _: dc.KW_ONLY
+
+    headers: ta.Mapping[str, str] | None = dc.field(default=None) | msh.update_field_metadata(omit_if=operator.not_)
+
     @cached.property
     def file_name_or_default(self) -> str:
         if self.file_name is not None:

omdev/cli/clicmds.py

@@ -0,0 +1,83 @@
+import argparse
+import inspect
+import os
+import subprocess
+import sys
+
+from omlish import __about__
+
+from . import install
+from .types import CliModule
+
+
+def _print_version(args) -> None:
+    print(__about__.__version__)
+
+
+def _print_revision(args) -> None:
+    print(__about__.__revision__)
+
+
+def _reinstall(args) -> None:
+    mod_name = globals()['__spec__'].name
+    tool_name = '.'.join([mod_name.partition('.')[0], 'tools', 'piptools'])
+
+    out = subprocess.check_output([
+        sys.executable,
+        '-m',
+        tool_name,
+        'list-root-dists',
+    ]).decode()
+
+    deps = sorted(
+        ({s for l in out.splitlines() if (s := l.strip())} | set(args.extra_deps or []))
+        - {install.DEFAULT_CLI_PKG}  # noqa
+    )
+
+    if deps:
+        print('Reinstalling with following additional dependencies:')
+        print('\n'.join('  ' + d for d in deps))
+    else:
+        print('No additional dependencies detected.')
+    print()
+    print('Continue with reinstall? (ctrl-c to cancel)')
+    input()
+
+    install_src = inspect.getsource(install)
+
+    os.execl(
+        sys.executable,
+        sys.executable,
+        '-c',
+        install_src,
+        *deps,
+    )
+
+
+# @omlish-manifest
+_CLI_MODULE = CliModule('cli', __name__)
+
+
+def _main(argv=None) -> None:
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+
+    parser_version = subparsers.add_parser('version')
+    parser_version.set_defaults(func=_print_version)
+
+    parser_revision = subparsers.add_parser('revision')
+    parser_revision.set_defaults(func=_print_revision)
+
+    parser_reinstall = subparsers.add_parser('reinstall')
+    parser_reinstall.add_argument('extra_deps', nargs='*')
+    parser_reinstall.set_defaults(func=_reinstall)
+
+    args = parser.parse_args(argv)
+    if not getattr(args, 'func', None):
+        parser.print_help()
+    else:
+        args.func(args)
+
+
+if __name__ == '__main__':
+    _main()

omdev/cli/main.py

@@ -1,5 +1,7 @@
 """
 TODO:
+ - cache ldr.discover() somehow if in uvx/pipx - very slow
+  - <venv-root>/.omdev-cli-manifest-cache.json - {pkg_name: manifests_json}
  - allow manually specifying manifest packages
  - https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#creating-executable-scripts
  - https://packaging.python.org/en/latest/specifications/entry-points/#entry-points
@@ -8,8 +10,8 @@ import argparse
 import os
 import runpy
 import sys
+import typing as ta
 
-from omlish import __about__
 from omlish import check
 
 from ..manifests.load import ManifestLoader
@@ -21,18 +23,7 @@ from .types import CliModule
 ##
 
 
-def _print_version() -> None:
-    print(__about__.__version__)
-
-
-def _print_revision() -> None:
-    print(__about__.__revision__)
-
-
-_CLI_FUNCS = [
-    CliFunc('version', _print_version),
-    CliFunc('revision', _print_revision),
-]
+_CLI_FUNCS: ta.Sequence[CliFunc] = []
 
 
 ##
@@ -46,6 +37,7 @@ def _main() -> None:
     ldr = ManifestLoader.from_entry_point(globals())
 
     pkgs = ldr.discover()
+
     if not pkgs:
         pkgs = []
         for n in os.listdir(os.getcwd()):

omdev/git.py

@@ -73,11 +73,15 @@ def get_git_revision(
     if cwd is None:
         cwd = os.getcwd()
 
-    if subprocess.run([  # noqa
-        'git',
-        'rev-parse',
-        '--is-inside-work-tree',
-    ], stderr=subprocess.PIPE).returncode:
+    if subprocess.run(  # noqa
+        [
+            'git',
+            'rev-parse',
+            '--is-inside-work-tree',
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    ).returncode:
         return None
 
     has_untracked = bool(subprocess.check_output([

omdev/manifests/load.py

@@ -1,7 +1,12 @@
+"""
+Should be kept somewhat lightweight - used in cli entrypoints.
+
+TODO:
+ - persisted caching support - {pkg_name: manifests}
+"""
 # ruff: noqa: UP006 UP007
 import dataclasses as dc
 import importlib.machinery
-import importlib.metadata
 import importlib.resources
 import json
 import typing as ta
@@ -74,19 +79,7 @@ class ManifestLoader:
         self._cls_cache[key] = cls
         return cls
 
-    def load_raw(self, pkg_name: str) -> ta.Optional[ta.Sequence[Manifest]]:
-        try:
-            return self._raw_cache[pkg_name]
-        except KeyError:
-            pass
-
-        t = importlib.resources.files(pkg_name).joinpath('.manifests.json')
-        if not t.is_file():
-            self._raw_cache[pkg_name] = None
-            return None
-
-        src = t.read_text('utf-8')
-        obj = json.loads(src)
+    def load_contents(self, obj: ta.Any, pkg_name: str) -> ta.Sequence[Manifest]:
         if not isinstance(obj, (list, tuple)):
             raise TypeError(obj)
 
@@ -105,6 +98,26 @@ class ManifestLoader:
 
             lst.append(m)
 
+        return lst
+
+    def load_raw(self, pkg_name: str) -> ta.Optional[ta.Sequence[Manifest]]:
+        try:
+            return self._raw_cache[pkg_name]
+        except KeyError:
+            pass
+
+        t = importlib.resources.files(pkg_name).joinpath('.manifests.json')
+        if not t.is_file():
+            self._raw_cache[pkg_name] = None
+            return None
+
+        src = t.read_text('utf-8')
+        obj = json.loads(src)
+        if not isinstance(obj, (list, tuple)):
+            raise TypeError(obj)
+
+        lst = self.load_contents(obj, pkg_name)
+
         self._raw_cache[pkg_name] = lst
         return lst
 
@@ -143,6 +156,9 @@ class ManifestLoader:
     ENTRY_POINT_GROUP = 'omlish.manifests'
 
     def discover(self) -> ta.Sequence[str]:
+        # This is a fat dep so do it late.
+        import importlib.metadata
+
         return [
             ep.value
             for ep in importlib.metadata.entry_points(group=self.ENTRY_POINT_GROUP)

omdev/pyproject/cli.py

@@ -47,6 +47,7 @@ from ..toml.parser import toml_loads
 from .configs import PyprojectConfig
 from .configs import PyprojectConfigPreparer
 from .configs import VenvConfig
+from .pkg import BasePyprojectPackageGenerator
 from .pkg import PyprojectPackageGenerator
 from .reqs import RequirementsRewriter
 
@@ -338,25 +339,36 @@ def _pkg_cmd(args) -> None:
         if run_build:
             os.makedirs(build_output_dir, exist_ok=True)
 
-        num_threads = max(mp.cpu_count() // 2, 1)
+        pgs: ta.List[BasePyprojectPackageGenerator] = [
+            PyprojectPackageGenerator(
+                dir_name,
+                pkgs_root,
+            )
+            for dir_name in run.cfg().pkgs
+        ]
+        pgs = list(itertools.chain.from_iterable([pg, *pg.children()] for pg in pgs))
+
+        num_threads = args.jobs or max(mp.cpu_count() // 2, 1)
+        futs: ta.List[cf.Future]
         with cf.ThreadPoolExecutor(num_threads) as ex:
-            futs = [
-                ex.submit(functools.partial(
-                    PyprojectPackageGenerator(
-                        dir_name,
-                        pkgs_root,
-                    ).gen,
-                    PyprojectPackageGenerator.GenOpts(
-                        run_build=run_build,
-                        build_output_dir=build_output_dir,
-                        add_revision=add_revision,
-                    ),
-                ))
-                for dir_name in run.cfg().pkgs
-            ]
+            futs = [ex.submit(pg.gen) for pg in pgs]
             for fut in futs:
                 fut.result()
 
+            if run_build:
+                futs = [
+                    ex.submit(functools.partial(
+                        pg.build,
+                        build_output_dir,
+                        BasePyprojectPackageGenerator.BuildOpts(
+                            add_revision=add_revision,
+                        ),
+                    ))
+                    for pg in pgs
+                ]
+                for fut in futs:
+                    fut.result()
+
     else:
         raise Exception(f'unknown subcommand: {cmd}')
 
@@ -380,6 +392,7 @@ def _build_parser() -> argparse.ArgumentParser:
     parser_resolve = subparsers.add_parser('pkg')
     parser_resolve.add_argument('-b', '--build', action='store_true')
     parser_resolve.add_argument('-r', '--revision', action='store_true')
+    parser_resolve.add_argument('-j', '--jobs', type=int)
     parser_resolve.add_argument('cmd', nargs='?')
     parser_resolve.add_argument('args', nargs=argparse.REMAINDER)
     parser_resolve.set_defaults(func=_pkg_cmd)

omdev/pyproject/pkg.py

@@ -186,12 +186,33 @@ class BasePyprojectPackageGenerator(abc.ABC):
 
     #
 
-    def _run_build(
+    def children(self) -> ta.Sequence['BasePyprojectPackageGenerator']:
+        return []
+
+    #
+
+    def gen(self) -> str:
+        log.info('Generating pyproject package: %s -> %s (%s)', self._dir_name, self._pkgs_root, self._pkg_suffix)
+
+        self._pkg_dir()
+        self._write_git_ignore()
+        self._symlink_source_dir()
+        self._write_file_contents()
+        self._symlink_standard_files()
+
+        return self._pkg_dir()
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class BuildOpts:
+        add_revision: bool = False
+        test: bool = False
+
+    def build(
             self,
-            build_output_dir: ta.Optional[str] = None,
-            *,
-            add_revision: bool = False,
-            test: bool = False,
+            output_dir: ta.Optional[str] = None,
+            opts: BuildOpts = BuildOpts(),
     ) -> None:
         subprocess_check_call(
             sys.executable,
@@ -202,10 +223,10 @@ class BasePyprojectPackageGenerator(abc.ABC):
 
         dist_dir = os.path.join(self._pkg_dir(), 'dist')
 
-        if add_revision:
+        if opts.add_revision:
             GitRevisionAdder().add_to(dist_dir)
 
-        if test:
+        if opts.test:
             for fn in os.listdir(dist_dir):
                 tmp_dir = tempfile.mkdtemp()
 
@@ -224,34 +245,9 @@ class BasePyprojectPackageGenerator(abc.ABC):
                     cwd=tmp_dir,
                 )
 
-        if build_output_dir is not None:
+        if output_dir is not None:
             for fn in os.listdir(dist_dir):
-                shutil.copyfile(os.path.join(dist_dir, fn), os.path.join(build_output_dir, fn))
-
-    #
-
-    @dc.dataclass(frozen=True)
-    class GenOpts:
-        run_build: bool = False
-        build_output_dir: ta.Optional[str] = None
-        add_revision: bool = False
-
-    def gen(self, opts: GenOpts = GenOpts()) -> str:
-        log.info('Generating pyproject package: %s -> %s (%s)', self._dir_name, self._pkgs_root, self._pkg_suffix)
-
-        self._pkg_dir()
-        self._write_git_ignore()
-        self._symlink_source_dir()
-        self._write_file_contents()
-        self._symlink_standard_files()
-
-        if opts.run_build:
-            self._run_build(
-                opts.build_output_dir,
-                add_revision=opts.add_revision,
-            )
-
-        return self._pkg_dir()
+                shutil.copyfile(os.path.join(dist_dir, fn), os.path.join(output_dir, fn))
 
 
 #
@@ -377,24 +373,25 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
 
     #
 
-    def gen(self, opts: BasePyprojectPackageGenerator.GenOpts = BasePyprojectPackageGenerator.GenOpts()) -> str:
-        ret = super().gen(opts)
+    @cached_nullary
+    def children(self) -> ta.Sequence[BasePyprojectPackageGenerator]:
+        out: ta.List[BasePyprojectPackageGenerator] = []
 
         if self.build_specs().setuptools.get('cexts'):
-            _PyprojectCextPackageGenerator(
+            out.append(_PyprojectCextPackageGenerator(
                 self._dir_name,
                 self._pkgs_root,
                 pkg_suffix='-cext',
-            ).gen(opts)
+            ))
 
         if self.build_specs().pyproject.get('cli_scripts'):
-            _PyprojectCliPackageGenerator(
+            out.append(_PyprojectCliPackageGenerator(
                 self._dir_name,
                 self._pkgs_root,
                 pkg_suffix='-cli',
-            ).gen(opts)
+            ))
 
-        return ret
+        return out
 
 
 #

omdev/scripts/execrss.py

@@ -8,6 +8,13 @@ def _get_rss() -> int:
     return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
 
 
+# @omlish-manifest
+_CLI_MODULE = {'$omdev.cli.types.CliModule': {
+    'cmd_name': 'execrss',
+    'mod_name': __name__,
+}}
+
+
 def _main() -> None:
     [src] = sys.argv[1:]
     start = _get_rss()

omdev/scripts/exectime.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# @omlish-script
+import sys
+import time
+
+
+# @omlish-manifest
+_CLI_MODULE = {'$omdev.cli.types.CliModule': {
+    'cmd_name': 'exectime',
+    'mod_name': __name__,
+}}
+
+
+def _main() -> None:
+    [src] = sys.argv[1:]
+    co = compile(src, '<string>', 'exec')
+    start = time.time_ns()
+    exec(co)
+    end = time.time_ns()
+    print(end - start)
+
+
+if __name__ == '__main__':
+    _main()

omdev/{tools → scripts}/importtrace.py

@@ -478,6 +478,13 @@ class SqliteWriter:
 ##
 
 
+# @omlish-manifest
+_CLI_MODULE = {'$omdev.cli.types.CliModule': {
+    'cmd_name': 'importtrace',
+    'mod_name': __name__,
+}}
+
+
 def _main() -> None:
     if sys.version_info < REQUIRED_PYTHON_VERSION:
         raise EnvironmentError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}')  # noqa

omdev/scripts/pyproject.py

@@ -242,11 +242,15 @@ def get_git_revision(
     if cwd is None:
         cwd = os.getcwd()
 
-    if subprocess.run([  # noqa
-        'git',
-        'rev-parse',
-        '--is-inside-work-tree',
-    ], stderr=subprocess.PIPE).returncode:
+    if subprocess.run(  # noqa
+        [
+            'git',
+            'rev-parse',
+            '--is-inside-work-tree',
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    ).returncode:
         return None
 
     has_untracked = bool(subprocess.check_output([
@@ -3879,12 +3883,33 @@ class BasePyprojectPackageGenerator(abc.ABC):
 
     #
 
-    def _run_build(
+    def children(self) -> ta.Sequence['BasePyprojectPackageGenerator']:
+        return []
+
+    #
+
+    def gen(self) -> str:
+        log.info('Generating pyproject package: %s -> %s (%s)', self._dir_name, self._pkgs_root, self._pkg_suffix)
+
+        self._pkg_dir()
+        self._write_git_ignore()
+        self._symlink_source_dir()
+        self._write_file_contents()
+        self._symlink_standard_files()
+
+        return self._pkg_dir()
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class BuildOpts:
+        add_revision: bool = False
+        test: bool = False
+
+    def build(
             self,
-            build_output_dir: ta.Optional[str] = None,
-            *,
-            add_revision: bool = False,
-            test: bool = False,
+            output_dir: ta.Optional[str] = None,
+            opts: BuildOpts = BuildOpts(),
     ) -> None:
         subprocess_check_call(
             sys.executable,
@@ -3895,10 +3920,10 @@ class BasePyprojectPackageGenerator(abc.ABC):
 
         dist_dir = os.path.join(self._pkg_dir(), 'dist')
 
-        if add_revision:
+        if opts.add_revision:
             GitRevisionAdder().add_to(dist_dir)
 
-        if test:
+        if opts.test:
             for fn in os.listdir(dist_dir):
                 tmp_dir = tempfile.mkdtemp()
 
@@ -3917,34 +3942,9 @@ class BasePyprojectPackageGenerator(abc.ABC):
                     cwd=tmp_dir,
                 )
 
-        if build_output_dir is not None:
+        if output_dir is not None:
             for fn in os.listdir(dist_dir):
-                shutil.copyfile(os.path.join(dist_dir, fn), os.path.join(build_output_dir, fn))
-
-    #
-
-    @dc.dataclass(frozen=True)
-    class GenOpts:
-        run_build: bool = False
-        build_output_dir: ta.Optional[str] = None
-        add_revision: bool = False
-
-    def gen(self, opts: GenOpts = GenOpts()) -> str:
-        log.info('Generating pyproject package: %s -> %s (%s)', self._dir_name, self._pkgs_root, self._pkg_suffix)
-
-        self._pkg_dir()
-        self._write_git_ignore()
-        self._symlink_source_dir()
-        self._write_file_contents()
-        self._symlink_standard_files()
-
-        if opts.run_build:
-            self._run_build(
-                opts.build_output_dir,
-                add_revision=opts.add_revision,
-            )
-
-        return self._pkg_dir()
+                shutil.copyfile(os.path.join(dist_dir, fn), os.path.join(output_dir, fn))
 
 
 #
@@ -4070,24 +4070,25 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
 
     #
 
-    def gen(self, opts: BasePyprojectPackageGenerator.GenOpts = BasePyprojectPackageGenerator.GenOpts()) -> str:
-        ret = super().gen(opts)
+    @cached_nullary
+    def children(self) -> ta.Sequence[BasePyprojectPackageGenerator]:
+        out: ta.List[BasePyprojectPackageGenerator] = []
 
         if self.build_specs().setuptools.get('cexts'):
-            _PyprojectCextPackageGenerator(
+            out.append(_PyprojectCextPackageGenerator(
                 self._dir_name,
                 self._pkgs_root,
                 pkg_suffix='-cext',
-            ).gen(opts)
+            ))
 
         if self.build_specs().pyproject.get('cli_scripts'):
-            _PyprojectCliPackageGenerator(
+            out.append(_PyprojectCliPackageGenerator(
                 self._dir_name,
                 self._pkgs_root,
                 pkg_suffix='-cli',
-            ).gen(opts)
+            ))
 
-        return ret
+        return out
 
 
 #
@@ -5275,25 +5276,36 @@ def _pkg_cmd(args) -> None:
         if run_build:
             os.makedirs(build_output_dir, exist_ok=True)
 
-        num_threads = max(mp.cpu_count() // 2, 1)
+        pgs: ta.List[BasePyprojectPackageGenerator] = [
+            PyprojectPackageGenerator(
+                dir_name,
+                pkgs_root,
+            )
+            for dir_name in run.cfg().pkgs
+        ]
+        pgs = list(itertools.chain.from_iterable([pg, *pg.children()] for pg in pgs))
+
+        num_threads = args.jobs or max(mp.cpu_count() // 2, 1)
+        futs: ta.List[cf.Future]
         with cf.ThreadPoolExecutor(num_threads) as ex:
-            futs = [
-                ex.submit(functools.partial(
-                    PyprojectPackageGenerator(
-                        dir_name,
-                        pkgs_root,
-                    ).gen,
-                    PyprojectPackageGenerator.GenOpts(
-                        run_build=run_build,
-                        build_output_dir=build_output_dir,
-                        add_revision=add_revision,
-                    ),
-                ))
-                for dir_name in run.cfg().pkgs
-            ]
+            futs = [ex.submit(pg.gen) for pg in pgs]
             for fut in futs:
                 fut.result()
 
+            if run_build:
+                futs = [
+                    ex.submit(functools.partial(
+                        pg.build,
+                        build_output_dir,
+                        BasePyprojectPackageGenerator.BuildOpts(
+                            add_revision=add_revision,
+                        ),
+                    ))
+                    for pg in pgs
+                ]
+                for fut in futs:
+                    fut.result()
+
     else:
         raise Exception(f'unknown subcommand: {cmd}')
 
@@ -5317,6 +5329,7 @@ def _build_parser() -> argparse.ArgumentParser:
     parser_resolve = subparsers.add_parser('pkg')
     parser_resolve.add_argument('-b', '--build', action='store_true')
     parser_resolve.add_argument('-r', '--revision', action='store_true')
+    parser_resolve.add_argument('-j', '--jobs', type=int)
     parser_resolve.add_argument('cmd', nargs='?')
     parser_resolve.add_argument('args', nargs=argparse.REMAINDER)
     parser_resolve.set_defaults(func=_pkg_cmd)

omdev/tools/importscan.py

@@ -17,6 +17,8 @@ import typing as ta
 from omlish import concurrent as cu
 from omlish import lang
 
+from ..cli import CliModule
+
 
 ##
 
@@ -160,6 +162,10 @@ def run(
 ##
 
 
+# @omlish-manifest
+_CLI_MODULE = CliModule('importscan', __name__)
+
+
 def _main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument('-j', '--jobs', type=int)

{omdev-0.0.0.dev40.dist-info → omdev-0.0.0.dev43.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev40
+Version: 0.0.0.dev43
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omlish ==0.0.0.dev40
+Requires-Dist: omlish ==0.0.0.dev43
 Provides-Extra: all
 Requires-Dist: pycparser ~=2.22 ; extra == 'all'
 Requires-Dist: cffi ~=1.17 ; extra == 'all'

{omdev-0.0.0.dev40.dist-info → omdev-0.0.0.dev43.dist-info}/RECORD

@@ -1,4 +1,4 @@
-omdev/.manifests.json,sha256=tZR-7JWuphPfRzUGk5qhIhRP8qJpZOu3Z-D12VjgQFo,2595
+omdev/.manifests.json,sha256=c-eZ9NasJYDz6P7LeyEV_-NLPoVpI3ph8LcO8Hg0JNY,3921
 omdev/__about__.py,sha256=LqSNNFFcT84xW3W8fIOJ78kPYJKFLIXZyDX-AJREvN0,1005
 omdev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/bracepy.py,sha256=HwBK5XmlOsF_juTel25fRLJK9vHSJCWXuCc-OZlevRQ,2619
@@ -6,7 +6,7 @@ omdev/classdot.py,sha256=K0YMTngaC6uuEKhDb95tFzW33Re_YEdgIWBBeze4PTI,1628
 omdev/cmake.py,sha256=Diy2ry65806dQP125DAstD3w46z_wszMH7PwC2-6iik,4578
 omdev/findimports.py,sha256=P8v4I1tm6g-PEWJiNwAKxErvWwL-Nop83vAuwq1kR5A,2246
 omdev/findmagic.py,sha256=DhBYHHP_dzwM5pIh21xnQPnkZ2YmAXCjithsr7X0ScU,2357
-omdev/git.py,sha256=OzP4xHVboaa7GhS-mg4F3lYWf3HLa5aMm6V6PtIw_3U,2137
+omdev/git.py,sha256=riM2KqSpQPC3N89uipd7Vm3kVgv4EYYDBU9tV1Z8juM,2208
 omdev/revisions.py,sha256=U657hf4zeEN32y3g4CzqCAodx_HlfkHj2cIIKALNFDo,5009
 omdev/secrets.py,sha256=ja0VsCB01MHxYwn5OHjFeXV9cRah9AQl-0uJzZELpic,256
 omdev/tokens.py,sha256=GusxQ1Cd_eiScuR8XTTtc9QFhOgYviYGBZmFnn3Hj7s,756
@@ -25,11 +25,11 @@ omdev/cache/compute/storage.py,sha256=woCUqHg8ZrwLEejRG3zu1L5ZXxGNNXveh3E8FnlEkj
 omdev/cache/compute/types.py,sha256=NpCTTJHDmpERjrbO6dh9TEzHuP6-vOuoX3ym9sA0ukc,2639
 omdev/cache/data/__init__.py,sha256=SQXtugLceRif463rcoklpQ33pxYLgEIm0xiI6NvOI6M,301
 omdev/cache/data/actions.py,sha256=KVYb3tBYP5c0g-wK1bXih_K7L0ER9UINKChhfc7mwKQ,1071
-omdev/cache/data/cache.py,sha256=WSsbFyFRT_IQFYQCrmUpaTvs9DRglLmCnhguOzdJ6p4,5753
+omdev/cache/data/cache.py,sha256=NRi1WJHVixt9tL_JpOPZqOow8aIaapXCMwIkk2JlM-k,7719
 omdev/cache/data/consts.py,sha256=d6W_aeMqgah6PmPYi9RA8Be54oQ4BcNCy8kDQ7FlB_Q,26
 omdev/cache/data/defaults.py,sha256=HrapVUIf9Ozu3qSfRPyQj-vx-dz6Yyedjb-k3yV4CW8,277
 omdev/cache/data/manifests.py,sha256=CupK71fL3_PnDzUqjrWLNt64KfGKF-K4ycMkT5p0gPA,979
-omdev/cache/data/specs.py,sha256=h2yGkDAZ5tGpqJ280QyglFodTNf_WP1GHJo6koQ-sTk,2313
+omdev/cache/data/specs.py,sha256=4eQ1o8Oxd7AWkcCGy1FgH2Sa0B9uqhp1UFO45eUAH1c,2466
 omdev/cexts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/cexts/_boilerplate.cc,sha256=sbpXEgdFrkdzZXgaNWFFNN27fL9TZu6VrwvMY4-nnFM,1726
 omdev/cexts/build.py,sha256=F3z1-CjDlEM-Gzi5IunKUBO52qdH_pMsFylobTdGJnI,2654
@@ -55,8 +55,9 @@ omdev/cexts/_distutils/compilers/options.py,sha256=H7r5IcLvga5Fs3jjXWIT-6ap3JBdu
 omdev/cexts/_distutils/compilers/unixccompiler.py,sha256=o1h8QuyupLntv4F21_XjzAZmCiwwxJuTmOirvBSL-Qw,15419
 omdev/cli/__init__.py,sha256=V_l6VP1SZMlJbO-8CJwSuO9TThOy2S_oaPepNYgIrbE,37
 omdev/cli/__main__.py,sha256=d23loR_cKfTYZwYiqpt_CmKI7dd5WcYFgIYzqMep75E,68
+omdev/cli/clicmds.py,sha256=Qt75JOZ5nnI1OSvKFkNy25Zi1iaNx_hcPlK3PkgWgYY,1923
 omdev/cli/install.py,sha256=-pFczwyJn48IeqCswbJzrV15ZslQUNIcREiiOGSjDk0,3933
-omdev/cli/main.py,sha256=63IcpB8y2j3xp0I6kF6hdqqJ2bzrMsGElWNtcEsI8nU,1946
+omdev/cli/main.py,sha256=rGc8SIk-czOS9eILZbzu27nx3ezjc54fn880OWskpYE,1880
 omdev/cli/types.py,sha256=7_Owg0P8C8oOObSuOp6aEYSjkEukVFxTT00SRy1bLHM,250
 omdev/interp/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/interp/__main__.py,sha256=GMCqeGYltgt5dlJzHxY9gqisa8cRkrPfmZYuZnjg4WI,162
@@ -70,7 +71,7 @@ omdev/interp/system.py,sha256=bI-JhX4GVJqW7wMxnIa-DGJWnCLmFcIsnl9pc1RGY2g,3513
 omdev/interp/types.py,sha256=EMN3StEMkFoQAMUIZd7JYL4uUWqzFGY-2hTL8EBznYM,2437
 omdev/manifests/__init__.py,sha256=P2B0dpT8D7l5lJwRGPA92IcQj6oeXfd90X5-q9BJrKg,51
 omdev/manifests/build.py,sha256=_anZfhIAwUQo0J4BfKvDsvivrST2U5VigJaFKbJKXMQ,9067
-omdev/manifests/load.py,sha256=1QznPL4zNlu1eIFkoGrO78coIQCk9LlMC8FJ6PcvhNM,4373
+omdev/manifests/load.py,sha256=LtEsluDdd8CkNGj0QGBxee3twBn095Fru0xz2mtr7uk,4788
 omdev/manifests/types.py,sha256=Jv6PAdVLPb9Hh4y6vDhPlWuMNBBViin1bC_u83jfsH4,234
 omdev/mypy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/mypy/debug.py,sha256=WcZw-3Z1njg_KFGqi3DB6RuqbBa3dLArJnjVCuY1Mn0,3003
@@ -89,30 +90,31 @@ omdev/precheck/scripts.py,sha256=qq6MXkxgrYngPg5pWnXH4uRSuRkP3mFqbeml1UmvGBc,126
 omdev/pyproject/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/pyproject/__main__.py,sha256=gn3Rl1aYPYdiTtEqa9ifi0t-e4ZwPY0vhJ4UXvYdJDY,165
 omdev/pyproject/cexts.py,sha256=x13piOOnNrYbA17qZLDVuR0p1sqhgEwpk4FtImX-klM,4281
-omdev/pyproject/cli.py,sha256=hE2bjfXmJy1xnadSyIcFjF3kYYTqNK4Y5dqTjnMtjVs,10909
+omdev/pyproject/cli.py,sha256=QnlptoCJQ0yvJ1C2mXEYfOMS6agRQMn2nGS9KgWoJ5U,11358
 omdev/pyproject/configs.py,sha256=K9H5cGwVLgHi8wKwtYvlXHZ9ThtmnI4jo8JAb-t1-70,2859
-omdev/pyproject/pkg.py,sha256=-8meXIJSVkXrDJDsqEcdJ05bssAnTlLp2Y7bElcxI-g,14753
+omdev/pyproject/pkg.py,sha256=rNKzJOIgPDrtT2i14Pebldoboz45w00sKb5l_kYFaRI,14562
 omdev/pyproject/reqs.py,sha256=jgDH_nmOkcgxKshbeFt-byPAuGfwoxHuyzaolGu-lms,2299
 omdev/scripts/__init__.py,sha256=MKCvUAEQwsIvwLixwtPlpBqmkMXLCnjjXyAXvVpDwVk,91
 omdev/scripts/bumpversion.py,sha256=Kn7fo73Hs8uJh3Hi3EIyLOlzLPWAC6dwuD_lZ3cIzuY,1064
-omdev/scripts/execrss.py,sha256=HzDNmwXOO8fMwIRXw9q8CUnVfLFCQASyU2tfY_y2Vf8,324
+omdev/scripts/execrss.py,sha256=d6purJqU99OkMcNxCS1kG2CMfSsw7wjnvBQW7SjyJ70,448
+omdev/scripts/exectime.py,sha256=LRVIJsnvGYUqH-zfCdFnrQCZv1KLXJPtWBWplXMDwhI,407
+omdev/scripts/importtrace.py,sha256=bp6lnrDdh0zPR0cmI3nvlU5QPWybiImpe9SNoJFOrac,13558
 omdev/scripts/interp.py,sha256=CB9Eg8qPULFIQmMDsbLTGZ74c3xASWrQJ9KriLSb19A,70949
-omdev/scripts/pyproject.py,sha256=zLDXrPavD4MzWTTwOKEPPL8y71_XmLlFf7r7AYXObq0,157080
+omdev/scripts/pyproject.py,sha256=lk0Gk6chRLHYRpES32uXDgxsHti4Ui8oGGyJrJu_dnk,157362
 omdev/toml/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/toml/parser.py,sha256=84bn09uhYHwQGyfww6Rw6y1RxPAE_HDltODOSakcqDM,29186
 omdev/toml/writer.py,sha256=lk3on3YXVbWuLJa-xsOzOhs1bBAT1vXqw4mBbluZl_w,3040
 omdev/tools/__init__.py,sha256=iVJAOQ0viGTQOm0DLX4uZLro-9jOioYJGLg9s0kDx1A,78
 omdev/tools/dockertools.py,sha256=x00GV8j1KReMXwxJ641GlcsVwHoWeuzdIKVBp36BqwU,5298
 omdev/tools/gittools.py,sha256=s_-cxh-4Nv2a_bJtHZ6AiKF8kx2IjDrt_GOH8Wjz-3M,669
-omdev/tools/importscan.py,sha256=XRLiasVSaTIp-jnO0-Nfhi0t6gnv_hVy5j2nVfEvuMI,3831
-omdev/tools/importtrace.py,sha256=oDry9CwIv5h96wSaTVKJ0qQ5vMGxYE5oBtfF-GYNLJs,13430
+omdev/tools/importscan.py,sha256=usF35AjdMZacpe8nfP-wfzxelExT5sEQUORNcBKqr5M,3929
 omdev/tools/piptools.py,sha256=lhwzGXD-v0KFEQNyvzvdO2Kw1OA_2AfGPBs_rIkz8iE,2772
 omdev/tools/proftools.py,sha256=xKSm_yPoCnfsvS3iT9MblDqFMuZmGfI3_koGj8amMyU,145
 omdev/tools/rst.py,sha256=6dWk8QZHoGiLSuBw3TKsXZjjFK6wWBEtPi9krdCLKKg,977
 omdev/tools/sqlrepl.py,sha256=tmFZh80-xsGM62dyQ7_UGLebChrj7IHbIPYBWDJMgVk,5741
-omdev-0.0.0.dev40.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-omdev-0.0.0.dev40.dist-info/METADATA,sha256=NwNqYM23XZBQMNs1KcxHQ5imSOLX7UhGWIyHrkcrmbo,1252
-omdev-0.0.0.dev40.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-omdev-0.0.0.dev40.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
-omdev-0.0.0.dev40.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
-omdev-0.0.0.dev40.dist-info/RECORD,,
+omdev-0.0.0.dev43.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+omdev-0.0.0.dev43.dist-info/METADATA,sha256=5Hv8QOnRyAlCu0hmOy2f4I772ceLh29Q1XPQmufh8fw,1252
+omdev-0.0.0.dev43.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+omdev-0.0.0.dev43.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
+omdev-0.0.0.dev43.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
+omdev-0.0.0.dev43.dist-info/RECORD,,