omdev 0.0.0.dev50__tar.gz → 0.0.0.dev52__tar.gz

{omdev-0.0.0.dev50/omdev.egg-info → omdev-0.0.0.dev52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev50
+Version: 0.0.0.dev52
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev50
+Requires-Dist: omlish==0.0.0.dev52
 Provides-Extra: all
 Requires-Dist: pycparser~=2.22; extra == "all"
 Requires-Dist: cffi~=1.17; extra == "all"

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52}/omdev/.manifests.json

@@ -155,6 +155,18 @@
       }
     }
   },
+  {
+    "module": ".tools.mkrelimp",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/tools/mkrelimp.py",
+    "line": 148,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": "mkrelimp",
+        "mod_name": "omdev.tools.mkrelimp"
+      }
+    }
+  },
   {
     "module": ".tools.piptools",
     "attr": "_CLI_MODULE",

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52}/omdev/interp/pyenv.py

@@ -76,11 +76,12 @@ class Pyenv:
             return []
         ret = []
         vp = os.path.join(root, 'versions')
-        for dn in os.listdir(vp):
-            ep = os.path.join(vp, dn, 'bin', 'python')
-            if not os.path.isfile(ep):
-                continue
-            ret.append((dn, ep))
+        if os.path.isdir(vp):
+            for dn in os.listdir(vp):
+                ep = os.path.join(vp, dn, 'bin', 'python')
+                if not os.path.isfile(ep):
+                    continue
+                ret.append((dn, ep))
         return ret
 
     def installable_versions(self) -> ta.List[str]:

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52}/omdev/manifests/build.py

@@ -13,9 +13,13 @@ See (entry_points):
 # ruff: noqa: UP006 UP007
 import argparse
 import collections
+import concurrent.futures as cf
 import dataclasses as dc
+import functools
 import inspect
+import itertools
 import json
+import multiprocessing as mp
 import os.path
 import re
 import shlex
@@ -195,6 +199,7 @@ def build_module_manifests(
 
 
 def build_package_manifests(
+        ex: cf.Executor,
         name: str,
         base: str,
         *,
@@ -204,14 +209,20 @@ def build_package_manifests(
     if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
         raise Exception(pkg_dir)
 
-    manifests: ta.List[Manifest] = []
-
-    for file in sorted(findmagic.find_magic(
-            [pkg_dir],
-            [MANIFEST_MAGIC],
-            ['py'],
-    )):
-        manifests.extend(build_module_manifests(os.path.relpath(file, base), base))
+    files = sorted(findmagic.find_magic(
+        [pkg_dir],
+        [MANIFEST_MAGIC],
+        ['py'],
+    ))
+    futs = [
+        ex.submit(functools.partial(
+            build_module_manifests,
+            os.path.relpath(file, base),
+            base,
+        ))
+        for file in files
+    ]
+    manifests: ta.List[Manifest] = list(itertools.chain.from_iterable(fut.result() for fut in futs))
 
     if write:
         with open(os.path.join(pkg_dir, '.manifests.json'), 'w') as f:
@@ -266,14 +277,22 @@ if __name__ == '__main__':
     def _gen_cmd(args) -> None:
         base = _get_base(args)
 
-        for pkg in args.package:
-            ms = build_package_manifests(
-                pkg,
-                base,
-                write=args.write or False,
-            )
+        num_threads = args.jobs or max(mp.cpu_count() // 2, 1)
+        with cf.ThreadPoolExecutor(num_threads) as ex:
+            futs = [
+                ex.submit(functools.partial(
+                    build_package_manifests,
+                    ex,
+                    pkg,
+                    base,
+                    write=args.write or False,
+                ))
+                for pkg in args.package
+            ]
+            mss = [fut.result() for fut in futs]
             if not args.quiet:
-                print(json_dumps_pretty([dc.asdict(m) for m in ms]))
+                for ms in mss:
+                    print(json_dumps_pretty([dc.asdict(m) for m in ms]))
 
     def _check_cmd(args) -> None:
         base = _get_base(args)
@@ -294,6 +313,7 @@ if __name__ == '__main__':
         parser_gen.add_argument('-b', '--base')
         parser_gen.add_argument('-w', '--write', action='store_true')
         parser_gen.add_argument('-q', '--quiet', action='store_true')
+        parser_gen.add_argument('-j', '--jobs', type=int)
         parser_gen.add_argument('package', nargs='*')
         parser_gen.set_defaults(func=_gen_cmd)
 

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52}/omdev/pyproject/reqs.py

@@ -8,6 +8,7 @@ import tempfile
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
+from omlish.lite.logs import log
 
 
 class RequirementsRewriter:
@@ -57,6 +58,7 @@ class RequirementsRewriter:
 
         with open(out_file, 'w') as f:
             f.write(''.join(out_lines))
+        log.info('Rewrote requirements file %s to %s', in_file, out_file)
         return out_file
 
     def rewrite(self, in_req: str, *, for_file: bool = False) -> str:

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52}/omdev/scripts/interp.py

@@ -1851,11 +1851,12 @@ class Pyenv:
             return []
         ret = []
         vp = os.path.join(root, 'versions')
-        for dn in os.listdir(vp):
-            ep = os.path.join(vp, dn, 'bin', 'python')
-            if not os.path.isfile(ep):
-                continue
-            ret.append((dn, ep))
+        if os.path.isdir(vp):
+            for dn in os.listdir(vp):
+                ep = os.path.join(vp, dn, 'bin', 'python')
+                if not os.path.isfile(ep):
+                    continue
+                ret.append((dn, ep))
         return ret
 
     def installable_versions(self) -> ta.List[str]:

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52}/omdev/scripts/pyproject.py

@@ -2539,82 +2539,6 @@ class SpecifierSet(BaseSpecifier):
             return iter(filtered)
 
 
-########################################
-# ../reqs.py
-"""
-TODO:
- - embed pip._internal.req.parse_requirements, add additional env stuff? breaks compat with raw pip
-"""
-
-
-class RequirementsRewriter:
-    def __init__(
-            self,
-            venv: ta.Optional[str] = None,
-    ) -> None:
-        super().__init__()
-        self._venv = venv
-
-    @cached_nullary
-    def _tmp_dir(self) -> str:
-        return tempfile.mkdtemp('-omlish-reqs')
-
-    VENV_MAGIC = '# @omlish-venv'
-
-    def rewrite_file(self, in_file: str) -> str:
-        with open(in_file) as f:
-            src = f.read()
-
-        in_lines = src.splitlines(keepends=True)
-        out_lines = []
-
-        for l in in_lines:
-            if self.VENV_MAGIC in l:
-                lp, _, rp = l.partition(self.VENV_MAGIC)
-                rp = rp.partition('#')[0]
-                omit = False
-                for v in rp.split():
-                    if v[0] == '!':
-                        if self._venv is not None and self._venv == v[1:]:
-                            omit = True
-                            break
-                    else:
-                        raise NotImplementedError
-
-                if omit:
-                    out_lines.append('# OMITTED:  ' + l)
-                    continue
-
-            out_req = self.rewrite(l.rstrip('\n'), for_file=True)
-            out_lines.append(out_req + '\n')
-
-        out_file = os.path.join(self._tmp_dir(), os.path.basename(in_file))
-        if os.path.exists(out_file):
-            raise Exception(f'file exists: {out_file}')
-
-        with open(out_file, 'w') as f:
-            f.write(''.join(out_lines))
-        return out_file
-
-    def rewrite(self, in_req: str, *, for_file: bool = False) -> str:
-        if in_req.strip().startswith('-r'):
-            l = in_req.strip()
-            lp, _, rp = l.partition(' ')
-            if lp == '-r':
-                inc_in_file, _, rest = rp.partition(' ')
-            else:
-                inc_in_file, rest = lp[2:], rp
-
-            inc_out_file = self.rewrite_file(inc_in_file)
-            if for_file:
-                return ' '.join(['-r ', inc_out_file, rest])
-            else:
-                return '-r' + inc_out_file
-
-        else:
-            return in_req
-
-
 ########################################
 # ../../../omlish/lite/logs.py
 """
@@ -3390,6 +3314,83 @@ class PyprojectConfigPreparer:
         return pcfg
 
 
+########################################
+# ../reqs.py
+"""
+TODO:
+ - embed pip._internal.req.parse_requirements, add additional env stuff? breaks compat with raw pip
+"""
+
+
+class RequirementsRewriter:
+    def __init__(
+            self,
+            venv: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+        self._venv = venv
+
+    @cached_nullary
+    def _tmp_dir(self) -> str:
+        return tempfile.mkdtemp('-omlish-reqs')
+
+    VENV_MAGIC = '# @omlish-venv'
+
+    def rewrite_file(self, in_file: str) -> str:
+        with open(in_file) as f:
+            src = f.read()
+
+        in_lines = src.splitlines(keepends=True)
+        out_lines = []
+
+        for l in in_lines:
+            if self.VENV_MAGIC in l:
+                lp, _, rp = l.partition(self.VENV_MAGIC)
+                rp = rp.partition('#')[0]
+                omit = False
+                for v in rp.split():
+                    if v[0] == '!':
+                        if self._venv is not None and self._venv == v[1:]:
+                            omit = True
+                            break
+                    else:
+                        raise NotImplementedError
+
+                if omit:
+                    out_lines.append('# OMITTED:  ' + l)
+                    continue
+
+            out_req = self.rewrite(l.rstrip('\n'), for_file=True)
+            out_lines.append(out_req + '\n')
+
+        out_file = os.path.join(self._tmp_dir(), os.path.basename(in_file))
+        if os.path.exists(out_file):
+            raise Exception(f'file exists: {out_file}')
+
+        with open(out_file, 'w') as f:
+            f.write(''.join(out_lines))
+        log.info('Rewrote requirements file %s to %s', in_file, out_file)
+        return out_file
+
+    def rewrite(self, in_req: str, *, for_file: bool = False) -> str:
+        if in_req.strip().startswith('-r'):
+            l = in_req.strip()
+            lp, _, rp = l.partition(' ')
+            if lp == '-r':
+                inc_in_file, _, rest = rp.partition(' ')
+            else:
+                inc_in_file, rest = lp[2:], rp
+
+            inc_out_file = self.rewrite_file(inc_in_file)
+            if for_file:
+                return ' '.join(['-r ', inc_out_file, rest])
+            else:
+                return '-r' + inc_out_file
+
+        else:
+            return in_req
+
+
 ########################################
 # ../../revisions.py
 """
@@ -4386,11 +4387,12 @@ class Pyenv:
             return []
         ret = []
         vp = os.path.join(root, 'versions')
-        for dn in os.listdir(vp):
-            ep = os.path.join(vp, dn, 'bin', 'python')
-            if not os.path.isfile(ep):
-                continue
-            ret.append((dn, ep))
+        if os.path.isdir(vp):
+            for dn in os.listdir(vp):
+                ep = os.path.join(vp, dn, 'bin', 'python')
+                if not os.path.isfile(ep):
+                    continue
+                ret.append((dn, ep))
         return ret
 
     def installable_versions(self) -> ta.List[str]:

omdev-0.0.0.dev52/omdev/secrets.py

@@ -0,0 +1,17 @@
+import os.path
+
+import yaml
+
+from omlish import secrets as sec
+
+
+SECRETS_PATH = os.getenv('SECRETS_PATH', os.path.expanduser('~/Dropbox/.dotfiles/secrets.yml'))
+
+
+def load_secrets() -> sec.Secrets:
+    dct: dict[str, sec.Secret] = {}
+    with open(SECRETS_PATH) as f:
+        for k, v in yaml.safe_load(f).items():
+            if isinstance(v, str):
+                dct[k] = sec.Secret(key=k, value=v)
+    return sec.MappingSecrets(dct)

omdev-0.0.0.dev52/omdev/tools/mkrelimp.py

@@ -0,0 +1,169 @@
+import argparse
+import itertools
+import logging
+import os.path
+import typing as ta
+
+import tokenize_rt as trt
+
+from omlish import logs
+
+from .. import tokens as tks
+from ..cli import CliModule
+
+
+T = ta.TypeVar('T')
+
+
+log = logging.getLogger(__name__)
+
+
+def indexfn(
+        fn: ta.Callable[[T], bool],
+        it: ta.Iterable[T],
+        start: int = 0,
+        stop: int | None = None,
+        step: int = 1,
+) -> int:
+    for i, e in enumerate(itertools.islice(it, start, stop, step)):
+        if fn(e):
+            return start + i * step
+    return -1
+
+
+def interleave(sep: T, it: ta.Iterable[T]) -> ta.Iterable[T]:
+    for i, e in enumerate(it):
+        if i > 0:
+            yield sep
+        yield e
+
+
+class Processor:
+    def __init__(
+            self,
+            base_dir: str,
+            mod_name: str | None = None,
+            *,
+            write: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._base_dir = base_dir
+        self._mod_name = mod_name if mod_name is not None else os.path.basename(base_dir)
+        self._write = write
+
+    def process_line_tks(
+            self,
+            in_tks: tks.Tokens,
+            src_file: str,
+    ) -> tks.Tokens:
+        lst = list(in_tks)
+        pfx = []
+        while lst and (tks.is_ws(lst[0]) or lst[0].name in ('INDENT', 'DEDENT')):
+            pfx.append(lst.pop(0))
+
+        if (
+                len(lst) < 3 or
+                lst[0].name != 'NAME' or
+                lst[0].src not in ('import', 'from') or
+                lst[2].name != 'NAME' or
+                lst[2].src != self._mod_name
+        ):
+            return in_tks
+
+        ##
+
+        ws_pos = indexfn(tks.is_ws, lst, 3)
+        imp_name_tks = list(lst[2:ws_pos])
+        imp_name_parts = [t.src for t in imp_name_tks if t.name == 'NAME']
+
+        ##
+
+        src_dir = os.path.dirname(src_file)
+        rel_path = os.path.relpath(os.path.join(self._base_dir, *imp_name_parts[1:]), src_dir)
+        rel_path_parts = rel_path.split(os.sep)
+        pd_pos = indexfn(lambda s: s != '..', rel_path_parts)
+        if pd_pos < 0:
+            rel_imp_name_parts = ['.' * (len(rel_path_parts) + 1)]
+        else:
+            rel_imp_name_parts = ['.' * pd_pos, *rel_path_parts[pd_pos:]]
+
+        ##
+
+        new_tks = list(interleave(
+            trt.Token(name='OP', src='.'),
+            [trt.Token(name='NAME', src=p) for p in rel_imp_name_parts],
+        ))
+        out_tks = [
+            *pfx,
+            *lst[:2],
+            *new_tks,
+            *lst[ws_pos:],
+        ]
+        return out_tks
+
+    def process_file(
+            self,
+            src_file: str,
+    ) -> None:
+        log.info('Processing file: %s : %s', self._mod_name, src_file)
+
+        with open(src_file) as f:
+            src = f.read()
+
+        ts = trt.src_to_tokens(src)
+        in_ls = tks.split_lines(ts)
+        out_ls = [
+            self.process_line_tks(
+                l,
+                src_file,
+            )
+            for l in in_ls
+        ]
+        out_src = tks.join_lines(out_ls)
+
+        if self._write:
+            with open(src_file, 'w') as f:
+                f.write(out_src)
+
+        else:
+            print(out_src)
+            print()
+
+    def process_dir(
+            self,
+            base_dir: str,
+    ) -> None:
+        for dp, _, fns in os.walk(base_dir):
+            for fn in fns:
+                if not fn.endswith('.py'):
+                    continue
+
+                self.process_file(os.path.join(dp, fn))
+
+    def process(self) -> None:
+        self.process_dir(self._base_dir)
+
+
+# @omlish-manifest
+_CLI_MODULE = CliModule('mkrelimp', __name__)
+
+
+def _main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('base_dir')
+    parser.add_argument('mod_name', nargs='?')
+    parser.add_argument('-w', '--write', action='store_true')
+    args = parser.parse_args()
+
+    logs.configure_standard_logging('INFO')
+
+    Processor(
+        args.base_dir,
+        args.mod_name,
+        write=args.write,
+    ).process()
+
+
+if __name__ == '__main__':
+    _main()

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52/omdev.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev50
+Version: 0.0.0.dev52
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev50
+Requires-Dist: omlish==0.0.0.dev52
 Provides-Extra: all
 Requires-Dist: pycparser~=2.22; extra == "all"
 Requires-Dist: cffi~=1.17; extra == "all"

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52}/omdev.egg-info/SOURCES.txt

@@ -119,6 +119,7 @@ omdev/tools/__init__.py
 omdev/tools/dockertools.py
 omdev/tools/gittools.py
 omdev/tools/importscan.py
+omdev/tools/mkrelimp.py
 omdev/tools/piptools.py
 omdev/tools/proftools.py
 omdev/tools/rst.py

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52}/omdev.egg-info/requires.txt

@@ -1,4 +1,4 @@
-omlish==0.0.0.dev50
+omlish==0.0.0.dev52
 
 [all]
 pycparser~=2.22

{omdev-0.0.0.dev50 → omdev-0.0.0.dev52}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 urls = {source = 'https://github.com/wrmsr/omlish'}
 license = {text = 'BSD-3-Clause'}
 requires-python = '~=3.12'
-version = '0.0.0.dev50'
+version = '0.0.0.dev52'
 classifiers = [
     'License :: OSI Approved :: BSD License',
     'Development Status :: 2 - Pre-Alpha',
@@ -22,7 +22,7 @@ classifiers = [
 ]
 description = 'omdev'
 dependencies = [
-    'omlish == 0.0.0.dev50',
+    'omlish == 0.0.0.dev52',
 ]
 
 [project.optional-dependencies]
@@ -73,6 +73,7 @@ exclude = [
     '*.c',
     '*.cc',
     '*.cu',
+    '*.g4',
     '*.h',
     '.manifests.json',
     'LICENSE',

omdev-0.0.0.dev50/omdev/secrets.py

@@ -1,12 +0,0 @@
-import os.path
-import typing as ta
-
-import yaml
-
-
-SECRETS_PATH = os.getenv('SECRETS_PATH', os.path.expanduser('~/Dropbox/.dotfiles/secrets.yml'))
-
-
-def load_secrets() -> dict[str, ta.Any]:
-    with open(SECRETS_PATH) as f:
-        return yaml.safe_load(f)