omdev 0.0.0.dev10__py3-none-any.whl → 0.0.0.dev12__py3-none-any.whl

omdev/scripts/bracepy.py

@@ -0,0 +1,105 @@
+# !/usr/bin/env python3
+"""
+https://github.com/umlet/pwk/blob/dc23b3400108a71947a695f1fa1df0f514b42528/pwk
+"""
+import io
+import tokenize
+
+
+def translate_brace_python(
+        s: str,
+        *,
+        indent_width: int = 4,
+) -> str:
+    lt = tokenize.tokenize(io.BytesIO(s.encode('utf-8')).readline)
+
+    ret = io.StringIO()
+
+    indent = 0
+    open_braces = 0
+
+    newline = False
+    indent_up = False
+    indent_down = False
+    skip = False
+
+    while True:
+        try:
+            t = next(lt)
+
+            if t.type == tokenize.ENCODING:
+                last_t = t
+                continue
+
+            if t.type == tokenize.OP and t.string == ';':
+                newline = True
+
+            elif t.type == tokenize.OP and t.string == ':':
+                if open_braces == 0:
+                    newline = True
+                    indent_up = True
+
+            elif t.type == tokenize.OP and t.string == '{':
+                if last_t.type == tokenize.OP and last_t.string == ':':  # noqa
+                    skip = True
+                else:
+                    open_braces += 1
+
+            elif t.type == tokenize.OP and t.string == '}':
+                if open_braces > 0:
+                    open_braces -= 1
+                elif open_braces == 0:
+                    if indent > 0:
+                        newline = True
+                        indent_down = True
+                        skip = True
+                    else:
+                        raise Exception('Too many closing braces')
+
+            if indent_up:
+                indent += indent_width
+            elif indent_down:
+                indent -= indent_width
+
+            if newline and indent_up:
+                ret.write(':\n' + ' ' * indent)
+            elif newline:
+                ret.write('\n' + ' ' * indent)
+            elif not skip:
+                ret.write(t.string)
+                ret.write(' ')
+
+            newline = False
+            indent_up = False
+            indent_down = False
+            skip = False
+            last_t = t
+
+        except StopIteration:
+            break
+        except tokenize.TokenError:
+            continue
+
+    ret.write('\n')
+    return ret.getvalue()
+
+
+def _main(argv=None) -> None:
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-x', '--exec', action='store_true')
+    parser.add_argument('cmd')
+
+    args = parser.parse_args(argv)
+
+    src = translate_brace_python(args.cmd)
+
+    if args.exec:
+        exec(src)
+    else:
+        print(src)
+
+
+if __name__ == '__main__':
+    _main()

omdev/scripts/findmagic.py

@@ -28,8 +28,11 @@ def find_magic(
                     continue
 
                 fp = os.path.join(dp, fn)
-                with open(fp) as f:
-                    src = f.read()
+                try:
+                    with open(fp) as f:
+                        src = f.read()
+                except UnicodeDecodeError:
+                    continue
 
                 if not any(
                         any(pat.fullmatch(l) for pat in pats)

omdev/scripts/interp.py

@@ -1669,8 +1669,10 @@ class Pyenv:
         return os.path.join(check_not_none(self.root()), 'bin', 'pyenv')
 
     def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
+        if (root := self.root()) is None:
+            return []
         ret = []
-        vp = os.path.join(self.root(), 'versions')
+        vp = os.path.join(root, 'versions')
         for dn in os.listdir(vp):
             ep = os.path.join(vp, dn, 'bin', 'python')
             if not os.path.isfile(ep):
@@ -1679,6 +1681,8 @@ class Pyenv:
         return ret
 
     def installable_versions(self) -> ta.List[str]:
+        if self.root() is None:
+            return []
         ret = []
         s = subprocess_check_output_str(self.exe(), 'install', '--list')
         for l in s.splitlines():

omdev/scripts/pyproject.py

@@ -39,6 +39,7 @@ import inspect
 import itertools
 import json
 import logging
+import multiprocessing as mp
 import os
 import os.path
 import re
@@ -3225,8 +3226,10 @@ class Pyenv:
         return os.path.join(check_not_none(self.root()), 'bin', 'pyenv')
 
     def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
+        if (root := self.root()) is None:
+            return []
         ret = []
-        vp = os.path.join(self.root(), 'versions')
+        vp = os.path.join(root, 'versions')
         for dn in os.listdir(vp):
             ep = os.path.join(vp, dn, 'bin', 'python')
             if not os.path.isfile(ep):
@@ -3235,6 +3238,8 @@ class Pyenv:
         return ret
 
     def installable_versions(self) -> ta.List[str]:
+        if self.root() is None:
+            return []
         ret = []
         s = subprocess_check_output_str(self.exe(), 'install', '--list')
         for l in s.splitlines():
@@ -3895,13 +3900,12 @@ def _venv_cmd(args) -> None:
         )
         return
 
-    venv.create()
-
     cmd = args.cmd
     if not cmd:
-        pass
+        venv.create()
 
     elif cmd == 'python':
+        venv.create()
         os.execl(
             (exe := venv.exe()),
             exe,
@@ -3909,10 +3913,12 @@ def _venv_cmd(args) -> None:
         )
 
     elif cmd == 'exe':
+        venv.create()
         check_not(args.args)
         print(venv.exe())
 
     elif cmd == 'run':
+        venv.create()
         sh = check_not_none(shutil.which('bash'))
         script = ' '.join(args.args)
         if not script:
@@ -3929,6 +3935,7 @@ def _venv_cmd(args) -> None:
         print('\n'.join(venv.srcs()))
 
     elif cmd == 'test':
+        venv.create()
         subprocess_check_call(venv.exe(), '-m', 'pytest', *(args.args or []), *venv.srcs())
 
     else:
@@ -3954,11 +3961,10 @@ def _pkg_cmd(args) -> None:
         build_output_dir = 'dist'
         run_build = bool(args.build)
 
-        num_threads = 8
-
         if run_build:
             os.makedirs(build_output_dir, exist_ok=True)
 
+        num_threads = max(mp.cpu_count() // 2, 1)
         with cf.ThreadPoolExecutor(num_threads) as ex:
             futs = [
                 ex.submit(functools.partial(

omdev/tools/gittools.py

@@ -0,0 +1,22 @@
+import subprocess
+
+from omlish import argparse as ap
+from omlish import logs
+
+
+class Cli(ap.Cli):
+    @ap.command()
+    def blob_sizes(self) -> None:
+        # https://stackoverflow.com/a/42544963
+        subprocess.check_call(  # noqa
+            "git rev-list --objects --all | "
+            "git cat-file --batch-check='%(objecttype) %(objectname) %(objectsize) %(rest)' | "
+            "sed -n 's/^blob //p' | "
+            "sort --numeric-sort --key=2",
+            shell=True,
+        )
+
+
+if __name__ == '__main__':
+    logs.configure_standard_logging('INFO')
+    Cli()()

omdev/tools/revisions.py

@@ -0,0 +1,173 @@
+"""
+TODO:
+ - omlish-lite, move to pyproject/
+  - vendor-lite wheel.wheelfile
+"""
+# ruff: noqa: TCH003 UP006 UP007
+# @omlish-lite
+import argparse
+import io
+import os.path
+import subprocess
+import tarfile
+import typing as ta
+import zipfile
+
+from omlish.lite.logs import configure_standard_logging
+from omlish.lite.logs import log
+
+from ..wheelfile import WheelFile
+
+
+class RevisionAdder:
+    def __init__(
+            self,
+            revision: str,
+            output_suffix: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+        self._revision = revision
+        self._output_suffix = output_suffix
+
+    REVISION_ATTR = '__revision__'
+
+    def add_to_contents(self, dct: ta.Dict[str, bytes]) -> bool:
+        changed = False
+        for n in dct:
+            if not n.endswith('__about__.py'):
+                continue
+            src = dct[n].decode('utf-8')
+            lines = src.splitlines(keepends=True)
+            for i, l in enumerate(lines):
+                if l != f'{self.REVISION_ATTR} = None\n':
+                    continue
+                lines[i] = f"{self.REVISION_ATTR} = '{self._revision}'\n"
+                changed = True
+            dct[n] = ''.join(lines).encode('utf-8')
+        return changed
+
+    def add_to_wheel(self, f: str) -> None:
+        if not f.endswith('.whl'):
+            raise Exception(f)
+        log.info('Scanning wheel %s', f)
+
+        zis: ta.Dict[str, zipfile.ZipInfo] = {}
+        dct: ta.Dict[str, bytes] = {}
+        with WheelFile(f) as wf:
+            for zi in wf.filelist:
+                if zi.filename == wf.record_path:
+                    continue
+                zis[zi.filename] = zi
+                dct[zi.filename] = wf.read(zi.filename)
+
+        if self.add_to_contents(dct):
+            of = f[:-4] + (self._output_suffix or '') + '.whl'
+            log.info('Repacking wheel %s', of)
+            with WheelFile(of, 'w') as wf:
+                for n, d in dct.items():
+                    log.info('Adding zipinfo %s', n)
+                    wf.writestr(zis[n], d)
+
+    def add_to_tgz(self, f: str) -> None:
+        if not f.endswith('.tar.gz'):
+            raise Exception(f)
+        log.info('Scanning tgz %s', f)
+
+        tis: ta.Dict[str, tarfile.TarInfo] = {}
+        dct: ta.Dict[str, bytes] = {}
+        with tarfile.open(f, 'r:gz') as tf:
+            for ti in tf:
+                tis[ti.name] = ti
+                if ti.type == tarfile.REGTYPE:
+                    with tf.extractfile(ti.name) as tif:  # type: ignore
+                        dct[ti.name] = tif.read()
+
+        if self.add_to_contents(dct):
+            of = f[:-7] + (self._output_suffix or '') + '.tar.gz'
+            log.info('Repacking tgz %s', of)
+            with tarfile.open(of, 'w:gz') as tf:
+                for n, ti in tis.items():
+                    log.info('Adding tarinfo %s', n)
+                    if n in dct:
+                        data = dct[n]
+                        ti.size = len(data)
+                        fo = io.BytesIO(data)
+                    else:
+                        fo = None
+                    tf.addfile(ti, fileobj=fo)
+
+    EXTS = ('.tar.gz', '.whl')
+
+    def add_to_file(self, f: str) -> None:
+        if f.endswith('.whl'):
+            self.add_to_wheel(f)
+
+        elif f.endswith('.tar.gz'):
+            self.add_to_tgz(f)
+
+    def add_to(self, tgt: str) -> None:
+        if os.path.isfile(tgt):
+            self.add_to_file(tgt)
+
+        elif os.path.isdir(tgt):
+            for dp, dns, fns in os.walk(tgt):  # noqa
+                for f in fns:
+                    if any(f.endswith(ext) for ext in self.EXTS):
+                        self.add_to_file(os.path.join(dp, f))
+
+
+#
+
+
+def get_revision() -> str:
+    return subprocess.check_output([
+        'git',
+        'describe',
+        '--match=NeVeRmAtCh',
+        '--always',
+        '--abbrev=40',
+        '--dirty',
+    ]).decode().strip()
+
+
+#
+
+
+def _add_cmd(args) -> None:
+    if (revision := args.revision) is None:
+        revision = get_revision()
+        log.info('Using revision %s', revision)
+
+    if not args.targets:
+        raise Exception('must specify targets')
+
+    ra = RevisionAdder(
+        revision,
+        output_suffix=args.suffix,
+    )
+    for tgt in args.targets:
+        ra.add_to(tgt)
+
+
+def _main(argv=None) -> None:
+    configure_standard_logging('INFO')
+
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers()
+
+    parser_add = subparsers.add_parser('add')
+    parser_add.add_argument('-r', '--revision')
+    parser_add.add_argument('-s', '--suffix')
+    parser_add.add_argument('targets', nargs='*')
+    parser_add.set_defaults(func=_add_cmd)
+
+    args = parser.parse_args(argv)
+    if not getattr(args, 'func', None):
+        parser.print_help()
+    else:
+        args.func(args)
+
+
+if __name__ == '__main__':
+    _main()

omdev/wheelfile.py

@@ -0,0 +1,246 @@
+# @omlish-lite
+# ruff: noqa: UP006 UP007
+# https://github.com/pypa/wheel/blob/7bb46d7727e6e89fe56b3c78297b3af2672bbbe2/src/wheel/wheelfile.py
+# MIT License
+#
+# Copyright (c) 2012 Daniel Holth <dholth@fastmail.fm> and contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+import base64
+import csv
+import hashlib
+import io
+import os.path
+import re
+import stat
+import time
+import typing as ta
+import zipfile
+
+
+class WheelError(Exception):
+    pass
+
+
+# Non-greedy matching of an optional build number may be too clever (more invalid wheel filenames will match). Separate
+# regex for .dist-info?
+WHEEL_INFO_RE = re.compile(
+    r'^'
+    r'(?P<namever>(?P<name>[^\s-]+?)-(?P<ver>[^\s-]+?))'
+    r'(-(?P<build>\d[^\s-]*))?-'
+    r'(?P<pyver>[^\s-]+?)-'
+    r'(?P<abi>[^\s-]+?)-'
+    r'(?P<plat>\S+)'
+    r'\.whl$',
+    re.VERBOSE,
+)
+
+
+class WheelFile(zipfile.ZipFile):
+    """
+    A ZipFile derivative class that also reads SHA-256 hashes from .dist-info/RECORD and checks any read files against
+    those.
+    """
+
+    _default_algorithm = hashlib.sha256
+
+    def __init__(
+            self,
+            file: str,
+            mode: str = 'r',  # ta.Literal["r", "w", "x", "a"]
+            compression: int = zipfile.ZIP_DEFLATED,
+    ) -> None:
+        basename = os.path.basename(file)
+        self.parsed_filename = WHEEL_INFO_RE.match(basename)
+        if not basename.endswith('.whl') or self.parsed_filename is None:
+            raise WheelError(f'Bad wheel filename {basename!r}')
+
+        super().__init__(  # type: ignore
+            file,
+            mode,
+            compression=compression,
+            allowZip64=True,
+        )
+
+        self.dist_info_path = '{}.dist-info'.format(self.parsed_filename.group('namever'))
+        self.record_path = self.dist_info_path + '/RECORD'
+        self._file_hashes: ta.Dict[str, ta.Union[ta.Tuple[None, None], ta.Tuple[int, bytes]]] = {}
+        self._file_sizes: ta.Dict[str, int] = {}
+
+        if mode == 'r':
+            # Ignore RECORD and any embedded wheel signatures
+            self._file_hashes[self.record_path] = None, None
+            self._file_hashes[self.record_path + '.jws'] = None, None
+            self._file_hashes[self.record_path + '.p7s'] = None, None
+
+            # Fill in the expected hashes by reading them from RECORD
+            try:
+                record = self.open(self.record_path)
+            except KeyError:
+                raise WheelError(f'Missing {self.record_path} file') from None
+
+            with record:
+                for line in csv.reader(io.TextIOWrapper(record, newline='', encoding='utf-8')):
+                    path, hash_sum, size = line
+                    if not hash_sum:
+                        continue
+
+                    algorithm, hash_sum = hash_sum.split('=')
+                    try:
+                        hashlib.new(algorithm)
+                    except ValueError:
+                        raise WheelError(f'Unsupported hash algorithm: {algorithm}') from None
+
+                    if algorithm.lower() in {'md5', 'sha1'}:
+                        raise WheelError(f'Weak hash algorithm ({algorithm}) is not permitted by PEP 427')
+
+                    self._file_hashes[path] = (  # type: ignore
+                        algorithm,
+                        self._urlsafe_b64decode(hash_sum.encode('ascii')),
+                    )
+
+    @staticmethod
+    def _urlsafe_b64encode(data: bytes) -> bytes:
+        """urlsafe_b64encode without padding"""
+        return base64.urlsafe_b64encode(data).rstrip(b'=')
+
+    @staticmethod
+    def _urlsafe_b64decode(data: bytes) -> bytes:
+        """urlsafe_b64decode without padding"""
+        pad = b'=' * (4 - (len(data) & 3))
+        return base64.urlsafe_b64decode(data + pad)
+
+    def open(  # type: ignore  # noqa
+            self,
+            name_or_info: ta.Union[str, zipfile.ZipInfo],
+            mode: str = 'r',  # ta.Literal["r", "w"]
+            pwd: ta.Optional[bytes] = None,
+    ) -> ta.IO[bytes]:
+        def _update_crc(newdata: bytes) -> None:
+            eof = ef._eof  # type: ignore  # noqa
+            update_crc_orig(newdata)
+            running_hash.update(newdata)
+            if eof and running_hash.digest() != expected_hash:
+                raise WheelError(f"Hash mismatch for file '{ef_name}'")
+
+        ef_name = name_or_info.filename if isinstance(name_or_info, zipfile.ZipInfo) else name_or_info
+        if (
+                mode == 'r'
+                and not ef_name.endswith('/')
+                and ef_name not in self._file_hashes
+        ):
+            raise WheelError(f"No hash found for file '{ef_name}'")
+
+        ef = super().open(name_or_info, mode, pwd)  # noqa
+        if mode == 'r' and not ef_name.endswith('/'):
+            algorithm, expected_hash = self._file_hashes[ef_name]
+            if expected_hash is not None:
+                # Monkey patch the _update_crc method to also check for the hash from RECORD
+                running_hash = hashlib.new(algorithm)  # type: ignore
+                update_crc_orig, ef._update_crc = ef._update_crc, _update_crc  # type: ignore  # noqa
+
+        return ef
+
+    def write_files(self, base_dir: str) -> None:
+        deferred: list[tuple[str, str]] = []
+        for root, dirnames, filenames in os.walk(base_dir):
+            # Sort the directory names so that `os.walk` will walk them in a defined order on the next iteration.
+            dirnames.sort()
+            for name in sorted(filenames):
+                path = os.path.normpath(os.path.join(root, name))
+                if os.path.isfile(path):
+                    arcname = os.path.relpath(path, base_dir).replace(os.path.sep, '/')
+                    if arcname == self.record_path:
+                        pass
+                    elif root.endswith('.dist-info'):
+                        deferred.append((path, arcname))
+                    else:
+                        self.write(path, arcname)
+
+        deferred.sort()
+        for path, arcname in deferred:
+            self.write(path, arcname)
+
+    def write(  # type: ignore  # noqa
+            self,
+            filename: str,
+            arcname: ta.Optional[str] = None,
+            compress_type: ta.Optional[int] = None,
+    ) -> None:
+        with open(filename, 'rb') as f:
+            st = os.fstat(f.fileno())
+            data = f.read()
+
+        zinfo = zipfile.ZipInfo(
+            arcname or filename,
+            date_time=self._get_zipinfo_datetime(st.st_mtime),
+        )
+        zinfo.external_attr = (stat.S_IMODE(st.st_mode) | stat.S_IFMT(st.st_mode)) << 16
+        zinfo.compress_type = compress_type or self.compression
+        self.writestr(zinfo, data, compress_type)
+
+    _MINIMUM_TIMESTAMP = 315532800  # 1980-01-01 00:00:00 UTC
+
+    @classmethod
+    def _get_zipinfo_datetime(cls, timestamp: ta.Optional[float] = None) -> ta.Any:
+        # Some applications need reproducible .whl files, but they can't do this without forcing the timestamp of the
+        # individual ZipInfo objects. See issue #143.
+        timestamp = int(os.environ.get('SOURCE_DATE_EPOCH', timestamp or time.time()))
+        timestamp = max(timestamp, cls._MINIMUM_TIMESTAMP)
+        return time.gmtime(timestamp)[0:6]
+
+    def writestr(  # type: ignore  # noqa
+            self,
+            zinfo_or_arcname: ta.Union[str, zipfile.ZipInfo],
+            data: ta.Any,  # SizedBuffer | str,
+            compress_type: ta.Optional[int] = None,
+    ) -> None:
+        if isinstance(zinfo_or_arcname, str):
+            zinfo_or_arcname = zipfile.ZipInfo(
+                zinfo_or_arcname,
+                date_time=self._get_zipinfo_datetime(),
+            )
+            zinfo_or_arcname.compress_type = self.compression
+            zinfo_or_arcname.external_attr = (0o664 | stat.S_IFREG) << 16
+
+        if isinstance(data, str):
+            data = data.encode('utf-8')
+
+        super().writestr(zinfo_or_arcname, data, compress_type)
+        fname = (
+            zinfo_or_arcname.filename
+            if isinstance(zinfo_or_arcname, zipfile.ZipInfo)
+            else zinfo_or_arcname
+        )
+        if fname != self.record_path:
+            hash_ = self._default_algorithm(data)  # type: ignore
+            self._file_hashes[fname] = (  # type: ignore
+                hash_.name,
+                self._urlsafe_b64encode(hash_.digest()).decode('ascii'),
+            )
+            self._file_sizes[fname] = len(data)
+
+    def close(self) -> None:
+        # Write RECORD
+        if self.fp is not None and self.mode == 'w' and self._file_hashes:
+            data = io.StringIO()
+            writer = csv.writer(data, delimiter=',', quotechar='"', lineterminator='\n')
+            writer.writerows((
+                (fname, algorithm + '=' + hash_, self._file_sizes[fname])  # type: ignore
+                for fname, (algorithm, hash_) in self._file_hashes.items()
+            ))
+            writer.writerow((format(self.record_path), '', ''))
+            self.writestr(self.record_path, data.getvalue())
+
+        super().close()

{omdev-0.0.0.dev10.dist-info → omdev-0.0.0.dev12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev10
+Version: 0.0.0.dev12
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,13 +12,14 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish ==0.0.0.dev10
+Requires-Dist: omlish ==0.0.0.dev12
 Provides-Extra: all
 Requires-Dist: pycparser >=2.22 ; extra == 'all'
 Requires-Dist: cffi >=1.17 ; extra == 'all'
 Requires-Dist: pcpp >=1.30 ; extra == 'all'
 Requires-Dist: mypy >=1.11 ; extra == 'all'
 Requires-Dist: tokenize-rt >=6 ; extra == 'all'
+Requires-Dist: wheel >=0.44 ; extra == 'all'
 Provides-Extra: c
 Requires-Dist: pycparser >=2.22 ; extra == 'c'
 Requires-Dist: cffi >=1.17 ; extra == 'c'
@@ -27,4 +28,6 @@ Provides-Extra: mypy
 Requires-Dist: mypy >=1.11 ; extra == 'mypy'
 Provides-Extra: tokens
 Requires-Dist: tokenize-rt >=6 ; extra == 'tokens'
+Provides-Extra: wheel
+Requires-Dist: wheel >=0.44 ; extra == 'wheel'