omdev 0.0.0.dev11__tar.gz → 0.0.0.dev12__tar.gz

{omdev-0.0.0.dev11/omdev.egg-info → omdev-0.0.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev11
+Version: 0.0.0.dev12
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,13 +12,14 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev11
+Requires-Dist: omlish==0.0.0.dev12
 Provides-Extra: all
 Requires-Dist: pycparser>=2.22; extra == "all"
 Requires-Dist: cffi>=1.17; extra == "all"
 Requires-Dist: pcpp>=1.30; extra == "all"
 Requires-Dist: mypy>=1.11; extra == "all"
 Requires-Dist: tokenize_rt>=6; extra == "all"
+Requires-Dist: wheel>=0.44; extra == "all"
 Provides-Extra: c
 Requires-Dist: pycparser>=2.22; extra == "c"
 Requires-Dist: cffi>=1.17; extra == "c"
@@ -27,3 +28,5 @@ Provides-Extra: mypy
 Requires-Dist: mypy>=1.11; extra == "mypy"
 Provides-Extra: tokens
 Requires-Dist: tokenize_rt>=6; extra == "tokens"
+Provides-Extra: wheel
+Requires-Dist: wheel>=0.44; extra == "wheel"

{omdev-0.0.0.dev11 → omdev-0.0.0.dev12}/omdev/__about__.py

@@ -25,6 +25,10 @@ class Project(ProjectBase):
         'tokens': [
             'tokenize_rt >= 6',
         ],
+
+        'wheel': [
+            'wheel >= 0.44',
+        ],
     }
 
 

{omdev-0.0.0.dev11 → omdev-0.0.0.dev12}/omdev/cmake.py

@@ -1,5 +1,4 @@
 import abc
-import io
 import typing as ta
 
 from omlish import dataclasses as dc
@@ -80,7 +79,7 @@ class CmakeGen:
 
     def __init__(
             self,
-            out: io.TextIOBase,
+            out: ta.TextIO,
             *,
             indent: int = 4,
     ) -> None:

omdev-0.0.0.dev12/omdev/exts/cmake.py

@@ -0,0 +1,342 @@
+"""
+TODO:
+ - symlink headers, included src files (hamt_impl, ...)
+ - point / copy output to dst dirs
+ - libs
+  - ..
+   - pybind
+   - catch2?
+   - json? https://github.com/nlohmann/json
+  - FindPackages? FetchContent? built_ext won't have that
+  - move omml git / data retriever stuff into omdev, get just the one header file from git via sha?
+ - support local built pys 
+
+==
+
+Done:
+ - https://intellij-support.jetbrains.com/hc/en-us/community/posts/206608485-Multiple-Jetbrain-IDE-sharing-the-same-project-directory really?
+  - aight, generate a whole cmake subdir with symlinks to src files lol
+
+"""  # noqa
+import argparse
+import dataclasses as dc
+import io
+import logging
+import os.path
+import shutil
+import sys
+import sysconfig
+import typing as ta
+
+from omlish import check
+from omlish import lang
+from omlish import logs
+
+from .. import cmake
+from ..scripts import findmagic
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+MAGIC = '@omdev-ext'
+MAGIC_COMMENT = f'// {MAGIC}'
+
+FILE_EXTENSIONS = ('c', 'cc', 'cpp')
+
+
+def _sep_str_grps(*ls: ta.Sequence[str]) -> list[str]:
+    o = []
+    for i, l in enumerate(ls):
+        if not l:
+            continue
+        if i:
+            o.append('')
+        o.extend(check.not_isinstance(l, str))
+    return o
+
+
+class CmakeProjectGen:
+    def __init__(
+            self,
+            exts: ta.Sequence[str],
+            prj_root: str | None = None,
+    ) -> None:
+        super().__init__()
+        self._exts = check.not_isinstance(exts, str)
+        self._prj_root = os.path.abspath(prj_root) if prj_root is not None else os.getcwd()
+
+    #
+
+    @property
+    def prj_root(self) -> str:
+        return self._prj_root
+
+    @lang.cached_function
+    def prj_name(self) -> str:
+        return os.path.basename(os.path.dirname(self.prj_root))
+
+    @lang.cached_function
+    def cmake_dir(self) -> str:
+        cmake_dir = os.path.join(self.prj_root, 'cmake')
+        if os.path.exists(cmake_dir):
+            for e in os.listdir(cmake_dir):
+                if e == '.idea':
+                    continue
+                ep = os.path.join(cmake_dir, e)
+                if os.path.isfile(ep):
+                    os.unlink(ep)
+                else:
+                    shutil.rmtree(ep)
+        else:
+            os.mkdir(cmake_dir)
+        return cmake_dir
+
+    #
+
+    def write_git_ignore(self) -> None:
+        with open(os.path.join(self.cmake_dir(), '.gitignore'), 'w') as f:
+            f.write('\n'.join(sorted(['/cmake-*', '/build'])))
+
+    #
+
+    def write_idea_name(self) -> None:
+        idea_dir = os.path.join(self.cmake_dir(), '.idea')
+        if not os.path.isdir(idea_dir):
+            os.mkdir(idea_dir)
+        idea_name_file = os.path.join(idea_dir, '.name')
+        if not os.path.isfile(idea_name_file):
+            with open(idea_name_file, 'w') as f:
+                f.write(self.prj_name())
+
+    #
+
+    @dc.dataclass(frozen=True, kw_only=True)
+    class PyInfo:
+        venv_exe: str
+        venv_root: str
+        real_exe: str
+        root: str
+        suffix: str
+
+    @lang.cached_function
+    def py_info(self) -> PyInfo:
+        venv_exe = sys.executable
+        real_exe = os.path.realpath(venv_exe)
+        return self.PyInfo(
+            venv_exe=venv_exe,
+            venv_root=os.path.abspath(os.path.join(os.path.dirname(venv_exe), '..')),
+            real_exe=real_exe,
+            root=os.path.abspath(os.path.join(os.path.dirname(real_exe), '..')),
+            suffix='.'.join(map(str, sys.version_info[:2])),
+        )
+
+    #
+
+    @lang.cached_function
+    def ext_files(self) -> ta.Sequence[str]:
+        out = []
+        for e in self._exts:
+            e = os.path.abspath(e)
+            if os.path.isfile(e):
+                out.append(e)
+            elif os.path.isdir(e):
+                out.extend(
+                    findmagic.find_magic(
+                        [e],
+                        [MAGIC_COMMENT],
+                        FILE_EXTENSIONS,
+                    ),
+                )
+            else:
+                raise KeyError(e)
+        return out
+
+    #
+
+    class _CmakeListsGen:
+        def __init__(
+                self,
+                p: 'CmakeProjectGen',
+                out: ta.TextIO,
+        ) -> None:
+            super().__init__()
+            self.p = p
+            self.g = cmake.CmakeGen(out)
+
+        @lang.cached_property
+        def var_prefix(self) -> str:
+            return self.p.prj_name().upper()
+
+        @lang.cached_property
+        def py(self) -> 'CmakeProjectGen.PyInfo':
+            return self.p.py_info()
+
+        def _add_ext(self, ext_src: str) -> None:
+            ext_name = ext_src.rpartition('.')[0].replace('/', '__')
+
+            log.info('Adding cmake c extension: %s -> %s', ext_src, ext_name)
+
+            so_name = ''.join([
+                os.path.basename(ext_src).split('.')[0],
+                '.',
+                sysconfig.get_config_var('SOABI'),
+                sysconfig.get_config_var('SHLIB_SUFFIX'),
+            ])
+
+            sl = os.path.join(self.p.cmake_dir(), ext_src)
+            sal = os.path.abspath(sl)
+            sd = os.path.dirname(sal)
+            os.makedirs(sd, exist_ok=True)
+            rp = os.path.relpath(os.path.abspath(ext_src), sd)
+            os.symlink(rp, sal)
+
+            ml = cmake.ModuleLibrary(
+                ext_name,
+                src_files=[
+                    sl,
+                ],
+                include_dirs=[
+                    f'${{{self.var_prefix}_INCLUDE_DIRECTORIES}}',
+                ],
+                compile_opts=[
+                    f'${{{self.var_prefix}_COMPILE_OPTIONS}}',
+                ],
+                link_dirs=[
+                    f'${{{self.var_prefix}_LINK_DIRECTORIES}}',
+                ],
+                link_libs=[
+                    f'${{{self.var_prefix}_LINK_LIBRARIES}}',
+                ],
+                extra_cmds=[
+                    cmake.Command(
+                        'add_custom_command',
+                        ['TARGET', ext_name, 'POST_BUILD'],
+                        [
+                            ' '.join([
+                                'COMMAND ${CMAKE_COMMAND} -E ',
+                                f'copy $<TARGET_FILE_NAME:{ext_name}> ../../{os.path.dirname(ext_src)}/{so_name}',
+                            ]),
+                            'COMMAND_EXPAND_LISTS',
+                        ],
+                    ),
+                ],
+            )
+            self.g.write_target(ml)
+
+        def run(self) -> None:
+            self.g.write(self.g.preamble)
+            self.g.write('')
+
+            self.g.write(f'project({self.p.prj_name()})')
+            self.g.write('')
+
+            self.g.write_var(cmake.Var(
+                f'{self.var_prefix}_INCLUDE_DIRECTORIES',
+                _sep_str_grps(
+                    [f'{self.py.venv_root}/include'],
+                    [f'{self.py.root}/include/python{self.py.suffix}'],
+                    [
+                        # $ENV{HOME}/src/python/cpython
+                        # $ENV{HOME}/src/python/cpython/include
+                    ],
+                ),
+            ))
+
+            self.g.write_var(cmake.Var(
+                f'{self.var_prefix}_COMPILE_OPTIONS',
+                _sep_str_grps(
+                    [
+                        '-Wsign-compare',
+                        '-Wunreachable-code',
+                        '-DNDEBUG',
+                        '-g',
+                        '-fwrapv',
+                        '-O3',
+                        '-Wall',
+                    ],
+                    [
+                        '-g',
+                        '-c',
+                    ],
+                    ['-std=c++20'],
+                ),
+            ))
+
+            self.g.write_var(cmake.Var(
+                f'{self.var_prefix}_LINK_DIRECTORIES',
+                _sep_str_grps(
+                    [f'{self.py.root}/lib'],
+                    # ['$ENV{HOME}/src/python/cpython'],
+                ),
+            ))
+
+            self.g.write_var(cmake.Var(
+                f'{self.var_prefix}_LINK_LIBRARIES',
+                _sep_str_grps(
+                    *([[
+                        '-bundle',
+                        '"-undefined dynamic_lookup"',
+                    ]] if sys.platform == 'darwin' else []),
+                ),
+            ))
+
+            for ext_src in self.p.ext_files():
+                self._add_ext(os.path.relpath(ext_src, self.p.prj_root))
+
+    #
+
+    def run(self) -> None:
+        if not os.path.isfile(os.path.join(self._prj_root, 'pyproject.toml')):
+            raise Exception('Must be run in project root')
+
+        self.ext_files()
+
+        log.info('Generating cmake project %s', self.prj_name())
+
+        self.cmake_dir()
+        self.write_git_ignore()
+        self.write_idea_name()
+
+        out = io.StringIO()
+        clg = self._CmakeListsGen(self, out)
+        clg.run()
+
+        with open(os.path.join(self.cmake_dir(), 'CMakeLists.txt'), 'w') as f:
+            f.write(out.getvalue())
+
+
+##
+
+
+def _gen_cmd(args) -> None:
+    if not args.exts:
+        raise Exception('must specify exts')
+
+    cpg = CmakeProjectGen(args.exts)
+    cpg.run()
+
+
+def _main(argv=None) -> None:
+    logs.configure_standard_logging('INFO')
+
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers()
+
+    parser_gen = subparsers.add_parser('gen')
+    parser_gen.add_argument('exts', nargs='*')
+    parser_gen.set_defaults(func=_gen_cmd)
+
+    args = parser.parse_args(argv)
+    if not getattr(args, 'func', None):
+        parser.print_help()
+    else:
+        args.func(args)
+
+
+if __name__ == '__main__':
+    _main()

{omdev-0.0.0.dev11 → omdev-0.0.0.dev12}/omdev/exts/scan.py

@@ -12,7 +12,7 @@ log = logging.getLogger(__name__)
 SCAN_COMMENT = '// @omdev-ext'
 
 
-def _scan_one(
+def scan_one(
         input_path: str,
         **kwargs: ta.Any,
 ) -> None:
@@ -42,7 +42,7 @@ def _scan_cmd(args) -> None:
         log.info('Scanning %s', i)
         for we_dirpath, we_dirnames, we_filenames in os.walk(i):  # noqa
             for fname in we_filenames:
-                _scan_one(
+                scan_one(
                     os.path.abspath(os.path.join(we_dirpath, fname)),
                 )
 

{omdev-0.0.0.dev11 → omdev-0.0.0.dev12}/omdev/scripts/findmagic.py

@@ -28,8 +28,11 @@ def find_magic(
                     continue
 
                 fp = os.path.join(dp, fn)
-                with open(fp) as f:
-                    src = f.read()
+                try:
+                    with open(fp) as f:
+                        src = f.read()
+                except UnicodeDecodeError:
+                    continue
 
                 if not any(
                         any(pat.fullmatch(l) for pat in pats)

omdev-0.0.0.dev12/omdev/tools/revisions.py

@@ -0,0 +1,173 @@
+"""
+TODO:
+ - omlish-lite, move to pyproject/
+  - vendor-lite wheel.wheelfile
+"""
+# ruff: noqa: TCH003 UP006 UP007
+# @omlish-lite
+import argparse
+import io
+import os.path
+import subprocess
+import tarfile
+import typing as ta
+import zipfile
+
+from omlish.lite.logs import configure_standard_logging
+from omlish.lite.logs import log
+
+from ..wheelfile import WheelFile
+
+
+class RevisionAdder:
+    def __init__(
+            self,
+            revision: str,
+            output_suffix: ta.Optional[str] = None,
+    ) -> None:
+        super().__init__()
+        self._revision = revision
+        self._output_suffix = output_suffix
+
+    REVISION_ATTR = '__revision__'
+
+    def add_to_contents(self, dct: ta.Dict[str, bytes]) -> bool:
+        changed = False
+        for n in dct:
+            if not n.endswith('__about__.py'):
+                continue
+            src = dct[n].decode('utf-8')
+            lines = src.splitlines(keepends=True)
+            for i, l in enumerate(lines):
+                if l != f'{self.REVISION_ATTR} = None\n':
+                    continue
+                lines[i] = f"{self.REVISION_ATTR} = '{self._revision}'\n"
+                changed = True
+            dct[n] = ''.join(lines).encode('utf-8')
+        return changed
+
+    def add_to_wheel(self, f: str) -> None:
+        if not f.endswith('.whl'):
+            raise Exception(f)
+        log.info('Scanning wheel %s', f)
+
+        zis: ta.Dict[str, zipfile.ZipInfo] = {}
+        dct: ta.Dict[str, bytes] = {}
+        with WheelFile(f) as wf:
+            for zi in wf.filelist:
+                if zi.filename == wf.record_path:
+                    continue
+                zis[zi.filename] = zi
+                dct[zi.filename] = wf.read(zi.filename)
+
+        if self.add_to_contents(dct):
+            of = f[:-4] + (self._output_suffix or '') + '.whl'
+            log.info('Repacking wheel %s', of)
+            with WheelFile(of, 'w') as wf:
+                for n, d in dct.items():
+                    log.info('Adding zipinfo %s', n)
+                    wf.writestr(zis[n], d)
+
+    def add_to_tgz(self, f: str) -> None:
+        if not f.endswith('.tar.gz'):
+            raise Exception(f)
+        log.info('Scanning tgz %s', f)
+
+        tis: ta.Dict[str, tarfile.TarInfo] = {}
+        dct: ta.Dict[str, bytes] = {}
+        with tarfile.open(f, 'r:gz') as tf:
+            for ti in tf:
+                tis[ti.name] = ti
+                if ti.type == tarfile.REGTYPE:
+                    with tf.extractfile(ti.name) as tif:  # type: ignore
+                        dct[ti.name] = tif.read()
+
+        if self.add_to_contents(dct):
+            of = f[:-7] + (self._output_suffix or '') + '.tar.gz'
+            log.info('Repacking tgz %s', of)
+            with tarfile.open(of, 'w:gz') as tf:
+                for n, ti in tis.items():
+                    log.info('Adding tarinfo %s', n)
+                    if n in dct:
+                        data = dct[n]
+                        ti.size = len(data)
+                        fo = io.BytesIO(data)
+                    else:
+                        fo = None
+                    tf.addfile(ti, fileobj=fo)
+
+    EXTS = ('.tar.gz', '.whl')
+
+    def add_to_file(self, f: str) -> None:
+        if f.endswith('.whl'):
+            self.add_to_wheel(f)
+
+        elif f.endswith('.tar.gz'):
+            self.add_to_tgz(f)
+
+    def add_to(self, tgt: str) -> None:
+        if os.path.isfile(tgt):
+            self.add_to_file(tgt)
+
+        elif os.path.isdir(tgt):
+            for dp, dns, fns in os.walk(tgt):  # noqa
+                for f in fns:
+                    if any(f.endswith(ext) for ext in self.EXTS):
+                        self.add_to_file(os.path.join(dp, f))
+
+
+#
+
+
+def get_revision() -> str:
+    return subprocess.check_output([
+        'git',
+        'describe',
+        '--match=NeVeRmAtCh',
+        '--always',
+        '--abbrev=40',
+        '--dirty',
+    ]).decode().strip()
+
+
+#
+
+
+def _add_cmd(args) -> None:
+    if (revision := args.revision) is None:
+        revision = get_revision()
+        log.info('Using revision %s', revision)
+
+    if not args.targets:
+        raise Exception('must specify targets')
+
+    ra = RevisionAdder(
+        revision,
+        output_suffix=args.suffix,
+    )
+    for tgt in args.targets:
+        ra.add_to(tgt)
+
+
+def _main(argv=None) -> None:
+    configure_standard_logging('INFO')
+
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers()
+
+    parser_add = subparsers.add_parser('add')
+    parser_add.add_argument('-r', '--revision')
+    parser_add.add_argument('-s', '--suffix')
+    parser_add.add_argument('targets', nargs='*')
+    parser_add.set_defaults(func=_add_cmd)
+
+    args = parser.parse_args(argv)
+    if not getattr(args, 'func', None):
+        parser.print_help()
+    else:
+        args.func(args)
+
+
+if __name__ == '__main__':
+    _main()