omdev 0.0.0.dev10__tar.gz → 0.0.0.dev12__tar.gz

{omdev-0.0.0.dev10/omdev.egg-info → omdev-0.0.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev10
+Version: 0.0.0.dev12
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,13 +12,14 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev10
+Requires-Dist: omlish==0.0.0.dev12
 Provides-Extra: all
 Requires-Dist: pycparser>=2.22; extra == "all"
 Requires-Dist: cffi>=1.17; extra == "all"
 Requires-Dist: pcpp>=1.30; extra == "all"
 Requires-Dist: mypy>=1.11; extra == "all"
 Requires-Dist: tokenize_rt>=6; extra == "all"
+Requires-Dist: wheel>=0.44; extra == "all"
 Provides-Extra: c
 Requires-Dist: pycparser>=2.22; extra == "c"
 Requires-Dist: cffi>=1.17; extra == "c"
@@ -27,3 +28,5 @@ Provides-Extra: mypy
 Requires-Dist: mypy>=1.11; extra == "mypy"
 Provides-Extra: tokens
 Requires-Dist: tokenize_rt>=6; extra == "tokens"
+Provides-Extra: wheel
+Requires-Dist: wheel>=0.44; extra == "wheel"

omdev-0.0.0.dev12/README.rst

@@ -0,0 +1,5 @@
+*omlish*
+
+Packages installable from git via:
+::
+  pip install 'git+https://github.com/wrmsr/omlish@master#subdirectory=.pkg/<pkg>'

{omdev-0.0.0.dev10 → omdev-0.0.0.dev12}/omdev/__about__.py

@@ -25,6 +25,10 @@ class Project(ProjectBase):
         'tokens': [
             'tokenize_rt >= 6',
         ],
+
+        'wheel': [
+            'wheel >= 0.44',
+        ],
     }
 
 

{omdev-0.0.0.dev10 → omdev-0.0.0.dev12}/omdev/cmake.py

@@ -1,5 +1,4 @@
 import abc
-import io
 import typing as ta
 
 from omlish import dataclasses as dc
@@ -80,7 +79,7 @@ class CmakeGen:
 
     def __init__(
             self,
-            out: io.TextIOBase,
+            out: ta.TextIO,
             *,
             indent: int = 4,
     ) -> None:

omdev-0.0.0.dev12/omdev/exts/cmake.py

@@ -0,0 +1,342 @@
+"""
+TODO:
+ - symlink headers, included src files (hamt_impl, ...)
+ - point / copy output to dst dirs
+ - libs
+  - ..
+   - pybind
+   - catch2?
+   - json? https://github.com/nlohmann/json
+  - FindPackages? FetchContent? built_ext won't have that
+  - move omml git / data retriever stuff into omdev, get just the one header file from git via sha?
+ - support local built pys 
+
+==
+
+Done:
+ - https://intellij-support.jetbrains.com/hc/en-us/community/posts/206608485-Multiple-Jetbrain-IDE-sharing-the-same-project-directory really?
+  - aight, generate a whole cmake subdir with symlinks to src files lol
+
+"""  # noqa
+import argparse
+import dataclasses as dc
+import io
+import logging
+import os.path
+import shutil
+import sys
+import sysconfig
+import typing as ta
+
+from omlish import check
+from omlish import lang
+from omlish import logs
+
+from .. import cmake
+from ..scripts import findmagic
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+MAGIC = '@omdev-ext'
+MAGIC_COMMENT = f'// {MAGIC}'
+
+FILE_EXTENSIONS = ('c', 'cc', 'cpp')
+
+
+def _sep_str_grps(*ls: ta.Sequence[str]) -> list[str]:
+    o = []
+    for i, l in enumerate(ls):
+        if not l:
+            continue
+        if i:
+            o.append('')
+        o.extend(check.not_isinstance(l, str))
+    return o
+
+
+class CmakeProjectGen:
+    def __init__(
+            self,
+            exts: ta.Sequence[str],
+            prj_root: str | None = None,
+    ) -> None:
+        super().__init__()
+        self._exts = check.not_isinstance(exts, str)
+        self._prj_root = os.path.abspath(prj_root) if prj_root is not None else os.getcwd()
+
+    #
+
+    @property
+    def prj_root(self) -> str:
+        return self._prj_root
+
+    @lang.cached_function
+    def prj_name(self) -> str:
+        return os.path.basename(os.path.dirname(self.prj_root))
+
+    @lang.cached_function
+    def cmake_dir(self) -> str:
+        cmake_dir = os.path.join(self.prj_root, 'cmake')
+        if os.path.exists(cmake_dir):
+            for e in os.listdir(cmake_dir):
+                if e == '.idea':
+                    continue
+                ep = os.path.join(cmake_dir, e)
+                if os.path.isfile(ep):
+                    os.unlink(ep)
+                else:
+                    shutil.rmtree(ep)
+        else:
+            os.mkdir(cmake_dir)
+        return cmake_dir
+
+    #
+
+    def write_git_ignore(self) -> None:
+        with open(os.path.join(self.cmake_dir(), '.gitignore'), 'w') as f:
+            f.write('\n'.join(sorted(['/cmake-*', '/build'])))
+
+    #
+
+    def write_idea_name(self) -> None:
+        idea_dir = os.path.join(self.cmake_dir(), '.idea')
+        if not os.path.isdir(idea_dir):
+            os.mkdir(idea_dir)
+        idea_name_file = os.path.join(idea_dir, '.name')
+        if not os.path.isfile(idea_name_file):
+            with open(idea_name_file, 'w') as f:
+                f.write(self.prj_name())
+
+    #
+
+    @dc.dataclass(frozen=True, kw_only=True)
+    class PyInfo:
+        venv_exe: str
+        venv_root: str
+        real_exe: str
+        root: str
+        suffix: str
+
+    @lang.cached_function
+    def py_info(self) -> PyInfo:
+        venv_exe = sys.executable
+        real_exe = os.path.realpath(venv_exe)
+        return self.PyInfo(
+            venv_exe=venv_exe,
+            venv_root=os.path.abspath(os.path.join(os.path.dirname(venv_exe), '..')),
+            real_exe=real_exe,
+            root=os.path.abspath(os.path.join(os.path.dirname(real_exe), '..')),
+            suffix='.'.join(map(str, sys.version_info[:2])),
+        )
+
+    #
+
+    @lang.cached_function
+    def ext_files(self) -> ta.Sequence[str]:
+        out = []
+        for e in self._exts:
+            e = os.path.abspath(e)
+            if os.path.isfile(e):
+                out.append(e)
+            elif os.path.isdir(e):
+                out.extend(
+                    findmagic.find_magic(
+                        [e],
+                        [MAGIC_COMMENT],
+                        FILE_EXTENSIONS,
+                    ),
+                )
+            else:
+                raise KeyError(e)
+        return out
+
+    #
+
+    class _CmakeListsGen:
+        def __init__(
+                self,
+                p: 'CmakeProjectGen',
+                out: ta.TextIO,
+        ) -> None:
+            super().__init__()
+            self.p = p
+            self.g = cmake.CmakeGen(out)
+
+        @lang.cached_property
+        def var_prefix(self) -> str:
+            return self.p.prj_name().upper()
+
+        @lang.cached_property
+        def py(self) -> 'CmakeProjectGen.PyInfo':
+            return self.p.py_info()
+
+        def _add_ext(self, ext_src: str) -> None:
+            ext_name = ext_src.rpartition('.')[0].replace('/', '__')
+
+            log.info('Adding cmake c extension: %s -> %s', ext_src, ext_name)
+
+            so_name = ''.join([
+                os.path.basename(ext_src).split('.')[0],
+                '.',
+                sysconfig.get_config_var('SOABI'),
+                sysconfig.get_config_var('SHLIB_SUFFIX'),
+            ])
+
+            sl = os.path.join(self.p.cmake_dir(), ext_src)
+            sal = os.path.abspath(sl)
+            sd = os.path.dirname(sal)
+            os.makedirs(sd, exist_ok=True)
+            rp = os.path.relpath(os.path.abspath(ext_src), sd)
+            os.symlink(rp, sal)
+
+            ml = cmake.ModuleLibrary(
+                ext_name,
+                src_files=[
+                    sl,
+                ],
+                include_dirs=[
+                    f'${{{self.var_prefix}_INCLUDE_DIRECTORIES}}',
+                ],
+                compile_opts=[
+                    f'${{{self.var_prefix}_COMPILE_OPTIONS}}',
+                ],
+                link_dirs=[
+                    f'${{{self.var_prefix}_LINK_DIRECTORIES}}',
+                ],
+                link_libs=[
+                    f'${{{self.var_prefix}_LINK_LIBRARIES}}',
+                ],
+                extra_cmds=[
+                    cmake.Command(
+                        'add_custom_command',
+                        ['TARGET', ext_name, 'POST_BUILD'],
+                        [
+                            ' '.join([
+                                'COMMAND ${CMAKE_COMMAND} -E ',
+                                f'copy $<TARGET_FILE_NAME:{ext_name}> ../../{os.path.dirname(ext_src)}/{so_name}',
+                            ]),
+                            'COMMAND_EXPAND_LISTS',
+                        ],
+                    ),
+                ],
+            )
+            self.g.write_target(ml)
+
+        def run(self) -> None:
+            self.g.write(self.g.preamble)
+            self.g.write('')
+
+            self.g.write(f'project({self.p.prj_name()})')
+            self.g.write('')
+
+            self.g.write_var(cmake.Var(
+                f'{self.var_prefix}_INCLUDE_DIRECTORIES',
+                _sep_str_grps(
+                    [f'{self.py.venv_root}/include'],
+                    [f'{self.py.root}/include/python{self.py.suffix}'],
+                    [
+                        # $ENV{HOME}/src/python/cpython
+                        # $ENV{HOME}/src/python/cpython/include
+                    ],
+                ),
+            ))
+
+            self.g.write_var(cmake.Var(
+                f'{self.var_prefix}_COMPILE_OPTIONS',
+                _sep_str_grps(
+                    [
+                        '-Wsign-compare',
+                        '-Wunreachable-code',
+                        '-DNDEBUG',
+                        '-g',
+                        '-fwrapv',
+                        '-O3',
+                        '-Wall',
+                    ],
+                    [
+                        '-g',
+                        '-c',
+                    ],
+                    ['-std=c++20'],
+                ),
+            ))
+
+            self.g.write_var(cmake.Var(
+                f'{self.var_prefix}_LINK_DIRECTORIES',
+                _sep_str_grps(
+                    [f'{self.py.root}/lib'],
+                    # ['$ENV{HOME}/src/python/cpython'],
+                ),
+            ))
+
+            self.g.write_var(cmake.Var(
+                f'{self.var_prefix}_LINK_LIBRARIES',
+                _sep_str_grps(
+                    *([[
+                        '-bundle',
+                        '"-undefined dynamic_lookup"',
+                    ]] if sys.platform == 'darwin' else []),
+                ),
+            ))
+
+            for ext_src in self.p.ext_files():
+                self._add_ext(os.path.relpath(ext_src, self.p.prj_root))
+
+    #
+
+    def run(self) -> None:
+        if not os.path.isfile(os.path.join(self._prj_root, 'pyproject.toml')):
+            raise Exception('Must be run in project root')
+
+        self.ext_files()
+
+        log.info('Generating cmake project %s', self.prj_name())
+
+        self.cmake_dir()
+        self.write_git_ignore()
+        self.write_idea_name()
+
+        out = io.StringIO()
+        clg = self._CmakeListsGen(self, out)
+        clg.run()
+
+        with open(os.path.join(self.cmake_dir(), 'CMakeLists.txt'), 'w') as f:
+            f.write(out.getvalue())
+
+
+##
+
+
+def _gen_cmd(args) -> None:
+    if not args.exts:
+        raise Exception('must specify exts')
+
+    cpg = CmakeProjectGen(args.exts)
+    cpg.run()
+
+
+def _main(argv=None) -> None:
+    logs.configure_standard_logging('INFO')
+
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers()
+
+    parser_gen = subparsers.add_parser('gen')
+    parser_gen.add_argument('exts', nargs='*')
+    parser_gen.set_defaults(func=_gen_cmd)
+
+    args = parser.parse_args(argv)
+    if not getattr(args, 'func', None):
+        parser.print_help()
+    else:
+        args.func(args)
+
+
+if __name__ == '__main__':
+    _main()

{omdev-0.0.0.dev10 → omdev-0.0.0.dev12}/omdev/exts/scan.py

@@ -12,7 +12,7 @@ log = logging.getLogger(__name__)
 SCAN_COMMENT = '// @omdev-ext'
 
 
-def _scan_one(
+def scan_one(
         input_path: str,
         **kwargs: ta.Any,
 ) -> None:
@@ -42,7 +42,7 @@ def _scan_cmd(args) -> None:
         log.info('Scanning %s', i)
         for we_dirpath, we_dirnames, we_filenames in os.walk(i):  # noqa
             for fname in we_filenames:
-                _scan_one(
+                scan_one(
                     os.path.abspath(os.path.join(we_dirpath, fname)),
                 )
 

{omdev-0.0.0.dev10 → omdev-0.0.0.dev12}/omdev/interp/pyenv.py

@@ -67,8 +67,10 @@ class Pyenv:
         return os.path.join(check_not_none(self.root()), 'bin', 'pyenv')
 
     def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
+        if (root := self.root()) is None:
+            return []
         ret = []
-        vp = os.path.join(self.root(), 'versions')
+        vp = os.path.join(root, 'versions')
         for dn in os.listdir(vp):
             ep = os.path.join(vp, dn, 'bin', 'python')
             if not os.path.isfile(ep):
@@ -77,6 +79,8 @@ class Pyenv:
         return ret
 
     def installable_versions(self) -> ta.List[str]:
+        if self.root() is None:
+            return []
         ret = []
         s = subprocess_check_output_str(self.exe(), 'install', '--list')
         for l in s.splitlines():

{omdev-0.0.0.dev10 → omdev-0.0.0.dev12}/omdev/pyproject/cli.py

@@ -26,6 +26,7 @@ import dataclasses as dc
 import functools
 import glob
 import itertools
+import multiprocessing as mp
 import os.path
 import shlex
 import shutil
@@ -247,13 +248,12 @@ def _venv_cmd(args) -> None:
         )
         return
 
-    venv.create()
-
     cmd = args.cmd
     if not cmd:
-        pass
+        venv.create()
 
     elif cmd == 'python':
+        venv.create()
         os.execl(
             (exe := venv.exe()),
             exe,
@@ -261,10 +261,12 @@ def _venv_cmd(args) -> None:
         )
 
     elif cmd == 'exe':
+        venv.create()
         check_not(args.args)
         print(venv.exe())
 
     elif cmd == 'run':
+        venv.create()
         sh = check_not_none(shutil.which('bash'))
         script = ' '.join(args.args)
         if not script:
@@ -281,6 +283,7 @@ def _venv_cmd(args) -> None:
         print('\n'.join(venv.srcs()))
 
     elif cmd == 'test':
+        venv.create()
         subprocess_check_call(venv.exe(), '-m', 'pytest', *(args.args or []), *venv.srcs())
 
     else:
@@ -306,11 +309,10 @@ def _pkg_cmd(args) -> None:
         build_output_dir = 'dist'
         run_build = bool(args.build)
 
-        num_threads = 8
-
         if run_build:
             os.makedirs(build_output_dir, exist_ok=True)
 
+        num_threads = max(mp.cpu_count() // 2, 1)
         with cf.ThreadPoolExecutor(num_threads) as ex:
             futs = [
                 ex.submit(functools.partial(

omdev-0.0.0.dev12/omdev/scripts/bracepy.py

@@ -0,0 +1,105 @@
+# !/usr/bin/env python3
+"""
+https://github.com/umlet/pwk/blob/dc23b3400108a71947a695f1fa1df0f514b42528/pwk
+"""
+import io
+import tokenize
+
+
+def translate_brace_python(
+        s: str,
+        *,
+        indent_width: int = 4,
+) -> str:
+    lt = tokenize.tokenize(io.BytesIO(s.encode('utf-8')).readline)
+
+    ret = io.StringIO()
+
+    indent = 0
+    open_braces = 0
+
+    newline = False
+    indent_up = False
+    indent_down = False
+    skip = False
+
+    while True:
+        try:
+            t = next(lt)
+
+            if t.type == tokenize.ENCODING:
+                last_t = t
+                continue
+
+            if t.type == tokenize.OP and t.string == ';':
+                newline = True
+
+            elif t.type == tokenize.OP and t.string == ':':
+                if open_braces == 0:
+                    newline = True
+                    indent_up = True
+
+            elif t.type == tokenize.OP and t.string == '{':
+                if last_t.type == tokenize.OP and last_t.string == ':':  # noqa
+                    skip = True
+                else:
+                    open_braces += 1
+
+            elif t.type == tokenize.OP and t.string == '}':
+                if open_braces > 0:
+                    open_braces -= 1
+                elif open_braces == 0:
+                    if indent > 0:
+                        newline = True
+                        indent_down = True
+                        skip = True
+                    else:
+                        raise Exception('Too many closing braces')
+
+            if indent_up:
+                indent += indent_width
+            elif indent_down:
+                indent -= indent_width
+
+            if newline and indent_up:
+                ret.write(':\n' + ' ' * indent)
+            elif newline:
+                ret.write('\n' + ' ' * indent)
+            elif not skip:
+                ret.write(t.string)
+                ret.write(' ')
+
+            newline = False
+            indent_up = False
+            indent_down = False
+            skip = False
+            last_t = t
+
+        except StopIteration:
+            break
+        except tokenize.TokenError:
+            continue
+
+    ret.write('\n')
+    return ret.getvalue()
+
+
+def _main(argv=None) -> None:
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-x', '--exec', action='store_true')
+    parser.add_argument('cmd')
+
+    args = parser.parse_args(argv)
+
+    src = translate_brace_python(args.cmd)
+
+    if args.exec:
+        exec(src)
+    else:
+        print(src)
+
+
+if __name__ == '__main__':
+    _main()

{omdev-0.0.0.dev10 → omdev-0.0.0.dev12}/omdev/scripts/findmagic.py

@@ -28,8 +28,11 @@ def find_magic(
                     continue
 
                 fp = os.path.join(dp, fn)
-                with open(fp) as f:
-                    src = f.read()
+                try:
+                    with open(fp) as f:
+                        src = f.read()
+                except UnicodeDecodeError:
+                    continue
 
                 if not any(
                         any(pat.fullmatch(l) for pat in pats)

{omdev-0.0.0.dev10 → omdev-0.0.0.dev12}/omdev/scripts/interp.py

@@ -1669,8 +1669,10 @@ class Pyenv:
         return os.path.join(check_not_none(self.root()), 'bin', 'pyenv')
 
     def version_exes(self) -> ta.List[ta.Tuple[str, str]]:
+        if (root := self.root()) is None:
+            return []
         ret = []
-        vp = os.path.join(self.root(), 'versions')
+        vp = os.path.join(root, 'versions')
         for dn in os.listdir(vp):
             ep = os.path.join(vp, dn, 'bin', 'python')
             if not os.path.isfile(ep):
@@ -1679,6 +1681,8 @@ class Pyenv:
         return ret
 
     def installable_versions(self) -> ta.List[str]:
+        if self.root() is None:
+            return []
         ret = []
         s = subprocess_check_output_str(self.exe(), 'install', '--list')
         for l in s.splitlines():