omdev 0.0.0.dev17__tar.gz → 0.0.0.dev19__tar.gz

{omdev-0.0.0.dev17/omdev.egg-info → omdev-0.0.0.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev17
+Version: 0.0.0.dev19
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev17
+Requires-Dist: omlish==0.0.0.dev19
 Provides-Extra: all
 Requires-Dist: pycparser~=2.22; extra == "all"
 Requires-Dist: cffi~=1.17; extra == "all"

{omdev-0.0.0.dev17 → omdev-0.0.0.dev19}/omdev/__about__.py

@@ -33,6 +33,8 @@ class Project(ProjectBase):
 
 
 class Setuptools(SetuptoolsBase):
+    cexts = True
+
     find_packages = {
         'include': [Project.name, f'{Project.name}.*'],
         'exclude': [*SetuptoolsBase.find_packages['exclude']],

{omdev-0.0.0.dev17 → omdev-0.0.0.dev19}/omdev/cexts/cmake.py

@@ -1,6 +1,7 @@
 """
 FIXME:
  - debug tables don't handle symlinks
+ - use relapths in cml.txt
 
 TODO:
  - symlink headers, included src files (hamt_impl, ...)
@@ -37,6 +38,7 @@ from omlish import logs
 
 from .. import cmake
 from .. import findmagic
+from .magic import CextMagic
 
 
 log = logging.getLogger(__name__)
@@ -45,12 +47,6 @@ log = logging.getLogger(__name__)
 ##
 
 
-MAGIC = '@omdev-cext'
-MAGIC_COMMENT = f'// {MAGIC}'
-
-FILE_EXTENSIONS = ('c', 'cc', 'cpp')
-
-
 def _sep_str_grps(*ls: ta.Sequence[str]) -> list[str]:
     o = []
     for i, l in enumerate(ls):
@@ -87,7 +83,7 @@ class CmakeProjectGen:
 
     @lang.cached_function
     def cmake_dir(self) -> str:
-        cmake_dir = os.path.join(self.prj_root, 'cmake')
+        cmake_dir = os.path.join(self.prj_root, 'cmake', self.prj_name())
         if os.path.exists(cmake_dir):
             for e in os.listdir(cmake_dir):
                 if e == '.idea':
@@ -98,7 +94,7 @@ class CmakeProjectGen:
                 else:
                     shutil.rmtree(ep)
         else:
-            os.mkdir(cmake_dir)
+            os.makedirs(cmake_dir)
         return cmake_dir
 
     #
@@ -109,17 +105,6 @@ class CmakeProjectGen:
 
     #
 
-    def write_idea_name(self) -> None:
-        idea_dir = os.path.join(self.cmake_dir(), '.idea')
-        if not os.path.isdir(idea_dir):
-            os.mkdir(idea_dir)
-        idea_name_file = os.path.join(idea_dir, '.name')
-        if not os.path.isfile(idea_name_file):
-            with open(idea_name_file, 'w') as f:
-                f.write(self.prj_name())
-
-    #
-
     @dc.dataclass(frozen=True, kw_only=True)
     class PyInfo:
         venv_exe: str
@@ -153,8 +138,8 @@ class CmakeProjectGen:
                 out.extend(
                     findmagic.find_magic(
                         [e],
-                        [MAGIC_COMMENT],
-                        FILE_EXTENSIONS,
+                        [CextMagic.MAGIC_COMMENT],
+                        CextMagic.FILE_EXTENSIONS,
                     ),
                 )
             else:
@@ -224,7 +209,7 @@ class CmakeProjectGen:
                         [
                             ' '.join([
                                 'COMMAND ${CMAKE_COMMAND} -E ',
-                                f'copy $<TARGET_FILE_NAME:{ext_name}> ../../{os.path.dirname(ext_src)}/{so_name}',
+                                f'copy $<TARGET_FILE_NAME:{ext_name}> ../../../{os.path.dirname(ext_src)}/{so_name}',
                             ]),
                             'COMMAND_EXPAND_LISTS',
                         ],
@@ -311,7 +296,6 @@ class CmakeProjectGen:
 
         self.cmake_dir()
         self.write_git_ignore()
-        self.write_idea_name()
 
         out = io.StringIO()
         clg = self._CmakeListsGen(self, out)

omdev-0.0.0.dev19/omdev/cexts/magic.py

@@ -0,0 +1,7 @@
+# @omlish-lite
+
+class CextMagic:
+    MAGIC = '@omdev-cext'
+    MAGIC_COMMENT = f'// {MAGIC}'
+
+    FILE_EXTENSIONS = ('c', 'cc', 'cpp')

{omdev-0.0.0.dev17 → omdev-0.0.0.dev19}/omdev/cexts/scan.py

@@ -5,18 +5,17 @@ import typing as ta
 
 from omlish import logs
 
-
-log = logging.getLogger(__name__)
+from .magic import CextMagic
 
 
-SCAN_COMMENT = '// @omdev-cext'
+log = logging.getLogger(__name__)
 
 
 def scan_one(
         input_path: str,
         **kwargs: ta.Any,
 ) -> None:
-    if not any(input_path.endswith(fx) for fx in ('.c', '.cc', '.cpp')):
+    if not any(input_path.endswith('.' + fx) for fx in CextMagic.FILE_EXTENSIONS):
         return
 
     with open(input_path, 'rb') as f:
@@ -27,9 +26,9 @@ def scan_one(
     except UnicodeDecodeError:
         return
 
-    sls = [l for l in src.splitlines() if l.startswith(SCAN_COMMENT)]
+    sls = [l for l in src.splitlines() if l.startswith(CextMagic.MAGIC_COMMENT)]
     for sl in sls:
-        sas = sl[len(SCAN_COMMENT):].split()  # noqa
+        sas = sl[len(CextMagic.MAGIC_COMMENT):].split()  # noqa
 
         log.info('Found ext: %s', input_path)
 

{omdev-0.0.0.dev17 → omdev-0.0.0.dev19}/omdev/pyproject/cli.py

@@ -301,10 +301,10 @@ def _pkg_cmd(args) -> None:
         raise Exception('must specify command')
 
     elif cmd == 'gen':
-        build_root = os.path.join('.pkg')
+        pkgs_root = os.path.join('.pkg')
 
-        if os.path.exists(build_root):
-            shutil.rmtree(build_root)
+        if os.path.exists(pkgs_root):
+            shutil.rmtree(pkgs_root)
 
         build_output_dir = 'dist'
         run_build = bool(args.build)
@@ -319,11 +319,13 @@ def _pkg_cmd(args) -> None:
                 ex.submit(functools.partial(
                     PyprojectPackageGenerator(
                         dir_name,
-                        build_root,
+                        pkgs_root,
                     ).gen,
-                    run_build=run_build,
-                    build_output_dir=build_output_dir,
-                    add_revision=add_revision,
+                    PyprojectPackageGenerator.GenOpts(
+                        run_build=run_build,
+                        build_output_dir=build_output_dir,
+                        add_revision=add_revision,
+                    ),
                 ))
                 for dir_name in run.cfg().pkgs
             ]

omdev-0.0.0.dev19/omdev/pyproject/pkg.py

@@ -0,0 +1,388 @@
+"""
+TODO:
+ - ext scanning
+ - __revision__
+ - entry_points
+
+** NOTE **
+setuptools now (2024/09/02) has experimental support for extensions in pure pyproject.toml - but we still want a
+separate '-cext' package
+  https://setuptools.pypa.io/en/latest/userguide/ext_modules.html
+  https://github.com/pypa/setuptools/commit/1a9d87308dc0d8aabeaae0dce989b35dfb7699f0#diff-61d113525e9cc93565799a4bb8b34a68e2945b8a3f7d90c81380614a4ea39542R7-R8
+
+--
+
+https://setuptools.pypa.io/en/latest/references/keywords.html
+https://packaging.python.org/en/latest/specifications/pyproject-toml
+
+How to build a C extension in keeping with PEP 517, i.e. with pyproject.toml instead of setup.py?
+https://stackoverflow.com/a/66479252
+
+https://github.com/pypa/sampleproject/blob/db5806e0a3204034c51b1c00dde7d5eb3fa2532e/setup.py
+
+https://pip.pypa.io/en/stable/cli/pip_install/#vcs-support
+vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir
+'git+https://github.com/wrmsr/omlish@master#subdirectory=.pip/omlish'
+"""  # noqa
+# ruff: noqa: UP006 UP007
+import abc
+import dataclasses as dc
+import importlib
+import os.path
+import shutil
+import subprocess
+import sys
+import types
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.logs import log
+
+from ..cexts.magic import CextMagic
+from ..findmagic import find_magic
+from ..toml.writer import TomlWriter
+from ..tools.revisions import GitRevisionAdder
+
+
+#
+
+
+class BasePyprojectPackageGenerator(abc.ABC):
+    def __init__(
+            self,
+            dir_name: str,
+            pkgs_root: str,
+            *,
+            pkg_suffix: str = '',
+    ) -> None:
+        super().__init__()
+        self._dir_name = dir_name
+        self._pkgs_root = pkgs_root
+        self._pkg_suffix = pkg_suffix
+
+    #
+
+    @cached_nullary
+    def about(self) -> types.ModuleType:
+        return importlib.import_module(f'{self._dir_name}.__about__')
+
+    #
+
+    @cached_nullary
+    def _pkg_dir(self) -> str:
+        pkg_dir: str = os.path.join(self._pkgs_root, self._dir_name + self._pkg_suffix)
+        if os.path.isdir(pkg_dir):
+            shutil.rmtree(pkg_dir)
+        os.makedirs(pkg_dir)
+        return pkg_dir
+
+    #
+
+    _GIT_IGNORE: ta.Sequence[str] = [
+        '/*.egg-info/',
+        '/dist',
+    ]
+
+    def _write_git_ignore(self) -> None:
+        with open(os.path.join(self._pkg_dir(), '.gitignore'), 'w') as f:
+            f.write('\n'.join(self._GIT_IGNORE))
+
+    #
+
+    def _symlink_source_dir(self) -> None:
+        os.symlink(
+            os.path.relpath(self._dir_name, self._pkg_dir()),
+            os.path.join(self._pkg_dir(), self._dir_name),
+        )
+
+    #
+
+    @cached_nullary
+    def project_cls(self) -> type:
+        return self.about().Project
+
+    @cached_nullary
+    def setuptools_cls(self) -> type:
+        return self.about().Setuptools
+
+    @staticmethod
+    def _build_cls_dct(cls: type) -> ta.Dict[str, ta.Any]:  # noqa
+        dct = {}
+        for b in reversed(cls.__mro__):
+            for k, v in b.__dict__.items():
+                if k.startswith('_'):
+                    continue
+                dct[k] = v
+        return dct
+
+    @staticmethod
+    def _move_dict_key(
+            sd: ta.Dict[str, ta.Any],
+            sk: str,
+            dd: ta.Dict[str, ta.Any],
+            dk: str,
+    ) -> None:
+        if sk in sd:
+            dd[dk] = sd.pop(sk)
+
+    @dc.dataclass(frozen=True)
+    class Specs:
+        pyproject: ta.Dict[str, ta.Any]
+        setuptools: ta.Dict[str, ta.Any]
+
+    def build_specs(self) -> Specs:
+        return self.Specs(
+            self._build_cls_dct(self.project_cls()),
+            self._build_cls_dct(self.setuptools_cls()),
+        )
+
+    #
+
+    @abc.abstractmethod
+    def _write_file_contents(self) -> None:
+        raise NotImplementedError
+
+    #
+
+    _STANDARD_FILES: ta.Sequence[str] = [
+        'LICENSE',
+        'README.rst',
+    ]
+
+    def _symlink_standard_files(self) -> None:
+        for fn in self._STANDARD_FILES:
+            if os.path.exists(fn):
+                os.symlink(os.path.relpath(fn, self._pkg_dir()), os.path.join(self._pkg_dir(), fn))
+
+    #
+
+    def _run_build(
+            self,
+            build_output_dir: ta.Optional[str] = None,
+            *,
+            add_revision: bool = False,
+    ) -> None:
+        subprocess.check_call(
+            [
+                sys.executable,
+                '-m',
+                'build',
+            ],
+            cwd=self._pkg_dir(),
+        )
+
+        dist_dir = os.path.join(self._pkg_dir(), 'dist')
+
+        if add_revision:
+            GitRevisionAdder().add_to(dist_dir)
+
+        if build_output_dir is not None:
+            for fn in os.listdir(dist_dir):
+                shutil.copyfile(os.path.join(dist_dir, fn), os.path.join(build_output_dir, fn))
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class GenOpts:
+        run_build: bool = False
+        build_output_dir: ta.Optional[str] = None
+        add_revision: bool = False
+
+    def gen(self, opts: GenOpts = GenOpts()) -> str:
+        log.info('Generating pyproject package: %s -> %s (%s)', self._dir_name, self._pkgs_root, self._pkg_suffix)
+
+        self._pkg_dir()
+        self._write_git_ignore()
+        self._symlink_source_dir()
+        self._write_file_contents()
+        self._symlink_standard_files()
+
+        if opts.run_build:
+            self._run_build(
+                opts.build_output_dir,
+                add_revision=opts.add_revision,
+            )
+
+        return self._pkg_dir()
+
+
+#
+
+
+class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class FileContents:
+        pyproject_dct: ta.Mapping[str, ta.Any]
+        manifest_in: ta.Optional[ta.Sequence[str]]
+
+    @cached_nullary
+    def file_contents(self) -> FileContents:
+        specs = self.build_specs()
+
+        #
+
+        pyp_dct = {}
+
+        pyp_dct['build-system'] = {
+            'requires': ['setuptools'],
+            'build-backend': 'setuptools.build_meta',
+        }
+
+        prj = specs.pyproject
+        prj['name'] += self._pkg_suffix
+
+        pyp_dct['project'] = prj
+
+        self._move_dict_key(prj, 'optional_dependencies', pyp_dct, extrask := 'project.optional-dependencies')
+        if (extras := pyp_dct.get(extrask)):
+            pyp_dct[extrask] = {
+                'all': [
+                    e
+                    for lst in extras.values()
+                    for e in lst
+                ],
+                **extras,
+            }
+
+        #
+
+        st = specs.setuptools
+        pyp_dct['tool.setuptools'] = st
+
+        st.pop('cexts', None)
+
+        self._move_dict_key(st, 'find_packages', pyp_dct, 'tool.setuptools.packages.find')
+
+        mani_in = st.pop('manifest_in', None)
+
+        #
+
+        return self.FileContents(
+            pyp_dct,
+            mani_in,
+        )
+
+    def _write_file_contents(self) -> None:
+        fc = self.file_contents()
+
+        with open(os.path.join(self._pkg_dir(), 'pyproject.toml'), 'w') as f:
+            TomlWriter(f).write_root(fc.pyproject_dct)
+
+        if fc.manifest_in:
+            with open(os.path.join(self._pkg_dir(), 'MANIFEST.in'), 'w') as f:
+                f.write('\n'.join(fc.manifest_in))  # noqa
+
+    #
+
+    def gen(self, opts: BasePyprojectPackageGenerator.GenOpts = BasePyprojectPackageGenerator.GenOpts()) -> str:
+        ret = super().gen(opts)
+
+        if self.build_specs().setuptools.get('cexts'):
+            _PyprojectCextPackageGenerator(
+                self._dir_name,
+                self._pkgs_root,
+                pkg_suffix='-cext',
+            ).gen(opts)
+
+        return ret
+
+
+#
+
+
+class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
+
+    #
+
+    @cached_nullary
+    def find_cext_srcs(self) -> ta.Sequence[str]:
+        return sorted(find_magic(
+            [self._dir_name],
+            [CextMagic.MAGIC_COMMENT],
+            CextMagic.FILE_EXTENSIONS,
+        ))
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class FileContents:
+        pyproject_dct: ta.Mapping[str, ta.Any]
+        setup_py: str
+
+    @cached_nullary
+    def file_contents(self) -> FileContents:
+        specs = self.build_specs()
+
+        #
+
+        pyp_dct = {}
+
+        pyp_dct['build-system'] = {
+            'requires': ['setuptools'],
+            'build-backend': 'setuptools.build_meta',
+        }
+
+        prj = specs.pyproject
+        prj['dependencies'] = [f'{prj["name"]} == {prj["version"]}']
+        prj['name'] += self._pkg_suffix
+        prj.pop('optional_dependencies', None)
+
+        pyp_dct['project'] = prj
+
+        #
+
+        st = specs.setuptools
+        pyp_dct['tool.setuptools'] = st
+
+        st.pop('cexts', None)
+        st.pop('find_packages', None)
+        st.pop('manifest_in', None)
+
+        pyp_dct['tool.setuptools.packages.find'] = {
+            'include': [],
+        }
+
+        #
+
+        ext_lines = []
+
+        for ext_src in self.find_cext_srcs():
+            ext_name = ext_src.rpartition('.')[0].replace(os.sep, '.')
+            ext_lines.extend([
+                'st.Extension(',
+                f"    name='{ext_name}',",
+                f"    sources=['{ext_src}'],",
+                "    extra_compile_args=['-std=c++20'],",
+                '),',
+            ])
+
+        src = '\n'.join([
+            'import setuptools as st',
+            '',
+            '',
+            'st.setup(',
+            '    ext_modules=[',
+            *['        ' + l for l in ext_lines],
+            '    ]',
+            ')',
+            '',
+        ])
+
+        #
+
+        return self.FileContents(
+            pyp_dct,
+            src,
+        )
+
+    def _write_file_contents(self) -> None:
+        fc = self.file_contents()
+
+        with open(os.path.join(self._pkg_dir(), 'pyproject.toml'), 'w') as f:
+            TomlWriter(f).write_root(fc.pyproject_dct)
+
+        with open(os.path.join(self._pkg_dir(), 'setup.py'), 'w') as f:
+            f.write(fc.setup_py)

{omdev-0.0.0.dev17 → omdev-0.0.0.dev19}/omdev/scripts/interp.py

@@ -1206,7 +1206,7 @@ class JsonLogFormatter(logging.Formatter):
 STANDARD_LOG_FORMAT_PARTS = [
     ('asctime', '%(asctime)-15s'),
     ('process', 'pid=%(process)-6s'),
-    ('thread', 'tid=%(thread)-16s'),
+    ('thread', 'tid=%(thread)-10x'),
     ('levelname', '%(levelname)-8s'),
     ('name', '%(name)s'),
     ('separator', '::'),