omdev 0.0.0.dev416__py3-none-any.whl → 0.0.0.dev500__py3-none-any.whl

omdev/pyproject/pkg.py

@@ -28,16 +28,19 @@ vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir
 import abc
 import dataclasses as dc
 import importlib
+import inspect
 import os.path
 import shutil
 import sys
 import tempfile
+import textwrap
 import types
 import typing as ta
 
 from omlish.formats.toml.writer import TomlWriter
+from omlish.lite.abstract import Abstract
 from omlish.lite.cached import cached_nullary
-from omlish.lite.logs import log
+from omlish.logs.modules import get_module_logger
 from omlish.subprocesses.sync import subprocesses
 
 from ..cexts.magic import CextMagic
@@ -45,10 +48,13 @@ from ..magic.find import find_magic_files
 from ..packaging.revisions import GitRevisionAdder
 
 
+log = get_module_logger(globals())  # noqa
+
+
 ##
 
 
-class BasePyprojectPackageGenerator(abc.ABC):
+class BasePyprojectPackageGenerator(Abstract):
     def __init__(
             self,
             dir_name: str,
@@ -87,7 +93,7 @@ class BasePyprojectPackageGenerator(abc.ABC):
 
     def _write_git_ignore(self) -> None:
         with open(os.path.join(self._pkg_dir(), '.gitignore'), 'w') as f:
-            f.write('\n'.join(self._GIT_IGNORE))
+            f.write('\n'.join([*self._GIT_IGNORE, '']))
 
     #
 
@@ -256,6 +262,7 @@ class BasePyprojectPackageGenerator(abc.ABC):
                 )
 
         if output_dir is not None:
+            log.info(lambda: f'Copying {dist_dir} to {output_dir}')
             for fn in os.listdir(dist_dir):
                 shutil.copyfile(os.path.join(dist_dir, fn), os.path.join(output_dir, fn))
 
@@ -311,7 +318,11 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
         st = dict(specs.setuptools)
         pyp_dct['tool.setuptools'] = st
 
-        st.pop('cexts', None)
+        for k in [
+            'cext',
+            'rs',
+        ]:
+            st.pop(k, None)
 
         #
 
@@ -333,7 +344,7 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
         #         '*.c',
         #         '*.cc',
         #         '*.h',
-        #         '.manifests.json',
+        #         '.omlish-manifests.json',
         #         'LICENSE',
         #     ],
         # }
@@ -384,13 +395,20 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
     def children(self) -> ta.Sequence[BasePyprojectPackageGenerator]:
         out: ta.List[BasePyprojectPackageGenerator] = []
 
-        if self.build_specs().setuptools.get('cexts'):
+        if self.build_specs().setuptools.get('cext'):
             out.append(_PyprojectCextPackageGenerator(
                 self._dir_name,
                 self._pkgs_root,
                 pkg_suffix='-cext',
             ))
 
+        if self.build_specs().setuptools.get('rs'):
+            out.append(_PyprojectRsPackageGenerator(
+                self._dir_name,
+                self._pkgs_root,
+                pkg_suffix='-rs',
+            ))
+
         if self.build_specs().pyproject.get('cli_scripts'):
             out.append(_PyprojectCliPackageGenerator(
                 self._dir_name,
@@ -401,10 +419,67 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
         return out
 
 
-#
+##
+
+
+class _PyprojectExtensionPackageGenerator(BasePyprojectPackageGenerator, Abstract):
+    #
+
+    def _build_project_dict(self) -> ta.Dict[str, ta.Any]:
+        prj = dict(self.build_specs().pyproject)
+
+        prj['dependencies'] = [f'{prj["name"]} == {prj["version"]}']
+        prj['name'] += self._pkg_suffix
+
+        for k in [
+            'optional_dependencies',
+            'entry_points',
+            'scripts',
+            'cli_scripts',
+        ]:
+            prj.pop(k, None)
+
+        return prj
+
+    def _build_setuptools_dict(self) -> ta.Dict[str, ta.Any]:
+        st = dict(self.build_specs().setuptools)
+
+        for k in [
+            'cext',
+            'rs',
+
+            'find_packages',
+            'package_data',
+            'manifest_in',
+        ]:
+            st.pop(k, None)
+
+        return st
 
+    #
+
+    @dc.dataclass(frozen=True)
+    class FileContents:
+        pyproject_dct: ta.Mapping[str, ta.Any]
+        setup_py: str
+
+    @abc.abstractmethod
+    def file_contents(self) -> FileContents:
+        raise NotImplementedError
 
-class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
+    #
+
+    def _write_file_contents(self) -> None:
+        fc = self.file_contents()
+
+        with open(os.path.join(self._pkg_dir(), 'pyproject.toml'), 'w') as f:
+            TomlWriter(f).write_root(fc.pyproject_dct)
+
+        with open(os.path.join(self._pkg_dir(), 'setup.py'), 'w') as f:
+            f.write(fc.setup_py)
+
+
+class _PyprojectCextPackageGenerator(_PyprojectExtensionPackageGenerator):
     @cached_nullary
     def find_cext_srcs(self) -> ta.Sequence[str]:
         return sorted(find_magic_files(
@@ -415,14 +490,10 @@ class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
 
     #
 
-    @dc.dataclass(frozen=True)
-    class FileContents:
-        pyproject_dct: ta.Mapping[str, ta.Any]
-        setup_py: str
-
     @cached_nullary
-    def file_contents(self) -> FileContents:
-        specs = self.build_specs()
+    def file_contents(self) -> _PyprojectExtensionPackageGenerator.FileContents:
+        prj = self._build_project_dict()
+        st = self._build_setuptools_dict()
 
         #
 
@@ -433,33 +504,9 @@ class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
             'build-backend': 'setuptools.build_meta',
         }
 
-        prj = specs.pyproject
-        prj['dependencies'] = [f'{prj["name"]} == {prj["version"]}']
-        prj['name'] += self._pkg_suffix
-        for k in [
-            'optional_dependencies',
-            'entry_points',
-            'scripts',
-            'cli_scripts',
-        ]:
-            prj.pop(k, None)
-
         pyp_dct['project'] = prj
-
-        #
-
-        st = dict(specs.setuptools)
         pyp_dct['tool.setuptools'] = st
 
-        for k in [
-            'cexts',
-
-            'find_packages',
-            'package_data',
-            'manifest_in',
-        ]:
-            st.pop(k, None)
-
         pyp_dct['tool.setuptools.packages.find'] = {
             'include': [],
         }
@@ -469,12 +516,17 @@ class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
         ext_lines = []
 
         for ext_src in self.find_cext_srcs():
+            ext_lang = ext_src.rpartition('.')[2]
+            compile_args = {
+                'c': ['-std=c11'],
+                'cc': ['-std=c++20'],
+            }.get(ext_lang, [])
             ext_name = ext_src.rpartition('.')[0].replace(os.sep, '.')
             ext_lines.extend([
                 'st.Extension(',
                 f"    name='{ext_name}',",
                 f"    sources=['{ext_src}'],",
-                "    extra_compile_args=['-std=c++20'],",
+                f'    extra_compile_args={compile_args!r},',
                 '),',
             ])
 
@@ -485,7 +537,7 @@ class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
             'st.setup(',
             '    ext_modules=[',
             *['        ' + l for l in ext_lines],
-            '    ]',
+            '    ],',
             ')',
             '',
         ])
@@ -497,14 +549,110 @@ class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
             src,
         )
 
-    def _write_file_contents(self) -> None:
-        fc = self.file_contents()
 
-        with open(os.path.join(self._pkg_dir(), 'pyproject.toml'), 'w') as f:
-            TomlWriter(f).write_root(fc.pyproject_dct)
+class _PyprojectRsPackageGenerator(_PyprojectExtensionPackageGenerator):
+    @cached_nullary
+    def find_rs_dirs(self) -> ta.Sequence[str]:
+        return sorted(
+            dp
+            for dp, dns, fns in os.walk(self._dir_name)
+            for fn in fns
+            if fn == '.omlish-rs-ext'
+        )
 
-        with open(os.path.join(self._pkg_dir(), 'setup.py'), 'w') as f:
-            f.write(fc.setup_py)
+    #
+
+    @staticmethod
+    def _sdist_patch_body() -> None:
+        def _patch_sdist():
+            def _sdist_add_defaults(old, self):
+                import os.path
+
+                old(self)
+
+                if self.distribution.rust_extensions and len(self.distribution.rust_extensions) > 0:
+                    build_rust = self.get_finalized_command('build_rust')  # noqa
+                    for ext in build_rust.extensions:
+                        ext_dir = os.path.dirname(ext.path)
+                        for n in os.listdir(ext_dir):
+                            if n.startswith('.') or n == 'target':
+                                continue
+                            p = os.path.join(ext_dir, n)
+                            if os.path.isfile(p):
+                                self.filelist.append(p)
+                            elif os.path.isdir(p):
+                                self.filelist.extend(os.path.join(dp, f) for dp, dn, fn in os.walk(p) for f in fn)
+
+            # Sadly, we can't just subclass sdist and override it via cmdclass because manifest_maker calls
+            # `sdist.add_defaults` as an unbound function, not a bound method:
+            # https://github.com/pypa/setuptools/blob/9c4d383631d3951fcae0afd73b5d08ff5a262976/setuptools/command/egg_info.py#L581
+            from setuptools.command.sdist import sdist  # noqa
+            setattr(sdist, 'add_defaults', (lambda old: lambda sdist: _sdist_add_defaults(old, sdist))(sdist.add_defaults))  # noqa
+
+        _patch_sdist()
+
+    @cached_nullary
+    def sdist_patch_code(self) -> str:
+        return textwrap.dedent(''.join(inspect.getsource(self._sdist_patch_body).splitlines(keepends=True)[2:])).strip()
+
+    #
+
+    @cached_nullary
+    def file_contents(self) -> _PyprojectExtensionPackageGenerator.FileContents:
+        prj = self._build_project_dict()
+        st = self._build_setuptools_dict()
+
+        #
+
+        pyp_dct = {}
+
+        pyp_dct['build-system'] = {
+            'requires': ['setuptools', 'setuptools-rust'],
+            'build-backend': 'setuptools.build_meta',
+        }
+
+        pyp_dct['project'] = prj
+        pyp_dct['tool.setuptools'] = st
+
+        pyp_dct['tool.setuptools.packages.find'] = {
+            'include': [],
+        }
+
+        #
+
+        ext_lines: list = []
+
+        for ext_dir in self.find_rs_dirs():  # noqa
+            ext_name = ext_dir.replace(os.sep, '.')
+            ext_lines.extend([
+                'st_rs.RustExtension(',
+                f"    '{ext_name}',",
+                f"    path='{ext_dir}/Cargo.toml',",
+                '),',
+            ])
+
+        src = '\n'.join([
+            'import setuptools as st',
+            'import setuptools_rust as st_rs',
+            '',
+            '',
+            self.sdist_patch_code(),
+            '',
+            '',
+            'st.setup(',
+            '    rust_extensions=[',
+            *['        ' + l for l in ext_lines],
+            '    ],',
+            ')',
+            '',
+        ])
+
+        #
+
+        return self.FileContents(
+            pyp_dct,
+            src,
+        )
 
 
 ##
@@ -549,7 +697,8 @@ class _PyprojectCliPackageGenerator(BasePyprojectPackageGenerator):
         pyp_dct['tool.setuptools'] = st
 
         for k in [
-            'cexts',
+            'cext',
+            'rs',
 
             'find_packages',
             'package_data',

omdev/pyproject/reqs.py

@@ -4,11 +4,18 @@ TODO:
 """
 # ruff: noqa: UP007 UP045
 import os.path
+import re
 import tempfile
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
-from omlish.lite.logs import log
+from omlish.logs.modules import get_module_logger
+
+from ..packaging.requires import RequiresParserSyntaxError
+from ..packaging.requires import parse_requirement
+
+
+log = get_module_logger(globals())  # noqa
 
 
 ##
@@ -17,11 +24,14 @@ from omlish.lite.logs import log
 class RequirementsRewriter:
     def __init__(
             self,
+            *,
             venv: ta.Optional[str] = None,
+            only_pats: ta.Optional[ta.Sequence[re.Pattern]] = None,
     ) -> None:
         super().__init__()
 
         self._venv = venv
+        self._only_pats = only_pats
 
     @cached_nullary
     def _tmp_dir(self) -> str:
@@ -37,17 +47,33 @@ class RequirementsRewriter:
         out_lines = []
 
         for l in in_lines:
-            if self.VENV_MAGIC in l:
-                lp, _, rp = l.partition(self.VENV_MAGIC)
-                rp = rp.partition('#')[0]
+            if l.split('#')[0].strip():
                 omit = False
-                for v in rp.split():
-                    if v[0] == '!':
-                        if self._venv is not None and self._venv == v[1:]:
-                            omit = True
-                            break
+
+                if self.VENV_MAGIC in l:
+                    lp, _, rp = l.partition(self.VENV_MAGIC)
+                    rp = rp.partition('#')[0]
+                    vs = set(rp.split())
+                    nvs = {v[1:] for v in vs if v.startswith('!')}
+                    pvs = {v for v in vs if not v.startswith('!')}
+                    if (
+                            (nvs and self._venv in nvs) or
+                            (pvs and self._venv not in pvs)
+                    ):
+                        omit = True
+
+                if (
+                        not omit and
+                        (ops := self._only_pats) is not None and
+                        not l.strip().startswith('-')
+                ):
+                    try:
+                        pr = parse_requirement(l.split('#')[0].strip())
+                    except RequiresParserSyntaxError:
+                        pass
                     else:
-                        raise NotImplementedError
+                        if not any(op.fullmatch(pr.name) for op in ops):
+                            omit = True
 
                 if omit:
                     out_lines.append('# OMITTED:  ' + l)

omdev/pyproject/tools/aboutdeps.py

@@ -0,0 +1,60 @@
+"""
+TODO:
+ - pyproject api!
+"""
+import importlib.metadata
+import re
+import sys
+
+from omlish.formats.toml.parser import toml_loads
+
+from ...packaging.names import canonicalize_name
+from ...packaging.requires import parse_requirement
+from ...packaging.specifiers import Specifier
+
+
+##
+
+
+def _main() -> None:
+    dist_dct = {}
+    for dist in importlib.metadata.distributions(paths=sys.path):
+        dist_cn = canonicalize_name(dist.metadata['Name'], validate=True)
+        if dist_cn in dist_dct:
+            continue
+        dist_dct[dist_cn] = dist
+
+    #
+
+    with open('pyproject.toml') as f:
+        dct = toml_loads(f.read())
+
+    pkgs = dct['tool']['omlish']['pyproject']['pkgs']
+
+    for pkg in pkgs:
+        pkg_about = importlib.import_module('.'.join([pkg, '__about__']))
+        pkg_prj = pkg_about.Project
+
+        pkg_opt_deps = {d for ds in pkg_prj.optional_dependencies.values() for d in ds}
+        for opt_dep in sorted(pkg_opt_deps):
+            opt_req = parse_requirement(opt_dep)
+
+            opt_cn = canonicalize_name(opt_req.name, validate=True)
+
+            opt_spec = Specifier(opt_req.specifier)
+            if re.fullmatch(r'~=\s*\d+(\.\d+)*', str(opt_spec)):
+                opt_spec = Specifier(str(opt_spec) + '.0')
+
+            if opt_cn in pkgs:
+                continue
+
+            opt_dist = dist_dct[opt_cn]
+            opt_ver = opt_dist.version
+
+            # print((opt_dep, opt_spec, opt_ver))
+            if not opt_spec.contains(opt_ver):
+                print(f'{pkg} :: {opt_cn} : {opt_spec} ! {opt_ver}')
+
+
+if __name__ == '__main__':
+    _main()

omdev/pyproject/venvs.py

@@ -1,11 +1,12 @@
 # ruff: noqa: UP006 UP007 UP045
 import glob
 import os.path
+import re
 import typing as ta
 
 from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
-from omlish.lite.logs import log
+from omlish.logs.modules import get_module_logger
 
 from ..interp.venvs import InterpVenv
 from ..interp.venvs import InterpVenvRequirementsProcessor
@@ -13,6 +14,9 @@ from .configs import VenvConfig
 from .reqs import RequirementsRewriter
 
 
+log = get_module_logger(globals())  # noqa
+
+
 ##
 
 
@@ -39,7 +43,13 @@ class Venv:
 
     @cached_nullary
     def _iv(self) -> InterpVenv:
-        rr = RequirementsRewriter(self._name)
+        rr = RequirementsRewriter(
+            venv=self._name,
+            only_pats=(
+                [re.compile(p) for p in self._cfg.requires_pats]
+                if self._cfg.requires_pats is not None else None
+            ),
+        )
 
         return InterpVenv(
             self.dir_name,