omdev 0.0.0.dev180__py3-none-any.whl → 0.0.0.dev182__py3-none-any.whl

omdev/interp/pyenv/pyenv.py

@@ -10,31 +10,13 @@ TODO:
  - optionally install / upgrade pyenv itself
  - new vers dont need these custom mac opts, only run on old vers
 """
-import abc
-import dataclasses as dc
-import itertools
 import os.path
 import shutil
-import sys
 import typing as ta
 
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.cached import async_cached_nullary
-from omlish.lite.cached import cached_nullary
 from omlish.lite.check import check
-from omlish.lite.logs import log
-
-from ...packaging.versions import InvalidVersion
-from ...packaging.versions import Version
-from ..inspect import InterpInspector
-from ..providers.base import InterpProvider
-from ..types import Interp
-from ..types import InterpOpts
-from ..types import InterpSpecifier
-from ..types import InterpVersion
-
-
-##
 
 
 class Pyenv:
@@ -102,365 +84,3 @@ class Pyenv:
             return False
         await asyncio_subprocesses.check_call('git', 'pull', cwd=root)
         return True
-
-
-##
-
-
-@dc.dataclass(frozen=True)
-class PyenvInstallOpts:
-    opts: ta.Sequence[str] = ()
-    conf_opts: ta.Sequence[str] = ()
-    cflags: ta.Sequence[str] = ()
-    ldflags: ta.Sequence[str] = ()
-    env: ta.Mapping[str, str] = dc.field(default_factory=dict)
-
-    def merge(self, *others: 'PyenvInstallOpts') -> 'PyenvInstallOpts':
-        return PyenvInstallOpts(
-            opts=list(itertools.chain.from_iterable(o.opts for o in [self, *others])),
-            conf_opts=list(itertools.chain.from_iterable(o.conf_opts for o in [self, *others])),
-            cflags=list(itertools.chain.from_iterable(o.cflags for o in [self, *others])),
-            ldflags=list(itertools.chain.from_iterable(o.ldflags for o in [self, *others])),
-            env=dict(itertools.chain.from_iterable(o.env.items() for o in [self, *others])),
-        )
-
-
-# TODO: https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#building-for-maximum-performance
-DEFAULT_PYENV_INSTALL_OPTS = PyenvInstallOpts(
-    opts=[
-        '-s',
-        '-v',
-        '-k',
-    ],
-    conf_opts=[
-        # FIXME: breaks on mac for older py's
-        '--enable-loadable-sqlite-extensions',
-
-        # '--enable-shared',
-
-        '--enable-optimizations',
-        '--with-lto',
-
-        # '--enable-profiling', # ?
-
-        # '--enable-ipv6', # ?
-    ],
-    cflags=[
-        # '-march=native',
-        # '-mtune=native',
-    ],
-)
-
-DEBUG_PYENV_INSTALL_OPTS = PyenvInstallOpts(opts=['-g'])
-
-THREADED_PYENV_INSTALL_OPTS = PyenvInstallOpts(conf_opts=['--disable-gil'])
-
-
-#
-
-
-class PyenvInstallOptsProvider(abc.ABC):
-    @abc.abstractmethod
-    def opts(self) -> ta.Awaitable[PyenvInstallOpts]:
-        raise NotImplementedError
-
-
-class LinuxPyenvInstallOpts(PyenvInstallOptsProvider):
-    async def opts(self) -> PyenvInstallOpts:
-        return PyenvInstallOpts()
-
-
-class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
-    @cached_nullary
-    def framework_opts(self) -> PyenvInstallOpts:
-        return PyenvInstallOpts(conf_opts=['--enable-framework'])
-
-    @cached_nullary
-    def has_brew(self) -> bool:
-        return shutil.which('brew') is not None
-
-    BREW_DEPS: ta.Sequence[str] = [
-        'openssl',
-        'readline',
-        'sqlite3',
-        'zlib',
-    ]
-
-    @async_cached_nullary
-    async def brew_deps_opts(self) -> PyenvInstallOpts:
-        cflags = []
-        ldflags = []
-        for dep in self.BREW_DEPS:
-            dep_prefix = await asyncio_subprocesses.check_output_str('brew', '--prefix', dep)
-            cflags.append(f'-I{dep_prefix}/include')
-            ldflags.append(f'-L{dep_prefix}/lib')
-        return PyenvInstallOpts(
-            cflags=cflags,
-            ldflags=ldflags,
-        )
-
-    @async_cached_nullary
-    async def brew_tcl_opts(self) -> PyenvInstallOpts:
-        if await asyncio_subprocesses.try_output('brew', '--prefix', 'tcl-tk') is None:
-            return PyenvInstallOpts()
-
-        tcl_tk_prefix = await asyncio_subprocesses.check_output_str('brew', '--prefix', 'tcl-tk')
-        tcl_tk_ver_str = await asyncio_subprocesses.check_output_str('brew', 'ls', '--versions', 'tcl-tk')
-        tcl_tk_ver = '.'.join(tcl_tk_ver_str.split()[1].split('.')[:2])
-
-        return PyenvInstallOpts(conf_opts=[
-            f"--with-tcltk-includes='-I{tcl_tk_prefix}/include'",
-            f"--with-tcltk-libs='-L{tcl_tk_prefix}/lib -ltcl{tcl_tk_ver} -ltk{tcl_tk_ver}'",
-        ])
-
-    # @cached_nullary
-    # def brew_ssl_opts(self) -> PyenvInstallOpts:
-    #     pkg_config_path = subprocess_check_output_str('brew', '--prefix', 'openssl')
-    #     if 'PKG_CONFIG_PATH' in os.environ:
-    #         pkg_config_path += ':' + os.environ['PKG_CONFIG_PATH']
-    #     return PyenvInstallOpts(env={'PKG_CONFIG_PATH': pkg_config_path})
-
-    async def opts(self) -> PyenvInstallOpts:
-        return PyenvInstallOpts().merge(
-            self.framework_opts(),
-            await self.brew_deps_opts(),
-            await self.brew_tcl_opts(),
-            # self.brew_ssl_opts(),
-        )
-
-
-PLATFORM_PYENV_INSTALL_OPTS: ta.Dict[str, PyenvInstallOptsProvider] = {
-    'darwin': DarwinPyenvInstallOpts(),
-    'linux': LinuxPyenvInstallOpts(),
-}
-
-
-##
-
-
-class PyenvVersionInstaller:
-    """
-    Messy: can install freethreaded build with a 't' suffixed version str _or_ by THREADED_PYENV_INSTALL_OPTS - need
-    latter to build custom interp with ft, need former to use canned / blessed interps. Muh.
-    """
-
-    def __init__(
-            self,
-            version: str,
-            opts: ta.Optional[PyenvInstallOpts] = None,
-            interp_opts: InterpOpts = InterpOpts(),
-            *,
-            pyenv: Pyenv,
-
-            install_name: ta.Optional[str] = None,
-            no_default_opts: bool = False,
-    ) -> None:
-        super().__init__()
-
-        self._version = version
-        self._given_opts = opts
-        self._interp_opts = interp_opts
-        self._given_install_name = install_name
-
-        self._no_default_opts = no_default_opts
-        self._pyenv = pyenv
-
-    @property
-    def version(self) -> str:
-        return self._version
-
-    @async_cached_nullary
-    async def opts(self) -> PyenvInstallOpts:
-        opts = self._given_opts
-        if self._no_default_opts:
-            if opts is None:
-                opts = PyenvInstallOpts()
-        else:
-            lst = [self._given_opts if self._given_opts is not None else DEFAULT_PYENV_INSTALL_OPTS]
-            if self._interp_opts.debug:
-                lst.append(DEBUG_PYENV_INSTALL_OPTS)
-            if self._interp_opts.threaded:
-                lst.append(THREADED_PYENV_INSTALL_OPTS)
-            lst.append(await PLATFORM_PYENV_INSTALL_OPTS[sys.platform].opts())
-            opts = PyenvInstallOpts().merge(*lst)
-        return opts
-
-    @cached_nullary
-    def install_name(self) -> str:
-        if self._given_install_name is not None:
-            return self._given_install_name
-        return self._version + ('-debug' if self._interp_opts.debug else '')
-
-    @async_cached_nullary
-    async def install_dir(self) -> str:
-        return str(os.path.join(check.not_none(await self._pyenv.root()), 'versions', self.install_name()))
-
-    @async_cached_nullary
-    async def install(self) -> str:
-        opts = await self.opts()
-        env = {**os.environ, **opts.env}
-        for k, l in [
-            ('CFLAGS', opts.cflags),
-            ('LDFLAGS', opts.ldflags),
-            ('PYTHON_CONFIGURE_OPTS', opts.conf_opts),
-        ]:
-            v = ' '.join(l)
-            if k in os.environ:
-                v += ' ' + os.environ[k]
-            env[k] = v
-
-        conf_args = [
-            *opts.opts,
-            self._version,
-        ]
-
-        full_args: ta.List[str]
-        if self._given_install_name is not None:
-            full_args = [
-                os.path.join(check.not_none(await self._pyenv.root()), 'plugins', 'python-build', 'bin', 'python-build'),  # noqa
-                *conf_args,
-                await self.install_dir(),
-            ]
-        else:
-            full_args = [
-                await self._pyenv.exe(),
-                'install',
-                *conf_args,
-            ]
-
-        await asyncio_subprocesses.check_call(
-            *full_args,
-            env=env,
-        )
-
-        exe = os.path.join(await self.install_dir(), 'bin', 'python')
-        if not os.path.isfile(exe):
-            raise RuntimeError(f'Interpreter not found: {exe}')
-        return exe
-
-
-##
-
-
-class PyenvInterpProvider(InterpProvider):
-    @dc.dataclass(frozen=True)
-    class Options:
-        inspect: bool = False
-
-        try_update: bool = False
-
-    def __init__(
-            self,
-            options: Options = Options(),
-            *,
-            pyenv: Pyenv,
-            inspector: InterpInspector,
-    ) -> None:
-        super().__init__()
-
-        self._options = options
-
-        self._pyenv = pyenv
-        self._inspector = inspector
-
-    #
-
-    @staticmethod
-    def guess_version(s: str) -> ta.Optional[InterpVersion]:
-        def strip_sfx(s: str, sfx: str) -> ta.Tuple[str, bool]:
-            if s.endswith(sfx):
-                return s[:-len(sfx)], True
-            return s, False
-        ok = {}
-        s, ok['debug'] = strip_sfx(s, '-debug')
-        s, ok['threaded'] = strip_sfx(s, 't')
-        try:
-            v = Version(s)
-        except InvalidVersion:
-            return None
-        return InterpVersion(v, InterpOpts(**ok))
-
-    class Installed(ta.NamedTuple):
-        name: str
-        exe: str
-        version: InterpVersion
-
-    async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
-        iv: ta.Optional[InterpVersion]
-        if self._options.inspect:
-            try:
-                iv = check.not_none(await self._inspector.inspect(ep)).iv
-            except Exception as e:  # noqa
-                return None
-        else:
-            iv = self.guess_version(vn)
-        if iv is None:
-            return None
-        return PyenvInterpProvider.Installed(
-            name=vn,
-            exe=ep,
-            version=iv,
-        )
-
-    async def installed(self) -> ta.Sequence[Installed]:
-        ret: ta.List[PyenvInterpProvider.Installed] = []
-        for vn, ep in await self._pyenv.version_exes():
-            if (i := await self._make_installed(vn, ep)) is None:
-                log.debug('Invalid pyenv version: %s', vn)
-                continue
-            ret.append(i)
-        return ret
-
-    #
-
-    async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
-        return [i.version for i in await self.installed()]
-
-    async def get_installed_version(self, version: InterpVersion) -> Interp:
-        for i in await self.installed():
-            if i.version == version:
-                return Interp(
-                    exe=i.exe,
-                    version=i.version,
-                )
-        raise KeyError(version)
-
-    #
-
-    async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
-        lst = []
-
-        for vs in await self._pyenv.installable_versions():
-            if (iv := self.guess_version(vs)) is None:
-                continue
-            if iv.opts.debug:
-                raise Exception('Pyenv installable versions not expected to have debug suffix')
-            for d in [False, True]:
-                lst.append(dc.replace(iv, opts=dc.replace(iv.opts, debug=d)))
-
-        return lst
-
-    async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
-        lst = await self._get_installable_versions(spec)
-
-        if self._options.try_update and not any(v in spec for v in lst):
-            if self._pyenv.update():
-                lst = await self._get_installable_versions(spec)
-
-        return lst
-
-    async def install_version(self, version: InterpVersion) -> Interp:
-        inst_version = str(version.version)
-        inst_opts = version.opts
-        if inst_opts.threaded:
-            inst_version += 't'
-            inst_opts = dc.replace(inst_opts, threaded=False)
-
-        installer = PyenvVersionInstaller(
-            inst_version,
-            interp_opts=inst_opts,
-            pyenv=self._pyenv,
-        )
-
-        exe = await installer.install()
-        return Interp(exe, version)

omdev/interp/venvs.py

@@ -0,0 +1,114 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import logging
+import os.path
+import typing as ta
+
+from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
+from omlish.lite.cached import async_cached_nullary
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.typing import Func2
+
+from .default import get_default_interp_resolver
+from .types import InterpSpecifier
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class InterpVenvConfig:
+    interp: ta.Optional[str] = None
+    requires: ta.Optional[ta.Sequence[str]] = None
+    use_uv: ta.Optional[bool] = None
+
+
+class InterpVenvRequirementsProcessor(Func2['InterpVenv', ta.Sequence[str], ta.Sequence[str]]):
+    pass
+
+
+class InterpVenv:
+    def __init__(
+            self,
+            path: str,
+            cfg: InterpVenvConfig,
+            *,
+            requirements_processor: ta.Optional[InterpVenvRequirementsProcessor] = None,
+            log: ta.Optional[logging.Logger] = None,
+    ) -> None:
+        super().__init__()
+
+        self._path = path
+        self._cfg = cfg
+
+        self._requirements_processor = requirements_processor
+        self._log = log
+
+    @property
+    def path(self) -> str:
+        return self._path
+
+    @property
+    def cfg(self) -> InterpVenvConfig:
+        return self._cfg
+
+    @async_cached_nullary
+    async def interp_exe(self) -> str:
+        i = InterpSpecifier.parse(check.not_none(self._cfg.interp))
+        return check.not_none(await get_default_interp_resolver().resolve(i, install=True)).exe
+
+    @cached_nullary
+    def exe(self) -> str:
+        ve = os.path.join(self._path, 'bin/python')
+        if not os.path.isfile(ve):
+            raise Exception(f'venv exe {ve} does not exist or is not a file!')
+        return ve
+
+    @async_cached_nullary
+    async def create(self) -> bool:
+        if os.path.exists(dn := self._path):
+            if not os.path.isdir(dn):
+                raise Exception(f'{dn} exists but is not a directory!')
+            return False
+
+        ie = await self.interp_exe()
+
+        if self._log is not None:
+            self._log.info('Using interpreter %s', ie)
+
+        await asyncio_subprocesses.check_call(ie, '-m', 'venv', dn)
+
+        ve = self.exe()
+        uv = self._cfg.use_uv
+
+        await asyncio_subprocesses.check_call(
+            ve,
+            '-m', 'pip',
+            'install', '-v', '--upgrade',
+            'pip',
+            'setuptools',
+            'wheel',
+            *(['uv'] if uv else []),
+        )
+
+        if sr := self._cfg.requires:
+            reqs = list(sr)
+            if self._requirements_processor is not None:
+                reqs = list(self._requirements_processor(self, reqs))
+
+            # TODO: automatically try slower uv download when it fails? lol
+            #   Caused by: Failed to download distribution due to network timeout. Try increasing UV_HTTP_TIMEOUT (current value: 30s).  # noqa
+            #   UV_CONCURRENT_DOWNLOADS=4 UV_HTTP_TIMEOUT=3600
+
+            await asyncio_subprocesses.check_call(
+                ve,
+                '-m',
+                *(['uv'] if uv else []),
+                'pip',
+                'install',
+                *([] if uv else ['-v']),
+                *reqs,
+            )
+
+        return True

omdev/pyproject/configs.py

@@ -4,15 +4,14 @@ import typing as ta
 
 from omlish.lite.marshal import unmarshal_obj
 
+from ..interp.venvs import InterpVenvConfig
+
 
 @dc.dataclass(frozen=True)
-class VenvConfig:
+class VenvConfig(InterpVenvConfig):
     inherits: ta.Optional[ta.Sequence[str]] = None
-    interp: ta.Optional[str] = None
-    requires: ta.Optional[ta.List[str]] = None
     docker: ta.Optional[str] = None
     srcs: ta.Optional[ta.List[str]] = None
-    use_uv: ta.Optional[bool] = None
 
 
 @dc.dataclass(frozen=True)

omdev/pyproject/inject.py

@@ -0,0 +1,12 @@
+# ruff: noqa: UP006 UP007
+import typing as ta
+
+from omlish.lite.inject import InjectorBindingOrBindings
+from omlish.lite.inject import InjectorBindings
+from omlish.lite.inject import inj
+
+
+def bind_pyproject() -> InjectorBindings:
+    lst: ta.List[InjectorBindingOrBindings] = []
+
+    return inj.as_bindings(*lst)

omdev/pyproject/venvs.py

@@ -3,14 +3,12 @@ import glob
 import os.path
 import typing as ta
 
-from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
 from omlish.lite.cached import async_cached_nullary
 from omlish.lite.cached import cached_nullary
-from omlish.lite.check import check
 from omlish.lite.logs import log
 
-from ..interp.default import get_default_interp_resolver
-from ..interp.types import InterpSpecifier
+from ..interp.venvs import InterpVenv
+from ..interp.venvs import InterpVenvRequirementsProcessor
 from .configs import VenvConfig
 from .reqs import RequirementsRewriter
 
@@ -38,60 +36,26 @@ class Venv:
     def dir_name(self) -> str:
         return os.path.join(self.DIR_NAME, self._name)
 
-    @async_cached_nullary
-    async def interp_exe(self) -> str:
-        i = InterpSpecifier.parse(check.not_none(self._cfg.interp))
-        return check.not_none(await get_default_interp_resolver().resolve(i, install=True)).exe
+    @cached_nullary
+    def _iv(self) -> InterpVenv:
+        rr = RequirementsRewriter(self._name)
+
+        return InterpVenv(
+            self.dir_name,
+            self._cfg,
+            requirements_processor=InterpVenvRequirementsProcessor(
+                lambda iv, reqs: [rr.rewrite(req) for req in reqs]  # noqa
+            ),
+            log=log,
+        )
 
     @cached_nullary
     def exe(self) -> str:
-        ve = os.path.join(self.dir_name, 'bin/python')
-        if not os.path.isfile(ve):
-            raise Exception(f'venv exe {ve} does not exist or is not a file!')
-        return ve
+        return self._iv().exe()
 
     @async_cached_nullary
     async def create(self) -> bool:
-        if os.path.exists(dn := self.dir_name):
-            if not os.path.isdir(dn):
-                raise Exception(f'{dn} exists but is not a directory!')
-            return False
-
-        log.info('Using interpreter %s', (ie := await self.interp_exe()))
-        await asyncio_subprocesses.check_call(ie, '-m', 'venv', dn)
-
-        ve = self.exe()
-        uv = self._cfg.use_uv
-
-        await asyncio_subprocesses.check_call(
-            ve,
-            '-m', 'pip',
-            'install', '-v', '--upgrade',
-            'pip',
-            'setuptools',
-            'wheel',
-            *(['uv'] if uv else []),
-        )
-
-        if sr := self._cfg.requires:
-            rr = RequirementsRewriter(self._name)
-            reqs = [rr.rewrite(req) for req in sr]
-
-            # TODO: automatically try slower uv download when it fails? lol
-            #   Caused by: Failed to download distribution due to network timeout. Try increasing UV_HTTP_TIMEOUT (current value: 30s).  # noqa
-            #   UV_CONCURRENT_DOWNLOADS=4 UV_HTTP_TIMEOUT=3600
-
-            await asyncio_subprocesses.check_call(
-                ve,
-                '-m',
-                *(['uv'] if uv else []),
-                'pip',
-                'install',
-                *([] if uv else ['-v']),
-                *reqs,
-            )
-
-        return True
+        return await self._iv().create()
 
     @staticmethod
     def _resolve_srcs(raw: ta.List[str]) -> ta.List[str]: