omdev 0.0.0.dev151__tar.gz → 0.0.0.dev153__tar.gz

{omdev-0.0.0.dev151/omdev.egg-info → omdev-0.0.0.dev153}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev151
+Version: 0.0.0.dev153
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev151
+Requires-Dist: omlish==0.0.0.dev153
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev151 → omdev-0.0.0.dev153}/omdev/cache/data/cache.py

@@ -24,8 +24,8 @@ import urllib.request
 from omlish import check
 from omlish import lang
 from omlish import marshal as msh
-from omlish import os as osu
 from omlish.formats import json
+from omlish.os.files import touch
 
 from ... import git
 from .actions import Action
@@ -262,7 +262,7 @@ class Cache:
         if not os.path.isdir(item_dir):
             self._fetch_item(spec, item_dir)
 
-        osu.touch(os.path.join(item_dir, 'accessed'))
+        touch(os.path.join(item_dir, 'accessed'))
 
         data_dir = os.path.join(item_dir, 'data')
         return self._return_val(spec, data_dir)

{omdev-0.0.0.dev151 → omdev-0.0.0.dev153}/omdev/imgur.py

@@ -124,7 +124,7 @@ def _main() -> None:
 
             print(resp.data.link)
 
-    Cli().call_and_exit()
+    Cli()(exit=True)
 
 
 # @omlish-manifest

{omdev-0.0.0.dev151 → omdev-0.0.0.dev153}/omdev/pycharm/cli.py

@@ -141,7 +141,7 @@ class Cli(ap.Cli):
 
 
 def _main() -> None:
-    Cli().call_and_exit()
+    Cli()(exit=True)
 
 
 if __name__ == '__main__':

omdev-0.0.0.dev153/omdev/pyproject/cli.py

@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+# @omlish-amalg ../scripts/pyproject.py
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - check / tests, src dir sets
+ - ci
+ - build / package / publish / version roll
+  - {pkg_name: [src_dirs]}, default excludes, generate MANIFST.in, ...
+ - env vars - PYTHONPATH
+
+See:
+ - https://pdm-project.org/en/latest/
+ - https://rye.astral.sh/philosophy/
+ - https://github.com/indygreg/python-build-standalone/blob/main/pythonbuild/cpython.py
+ - https://astral.sh/blog/uv
+ - https://github.com/jazzband/pip-tools
+ - https://github.com/Osiris-Team/1JPM
+ - https://github.com/brettcannon/microvenv
+ - https://github.com/pypa/pipx
+ - https://github.com/tox-dev/tox/
+"""
+import argparse
+import asyncio
+import concurrent.futures as cf
+import dataclasses as dc
+import functools
+import itertools
+import multiprocessing as mp
+import os.path
+import shlex
+import shutil
+import sys
+import typing as ta
+
+from omlish.argparse.cli import ArgparseCli
+from omlish.argparse.cli import argparse_arg
+from omlish.argparse.cli import argparse_command
+from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_call
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.logs import configure_standard_logging
+from omlish.lite.runtime import check_runtime_version
+
+from ..toml.parser import toml_loads
+from .configs import PyprojectConfig
+from .configs import PyprojectConfigPreparer
+from .pkg import BasePyprojectPackageGenerator
+from .pkg import PyprojectPackageGenerator
+from .venvs import Venv
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class VersionsFile:
+    name: ta.Optional[str] = '.versions'
+
+    @staticmethod
+    def parse(s: str) -> ta.Mapping[str, str]:
+        return {
+            k: v
+            for l in s.splitlines()
+            if (sl := l.split('#')[0].strip())
+            for k, _, v in (sl.partition('='),)
+        }
+
+    @cached_nullary
+    def contents(self) -> ta.Mapping[str, str]:
+        if not self.name or not os.path.exists(self.name):
+            return {}
+        with open(self.name) as f:
+            s = f.read()
+        return self.parse(s)
+
+    @staticmethod
+    def get_pythons(d: ta.Mapping[str, str]) -> ta.Mapping[str, str]:
+        pfx = 'PYTHON_'
+        return {k[len(pfx):].lower(): v for k, v in d.items() if k.startswith(pfx)}
+
+    @cached_nullary
+    def pythons(self) -> ta.Mapping[str, str]:
+        return self.get_pythons(self.contents())
+
+
+##
+
+
+@cached_nullary
+def _script_rel_path() -> str:
+    cwd = os.getcwd()
+    if not (f := __file__).startswith(cwd):
+        raise OSError(f'file {f} not in {cwd}')
+    return f[len(cwd):].lstrip(os.sep)
+
+
+##
+
+
+class Run:
+    def __init__(
+            self,
+            *,
+            raw_cfg: ta.Union[ta.Mapping[str, ta.Any], str, None] = None,
+    ) -> None:
+        super().__init__()
+
+        self._raw_cfg = raw_cfg
+
+    @cached_nullary
+    def raw_cfg(self) -> ta.Mapping[str, ta.Any]:
+        if self._raw_cfg is None:
+            with open('pyproject.toml') as f:
+                buf = f.read()
+        elif isinstance(self._raw_cfg, str):
+            buf = self._raw_cfg
+        else:
+            return self._raw_cfg
+        return toml_loads(buf)
+
+    @cached_nullary
+    def cfg(self) -> PyprojectConfig:
+        dct = self.raw_cfg()['tool']['omlish']['pyproject']
+        return PyprojectConfigPreparer(
+            python_versions=VersionsFile().pythons(),
+        ).prepare_config(dct)
+
+    @cached_nullary
+    def venvs(self) -> ta.Mapping[str, Venv]:
+        return {
+            n: Venv(n, c)
+            for n, c in self.cfg().venvs.items()
+            if not n.startswith('_')
+        }
+
+
+##
+
+
+class PyprojectCli(ArgparseCli):
+    _docker_container = argparse_arg('--_docker_container', help=argparse.SUPPRESS)
+
+    @argparse_command(
+        argparse_arg('name'),
+        argparse_arg('-e', '--docker-env', action='append'),
+        argparse_arg('cmd', nargs='?'),
+        argparse_arg('args', nargs=argparse.REMAINDER),
+    )
+    async def venv(self) -> None:
+        venv = Run().venvs()[self.args.name]
+        if (sd := venv.cfg.docker) is not None and sd != (cd := self.args._docker_container):  # noqa
+            script = ' '.join([
+                'python3',
+                shlex.quote(_script_rel_path()),
+                f'--_docker_container={shlex.quote(sd)}',
+                *map(shlex.quote, sys.argv[1:]),
+            ])
+
+            docker_env = {
+                'DOCKER_HOST_PLATFORM': os.environ.get('DOCKER_HOST_PLATFORM', sys.platform),
+            }
+            for e in self.args.docker_env or []:
+                if '=' in e:
+                    k, _, v = e.split('=')
+                    docker_env[k] = v
+                else:
+                    docker_env[e] = os.environ.get(e, '')
+
+            await asyncio_subprocess_check_call(
+                'docker',
+                'compose',
+                '-f', 'docker/compose.yml',
+                'exec',
+                *itertools.chain.from_iterable(
+                    ('-e', f'{k}={v}')
+                    for k, v in docker_env.items()
+                ),
+                '-it', sd,
+                'bash', '--login', '-c', script,
+            )
+
+            return
+
+        cmd = self.args.cmd
+        if not cmd:
+            await venv.create()
+
+        elif cmd == 'python':
+            await venv.create()
+            os.execl(
+                (exe := venv.exe()),
+                exe,
+                *self.args.args,
+            )
+
+        elif cmd == 'exe':
+            await venv.create()
+            check.arg(not self.args.args)
+            print(venv.exe())
+
+        elif cmd == 'run':
+            await venv.create()
+            sh = check.not_none(shutil.which('bash'))
+            script = ' '.join(self.args.args)
+            if not script:
+                script = sh
+            os.execl(
+                (bash := check.not_none(sh)),
+                bash,
+                '-c',
+                f'. {venv.dir_name}/bin/activate && ' + script,
+            )
+
+        elif cmd == 'srcs':
+            check.arg(not self.args.args)
+            print('\n'.join(venv.srcs()))
+
+        elif cmd == 'test':
+            await venv.create()
+            await asyncio_subprocess_check_call(venv.exe(), '-m', 'pytest', *(self.args.args or []), *venv.srcs())
+
+        else:
+            raise Exception(f'unknown subcommand: {cmd}')
+
+    @argparse_command(
+        argparse_arg('-b', '--build', action='store_true'),
+        argparse_arg('-r', '--revision', action='store_true'),
+        argparse_arg('-j', '--jobs', type=int),
+        argparse_arg('cmd', nargs='?'),
+        argparse_arg('args', nargs=argparse.REMAINDER),
+    )
+    async def pkg(self) -> None:
+        run = Run()
+
+        cmd = self.args.cmd
+        if not cmd:
+            raise Exception('must specify command')
+
+        elif cmd == 'gen':
+            pkgs_root = os.path.join('.pkg')
+
+            if os.path.exists(pkgs_root):
+                shutil.rmtree(pkgs_root)
+
+            build_output_dir = 'dist'
+            run_build = bool(self.args.build)
+            add_revision = bool(self.args.revision)
+
+            if run_build:
+                os.makedirs(build_output_dir, exist_ok=True)
+
+            pgs: ta.List[BasePyprojectPackageGenerator] = [
+                PyprojectPackageGenerator(
+                    dir_name,
+                    pkgs_root,
+                )
+                for dir_name in run.cfg().pkgs
+            ]
+            pgs = list(itertools.chain.from_iterable([pg, *pg.children()] for pg in pgs))
+
+            num_threads = self.args.jobs or int(max(mp.cpu_count() // 1.5, 1))
+            futs: ta.List[cf.Future]
+            with cf.ThreadPoolExecutor(num_threads) as ex:
+                futs = [ex.submit(pg.gen) for pg in pgs]
+                for fut in futs:
+                    fut.result()
+
+                if run_build:
+                    futs = [
+                        ex.submit(functools.partial(
+                            pg.build,
+                            build_output_dir,
+                            BasePyprojectPackageGenerator.BuildOpts(
+                                add_revision=add_revision,
+                            ),
+                        ))
+                        for pg in pgs
+                    ]
+                    for fut in futs:
+                        fut.result()
+
+        else:
+            raise Exception(f'unknown subcommand: {cmd}')
+
+
+##
+
+
+async def _async_main(argv: ta.Optional[ta.Sequence[str]] = None) -> None:
+    check_runtime_version()
+    configure_standard_logging()
+
+    await PyprojectCli(argv).async_cli_run()
+
+
+def _main(argv: ta.Optional[ta.Sequence[str]] = None) -> None:
+    asyncio.run(_async_main(argv))
+
+
+if __name__ == '__main__':
+    _main()

omdev-0.0.0.dev153/omdev/pyproject/venvs.py

@@ -0,0 +1,114 @@
+# ruff: noqa: UP006 UP007
+import glob
+import os.path
+import typing as ta
+
+from omlish.lite.asyncio.subprocesses import asyncio_subprocess_check_call
+from omlish.lite.cached import async_cached_nullary
+from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check
+from omlish.lite.logs import log
+
+from ..interp.resolvers import DEFAULT_INTERP_RESOLVER
+from ..interp.types import InterpSpecifier
+from .configs import VenvConfig
+from .reqs import RequirementsRewriter
+
+
+##
+
+
+class Venv:
+    def __init__(
+            self,
+            name: str,
+            cfg: VenvConfig,
+    ) -> None:
+        super().__init__()
+        self._name = name
+        self._cfg = cfg
+
+    @property
+    def cfg(self) -> VenvConfig:
+        return self._cfg
+
+    DIR_NAME = '.venvs'
+
+    @property
+    def dir_name(self) -> str:
+        return os.path.join(self.DIR_NAME, self._name)
+
+    @async_cached_nullary
+    async def interp_exe(self) -> str:
+        i = InterpSpecifier.parse(check.not_none(self._cfg.interp))
+        return check.not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=True)).exe
+
+    @cached_nullary
+    def exe(self) -> str:
+        ve = os.path.join(self.dir_name, 'bin/python')
+        if not os.path.isfile(ve):
+            raise Exception(f'venv exe {ve} does not exist or is not a file!')
+        return ve
+
+    @async_cached_nullary
+    async def create(self) -> bool:
+        if os.path.exists(dn := self.dir_name):
+            if not os.path.isdir(dn):
+                raise Exception(f'{dn} exists but is not a directory!')
+            return False
+
+        log.info('Using interpreter %s', (ie := await self.interp_exe()))
+        await asyncio_subprocess_check_call(ie, '-m', 'venv', dn)
+
+        ve = self.exe()
+        uv = self._cfg.use_uv
+
+        await asyncio_subprocess_check_call(
+            ve,
+            '-m', 'pip',
+            'install', '-v', '--upgrade',
+            'pip',
+            'setuptools',
+            'wheel',
+            *(['uv'] if uv else []),
+        )
+
+        if sr := self._cfg.requires:
+            rr = RequirementsRewriter(self._name)
+            reqs = [rr.rewrite(req) for req in sr]
+
+            # TODO: automatically try slower uv download when it fails? lol
+            #   Caused by: Failed to download distribution due to network timeout. Try increasing UV_HTTP_TIMEOUT (current value: 30s).  # noqa
+            #   UV_CONCURRENT_DOWNLOADS=4 UV_HTTP_TIMEOUT=3600
+
+            await asyncio_subprocess_check_call(
+                ve,
+                '-m',
+                *(['uv'] if uv else []),
+                'pip',
+                'install',
+                *([] if uv else ['-v']),
+                *reqs,
+            )
+
+        return True
+
+    @staticmethod
+    def _resolve_srcs(raw: ta.List[str]) -> ta.List[str]:
+        out: list[str] = []
+        seen: ta.Set[str] = set()
+        for r in raw:
+            es: list[str]
+            if any(c in r for c in '*?'):
+                es = list(glob.glob(r, recursive=True))
+            else:
+                es = [r]
+            for e in es:
+                if e not in seen:
+                    seen.add(e)
+                    out.append(e)
+        return out
+
+    @cached_nullary
+    def srcs(self) -> ta.Sequence[str]:
+        return self._resolve_srcs(self._cfg.srcs or [])

{omdev-0.0.0.dev151 → omdev-0.0.0.dev153}/omdev/scripts/interp.py

@@ -55,7 +55,7 @@ VersionCmpLocalType = ta.Union['NegativeInfinityVersionType', _VersionCmpLocalTy
 VersionCmpKey = ta.Tuple[int, ta.Tuple[int, ...], VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpPrePostDevType, VersionCmpLocalType]  # noqa
 VersionComparisonMethod = ta.Callable[[VersionCmpKey, VersionCmpKey], bool]
 
-# ../../omlish/lite/asyncio/asyncio.py
+# ../../omlish/asyncs/asyncio/timeouts.py
 AwaitableT = ta.TypeVar('AwaitableT', bound=ta.Awaitable)
 
 # ../../omlish/lite/cached.py
@@ -76,7 +76,7 @@ UnparsedVersionVar = ta.TypeVar('UnparsedVersionVar', bound=UnparsedVersion)
 CallableVersionOperator = ta.Callable[['Version', str], bool]
 
 # ../../omlish/lite/subprocesses.py
-SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']
+SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']  # ta.TypeAlias
 
 
 ########################################
@@ -487,54 +487,7 @@ def canonicalize_version(
 
 
 ########################################
-# ../../../omlish/lite/asyncio/asyncio.py
-
-
-##
-
-
-ASYNCIO_DEFAULT_BUFFER_LIMIT = 2 ** 16
-
-
-async def asyncio_open_stream_reader(
-        f: ta.IO,
-        loop: ta.Any = None,
-        *,
-        limit: int = ASYNCIO_DEFAULT_BUFFER_LIMIT,
-) -> asyncio.StreamReader:
-    if loop is None:
-        loop = asyncio.get_running_loop()
-
-    reader = asyncio.StreamReader(limit=limit, loop=loop)
-    await loop.connect_read_pipe(
-        lambda: asyncio.StreamReaderProtocol(reader, loop=loop),
-        f,
-    )
-
-    return reader
-
-
-async def asyncio_open_stream_writer(
-        f: ta.IO,
-        loop: ta.Any = None,
-) -> asyncio.StreamWriter:
-    if loop is None:
-        loop = asyncio.get_running_loop()
-
-    writer_transport, writer_protocol = await loop.connect_write_pipe(
-        lambda: asyncio.streams.FlowControlMixin(loop=loop),
-        f,
-    )
-
-    return asyncio.streams.StreamWriter(
-        writer_transport,
-        writer_protocol,
-        None,
-        loop,
-    )
-
-
-##
+# ../../../omlish/asyncs/asyncio/timeouts.py
 
 
 def asyncio_maybe_timeout(
@@ -2455,22 +2408,25 @@ async def asyncio_subprocess_communicate(
     return await AsyncioProcessCommunicator(proc).communicate(input, timeout)  # noqa
 
 
-##
-
-
-async def _asyncio_subprocess_check_run(
+async def asyncio_subprocess_run(
         *args: str,
         input: ta.Any = None,  # noqa
         timeout: ta.Optional[float] = None,
+        check: bool = False,  # noqa
+        capture_output: ta.Optional[bool] = None,
         **kwargs: ta.Any,
 ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
+    if capture_output:
+        kwargs.setdefault('stdout', subprocess.PIPE)
+        kwargs.setdefault('stderr', subprocess.PIPE)
+
     args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
 
     proc: asyncio.subprocess.Process
     async with asyncio_subprocess_popen(*args, **kwargs) as proc:
         stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
 
-    if proc.returncode:
+    if check and proc.returncode:
         raise subprocess.CalledProcessError(
             proc.returncode,
             args,
@@ -2481,6 +2437,9 @@ async def _asyncio_subprocess_check_run(
     return stdout, stderr
 
 
+##
+
+
 async def asyncio_subprocess_check_call(
         *args: str,
         stdout: ta.Any = sys.stderr,
@@ -2488,11 +2447,12 @@ async def asyncio_subprocess_check_call(
         timeout: ta.Optional[float] = None,
         **kwargs: ta.Any,
 ) -> None:
-    _, _ = await _asyncio_subprocess_check_run(
+    _, _ = await asyncio_subprocess_run(
         *args,
         stdout=stdout,
         input=input,
         timeout=timeout,
+        check=True,
         **kwargs,
     )
 
@@ -2503,11 +2463,12 @@ async def asyncio_subprocess_check_output(
         timeout: ta.Optional[float] = None,
         **kwargs: ta.Any,
 ) -> bytes:
-    stdout, stderr = await _asyncio_subprocess_check_run(
+    stdout, stderr = await asyncio_subprocess_run(
         *args,
         stdout=asyncio.subprocess.PIPE,
         input=input,
         timeout=timeout,
+        check=True,
         **kwargs,
     )