omdev 0.0.0.dev25__py3-none-any.whl → 0.0.0.dev27__py3-none-any.whl

omdev/scripts/pyproject.py

@@ -3,7 +3,7 @@
 # @omlish-lite
 # @omlish-script
 # @omdev-amalg-output ../pyproject/cli.py
-# ruff: noqa: UP006 UP007
+# ruff: noqa: N802 TCH003 UP006 UP007 UP036
 """
 TODO:
  - check / tests, src dir sets
@@ -66,6 +66,17 @@ import weakref  # noqa
 import zipfile
 
 
+########################################
+
+
+if sys.version_info < (3, 8):
+    raise OSError(
+        f'Requires python (3, 8), got {sys.version_info} from {sys.executable}')  # noqa
+
+
+########################################
+
+
 # ../../toml/parser.py
 TomlParseFloat = ta.Callable[[str], ta.Any]
 TomlKey = ta.Tuple[str, ...]
@@ -101,7 +112,6 @@ class CextMagic:
 
 ########################################
 # ../../findmagic.py
-# @omlish-script
 
 
 def compile_magic_pat(m: str) -> re.Pattern:
@@ -160,6 +170,105 @@ def find_magic(
                 yield out
 
 
+########################################
+# ../../git.py
+
+
+def git_clone_subtree(
+        *,
+        base_dir: str,
+        repo_url: str,
+        repo_dir: str,
+        branch: ta.Optional[str] = None,
+        rev: ta.Optional[str] = None,
+        repo_subtrees: ta.Sequence[str],
+) -> None:
+    if not bool(branch) ^ bool(rev):
+        raise ValueError('must set branch or rev')
+
+    if isinstance(repo_subtrees, str):
+        raise TypeError(repo_subtrees)
+
+    git_opts = [
+        '-c', 'advice.detachedHead=false',
+    ]
+
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'clone',
+            '-n',
+            '--depth=1',
+            '--filter=tree:0',
+            *(['-b', branch] if branch else []),
+            '--single-branch',
+            repo_url,
+            repo_dir,
+        ],
+        cwd=base_dir,
+    )
+
+    rd = os.path.join(base_dir, repo_dir)
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'sparse-checkout',
+            'set',
+            '--no-cone',
+            *repo_subtrees,
+        ],
+        cwd=rd,
+    )
+
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'checkout',
+            *([rev] if rev else []),
+        ],
+        cwd=rd,
+    )
+
+
+def get_git_revision(
+        *,
+        cwd: ta.Optional[str] = None,
+) -> ta.Optional[str]:
+    subprocess.check_output(['git', '--version'])
+
+    if cwd is None:
+        cwd = os.getcwd()
+
+    if subprocess.run([  # noqa
+        'git',
+        'rev-parse',
+        '--is-inside-work-tree',
+    ], stderr=subprocess.PIPE).returncode:
+        return None
+
+    has_untracked = bool(subprocess.check_output([
+        'git',
+        'ls-files',
+        '.',
+        '--exclude-standard',
+        '--others',
+    ], cwd=cwd).decode().strip())
+
+    dirty_rev = subprocess.check_output([
+        'git',
+        'describe',
+        '--match=NeVeRmAtCh',
+        '--always',
+        '--abbrev=40',
+        '--dirty',
+    ], cwd=cwd).decode().strip()
+
+    return dirty_rev + ('-untracked' if has_untracked else '')
+
+
 ########################################
 # ../../toml/parser.py
 # SPDX-License-Identifier: MIT
@@ -203,7 +312,6 @@ def find_magic(
 # License Agreement.
 #
 # https://github.com/python/cpython/blob/f5009b69e0cd94b990270e04e65b9d4d2b365844/Lib/tomllib/_parser.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -1109,7 +1217,6 @@ class TomlWriter:
 # Apache License, Version 2.0, and the BSD License. See the LICENSE file in the root of this repository for complete
 # details.
 # https://github.com/pypa/packaging/blob/2c885fe91a54559e2382902dce28428ad2887be5/src/packaging/version.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -1495,7 +1602,6 @@ def canonicalize_version(
 
 ########################################
 # ../../wheelfile.py
-# ruff: noqa: UP006 UP007
 # https://github.com/pypa/wheel/blob/7bb46d7727e6e89fe56b3c78297b3af2672bbbe2/src/wheel/wheelfile.py
 # MIT License
 #
@@ -1755,7 +1861,6 @@ class cached_nullary:  # noqa
 
 ########################################
 # ../../../omlish/lite/check.py
-# ruff: noqa: UP006 UP007
 
 
 def check_isinstance(v: T, spec: ta.Union[ta.Type[T], tuple]) -> T:
@@ -1782,6 +1887,12 @@ def check_not(v: ta.Any) -> None:
     return v
 
 
+def check_non_empty_str(v: ta.Optional[str]) -> str:
+    if not v:
+        raise ValueError
+    return v
+
+
 ########################################
 # ../../../omlish/lite/json.py
 
@@ -1815,7 +1926,6 @@ json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON
 
 ########################################
 # ../../../omlish/lite/reflect.py
-# ruff: noqa: UP006
 
 
 _GENERIC_ALIAS_TYPES = (
@@ -1898,7 +2008,6 @@ def is_sunder(name: str) -> bool:
 TODO:
  - embed pip._internal.req.parse_requirements, add additional env stuff? breaks compat with raw pip
 """
-# ruff: noqa: UP007
 
 
 class RequirementsRewriter:
@@ -1993,7 +2102,6 @@ class RequirementsRewriter:
 # Apache License, Version 2.0, and the BSD License. See the LICENSE file in the root of this repository for complete
 # details.
 # https://github.com/pypa/packaging/blob/2c885fe91a54559e2382902dce28428ad2887be5/src/packaging/specifiers.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -2499,7 +2607,6 @@ TODO:
  - translate json keys
  - debug
 """
-# ruff: noqa: UP006 UP007 N802
 
 
 log = logging.getLogger(__name__)
@@ -2698,46 +2805,51 @@ def configure_standard_logging(
         *,
         json: bool = False,
         target: ta.Optional[logging.Logger] = None,
-        no_check: bool = False,
+        force: bool = False,
 ) -> ta.Optional[StandardLogHandler]:
-    if target is None:
-        target = logging.root
+    logging._acquireLock()  # type: ignore  # noqa
+    try:
+        if target is None:
+            target = logging.root
 
-    #
+        #
 
-    if not no_check:
-        if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
-            return None
+        if not force:
+            if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
+                return None
 
-    #
+        #
 
-    handler = logging.StreamHandler()
+        handler = logging.StreamHandler()
 
-    #
+        #
 
-    formatter: logging.Formatter
-    if json:
-        formatter = JsonLogFormatter()
-    else:
-        formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
-    handler.setFormatter(formatter)
+        formatter: logging.Formatter
+        if json:
+            formatter = JsonLogFormatter()
+        else:
+            formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
+        handler.setFormatter(formatter)
 
-    #
+        #
 
-    handler.addFilter(TidLogFilter())
+        handler.addFilter(TidLogFilter())
 
-    #
+        #
 
-    target.addHandler(handler)
+        target.addHandler(handler)
 
-    #
+        #
 
-    if level is not None:
-        target.setLevel(level)
+        if level is not None:
+            target.setLevel(level)
 
-    #
+        #
+
+        return StandardLogHandler(handler)
 
-    return StandardLogHandler(handler)
+    finally:
+        logging._releaseLock()  # type: ignore  # noqa
 
 
 ########################################
@@ -2747,7 +2859,6 @@ TODO:
  - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
  - nonstrict toggle
 """
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -3063,7 +3174,6 @@ def check_runtime_version() -> None:
 
 ########################################
 # ../../interp/types.py
-# ruff: noqa: UP006
 
 
 # See https://peps.python.org/pep-3149/
@@ -3155,7 +3265,6 @@ class Interp:
 
 ########################################
 # ../configs.py
-# ruff: noqa: UP006 UP007
 
 
 @dc.dataclass(frozen=True)
@@ -3259,31 +3368,6 @@ TODO:
  - omlish-lite, move to pyproject/
   - vendor-lite wheel.wheelfile
 """
-# ruff: noqa: TCH003 UP006 UP007
-
-
-##
-
-
-def get_git_revision() -> str:
-    has_untracked = bool(subprocess.check_output([
-        'git',
-        'ls-files',
-        '.',
-        '--exclude-standard',
-        '--others',
-    ]).decode().strip())
-
-    dirty_rev = subprocess.check_output([
-        'git',
-        'describe',
-        '--match=NeVeRmAtCh',
-        '--always',
-        '--abbrev=40',
-        '--dirty',
-    ]).decode().strip()
-
-    return dirty_rev + ('-untracked' if has_untracked else '')
 
 
 ##
@@ -3303,7 +3387,7 @@ class GitRevisionAdder:
     def revision(self) -> str:
         if self._given_revision is not None:
             return self._given_revision
-        return get_git_revision()
+        return check_non_empty_str(get_git_revision())
 
     REVISION_ATTR = '__revision__'
 
@@ -3399,7 +3483,6 @@ class GitRevisionAdder:
 
 ########################################
 # ../../../omlish/lite/subprocesses.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -3506,7 +3589,6 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
 
 ########################################
 # ../../interp/inspect.py
-# ruff: noqa: UP006 UP007
 
 
 @dc.dataclass(frozen=True)
@@ -3629,7 +3711,6 @@ https://pip.pypa.io/en/stable/cli/pip_install/#vcs-support
 vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir
 'git+https://github.com/wrmsr/omlish@master#subdirectory=.pip/omlish'
 """  # noqa
-# ruff: noqa: UP006 UP007
 
 
 #
@@ -3726,6 +3807,35 @@ class BasePyprojectPackageGenerator(abc.ABC):
 
     #
 
+    class _PkgData(ta.NamedTuple):
+        inc: ta.List[str]
+        exc: ta.List[str]
+
+    @cached_nullary
+    def _collect_pkg_data(self) -> _PkgData:
+        inc: ta.List[str] = []
+        exc: ta.List[str] = []
+
+        for p, ds, fs in os.walk(self._dir_name):  # noqa
+            for f in fs:
+                if f != '.pkgdata':
+                    continue
+                rp = os.path.relpath(p, self._dir_name)
+                log.info('Found pkgdata %s for pkg %s', rp, self._dir_name)
+                with open(os.path.join(p, f)) as fo:
+                    src = fo.read()
+                for l in src.splitlines():
+                    if not (l := l.strip()):
+                        continue
+                    if l.startswith('!'):
+                        exc.append(os.path.join(rp, l[1:]))
+                    else:
+                        inc.append(os.path.join(rp, l))
+
+        return self._PkgData(inc, exc)
+
+    #
+
     @abc.abstractmethod
     def _write_file_contents(self) -> None:
         raise NotImplementedError
@@ -3749,13 +3859,12 @@ class BasePyprojectPackageGenerator(abc.ABC):
             build_output_dir: ta.Optional[str] = None,
             *,
             add_revision: bool = False,
+            test: bool = False,
     ) -> None:
-        subprocess.check_call(
-            [
-                sys.executable,
-                '-m',
-                'build',
-            ],
+        subprocess_check_call(
+            sys.executable,
+            '-m',
+            'build',
             cwd=self._pkg_dir(),
         )
 
@@ -3764,6 +3873,25 @@ class BasePyprojectPackageGenerator(abc.ABC):
         if add_revision:
             GitRevisionAdder().add_to(dist_dir)
 
+        if test:
+            for fn in os.listdir(dist_dir):
+                tmp_dir = tempfile.mkdtemp()
+
+                subprocess_check_call(
+                    sys.executable,
+                    '-m', 'venv',
+                    'test-install',
+                    cwd=tmp_dir,
+                )
+
+                subprocess_check_call(
+                    os.path.join(tmp_dir, 'test-install', 'bin', 'python3'),
+                    '-m', 'pip',
+                    'install',
+                    os.path.abspath(os.path.join(dist_dir, fn)),
+                    cwd=tmp_dir,
+                )
+
         if build_output_dir is not None:
             for fn in os.listdir(dist_dir):
                 shutil.copyfile(os.path.join(dist_dir, fn), os.path.join(build_output_dir, fn))
@@ -3837,13 +3965,56 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
 
         #
 
-        st = specs.setuptools
+        st = dict(specs.setuptools)
         pyp_dct['tool.setuptools'] = st
 
         st.pop('cexts', None)
 
-        self._move_dict_key(st, 'find_packages', pyp_dct, 'tool.setuptools.packages.find')
-        self._move_dict_key(st, 'package_data', pyp_dct, 'tool.setuptools.package-data')
+        #
+
+        # TODO: default
+        # find_packages = {
+        #     'include': [Project.name, f'{Project.name}.*'],
+        #     'exclude': [*SetuptoolsBase.find_packages['exclude']],
+        # }
+
+        fp = dict(st.pop('find_packages', {}))
+
+        pyp_dct['tool.setuptools.packages.find'] = fp
+
+        #
+
+        # TODO: default
+        # package_data = {
+        #     '*': [
+        #         '*.c',
+        #         '*.cc',
+        #         '*.h',
+        #         '.manifests.json',
+        #         'LICENSE',
+        #     ],
+        # }
+
+        pd = dict(st.pop('package_data', {}))
+        epd = dict(st.pop('exclude_package_data', {}))
+
+        cpd = self._collect_pkg_data()
+        if cpd.inc:
+            pd['*'] = [*pd.get('*', []), *sorted(set(cpd.inc))]
+        if cpd.exc:
+            epd['*'] = [*epd.get('*', []), *sorted(set(cpd.exc))]
+
+        if pd:
+            pyp_dct['tool.setuptools.package-data'] = pd
+        if epd:
+            pyp_dct['tool.setuptools.exclude-package-data'] = epd
+
+        #
+
+        # TODO: default
+        # manifest_in = [
+        #     'global-exclude **/conftest.py',
+        # ]
 
         mani_in = st.pop('manifest_in', None)
 
@@ -3923,7 +4094,7 @@ class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
 
         #
 
-        st = specs.setuptools
+        st = dict(specs.setuptools)
         pyp_dct['tool.setuptools'] = st
 
         for k in [
@@ -4051,7 +4222,6 @@ TODO:
  - optionally install / upgrade pyenv itself
  - new vers dont need these custom mac opts, only run on old vers
 """
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -4446,7 +4616,6 @@ TODO:
  - python, python3, python3.12, ...
  - check if path py's are venvs: sys.prefix != sys.base_prefix
 """
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -4555,7 +4724,6 @@ class SystemInterpProvider(InterpProvider):
 
 ########################################
 # ../../interp/resolvers.py
-# ruff: noqa: UP006 UP007
 
 
 INTERP_PROVIDER_TYPES_BY_NAME: ta.Mapping[str, ta.Type[InterpProvider]] = {

omdev/tools/dockertools.py

@@ -1,11 +1,18 @@
+"""
+TODO:
+ - check for updates
+"""
 import os
+import re
 import shutil
 import subprocess
+import typing as ta
 
 from omlish import argparse as ap
 from omlish import check
 from omlish import lang
 from omlish import logs
+from omlish.formats import yaml
 
 
 @lang.cached_function
@@ -75,6 +82,101 @@ class Cli(ap.Cli):
             'sh', '-c', 'echo 0 > /proc/sys/kernel/yama/ptrace_scope',
         )
 
+    @ap.command(
+        ap.arg('-f', '--file'),
+        ap.arg('-w', '--write', action='store_true'),
+        ap.arg('-q', '--quiet', action='store_true'),
+        ap.arg('base', type=int),
+    )
+    def reset_compose_ports(self) -> None:
+        base_port = int(self.args.base)
+        if not base_port:
+            raise Exception('Invalid base port')
+
+        if self.args.file:
+            yml_file = self.args.file
+        else:
+            yml_file = os.path.join('docker', 'compose.yml')
+
+        with open(yml_file) as f:
+            yml_src = f.read()
+
+        #
+
+        port_pat = re.compile(r'(?P<l>\d+):(?P<r>\d+)')
+        port_line_pat = re.compile(r"(  )+- '(?P<l>\d+):(?P<r>\d+)'\s*")
+
+        class PortEntry(ta.NamedTuple):
+            l: int
+            s: str
+
+        dct: dict[str, list[PortEntry]] = {}
+
+        with lang.disposing(yaml.WrappedLoaders.base(yml_src)) as loader:
+            val = check.not_none(loader.get_single_data())  # type: ignore
+            root = check.isinstance(val.value, ta.Mapping)
+
+            services = check.isinstance(
+                check.single(
+                    v.value  # type: ignore
+                    for k, v in root.items()
+                    if k.value == 'services'  # type: ignore
+                ),
+                ta.Mapping,
+            )
+            for name_w, cfg_w in services.items():
+                name = check.isinstance(name_w.value, str)  # type: ignore
+                cfg = check.isinstance(cfg_w.value, ta.Mapping)  # type: ignore
+
+                ports = check.opt_single(v.value for k, v in cfg.items() if k.value == 'ports')  # type: ignore
+                if not ports:
+                    continue
+
+                lst: list[PortEntry] = []
+                for port_w in ports:
+                    port = check.isinstance(port_w.value, str)
+                    if not re.fullmatch(port_pat, port):
+                        raise Exception(f'Bad port: {port}')
+
+                    lst.append(PortEntry(
+                        l=port_w.node.start_mark.line,
+                        s=port,
+                    ))
+
+                dct[name] = lst
+
+        #
+
+        src_lines = yml_src.splitlines(keepends=True)
+        cur_port = base_port
+
+        for ps in dct.values():
+            for p in ps:
+                l = src_lines[p.l]
+                if not (m := port_line_pat.fullmatch(l)):
+                    raise Exception(f'Bad port line: {p} {l!r}')
+
+                p_l, p_r = map(int, p.s.split(':'))
+                l_l, l_r = map(int, [(gd := m.groupdict())['l'], gd['r']])
+                if p_l != l_l or p_r != l_r:
+                    raise Exception(f'Port mismatch: {p}')
+
+                new_l = l.partition("'")[0] + f"'{cur_port}:{l_r}'\n"
+                src_lines[p.l] = new_l
+
+                cur_port += 1
+
+        new_src = ''.join(src_lines)
+
+        #
+
+        if not self.args.quiet:
+            print(new_src)
+
+        if self.args.write:
+            with open(yml_file, 'w') as f:
+                f.write(new_src)
+
 
 if __name__ == '__main__':
     logs.configure_standard_logging('INFO')

{omdev-0.0.0.dev25.dist-info → omdev-0.0.0.dev27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev25
+Version: 0.0.0.dev27
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omlish ==0.0.0.dev25
+Requires-Dist: omlish ==0.0.0.dev27
 Provides-Extra: all
 Requires-Dist: pycparser ~=2.22 ; extra == 'all'
 Requires-Dist: cffi ~=1.17 ; extra == 'all'