omdev 0.0.0.dev25__py3-none-any.whl → 0.0.0.dev27__py3-none-any.whl

omdev/manifests.py

@@ -1,5 +1,5 @@
 """
-!!! manifests! get-manifest, _manifest.py
+!!! manifests! get-manifest, .manifest.json
  - dumb dicts, root keys are 'types'
  - get put in _manifest.py, root level dict or smth
   - IMPORT files w comment
@@ -10,7 +10,7 @@
  - # @omlish-manifest \n _CACHE_MANIFEST = {'cache': {'name': 'llm', …
  - also can do prechecks!
 """
-# ruff: noqa: UP006
+# ruff: noqa: UP006 UP007
 # @omlish-lite
 import argparse
 import collections
@@ -22,10 +22,13 @@ import re
 import shlex
 import subprocess
 import sys
+import time
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
 from omlish.lite.json import json_dumps_pretty
+from omlish.lite.logs import configure_standard_logging
+from omlish.lite.logs import log
 
 from . import findmagic
 
@@ -84,13 +87,17 @@ def build_module_manifests(
         base: str,
         *,
         shell_wrap: bool = True,
+        warn_threshold_s: ta.Optional[float] = 1.,
 ) -> ta.Sequence[Manifest]:
-    print((file, base))
+    log.info('Extracting manifests from file %s', file)
 
     if not file.endswith('.py'):
         raise Exception(file)
 
     mod_name = file.rpartition('.')[0].replace(os.sep, '.')
+    mod_base = mod_name.split('.')[0]
+    if mod_base != (first_dir := file.split(os.path.sep)[0]):
+        raise Exception(f'Unexpected module base: {mod_base=} != {first_dir=}')
 
     with open(os.path.join(base, file)) as f:
         src = f.read()
@@ -103,10 +110,10 @@ def build_module_manifests(
                 raise Exception(nl)
 
             origins.append(ManifestOrigin(
-                module=mod_name,
+                module='.'.join(['', *mod_name.split('.')[1:]]),
                 attr=m.groupdict()['name'],
 
-                file=file,
+                file=os.path.join(*os.path.split(file)[1:]),  # noqa
                 line=i + 1,
             ))
 
@@ -132,8 +139,15 @@ def build_module_manifests(
     if shell_wrap:
         args = ['sh', '-c', ' '.join(map(shlex.quote, args))]
 
+    start_time = time.time()
+
     subproc_out = subprocess.check_output(args)
 
+    end_time = time.time()
+
+    if warn_threshold_s is not None and (elapsed_time := (end_time - start_time)) >= warn_threshold_s:
+        log.warning('Manifest extraction took a long time: %s, %.2f s', file, elapsed_time)
+
     sp_lines = subproc_out.decode().strip().splitlines()
     if len(sp_lines) != 1:
         raise Exception('Unexpected subprocess output')
@@ -145,11 +159,17 @@ def build_module_manifests(
     out: ta.List[Manifest] = []
 
     for o in origins:
-        manifest = dct[o.attr]
+        value = dct[o.attr]
+
+        if not (
+                isinstance(value, ta.Mapping) and
+                all(isinstance(k, str) and k.startswith('$') and len(k) > 1 for k in value)
+        ):
+            raise TypeError(f'Manifests must be mapping of strings starting with $: {value!r}')
 
         out.append(Manifest(
             **dc.asdict(o),
-            value=manifest,
+            value=value,
         ))
 
     return out
@@ -175,7 +195,7 @@ def build_package_manifests(
         manifests.extend(build_module_manifests(os.path.relpath(file, base), base))
 
     if write:
-        with open(os.path.join(pkg_dir, '_manifests.json'), 'w') as f:
+        with open(os.path.join(pkg_dir, '.manifests.json'), 'w') as f:
             f.write(json_dumps_pretty([dc.asdict(m) for m in manifests]))
             f.write('\n')
 
@@ -202,9 +222,11 @@ if __name__ == '__main__':
                 write=args.write or False,
             )
             if not args.quiet:
-                print(json_dumps_pretty(ms))
+                print(json_dumps_pretty([dc.asdict(m) for m in ms]))
 
     def _main(argv=None) -> None:
+        configure_standard_logging('INFO')
+
         parser = argparse.ArgumentParser()
         subparsers = parser.add_subparsers()
 

omdev/pyproject/pkg.py

@@ -30,13 +30,14 @@ import dataclasses as dc
 import importlib
 import os.path
 import shutil
-import subprocess
 import sys
+import tempfile
 import types
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
 from omlish.lite.logs import log
+from omlish.lite.subprocesses import subprocess_check_call
 
 from ..cexts.magic import CextMagic
 from ..findmagic import find_magic
@@ -138,6 +139,35 @@ class BasePyprojectPackageGenerator(abc.ABC):
 
     #
 
+    class _PkgData(ta.NamedTuple):
+        inc: ta.List[str]
+        exc: ta.List[str]
+
+    @cached_nullary
+    def _collect_pkg_data(self) -> _PkgData:
+        inc: ta.List[str] = []
+        exc: ta.List[str] = []
+
+        for p, ds, fs in os.walk(self._dir_name):  # noqa
+            for f in fs:
+                if f != '.pkgdata':
+                    continue
+                rp = os.path.relpath(p, self._dir_name)
+                log.info('Found pkgdata %s for pkg %s', rp, self._dir_name)
+                with open(os.path.join(p, f)) as fo:
+                    src = fo.read()
+                for l in src.splitlines():
+                    if not (l := l.strip()):
+                        continue
+                    if l.startswith('!'):
+                        exc.append(os.path.join(rp, l[1:]))
+                    else:
+                        inc.append(os.path.join(rp, l))
+
+        return self._PkgData(inc, exc)
+
+    #
+
     @abc.abstractmethod
     def _write_file_contents(self) -> None:
         raise NotImplementedError
@@ -161,13 +191,12 @@ class BasePyprojectPackageGenerator(abc.ABC):
             build_output_dir: ta.Optional[str] = None,
             *,
             add_revision: bool = False,
+            test: bool = False,
     ) -> None:
-        subprocess.check_call(
-            [
-                sys.executable,
-                '-m',
-                'build',
-            ],
+        subprocess_check_call(
+            sys.executable,
+            '-m',
+            'build',
             cwd=self._pkg_dir(),
         )
 
@@ -176,6 +205,25 @@ class BasePyprojectPackageGenerator(abc.ABC):
         if add_revision:
             GitRevisionAdder().add_to(dist_dir)
 
+        if test:
+            for fn in os.listdir(dist_dir):
+                tmp_dir = tempfile.mkdtemp()
+
+                subprocess_check_call(
+                    sys.executable,
+                    '-m', 'venv',
+                    'test-install',
+                    cwd=tmp_dir,
+                )
+
+                subprocess_check_call(
+                    os.path.join(tmp_dir, 'test-install', 'bin', 'python3'),
+                    '-m', 'pip',
+                    'install',
+                    os.path.abspath(os.path.join(dist_dir, fn)),
+                    cwd=tmp_dir,
+                )
+
         if build_output_dir is not None:
             for fn in os.listdir(dist_dir):
                 shutil.copyfile(os.path.join(dist_dir, fn), os.path.join(build_output_dir, fn))
@@ -249,13 +297,56 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
 
         #
 
-        st = specs.setuptools
+        st = dict(specs.setuptools)
         pyp_dct['tool.setuptools'] = st
 
         st.pop('cexts', None)
 
-        self._move_dict_key(st, 'find_packages', pyp_dct, 'tool.setuptools.packages.find')
-        self._move_dict_key(st, 'package_data', pyp_dct, 'tool.setuptools.package-data')
+        #
+
+        # TODO: default
+        # find_packages = {
+        #     'include': [Project.name, f'{Project.name}.*'],
+        #     'exclude': [*SetuptoolsBase.find_packages['exclude']],
+        # }
+
+        fp = dict(st.pop('find_packages', {}))
+
+        pyp_dct['tool.setuptools.packages.find'] = fp
+
+        #
+
+        # TODO: default
+        # package_data = {
+        #     '*': [
+        #         '*.c',
+        #         '*.cc',
+        #         '*.h',
+        #         '.manifests.json',
+        #         'LICENSE',
+        #     ],
+        # }
+
+        pd = dict(st.pop('package_data', {}))
+        epd = dict(st.pop('exclude_package_data', {}))
+
+        cpd = self._collect_pkg_data()
+        if cpd.inc:
+            pd['*'] = [*pd.get('*', []), *sorted(set(cpd.inc))]
+        if cpd.exc:
+            epd['*'] = [*epd.get('*', []), *sorted(set(cpd.exc))]
+
+        if pd:
+            pyp_dct['tool.setuptools.package-data'] = pd
+        if epd:
+            pyp_dct['tool.setuptools.exclude-package-data'] = epd
+
+        #
+
+        # TODO: default
+        # manifest_in = [
+        #     'global-exclude **/conftest.py',
+        # ]
 
         mani_in = st.pop('manifest_in', None)
 
@@ -335,7 +426,7 @@ class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
 
         #
 
-        st = specs.setuptools
+        st = dict(specs.setuptools)
         pyp_dct['tool.setuptools'] = st
 
         for k in [

omdev/revisions.py

@@ -9,45 +9,22 @@ TODO:
 import argparse
 import io
 import os.path
-import subprocess
 import tarfile
 import typing as ta
 import zipfile
 
 from omlish.lite.cached import cached_nullary
+from omlish.lite.check import check_non_empty_str
 from omlish.lite.logs import configure_standard_logging
 from omlish.lite.logs import log
 
+from .git import get_git_revision
 from .wheelfile import WheelFile
 
 
 ##
 
 
-def get_git_revision() -> str:
-    has_untracked = bool(subprocess.check_output([
-        'git',
-        'ls-files',
-        '.',
-        '--exclude-standard',
-        '--others',
-    ]).decode().strip())
-
-    dirty_rev = subprocess.check_output([
-        'git',
-        'describe',
-        '--match=NeVeRmAtCh',
-        '--always',
-        '--abbrev=40',
-        '--dirty',
-    ]).decode().strip()
-
-    return dirty_rev + ('-untracked' if has_untracked else '')
-
-
-##
-
-
 class GitRevisionAdder:
     def __init__(
             self,
@@ -62,7 +39,7 @@ class GitRevisionAdder:
     def revision(self) -> str:
         if self._given_revision is not None:
             return self._given_revision
-        return get_git_revision()
+        return check_non_empty_str(get_git_revision())
 
     REVISION_ATTR = '__revision__'
 

omdev/scripts/interp.py

@@ -3,7 +3,7 @@
 # @omlish-lite
 # @omlish-script
 # @omdev-amalg-output ../interp/cli.py
-# ruff: noqa: UP007
+# ruff: noqa: N802 UP006 UP007 UP036
 """
 TODO:
  - partial best-matches - '3.12'
@@ -31,6 +31,17 @@ import threading
 import typing as ta
 
 
+########################################
+
+
+if sys.version_info < (3, 8):
+    raise OSError(
+        f'Requires python (3, 8), got {sys.version_info} from {sys.executable}')  # noqa
+
+
+########################################
+
+
 # ../../versioning/versions.py
 VersionLocalType = ta.Tuple[ta.Union[int, str], ...]
 VersionCmpPrePostDevType = ta.Union['InfinityVersionType', 'NegativeInfinityVersionType', ta.Tuple[str, int]]
@@ -72,7 +83,6 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
 # Apache License, Version 2.0, and the BSD License. See the LICENSE file in the root of this repository for complete
 # details.
 # https://github.com/pypa/packaging/blob/2c885fe91a54559e2382902dce28428ad2887be5/src/packaging/version.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -479,7 +489,6 @@ class cached_nullary:  # noqa
 
 ########################################
 # ../../../omlish/lite/check.py
-# ruff: noqa: UP006 UP007
 
 
 def check_isinstance(v: T, spec: ta.Union[ta.Type[T], tuple]) -> T:
@@ -506,6 +515,12 @@ def check_not(v: ta.Any) -> None:
     return v
 
 
+def check_non_empty_str(v: ta.Optional[str]) -> str:
+    if not v:
+        raise ValueError
+    return v
+
+
 ########################################
 # ../../../omlish/lite/json.py
 
@@ -539,7 +554,6 @@ json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON
 
 ########################################
 # ../../../omlish/lite/reflect.py
-# ruff: noqa: UP006
 
 
 _GENERIC_ALIAS_TYPES = (
@@ -640,7 +654,6 @@ def is_sunder(name: str) -> bool:
 # Apache License, Version 2.0, and the BSD License. See the LICENSE file in the root of this repository for complete
 # details.
 # https://github.com/pypa/packaging/blob/2c885fe91a54559e2382902dce28428ad2887be5/src/packaging/specifiers.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -1146,7 +1159,6 @@ TODO:
  - translate json keys
  - debug
 """
-# ruff: noqa: UP006 UP007 N802
 
 
 log = logging.getLogger(__name__)
@@ -1345,46 +1357,51 @@ def configure_standard_logging(
         *,
         json: bool = False,
         target: ta.Optional[logging.Logger] = None,
-        no_check: bool = False,
+        force: bool = False,
 ) -> ta.Optional[StandardLogHandler]:
-    if target is None:
-        target = logging.root
+    logging._acquireLock()  # type: ignore  # noqa
+    try:
+        if target is None:
+            target = logging.root
 
-    #
+        #
 
-    if not no_check:
-        if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
-            return None
+        if not force:
+            if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
+                return None
 
-    #
+        #
 
-    handler = logging.StreamHandler()
+        handler = logging.StreamHandler()
 
-    #
+        #
 
-    formatter: logging.Formatter
-    if json:
-        formatter = JsonLogFormatter()
-    else:
-        formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
-    handler.setFormatter(formatter)
+        formatter: logging.Formatter
+        if json:
+            formatter = JsonLogFormatter()
+        else:
+            formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
+        handler.setFormatter(formatter)
 
-    #
+        #
 
-    handler.addFilter(TidLogFilter())
+        handler.addFilter(TidLogFilter())
 
-    #
+        #
 
-    target.addHandler(handler)
+        target.addHandler(handler)
 
-    #
+        #
 
-    if level is not None:
-        target.setLevel(level)
+        if level is not None:
+            target.setLevel(level)
 
-    #
+        #
+
+        return StandardLogHandler(handler)
 
-    return StandardLogHandler(handler)
+    finally:
+        logging._releaseLock()  # type: ignore  # noqa
 
 
 ########################################
@@ -1407,7 +1424,6 @@ def check_runtime_version() -> None:
 
 ########################################
 # ../types.py
-# ruff: noqa: UP006
 
 
 # See https://peps.python.org/pep-3149/
@@ -1499,7 +1515,6 @@ class Interp:
 
 ########################################
 # ../../../omlish/lite/subprocesses.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -1606,7 +1621,6 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
 
 ########################################
 # ../inspect.py
-# ruff: noqa: UP006 UP007
 
 
 @dc.dataclass(frozen=True)
@@ -1770,7 +1784,6 @@ TODO:
  - optionally install / upgrade pyenv itself
  - new vers dont need these custom mac opts, only run on old vers
 """
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -2165,7 +2178,6 @@ TODO:
  - python, python3, python3.12, ...
  - check if path py's are venvs: sys.prefix != sys.base_prefix
 """
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -2274,7 +2286,6 @@ class SystemInterpProvider(InterpProvider):
 
 ########################################
 # ../resolvers.py
-# ruff: noqa: UP006 UP007
 
 
 INTERP_PROVIDER_TYPES_BY_NAME: ta.Mapping[str, ta.Type[InterpProvider]] = {