omdev 0.0.0.dev24__py3-none-any.whl → 0.0.0.dev26__py3-none-any.whl

omdev/datacache/specs.py

@@ -0,0 +1,93 @@
+import hashlib
+import typing as ta
+import urllib.parse
+import urllib.request
+
+from omlish import cached
+from omlish import check
+from omlish import dataclasses as dc
+from omlish import lang
+from omlish import marshal as msh
+from omlish.formats import json
+
+from .consts import MARSHAL_VERSION
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CacheDataSpec(lang.Abstract, lang.Sealed):
+    marshal_version: int = dc.field(default=MARSHAL_VERSION, kw_only=True)
+
+    @cached.property
+    def json(self) -> str:
+        return json.dumps_compact(msh.marshal(self, CacheDataSpec))
+
+    @cached.property
+    def digest(self) -> str:
+        return hashlib.md5(self.json.encode('utf-8')).hexdigest()  # noqa
+
+
+##
+
+
+def _maybe_sorted_strs(v: ta.Iterable[str] | None) -> ta.Sequence[str] | None:
+    if v is None:
+        return None
+    return sorted(set(check.not_isinstance(v, str)))
+
+
+@dc.dataclass(frozen=True)
+class GitCacheDataSpec(CacheDataSpec):
+    url: str
+
+    branch: str | None = dc.field(default=None, kw_only=True)
+    rev: str | None = dc.field(default=None, kw_only=True)
+
+    subtrees: ta.Sequence[str] = dc.xfield(default=None, kw_only=True, coerce=_maybe_sorted_strs)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class HttpCacheDataSpec(CacheDataSpec):
+    url: str = dc.xfield(validate=lambda u: bool(urllib.parse.urlparse(u)))
+    file_name: str | None = None
+
+    @cached.property
+    def file_name_or_default(self) -> str:
+        if self.file_name is not None:
+            return self.file_name
+        return urllib.parse.urlparse(self.url).path.split('/')[-1]
+
+
+##
+
+
+def _repo_str(s: str) -> str:
+    u, r = check.non_empty_str(s).split('/')
+    check.non_empty_str(u)
+    check.non_empty_str(r)
+    return s
+
+
+@dc.dataclass(frozen=True)
+class GithubContentCacheDataSpec(CacheDataSpec):
+    repo: str = dc.field(validate=_repo_str)  # type: ignore
+    rev: str
+    files: lang.SequenceNotStr[str]
+
+
+##
+
+
+@lang.cached_function
+def _install_standard_marshalling() -> None:
+    specs_poly = msh.polymorphism_from_subclasses(CacheDataSpec)
+    msh.STANDARD_MARSHALER_FACTORIES[0:0] = [msh.PolymorphismMarshalerFactory(specs_poly)]
+    msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [msh.PolymorphismUnmarshalerFactory(specs_poly)]
+
+
+_install_standard_marshalling()

omdev/findmagic.py

@@ -17,6 +17,13 @@ def find_magic(
         *,
         py: bool = False,
 ) -> ta.Iterator[str]:
+    if isinstance(roots, str):
+        raise TypeError(roots)
+    if isinstance(magics, str):
+        raise TypeError(magics)
+    if isinstance(exts, str):
+        raise TypeError(exts)
+
     if not magics:
         raise Exception('Must specify magics')
     if not exts:

omdev/git.py

@@ -0,0 +1,62 @@
+import os.path
+import subprocess
+import typing as ta
+
+
+def clone_subtree(
+        *,
+        base_dir: str,
+        repo_url: str,
+        repo_dir: str,
+        branch: str | None = None,
+        rev: str | None = None,
+        repo_subtrees: ta.Sequence[str],
+) -> None:
+    if not bool(branch) ^ bool(rev):
+        raise ValueError('must set branch or rev')
+
+    if isinstance(repo_subtrees, str):
+        raise TypeError(repo_subtrees)
+
+    git_opts = [
+        '-c', 'advice.detachedHead=false',
+    ]
+
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'clone',
+            '-n',
+            '--depth=1',
+            '--filter=tree:0',
+            *(['-b', branch] if branch else []),
+            '--single-branch',
+            repo_url,
+            repo_dir,
+        ],
+        cwd=base_dir,
+    )
+
+    rd = os.path.join(base_dir, repo_dir)
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'sparse-checkout',
+            'set',
+            '--no-cone',
+            *repo_subtrees,
+        ],
+        cwd=rd,
+    )
+
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'checkout',
+            *([rev] if rev else []),
+        ],
+        cwd=rd,
+    )

omdev/manifests.py

@@ -0,0 +1,247 @@
+"""
+!!! manifests! get-manifest, _manifest.py
+ - dumb dicts, root keys are 'types'
+ - get put in _manifest.py, root level dict or smth
+  - IMPORT files w comment
+  - comment must immediately precede a global val setter
+  - val is grabbed from imported module dict by name
+  - value is repr'd somehow (roundtrip checked) (naw, json lol)
+  - dumped in _manifest.py
+ - # @omlish-manifest \n _CACHE_MANIFEST = {'cache': {'name': 'llm', …
+ - also can do prechecks!
+"""
+# ruff: noqa: UP006 UP007
+# @omlish-lite
+import argparse
+import collections
+import dataclasses as dc
+import inspect
+import json
+import os.path
+import re
+import shlex
+import subprocess
+import sys
+import time
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.json import json_dumps_pretty
+from omlish.lite.logs import configure_standard_logging
+from omlish.lite.logs import log
+
+from . import findmagic
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class ManifestOrigin:
+    module: str
+    attr: str
+
+    file: str
+    line: int
+
+
+@dc.dataclass(frozen=True)
+class Manifest(ManifestOrigin):
+    value: ta.Any
+
+
+MANIFEST_MAGIC = '# @omlish-manifest'
+
+_MANIFEST_GLOBAL_PAT = re.compile(r'^(?P<name>[A-Za-z_][A-Za-z0-9_]*)\s*=.*')
+
+
+def _dump_module_manifests(spec: str, *attrs: str) -> None:
+    import importlib
+    import json
+
+    mod = importlib.import_module(spec)
+
+    out = {}
+    for attr in attrs:
+        manifest = getattr(mod, attr)
+
+        manifest_json = json.dumps(manifest)
+        rt_manifest = json.loads(manifest_json)
+
+        if rt_manifest != manifest:
+            raise Exception(f'Manifest failed to roundtrip: {manifest} != {rt_manifest}')
+
+        out[attr] = rt_manifest
+
+    out_json = json.dumps(out, indent=None, separators=(',', ':'))
+    print(out_json)
+
+
+@cached_nullary
+def _payload_src() -> str:
+    return inspect.getsource(_dump_module_manifests)
+
+
+def build_module_manifests(
+        file: str,
+        base: str,
+        *,
+        shell_wrap: bool = True,
+        warn_threshold_s: ta.Optional[float] = 1.,
+) -> ta.Sequence[Manifest]:
+    log.info('Extracting manifests from file %s', file)
+
+    if not file.endswith('.py'):
+        raise Exception(file)
+
+    mod_name = file.rpartition('.')[0].replace(os.sep, '.')
+    mod_base = mod_name.split('.')[0]
+    if mod_base != (first_dir := file.split(os.path.sep)[0]):
+        raise Exception(f'Unexpected module base: {mod_base=} != {first_dir=}')
+
+    with open(os.path.join(base, file)) as f:
+        src = f.read()
+
+    origins: ta.List[ManifestOrigin] = []
+    lines = src.splitlines(keepends=True)
+    for i, l in enumerate(lines):
+        if l.startswith(MANIFEST_MAGIC):
+            if (m := _MANIFEST_GLOBAL_PAT.match(nl := lines[i + 1])) is None:
+                raise Exception(nl)
+
+            origins.append(ManifestOrigin(
+                module='.'.join(['', *mod_name.split('.')[1:]]),
+                attr=m.groupdict()['name'],
+
+                file=os.path.join(*os.path.split(file)[1:]),  # noqa
+                line=i + 1,
+            ))
+
+    if not origins:
+        raise Exception('no manifests found')
+
+    if (dups := [k for k, v in collections.Counter(o.attr for o in origins).items() if v > 1]):
+        raise Exception(f'Duplicate attrs: {dups}')
+
+    attrs = [o.attr for o in origins]
+
+    subproc_src = '\n\n'.join([
+        _payload_src(),
+        f'_dump_module_manifests({mod_name!r}, {", ".join(repr(a) for a in attrs)})\n',
+    ])
+
+    args = [
+        sys.executable,
+        '-c',
+        subproc_src,
+    ]
+
+    if shell_wrap:
+        args = ['sh', '-c', ' '.join(map(shlex.quote, args))]
+
+    start_time = time.time()
+
+    subproc_out = subprocess.check_output(args)
+
+    end_time = time.time()
+
+    if warn_threshold_s is not None and (elapsed_time := (end_time - start_time)) >= warn_threshold_s:
+        log.warning('Manifest extraction took a long time: %s, %.2f s', file, elapsed_time)
+
+    sp_lines = subproc_out.decode().strip().splitlines()
+    if len(sp_lines) != 1:
+        raise Exception('Unexpected subprocess output')
+
+    dct = json.loads(sp_lines[0])
+    if set(dct) != set(attrs):
+        raise Exception('Unexpected subprocess output keys')
+
+    out: ta.List[Manifest] = []
+
+    for o in origins:
+        value = dct[o.attr]
+
+        if not (
+                isinstance(value, ta.Mapping) and
+                all(isinstance(k, str) and k.startswith('$') and len(k) > 1 for k in value)
+        ):
+            raise TypeError(f'Manifests must be mapping of strings starting with $: {value!r}')
+
+        out.append(Manifest(
+            **dc.asdict(o),
+            value=value,
+        ))
+
+    return out
+
+
+def build_package_manifests(
+        name: str,
+        base: str,
+        *,
+        write: bool = False,
+) -> ta.List[Manifest]:
+    pkg_dir = os.path.join(base, name)
+    if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
+        raise Exception(pkg_dir)
+
+    manifests: ta.List[Manifest] = []
+
+    for file in findmagic.find_magic(
+            [pkg_dir],
+            [MANIFEST_MAGIC],
+            ['py'],
+    ):
+        manifests.extend(build_module_manifests(os.path.relpath(file, base), base))
+
+    if write:
+        with open(os.path.join(pkg_dir, '_manifests.json'), 'w') as f:
+            f.write(json_dumps_pretty([dc.asdict(m) for m in manifests]))
+            f.write('\n')
+
+    return manifests
+
+
+##
+
+
+if __name__ == '__main__':
+    def _gen_cmd(args) -> None:
+        if args.base is not None:
+            base = args.base
+        else:
+            base = os.getcwd()
+        base = os.path.abspath(base)
+        if not os.path.isdir(base):
+            raise RuntimeError(base)
+
+        for pkg in args.package:
+            ms = build_package_manifests(
+                pkg,
+                base,
+                write=args.write or False,
+            )
+            if not args.quiet:
+                print(json_dumps_pretty([dc.asdict(m) for m in ms]))
+
+    def _main(argv=None) -> None:
+        configure_standard_logging('INFO')
+
+        parser = argparse.ArgumentParser()
+        subparsers = parser.add_subparsers()
+
+        parser_gen = subparsers.add_parser('gen')
+        parser_gen.add_argument('-b', '--base')
+        parser_gen.add_argument('-w', '--write', action='store_true')
+        parser_gen.add_argument('-q', '--quiet', action='store_true')
+        parser_gen.add_argument('package', nargs='*')
+
+        parser_gen.set_defaults(func=_gen_cmd)
+
+        args = parser.parse_args(argv)
+        if not getattr(args, 'func', None):
+            parser.print_help()
+        else:
+            args.func(args)
+
+    _main()

omdev/pyproject/pkg.py

@@ -30,13 +30,14 @@ import dataclasses as dc
 import importlib
 import os.path
 import shutil
-import subprocess
 import sys
+import tempfile
 import types
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
 from omlish.lite.logs import log
+from omlish.lite.subprocesses import subprocess_check_call
 
 from ..cexts.magic import CextMagic
 from ..findmagic import find_magic
@@ -161,13 +162,12 @@ class BasePyprojectPackageGenerator(abc.ABC):
             build_output_dir: ta.Optional[str] = None,
             *,
             add_revision: bool = False,
+            test: bool = False,
     ) -> None:
-        subprocess.check_call(
-            [
-                sys.executable,
-                '-m',
-                'build',
-            ],
+        subprocess_check_call(
+            sys.executable,
+            '-m',
+            'build',
             cwd=self._pkg_dir(),
         )
 
@@ -176,6 +176,25 @@ class BasePyprojectPackageGenerator(abc.ABC):
         if add_revision:
             GitRevisionAdder().add_to(dist_dir)
 
+        if test:
+            for fn in os.listdir(dist_dir):
+                tmp_dir = tempfile.mkdtemp()
+
+                subprocess_check_call(
+                    sys.executable,
+                    '-m', 'venv',
+                    'test-install',
+                    cwd=tmp_dir,
+                )
+
+                subprocess_check_call(
+                    os.path.join(tmp_dir, 'test-install', 'bin', 'python3'),
+                    '-m', 'pip',
+                    'install',
+                    os.path.abspath(os.path.join(dist_dir, fn)),
+                    cwd=tmp_dir,
+                )
+
         if build_output_dir is not None:
             for fn in os.listdir(dist_dir):
                 shutil.copyfile(os.path.join(dist_dir, fn), os.path.join(build_output_dir, fn))
@@ -255,6 +274,7 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
         st.pop('cexts', None)
 
         self._move_dict_key(st, 'find_packages', pyp_dct, 'tool.setuptools.packages.find')
+        self._move_dict_key(st, 'package_data', pyp_dct, 'tool.setuptools.package-data')
 
         mani_in = st.pop('manifest_in', None)
 
@@ -337,9 +357,14 @@ class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
         st = specs.setuptools
         pyp_dct['tool.setuptools'] = st
 
-        st.pop('cexts', None)
-        st.pop('find_packages', None)
-        st.pop('manifest_in', None)
+        for k in [
+            'cexts',
+
+            'find_packages',
+            'package_data',
+            'manifest_in',
+        ]:
+            st.pop(k, None)
 
         pyp_dct['tool.setuptools.packages.find'] = {
             'include': [],

omdev/scripts/interp.py

@@ -3,7 +3,7 @@
 # @omlish-lite
 # @omlish-script
 # @omdev-amalg-output ../interp/cli.py
-# ruff: noqa: UP007
+# ruff: noqa: N802 UP006 UP007 UP036
 """
 TODO:
  - partial best-matches - '3.12'
@@ -31,6 +31,17 @@ import threading
 import typing as ta
 
 
+########################################
+
+
+if sys.version_info < (3, 8):
+    raise OSError(
+        f'Requires python (3, 8), got {sys.version_info} from {sys.executable}')  # noqa
+
+
+########################################
+
+
 # ../../versioning/versions.py
 VersionLocalType = ta.Tuple[ta.Union[int, str], ...]
 VersionCmpPrePostDevType = ta.Union['InfinityVersionType', 'NegativeInfinityVersionType', ta.Tuple[str, int]]
@@ -72,7 +83,6 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
 # Apache License, Version 2.0, and the BSD License. See the LICENSE file in the root of this repository for complete
 # details.
 # https://github.com/pypa/packaging/blob/2c885fe91a54559e2382902dce28428ad2887be5/src/packaging/version.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -479,7 +489,6 @@ class cached_nullary:  # noqa
 
 ########################################
 # ../../../omlish/lite/check.py
-# ruff: noqa: UP006 UP007
 
 
 def check_isinstance(v: T, spec: ta.Union[ta.Type[T], tuple]) -> T:
@@ -539,7 +548,6 @@ json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON
 
 ########################################
 # ../../../omlish/lite/reflect.py
-# ruff: noqa: UP006
 
 
 _GENERIC_ALIAS_TYPES = (
@@ -640,7 +648,6 @@ def is_sunder(name: str) -> bool:
 # Apache License, Version 2.0, and the BSD License. See the LICENSE file in the root of this repository for complete
 # details.
 # https://github.com/pypa/packaging/blob/2c885fe91a54559e2382902dce28428ad2887be5/src/packaging/specifiers.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -1146,7 +1153,6 @@ TODO:
  - translate json keys
  - debug
 """
-# ruff: noqa: UP006 UP007 N802
 
 
 log = logging.getLogger(__name__)
@@ -1345,46 +1351,51 @@ def configure_standard_logging(
         *,
         json: bool = False,
         target: ta.Optional[logging.Logger] = None,
-        no_check: bool = False,
+        force: bool = False,
 ) -> ta.Optional[StandardLogHandler]:
-    if target is None:
-        target = logging.root
+    logging._acquireLock()  # type: ignore  # noqa
+    try:
+        if target is None:
+            target = logging.root
 
-    #
+        #
 
-    if not no_check:
-        if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
-            return None
+        if not force:
+            if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
+                return None
 
-    #
+        #
 
-    handler = logging.StreamHandler()
+        handler = logging.StreamHandler()
 
-    #
+        #
 
-    formatter: logging.Formatter
-    if json:
-        formatter = JsonLogFormatter()
-    else:
-        formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
-    handler.setFormatter(formatter)
+        formatter: logging.Formatter
+        if json:
+            formatter = JsonLogFormatter()
+        else:
+            formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
+        handler.setFormatter(formatter)
 
-    #
+        #
 
-    handler.addFilter(TidLogFilter())
+        handler.addFilter(TidLogFilter())
 
-    #
+        #
 
-    target.addHandler(handler)
+        target.addHandler(handler)
 
-    #
+        #
 
-    if level is not None:
-        target.setLevel(level)
+        if level is not None:
+            target.setLevel(level)
 
-    #
+        #
+
+        return StandardLogHandler(handler)
 
-    return StandardLogHandler(handler)
+    finally:
+        logging._releaseLock()  # type: ignore  # noqa
 
 
 ########################################
@@ -1407,7 +1418,6 @@ def check_runtime_version() -> None:
 
 ########################################
 # ../types.py
-# ruff: noqa: UP006
 
 
 # See https://peps.python.org/pep-3149/
@@ -1499,7 +1509,6 @@ class Interp:
 
 ########################################
 # ../../../omlish/lite/subprocesses.py
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -1606,7 +1615,6 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
 
 ########################################
 # ../inspect.py
-# ruff: noqa: UP006 UP007
 
 
 @dc.dataclass(frozen=True)
@@ -1770,7 +1778,6 @@ TODO:
  - optionally install / upgrade pyenv itself
  - new vers dont need these custom mac opts, only run on old vers
 """
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -2165,7 +2172,6 @@ TODO:
  - python, python3, python3.12, ...
  - check if path py's are venvs: sys.prefix != sys.base_prefix
 """
-# ruff: noqa: UP006 UP007
 
 
 ##
@@ -2274,7 +2280,6 @@ class SystemInterpProvider(InterpProvider):
 
 ########################################
 # ../resolvers.py
-# ruff: noqa: UP006 UP007
 
 
 INTERP_PROVIDER_TYPES_BY_NAME: ta.Mapping[str, ta.Type[InterpProvider]] = {