omdev 0.0.0.dev25__py3-none-any.whl → 0.0.0.dev27__py3-none-any.whl

omdev/amalg/amalg.py

@@ -29,13 +29,16 @@ import io
 import logging
 import os.path
 import re
+import textwrap
 import typing as ta
 
 import tokenize_rt as trt
 
 from omlish import check
 from omlish import collections as col
+from omlish import lang
 from omlish import logs
+from omlish.lite.runtime import REQUIRED_PYTHON_VERSION
 
 from .. import findmagic
 from .. import tokens as tks
@@ -91,6 +94,7 @@ def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
 
 STRIPPED_HEADER_MAGICS = [
     '# @omlish-lite',
+    '# @omlish-script',
 ]
 
 STRIPPED_HEADER_PATS = [findmagic.compile_magic_pat(m) for m in STRIPPED_HEADER_MAGICS]
@@ -268,6 +272,8 @@ class SrcFile:
     typings: ta.Sequence[Typing] = dc.field(repr=False)
     content_lines: ta.Sequence[Tokens] = dc.field(repr=False)
 
+    ruff_noqa: ta.AbstractSet[str] = dc.field(repr=False)
+
 
 def make_src_file(
         path: str,
@@ -283,6 +289,7 @@ def make_src_file(
     hls, cls = split_header_lines(lines)
 
     hls = strip_header_lines(hls)
+    rnls, hls = col.partition(hls, lambda l: tks.join_toks(l).startswith('# ruff: noqa: '))
 
     imps: list[Import] = []
     tys: list[Typing] = []
@@ -316,6 +323,8 @@ def make_src_file(
         imports=imps,
         typings=tys,
         content_lines=ctls,
+
+        ruff_noqa=set(lang.flatten(tks.join_toks(l).strip().split()[3:] for l in rnls)),  # noqa
     )
 
 
@@ -324,10 +333,11 @@ def make_src_file(
 
 SECTION_SEP = '#' * 40 + '\n'
 
-RUFF_DISABLES: ta.Sequence[str] = [
-    # 'UP006',  # non-pep585-annotation
-    # 'UP007',  # non-pep604-annotation
-]
+RUFF_DISABLES: ta.AbstractSet[str] = {
+    'UP006',  # non-pep585-annotation
+    'UP007',  # non-pep604-annotation
+    'UP036',  # outdated-version-block
+}
 
 OUTPUT_COMMENT = '# @omdev-amalg-output '
 SCAN_COMMENT = '# @omdev-amalg '
@@ -362,46 +372,70 @@ def gen_amalg(
 
     ##
 
+    hls = []
+
     mf = src_files[main_path]
     if mf.header_lines:
-        hls = [
+        hls.extend([
             hl
             for hlts in mf.header_lines
             if not (hl := tks.join_toks(hlts)).startswith(SCAN_COMMENT)
-        ]
-        if output_dir is not None:
-            ogf = os.path.relpath(main_path, output_dir)
-        else:
-            ogf = os.path.basename(main_path)
-        nhls = []
-        nhls.extend([
-            '#!/usr/bin/env python3\n',
-            '# noinspection DuplicatedCode\n',
-            '# @omlish-lite\n',
-            '# @omlish-script\n',
-            f'{OUTPUT_COMMENT.strip()} {ogf}\n',
         ])
-        hls = [*nhls, *hls]
-        out.write(''.join(hls))
 
-    if RUFF_DISABLES:
-        out.write(f'# ruff: noqa: {" ".join(RUFF_DISABLES)}\n')
+    if output_dir is not None:
+        ogf = os.path.relpath(main_path, output_dir)
+    else:
+        ogf = os.path.basename(main_path)
+
+    nhls = []
+    nhls.extend([
+        '#!/usr/bin/env python3\n',
+        '# noinspection DuplicatedCode\n',
+        '# @omlish-lite\n',
+        '# @omlish-script\n',
+        f'{OUTPUT_COMMENT.strip()} {ogf}\n',
+    ])
+
+    ruff_disables = sorted({
+        *lang.flatten(f.ruff_noqa for f in src_files.values()),
+        *RUFF_DISABLES,
+    })
+    if ruff_disables:
+        nhls.append(f'# ruff: noqa: {" ".join(sorted(ruff_disables))}\n')
+
+    hls = [*nhls, *hls]
+    out.write(''.join(hls))
 
     ##
 
     all_imps = [i for f in src_files.values() for i in f.imports]
     gl_imps = [i for i in all_imps if i.mod_path is None]
 
-    dct: dict = {}
+    dct: dict = {
+        ('sys', None, None): ['import sys\n'],
+    }
     for imp in gl_imps:
         dct.setdefault((imp.mod, imp.item, imp.as_), []).append(imp)
     for _, l in sorted(dct.items()):
-        out.write(tks.join_toks(l[0].toks))
+        il = l[0]
+        out.write(il if isinstance(il, str) else tks.join_toks(il.toks))
     if dct:
         out.write('\n\n')
 
     ##
 
+    out.write(SECTION_SEP)
+    out.write('\n\n')
+
+    out.write(textwrap.dedent(f"""
+    if sys.version_info < {REQUIRED_PYTHON_VERSION!r}:
+        raise OSError(
+            f'Requires python {REQUIRED_PYTHON_VERSION!r}, got {{sys.version_info}} from {{sys.executable}}')  # noqa
+    """).lstrip())
+    out.write('\n\n')
+
+    ##
+
     ts = list(col.toposort({  # noqa
         f.path: {mp for i in f.imports if (mp := i.mod_path) is not None}
         for f in src_files.values()
@@ -418,6 +452,9 @@ def gen_amalg(
             if ty.src not in tys:
                 tyd.setdefault(f.path, []).append(ty)
                 tys.add(ty.src)
+    if tys:
+        out.write(SECTION_SEP)
+        out.write('\n\n')
     for i, (sf, ftys) in enumerate(tyd.items()):
         f = src_files[sf]
         if i:

omdev/datacache/__init__.py

@@ -0,0 +1,19 @@
+from .cache import (  # noqa
+    DataCache,
+)
+
+from .defaults import (  # noqa
+    default,
+    default_dir,
+)
+
+from .manifests import (  # noqa
+    CacheDataManifest,
+)
+
+from .specs import (  # noqa
+    CacheDataSpec,
+    GitCacheDataSpec,
+    GithubContentCacheDataSpec,
+    HttpCacheDataSpec,
+)

omdev/datacache/actions.py

@@ -0,0 +1,36 @@
+import typing as ta
+
+from omlish import dataclasses as dc
+from omlish import lang
+from omlish import marshal as msh
+
+from .consts import SERIALIZATION_VERSION
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Action(lang.Abstract, lang.Sealed):
+    serialization_version: int = dc.field(default=SERIALIZATION_VERSION, kw_only=True)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class ExtractAction(Action, lang.Final):
+    files: ta.Sequence[str]
+
+
+##
+
+
+@lang.cached_function
+def _install_standard_marshalling() -> None:
+    actions_poly = msh.polymorphism_from_subclasses(Action)
+    msh.STANDARD_MARSHALER_FACTORIES[0:0] = [msh.PolymorphismMarshalerFactory(actions_poly)]
+    msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [msh.PolymorphismUnmarshalerFactory(actions_poly)]
+
+
+_install_standard_marshalling()

omdev/datacache/cache.py

@@ -0,0 +1,164 @@
+"""
+TODO:
+ - huggingface_hub
+ - postprocessing?
+  - unarchive
+ - stupid little progress bars
+ - return file path for single files
+  - thus, HttpSpec.url has derive=lambda url: ...
+"""
+import logging
+import os.path
+import shutil
+import tempfile
+import urllib.parse
+import urllib.request
+
+from omlish import check
+from omlish import lang
+from omlish import marshal as msh
+from omlish.formats import json
+
+from .. import git
+from .manifests import CacheDataManifest
+from .specs import CacheDataSpec
+from .specs import GitCacheDataSpec
+from .specs import GithubContentCacheDataSpec
+from .specs import HttpCacheDataSpec
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class DataCache:
+    def __init__(self, base_dir: str) -> None:
+        super().__init__()
+        self._base_dir = base_dir
+
+        self._items_dir = os.path.join(base_dir, 'items')
+
+    def _fetch_url(self, url: str, out_file: str) -> None:
+        log.info('Fetching url: %s -> %s', url, out_file)
+
+        urllib.request.urlretrieve(url, out_file)  # noqa
+
+    def _fetch_into(self, spec: CacheDataSpec, data_dir: str) -> None:
+        log.info('Fetching spec: %s %r -> %s', spec.digest, spec, data_dir)
+
+        if isinstance(spec, HttpCacheDataSpec):
+            self._fetch_url(spec.url, os.path.join(data_dir, spec.file_name_or_default))
+
+        elif isinstance(spec, GithubContentCacheDataSpec):
+            for repo_file in spec.files:
+                out_file = os.path.join(data_dir, repo_file)
+                if not os.path.abspath(out_file).startswith(os.path.abspath(data_dir)):
+                    raise RuntimeError(out_file)  # noqa
+
+                url = f'https://raw.githubusercontent.com/{spec.repo}/{spec.rev}/{repo_file}'
+                os.makedirs(os.path.dirname(out_file), exist_ok=True)
+                self._fetch_url(url, os.path.join(data_dir, out_file))
+
+        elif isinstance(spec, GitCacheDataSpec):
+            if not spec.subtrees:
+                raise NotImplementedError
+
+            tmp_dir = tempfile.mkdtemp()
+
+            log.info('Cloning git repo: %s -> %s', spec.url, tmp_dir)
+
+            git.git_clone_subtree(
+                base_dir=tmp_dir,
+                repo_url=spec.url,
+                repo_dir='data',
+                branch=spec.branch,
+                rev=spec.rev,
+                repo_subtrees=spec.subtrees,
+            )
+
+            repo_dir = os.path.join(tmp_dir, 'data')
+            if not os.path.isdir(repo_dir):
+                raise RuntimeError(repo_dir)
+
+            git_dir = os.path.join(repo_dir, '.git')
+            if not os.path.isdir(git_dir):
+                raise RuntimeError(git_dir)
+            shutil.rmtree(git_dir)
+
+            os.rmdir(data_dir)
+            os.rename(repo_dir, data_dir)
+
+        else:
+            raise TypeError(spec)
+
+    def _return_val(self, spec: CacheDataSpec, data_dir: str) -> str:
+        check.state(os.path.isdir(data_dir))
+
+        if isinstance(spec, HttpCacheDataSpec):
+            data_file = os.path.join(data_dir, spec.file_name_or_default)
+            if not os.path.isfile(data_file):
+                raise RuntimeError(data_file)  # noqa
+            return data_file
+
+        elif isinstance(spec, GithubContentCacheDataSpec):
+            if len(spec.files) != 1:
+                return data_dir
+            data_file = os.path.join(data_dir, check.single(spec.files))
+            if not os.path.isfile(data_file):
+                raise RuntimeError(data_file)  # noqa
+            return data_file
+
+        else:
+            return data_dir
+
+    def get(self, spec: CacheDataSpec) -> str:
+        os.makedirs(self._items_dir, exist_ok=True)
+
+        #
+
+        item_dir = os.path.join(self._items_dir, spec.digest)
+        if os.path.isdir(item_dir):
+            data_dir = os.path.join(item_dir, 'data')
+            return self._return_val(spec, data_dir)
+
+        #
+
+        tmp_dir = tempfile.mkdtemp()
+
+        #
+
+        fetch_dir = os.path.join(tmp_dir, 'data')
+        os.mkdir(fetch_dir)
+
+        start_at = lang.utcnow()
+        self._fetch_into(spec, fetch_dir)
+        end_at = lang.utcnow()
+
+        #
+
+        manifest = CacheDataManifest(
+            spec,
+            start_at=start_at,
+            end_at=end_at,
+        )
+        manifest_json = json.dumps_pretty(msh.marshal(manifest))
+
+        manifest_file = os.path.join(tmp_dir, 'manifest.json')
+        with open(manifest_file, 'w') as f:
+            f.write(manifest_json)
+
+        #
+
+        # for p, ds, fs in os.walk(tmp_dir):
+        #     for n in [*ds, *fs]:
+        #         np = os.path.join(p, n)
+        #         os.chmod(np, os.stat(np).st_mode & ~0o222)
+
+        #
+
+        shutil.move(tmp_dir, item_dir)
+
+        data_dir = os.path.join(item_dir, 'data')
+        return self._return_val(spec, data_dir)

omdev/datacache/consts.py

@@ -0,0 +1 @@
+SERIALIZATION_VERSION = 0

omdev/datacache/defaults.py

@@ -0,0 +1,18 @@
+import os.path
+
+from omlish import lang
+
+from .cache import DataCache
+
+
+##
+
+
+@lang.cached_function(lock=True)
+def default_dir() -> str:
+    return os.path.expanduser('~/.cache/omlish/data')
+
+
+@lang.cached_function(lock=True)
+def default() -> DataCache:
+    return DataCache(default_dir())

omdev/datacache/manifests.py

@@ -0,0 +1,40 @@
+import datetime
+
+from omlish import __about__ as about
+from omlish import cached
+from omlish import dataclasses as dc
+
+from ..git import get_git_revision
+from .consts import SERIALIZATION_VERSION
+from .specs import CacheDataSpec
+
+
+##
+
+
+@cached.function
+def _lib_revision() -> str | None:
+    if (rev := about.__revision__) is not None:
+        return rev  # type: ignore
+
+    return get_git_revision()
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CacheDataManifest:
+    spec: CacheDataSpec
+
+    start_at: datetime.datetime = dc.field(kw_only=True)
+    end_at: datetime.datetime = dc.field(kw_only=True)
+
+    lib_version: str = dc.field(default_factory=lambda: about.__version__, kw_only=True)
+    lib_revision: str = dc.field(default_factory=_lib_revision, kw_only=True)
+
+    serialization_version: int = dc.field(default=SERIALIZATION_VERSION, kw_only=True)
+
+    @dc.validate
+    def _validate_serialization_versions(self) -> bool:
+        return self.serialization_version == self.spec.serialization_version

omdev/datacache/specs.py

@@ -0,0 +1,93 @@
+import hashlib
+import typing as ta
+import urllib.parse
+import urllib.request
+
+from omlish import cached
+from omlish import check
+from omlish import dataclasses as dc
+from omlish import lang
+from omlish import marshal as msh
+from omlish.formats import json
+
+from .consts import SERIALIZATION_VERSION
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CacheDataSpec(lang.Abstract, lang.Sealed):
+    serialization_version: int = dc.field(default=SERIALIZATION_VERSION, kw_only=True)
+
+    @cached.property
+    def json(self) -> str:
+        return json.dumps_compact(msh.marshal(self, CacheDataSpec), sort_keys=True)
+
+    @cached.property
+    def digest(self) -> str:
+        return hashlib.md5(self.json.encode('utf-8')).hexdigest()  # noqa
+
+
+##
+
+
+def _maybe_sorted_strs(v: ta.Iterable[str] | None) -> ta.Sequence[str] | None:
+    if v is None:
+        return None
+    return sorted(set(check.not_isinstance(v, str)))
+
+
+@dc.dataclass(frozen=True)
+class GitCacheDataSpec(CacheDataSpec):
+    url: str
+
+    branch: str | None = dc.field(default=None, kw_only=True)
+    rev: str | None = dc.field(default=None, kw_only=True)
+
+    subtrees: ta.Sequence[str] = dc.xfield(default=None, kw_only=True, coerce=_maybe_sorted_strs)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class HttpCacheDataSpec(CacheDataSpec):
+    url: str = dc.xfield(validate=lambda u: bool(urllib.parse.urlparse(u)))
+    file_name: str | None = None
+
+    @cached.property
+    def file_name_or_default(self) -> str:
+        if self.file_name is not None:
+            return self.file_name
+        return urllib.parse.urlparse(self.url).path.split('/')[-1]
+
+
+##
+
+
+def _repo_str(s: str) -> str:
+    u, r = check.non_empty_str(s).split('/')
+    check.non_empty_str(u)
+    check.non_empty_str(r)
+    return s
+
+
+@dc.dataclass(frozen=True)
+class GithubContentCacheDataSpec(CacheDataSpec):
+    repo: str = dc.field(validate=_repo_str)  # type: ignore
+    rev: str
+    files: lang.SequenceNotStr[str]
+
+
+##
+
+
+@lang.cached_function
+def _install_standard_marshalling() -> None:
+    specs_poly = msh.polymorphism_from_subclasses(CacheDataSpec)
+    msh.STANDARD_MARSHALER_FACTORIES[0:0] = [msh.PolymorphismMarshalerFactory(specs_poly)]
+    msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [msh.PolymorphismUnmarshalerFactory(specs_poly)]
+
+
+_install_standard_marshalling()

omdev/git.py

@@ -0,0 +1,100 @@
+# ruff: noqa: UP007
+# @omlish-lite
+import os.path
+import subprocess
+import typing as ta
+
+
+def git_clone_subtree(
+        *,
+        base_dir: str,
+        repo_url: str,
+        repo_dir: str,
+        branch: ta.Optional[str] = None,
+        rev: ta.Optional[str] = None,
+        repo_subtrees: ta.Sequence[str],
+) -> None:
+    if not bool(branch) ^ bool(rev):
+        raise ValueError('must set branch or rev')
+
+    if isinstance(repo_subtrees, str):
+        raise TypeError(repo_subtrees)
+
+    git_opts = [
+        '-c', 'advice.detachedHead=false',
+    ]
+
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'clone',
+            '-n',
+            '--depth=1',
+            '--filter=tree:0',
+            *(['-b', branch] if branch else []),
+            '--single-branch',
+            repo_url,
+            repo_dir,
+        ],
+        cwd=base_dir,
+    )
+
+    rd = os.path.join(base_dir, repo_dir)
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'sparse-checkout',
+            'set',
+            '--no-cone',
+            *repo_subtrees,
+        ],
+        cwd=rd,
+    )
+
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'checkout',
+            *([rev] if rev else []),
+        ],
+        cwd=rd,
+    )
+
+
+def get_git_revision(
+        *,
+        cwd: ta.Optional[str] = None,
+) -> ta.Optional[str]:
+    subprocess.check_output(['git', '--version'])
+
+    if cwd is None:
+        cwd = os.getcwd()
+
+    if subprocess.run([  # noqa
+        'git',
+        'rev-parse',
+        '--is-inside-work-tree',
+    ], stderr=subprocess.PIPE).returncode:
+        return None
+
+    has_untracked = bool(subprocess.check_output([
+        'git',
+        'ls-files',
+        '.',
+        '--exclude-standard',
+        '--others',
+    ], cwd=cwd).decode().strip())
+
+    dirty_rev = subprocess.check_output([
+        'git',
+        'describe',
+        '--match=NeVeRmAtCh',
+        '--always',
+        '--abbrev=40',
+        '--dirty',
+    ], cwd=cwd).decode().strip()
+
+    return dirty_rev + ('-untracked' if has_untracked else '')