omdev 0.0.0.dev25__tar.gz → 0.0.0.dev26__tar.gz

{omdev-0.0.0.dev25/omdev.egg-info → omdev-0.0.0.dev26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev25
+Version: 0.0.0.dev26
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev25
+Requires-Dist: omlish==0.0.0.dev26
 Provides-Extra: all
 Requires-Dist: pycparser~=2.22; extra == "all"
 Requires-Dist: cffi~=1.17; extra == "all"

{omdev-0.0.0.dev25 → omdev-0.0.0.dev26}/omdev/amalg/amalg.py

@@ -29,13 +29,16 @@ import io
 import logging
 import os.path
 import re
+import textwrap
 import typing as ta
 
 import tokenize_rt as trt
 
 from omlish import check
 from omlish import collections as col
+from omlish import lang
 from omlish import logs
+from omlish.lite.runtime import REQUIRED_PYTHON_VERSION
 
 from .. import findmagic
 from .. import tokens as tks
@@ -91,6 +94,7 @@ def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
 
 STRIPPED_HEADER_MAGICS = [
     '# @omlish-lite',
+    '# @omlish-script',
 ]
 
 STRIPPED_HEADER_PATS = [findmagic.compile_magic_pat(m) for m in STRIPPED_HEADER_MAGICS]
@@ -268,6 +272,8 @@ class SrcFile:
     typings: ta.Sequence[Typing] = dc.field(repr=False)
     content_lines: ta.Sequence[Tokens] = dc.field(repr=False)
 
+    ruff_noqa: ta.AbstractSet[str] = dc.field(repr=False)
+
 
 def make_src_file(
         path: str,
@@ -283,6 +289,7 @@ def make_src_file(
     hls, cls = split_header_lines(lines)
 
     hls = strip_header_lines(hls)
+    rnls, hls = col.partition(hls, lambda l: tks.join_toks(l).startswith('# ruff: noqa: '))
 
     imps: list[Import] = []
     tys: list[Typing] = []
@@ -316,6 +323,8 @@ def make_src_file(
         imports=imps,
         typings=tys,
         content_lines=ctls,
+
+        ruff_noqa=set(lang.flatten(tks.join_toks(l).strip().split()[3:] for l in rnls)),  # noqa
     )
 
 
@@ -324,10 +333,11 @@ def make_src_file(
 
 SECTION_SEP = '#' * 40 + '\n'
 
-RUFF_DISABLES: ta.Sequence[str] = [
-    # 'UP006',  # non-pep585-annotation
-    # 'UP007',  # non-pep604-annotation
-]
+RUFF_DISABLES: ta.AbstractSet[str] = {
+    'UP006',  # non-pep585-annotation
+    'UP007',  # non-pep604-annotation
+    'UP036',  # outdated-version-block
+}
 
 OUTPUT_COMMENT = '# @omdev-amalg-output '
 SCAN_COMMENT = '# @omdev-amalg '
@@ -362,46 +372,70 @@ def gen_amalg(
 
     ##
 
+    hls = []
+
     mf = src_files[main_path]
     if mf.header_lines:
-        hls = [
+        hls.extend([
             hl
             for hlts in mf.header_lines
             if not (hl := tks.join_toks(hlts)).startswith(SCAN_COMMENT)
-        ]
-        if output_dir is not None:
-            ogf = os.path.relpath(main_path, output_dir)
-        else:
-            ogf = os.path.basename(main_path)
-        nhls = []
-        nhls.extend([
-            '#!/usr/bin/env python3\n',
-            '# noinspection DuplicatedCode\n',
-            '# @omlish-lite\n',
-            '# @omlish-script\n',
-            f'{OUTPUT_COMMENT.strip()} {ogf}\n',
         ])
-        hls = [*nhls, *hls]
-        out.write(''.join(hls))
 
-    if RUFF_DISABLES:
-        out.write(f'# ruff: noqa: {" ".join(RUFF_DISABLES)}\n')
+    if output_dir is not None:
+        ogf = os.path.relpath(main_path, output_dir)
+    else:
+        ogf = os.path.basename(main_path)
+
+    nhls = []
+    nhls.extend([
+        '#!/usr/bin/env python3\n',
+        '# noinspection DuplicatedCode\n',
+        '# @omlish-lite\n',
+        '# @omlish-script\n',
+        f'{OUTPUT_COMMENT.strip()} {ogf}\n',
+    ])
+
+    ruff_disables = sorted({
+        *lang.flatten(f.ruff_noqa for f in src_files.values()),
+        *RUFF_DISABLES,
+    })
+    if ruff_disables:
+        nhls.append(f'# ruff: noqa: {" ".join(sorted(ruff_disables))}\n')
+
+    hls = [*nhls, *hls]
+    out.write(''.join(hls))
 
     ##
 
     all_imps = [i for f in src_files.values() for i in f.imports]
     gl_imps = [i for i in all_imps if i.mod_path is None]
 
-    dct: dict = {}
+    dct: dict = {
+        ('sys', None, None): ['import sys\n'],
+    }
     for imp in gl_imps:
         dct.setdefault((imp.mod, imp.item, imp.as_), []).append(imp)
     for _, l in sorted(dct.items()):
-        out.write(tks.join_toks(l[0].toks))
+        il = l[0]
+        out.write(il if isinstance(il, str) else tks.join_toks(il.toks))
     if dct:
         out.write('\n\n')
 
     ##
 
+    out.write(SECTION_SEP)
+    out.write('\n\n')
+
+    out.write(textwrap.dedent(f"""
+    if sys.version_info < {REQUIRED_PYTHON_VERSION!r}:
+        raise OSError(
+            f'Requires python {REQUIRED_PYTHON_VERSION!r}, got {{sys.version_info}} from {{sys.executable}}')  # noqa
+    """).lstrip())
+    out.write('\n\n')
+
+    ##
+
     ts = list(col.toposort({  # noqa
         f.path: {mp for i in f.imports if (mp := i.mod_path) is not None}
         for f in src_files.values()
@@ -418,6 +452,9 @@ def gen_amalg(
             if ty.src not in tys:
                 tyd.setdefault(f.path, []).append(ty)
                 tys.add(ty.src)
+    if tys:
+        out.write(SECTION_SEP)
+        out.write('\n\n')
     for i, (sf, ftys) in enumerate(tyd.items()):
         f = src_files[sf]
         if i:

omdev-0.0.0.dev26/omdev/datacache/__init__.py

@@ -0,0 +1,19 @@
+from .cache import (  # noqa
+    DataCache,
+)
+
+from .default import (  # noqa
+    default,
+    default_dir,
+)
+
+from .manifests import (  # noqa
+    CacheDataManifest,
+)
+
+from .specs import (  # noqa
+    CacheDataSpec,
+    GitCacheDataSpec,
+    GithubContentCacheDataSpec,
+    HttpCacheDataSpec,
+)

omdev-0.0.0.dev26/omdev/datacache/cache.py

@@ -0,0 +1,149 @@
+"""
+TODO:
+ - huggingface_hub
+ - postprocessing?
+  - unarchive
+ - stupid little progress bars
+ - return file path for single files
+  - thus, HttpSpec.url has derive=lambda url: ...
+"""
+import logging
+import os.path
+import shutil
+import tempfile
+import urllib.parse
+import urllib.request
+
+from omlish import check
+from omlish import lang
+from omlish import marshal as msh
+from omlish.formats import json
+
+from .. import git
+from .manifests import CacheDataManifest
+from .specs import CacheDataSpec
+from .specs import GitCacheDataSpec
+from .specs import GithubContentCacheDataSpec
+from .specs import HttpCacheDataSpec
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class DataCache:
+    def __init__(self, base_dir: str) -> None:
+        super().__init__()
+        self._base_dir = base_dir
+
+        self._items_dir = os.path.join(base_dir, 'items')
+
+    def _fetch_url(self, url: str, out_file: str) -> None:
+        log.info('Fetching url: %s -> %s', url, out_file)
+
+        urllib.request.urlretrieve(url, out_file)  # noqa
+
+    def _fetch_into(self, spec: CacheDataSpec, data_dir: str) -> None:
+        log.info('Fetching spec: %s %r', spec.digest, spec)
+
+        if isinstance(spec, HttpCacheDataSpec):
+            self._fetch_url(spec.url, os.path.join(data_dir, spec.file_name_or_default))
+
+        elif isinstance(spec, GithubContentCacheDataSpec):
+            for repo_file in spec.files:
+                out_file = os.path.join(data_dir, repo_file)
+                if not os.path.abspath(out_file).startswith(os.path.abspath(data_dir)):
+                    raise RuntimeError(out_file)  # noqa
+
+                url = f'https://raw.githubusercontent.com/{spec.repo}/{spec.rev}/{repo_file}'
+                os.makedirs(os.path.dirname(out_file), exist_ok=True)
+                self._fetch_url(url, os.path.join(data_dir, out_file))
+
+        elif isinstance(spec, GitCacheDataSpec):
+            if not spec.subtrees:
+                raise NotImplementedError
+
+            tmp_dir = tempfile.mkdtemp()
+
+            log.info('Cloning git repo: %s -> %s', spec.url, tmp_dir)
+
+            git.clone_subtree(
+                base_dir=tmp_dir,
+                repo_url=spec.url,
+                repo_dir='data',
+                branch=spec.branch,
+                rev=spec.rev,
+                repo_subtrees=spec.subtrees,
+            )
+
+            repo_dir = os.path.join(tmp_dir, 'data')
+            if not os.path.isdir(repo_dir):
+                raise RuntimeError(repo_dir)
+
+            git_dir = os.path.join(repo_dir, '.git')
+            if not os.path.isdir(git_dir):
+                raise RuntimeError(git_dir)
+            shutil.rmtree(git_dir)
+
+            os.rmdir(data_dir)
+            os.rename(repo_dir, data_dir)
+
+        else:
+            raise TypeError(spec)
+
+    def _return_val(self, spec: CacheDataSpec, data_dir: str) -> str:
+        check.state(os.path.isdir(data_dir))
+
+        if isinstance(spec, HttpCacheDataSpec):
+            data_file = os.path.join(data_dir, spec.file_name_or_default)
+            if not os.path.isfile(data_file):
+                raise RuntimeError(data_file)  # noqa
+            return data_file
+
+        else:
+            return data_dir
+
+    def get(self, spec: CacheDataSpec) -> str:
+        os.makedirs(self._items_dir, exist_ok=True)
+
+        #
+
+        item_dir = os.path.join(self._items_dir, spec.digest)
+        if os.path.isdir(item_dir):
+            data_dir = os.path.join(item_dir, 'data')
+            return self._return_val(spec, data_dir)
+
+        #
+
+        tmp_dir = tempfile.mkdtemp()
+
+        #
+
+        fetch_dir = os.path.join(tmp_dir, 'data')
+        os.mkdir(fetch_dir)
+
+        start_at = lang.utcnow()
+        self._fetch_into(spec, fetch_dir)
+        end_at = lang.utcnow()
+
+        #
+
+        manifest = CacheDataManifest(
+            spec,
+            start_at=start_at,
+            end_at=end_at,
+        )
+        manifest_json = json.dumps_pretty(msh.marshal(manifest))
+
+        manifest_file = os.path.join(tmp_dir, 'manifest.json')
+        with open(manifest_file, 'w') as f:
+            f.write(manifest_json)
+
+        ##
+
+        os.rename(tmp_dir, item_dir)
+
+        data_dir = os.path.join(item_dir, 'data')
+        return self._return_val(spec, data_dir)

omdev-0.0.0.dev26/omdev/datacache/consts.py

@@ -0,0 +1 @@
+MARSHAL_VERSION = 0

omdev-0.0.0.dev26/omdev/datacache/default.py

@@ -0,0 +1,51 @@
+import os.path
+
+from omlish import lang
+
+from .cache import DataCache
+
+
+##
+
+
+@lang.cached_function(lock=True)
+def default_dir() -> str:
+    return os.path.expanduser('~/.cache/omlish/data')
+
+
+@lang.cached_function(lock=True)
+def default() -> DataCache:
+    return DataCache(default_dir())
+
+
+def _main() -> None:
+    from omlish import logs
+
+    logs.configure_standard_logging('INFO')
+
+    #
+
+    from .specs import GitCacheDataSpec
+    from .specs import GithubContentCacheDataSpec
+    from .specs import HttpCacheDataSpec
+
+    for spec in [
+        GitCacheDataSpec(
+            'https://github.com/wrmsr/deep_learning_cookbook',
+            rev='138a99b09ffa3a728d261e461440f029e512ac93',
+            subtrees=['data/wp_movies_10k.ndjson'],
+        ),
+        GithubContentCacheDataSpec(
+            'karpathy/char-rnn',
+            'master',
+            ['data/tinyshakespeare/input.txt'],
+        ),
+        HttpCacheDataSpec('https://github.com/VanushVaswani/keras_mnistm/releases/download/1.0/keras_mnistm.pkl.gz'),
+    ]:
+        print(spec)
+        for _ in range(2):
+            print(default().get(spec))
+
+
+if __name__ == '__main__':
+    _main()

omdev-0.0.0.dev26/omdev/datacache/manifests.py

@@ -0,0 +1,40 @@
+import datetime
+
+from omlish import __about__ as about
+from omlish import cached
+from omlish import dataclasses as dc
+
+from ..revisions import get_git_revision
+from .consts import MARSHAL_VERSION
+from .specs import CacheDataSpec
+
+
+##
+
+
+@cached.function
+def _lib_revision() -> str | None:
+    if (rev := about.__revision__) is not None:
+        return rev  # type: ignore
+
+    return get_git_revision()
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CacheDataManifest:
+    spec: CacheDataSpec
+
+    start_at: datetime.datetime = dc.field(kw_only=True)
+    end_at: datetime.datetime = dc.field(kw_only=True)
+
+    lib_version: str = dc.field(default_factory=lambda: about.__version__, kw_only=True)
+    lib_revision: str = dc.field(default_factory=_lib_revision, kw_only=True)
+
+    marshal_version: int = dc.field(default=MARSHAL_VERSION, kw_only=True)
+
+    @dc.validate
+    def _validate_marshal_versions(self) -> bool:
+        return self.marshal_version == self.spec.marshal_version

omdev-0.0.0.dev26/omdev/datacache/specs.py

@@ -0,0 +1,93 @@
+import hashlib
+import typing as ta
+import urllib.parse
+import urllib.request
+
+from omlish import cached
+from omlish import check
+from omlish import dataclasses as dc
+from omlish import lang
+from omlish import marshal as msh
+from omlish.formats import json
+
+from .consts import MARSHAL_VERSION
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CacheDataSpec(lang.Abstract, lang.Sealed):
+    marshal_version: int = dc.field(default=MARSHAL_VERSION, kw_only=True)
+
+    @cached.property
+    def json(self) -> str:
+        return json.dumps_compact(msh.marshal(self, CacheDataSpec))
+
+    @cached.property
+    def digest(self) -> str:
+        return hashlib.md5(self.json.encode('utf-8')).hexdigest()  # noqa
+
+
+##
+
+
+def _maybe_sorted_strs(v: ta.Iterable[str] | None) -> ta.Sequence[str] | None:
+    if v is None:
+        return None
+    return sorted(set(check.not_isinstance(v, str)))
+
+
+@dc.dataclass(frozen=True)
+class GitCacheDataSpec(CacheDataSpec):
+    url: str
+
+    branch: str | None = dc.field(default=None, kw_only=True)
+    rev: str | None = dc.field(default=None, kw_only=True)
+
+    subtrees: ta.Sequence[str] = dc.xfield(default=None, kw_only=True, coerce=_maybe_sorted_strs)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class HttpCacheDataSpec(CacheDataSpec):
+    url: str = dc.xfield(validate=lambda u: bool(urllib.parse.urlparse(u)))
+    file_name: str | None = None
+
+    @cached.property
+    def file_name_or_default(self) -> str:
+        if self.file_name is not None:
+            return self.file_name
+        return urllib.parse.urlparse(self.url).path.split('/')[-1]
+
+
+##
+
+
+def _repo_str(s: str) -> str:
+    u, r = check.non_empty_str(s).split('/')
+    check.non_empty_str(u)
+    check.non_empty_str(r)
+    return s
+
+
+@dc.dataclass(frozen=True)
+class GithubContentCacheDataSpec(CacheDataSpec):
+    repo: str = dc.field(validate=_repo_str)  # type: ignore
+    rev: str
+    files: lang.SequenceNotStr[str]
+
+
+##
+
+
+@lang.cached_function
+def _install_standard_marshalling() -> None:
+    specs_poly = msh.polymorphism_from_subclasses(CacheDataSpec)
+    msh.STANDARD_MARSHALER_FACTORIES[0:0] = [msh.PolymorphismMarshalerFactory(specs_poly)]
+    msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [msh.PolymorphismUnmarshalerFactory(specs_poly)]
+
+
+_install_standard_marshalling()

omdev-0.0.0.dev26/omdev/git.py

@@ -0,0 +1,62 @@
+import os.path
+import subprocess
+import typing as ta
+
+
+def clone_subtree(
+        *,
+        base_dir: str,
+        repo_url: str,
+        repo_dir: str,
+        branch: str | None = None,
+        rev: str | None = None,
+        repo_subtrees: ta.Sequence[str],
+) -> None:
+    if not bool(branch) ^ bool(rev):
+        raise ValueError('must set branch or rev')
+
+    if isinstance(repo_subtrees, str):
+        raise TypeError(repo_subtrees)
+
+    git_opts = [
+        '-c', 'advice.detachedHead=false',
+    ]
+
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'clone',
+            '-n',
+            '--depth=1',
+            '--filter=tree:0',
+            *(['-b', branch] if branch else []),
+            '--single-branch',
+            repo_url,
+            repo_dir,
+        ],
+        cwd=base_dir,
+    )
+
+    rd = os.path.join(base_dir, repo_dir)
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'sparse-checkout',
+            'set',
+            '--no-cone',
+            *repo_subtrees,
+        ],
+        cwd=rd,
+    )
+
+    subprocess.check_call(
+        [
+            'git',
+            *git_opts,
+            'checkout',
+            *([rev] if rev else []),
+        ],
+        cwd=rd,
+    )

{omdev-0.0.0.dev25 → omdev-0.0.0.dev26}/omdev/manifests.py

@@ -10,7 +10,7 @@
  - # @omlish-manifest \n _CACHE_MANIFEST = {'cache': {'name': 'llm', …
  - also can do prechecks!
 """
-# ruff: noqa: UP006
+# ruff: noqa: UP006 UP007
 # @omlish-lite
 import argparse
 import collections
@@ -22,10 +22,13 @@ import re
 import shlex
 import subprocess
 import sys
+import time
 import typing as ta
 
 from omlish.lite.cached import cached_nullary
 from omlish.lite.json import json_dumps_pretty
+from omlish.lite.logs import configure_standard_logging
+from omlish.lite.logs import log
 
 from . import findmagic
 
@@ -84,13 +87,17 @@ def build_module_manifests(
         base: str,
         *,
         shell_wrap: bool = True,
+        warn_threshold_s: ta.Optional[float] = 1.,
 ) -> ta.Sequence[Manifest]:
-    print((file, base))
+    log.info('Extracting manifests from file %s', file)
 
     if not file.endswith('.py'):
         raise Exception(file)
 
     mod_name = file.rpartition('.')[0].replace(os.sep, '.')
+    mod_base = mod_name.split('.')[0]
+    if mod_base != (first_dir := file.split(os.path.sep)[0]):
+        raise Exception(f'Unexpected module base: {mod_base=} != {first_dir=}')
 
     with open(os.path.join(base, file)) as f:
         src = f.read()
@@ -103,10 +110,10 @@ def build_module_manifests(
                 raise Exception(nl)
 
             origins.append(ManifestOrigin(
-                module=mod_name,
+                module='.'.join(['', *mod_name.split('.')[1:]]),
                 attr=m.groupdict()['name'],
 
-                file=file,
+                file=os.path.join(*os.path.split(file)[1:]),  # noqa
                 line=i + 1,
             ))
 
@@ -132,8 +139,15 @@ def build_module_manifests(
     if shell_wrap:
         args = ['sh', '-c', ' '.join(map(shlex.quote, args))]
 
+    start_time = time.time()
+
     subproc_out = subprocess.check_output(args)
 
+    end_time = time.time()
+
+    if warn_threshold_s is not None and (elapsed_time := (end_time - start_time)) >= warn_threshold_s:
+        log.warning('Manifest extraction took a long time: %s, %.2f s', file, elapsed_time)
+
     sp_lines = subproc_out.decode().strip().splitlines()
     if len(sp_lines) != 1:
         raise Exception('Unexpected subprocess output')
@@ -145,11 +159,17 @@ def build_module_manifests(
     out: ta.List[Manifest] = []
 
     for o in origins:
-        manifest = dct[o.attr]
+        value = dct[o.attr]
+
+        if not (
+                isinstance(value, ta.Mapping) and
+                all(isinstance(k, str) and k.startswith('$') and len(k) > 1 for k in value)
+        ):
+            raise TypeError(f'Manifests must be mapping of strings starting with $: {value!r}')
 
         out.append(Manifest(
             **dc.asdict(o),
-            value=manifest,
+            value=value,
         ))
 
     return out
@@ -202,9 +222,11 @@ if __name__ == '__main__':
                 write=args.write or False,
             )
             if not args.quiet:
-                print(json_dumps_pretty(ms))
+                print(json_dumps_pretty([dc.asdict(m) for m in ms]))
 
     def _main(argv=None) -> None:
+        configure_standard_logging('INFO')
+
         parser = argparse.ArgumentParser()
         subparsers = parser.add_subparsers()