omdev 0.0.0.dev24__py3-none-any.whl → 0.0.0.dev26__py3-none-any.whl

omdev/_manifests.json

@@ -0,0 +1 @@
+[]

omdev/amalg/amalg.py

@@ -29,13 +29,16 @@ import io
 import logging
 import os.path
 import re
+import textwrap
 import typing as ta
 
 import tokenize_rt as trt
 
 from omlish import check
 from omlish import collections as col
+from omlish import lang
 from omlish import logs
+from omlish.lite.runtime import REQUIRED_PYTHON_VERSION
 
 from .. import findmagic
 from .. import tokens as tks
@@ -91,6 +94,7 @@ def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
 
 STRIPPED_HEADER_MAGICS = [
     '# @omlish-lite',
+    '# @omlish-script',
 ]
 
 STRIPPED_HEADER_PATS = [findmagic.compile_magic_pat(m) for m in STRIPPED_HEADER_MAGICS]
@@ -268,6 +272,8 @@ class SrcFile:
     typings: ta.Sequence[Typing] = dc.field(repr=False)
     content_lines: ta.Sequence[Tokens] = dc.field(repr=False)
 
+    ruff_noqa: ta.AbstractSet[str] = dc.field(repr=False)
+
 
 def make_src_file(
         path: str,
@@ -283,6 +289,7 @@ def make_src_file(
     hls, cls = split_header_lines(lines)
 
     hls = strip_header_lines(hls)
+    rnls, hls = col.partition(hls, lambda l: tks.join_toks(l).startswith('# ruff: noqa: '))
 
     imps: list[Import] = []
     tys: list[Typing] = []
@@ -316,6 +323,8 @@ def make_src_file(
         imports=imps,
         typings=tys,
         content_lines=ctls,
+
+        ruff_noqa=set(lang.flatten(tks.join_toks(l).strip().split()[3:] for l in rnls)),  # noqa
     )
 
 
@@ -324,10 +333,11 @@ def make_src_file(
 
 SECTION_SEP = '#' * 40 + '\n'
 
-RUFF_DISABLES: ta.Sequence[str] = [
-    # 'UP006',  # non-pep585-annotation
-    # 'UP007',  # non-pep604-annotation
-]
+RUFF_DISABLES: ta.AbstractSet[str] = {
+    'UP006',  # non-pep585-annotation
+    'UP007',  # non-pep604-annotation
+    'UP036',  # outdated-version-block
+}
 
 OUTPUT_COMMENT = '# @omdev-amalg-output '
 SCAN_COMMENT = '# @omdev-amalg '
@@ -362,46 +372,70 @@ def gen_amalg(
 
     ##
 
+    hls = []
+
     mf = src_files[main_path]
     if mf.header_lines:
-        hls = [
+        hls.extend([
             hl
             for hlts in mf.header_lines
             if not (hl := tks.join_toks(hlts)).startswith(SCAN_COMMENT)
-        ]
-        if output_dir is not None:
-            ogf = os.path.relpath(main_path, output_dir)
-        else:
-            ogf = os.path.basename(main_path)
-        nhls = []
-        nhls.extend([
-            '#!/usr/bin/env python3\n',
-            '# noinspection DuplicatedCode\n',
-            '# @omlish-lite\n',
-            '# @omlish-script\n',
-            f'{OUTPUT_COMMENT.strip()} {ogf}\n',
         ])
-        hls = [*nhls, *hls]
-        out.write(''.join(hls))
 
-    if RUFF_DISABLES:
-        out.write(f'# ruff: noqa: {" ".join(RUFF_DISABLES)}\n')
+    if output_dir is not None:
+        ogf = os.path.relpath(main_path, output_dir)
+    else:
+        ogf = os.path.basename(main_path)
+
+    nhls = []
+    nhls.extend([
+        '#!/usr/bin/env python3\n',
+        '# noinspection DuplicatedCode\n',
+        '# @omlish-lite\n',
+        '# @omlish-script\n',
+        f'{OUTPUT_COMMENT.strip()} {ogf}\n',
+    ])
+
+    ruff_disables = sorted({
+        *lang.flatten(f.ruff_noqa for f in src_files.values()),
+        *RUFF_DISABLES,
+    })
+    if ruff_disables:
+        nhls.append(f'# ruff: noqa: {" ".join(sorted(ruff_disables))}\n')
+
+    hls = [*nhls, *hls]
+    out.write(''.join(hls))
 
     ##
 
     all_imps = [i for f in src_files.values() for i in f.imports]
     gl_imps = [i for i in all_imps if i.mod_path is None]
 
-    dct: dict = {}
+    dct: dict = {
+        ('sys', None, None): ['import sys\n'],
+    }
     for imp in gl_imps:
         dct.setdefault((imp.mod, imp.item, imp.as_), []).append(imp)
     for _, l in sorted(dct.items()):
-        out.write(tks.join_toks(l[0].toks))
+        il = l[0]
+        out.write(il if isinstance(il, str) else tks.join_toks(il.toks))
     if dct:
         out.write('\n\n')
 
     ##
 
+    out.write(SECTION_SEP)
+    out.write('\n\n')
+
+    out.write(textwrap.dedent(f"""
+    if sys.version_info < {REQUIRED_PYTHON_VERSION!r}:
+        raise OSError(
+            f'Requires python {REQUIRED_PYTHON_VERSION!r}, got {{sys.version_info}} from {{sys.executable}}')  # noqa
+    """).lstrip())
+    out.write('\n\n')
+
+    ##
+
     ts = list(col.toposort({  # noqa
         f.path: {mp for i in f.imports if (mp := i.mod_path) is not None}
         for f in src_files.values()
@@ -418,6 +452,9 @@ def gen_amalg(
             if ty.src not in tys:
                 tyd.setdefault(f.path, []).append(ty)
                 tys.add(ty.src)
+    if tys:
+        out.write(SECTION_SEP)
+        out.write('\n\n')
     for i, (sf, ftys) in enumerate(tyd.items()):
         f = src_files[sf]
         if i:

omdev/cexts/_boilerplate.cc

@@ -0,0 +1,82 @@
+// @omdev-cext
+#define PY_SSIZE_T_CLEAN
+#include "Python.h"
+#include "structmember.h"
+
+#include <unistd.h>
+
+//
+
+#define _MODULE_NAME "_boilerplate"
+#define _PACKAGE_NAME "omdev.cexts"
+#define _MODULE_FULL_NAME _PACKAGE_NAME "." _MODULE_NAME
+
+typedef struct boilerplate_state {
+} boilerplate_state;
+
+static inline boilerplate_state * get_boilerplate_state(PyObject *module)
+{
+    void *state = PyModule_GetState(module);
+    assert(state != NULL);
+    return (boilerplate_state *)state;
+}
+
+//
+
+PyDoc_STRVAR(boilerplate_doc, "boilerplate");
+
+static int boilerplate_exec(PyObject *module)
+{
+    get_boilerplate_state(module);
+    return 0;
+}
+
+static int boilerplate_traverse(PyObject *module, visitproc visit, void *arg)
+{
+    get_boilerplate_state(module);
+    return 0;
+}
+
+static int boilerplate_clear(PyObject *module)
+{
+    get_boilerplate_state(module);
+    return 0;
+}
+
+static void boilerplate_free(void *module)
+{
+    boilerplate_clear((PyObject *)module);
+}
+
+static PyMethodDef boilerplate_methods[] = {
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef_Slot boilerplate_slots[] = {
+    {Py_mod_exec, (void *) boilerplate_exec},
+// #if PY_VERSION_HEX >= 0x030D0000
+//     {Py_mod_gil, Py_MOD_GIL_NOT_USED},
+// #endif
+    {0, NULL}
+};
+
+static struct PyModuleDef boilerplate_module = {
+    .m_base = PyModuleDef_HEAD_INIT,
+    .m_name = _MODULE_NAME,
+    .m_doc = boilerplate_doc,
+    .m_size = sizeof(boilerplate_state),
+    .m_methods = boilerplate_methods,
+    .m_slots = boilerplate_slots,
+    .m_traverse = boilerplate_traverse,
+    .m_clear = boilerplate_clear,
+    .m_free = boilerplate_free,
+};
+
+extern "C" {
+
+PyMODINIT_FUNC PyInit__boilerplate(void)
+{
+    return PyModuleDef_Init(&boilerplate_module);
+}
+
+}

omdev/cexts/_distutils/LICENSE

@@ -0,0 +1,12 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

omdev/cexts/build.py

@@ -1,43 +1,87 @@
+import dataclasses as dc
 import os.path
 import sys
 import sysconfig
+import typing as ta
+
+from omlish import check
+from omlish import lang
 
 from . import _distutils as du
 
 
-def build_ext(
-        fullname: str,
-        src_path: str,
-) -> str:
+CPP_STD = 'c++20'
+
+
+@dc.dataclass(frozen=True)
+class BuildExt:
+    full_name: str
+    src_file: str
+
+    inplace: bool = dc.field(default=True, kw_only=True)
+    debug: bool = dc.field(default=True, kw_only=True)
+    force: bool = dc.field(default=False, kw_only=True)
+
+    dry_run: bool = dc.field(default=False, kw_only=True)
+    verbose: bool = dc.field(default=False, kw_only=True)
+
+    extra_src_files: lang.SequenceNotStr[str] | None = dc.field(default=None, kw_only=True)
+    include_dirs: lang.SequenceNotStr[str] | None = dc.field(default=None, kw_only=True)
+    compile_args: lang.SequenceNotStr[str] | None = dc.field(default=None, kw_only=True)
+    link_args: lang.SequenceNotStr[str] | None = dc.field(default=None, kw_only=True)
+    define_macros: ta.Sequence[tuple[str, str]] | None = dc.field(default=None, kw_only=True)
+    undef_macros: lang.SequenceNotStr[str] | None = dc.field(default=None, kw_only=True)
+
+    def __post_init__(self) -> None:
+        check.not_isinstance(self.compile_args, str)
+        check.not_isinstance(self.link_args, str)
+
+
+def build_ext(ext: BuildExt) -> str:
     extra_link_args: list[str] = []
     if sys.platform == 'darwin':
         extra_link_args.append('-Wl,-no_fixup_chains')
 
-    ext = du.Extension(
-        fullname,
-        sources=[src_path],
-        include_dirs=[os.path.dirname(src_path)],
+    du_ext = du.Extension(
+        ext.full_name,
+        sources=[
+            ext.src_file,
+            *(ext.extra_src_files or []),
+        ],
+        include_dirs=[
+            os.path.dirname(ext.src_file),
+            *(ext.include_dirs or []),
+        ],
         extra_compile_args=[
-            *(['-std=c++20'] if any(src_path.endswith(sf) for sf in ('cc', 'cpp')) else []),
+            *([f'-std={CPP_STD}'] if any(ext.src_file.endswith(sf) for sf in ('cc', 'cpp')) else []),
+            *(ext.compile_args or []),
+        ],
+        extra_link_args=[
+            *extra_link_args,
+            *(ext.link_args or []),
         ],
-        extra_link_args=extra_link_args,
-        undef_macros=['BARF'],
+        define_macros=(list(ext.define_macros) if ext.define_macros is not None else None),
+        undef_macros=(list(ext.undef_macros) if ext.undef_macros is not None else None),
     )
 
     cmd_obj = du.BuildExt(du.BuildExt.Options(
-        inplace=True,
-        debug=True,
+        inplace=ext.inplace,
+        debug=ext.debug,
+        force=ext.force,
+
+        dry_run=ext.dry_run,
+        verbose=ext.verbose,
     ))
-    cmd_obj.build_extension(ext)
+    cmd_obj.build_extension(du_ext)
 
-    so_path = os.path.join(
-        os.path.dirname(src_path),
+    so_file = os.path.join(
+        os.path.dirname(ext.src_file),
         ''.join([
-            fullname.rpartition('.')[2],
+            ext.full_name.rpartition('.')[2],
             '.',
             sysconfig.get_config_var('SOABI'),
             sysconfig.get_config_var('SHLIB_SUFFIX'),
         ]),
     )
 
-    return so_path
+    return so_file

omdev/cexts/importhook.py

@@ -42,7 +42,7 @@ class CextImportLoader(importlib.machinery.ExtensionFileLoader):
         super().__init__(module_name, filename)
 
     def create_module(self, spec: importlib.machinery.ModuleSpec) -> types.ModuleType:
-        so_path = build.build_ext(spec.name, check.non_empty_str(spec.origin))
+        so_path = build.build_ext(build.BuildExt(spec.name, check.non_empty_str(spec.origin)))
         self.path = so_path  # noqa
         spec.origin = so_path
         return super().create_module(spec)

omdev/datacache/__init__.py

@@ -0,0 +1,19 @@
+from .cache import (  # noqa
+    DataCache,
+)
+
+from .default import (  # noqa
+    default,
+    default_dir,
+)
+
+from .manifests import (  # noqa
+    CacheDataManifest,
+)
+
+from .specs import (  # noqa
+    CacheDataSpec,
+    GitCacheDataSpec,
+    GithubContentCacheDataSpec,
+    HttpCacheDataSpec,
+)

omdev/datacache/cache.py

@@ -0,0 +1,149 @@
+"""
+TODO:
+ - huggingface_hub
+ - postprocessing?
+  - unarchive
+ - stupid little progress bars
+ - return file path for single files
+  - thus, HttpSpec.url has derive=lambda url: ...
+"""
+import logging
+import os.path
+import shutil
+import tempfile
+import urllib.parse
+import urllib.request
+
+from omlish import check
+from omlish import lang
+from omlish import marshal as msh
+from omlish.formats import json
+
+from .. import git
+from .manifests import CacheDataManifest
+from .specs import CacheDataSpec
+from .specs import GitCacheDataSpec
+from .specs import GithubContentCacheDataSpec
+from .specs import HttpCacheDataSpec
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class DataCache:
+    def __init__(self, base_dir: str) -> None:
+        super().__init__()
+        self._base_dir = base_dir
+
+        self._items_dir = os.path.join(base_dir, 'items')
+
+    def _fetch_url(self, url: str, out_file: str) -> None:
+        log.info('Fetching url: %s -> %s', url, out_file)
+
+        urllib.request.urlretrieve(url, out_file)  # noqa
+
+    def _fetch_into(self, spec: CacheDataSpec, data_dir: str) -> None:
+        log.info('Fetching spec: %s %r', spec.digest, spec)
+
+        if isinstance(spec, HttpCacheDataSpec):
+            self._fetch_url(spec.url, os.path.join(data_dir, spec.file_name_or_default))
+
+        elif isinstance(spec, GithubContentCacheDataSpec):
+            for repo_file in spec.files:
+                out_file = os.path.join(data_dir, repo_file)
+                if not os.path.abspath(out_file).startswith(os.path.abspath(data_dir)):
+                    raise RuntimeError(out_file)  # noqa
+
+                url = f'https://raw.githubusercontent.com/{spec.repo}/{spec.rev}/{repo_file}'
+                os.makedirs(os.path.dirname(out_file), exist_ok=True)
+                self._fetch_url(url, os.path.join(data_dir, out_file))
+
+        elif isinstance(spec, GitCacheDataSpec):
+            if not spec.subtrees:
+                raise NotImplementedError
+
+            tmp_dir = tempfile.mkdtemp()
+
+            log.info('Cloning git repo: %s -> %s', spec.url, tmp_dir)
+
+            git.clone_subtree(
+                base_dir=tmp_dir,
+                repo_url=spec.url,
+                repo_dir='data',
+                branch=spec.branch,
+                rev=spec.rev,
+                repo_subtrees=spec.subtrees,
+            )
+
+            repo_dir = os.path.join(tmp_dir, 'data')
+            if not os.path.isdir(repo_dir):
+                raise RuntimeError(repo_dir)
+
+            git_dir = os.path.join(repo_dir, '.git')
+            if not os.path.isdir(git_dir):
+                raise RuntimeError(git_dir)
+            shutil.rmtree(git_dir)
+
+            os.rmdir(data_dir)
+            os.rename(repo_dir, data_dir)
+
+        else:
+            raise TypeError(spec)
+
+    def _return_val(self, spec: CacheDataSpec, data_dir: str) -> str:
+        check.state(os.path.isdir(data_dir))
+
+        if isinstance(spec, HttpCacheDataSpec):
+            data_file = os.path.join(data_dir, spec.file_name_or_default)
+            if not os.path.isfile(data_file):
+                raise RuntimeError(data_file)  # noqa
+            return data_file
+
+        else:
+            return data_dir
+
+    def get(self, spec: CacheDataSpec) -> str:
+        os.makedirs(self._items_dir, exist_ok=True)
+
+        #
+
+        item_dir = os.path.join(self._items_dir, spec.digest)
+        if os.path.isdir(item_dir):
+            data_dir = os.path.join(item_dir, 'data')
+            return self._return_val(spec, data_dir)
+
+        #
+
+        tmp_dir = tempfile.mkdtemp()
+
+        #
+
+        fetch_dir = os.path.join(tmp_dir, 'data')
+        os.mkdir(fetch_dir)
+
+        start_at = lang.utcnow()
+        self._fetch_into(spec, fetch_dir)
+        end_at = lang.utcnow()
+
+        #
+
+        manifest = CacheDataManifest(
+            spec,
+            start_at=start_at,
+            end_at=end_at,
+        )
+        manifest_json = json.dumps_pretty(msh.marshal(manifest))
+
+        manifest_file = os.path.join(tmp_dir, 'manifest.json')
+        with open(manifest_file, 'w') as f:
+            f.write(manifest_json)
+
+        ##
+
+        os.rename(tmp_dir, item_dir)
+
+        data_dir = os.path.join(item_dir, 'data')
+        return self._return_val(spec, data_dir)

omdev/datacache/consts.py

@@ -0,0 +1 @@
+MARSHAL_VERSION = 0

omdev/datacache/default.py

@@ -0,0 +1,51 @@
+import os.path
+
+from omlish import lang
+
+from .cache import DataCache
+
+
+##
+
+
+@lang.cached_function(lock=True)
+def default_dir() -> str:
+    return os.path.expanduser('~/.cache/omlish/data')
+
+
+@lang.cached_function(lock=True)
+def default() -> DataCache:
+    return DataCache(default_dir())
+
+
+def _main() -> None:
+    from omlish import logs
+
+    logs.configure_standard_logging('INFO')
+
+    #
+
+    from .specs import GitCacheDataSpec
+    from .specs import GithubContentCacheDataSpec
+    from .specs import HttpCacheDataSpec
+
+    for spec in [
+        GitCacheDataSpec(
+            'https://github.com/wrmsr/deep_learning_cookbook',
+            rev='138a99b09ffa3a728d261e461440f029e512ac93',
+            subtrees=['data/wp_movies_10k.ndjson'],
+        ),
+        GithubContentCacheDataSpec(
+            'karpathy/char-rnn',
+            'master',
+            ['data/tinyshakespeare/input.txt'],
+        ),
+        HttpCacheDataSpec('https://github.com/VanushVaswani/keras_mnistm/releases/download/1.0/keras_mnistm.pkl.gz'),
+    ]:
+        print(spec)
+        for _ in range(2):
+            print(default().get(spec))
+
+
+if __name__ == '__main__':
+    _main()

omdev/datacache/manifests.py

@@ -0,0 +1,40 @@
+import datetime
+
+from omlish import __about__ as about
+from omlish import cached
+from omlish import dataclasses as dc
+
+from ..revisions import get_git_revision
+from .consts import MARSHAL_VERSION
+from .specs import CacheDataSpec
+
+
+##
+
+
+@cached.function
+def _lib_revision() -> str | None:
+    if (rev := about.__revision__) is not None:
+        return rev  # type: ignore
+
+    return get_git_revision()
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class CacheDataManifest:
+    spec: CacheDataSpec
+
+    start_at: datetime.datetime = dc.field(kw_only=True)
+    end_at: datetime.datetime = dc.field(kw_only=True)
+
+    lib_version: str = dc.field(default_factory=lambda: about.__version__, kw_only=True)
+    lib_revision: str = dc.field(default_factory=_lib_revision, kw_only=True)
+
+    marshal_version: int = dc.field(default=MARSHAL_VERSION, kw_only=True)
+
+    @dc.validate
+    def _validate_marshal_versions(self) -> bool:
+        return self.marshal_version == self.spec.marshal_version