omdev 0.0.0.dev29__py3-none-any.whl → 0.0.0.dev31__py3-none-any.whl

omdev/cache/data/__init__.py

@@ -1,3 +1,7 @@
+from .actions import (  # noqa
+    ExtractAction,
+)
+
 from .cache import (  # noqa
     Cache,
 )
@@ -8,12 +12,12 @@ from .defaults import (  # noqa
 )
 
 from .manifests import (  # noqa
-    CacheDataManifest,
+    Manifest,
 )
 
 from .specs import (  # noqa
-    CacheDataSpec,
-    GitCacheDataSpec,
-    GithubContentCacheDataSpec,
-    HttpCacheDataSpec,
+    Spec,
+    GitSpec,
+    GithubContentSpec,
+    UrlSpec,
 )

omdev/cache/data/actions.py

@@ -6,6 +6,7 @@ TODO:
 """
 import typing as ta
 
+from omlish import check
 from omlish import dataclasses as dc
 from omlish import lang
 from omlish import marshal as msh
@@ -24,9 +25,14 @@ class Action(lang.Abstract, lang.Sealed):
 ##
 
 
+def _non_empty_strs(v: ta.Sequence[str]) -> ta.Sequence[str]:
+    return [check.non_empty_str(s) for s in ([v] if isinstance(v, str) else v)]
+
+
 @dc.dataclass(frozen=True)
 class ExtractAction(Action, lang.Final):
-    files: ta.Sequence[str]
+    files: ta.Sequence[str] = dc.xfield(coerce=_non_empty_strs)
+    keep_archive: bool = False
 
 
 ##
@@ -34,7 +40,7 @@ class ExtractAction(Action, lang.Final):
 
 @lang.cached_function
 def _install_standard_marshalling() -> None:
-    actions_poly = msh.polymorphism_from_subclasses(Action)
+    actions_poly = msh.polymorphism_from_subclasses(Action, naming=msh.Naming.SNAKE, strip_suffix=True)
     msh.STANDARD_MARSHALER_FACTORIES[0:0] = [msh.PolymorphismMarshalerFactory(actions_poly)]
     msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [msh.PolymorphismUnmarshalerFactory(actions_poly)]
 

omdev/cache/data/cache.py

@@ -2,13 +2,18 @@
 TODO:
  - mirrors
  - huggingface_hub
+  - datasets
+ - verify md5 (action)
  - stupid little progress bars
  - groups of multiple files downloaded - 'spec set'? idk
   - torchvision.datasets.FashionMNIST
+ - chaining? or is this compcache..
+ - download resume ala hf_hub
 """
 import logging
 import os.path
 import shutil
+import subprocess
 import tempfile
 import urllib.parse
 import urllib.request
@@ -16,14 +21,17 @@ import urllib.request
 from omlish import check
 from omlish import lang
 from omlish import marshal as msh
+from omlish import os as osu
 from omlish.formats import json
 
 from ... import git
-from .manifests import CacheDataManifest
-from .specs import CacheDataSpec
-from .specs import GitCacheDataSpec
-from .specs import GithubContentCacheDataSpec
-from .specs import HttpCacheDataSpec
+from .actions import Action
+from .actions import ExtractAction
+from .manifests import Manifest
+from .specs import GithubContentSpec
+from .specs import GitSpec
+from .specs import Spec
+from .specs import UrlSpec
 
 
 log = logging.getLogger(__name__)
@@ -39,18 +47,20 @@ class Cache:
 
         self._items_dir = os.path.join(base_dir, 'items')
 
+    #
+
     def _fetch_url(self, url: str, out_file: str) -> None:
         log.info('Fetching url: %s -> %s', url, out_file)
 
         urllib.request.urlretrieve(url, out_file)  # noqa
 
-    def _fetch_into(self, spec: CacheDataSpec, data_dir: str) -> None:
+    def _fetch_into(self, spec: Spec, data_dir: str) -> None:
         log.info('Fetching spec: %s %r -> %s', spec.digest, spec, data_dir)
 
-        if isinstance(spec, HttpCacheDataSpec):
+        if isinstance(spec, UrlSpec):
             self._fetch_url(spec.url, os.path.join(data_dir, spec.file_name_or_default))
 
-        elif isinstance(spec, GithubContentCacheDataSpec):
+        elif isinstance(spec, GithubContentSpec):
             for repo_file in spec.files:
                 out_file = os.path.join(data_dir, repo_file)
                 if not os.path.abspath(out_file).startswith(os.path.abspath(data_dir)):
@@ -60,7 +70,7 @@ class Cache:
                 os.makedirs(os.path.dirname(out_file), exist_ok=True)
                 self._fetch_url(url, os.path.join(data_dir, out_file))
 
-        elif isinstance(spec, GitCacheDataSpec):
+        elif isinstance(spec, GitSpec):
             if not spec.subtrees:
                 raise NotImplementedError
 
@@ -92,38 +102,52 @@ class Cache:
         else:
             raise TypeError(spec)
 
-    def _return_val(self, spec: CacheDataSpec, data_dir: str) -> str:
+    def _perform_action(self, action: Action, data_dir: str) -> None:
+        if isinstance(action, ExtractAction):
+            for f in action.files:
+                file = os.path.join(data_dir, f)
+                if not os.path.isfile(file):
+                    raise Exception(f'Not file: {file}')
+
+                if file.endswith('.tar.gz'):
+                    subprocess.check_call(['tar', 'xzf', file], cwd=data_dir)
+
+                elif file.endswith('.zip'):
+                    subprocess.check_call(['unzip', file], cwd=data_dir)
+
+                else:
+                    raise Exception(f'Unhandled archive extension: {file}')
+
+                if not action.keep_archive:
+                    os.unlink(file)
+
+        else:
+            raise TypeError(action)
+
+    def _return_val(self, spec: Spec, data_dir: str) -> str:
         check.state(os.path.isdir(data_dir))
 
-        if isinstance(spec, HttpCacheDataSpec):
-            data_file = os.path.join(data_dir, spec.file_name_or_default)
-            if not os.path.isfile(data_file):
-                raise RuntimeError(data_file)  # noqa
-            return data_file
+        if any(isinstance(a, ExtractAction) for a in spec.actions):
+            return data_dir
 
-        elif isinstance(spec, GithubContentCacheDataSpec):
+        single_file: str
+        if isinstance(spec, UrlSpec):
+            single_file = os.path.join(data_dir, spec.file_name_or_default)
+
+        elif isinstance(spec, GithubContentSpec):
             if len(spec.files) != 1:
                 return data_dir
-            data_file = os.path.join(data_dir, check.single(spec.files))
-            if not os.path.isfile(data_file):
-                raise RuntimeError(data_file)  # noqa
-            return data_file
+            single_file = os.path.join(data_dir, check.single(spec.files))
 
         else:
             return data_dir
 
-    def get(self, spec: CacheDataSpec) -> str:
-        os.makedirs(self._items_dir, exist_ok=True)
+        if not os.path.isfile(single_file):
+            raise RuntimeError(single_file)  # noqa
 
-        #
-
-        item_dir = os.path.join(self._items_dir, spec.digest)
-        if os.path.isdir(item_dir):
-            data_dir = os.path.join(item_dir, 'data')
-            return self._return_val(spec, data_dir)
-
-        #
+        return single_file
 
+    def _fetch_item(self, spec: Spec, item_dir: str) -> None:
         tmp_dir = tempfile.mkdtemp()
 
         #
@@ -133,11 +157,13 @@ class Cache:
 
         start_at = lang.utcnow()
         self._fetch_into(spec, fetch_dir)
+        for action in spec.actions:
+            self._perform_action(action, fetch_dir)
         end_at = lang.utcnow()
 
         #
 
-        manifest = CacheDataManifest(
+        manifest = Manifest(
             spec,
             start_at=start_at,
             end_at=end_at,
@@ -159,5 +185,14 @@ class Cache:
 
         shutil.move(tmp_dir, item_dir)
 
+    def get(self, spec: Spec) -> str:
+        os.makedirs(self._items_dir, exist_ok=True)
+
+        item_dir = os.path.join(self._items_dir, spec.digest)
+        if not os.path.isdir(item_dir):
+            self._fetch_item(spec, item_dir)
+
+        osu.touch(os.path.join(item_dir, 'accessed'))
+
         data_dir = os.path.join(item_dir, 'data')
         return self._return_val(spec, data_dir)

omdev/cache/data/manifests.py

@@ -6,7 +6,7 @@ from omlish import dataclasses as dc
 
 from ...git import get_git_revision
 from .consts import SERIALIZATION_VERSION
-from .specs import CacheDataSpec
+from .specs import Spec
 
 
 ##
@@ -24,8 +24,8 @@ def _lib_revision() -> str | None:
 
 
 @dc.dataclass(frozen=True)
-class CacheDataManifest:
-    spec: CacheDataSpec
+class Manifest:
+    spec: Spec
 
     start_at: datetime.datetime = dc.field(kw_only=True)
     end_at: datetime.datetime = dc.field(kw_only=True)

omdev/cache/data/specs.py

@@ -10,6 +10,7 @@ from omlish import lang
 from omlish import marshal as msh
 from omlish.formats import json
 
+from .actions import Action
 from .consts import SERIALIZATION_VERSION
 
 
@@ -17,12 +18,14 @@ from .consts import SERIALIZATION_VERSION
 
 
 @dc.dataclass(frozen=True)
-class CacheDataSpec(lang.Abstract, lang.Sealed):
+class Spec(lang.Abstract, lang.Sealed):
     serialization_version: int = dc.field(default=SERIALIZATION_VERSION, kw_only=True)
 
+    actions: ta.Sequence[Action] = dc.field(default=(), kw_only=True)
+
     @cached.property
     def json(self) -> str:
-        return json.dumps_compact(msh.marshal(self, CacheDataSpec), sort_keys=True)
+        return json.dumps_compact(msh.marshal(self, Spec), sort_keys=True)
 
     @cached.property
     def digest(self) -> str:
@@ -39,7 +42,7 @@ def _maybe_sorted_strs(v: ta.Iterable[str] | None) -> ta.Sequence[str] | None:
 
 
 @dc.dataclass(frozen=True)
-class GitCacheDataSpec(CacheDataSpec):
+class GitSpec(Spec):
     url: str
 
     branch: str | None = dc.field(default=None, kw_only=True)
@@ -52,7 +55,7 @@ class GitCacheDataSpec(CacheDataSpec):
 
 
 @dc.dataclass(frozen=True)
-class HttpCacheDataSpec(CacheDataSpec):
+class UrlSpec(Spec):
     url: str = dc.xfield(validate=lambda u: bool(urllib.parse.urlparse(u)))
     file_name: str | None = None
 
@@ -74,7 +77,7 @@ def _repo_str(s: str) -> str:
 
 
 @dc.dataclass(frozen=True)
-class GithubContentCacheDataSpec(CacheDataSpec):
+class GithubContentSpec(Spec):
     repo: str = dc.field(validate=_repo_str)  # type: ignore
     rev: str
     files: lang.SequenceNotStr[str]
@@ -85,7 +88,7 @@ class GithubContentCacheDataSpec(CacheDataSpec):
 
 @lang.cached_function
 def _install_standard_marshalling() -> None:
-    specs_poly = msh.polymorphism_from_subclasses(CacheDataSpec)
+    specs_poly = msh.polymorphism_from_subclasses(Spec, naming=msh.Naming.SNAKE, strip_suffix=True)
     msh.STANDARD_MARSHALER_FACTORIES[0:0] = [msh.PolymorphismMarshalerFactory(specs_poly)]
     msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [msh.PolymorphismUnmarshalerFactory(specs_poly)]
 

omdev/interp/pyenv.py

@@ -3,7 +3,14 @@ TODO:
  - custom tags
  - optionally install / upgrade pyenv itself
  - new vers dont need these custom mac opts, only run on old vers
-"""
+
+TODO opts:
+ - --enable-loadable-sqlite-extensions LDFLAGS="-L/opt/homebrew/opt/sqlite/lib" CPPFLAGS="-I/opt/homebrew/opt/sqlite/include"
+ - --enable-shared
+ - --enable-optimizations
+ - --enable-profiling ?
+ - --enable-ipv6 ?
+"""  # noqa
 # ruff: noqa: UP006 UP007
 import abc
 import dataclasses as dc

omdev/manifests.py

@@ -1,20 +1,15 @@
 """
-!!! manifests! get-manifest, .manifest.json
- - dumb dicts, root keys are 'types'
- - get put in _manifest.py, root level dict or smth
-  - IMPORT files w comment
-  - comment must immediately precede a global val setter
-  - val is grabbed from imported module dict by name
-  - value is repr'd somehow (roundtrip checked) (naw, json lol)
-  - dumped in _manifest.py
- - # @omlish-manifest \n _CACHE_MANIFEST = {'cache': {'name': 'llm', …
- - also can do prechecks!
+TODO:
+ - split, cli.py / types.py
+ - opt to load only specified types
+ - prechecks
 """
 # ruff: noqa: UP006 UP007
 # @omlish-lite
 import argparse
 import collections
 import dataclasses as dc
+import importlib
 import inspect
 import json
 import os.path
@@ -50,12 +45,50 @@ class Manifest(ManifestOrigin):
     value: ta.Any
 
 
+def load_manifest_entry(
+        entry: ta.Mapping[str, ta.Any],
+        *,
+        raw_values: bool = False,
+) -> Manifest:
+    manifest = Manifest(**entry)
+
+    [(key, value_dct)] = manifest.value.items()
+    if not key.startswith('$'):
+        raise Exception(f'Bad key: {key}')
+
+    if not raw_values:
+        parts = key[1:].split('.')
+        pos = next(i for i, p in enumerate(parts) if p[0].isupper())
+        mod_name = '.'.join(parts[:pos])
+        mod = importlib.import_module(mod_name)
+
+        obj: ta.Any = mod
+        for ca in parts[pos:]:
+            obj = getattr(obj, ca)
+        cls = obj
+        if not isinstance(cls, type):
+            raise TypeError(cls)
+
+        if not dc.is_dataclass(cls):
+            raise TypeError(cls)
+        obj = cls(**value_dct)  # noqa
+
+        manifest = dc.replace(manifest, value=obj)
+
+    return manifest
+
+
+##
+
+
 MANIFEST_MAGIC = '# @omlish-manifest'
 
 _MANIFEST_GLOBAL_PAT = re.compile(r'^(?P<name>[A-Za-z_][A-Za-z0-9_]*)\s*=.*')
 
 
 def _dump_module_manifests(spec: str, *attrs: str) -> None:
+    import collections.abc
+    import dataclasses as dc  # noqa
     import importlib
     import json
 
@@ -65,13 +98,35 @@ def _dump_module_manifests(spec: str, *attrs: str) -> None:
     for attr in attrs:
         manifest = getattr(mod, attr)
 
-        manifest_json = json.dumps(manifest)
-        rt_manifest = json.loads(manifest_json)
+        if dc.is_dataclass(manifest):
+            cls = type(manifest)
+            manifest_json = json.dumps(dc.asdict(manifest))  # type: ignore
+            manifest_dct = json.loads(manifest_json)
+
+            rt_manifest = cls(**manifest_dct)  # type: ignore
+            if rt_manifest != manifest:
+                raise Exception(f'Manifest failed to roundtrip: {manifest} -> {manifest_dct} != {rt_manifest}')
+
+            key = f'${cls.__module__}.{cls.__qualname__}'
+            out[attr] = {key: manifest_dct}
+
+        elif isinstance(manifest, collections.abc.Mapping):
+            [(key, manifest_dct)] = manifest.items()
+            if not key.startswith('$'):  # noqa
+                raise Exception(f'Bad key: {key}')
+
+            if not isinstance(manifest_dct, collections.abc.Mapping):
+                raise Exception(f'Bad value: {manifest_dct}')
 
-        if rt_manifest != manifest:
-            raise Exception(f'Manifest failed to roundtrip: {manifest} != {rt_manifest}')
+            manifest_json = json.dumps(manifest_dct)
+            rt_manifest_dct = json.loads(manifest_json)
+            if manifest_dct != rt_manifest_dct:
+                raise Exception(f'Manifest failed to roundtrip: {manifest_dct} != {rt_manifest_dct}')
 
-        out[attr] = rt_manifest
+            out[attr] = {key: manifest_dct}
+
+        else:
+            raise TypeError(f'Manifest must be dataclass or mapping: {manifest!r}')
 
     out_json = json.dumps(out, indent=None, separators=(',', ':'))
     print(out_json)
@@ -113,7 +168,7 @@ def build_module_manifests(
                 module='.'.join(['', *mod_name.split('.')[1:]]),
                 attr=m.groupdict()['name'],
 
-                file=os.path.join(*os.path.split(file)[1:]),  # noqa
+                file=file,
                 line=i + 1,
             ))
 
@@ -205,8 +260,30 @@ def build_package_manifests(
 ##
 
 
+def check_package_manifests(
+        name: str,
+        base: str,
+) -> None:
+    pkg_dir = os.path.join(base, name)
+    if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
+        raise Exception(pkg_dir)
+
+    manifests_file = os.path.join(pkg_dir, '.manifests.json')
+    if not os.path.isfile(manifests_file):
+        raise Exception(f'No manifests file: {manifests_file}')
+
+    with open(manifests_file) as f:
+        manifests_json = json.load(f)
+
+    for entry in manifests_json:
+        load_manifest_entry(entry)
+
+
+##
+
+
 if __name__ == '__main__':
-    def _gen_cmd(args) -> None:
+    def _get_base(args) -> str:
         if args.base is not None:
             base = args.base
         else:
@@ -214,6 +291,10 @@ if __name__ == '__main__':
         base = os.path.abspath(base)
         if not os.path.isdir(base):
             raise RuntimeError(base)
+        return base
+
+    def _gen_cmd(args) -> None:
+        base = _get_base(args)
 
         for pkg in args.package:
             ms = build_package_manifests(
@@ -224,6 +305,15 @@ if __name__ == '__main__':
             if not args.quiet:
                 print(json_dumps_pretty([dc.asdict(m) for m in ms]))
 
+    def _check_cmd(args) -> None:
+        base = _get_base(args)
+
+        for pkg in args.package:
+            check_package_manifests(
+                pkg,
+                base,
+            )
+
     def _main(argv=None) -> None:
         configure_standard_logging('INFO')
 
@@ -235,9 +325,13 @@ if __name__ == '__main__':
         parser_gen.add_argument('-w', '--write', action='store_true')
         parser_gen.add_argument('-q', '--quiet', action='store_true')
         parser_gen.add_argument('package', nargs='*')
-
         parser_gen.set_defaults(func=_gen_cmd)
 
+        parser_check = subparsers.add_parser('check')
+        parser_check.add_argument('-b', '--base')
+        parser_check.add_argument('package', nargs='*')
+        parser_check.set_defaults(func=_check_cmd)
+
         args = parser.parse_args(argv)
         if not getattr(args, 'func', None):
             parser.print_help()

omdev/pyproject/cli.py

@@ -8,9 +8,8 @@ TODO:
  - build / package / publish / version roll
   - {pkg_name: [src_dirs]}, default excludes, generate MANIFST.in, ...
  - env vars - PYTHONPATH
- - optional uv backend
 
-lookit:
+See:
  - https://pdm-project.org/en/latest/
  - https://rye.astral.sh/philosophy/
  - https://github.com/indygreg/python-build-standalone/blob/main/pythonbuild/cpython.py
@@ -158,6 +157,11 @@ class Venv:
         if (sr := self._cfg.requires):
             rr = RequirementsRewriter(self._name)
             reqs = [rr.rewrite(req) for req in sr]
+
+            # TODO: automatically try slower uv download when it fails? lol
+            #   Caused by: Failed to download distribution due to network timeout. Try increasing UV_HTTP_TIMEOUT (current value: 30s).  # noqa
+            #   UV_CONCURRENT_DOWNLOADS=4 UV_HTTP_TIMEOUT=3600
+
             subprocess_check_call(
                 ve,
                 '-m',

omdev/scripts/interp.py

@@ -13,6 +13,7 @@ TODO:
 import abc
 import argparse
 import collections
+import contextlib
 import dataclasses as dc
 import datetime
 import functools
@@ -1352,6 +1353,24 @@ class StandardLogHandler(ProxyLogHandler):
 ##
 
 
+@contextlib.contextmanager
+def _locking_logging_module_lock() -> ta.Iterator[None]:
+    if hasattr(logging, '_acquireLock'):
+        logging._acquireLock()  # noqa
+        try:
+            yield
+        finally:
+            logging._releaseLock()  # type: ignore  # noqa
+
+    elif hasattr(logging, '_lock'):
+        # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
+        with logging._lock:  # noqa
+            yield
+
+    else:
+        raise Exception("Can't find lock in logging module")
+
+
 def configure_standard_logging(
         level: ta.Union[int, str] = logging.INFO,
         *,
@@ -1359,8 +1378,7 @@ def configure_standard_logging(
         target: ta.Optional[logging.Logger] = None,
         force: bool = False,
 ) -> ta.Optional[StandardLogHandler]:
-    logging._acquireLock()  # type: ignore  # noqa
-    try:
+    with _locking_logging_module_lock():
         if target is None:
             target = logging.root
 
@@ -1400,9 +1418,6 @@ def configure_standard_logging(
 
         return StandardLogHandler(handler)
 
-    finally:
-        logging._releaseLock()  # type: ignore  # noqa
-
 
 ########################################
 # ../../../omlish/lite/runtime.py
@@ -1783,7 +1798,14 @@ TODO:
  - custom tags
  - optionally install / upgrade pyenv itself
  - new vers dont need these custom mac opts, only run on old vers
-"""
+
+TODO opts:
+ - --enable-loadable-sqlite-extensions LDFLAGS="-L/opt/homebrew/opt/sqlite/lib" CPPFLAGS="-I/opt/homebrew/opt/sqlite/include"
+ - --enable-shared
+ - --enable-optimizations
+ - --enable-profiling ?
+ - --enable-ipv6 ?
+"""  # noqa
 
 
 ##