omdev 0.0.0.dev28__py3-none-any.whl → 0.0.0.dev30__py3-none-any.whl

omdev/cache/data/__init__.py

@@ -1,5 +1,9 @@
+from .actions import (  # noqa
+    ExtractAction,
+)
+
 from .cache import (  # noqa
-    DataCache,
+    Cache,
 )
 
 from .defaults import (  # noqa
@@ -8,12 +12,12 @@ from .defaults import (  # noqa
 )
 
 from .manifests import (  # noqa
-    CacheDataManifest,
+    Manifest,
 )
 
 from .specs import (  # noqa
-    CacheDataSpec,
-    GitCacheDataSpec,
-    GithubContentCacheDataSpec,
-    HttpCacheDataSpec,
+    Spec,
+    GitSpec,
+    GithubContentSpec,
+    UrlSpec,
 )

omdev/cache/data/actions.py

@@ -6,6 +6,7 @@ TODO:
 """
 import typing as ta
 
+from omlish import check
 from omlish import dataclasses as dc
 from omlish import lang
 from omlish import marshal as msh
@@ -24,9 +25,14 @@ class Action(lang.Abstract, lang.Sealed):
 ##
 
 
+def _non_empty_strs(v: ta.Sequence[str]) -> ta.Sequence[str]:
+    return [check.non_empty_str(s) for s in ([v] if isinstance(v, str) else v)]
+
+
 @dc.dataclass(frozen=True)
 class ExtractAction(Action, lang.Final):
-    files: ta.Sequence[str]
+    files: ta.Sequence[str] = dc.xfield(coerce=_non_empty_strs)
+    keep_archive: bool = False
 
 
 ##
@@ -34,7 +40,7 @@ class ExtractAction(Action, lang.Final):
 
 @lang.cached_function
 def _install_standard_marshalling() -> None:
-    actions_poly = msh.polymorphism_from_subclasses(Action)
+    actions_poly = msh.polymorphism_from_subclasses(Action, naming=msh.Naming.SNAKE, strip_suffix=True)
     msh.STANDARD_MARSHALER_FACTORIES[0:0] = [msh.PolymorphismMarshalerFactory(actions_poly)]
     msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [msh.PolymorphismUnmarshalerFactory(actions_poly)]
 

omdev/cache/data/cache.py

@@ -2,13 +2,18 @@
 TODO:
  - mirrors
  - huggingface_hub
+  - datasets
+ - verify md5 (action)
  - stupid little progress bars
  - groups of multiple files downloaded - 'spec set'? idk
   - torchvision.datasets.FashionMNIST
+ - chaining? or is this compcache..
+ - download resume ala hf_hub
 """
 import logging
 import os.path
 import shutil
+import subprocess
 import tempfile
 import urllib.parse
 import urllib.request
@@ -16,14 +21,17 @@ import urllib.request
 from omlish import check
 from omlish import lang
 from omlish import marshal as msh
+from omlish import os as osu
 from omlish.formats import json
 
 from ... import git
-from .manifests import CacheDataManifest
-from .specs import CacheDataSpec
-from .specs import GitCacheDataSpec
-from .specs import GithubContentCacheDataSpec
-from .specs import HttpCacheDataSpec
+from .actions import Action
+from .actions import ExtractAction
+from .manifests import Manifest
+from .specs import GithubContentSpec
+from .specs import GitSpec
+from .specs import Spec
+from .specs import UrlSpec
 
 
 log = logging.getLogger(__name__)
@@ -32,25 +40,27 @@ log = logging.getLogger(__name__)
 ##
 
 
-class DataCache:
+class Cache:
     def __init__(self, base_dir: str) -> None:
         super().__init__()
         self._base_dir = base_dir
 
         self._items_dir = os.path.join(base_dir, 'items')
 
+    #
+
     def _fetch_url(self, url: str, out_file: str) -> None:
         log.info('Fetching url: %s -> %s', url, out_file)
 
         urllib.request.urlretrieve(url, out_file)  # noqa
 
-    def _fetch_into(self, spec: CacheDataSpec, data_dir: str) -> None:
+    def _fetch_into(self, spec: Spec, data_dir: str) -> None:
         log.info('Fetching spec: %s %r -> %s', spec.digest, spec, data_dir)
 
-        if isinstance(spec, HttpCacheDataSpec):
+        if isinstance(spec, UrlSpec):
             self._fetch_url(spec.url, os.path.join(data_dir, spec.file_name_or_default))
 
-        elif isinstance(spec, GithubContentCacheDataSpec):
+        elif isinstance(spec, GithubContentSpec):
             for repo_file in spec.files:
                 out_file = os.path.join(data_dir, repo_file)
                 if not os.path.abspath(out_file).startswith(os.path.abspath(data_dir)):
@@ -60,7 +70,7 @@ class DataCache:
                 os.makedirs(os.path.dirname(out_file), exist_ok=True)
                 self._fetch_url(url, os.path.join(data_dir, out_file))
 
-        elif isinstance(spec, GitCacheDataSpec):
+        elif isinstance(spec, GitSpec):
             if not spec.subtrees:
                 raise NotImplementedError
 
@@ -92,38 +102,52 @@ class DataCache:
         else:
             raise TypeError(spec)
 
-    def _return_val(self, spec: CacheDataSpec, data_dir: str) -> str:
+    def _perform_action(self, action: Action, data_dir: str) -> None:
+        if isinstance(action, ExtractAction):
+            for f in action.files:
+                file = os.path.join(data_dir, f)
+                if not os.path.isfile(file):
+                    raise Exception(f'Not file: {file}')
+
+                if file.endswith('.tar.gz'):
+                    subprocess.check_call(['tar', 'xzf', file], cwd=data_dir)
+
+                elif file.endswith('.zip'):
+                    subprocess.check_call(['unzip', file], cwd=data_dir)
+
+                else:
+                    raise Exception(f'Unhandled archive extension: {file}')
+
+                if not action.keep_archive:
+                    os.unlink(file)
+
+        else:
+            raise TypeError(action)
+
+    def _return_val(self, spec: Spec, data_dir: str) -> str:
         check.state(os.path.isdir(data_dir))
 
-        if isinstance(spec, HttpCacheDataSpec):
-            data_file = os.path.join(data_dir, spec.file_name_or_default)
-            if not os.path.isfile(data_file):
-                raise RuntimeError(data_file)  # noqa
-            return data_file
+        if any(isinstance(a, ExtractAction) for a in spec.actions):
+            return data_dir
 
-        elif isinstance(spec, GithubContentCacheDataSpec):
+        single_file: str
+        if isinstance(spec, UrlSpec):
+            single_file = os.path.join(data_dir, spec.file_name_or_default)
+
+        elif isinstance(spec, GithubContentSpec):
             if len(spec.files) != 1:
                 return data_dir
-            data_file = os.path.join(data_dir, check.single(spec.files))
-            if not os.path.isfile(data_file):
-                raise RuntimeError(data_file)  # noqa
-            return data_file
+            single_file = os.path.join(data_dir, check.single(spec.files))
 
         else:
             return data_dir
 
-    def get(self, spec: CacheDataSpec) -> str:
-        os.makedirs(self._items_dir, exist_ok=True)
+        if not os.path.isfile(single_file):
+            raise RuntimeError(single_file)  # noqa
 
-        #
-
-        item_dir = os.path.join(self._items_dir, spec.digest)
-        if os.path.isdir(item_dir):
-            data_dir = os.path.join(item_dir, 'data')
-            return self._return_val(spec, data_dir)
-
-        #
+        return single_file
 
+    def _fetch_item(self, spec: Spec, item_dir: str) -> None:
         tmp_dir = tempfile.mkdtemp()
 
         #
@@ -133,11 +157,13 @@ class DataCache:
 
         start_at = lang.utcnow()
         self._fetch_into(spec, fetch_dir)
+        for action in spec.actions:
+            self._perform_action(action, fetch_dir)
         end_at = lang.utcnow()
 
         #
 
-        manifest = CacheDataManifest(
+        manifest = Manifest(
             spec,
             start_at=start_at,
             end_at=end_at,
@@ -159,5 +185,14 @@ class DataCache:
 
         shutil.move(tmp_dir, item_dir)
 
+    def get(self, spec: Spec) -> str:
+        os.makedirs(self._items_dir, exist_ok=True)
+
+        item_dir = os.path.join(self._items_dir, spec.digest)
+        if not os.path.isdir(item_dir):
+            self._fetch_item(spec, item_dir)
+
+        osu.touch(os.path.join(item_dir, 'accessed'))
+
         data_dir = os.path.join(item_dir, 'data')
         return self._return_val(spec, data_dir)

omdev/cache/data/defaults.py

@@ -2,7 +2,7 @@ import os.path
 
 from omlish import lang
 
-from .cache import DataCache
+from .cache import Cache
 
 
 ##
@@ -14,5 +14,5 @@ def default_dir() -> str:
 
 
 @lang.cached_function(lock=True)
-def default() -> DataCache:
-    return DataCache(default_dir())
+def default() -> Cache:
+    return Cache(default_dir())

omdev/cache/data/manifests.py

@@ -6,7 +6,7 @@ from omlish import dataclasses as dc
 
 from ...git import get_git_revision
 from .consts import SERIALIZATION_VERSION
-from .specs import CacheDataSpec
+from .specs import Spec
 
 
 ##
@@ -24,8 +24,8 @@ def _lib_revision() -> str | None:
 
 
 @dc.dataclass(frozen=True)
-class CacheDataManifest:
-    spec: CacheDataSpec
+class Manifest:
+    spec: Spec
 
     start_at: datetime.datetime = dc.field(kw_only=True)
     end_at: datetime.datetime = dc.field(kw_only=True)

omdev/cache/data/specs.py

@@ -10,6 +10,7 @@ from omlish import lang
 from omlish import marshal as msh
 from omlish.formats import json
 
+from .actions import Action
 from .consts import SERIALIZATION_VERSION
 
 
@@ -17,12 +18,14 @@ from .consts import SERIALIZATION_VERSION
 
 
 @dc.dataclass(frozen=True)
-class CacheDataSpec(lang.Abstract, lang.Sealed):
+class Spec(lang.Abstract, lang.Sealed):
     serialization_version: int = dc.field(default=SERIALIZATION_VERSION, kw_only=True)
 
+    actions: ta.Sequence[Action] = dc.field(default=(), kw_only=True)
+
     @cached.property
     def json(self) -> str:
-        return json.dumps_compact(msh.marshal(self, CacheDataSpec), sort_keys=True)
+        return json.dumps_compact(msh.marshal(self, Spec), sort_keys=True)
 
     @cached.property
     def digest(self) -> str:
@@ -39,7 +42,7 @@ def _maybe_sorted_strs(v: ta.Iterable[str] | None) -> ta.Sequence[str] | None:
 
 
 @dc.dataclass(frozen=True)
-class GitCacheDataSpec(CacheDataSpec):
+class GitSpec(Spec):
     url: str
 
     branch: str | None = dc.field(default=None, kw_only=True)
@@ -52,7 +55,7 @@ class GitCacheDataSpec(CacheDataSpec):
 
 
 @dc.dataclass(frozen=True)
-class HttpCacheDataSpec(CacheDataSpec):
+class UrlSpec(Spec):
     url: str = dc.xfield(validate=lambda u: bool(urllib.parse.urlparse(u)))
     file_name: str | None = None
 
@@ -74,7 +77,7 @@ def _repo_str(s: str) -> str:
 
 
 @dc.dataclass(frozen=True)
-class GithubContentCacheDataSpec(CacheDataSpec):
+class GithubContentSpec(Spec):
     repo: str = dc.field(validate=_repo_str)  # type: ignore
     rev: str
     files: lang.SequenceNotStr[str]
@@ -85,7 +88,7 @@ class GithubContentCacheDataSpec(CacheDataSpec):
 
 @lang.cached_function
 def _install_standard_marshalling() -> None:
-    specs_poly = msh.polymorphism_from_subclasses(CacheDataSpec)
+    specs_poly = msh.polymorphism_from_subclasses(Spec, naming=msh.Naming.SNAKE, strip_suffix=True)
     msh.STANDARD_MARSHALER_FACTORIES[0:0] = [msh.PolymorphismMarshalerFactory(specs_poly)]
     msh.STANDARD_UNMARSHALER_FACTORIES[0:0] = [msh.PolymorphismUnmarshalerFactory(specs_poly)]
 

omdev/manifests.py

@@ -113,7 +113,7 @@ def build_module_manifests(
                 module='.'.join(['', *mod_name.split('.')[1:]]),
                 attr=m.groupdict()['name'],
 
-                file=os.path.join(*os.path.split(file)[1:]),  # noqa
+                file=file,
                 line=i + 1,
             ))
 

omdev/precheck/base.py

@@ -0,0 +1,37 @@
+import abc
+import dataclasses as dc
+import typing as ta
+
+
+PrecheckConfigT = ta.TypeVar('PrecheckConfigT', bound='Precheck.Config')
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class PrecheckContext:
+    src_roots: ta.Sequence[str]
+
+
+##
+
+
+class Precheck(abc.ABC, ta.Generic[PrecheckConfigT]):
+    @dc.dataclass(frozen=True)
+    class Config:
+        pass
+
+    def __init__(self, context: PrecheckContext, config: PrecheckConfigT) -> None:
+        super().__init__()
+        self._context = context
+        self._config = config
+
+    @dc.dataclass(frozen=True)
+    class Violation:
+        pc: 'Precheck'
+        msg: str
+
+    @abc.abstractmethod
+    def run(self) -> ta.AsyncIterator[Violation]:
+        raise NotImplementedError

omdev/precheck/git.py

@@ -0,0 +1,34 @@
+import asyncio
+import dataclasses as dc
+import typing as ta
+
+from .base import Precheck
+from .base import PrecheckContext
+
+
+##
+
+
+class GitBlacklistPrecheck(Precheck['GitBlacklistPrecheck.Config']):
+    """
+    TODO:
+     - globs
+     - regex
+    """
+
+    @dc.dataclass(frozen=True)
+    class Config(Precheck.Config):
+        files: ta.Sequence[str] = (
+            '.env',
+            'secrets.yml',
+        )
+
+    def __init__(self, context: PrecheckContext, config: Config = Config()) -> None:
+        super().__init__(context, config)
+
+    async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
+        for f in self._config.files:
+            proc = await asyncio.create_subprocess_exec('git', 'status', '-s', f)
+            await proc.communicate()
+            if proc.returncode:
+                yield Precheck.Violation(self, f)

omdev/precheck/lite.py

@@ -0,0 +1,135 @@
+import asyncio
+import dataclasses as dc
+import glob
+import inspect
+import logging
+import os.path
+import subprocess
+import textwrap
+import typing as ta
+
+from omdev import findmagic
+from omlish import cached
+
+from .base import Precheck
+from .base import PrecheckContext
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class LitePython8Precheck(Precheck['LitePython8Precheck.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Precheck.Config):
+        pass
+
+    def __init__(self, context: PrecheckContext, config: Config = Config()) -> None:
+        super().__init__(context, config)
+
+    #
+
+    @staticmethod
+    def _load_file_module(fp: str) -> None:
+        import os.path  # noqa
+        import types  # noqa
+
+        fp = os.path.abspath(fp)
+
+        with open(fp) as f:
+            src = f.read()
+
+        mn = os.path.basename(fp).rpartition('.')[0]
+
+        mod = types.ModuleType(mn)
+        mod.__name__ = mn
+        mod.__file__ = fp
+        mod.__builtins__ = __builtins__  # type: ignore
+        mod.__spec__ = None
+
+        code = compile(src, fp, 'exec')
+        exec(code, mod.__dict__, mod.__dict__)
+
+    @cached.function
+    def _load_file_module_payload(self) -> str:
+        return '\n'.join([
+            'import sys',
+            'fp = sys.argv[-1]',
+            '',
+            textwrap.dedent('\n'.join(inspect.getsource(LitePython8Precheck._load_file_module).splitlines()[2:])),
+        ])
+
+    #
+
+    async def _run_script(self, fp: str) -> list[Precheck.Violation]:
+        log.debug('%s: loading script %s', self.__class__.__name__, fp)
+
+        vs: list[Precheck.Violation] = []
+
+        proc = await asyncio.create_subprocess_exec(
+            '.venvs/8/bin/python',
+            '-c',
+            self._load_file_module_payload(),
+            fp,
+            stderr=subprocess.PIPE,
+        )
+
+        _, stderr = await proc.communicate()
+        if proc.returncode != 0:
+            vs.append(Precheck.Violation(self, f'lite script {fp} failed to load in python8: {stderr.decode()}'))
+
+        return vs
+
+    async def _run_one_module(self, fp: str) -> list[Precheck.Violation]:
+        vs: list[Precheck.Violation] = []
+
+        mod = fp.rpartition('.')[0].replace(os.sep, '.')
+
+        log.debug('%s: loading module %s', self.__class__.__name__, mod)
+
+        proc = await asyncio.create_subprocess_exec(
+            '.venvs/8/bin/python',
+            '-c',
+            f'import {mod}',
+            stderr=subprocess.PIPE,
+        )
+
+        _, stderr = await proc.communicate()
+        if proc.returncode != 0:
+            vs.append(Precheck.Violation(self, f'lite module {fp} failed to import in python8: {stderr.decode()}'))  # noqa
+
+        return vs
+
+    async def _run_module(self, fp: str) -> list[Precheck.Violation]:
+        vs: list[Precheck.Violation] = []
+
+        if fp.endswith('__init__.py'):
+            pfps = glob.glob(os.path.join(os.path.dirname(fp), '**/*.py'), recursive=True)
+        else:
+            pfps = [fp]
+
+        for pfp in pfps:
+            vs.extend(await self._run_one_module(pfp))
+
+        return vs
+
+    async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
+        for fp in findmagic.find_magic(
+                self._context.src_roots,
+                ['# @omlish-lite'],
+                ['py'],
+        ):
+            with open(fp) as f:  # noqa  # FIXME
+                src = f.read()
+
+            is_script = '# @omlish-script' in src.splitlines()
+
+            if is_script:
+                for v in await self._run_script(fp):
+                    yield v
+
+            else:
+                for v in await self._run_module(fp):
+                    yield v