omdev 0.0.0.dev27__py3-none-any.whl → 0.0.0.dev29__py3-none-any.whl

omdev/cache/comp/cache.py

@@ -0,0 +1,137 @@
+"""
+TODO:
+ - decorator
+ - thread local cache instance - but shared
+ - arbitrary user-specified cache keys
+ - filesystem OPTIONAL
+ - locking
+ - Keyer scheme
+ - per-module-ish CACHE_VERSION convention
+ - are pickles stable?
+ - ComputeCache class
+ - Cacheable - fn is one
+ - ttl
+ - nice to have: np mmap
+ - compress?
+ - decos, descriptors, etc
+ - overlap w/ jobs/dags/batches/whatever
+ - joblib
+ - keep src anyway, but just for warn
+  - strip comments?
+ - ** INPUTS **
+  - if underlying impl changes, bust
+  - kinda reacty/reffy/signally
+ - decorator unwrapping and shit
+ - proactive deep invalidate
+ - tracked and versioned 'ops' but not result cached
+  - 'Versioned'
+
+manifest stuff
+ - serialization_version
+ - lib_version
+ - lib_revision
+
+fn manifest stuff
+ - source
+ - qualname
+ - location
+
+See:
+ - https://github.com/amakelov/mandala
+ - https://jax.readthedocs.io/en/latest/autodidax.html
+ - tinyjit
+ - https://docs.python.org/3/library/pickle.html#pickle.Pickler.dispatch_table
+
+names:
+ - CacheKey = unambiguous, fully qualified, unhashed map key - usually Cacheable + args
+ - Cacheable = usually a fn
+ - CacheableName = qualname of a cacheable
+  - dir structure: __package__/__qualname__/... ?
+"""
+import copy
+import typing as ta
+
+from omlish import collections as col
+from omlish import dataclasses as dc
+
+from .types import CacheableName
+from .types import CacheableResolver
+from .types import CacheableVersionMap
+from .types import CacheKey
+from .types import CacheResult
+
+
+class Cache:
+    def __init__(
+            self,
+            resolver: CacheableResolver,
+    ) -> None:
+        super().__init__()
+
+        self._resolver = resolver
+
+        self._dct: dict[CacheKey, Cache.Entry] = {}
+
+        self._stats = Cache.Stats()
+
+    @dc.dataclass()
+    class Stats:
+        num_hits: int = 0
+        num_misses: int = 0
+        num_invalidates: int = 0
+        num_puts: int = 0
+
+    @property
+    def stats(self) -> Stats:
+        return copy.deepcopy(self._stats)
+
+    @dc.dataclass(frozen=True)
+    class Entry:
+        key: CacheKey
+        versions: CacheableVersionMap
+        value: ta.Any
+
+        @dc.validate
+        def _check_types(self) -> bool:
+            return (
+                isinstance(self.key, CacheKey) and
+                isinstance(self.versions, col.frozendict)
+            )
+
+    def _build_version_map(self, names: ta.Iterable[CacheableName]) -> CacheableVersionMap:
+        dct = {}
+        for n in names:
+            c = self._resolver.resolve(n)
+            dct[n] = c.version
+        return col.frozendict(dct)
+
+    def get(self, key: CacheKey) -> CacheResult | None:
+        try:
+            entry = self._dct[key]
+        except KeyError:
+            self._stats.num_misses += 1
+            return None
+
+        new_versions = self._build_version_map(entry.versions)
+        if entry.versions != new_versions:
+            del self._dct[key]
+            self._stats.num_invalidates += 1
+            return None
+
+        self._stats.num_hits += 1
+        return CacheResult(
+            True,
+            entry.versions,
+            entry.value,
+        )
+
+    def put(self, key: CacheKey, versions: CacheableVersionMap, val: ta.Any) -> None:
+        if key in self._dct:
+            raise KeyError(key)
+
+        self._dct[key] = Cache.Entry(
+            key,
+            versions,
+            val,
+        )
+        self._stats.num_puts += 1

omdev/cache/comp/contexts.py

@@ -0,0 +1,136 @@
+import contextlib
+import typing as ta
+
+from omlish import check
+from omlish import lang
+
+from .cache import Cache
+from .types import Cacheable
+from .types import CacheableVersionMap
+from .types import CacheKey
+from .types import CacheResult
+from .types import merge_version_maps
+
+
+CacheT = ta.TypeVar('CacheT', bound='Cache')
+
+
+##
+
+
+_CURRENT_CACHE: Cache | None = None
+
+
+@contextlib.contextmanager
+def cache_context(cache: CacheT) -> ta.Iterator[CacheT]:
+    global _CURRENT_CACHE
+    prev = _CURRENT_CACHE
+    try:
+        _CURRENT_CACHE = cache
+        yield cache
+    finally:
+        check.is_(_CURRENT_CACHE, cache)
+        _CURRENT_CACHE = prev
+
+
+def get_current_cache() -> Cache | None:
+    return _CURRENT_CACHE
+
+
+##
+
+
+class CacheableContext(lang.Final):
+    def __init__(
+            self,
+            cacheable: Cacheable,
+            key: CacheKey,
+            *,
+            parent: ta.Optional['CacheableContext'] = None,
+    ) -> None:
+        super().__init__()
+        self._cacheable = cacheable
+        self._key = key
+        self._parent = parent
+
+        self._result: CacheResult | None = None
+        self._children: list[CacheableContext] = []
+
+        if parent is not None:
+            check.state(not parent.has_result)
+            parent._children.append(self)  # noqa
+
+    #
+
+    @property
+    def cacheable(self) -> Cacheable:
+        return self._cacheable
+
+    @property
+    def key(self) -> CacheKey:
+        return self._key
+
+    @property
+    def parent(self) -> ta.Optional['CacheableContext']:
+        return self._parent
+
+    @property
+    def children(self) -> ta.Sequence['CacheableContext']:
+        return self._children
+
+    #
+
+    @property
+    def has_result(self) -> bool:
+        return self._result is not None
+
+    def result(self) -> CacheResult:
+        return check.not_none(self._result)
+
+    def set_hit(self, result: CacheResult) -> None:
+        check.state(result.hit)
+        self._result = check.replacing_none(self._result, result)
+        self.result_versions()
+
+    def set_miss(self, val: ta.Any) -> None:
+        self._result = check.replacing_none(self._result, CacheResult(
+            False,
+            CacheableVersionMap(),
+            val,
+        ))
+        self.result_versions()
+
+    @lang.cached_function
+    def result_versions(self) -> CacheableVersionMap:
+        r = check.not_none(self._result)
+        return merge_version_maps(
+            self._cacheable.as_version_map,
+            r.versions,
+            *[c.result_versions() for c in self._children],
+        )
+
+
+#
+
+
+_CURRENT_CACHEABLE_CONTEXT: CacheableContext | None = None
+
+
+@contextlib.contextmanager
+def cacheable_context(
+        cacheable: Cacheable,
+        key: CacheKey,
+) -> ta.Iterator[CacheableContext]:
+    global _CURRENT_CACHEABLE_CONTEXT
+    prev = _CURRENT_CACHEABLE_CONTEXT
+    ctx = CacheableContext(
+        cacheable,
+        key,
+        parent=prev,
+    )
+    try:
+        _CURRENT_CACHEABLE_CONTEXT = ctx
+        yield ctx
+    finally:
+        check.is_(_CURRENT_CACHEABLE_CONTEXT, ctx)
+        _CURRENT_CACHEABLE_CONTEXT = prev

omdev/cache/comp/fns.py

@@ -0,0 +1,115 @@
+import functools
+import importlib
+import typing as ta
+
+from omlish import cached
+from omlish import check
+from omlish import collections as col
+from omlish import dataclasses as dc
+from omlish import lang
+
+from .contexts import cacheable_context
+from .contexts import get_current_cache
+from .types import Cacheable
+from .types import CacheableName
+from .types import CacheableResolver
+from .types import CacheKey
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class FnCacheableName(CacheableName, lang.Final):
+    module: str
+    qualname: str
+
+
+@dc.dataclass(frozen=True)
+class FnCacheable(Cacheable, lang.Final):
+    fn: ta.Callable
+    version: int = dc.xfield(override=True)
+
+    @cached.property
+    def name(self) -> FnCacheableName:
+        return FnCacheableName(self.fn.__module__, self.fn.__qualname__)  # noqa
+
+
+class FnCacheableResolver(CacheableResolver):
+    def resolve(self, name: CacheableName) -> Cacheable:
+        fname = check.isinstance(name, FnCacheableName)
+
+        mod = importlib.import_module(fname.module)
+        obj = mod
+        for a in fname.qualname.split('.'):
+            obj = getattr(obj, a)
+
+        check.callable(obj)
+        fc = check.isinstance(obj.__cacheable__, FnCacheable)
+
+        return fc
+
+
+@dc.dataclass(frozen=True)
+class FnCacheKey(CacheKey[FnCacheableName], lang.Final):
+    args: tuple
+    kwargs: col.frozendict[str, ta.Any]
+
+    @dc.validate
+    def _check_fn_types(self) -> bool:
+        return (
+            isinstance(self.name, FnCacheableName) and
+            isinstance(self.args, tuple) and
+            isinstance(self.kwargs, col.frozendict)
+        )
+
+
+##
+
+
+def cached_fn(version: int) -> ta.Callable[[T], T]:
+    def outer(fn):
+        @functools.wraps(fn)
+        def inner(*args, **kwargs):
+            # NOTE: just for testing :x allows updating
+            # TODO: proper wrapper obj probably (enforce name resolution)
+            cacheable = inner.__cacheable__  # type: ignore
+
+            if (cache := get_current_cache()) is not None:
+                key = FnCacheKey(
+                    cacheable.name,
+                    args,
+                    col.frozendict(kwargs),
+                )
+
+                with cacheable_context(
+                        cacheable,
+                        key,
+                ) as ctx:
+                    if (hit := cache.get(key)) is not None:
+                        ctx.set_hit(hit)
+                        return hit.value
+
+                    val = fn(*args, **kwargs)
+                    ctx.set_miss(val)
+                    cache.put(
+                        key,
+                        ctx.result_versions(),
+                        val,
+                    )
+                    return val
+
+            else:
+                return fn(*args, **kwargs)
+
+        inner.__cacheable__ = FnCacheable(  # type: ignore
+            fn,
+            version,
+        )
+
+        return inner
+
+    return outer  # noqa

omdev/cache/comp/resolvers.py

@@ -0,0 +1,23 @@
+from .types import Cacheable
+from .types import CacheableName
+from .types import CacheableResolver
+
+
+class CachingCacheableResolver(CacheableResolver):
+    def __init__(self, child: CacheableResolver) -> None:
+        super().__init__()
+
+        self._child = child
+        self._dct: dict[CacheableName, Cacheable] = {}
+
+    def clear(self) -> None:
+        self._dct.clear()
+
+    def resolve(self, name: CacheableName) -> Cacheable:
+        try:
+            return self._dct[name]
+        except KeyError:
+            pass
+        ret = self._child.resolve(name)
+        self._dct[name] = ret
+        return ret

omdev/cache/comp/types.py

@@ -0,0 +1,92 @@
+import abc
+import typing as ta
+
+from omlish import cached
+from omlish import collections as col
+from omlish import dataclasses as dc
+from omlish import lang
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+CacheableNameT = ta.TypeVar('CacheableNameT', bound='CacheableName')
+
+
+class CacheableName(lang.Abstract):
+    pass
+
+
+##
+
+
+CacheableVersion: ta.TypeAlias = ta.Hashable
+CacheableVersionMap: ta.TypeAlias = col.frozendict['CacheableName', CacheableVersion]
+
+
+def merge_version_maps(
+        *dcts: ta.Mapping[CacheableName, CacheableVersion],
+) -> CacheableVersionMap:
+    out: dict[CacheableName, CacheableVersion] = {}
+    for dct in dcts:
+        for name, version in dct.items():
+            try:
+                ex = out[name]
+            except KeyError:
+                out[name] = version
+            else:
+                if ex != version:
+                    raise Exception(f'Version mismatch: {ex} {version}')
+    return col.frozendict(out)
+
+
+##
+
+
+class Cacheable(lang.Abstract):
+    @property
+    @abc.abstractmethod
+    def name(self) -> CacheableName:
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def version(self) -> CacheableVersion:
+        raise NotImplementedError
+
+    @cached.property
+    def as_version_map(self) -> CacheableVersionMap:
+        return col.frozendict({self.name: self.version})
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+@dc.extra_params(cache_hash=True)
+class CacheKey(lang.Abstract, ta.Generic[CacheableNameT]):
+    name: CacheableNameT
+
+    @dc.validate
+    def _check_types(self) -> bool:
+        hash(self)
+        return isinstance(self.name, CacheableName)
+
+
+@dc.dataclass(frozen=True)
+class CacheResult(ta.Generic[T], lang.Final):
+    hit: bool
+    versions: CacheableVersionMap
+    value: T
+
+
+##
+
+
+class CacheableResolver(lang.Abstract):
+    @abc.abstractmethod
+    def resolve(self, name: CacheableName) -> Cacheable:
+        raise NotImplementedError

omdev/{datacache → cache/data}/__init__.py

@@ -1,5 +1,5 @@
 from .cache import (  # noqa
-    DataCache,
+    Cache,
 )
 
 from .defaults import (  # noqa

omdev/{datacache → cache/data}/actions.py

@@ -1,3 +1,9 @@
+"""
+TODO:
+ - extract
+ - shell
+ - checksum
+"""
 import typing as ta
 
 from omlish import dataclasses as dc

omdev/{datacache → cache/data}/cache.py

@@ -1,11 +1,10 @@
 """
 TODO:
+ - mirrors
  - huggingface_hub
- - postprocessing?
-  - unarchive
  - stupid little progress bars
- - return file path for single files
-  - thus, HttpSpec.url has derive=lambda url: ...
+ - groups of multiple files downloaded - 'spec set'? idk
+  - torchvision.datasets.FashionMNIST
 """
 import logging
 import os.path
@@ -19,7 +18,7 @@ from omlish import lang
 from omlish import marshal as msh
 from omlish.formats import json
 
-from .. import git
+from ... import git
 from .manifests import CacheDataManifest
 from .specs import CacheDataSpec
 from .specs import GitCacheDataSpec
@@ -33,7 +32,7 @@ log = logging.getLogger(__name__)
 ##
 
 
-class DataCache:
+class Cache:
     def __init__(self, base_dir: str) -> None:
         super().__init__()
         self._base_dir = base_dir

omdev/{datacache → cache/data}/defaults.py

@@ -2,7 +2,7 @@ import os.path
 
 from omlish import lang
 
-from .cache import DataCache
+from .cache import Cache
 
 
 ##
@@ -14,5 +14,5 @@ def default_dir() -> str:
 
 
 @lang.cached_function(lock=True)
-def default() -> DataCache:
-    return DataCache(default_dir())
+def default() -> Cache:
+    return Cache(default_dir())

omdev/{datacache → cache/data}/manifests.py

@@ -4,7 +4,7 @@ from omlish import __about__ as about
 from omlish import cached
 from omlish import dataclasses as dc
 
-from ..git import get_git_revision
+from ...git import get_git_revision
 from .consts import SERIALIZATION_VERSION
 from .specs import CacheDataSpec
 

omdev/precheck/base.py

@@ -0,0 +1,37 @@
+import abc
+import dataclasses as dc
+import typing as ta
+
+
+PrecheckConfigT = ta.TypeVar('PrecheckConfigT', bound='Precheck.Config')
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class PrecheckContext:
+    src_roots: ta.Sequence[str]
+
+
+##
+
+
+class Precheck(abc.ABC, ta.Generic[PrecheckConfigT]):
+    @dc.dataclass(frozen=True)
+    class Config:
+        pass
+
+    def __init__(self, context: PrecheckContext, config: PrecheckConfigT) -> None:
+        super().__init__()
+        self._context = context
+        self._config = config
+
+    @dc.dataclass(frozen=True)
+    class Violation:
+        pc: 'Precheck'
+        msg: str
+
+    @abc.abstractmethod
+    def run(self) -> ta.AsyncIterator[Violation]:
+        raise NotImplementedError

omdev/precheck/git.py

@@ -0,0 +1,34 @@
+import asyncio
+import dataclasses as dc
+import typing as ta
+
+from .base import Precheck
+from .base import PrecheckContext
+
+
+##
+
+
+class GitBlacklistPrecheck(Precheck['GitBlacklistPrecheck.Config']):
+    """
+    TODO:
+     - globs
+     - regex
+    """
+
+    @dc.dataclass(frozen=True)
+    class Config(Precheck.Config):
+        files: ta.Sequence[str] = (
+            '.env',
+            'secrets.yml',
+        )
+
+    def __init__(self, context: PrecheckContext, config: Config = Config()) -> None:
+        super().__init__(context, config)
+
+    async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
+        for f in self._config.files:
+            proc = await asyncio.create_subprocess_exec('git', 'status', '-s', f)
+            await proc.communicate()
+            if proc.returncode:
+                yield Precheck.Violation(self, f)

omdev/precheck/lite.py

@@ -0,0 +1,135 @@
+import asyncio
+import dataclasses as dc
+import glob
+import inspect
+import logging
+import os.path
+import subprocess
+import textwrap
+import typing as ta
+
+from omdev import findmagic
+from omlish import cached
+
+from .base import Precheck
+from .base import PrecheckContext
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+class LitePython8Precheck(Precheck['LitePython8Precheck.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Precheck.Config):
+        pass
+
+    def __init__(self, context: PrecheckContext, config: Config = Config()) -> None:
+        super().__init__(context, config)
+
+    #
+
+    @staticmethod
+    def _load_file_module(fp: str) -> None:
+        import os.path  # noqa
+        import types  # noqa
+
+        fp = os.path.abspath(fp)
+
+        with open(fp) as f:
+            src = f.read()
+
+        mn = os.path.basename(fp).rpartition('.')[0]
+
+        mod = types.ModuleType(mn)
+        mod.__name__ = mn
+        mod.__file__ = fp
+        mod.__builtins__ = __builtins__  # type: ignore
+        mod.__spec__ = None
+
+        code = compile(src, fp, 'exec')
+        exec(code, mod.__dict__, mod.__dict__)
+
+    @cached.function
+    def _load_file_module_payload(self) -> str:
+        return '\n'.join([
+            'import sys',
+            'fp = sys.argv[-1]',
+            '',
+            textwrap.dedent('\n'.join(inspect.getsource(LitePython8Precheck._load_file_module).splitlines()[2:])),
+        ])
+
+    #
+
+    async def _run_script(self, fp: str) -> list[Precheck.Violation]:
+        log.debug('%s: loading script %s', self.__class__.__name__, fp)
+
+        vs: list[Precheck.Violation] = []
+
+        proc = await asyncio.create_subprocess_exec(
+            '.venvs/8/bin/python',
+            '-c',
+            self._load_file_module_payload(),
+            fp,
+            stderr=subprocess.PIPE,
+        )
+
+        _, stderr = await proc.communicate()
+        if proc.returncode != 0:
+            vs.append(Precheck.Violation(self, f'lite script {fp} failed to load in python8: {stderr.decode()}'))
+
+        return vs
+
+    async def _run_one_module(self, fp: str) -> list[Precheck.Violation]:
+        vs: list[Precheck.Violation] = []
+
+        mod = fp.rpartition('.')[0].replace(os.sep, '.')
+
+        log.debug('%s: loading module %s', self.__class__.__name__, mod)
+
+        proc = await asyncio.create_subprocess_exec(
+            '.venvs/8/bin/python',
+            '-c',
+            f'import {mod}',
+            stderr=subprocess.PIPE,
+        )
+
+        _, stderr = await proc.communicate()
+        if proc.returncode != 0:
+            vs.append(Precheck.Violation(self, f'lite module {fp} failed to import in python8: {stderr.decode()}'))  # noqa
+
+        return vs
+
+    async def _run_module(self, fp: str) -> list[Precheck.Violation]:
+        vs: list[Precheck.Violation] = []
+
+        if fp.endswith('__init__.py'):
+            pfps = glob.glob(os.path.join(os.path.dirname(fp), '**/*.py'), recursive=True)
+        else:
+            pfps = [fp]
+
+        for pfp in pfps:
+            vs.extend(await self._run_one_module(pfp))
+
+        return vs
+
+    async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
+        for fp in findmagic.find_magic(
+                self._context.src_roots,
+                ['# @omlish-lite'],
+                ['py'],
+        ):
+            with open(fp) as f:  # noqa  # FIXME
+                src = f.read()
+
+            is_script = '# @omlish-script' in src.splitlines()
+
+            if is_script:
+                for v in await self._run_script(fp):
+                    yield v
+
+            else:
+                for v in await self._run_module(fp):
+                    yield v

omdev/precheck/precheck.py

@@ -17,28 +17,20 @@ TODO:
  - https://github.com/pre-commit/pre-commit-hooks?tab=readme-ov-file#forbid-new-submodules
  - don't check in .o's (omdev.ext import hook is dumb w build dir)
 """
-import abc
 import argparse
 import asyncio
-import dataclasses as dc
-import glob
-import inspect
 import logging
 import os.path
-import stat
-import subprocess
 import sys
-import textwrap
 import typing as ta
 
-from omdev import findimports
-from omdev import findmagic
-from omlish import cached
 from omlish import logs
 
-
-T = ta.TypeVar('T')
-PrecheckConfigT = ta.TypeVar('PrecheckConfigT', bound='Precheck.Config')
+from .base import Precheck
+from .base import PrecheckContext
+from .git import GitBlacklistPrecheck
+from .lite import LitePython8Precheck
+from .scripts import ScriptDepsPrecheck
 
 
 log = logging.getLogger(__name__)
@@ -47,214 +39,6 @@ log = logging.getLogger(__name__)
 ##
 
 
-@dc.dataclass(frozen=True, kw_only=True)
-class PrecheckContext:
-    src_roots: ta.Sequence[str]
-
-
-##
-
-
-class Precheck(abc.ABC, ta.Generic[PrecheckConfigT]):
-    @dc.dataclass(frozen=True)
-    class Config:
-        pass
-
-    def __init__(self, context: PrecheckContext, config: PrecheckConfigT) -> None:
-        super().__init__()
-        self._context = context
-        self._config = config
-
-    @dc.dataclass(frozen=True)
-    class Violation:
-        pc: 'Precheck'
-        msg: str
-
-    @abc.abstractmethod
-    def run(self) -> ta.AsyncIterator[Violation]:
-        raise NotImplementedError
-
-
-##
-
-
-class GitBlacklistPrecheck(Precheck['GitBlacklistPrecheck.Config']):
-    """
-    TODO:
-     - globs
-     - regex
-    """
-
-    @dc.dataclass(frozen=True)
-    class Config(Precheck.Config):
-        files: ta.Sequence[str] = (
-            '.env',
-            'secrets.yml',
-        )
-
-    def __init__(self, context: PrecheckContext, config: Config = Config()) -> None:
-        super().__init__(context, config)
-
-    async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
-        for f in self._config.files:
-            proc = await asyncio.create_subprocess_exec('git', 'status', '-s', f)
-            await proc.communicate()
-            if proc.returncode:
-                yield Precheck.Violation(self, f)
-
-
-##
-
-
-class ScriptDepsPrecheck(Precheck['ScriptDepsPrecheck.Config']):
-    @dc.dataclass(frozen=True)
-    class Config(Precheck.Config):
-        pass
-
-    def __init__(self, context: PrecheckContext, config: Config = Config()) -> None:
-        super().__init__(context, config)
-
-    async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
-        for fp in findmagic.find_magic(
-                self._context.src_roots,
-                ['# @omlish-script'],
-                ['py'],
-        ):
-            if not (stat.S_IXUSR & os.stat(fp).st_mode):
-                yield Precheck.Violation(self, f'script {fp} is not executable')
-
-            with open(fp) as f:  # noqa  # FIXME
-                src = f.read()
-
-            if not src.startswith('#!/usr/bin/env python3\n'):
-                yield Precheck.Violation(self, f'script {fp} lacks correct shebang')
-
-            imps = findimports.find_imports(fp)
-            deps = findimports.get_import_deps(imps)
-            if deps:
-                yield Precheck.Violation(self, f'script {fp} has deps: {deps}')
-
-
-##
-
-
-class LitePython8Precheck(Precheck['LitePython8Precheck.Config']):
-    @dc.dataclass(frozen=True)
-    class Config(Precheck.Config):
-        pass
-
-    def __init__(self, context: PrecheckContext, config: Config = Config()) -> None:
-        super().__init__(context, config)
-
-    #
-
-    @staticmethod
-    def _load_file_module(fp: str) -> None:
-        import os.path  # noqa
-        import types  # noqa
-
-        fp = os.path.abspath(fp)
-
-        with open(fp) as f:
-            src = f.read()
-
-        mn = os.path.basename(fp).rpartition('.')[0]
-
-        mod = types.ModuleType(mn)
-        mod.__name__ = mn
-        mod.__file__ = fp
-        mod.__builtins__ = __builtins__  # type: ignore
-        mod.__spec__ = None
-
-        code = compile(src, fp, 'exec')
-        exec(code, mod.__dict__, mod.__dict__)
-
-    @cached.function
-    def _load_file_module_payload(self) -> str:
-        return '\n'.join([
-            'import sys',
-            'fp = sys.argv[-1]',
-            '',
-            textwrap.dedent('\n'.join(inspect.getsource(LitePython8Precheck._load_file_module).splitlines()[2:])),
-        ])
-
-    #
-
-    async def _run_script(self, fp: str) -> list[Precheck.Violation]:
-        log.debug('%s: loading script %s', self.__class__.__name__, fp)
-
-        vs: list[Precheck.Violation] = []
-
-        proc = await asyncio.create_subprocess_exec(
-            '.venvs/8/bin/python',
-            '-c',
-            self._load_file_module_payload(),
-            fp,
-            stderr=subprocess.PIPE,
-        )
-
-        _, stderr = await proc.communicate()
-        if proc.returncode != 0:
-            vs.append(Precheck.Violation(self, f'lite script {fp} failed to load in python8: {stderr.decode()}'))
-
-        return vs
-
-    async def _run_one_module(self, fp: str) -> list[Precheck.Violation]:
-        vs: list[Precheck.Violation] = []
-
-        mod = fp.rpartition('.')[0].replace(os.sep, '.')
-
-        log.debug('%s: loading module %s', self.__class__.__name__, mod)
-
-        proc = await asyncio.create_subprocess_exec(
-            '.venvs/8/bin/python',
-            '-c',
-            f'import {mod}',
-            stderr=subprocess.PIPE,
-        )
-
-        _, stderr = await proc.communicate()
-        if proc.returncode != 0:
-            vs.append(Precheck.Violation(self, f'lite module {fp} failed to import in python8: {stderr.decode()}'))  # noqa
-
-        return vs
-
-    async def _run_module(self, fp: str) -> list[Precheck.Violation]:
-        vs: list[Precheck.Violation] = []
-
-        if fp.endswith('__init__.py'):
-            pfps = glob.glob(os.path.join(os.path.dirname(fp), '**/*.py'), recursive=True)
-        else:
-            pfps = [fp]
-
-        for pfp in pfps:
-            vs.extend(await self._run_one_module(pfp))
-
-        return vs
-
-    async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
-        for fp in findmagic.find_magic(
-                self._context.src_roots,
-                ['# @omlish-lite'],
-                ['py'],
-        ):
-            with open(fp) as f:  # noqa  # FIXME
-                src = f.read()
-
-            is_script = '# @omlish-script' in src.splitlines()
-
-            if is_script:
-                for v in await self._run_script(fp):
-                    yield v
-
-            else:
-                for v in await self._run_module(fp):
-                    yield v
-
-
-##
-
-
 def _check_cmd(args) -> None:
     if not os.path.isfile('pyproject.toml'):
         raise RuntimeError('must run in project root')

omdev/precheck/scripts.py

@@ -0,0 +1,42 @@
+import dataclasses as dc
+import os
+import stat
+import typing as ta
+
+from omdev import findimports
+from omdev import findmagic
+
+from .base import Precheck
+from .base import PrecheckContext
+
+
+##
+
+
+class ScriptDepsPrecheck(Precheck['ScriptDepsPrecheck.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Precheck.Config):
+        pass
+
+    def __init__(self, context: PrecheckContext, config: Config = Config()) -> None:
+        super().__init__(context, config)
+
+    async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
+        for fp in findmagic.find_magic(
+                self._context.src_roots,
+                ['# @omlish-script'],
+                ['py'],
+        ):
+            if not (stat.S_IXUSR & os.stat(fp).st_mode):
+                yield Precheck.Violation(self, f'script {fp} is not executable')
+
+            with open(fp) as f:  # noqa  # FIXME
+                src = f.read()
+
+            if not src.startswith('#!/usr/bin/env python3\n'):
+                yield Precheck.Violation(self, f'script {fp} lacks correct shebang')
+
+            imps = findimports.find_imports(fp)
+            deps = findimports.get_import_deps(imps)
+            if deps:
+                yield Precheck.Violation(self, f'script {fp} has deps: {deps}')

omdev/tools/piptools.py

@@ -0,0 +1,26 @@
+import io
+import urllib.request
+import xml.etree.ElementTree as ET  # noqa
+
+from omlish import argparse as ap
+from omlish import check
+
+
+PYPI_URL = 'https://pypi.org/'
+
+
+class Cli(ap.Cli):
+    @ap.command(
+        ap.arg('package'),
+    )
+    def lookup_latest_version(self) -> None:
+        pkg_name = check.non_empty_str(self.args.package)
+        with urllib.request.urlopen(f'{PYPI_URL}rss/project/{pkg_name}/releases.xml') as resp:  # noqa
+            rss = resp.read()
+        doc = ET.parse(io.BytesIO(rss))  # noqa
+        latest = check.not_none(doc.find('./channel/item/title')).text
+        print(latest)
+
+
+if __name__ == '__main__':
+    Cli()()

{omdev-0.0.0.dev27.dist-info → omdev-0.0.0.dev29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev27
+Version: 0.0.0.dev29
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omlish ==0.0.0.dev27
+Requires-Dist: omlish ==0.0.0.dev29
 Provides-Extra: all
 Requires-Dist: pycparser ~=2.22 ; extra == 'all'
 Requires-Dist: cffi ~=1.17 ; extra == 'all'

{omdev-0.0.0.dev27.dist-info → omdev-0.0.0.dev29.dist-info}/RECORD

@@ -14,6 +14,20 @@ omdev/wheelfile.py,sha256=yfupGcGkbFlmzGzKU64k_vmOKpaKnUlDWxeGn2KdekU,10005
 omdev/amalg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/amalg/__main__.py,sha256=OE1udULO1g4McUbeg1CoHbSm4hbQ2kcE3ffEGxlnPh4,69
 omdev/amalg/amalg.py,sha256=g7wwcPE2G9qmzh8M9eZAscOYWKo3ldI8bNxEXFnmzLE,14064
+omdev/cache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+omdev/cache/comp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+omdev/cache/comp/cache.py,sha256=0VDub_za4GlR5gIynqK4CnhhZFs-cdtlf_lwMhgtFa4,3387
+omdev/cache/comp/contexts.py,sha256=1pkZXeNG3Ehrwb6vZHWOgR0zDSXSTKsPrdSDW5tIRKs,3091
+omdev/cache/comp/fns.py,sha256=72yQ1e7NH8GxZ-ppcik1JLMEJGzrBvHgI5PSp4HcXVM,2943
+omdev/cache/comp/resolvers.py,sha256=Do7_PKy-09hyptgKel1vKQnHcpc3wjCP-1St3unTspc,614
+omdev/cache/comp/types.py,sha256=vFrcB8WftWDHQnrHG0lRXMBStWCm2M_Q4Xft-Rz7xnc,1907
+omdev/cache/data/__init__.py,sha256=7286T46a7gUEvC0gvSYZtfyS_h2oZJx0I2zE6UUTJkc,294
+omdev/cache/data/actions.py,sha256=OrCjo0INKeG8iEvXnQ9LI8PD8WF4d1vxrBwQzuA2ASw,829
+omdev/cache/data/cache.py,sha256=VQQpC5c2TcpLtFG87nkMdJ6_jgmRsocRuZe2FdzTddo,4768
+omdev/cache/data/consts.py,sha256=d6W_aeMqgah6PmPYi9RA8Be54oQ4BcNCy8kDQ7FlB_Q,26
+omdev/cache/data/defaults.py,sha256=HrapVUIf9Ozu3qSfRPyQj-vx-dz6Yyedjb-k3yV4CW8,277
+omdev/cache/data/manifests.py,sha256=6SqUiAlzkuDwIApEjzh5P6VimRVGk3iMwXjEPH2zvrc,1006
+omdev/cache/data/specs.py,sha256=ggvbp4nSEnXJImt_stVh4OsOYNzPIVGXAWplr17yEyg,2290
 omdev/cexts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/cexts/_boilerplate.cc,sha256=aOWF_5C2pqnIrkT1ykEaL7N2pIpamW6pdXriRbd3lvs,1725
 omdev/cexts/build.py,sha256=F3z1-CjDlEM-Gzi5IunKUBO52qdH_pMsFylobTdGJnI,2654
@@ -37,13 +51,6 @@ omdev/cexts/_distutils/compilers/__init__.py,sha256=amL_zrFlba0lHIvpqDne9uhqhLem
 omdev/cexts/_distutils/compilers/ccompiler.py,sha256=cTs88qrvj0hBVXHfemSDE_du_nEA4_qo3Qst5TpQkVI,43606
 omdev/cexts/_distutils/compilers/options.py,sha256=H7r5IcLvga5Fs3jjXWIT-6ap3JBduXRKgtpDmSGCZxs,3818
 omdev/cexts/_distutils/compilers/unixccompiler.py,sha256=o1h8QuyupLntv4F21_XjzAZmCiwwxJuTmOirvBSL-Qw,15419
-omdev/datacache/__init__.py,sha256=Jn_QRQn8L3-L0P-w9s5sBBku2tnFTOS3oc_U2PF8SSk,298
-omdev/datacache/actions.py,sha256=uqG6UM16o7GFDEoCWwzNKSE95fOGYmSLKGRvV7pokz4,783
-omdev/datacache/cache.py,sha256=nb7tu8FnPhpI8sK5uwoec4RZJ7eEd8wiEGwqYC6UeH0,4785
-omdev/datacache/consts.py,sha256=d6W_aeMqgah6PmPYi9RA8Be54oQ4BcNCy8kDQ7FlB_Q,26
-omdev/datacache/defaults.py,sha256=eRwkBRVaPSgYsfejgo-3SJGx2vyX_H4azhJ_un6-f74,289
-omdev/datacache/manifests.py,sha256=J8bK05xq0_r8dMBK7GsYT1-zO2jk-fx7Ati0S_IHN0Y,1005
-omdev/datacache/specs.py,sha256=ggvbp4nSEnXJImt_stVh4OsOYNzPIVGXAWplr17yEyg,2290
 omdev/interp/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/interp/__main__.py,sha256=gFhR9DikwDZk0LqgdR3qq_aXQHThUOPllDmHDOfnFAU,67
 omdev/interp/cli.py,sha256=8T3qLXTC2mni5FXDHkHN3mZG9_BnjkDMXYy6EYbAYR8,1679
@@ -58,7 +65,11 @@ omdev/mypy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/mypy/debug.py,sha256=WcZw-3Z1njg_KFGqi3DB6RuqbBa3dLArJnjVCuY1Mn0,3003
 omdev/precheck/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/precheck/__main__.py,sha256=wKF_2KP2Yn1hKDEOCGR_fm5zu9UHMWCZtuEmWjpprrU,72
-omdev/precheck/precheck.py,sha256=2yTjNGvjPYf3QxUBbCbehBYYuB8gDR_dYSTrlNCs9qU,8322
+omdev/precheck/base.py,sha256=a_lGoFM-QhL8u8XDUYFhb-feEyfPbP4j8lcmNO51sHY,732
+omdev/precheck/git.py,sha256=APC5Ln7x0zDrQiGPRWPsBcVJK3vWhbU-brqR5M63JQA,849
+omdev/precheck/lite.py,sha256=MLeDZP2UexNZzYTcSx4-LrhA97kCKn8tXrGkhsJb6I0,3649
+omdev/precheck/precheck.py,sha256=LUqh501TpLgBL_ZBW-enEMqaYnWzPMViECpnHGKaCIc,2462
+omdev/precheck/scripts.py,sha256=qq6MXkxgrYngPg5pWnXH4uRSuRkP3mFqbeml1UmvGBc,1265
 omdev/pyproject/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/pyproject/__main__.py,sha256=gFhR9DikwDZk0LqgdR3qq_aXQHThUOPllDmHDOfnFAU,67
 omdev/pyproject/cexts.py,sha256=x13piOOnNrYbA17qZLDVuR0p1sqhgEwpk4FtImX-klM,4281
@@ -79,13 +90,14 @@ omdev/tools/dockertools.py,sha256=0RoUThTqv4ySJZX0aufYeQWD2bp-BMplQ8Y4WvDpguA,52
 omdev/tools/gittools.py,sha256=zPy2D5WDs-CbwT86_T_hbaq5yCuss5e-ouUccXC6xlg,578
 omdev/tools/importscan.py,sha256=XRLiasVSaTIp-jnO0-Nfhi0t6gnv_hVy5j2nVfEvuMI,3831
 omdev/tools/importtrace.py,sha256=oDry9CwIv5h96wSaTVKJ0qQ5vMGxYE5oBtfF-GYNLJs,13430
+omdev/tools/piptools.py,sha256=etJBfjtUZ5HeQ7C4sxWmeshT-xXPdhScqEHWbca98n0,657
 omdev/tools/rst.py,sha256=6dWk8QZHoGiLSuBw3TKsXZjjFK6wWBEtPi9krdCLKKg,977
 omdev/tools/sqlrepl.py,sha256=v9uVQ4nvquSXcQVYIFq34ikumSILvKqzD6lUKLcncCE,5646
 omdev/versioning/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/versioning/specifiers.py,sha256=6Odf9e6farwlPRsD_YqwTfYKG-BXn_dIcKtqfkhfodI,17432
 omdev/versioning/versions.py,sha256=ei2eopEsJq3zSMJmezK1nzZgikgCdxFtnF3f69nCRZQ,12246
-omdev-0.0.0.dev27.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-omdev-0.0.0.dev27.dist-info/METADATA,sha256=pdGyBQjnHHcq_FVW-cWVPy3e3Gq8Iei4WkYaeyK8h3Q,1252
-omdev-0.0.0.dev27.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-omdev-0.0.0.dev27.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
-omdev-0.0.0.dev27.dist-info/RECORD,,
+omdev-0.0.0.dev29.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+omdev-0.0.0.dev29.dist-info/METADATA,sha256=_kVRNNJ4CIoIyEtgTCcAJzicnJevUfpkRjtXGcONxCQ,1252
+omdev-0.0.0.dev29.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+omdev-0.0.0.dev29.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
+omdev-0.0.0.dev29.dist-info/RECORD,,