omdev 0.0.0.dev29__py3-none-any.whl → 0.0.0.dev30__py3-none-any.whl

omdev/cache/comp/cache.py

@@ -1,137 +0,0 @@
-"""
-TODO:
- - decorator
- - thread local cache instance - but shared
- - arbitrary user-specified cache keys
- - filesystem OPTIONAL
- - locking
- - Keyer scheme
- - per-module-ish CACHE_VERSION convention
- - are pickles stable?
- - ComputeCache class
- - Cacheable - fn is one
- - ttl
- - nice to have: np mmap
- - compress?
- - decos, descriptors, etc
- - overlap w/ jobs/dags/batches/whatever
- - joblib
- - keep src anyway, but just for warn
-  - strip comments?
- - ** INPUTS **
-  - if underlying impl changes, bust
-  - kinda reacty/reffy/signally
- - decorator unwrapping and shit
- - proactive deep invalidate
- - tracked and versioned 'ops' but not result cached
-  - 'Versioned'
-
-manifest stuff
- - serialization_version
- - lib_version
- - lib_revision
-
-fn manifest stuff
- - source
- - qualname
- - location
-
-See:
- - https://github.com/amakelov/mandala
- - https://jax.readthedocs.io/en/latest/autodidax.html
- - tinyjit
- - https://docs.python.org/3/library/pickle.html#pickle.Pickler.dispatch_table
-
-names:
- - CacheKey = unambiguous, fully qualified, unhashed map key - usually Cacheable + args
- - Cacheable = usually a fn
- - CacheableName = qualname of a cacheable
-  - dir structure: __package__/__qualname__/... ?
-"""
-import copy
-import typing as ta
-
-from omlish import collections as col
-from omlish import dataclasses as dc
-
-from .types import CacheableName
-from .types import CacheableResolver
-from .types import CacheableVersionMap
-from .types import CacheKey
-from .types import CacheResult
-
-
-class Cache:
-    def __init__(
-            self,
-            resolver: CacheableResolver,
-    ) -> None:
-        super().__init__()
-
-        self._resolver = resolver
-
-        self._dct: dict[CacheKey, Cache.Entry] = {}
-
-        self._stats = Cache.Stats()
-
-    @dc.dataclass()
-    class Stats:
-        num_hits: int = 0
-        num_misses: int = 0
-        num_invalidates: int = 0
-        num_puts: int = 0
-
-    @property
-    def stats(self) -> Stats:
-        return copy.deepcopy(self._stats)
-
-    @dc.dataclass(frozen=True)
-    class Entry:
-        key: CacheKey
-        versions: CacheableVersionMap
-        value: ta.Any
-
-        @dc.validate
-        def _check_types(self) -> bool:
-            return (
-                isinstance(self.key, CacheKey) and
-                isinstance(self.versions, col.frozendict)
-            )
-
-    def _build_version_map(self, names: ta.Iterable[CacheableName]) -> CacheableVersionMap:
-        dct = {}
-        for n in names:
-            c = self._resolver.resolve(n)
-            dct[n] = c.version
-        return col.frozendict(dct)
-
-    def get(self, key: CacheKey) -> CacheResult | None:
-        try:
-            entry = self._dct[key]
-        except KeyError:
-            self._stats.num_misses += 1
-            return None
-
-        new_versions = self._build_version_map(entry.versions)
-        if entry.versions != new_versions:
-            del self._dct[key]
-            self._stats.num_invalidates += 1
-            return None
-
-        self._stats.num_hits += 1
-        return CacheResult(
-            True,
-            entry.versions,
-            entry.value,
-        )
-
-    def put(self, key: CacheKey, versions: CacheableVersionMap, val: ta.Any) -> None:
-        if key in self._dct:
-            raise KeyError(key)
-
-        self._dct[key] = Cache.Entry(
-            key,
-            versions,
-            val,
-        )
-        self._stats.num_puts += 1

omdev/cache/comp/contexts.py

@@ -1,136 +0,0 @@
-import contextlib
-import typing as ta
-
-from omlish import check
-from omlish import lang
-
-from .cache import Cache
-from .types import Cacheable
-from .types import CacheableVersionMap
-from .types import CacheKey
-from .types import CacheResult
-from .types import merge_version_maps
-
-
-CacheT = ta.TypeVar('CacheT', bound='Cache')
-
-
-##
-
-
-_CURRENT_CACHE: Cache | None = None
-
-
-@contextlib.contextmanager
-def cache_context(cache: CacheT) -> ta.Iterator[CacheT]:
-    global _CURRENT_CACHE
-    prev = _CURRENT_CACHE
-    try:
-        _CURRENT_CACHE = cache
-        yield cache
-    finally:
-        check.is_(_CURRENT_CACHE, cache)
-        _CURRENT_CACHE = prev
-
-
-def get_current_cache() -> Cache | None:
-    return _CURRENT_CACHE
-
-
-##
-
-
-class CacheableContext(lang.Final):
-    def __init__(
-            self,
-            cacheable: Cacheable,
-            key: CacheKey,
-            *,
-            parent: ta.Optional['CacheableContext'] = None,
-    ) -> None:
-        super().__init__()
-        self._cacheable = cacheable
-        self._key = key
-        self._parent = parent
-
-        self._result: CacheResult | None = None
-        self._children: list[CacheableContext] = []
-
-        if parent is not None:
-            check.state(not parent.has_result)
-            parent._children.append(self)  # noqa
-
-    #
-
-    @property
-    def cacheable(self) -> Cacheable:
-        return self._cacheable
-
-    @property
-    def key(self) -> CacheKey:
-        return self._key
-
-    @property
-    def parent(self) -> ta.Optional['CacheableContext']:
-        return self._parent
-
-    @property
-    def children(self) -> ta.Sequence['CacheableContext']:
-        return self._children
-
-    #
-
-    @property
-    def has_result(self) -> bool:
-        return self._result is not None
-
-    def result(self) -> CacheResult:
-        return check.not_none(self._result)
-
-    def set_hit(self, result: CacheResult) -> None:
-        check.state(result.hit)
-        self._result = check.replacing_none(self._result, result)
-        self.result_versions()
-
-    def set_miss(self, val: ta.Any) -> None:
-        self._result = check.replacing_none(self._result, CacheResult(
-            False,
-            CacheableVersionMap(),
-            val,
-        ))
-        self.result_versions()
-
-    @lang.cached_function
-    def result_versions(self) -> CacheableVersionMap:
-        r = check.not_none(self._result)
-        return merge_version_maps(
-            self._cacheable.as_version_map,
-            r.versions,
-            *[c.result_versions() for c in self._children],
-        )
-
-
-#
-
-
-_CURRENT_CACHEABLE_CONTEXT: CacheableContext | None = None
-
-
-@contextlib.contextmanager
-def cacheable_context(
-        cacheable: Cacheable,
-        key: CacheKey,
-) -> ta.Iterator[CacheableContext]:
-    global _CURRENT_CACHEABLE_CONTEXT
-    prev = _CURRENT_CACHEABLE_CONTEXT
-    ctx = CacheableContext(
-        cacheable,
-        key,
-        parent=prev,
-    )
-    try:
-        _CURRENT_CACHEABLE_CONTEXT = ctx
-        yield ctx
-    finally:
-        check.is_(_CURRENT_CACHEABLE_CONTEXT, ctx)
-        _CURRENT_CACHEABLE_CONTEXT = prev

omdev/cache/comp/fns.py

@@ -1,115 +0,0 @@
-import functools
-import importlib
-import typing as ta
-
-from omlish import cached
-from omlish import check
-from omlish import collections as col
-from omlish import dataclasses as dc
-from omlish import lang
-
-from .contexts import cacheable_context
-from .contexts import get_current_cache
-from .types import Cacheable
-from .types import CacheableName
-from .types import CacheableResolver
-from .types import CacheKey
-
-
-T = ta.TypeVar('T')
-
-
-##
-
-
-@dc.dataclass(frozen=True)
-class FnCacheableName(CacheableName, lang.Final):
-    module: str
-    qualname: str
-
-
-@dc.dataclass(frozen=True)
-class FnCacheable(Cacheable, lang.Final):
-    fn: ta.Callable
-    version: int = dc.xfield(override=True)
-
-    @cached.property
-    def name(self) -> FnCacheableName:
-        return FnCacheableName(self.fn.__module__, self.fn.__qualname__)  # noqa
-
-
-class FnCacheableResolver(CacheableResolver):
-    def resolve(self, name: CacheableName) -> Cacheable:
-        fname = check.isinstance(name, FnCacheableName)
-
-        mod = importlib.import_module(fname.module)
-        obj = mod
-        for a in fname.qualname.split('.'):
-            obj = getattr(obj, a)
-
-        check.callable(obj)
-        fc = check.isinstance(obj.__cacheable__, FnCacheable)
-
-        return fc
-
-
-@dc.dataclass(frozen=True)
-class FnCacheKey(CacheKey[FnCacheableName], lang.Final):
-    args: tuple
-    kwargs: col.frozendict[str, ta.Any]
-
-    @dc.validate
-    def _check_fn_types(self) -> bool:
-        return (
-            isinstance(self.name, FnCacheableName) and
-            isinstance(self.args, tuple) and
-            isinstance(self.kwargs, col.frozendict)
-        )
-
-
-##
-
-
-def cached_fn(version: int) -> ta.Callable[[T], T]:
-    def outer(fn):
-        @functools.wraps(fn)
-        def inner(*args, **kwargs):
-            # NOTE: just for testing :x allows updating
-            # TODO: proper wrapper obj probably (enforce name resolution)
-            cacheable = inner.__cacheable__  # type: ignore
-
-            if (cache := get_current_cache()) is not None:
-                key = FnCacheKey(
-                    cacheable.name,
-                    args,
-                    col.frozendict(kwargs),
-                )
-
-                with cacheable_context(
-                        cacheable,
-                        key,
-                ) as ctx:
-                    if (hit := cache.get(key)) is not None:
-                        ctx.set_hit(hit)
-                        return hit.value
-
-                    val = fn(*args, **kwargs)
-                    ctx.set_miss(val)
-                    cache.put(
-                        key,
-                        ctx.result_versions(),
-                        val,
-                    )
-                    return val
-
-            else:
-                return fn(*args, **kwargs)
-
-        inner.__cacheable__ = FnCacheable(  # type: ignore
-            fn,
-            version,
-        )
-
-        return inner
-
-    return outer  # noqa

omdev/cache/comp/resolvers.py

@@ -1,23 +0,0 @@
-from .types import Cacheable
-from .types import CacheableName
-from .types import CacheableResolver
-
-
-class CachingCacheableResolver(CacheableResolver):
-    def __init__(self, child: CacheableResolver) -> None:
-        super().__init__()
-
-        self._child = child
-        self._dct: dict[CacheableName, Cacheable] = {}
-
-    def clear(self) -> None:
-        self._dct.clear()
-
-    def resolve(self, name: CacheableName) -> Cacheable:
-        try:
-            return self._dct[name]
-        except KeyError:
-            pass
-        ret = self._child.resolve(name)
-        self._dct[name] = ret
-        return ret

omdev/cache/comp/types.py

@@ -1,92 +0,0 @@
-import abc
-import typing as ta
-
-from omlish import cached
-from omlish import collections as col
-from omlish import dataclasses as dc
-from omlish import lang
-
-
-T = ta.TypeVar('T')
-
-
-##
-
-
-CacheableNameT = ta.TypeVar('CacheableNameT', bound='CacheableName')
-
-
-class CacheableName(lang.Abstract):
-    pass
-
-
-##
-
-
-CacheableVersion: ta.TypeAlias = ta.Hashable
-CacheableVersionMap: ta.TypeAlias = col.frozendict['CacheableName', CacheableVersion]
-
-
-def merge_version_maps(
-        *dcts: ta.Mapping[CacheableName, CacheableVersion],
-) -> CacheableVersionMap:
-    out: dict[CacheableName, CacheableVersion] = {}
-    for dct in dcts:
-        for name, version in dct.items():
-            try:
-                ex = out[name]
-            except KeyError:
-                out[name] = version
-            else:
-                if ex != version:
-                    raise Exception(f'Version mismatch: {ex} {version}')
-    return col.frozendict(out)
-
-
-##
-
-
-class Cacheable(lang.Abstract):
-    @property
-    @abc.abstractmethod
-    def name(self) -> CacheableName:
-        raise NotImplementedError
-
-    @property
-    @abc.abstractmethod
-    def version(self) -> CacheableVersion:
-        raise NotImplementedError
-
-    @cached.property
-    def as_version_map(self) -> CacheableVersionMap:
-        return col.frozendict({self.name: self.version})
-
-
-##
-
-
-@dc.dataclass(frozen=True)
-@dc.extra_params(cache_hash=True)
-class CacheKey(lang.Abstract, ta.Generic[CacheableNameT]):
-    name: CacheableNameT
-
-    @dc.validate
-    def _check_types(self) -> bool:
-        hash(self)
-        return isinstance(self.name, CacheableName)
-
-
-@dc.dataclass(frozen=True)
-class CacheResult(ta.Generic[T], lang.Final):
-    hit: bool
-    versions: CacheableVersionMap
-    value: T
-
-
-##
-
-
-class CacheableResolver(lang.Abstract):
-    @abc.abstractmethod
-    def resolve(self, name: CacheableName) -> Cacheable:
-        raise NotImplementedError