omdev 0.0.0.dev27__py3-none-any.whl → 0.0.0.dev28__py3-none-any.whl

omdev/cache/comp/cache.py

@@ -0,0 +1,137 @@
+"""
+TODO:
+ - decorator
+ - thread local cache instance - but shared
+ - arbitrary user-specified cache keys
+ - filesystem OPTIONAL
+ - locking
+ - Keyer scheme
+ - per-module-ish CACHE_VERSION convention
+ - are pickles stable?
+ - ComputeCache class
+ - Cacheable - fn is one
+ - ttl
+ - nice to have: np mmap
+ - compress?
+ - decos, descriptors, etc
+ - overlap w/ jobs/dags/batches/whatever
+ - joblib
+ - keep src anyway, but just for warn
+  - strip comments?
+ - ** INPUTS **
+  - if underlying impl changes, bust
+  - kinda reacty/reffy/signally
+ - decorator unwrapping and shit
+ - proactive deep invalidate
+ - tracked and versioned 'ops' but not result cached
+  - 'Versioned'
+
+manifest stuff
+ - serialization_version
+ - lib_version
+ - lib_revision
+
+fn manifest stuff
+ - source
+ - qualname
+ - location
+
+See:
+ - https://github.com/amakelov/mandala
+ - https://jax.readthedocs.io/en/latest/autodidax.html
+ - tinyjit
+ - https://docs.python.org/3/library/pickle.html#pickle.Pickler.dispatch_table
+
+names:
+ - CacheKey = unambiguous, fully qualified, unhashed map key - usually Cacheable + args
+ - Cacheable = usually a fn
+ - CacheableName = qualname of a cacheable
+  - dir structure: __package__/__qualname__/... ?
+"""
+import copy
+import typing as ta
+
+from omlish import collections as col
+from omlish import dataclasses as dc
+
+from .types import CacheableName
+from .types import CacheableResolver
+from .types import CacheableVersionMap
+from .types import CacheKey
+from .types import CacheResult
+
+
+class Cache:
+    def __init__(
+            self,
+            resolver: CacheableResolver,
+    ) -> None:
+        super().__init__()
+
+        self._resolver = resolver
+
+        self._dct: dict[CacheKey, Cache.Entry] = {}
+
+        self._stats = Cache.Stats()
+
+    @dc.dataclass()
+    class Stats:
+        num_hits: int = 0
+        num_misses: int = 0
+        num_invalidates: int = 0
+        num_puts: int = 0
+
+    @property
+    def stats(self) -> Stats:
+        return copy.deepcopy(self._stats)
+
+    @dc.dataclass(frozen=True)
+    class Entry:
+        key: CacheKey
+        versions: CacheableVersionMap
+        value: ta.Any
+
+        @dc.validate
+        def _check_types(self) -> bool:
+            return (
+                isinstance(self.key, CacheKey) and
+                isinstance(self.versions, col.frozendict)
+            )
+
+    def _build_version_map(self, names: ta.Iterable[CacheableName]) -> CacheableVersionMap:
+        dct = {}
+        for n in names:
+            c = self._resolver.resolve(n)
+            dct[n] = c.version
+        return col.frozendict(dct)
+
+    def get(self, key: CacheKey) -> CacheResult | None:
+        try:
+            entry = self._dct[key]
+        except KeyError:
+            self._stats.num_misses += 1
+            return None
+
+        new_versions = self._build_version_map(entry.versions)
+        if entry.versions != new_versions:
+            del self._dct[key]
+            self._stats.num_invalidates += 1
+            return None
+
+        self._stats.num_hits += 1
+        return CacheResult(
+            True,
+            entry.versions,
+            entry.value,
+        )
+
+    def put(self, key: CacheKey, versions: CacheableVersionMap, val: ta.Any) -> None:
+        if key in self._dct:
+            raise KeyError(key)
+
+        self._dct[key] = Cache.Entry(
+            key,
+            versions,
+            val,
+        )
+        self._stats.num_puts += 1

omdev/cache/comp/contexts.py

@@ -0,0 +1,136 @@
+import contextlib
+import typing as ta
+
+from omlish import check
+from omlish import lang
+
+from .cache import Cache
+from .types import Cacheable
+from .types import CacheableVersionMap
+from .types import CacheKey
+from .types import CacheResult
+from .types import merge_version_maps
+
+
+CacheT = ta.TypeVar('CacheT', bound='Cache')
+
+
+##
+
+
+_CURRENT_CACHE: Cache | None = None
+
+
+@contextlib.contextmanager
+def cache_context(cache: CacheT) -> ta.Iterator[CacheT]:
+    global _CURRENT_CACHE
+    prev = _CURRENT_CACHE
+    try:
+        _CURRENT_CACHE = cache
+        yield cache
+    finally:
+        check.is_(_CURRENT_CACHE, cache)
+        _CURRENT_CACHE = prev
+
+
+def get_current_cache() -> Cache | None:
+    return _CURRENT_CACHE
+
+
+##
+
+
+class CacheableContext(lang.Final):
+    def __init__(
+            self,
+            cacheable: Cacheable,
+            key: CacheKey,
+            *,
+            parent: ta.Optional['CacheableContext'] = None,
+    ) -> None:
+        super().__init__()
+        self._cacheable = cacheable
+        self._key = key
+        self._parent = parent
+
+        self._result: CacheResult | None = None
+        self._children: list[CacheableContext] = []
+
+        if parent is not None:
+            check.state(not parent.has_result)
+            parent._children.append(self)  # noqa
+
+    #
+
+    @property
+    def cacheable(self) -> Cacheable:
+        return self._cacheable
+
+    @property
+    def key(self) -> CacheKey:
+        return self._key
+
+    @property
+    def parent(self) -> 'CacheableContext | None':
+        return self._parent
+
+    @property
+    def children(self) -> ta.Sequence['CacheableContext']:
+        return self._children
+
+    #
+
+    @property
+    def has_result(self) -> bool:
+        return self._result is not None
+
+    def result(self) -> CacheResult:
+        return check.not_none(self._result)
+
+    def set_hit(self, result: CacheResult) -> None:
+        check.state(result.hit)
+        self._result = check.replacing_none(self._result, result)
+        self.result_versions()
+
+    def set_miss(self, val: ta.Any) -> None:
+        self._result = check.replacing_none(self._result, CacheResult(
+            False,
+            CacheableVersionMap(),
+            val,
+        ))
+        self.result_versions()
+
+    @lang.cached_function
+    def result_versions(self) -> CacheableVersionMap:
+        r = check.not_none(self._result)
+        return merge_version_maps(
+            self._cacheable.as_version_map,
+            r.versions,
+            *[c.result_versions() for c in self._children],
+        )
+
+
+#
+
+
+_CURRENT_CACHEABLE_CONTEXT: CacheableContext | None = None
+
+
+@contextlib.contextmanager
+def cacheable_context(
+        cacheable: Cacheable,
+        key: CacheKey,
+) -> ta.Iterator[CacheableContext]:
+    global _CURRENT_CACHEABLE_CONTEXT
+    prev = _CURRENT_CACHEABLE_CONTEXT
+    ctx = CacheableContext(
+        cacheable,
+        key,
+        parent=prev,
+    )
+    try:
+        _CURRENT_CACHEABLE_CONTEXT = ctx
+        yield ctx
+    finally:
+        check.is_(_CURRENT_CACHEABLE_CONTEXT, ctx)
+        _CURRENT_CACHEABLE_CONTEXT = prev

omdev/cache/comp/fns.py

@@ -0,0 +1,115 @@
+import functools
+import importlib
+import typing as ta
+
+from omlish import cached
+from omlish import check
+from omlish import collections as col
+from omlish import dataclasses as dc
+from omlish import lang
+
+from .contexts import cacheable_context
+from .contexts import get_current_cache
+from .types import Cacheable
+from .types import CacheableName
+from .types import CacheableResolver
+from .types import CacheKey
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class FnCacheableName(CacheableName, lang.Final):
+    module: str
+    qualname: str
+
+
+@dc.dataclass(frozen=True)
+class FnCacheable(Cacheable, lang.Final):
+    fn: ta.Callable
+    version: int = dc.xfield(override=True)
+
+    @cached.property
+    def name(self) -> FnCacheableName:
+        return FnCacheableName(self.fn.__module__, self.fn.__qualname__)  # noqa
+
+
+class FnCacheableResolver(CacheableResolver):
+    def resolve(self, name: CacheableName) -> Cacheable:
+        fname = check.isinstance(name, FnCacheableName)
+
+        mod = importlib.import_module(fname.module)
+        obj = mod
+        for a in fname.qualname.split('.'):
+            obj = getattr(obj, a)
+
+        check.callable(obj)
+        fc = check.isinstance(obj.__cacheable__, FnCacheable)
+
+        return fc
+
+
+@dc.dataclass(frozen=True)
+class FnCacheKey(CacheKey[FnCacheableName], lang.Final):
+    args: tuple
+    kwargs: col.frozendict[str, ta.Any]
+
+    @dc.validate
+    def _check_fn_types(self) -> bool:
+        return (
+            isinstance(self.name, FnCacheableName) and
+            isinstance(self.args, tuple) and
+            isinstance(self.kwargs, col.frozendict)
+        )
+
+
+##
+
+
+def cached_fn(version: int) -> ta.Callable[[T], T]:
+    def outer(fn):
+        @functools.wraps(fn)
+        def inner(*args, **kwargs):
+            # NOTE: just for testing :x allows updating
+            # TODO: proper wrapper obj probably (enforce name resolution)
+            cacheable = inner.__cacheable__  # type: ignore
+
+            if (cache := get_current_cache()) is not None:
+                key = FnCacheKey(
+                    cacheable.name,
+                    args,
+                    col.frozendict(kwargs),
+                )
+
+                with cacheable_context(
+                        cacheable,
+                        key,
+                ) as ctx:
+                    if (hit := cache.get(key)) is not None:
+                        ctx.set_hit(hit)
+                        return hit.value
+
+                    val = fn(*args, **kwargs)
+                    ctx.set_miss(val)
+                    cache.put(
+                        key,
+                        ctx.result_versions(),
+                        val,
+                    )
+                    return val
+
+            else:
+                return fn(*args, **kwargs)
+
+        inner.__cacheable__ = FnCacheable(  # type: ignore
+            fn,
+            version,
+        )
+
+        return inner
+
+    return outer  # noqa

omdev/cache/comp/resolvers.py

@@ -0,0 +1,23 @@
+from .types import Cacheable
+from .types import CacheableName
+from .types import CacheableResolver
+
+
+class CachingCacheableResolver(CacheableResolver):
+    def __init__(self, child: CacheableResolver) -> None:
+        super().__init__()
+
+        self._child = child
+        self._dct: dict[CacheableName, Cacheable] = {}
+
+    def clear(self) -> None:
+        self._dct.clear()
+
+    def resolve(self, name: CacheableName) -> Cacheable:
+        try:
+            return self._dct[name]
+        except KeyError:
+            pass
+        ret = self._child.resolve(name)
+        self._dct[name] = ret
+        return ret

omdev/cache/comp/types.py

@@ -0,0 +1,92 @@
+import abc
+import typing as ta
+
+from omlish import cached
+from omlish import collections as col
+from omlish import dataclasses as dc
+from omlish import lang
+
+
+T = ta.TypeVar('T')
+
+
+##
+
+
+CacheableNameT = ta.TypeVar('CacheableNameT', bound='CacheableName')
+
+
+class CacheableName(lang.Abstract):
+    pass
+
+
+##
+
+
+CacheableVersion: ta.TypeAlias = ta.Hashable
+CacheableVersionMap: ta.TypeAlias = col.frozendict['CacheableName', CacheableVersion]
+
+
+def merge_version_maps(
+        *dcts: ta.Mapping[CacheableName, CacheableVersion],
+) -> CacheableVersionMap:
+    out: dict[CacheableName, CacheableVersion] = {}
+    for dct in dcts:
+        for name, version in dct.items():
+            try:
+                ex = out[name]
+            except KeyError:
+                out[name] = version
+            else:
+                if ex != version:
+                    raise Exception(f'Version mismatch: {ex} {version}')
+    return col.frozendict(out)
+
+
+##
+
+
+class Cacheable(lang.Abstract):
+    @property
+    @abc.abstractmethod
+    def name(self) -> CacheableName:
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def version(self) -> CacheableVersion:
+        raise NotImplementedError
+
+    @cached.property
+    def as_version_map(self) -> CacheableVersionMap:
+        return col.frozendict({self.name: self.version})
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+@dc.extra_params(cache_hash=True)
+class CacheKey(lang.Abstract, ta.Generic[CacheableNameT]):
+    name: CacheableNameT
+
+    @dc.validate
+    def _check_types(self) -> bool:
+        hash(self)
+        return isinstance(self.name, CacheableName)
+
+
+@dc.dataclass(frozen=True)
+class CacheResult(ta.Generic[T], lang.Final):
+    hit: bool
+    versions: CacheableVersionMap
+    value: T
+
+
+##
+
+
+class CacheableResolver(lang.Abstract):
+    @abc.abstractmethod
+    def resolve(self, name: CacheableName) -> Cacheable:
+        raise NotImplementedError

omdev/{datacache → cache/data}/actions.py

@@ -1,3 +1,9 @@
+"""
+TODO:
+ - extract
+ - shell
+ - checksum
+"""
 import typing as ta
 
 from omlish import dataclasses as dc

omdev/{datacache → cache/data}/cache.py

@@ -1,11 +1,10 @@
 """
 TODO:
+ - mirrors
  - huggingface_hub
- - postprocessing?
-  - unarchive
  - stupid little progress bars
- - return file path for single files
-  - thus, HttpSpec.url has derive=lambda url: ...
+ - groups of multiple files downloaded - 'spec set'? idk
+  - torchvision.datasets.FashionMNIST
 """
 import logging
 import os.path
@@ -19,7 +18,7 @@ from omlish import lang
 from omlish import marshal as msh
 from omlish.formats import json
 
-from .. import git
+from ... import git
 from .manifests import CacheDataManifest
 from .specs import CacheDataSpec
 from .specs import GitCacheDataSpec

omdev/{datacache → cache/data}/manifests.py

@@ -4,7 +4,7 @@ from omlish import __about__ as about
 from omlish import cached
 from omlish import dataclasses as dc
 
-from ..git import get_git_revision
+from ...git import get_git_revision
 from .consts import SERIALIZATION_VERSION
 from .specs import CacheDataSpec
 

{omdev-0.0.0.dev27.dist-info → omdev-0.0.0.dev28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev27
+Version: 0.0.0.dev28
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: ~=3.12
 License-File: LICENSE
-Requires-Dist: omlish ==0.0.0.dev27
+Requires-Dist: omlish ==0.0.0.dev28
 Provides-Extra: all
 Requires-Dist: pycparser ~=2.22 ; extra == 'all'
 Requires-Dist: cffi ~=1.17 ; extra == 'all'

{omdev-0.0.0.dev27.dist-info → omdev-0.0.0.dev28.dist-info}/RECORD

@@ -14,6 +14,20 @@ omdev/wheelfile.py,sha256=yfupGcGkbFlmzGzKU64k_vmOKpaKnUlDWxeGn2KdekU,10005
 omdev/amalg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/amalg/__main__.py,sha256=OE1udULO1g4McUbeg1CoHbSm4hbQ2kcE3ffEGxlnPh4,69
 omdev/amalg/amalg.py,sha256=g7wwcPE2G9qmzh8M9eZAscOYWKo3ldI8bNxEXFnmzLE,14064
+omdev/cache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+omdev/cache/comp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+omdev/cache/comp/cache.py,sha256=0VDub_za4GlR5gIynqK4CnhhZFs-cdtlf_lwMhgtFa4,3387
+omdev/cache/comp/contexts.py,sha256=LHCzt9x3s35Xc1jW5U7GWkhi-z-Z0_KA47IcfqgmNvU,3085
+omdev/cache/comp/fns.py,sha256=72yQ1e7NH8GxZ-ppcik1JLMEJGzrBvHgI5PSp4HcXVM,2943
+omdev/cache/comp/resolvers.py,sha256=Do7_PKy-09hyptgKel1vKQnHcpc3wjCP-1St3unTspc,614
+omdev/cache/comp/types.py,sha256=vFrcB8WftWDHQnrHG0lRXMBStWCm2M_Q4Xft-Rz7xnc,1907
+omdev/cache/data/__init__.py,sha256=Jn_QRQn8L3-L0P-w9s5sBBku2tnFTOS3oc_U2PF8SSk,298
+omdev/cache/data/actions.py,sha256=OrCjo0INKeG8iEvXnQ9LI8PD8WF4d1vxrBwQzuA2ASw,829
+omdev/cache/data/cache.py,sha256=HfxoXRmzVeqMDUKyqY3PSJShuqtcYjkuA14T2kz3uzE,4772
+omdev/cache/data/consts.py,sha256=d6W_aeMqgah6PmPYi9RA8Be54oQ4BcNCy8kDQ7FlB_Q,26
+omdev/cache/data/defaults.py,sha256=eRwkBRVaPSgYsfejgo-3SJGx2vyX_H4azhJ_un6-f74,289
+omdev/cache/data/manifests.py,sha256=6SqUiAlzkuDwIApEjzh5P6VimRVGk3iMwXjEPH2zvrc,1006
+omdev/cache/data/specs.py,sha256=ggvbp4nSEnXJImt_stVh4OsOYNzPIVGXAWplr17yEyg,2290
 omdev/cexts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/cexts/_boilerplate.cc,sha256=aOWF_5C2pqnIrkT1ykEaL7N2pIpamW6pdXriRbd3lvs,1725
 omdev/cexts/build.py,sha256=F3z1-CjDlEM-Gzi5IunKUBO52qdH_pMsFylobTdGJnI,2654
@@ -37,13 +51,6 @@ omdev/cexts/_distutils/compilers/__init__.py,sha256=amL_zrFlba0lHIvpqDne9uhqhLem
 omdev/cexts/_distutils/compilers/ccompiler.py,sha256=cTs88qrvj0hBVXHfemSDE_du_nEA4_qo3Qst5TpQkVI,43606
 omdev/cexts/_distutils/compilers/options.py,sha256=H7r5IcLvga5Fs3jjXWIT-6ap3JBduXRKgtpDmSGCZxs,3818
 omdev/cexts/_distutils/compilers/unixccompiler.py,sha256=o1h8QuyupLntv4F21_XjzAZmCiwwxJuTmOirvBSL-Qw,15419
-omdev/datacache/__init__.py,sha256=Jn_QRQn8L3-L0P-w9s5sBBku2tnFTOS3oc_U2PF8SSk,298
-omdev/datacache/actions.py,sha256=uqG6UM16o7GFDEoCWwzNKSE95fOGYmSLKGRvV7pokz4,783
-omdev/datacache/cache.py,sha256=nb7tu8FnPhpI8sK5uwoec4RZJ7eEd8wiEGwqYC6UeH0,4785
-omdev/datacache/consts.py,sha256=d6W_aeMqgah6PmPYi9RA8Be54oQ4BcNCy8kDQ7FlB_Q,26
-omdev/datacache/defaults.py,sha256=eRwkBRVaPSgYsfejgo-3SJGx2vyX_H4azhJ_un6-f74,289
-omdev/datacache/manifests.py,sha256=J8bK05xq0_r8dMBK7GsYT1-zO2jk-fx7Ati0S_IHN0Y,1005
-omdev/datacache/specs.py,sha256=ggvbp4nSEnXJImt_stVh4OsOYNzPIVGXAWplr17yEyg,2290
 omdev/interp/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/interp/__main__.py,sha256=gFhR9DikwDZk0LqgdR3qq_aXQHThUOPllDmHDOfnFAU,67
 omdev/interp/cli.py,sha256=8T3qLXTC2mni5FXDHkHN3mZG9_BnjkDMXYy6EYbAYR8,1679
@@ -84,8 +91,8 @@ omdev/tools/sqlrepl.py,sha256=v9uVQ4nvquSXcQVYIFq34ikumSILvKqzD6lUKLcncCE,5646
 omdev/versioning/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/versioning/specifiers.py,sha256=6Odf9e6farwlPRsD_YqwTfYKG-BXn_dIcKtqfkhfodI,17432
 omdev/versioning/versions.py,sha256=ei2eopEsJq3zSMJmezK1nzZgikgCdxFtnF3f69nCRZQ,12246
-omdev-0.0.0.dev27.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-omdev-0.0.0.dev27.dist-info/METADATA,sha256=pdGyBQjnHHcq_FVW-cWVPy3e3Gq8Iei4WkYaeyK8h3Q,1252
-omdev-0.0.0.dev27.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-omdev-0.0.0.dev27.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
-omdev-0.0.0.dev27.dist-info/RECORD,,
+omdev-0.0.0.dev28.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+omdev-0.0.0.dev28.dist-info/METADATA,sha256=QoShpM9YgegstZpXpp-S85RypV0WYAwkbIHwJ8ZIfsI,1252
+omdev-0.0.0.dev28.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+omdev-0.0.0.dev28.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
+omdev-0.0.0.dev28.dist-info/RECORD,,