omdev 0.0.0.dev29__py3-none-any.whl → 0.0.0.dev31__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omdev/cache/compute/cache.py +120 -0
- omdev/cache/compute/contexts.py +137 -0
- omdev/cache/compute/currents.py +78 -0
- omdev/cache/compute/fns.py +157 -0
- omdev/cache/compute/resolvers.py +23 -0
- omdev/cache/compute/storage.py +39 -0
- omdev/cache/compute/types.py +144 -0
- omdev/cache/data/__init__.py +9 -5
- omdev/cache/data/actions.py +8 -2
- omdev/cache/data/cache.py +66 -31
- omdev/cache/data/manifests.py +3 -3
- omdev/cache/data/specs.py +9 -6
- omdev/interp/pyenv.py +8 -1
- omdev/manifests.py +112 -18
- omdev/pyproject/cli.py +6 -2
- omdev/scripts/interp.py +28 -6
- omdev/scripts/pyproject.py +34 -8
- omdev/secrets.py +12 -0
- omdev/tools/piptools.py +24 -0
- {omdev-0.0.0.dev29.dist-info → omdev-0.0.0.dev31.dist-info}/METADATA +2 -2
- {omdev-0.0.0.dev29.dist-info → omdev-0.0.0.dev31.dist-info}/RECORD +25 -22
- omdev/cache/comp/cache.py +0 -137
- omdev/cache/comp/contexts.py +0 -136
- omdev/cache/comp/fns.py +0 -115
- omdev/cache/comp/resolvers.py +0 -23
- omdev/cache/comp/types.py +0 -92
- /omdev/cache/{comp → compute}/__init__.py +0 -0
- {omdev-0.0.0.dev29.dist-info → omdev-0.0.0.dev31.dist-info}/LICENSE +0 -0
- {omdev-0.0.0.dev29.dist-info → omdev-0.0.0.dev31.dist-info}/WHEEL +0 -0
- {omdev-0.0.0.dev29.dist-info → omdev-0.0.0.dev31.dist-info}/top_level.txt +0 -0
omdev/scripts/pyproject.py
CHANGED
|
@@ -11,9 +11,8 @@ TODO:
|
|
|
11
11
|
- build / package / publish / version roll
|
|
12
12
|
- {pkg_name: [src_dirs]}, default excludes, generate MANIFST.in, ...
|
|
13
13
|
- env vars - PYTHONPATH
|
|
14
|
-
- optional uv backend
|
|
15
14
|
|
|
16
|
-
|
|
15
|
+
See:
|
|
17
16
|
- https://pdm-project.org/en/latest/
|
|
18
17
|
- https://rye.astral.sh/philosophy/
|
|
19
18
|
- https://github.com/indygreg/python-build-standalone/blob/main/pythonbuild/cpython.py
|
|
@@ -30,6 +29,7 @@ import base64
|
|
|
30
29
|
import collections
|
|
31
30
|
import collections.abc
|
|
32
31
|
import concurrent.futures as cf
|
|
32
|
+
import contextlib
|
|
33
33
|
import csv
|
|
34
34
|
import dataclasses as dc
|
|
35
35
|
import datetime
|
|
@@ -2800,6 +2800,24 @@ class StandardLogHandler(ProxyLogHandler):
|
|
|
2800
2800
|
##
|
|
2801
2801
|
|
|
2802
2802
|
|
|
2803
|
+
@contextlib.contextmanager
|
|
2804
|
+
def _locking_logging_module_lock() -> ta.Iterator[None]:
|
|
2805
|
+
if hasattr(logging, '_acquireLock'):
|
|
2806
|
+
logging._acquireLock() # noqa
|
|
2807
|
+
try:
|
|
2808
|
+
yield
|
|
2809
|
+
finally:
|
|
2810
|
+
logging._releaseLock() # type: ignore # noqa
|
|
2811
|
+
|
|
2812
|
+
elif hasattr(logging, '_lock'):
|
|
2813
|
+
# https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
|
|
2814
|
+
with logging._lock: # noqa
|
|
2815
|
+
yield
|
|
2816
|
+
|
|
2817
|
+
else:
|
|
2818
|
+
raise Exception("Can't find lock in logging module")
|
|
2819
|
+
|
|
2820
|
+
|
|
2803
2821
|
def configure_standard_logging(
|
|
2804
2822
|
level: ta.Union[int, str] = logging.INFO,
|
|
2805
2823
|
*,
|
|
@@ -2807,8 +2825,7 @@ def configure_standard_logging(
|
|
|
2807
2825
|
target: ta.Optional[logging.Logger] = None,
|
|
2808
2826
|
force: bool = False,
|
|
2809
2827
|
) -> ta.Optional[StandardLogHandler]:
|
|
2810
|
-
|
|
2811
|
-
try:
|
|
2828
|
+
with _locking_logging_module_lock():
|
|
2812
2829
|
if target is None:
|
|
2813
2830
|
target = logging.root
|
|
2814
2831
|
|
|
@@ -2848,9 +2865,6 @@ def configure_standard_logging(
|
|
|
2848
2865
|
|
|
2849
2866
|
return StandardLogHandler(handler)
|
|
2850
2867
|
|
|
2851
|
-
finally:
|
|
2852
|
-
logging._releaseLock() # type: ignore # noqa
|
|
2853
|
-
|
|
2854
2868
|
|
|
2855
2869
|
########################################
|
|
2856
2870
|
# ../../../omlish/lite/marshal.py
|
|
@@ -4221,7 +4235,14 @@ TODO:
|
|
|
4221
4235
|
- custom tags
|
|
4222
4236
|
- optionally install / upgrade pyenv itself
|
|
4223
4237
|
- new vers dont need these custom mac opts, only run on old vers
|
|
4224
|
-
|
|
4238
|
+
|
|
4239
|
+
TODO opts:
|
|
4240
|
+
- --enable-loadable-sqlite-extensions LDFLAGS="-L/opt/homebrew/opt/sqlite/lib" CPPFLAGS="-I/opt/homebrew/opt/sqlite/include"
|
|
4241
|
+
- --enable-shared
|
|
4242
|
+
- --enable-optimizations
|
|
4243
|
+
- --enable-profiling ?
|
|
4244
|
+
- --enable-ipv6 ?
|
|
4245
|
+
""" # noqa
|
|
4225
4246
|
|
|
4226
4247
|
|
|
4227
4248
|
##
|
|
@@ -4929,6 +4950,11 @@ class Venv:
|
|
|
4929
4950
|
if (sr := self._cfg.requires):
|
|
4930
4951
|
rr = RequirementsRewriter(self._name)
|
|
4931
4952
|
reqs = [rr.rewrite(req) for req in sr]
|
|
4953
|
+
|
|
4954
|
+
# TODO: automatically try slower uv download when it fails? lol
|
|
4955
|
+
# Caused by: Failed to download distribution due to network timeout. Try increasing UV_HTTP_TIMEOUT (current value: 30s). # noqa
|
|
4956
|
+
# UV_CONCURRENT_DOWNLOADS=4 UV_HTTP_TIMEOUT=3600
|
|
4957
|
+
|
|
4932
4958
|
subprocess_check_call(
|
|
4933
4959
|
ve,
|
|
4934
4960
|
'-m',
|
omdev/secrets.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import os.path
|
|
2
|
+
import typing as ta
|
|
3
|
+
|
|
4
|
+
import yaml
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
SECRETS_PATH = os.getenv('SECRETS_PATH', os.path.expanduser('~/Dropbox/.dotfiles/secrets.yml'))
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def load_secrets() -> dict[str, ta.Any]:
|
|
11
|
+
with open(SECRETS_PATH) as f:
|
|
12
|
+
return yaml.safe_load(f)
|
omdev/tools/piptools.py
CHANGED
|
@@ -21,6 +21,30 @@ class Cli(ap.Cli):
|
|
|
21
21
|
latest = check.not_none(doc.find('./channel/item/title')).text
|
|
22
22
|
print(latest)
|
|
23
23
|
|
|
24
|
+
@ap.command(
|
|
25
|
+
ap.arg('file'),
|
|
26
|
+
ap.arg('-w', '--write', action='store_true'),
|
|
27
|
+
ap.arg('-q', '--quiet', action='store_true'),
|
|
28
|
+
)
|
|
29
|
+
def filter_dev_deps(self) -> None:
|
|
30
|
+
with open(self.args.file) as f:
|
|
31
|
+
src = f.read()
|
|
32
|
+
|
|
33
|
+
out = []
|
|
34
|
+
for l in src.splitlines(keepends=True):
|
|
35
|
+
if l.startswith('-e'):
|
|
36
|
+
continue
|
|
37
|
+
out.append(l)
|
|
38
|
+
|
|
39
|
+
new_src = ''.join(out)
|
|
40
|
+
|
|
41
|
+
if not self.args.quiet:
|
|
42
|
+
print(new_src)
|
|
43
|
+
|
|
44
|
+
if self.args.write:
|
|
45
|
+
with open(self.args.file, 'w') as f:
|
|
46
|
+
f.write(new_src)
|
|
47
|
+
|
|
24
48
|
|
|
25
49
|
if __name__ == '__main__':
|
|
26
50
|
Cli()()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: omdev
|
|
3
|
-
Version: 0.0.0.
|
|
3
|
+
Version: 0.0.0.dev31
|
|
4
4
|
Summary: omdev
|
|
5
5
|
Author: wrmsr
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
|
|
|
12
12
|
Classifier: Operating System :: POSIX
|
|
13
13
|
Requires-Python: ~=3.12
|
|
14
14
|
License-File: LICENSE
|
|
15
|
-
Requires-Dist: omlish ==0.0.0.
|
|
15
|
+
Requires-Dist: omlish ==0.0.0.dev31
|
|
16
16
|
Provides-Extra: all
|
|
17
17
|
Requires-Dist: pycparser ~=2.22 ; extra == 'all'
|
|
18
18
|
Requires-Dist: cffi ~=1.17 ; extra == 'all'
|
|
@@ -7,27 +7,30 @@ omdev/cmake.py,sha256=Diy2ry65806dQP125DAstD3w46z_wszMH7PwC2-6iik,4578
|
|
|
7
7
|
omdev/findimports.py,sha256=P8v4I1tm6g-PEWJiNwAKxErvWwL-Nop83vAuwq1kR5A,2246
|
|
8
8
|
omdev/findmagic.py,sha256=DhBYHHP_dzwM5pIh21xnQPnkZ2YmAXCjithsr7X0ScU,2357
|
|
9
9
|
omdev/git.py,sha256=OzP4xHVboaa7GhS-mg4F3lYWf3HLa5aMm6V6PtIw_3U,2137
|
|
10
|
-
omdev/manifests.py,sha256=
|
|
10
|
+
omdev/manifests.py,sha256=bn8FzgYvC7L70mRXvu8TSc41eGp-BSVzuRvGuYj3FC8,9225
|
|
11
11
|
omdev/revisions.py,sha256=U657hf4zeEN32y3g4CzqCAodx_HlfkHj2cIIKALNFDo,5009
|
|
12
|
+
omdev/secrets.py,sha256=ja0VsCB01MHxYwn5OHjFeXV9cRah9AQl-0uJzZELpic,256
|
|
12
13
|
omdev/tokens.py,sha256=GusxQ1Cd_eiScuR8XTTtc9QFhOgYviYGBZmFnn3Hj7s,756
|
|
13
14
|
omdev/wheelfile.py,sha256=yfupGcGkbFlmzGzKU64k_vmOKpaKnUlDWxeGn2KdekU,10005
|
|
14
15
|
omdev/amalg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
16
|
omdev/amalg/__main__.py,sha256=OE1udULO1g4McUbeg1CoHbSm4hbQ2kcE3ffEGxlnPh4,69
|
|
16
17
|
omdev/amalg/amalg.py,sha256=g7wwcPE2G9qmzh8M9eZAscOYWKo3ldI8bNxEXFnmzLE,14064
|
|
17
18
|
omdev/cache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
-
omdev/cache/
|
|
19
|
-
omdev/cache/
|
|
20
|
-
omdev/cache/
|
|
21
|
-
omdev/cache/
|
|
22
|
-
omdev/cache/
|
|
23
|
-
omdev/cache/
|
|
24
|
-
omdev/cache/
|
|
25
|
-
omdev/cache/
|
|
26
|
-
omdev/cache/data/
|
|
19
|
+
omdev/cache/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
+
omdev/cache/compute/cache.py,sha256=pEbcTSQavhX7M0olzR8AOCtHx7tMnMnEDEfTPQVY5uE,3602
|
|
21
|
+
omdev/cache/compute/contexts.py,sha256=OKOHOIj8JSEHmCKMY9G_zpKkLAkGuXcFXvzY5SXqCMA,3231
|
|
22
|
+
omdev/cache/compute/currents.py,sha256=D1Ls5Sd7FX2aPO5wpyEwTSmkz50sxkwCs37Amb0urH8,1425
|
|
23
|
+
omdev/cache/compute/fns.py,sha256=_1xU7qw1O57OsTp17RFjFTBO1M2t54RL-CGdl-0cTBk,4175
|
|
24
|
+
omdev/cache/compute/resolvers.py,sha256=9dq0mt__emp8CdNDWPVUr_dCkTOn7ar6nw0F2QH6XpQ,566
|
|
25
|
+
omdev/cache/compute/storage.py,sha256=woCUqHg8ZrwLEejRG3zu1L5ZXxGNNXveh3E8FnlEkjc,941
|
|
26
|
+
omdev/cache/compute/types.py,sha256=NpCTTJHDmpERjrbO6dh9TEzHuP6-vOuoX3ym9sA0ukc,2639
|
|
27
|
+
omdev/cache/data/__init__.py,sha256=SQXtugLceRif463rcoklpQ33pxYLgEIm0xiI6NvOI6M,301
|
|
28
|
+
omdev/cache/data/actions.py,sha256=TX6DPbOzQY6S2MSTPnsG53BQ61NNPWuLeCXa-MF-W2g,1109
|
|
29
|
+
omdev/cache/data/cache.py,sha256=WSsbFyFRT_IQFYQCrmUpaTvs9DRglLmCnhguOzdJ6p4,5753
|
|
27
30
|
omdev/cache/data/consts.py,sha256=d6W_aeMqgah6PmPYi9RA8Be54oQ4BcNCy8kDQ7FlB_Q,26
|
|
28
31
|
omdev/cache/data/defaults.py,sha256=HrapVUIf9Ozu3qSfRPyQj-vx-dz6Yyedjb-k3yV4CW8,277
|
|
29
|
-
omdev/cache/data/manifests.py,sha256=
|
|
30
|
-
omdev/cache/data/specs.py,sha256=
|
|
32
|
+
omdev/cache/data/manifests.py,sha256=CupK71fL3_PnDzUqjrWLNt64KfGKF-K4ycMkT5p0gPA,979
|
|
33
|
+
omdev/cache/data/specs.py,sha256=EB_JLIFe47ETCtAk8BD6oY6qhM7U_nOkILJfnLaVafI,2351
|
|
31
34
|
omdev/cexts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
35
|
omdev/cexts/_boilerplate.cc,sha256=aOWF_5C2pqnIrkT1ykEaL7N2pIpamW6pdXriRbd3lvs,1725
|
|
33
36
|
omdev/cexts/build.py,sha256=F3z1-CjDlEM-Gzi5IunKUBO52qdH_pMsFylobTdGJnI,2654
|
|
@@ -56,7 +59,7 @@ omdev/interp/__main__.py,sha256=gFhR9DikwDZk0LqgdR3qq_aXQHThUOPllDmHDOfnFAU,67
|
|
|
56
59
|
omdev/interp/cli.py,sha256=8T3qLXTC2mni5FXDHkHN3mZG9_BnjkDMXYy6EYbAYR8,1679
|
|
57
60
|
omdev/interp/inspect.py,sha256=SI4jQmWfXCnlceFTxlVRfTlEYYCjO_X12wuG5e74yto,2849
|
|
58
61
|
omdev/interp/providers.py,sha256=PFEjozW0c33eqg8sno-GHMKbhVUzQF9jrAx-M0uQimk,1787
|
|
59
|
-
omdev/interp/pyenv.py,sha256=
|
|
62
|
+
omdev/interp/pyenv.py,sha256=s7nY2c6g3dOzvEefB_lgDjs6XSJ_u1ClCXQlWnCk6oM,12688
|
|
60
63
|
omdev/interp/resolvers.py,sha256=tpzlmqGp1C4QKdA6TfcPmtmaygu7mb6WK2RPSbyNQ6s,3022
|
|
61
64
|
omdev/interp/standalone.py,sha256=XcltiL7ypcfV89C82_3knQ3Kx7aW4wnnxf2056ZXC3A,7731
|
|
62
65
|
omdev/interp/system.py,sha256=UFHfMR0CHCEnNx5fhrze8esAwigpRrJUA33ftq6nA0I,3514
|
|
@@ -73,15 +76,15 @@ omdev/precheck/scripts.py,sha256=qq6MXkxgrYngPg5pWnXH4uRSuRkP3mFqbeml1UmvGBc,126
|
|
|
73
76
|
omdev/pyproject/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
|
|
74
77
|
omdev/pyproject/__main__.py,sha256=gFhR9DikwDZk0LqgdR3qq_aXQHThUOPllDmHDOfnFAU,67
|
|
75
78
|
omdev/pyproject/cexts.py,sha256=x13piOOnNrYbA17qZLDVuR0p1sqhgEwpk4FtImX-klM,4281
|
|
76
|
-
omdev/pyproject/cli.py,sha256=
|
|
79
|
+
omdev/pyproject/cli.py,sha256=gRUsTcVQDx9u6hZNOGAHG-4pccF6tS3I1fHBzgS1Tb0,10908
|
|
77
80
|
omdev/pyproject/configs.py,sha256=K9H5cGwVLgHi8wKwtYvlXHZ9ThtmnI4jo8JAb-t1-70,2859
|
|
78
81
|
omdev/pyproject/pkg.py,sha256=GlZvDcLbo7HmiV2SgQnJdgAswr9IoJpy5gOeTRXG2RM,12576
|
|
79
82
|
omdev/pyproject/reqs.py,sha256=coq21cdWQIPs06-iuRnwc6F2Sf-IxpqoT6DEMhol2kA,2298
|
|
80
83
|
omdev/scripts/__init__.py,sha256=MKCvUAEQwsIvwLixwtPlpBqmkMXLCnjjXyAXvVpDwVk,91
|
|
81
84
|
omdev/scripts/bumpversion.py,sha256=Kn7fo73Hs8uJh3Hi3EIyLOlzLPWAC6dwuD_lZ3cIzuY,1064
|
|
82
85
|
omdev/scripts/execrss.py,sha256=HzDNmwXOO8fMwIRXw9q8CUnVfLFCQASyU2tfY_y2Vf8,324
|
|
83
|
-
omdev/scripts/interp.py,sha256=
|
|
84
|
-
omdev/scripts/pyproject.py,sha256=
|
|
86
|
+
omdev/scripts/interp.py,sha256=f_BDB7gQTnGMQk-kzUHcBbtl1RGpFIqQskOCODHir50,69488
|
|
87
|
+
omdev/scripts/pyproject.py,sha256=zL2MPdN1tyEBXzZc6rmB86cTdAZUCtmB9ICtaMGIxj4,153598
|
|
85
88
|
omdev/toml/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
|
|
86
89
|
omdev/toml/parser.py,sha256=84bn09uhYHwQGyfww6Rw6y1RxPAE_HDltODOSakcqDM,29186
|
|
87
90
|
omdev/toml/writer.py,sha256=dwz_Qw8z5Z_nmWpXqch63W6S_j6n256erb7AGFTVzB4,2872
|
|
@@ -90,14 +93,14 @@ omdev/tools/dockertools.py,sha256=0RoUThTqv4ySJZX0aufYeQWD2bp-BMplQ8Y4WvDpguA,52
|
|
|
90
93
|
omdev/tools/gittools.py,sha256=zPy2D5WDs-CbwT86_T_hbaq5yCuss5e-ouUccXC6xlg,578
|
|
91
94
|
omdev/tools/importscan.py,sha256=XRLiasVSaTIp-jnO0-Nfhi0t6gnv_hVy5j2nVfEvuMI,3831
|
|
92
95
|
omdev/tools/importtrace.py,sha256=oDry9CwIv5h96wSaTVKJ0qQ5vMGxYE5oBtfF-GYNLJs,13430
|
|
93
|
-
omdev/tools/piptools.py,sha256=
|
|
96
|
+
omdev/tools/piptools.py,sha256=P2Nq8OzRuLxay1uQgqaWf3Iz6PFVhFKmVaBwxNbbzwU,1274
|
|
94
97
|
omdev/tools/rst.py,sha256=6dWk8QZHoGiLSuBw3TKsXZjjFK6wWBEtPi9krdCLKKg,977
|
|
95
98
|
omdev/tools/sqlrepl.py,sha256=v9uVQ4nvquSXcQVYIFq34ikumSILvKqzD6lUKLcncCE,5646
|
|
96
99
|
omdev/versioning/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
|
|
97
100
|
omdev/versioning/specifiers.py,sha256=6Odf9e6farwlPRsD_YqwTfYKG-BXn_dIcKtqfkhfodI,17432
|
|
98
101
|
omdev/versioning/versions.py,sha256=ei2eopEsJq3zSMJmezK1nzZgikgCdxFtnF3f69nCRZQ,12246
|
|
99
|
-
omdev-0.0.0.
|
|
100
|
-
omdev-0.0.0.
|
|
101
|
-
omdev-0.0.0.
|
|
102
|
-
omdev-0.0.0.
|
|
103
|
-
omdev-0.0.0.
|
|
102
|
+
omdev-0.0.0.dev31.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
|
|
103
|
+
omdev-0.0.0.dev31.dist-info/METADATA,sha256=PPt5cs_yozGg6dBtCj2mS1oviINmxzj1NVOmhIxa0Vg,1252
|
|
104
|
+
omdev-0.0.0.dev31.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
|
|
105
|
+
omdev-0.0.0.dev31.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
|
|
106
|
+
omdev-0.0.0.dev31.dist-info/RECORD,,
|
omdev/cache/comp/cache.py
DELETED
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
TODO:
|
|
3
|
-
- decorator
|
|
4
|
-
- thread local cache instance - but shared
|
|
5
|
-
- arbitrary user-specified cache keys
|
|
6
|
-
- filesystem OPTIONAL
|
|
7
|
-
- locking
|
|
8
|
-
- Keyer scheme
|
|
9
|
-
- per-module-ish CACHE_VERSION convention
|
|
10
|
-
- are pickles stable?
|
|
11
|
-
- ComputeCache class
|
|
12
|
-
- Cacheable - fn is one
|
|
13
|
-
- ttl
|
|
14
|
-
- nice to have: np mmap
|
|
15
|
-
- compress?
|
|
16
|
-
- decos, descriptors, etc
|
|
17
|
-
- overlap w/ jobs/dags/batches/whatever
|
|
18
|
-
- joblib
|
|
19
|
-
- keep src anyway, but just for warn
|
|
20
|
-
- strip comments?
|
|
21
|
-
- ** INPUTS **
|
|
22
|
-
- if underlying impl changes, bust
|
|
23
|
-
- kinda reacty/reffy/signally
|
|
24
|
-
- decorator unwrapping and shit
|
|
25
|
-
- proactive deep invalidate
|
|
26
|
-
- tracked and versioned 'ops' but not result cached
|
|
27
|
-
- 'Versioned'
|
|
28
|
-
|
|
29
|
-
manifest stuff
|
|
30
|
-
- serialization_version
|
|
31
|
-
- lib_version
|
|
32
|
-
- lib_revision
|
|
33
|
-
|
|
34
|
-
fn manifest stuff
|
|
35
|
-
- source
|
|
36
|
-
- qualname
|
|
37
|
-
- location
|
|
38
|
-
|
|
39
|
-
See:
|
|
40
|
-
- https://github.com/amakelov/mandala
|
|
41
|
-
- https://jax.readthedocs.io/en/latest/autodidax.html
|
|
42
|
-
- tinyjit
|
|
43
|
-
- https://docs.python.org/3/library/pickle.html#pickle.Pickler.dispatch_table
|
|
44
|
-
|
|
45
|
-
names:
|
|
46
|
-
- CacheKey = unambiguous, fully qualified, unhashed map key - usually Cacheable + args
|
|
47
|
-
- Cacheable = usually a fn
|
|
48
|
-
- CacheableName = qualname of a cacheable
|
|
49
|
-
- dir structure: __package__/__qualname__/... ?
|
|
50
|
-
"""
|
|
51
|
-
import copy
|
|
52
|
-
import typing as ta
|
|
53
|
-
|
|
54
|
-
from omlish import collections as col
|
|
55
|
-
from omlish import dataclasses as dc
|
|
56
|
-
|
|
57
|
-
from .types import CacheableName
|
|
58
|
-
from .types import CacheableResolver
|
|
59
|
-
from .types import CacheableVersionMap
|
|
60
|
-
from .types import CacheKey
|
|
61
|
-
from .types import CacheResult
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
class Cache:
|
|
65
|
-
def __init__(
|
|
66
|
-
self,
|
|
67
|
-
resolver: CacheableResolver,
|
|
68
|
-
) -> None:
|
|
69
|
-
super().__init__()
|
|
70
|
-
|
|
71
|
-
self._resolver = resolver
|
|
72
|
-
|
|
73
|
-
self._dct: dict[CacheKey, Cache.Entry] = {}
|
|
74
|
-
|
|
75
|
-
self._stats = Cache.Stats()
|
|
76
|
-
|
|
77
|
-
@dc.dataclass()
|
|
78
|
-
class Stats:
|
|
79
|
-
num_hits: int = 0
|
|
80
|
-
num_misses: int = 0
|
|
81
|
-
num_invalidates: int = 0
|
|
82
|
-
num_puts: int = 0
|
|
83
|
-
|
|
84
|
-
@property
|
|
85
|
-
def stats(self) -> Stats:
|
|
86
|
-
return copy.deepcopy(self._stats)
|
|
87
|
-
|
|
88
|
-
@dc.dataclass(frozen=True)
|
|
89
|
-
class Entry:
|
|
90
|
-
key: CacheKey
|
|
91
|
-
versions: CacheableVersionMap
|
|
92
|
-
value: ta.Any
|
|
93
|
-
|
|
94
|
-
@dc.validate
|
|
95
|
-
def _check_types(self) -> bool:
|
|
96
|
-
return (
|
|
97
|
-
isinstance(self.key, CacheKey) and
|
|
98
|
-
isinstance(self.versions, col.frozendict)
|
|
99
|
-
)
|
|
100
|
-
|
|
101
|
-
def _build_version_map(self, names: ta.Iterable[CacheableName]) -> CacheableVersionMap:
|
|
102
|
-
dct = {}
|
|
103
|
-
for n in names:
|
|
104
|
-
c = self._resolver.resolve(n)
|
|
105
|
-
dct[n] = c.version
|
|
106
|
-
return col.frozendict(dct)
|
|
107
|
-
|
|
108
|
-
def get(self, key: CacheKey) -> CacheResult | None:
|
|
109
|
-
try:
|
|
110
|
-
entry = self._dct[key]
|
|
111
|
-
except KeyError:
|
|
112
|
-
self._stats.num_misses += 1
|
|
113
|
-
return None
|
|
114
|
-
|
|
115
|
-
new_versions = self._build_version_map(entry.versions)
|
|
116
|
-
if entry.versions != new_versions:
|
|
117
|
-
del self._dct[key]
|
|
118
|
-
self._stats.num_invalidates += 1
|
|
119
|
-
return None
|
|
120
|
-
|
|
121
|
-
self._stats.num_hits += 1
|
|
122
|
-
return CacheResult(
|
|
123
|
-
True,
|
|
124
|
-
entry.versions,
|
|
125
|
-
entry.value,
|
|
126
|
-
)
|
|
127
|
-
|
|
128
|
-
def put(self, key: CacheKey, versions: CacheableVersionMap, val: ta.Any) -> None:
|
|
129
|
-
if key in self._dct:
|
|
130
|
-
raise KeyError(key)
|
|
131
|
-
|
|
132
|
-
self._dct[key] = Cache.Entry(
|
|
133
|
-
key,
|
|
134
|
-
versions,
|
|
135
|
-
val,
|
|
136
|
-
)
|
|
137
|
-
self._stats.num_puts += 1
|
omdev/cache/comp/contexts.py
DELETED
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
import contextlib
|
|
2
|
-
import typing as ta
|
|
3
|
-
|
|
4
|
-
from omlish import check
|
|
5
|
-
from omlish import lang
|
|
6
|
-
|
|
7
|
-
from .cache import Cache
|
|
8
|
-
from .types import Cacheable
|
|
9
|
-
from .types import CacheableVersionMap
|
|
10
|
-
from .types import CacheKey
|
|
11
|
-
from .types import CacheResult
|
|
12
|
-
from .types import merge_version_maps
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
CacheT = ta.TypeVar('CacheT', bound='Cache')
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
##
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
_CURRENT_CACHE: Cache | None = None
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
@contextlib.contextmanager
|
|
25
|
-
def cache_context(cache: CacheT) -> ta.Iterator[CacheT]:
|
|
26
|
-
global _CURRENT_CACHE
|
|
27
|
-
prev = _CURRENT_CACHE
|
|
28
|
-
try:
|
|
29
|
-
_CURRENT_CACHE = cache
|
|
30
|
-
yield cache
|
|
31
|
-
finally:
|
|
32
|
-
check.is_(_CURRENT_CACHE, cache)
|
|
33
|
-
_CURRENT_CACHE = prev
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def get_current_cache() -> Cache | None:
|
|
37
|
-
return _CURRENT_CACHE
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
##
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
class CacheableContext(lang.Final):
|
|
44
|
-
def __init__(
|
|
45
|
-
self,
|
|
46
|
-
cacheable: Cacheable,
|
|
47
|
-
key: CacheKey,
|
|
48
|
-
*,
|
|
49
|
-
parent: ta.Optional['CacheableContext'] = None,
|
|
50
|
-
) -> None:
|
|
51
|
-
super().__init__()
|
|
52
|
-
self._cacheable = cacheable
|
|
53
|
-
self._key = key
|
|
54
|
-
self._parent = parent
|
|
55
|
-
|
|
56
|
-
self._result: CacheResult | None = None
|
|
57
|
-
self._children: list[CacheableContext] = []
|
|
58
|
-
|
|
59
|
-
if parent is not None:
|
|
60
|
-
check.state(not parent.has_result)
|
|
61
|
-
parent._children.append(self) # noqa
|
|
62
|
-
|
|
63
|
-
#
|
|
64
|
-
|
|
65
|
-
@property
|
|
66
|
-
def cacheable(self) -> Cacheable:
|
|
67
|
-
return self._cacheable
|
|
68
|
-
|
|
69
|
-
@property
|
|
70
|
-
def key(self) -> CacheKey:
|
|
71
|
-
return self._key
|
|
72
|
-
|
|
73
|
-
@property
|
|
74
|
-
def parent(self) -> ta.Optional['CacheableContext']:
|
|
75
|
-
return self._parent
|
|
76
|
-
|
|
77
|
-
@property
|
|
78
|
-
def children(self) -> ta.Sequence['CacheableContext']:
|
|
79
|
-
return self._children
|
|
80
|
-
|
|
81
|
-
#
|
|
82
|
-
|
|
83
|
-
@property
|
|
84
|
-
def has_result(self) -> bool:
|
|
85
|
-
return self._result is not None
|
|
86
|
-
|
|
87
|
-
def result(self) -> CacheResult:
|
|
88
|
-
return check.not_none(self._result)
|
|
89
|
-
|
|
90
|
-
def set_hit(self, result: CacheResult) -> None:
|
|
91
|
-
check.state(result.hit)
|
|
92
|
-
self._result = check.replacing_none(self._result, result)
|
|
93
|
-
self.result_versions()
|
|
94
|
-
|
|
95
|
-
def set_miss(self, val: ta.Any) -> None:
|
|
96
|
-
self._result = check.replacing_none(self._result, CacheResult(
|
|
97
|
-
False,
|
|
98
|
-
CacheableVersionMap(),
|
|
99
|
-
val,
|
|
100
|
-
))
|
|
101
|
-
self.result_versions()
|
|
102
|
-
|
|
103
|
-
@lang.cached_function
|
|
104
|
-
def result_versions(self) -> CacheableVersionMap:
|
|
105
|
-
r = check.not_none(self._result)
|
|
106
|
-
return merge_version_maps(
|
|
107
|
-
self._cacheable.as_version_map,
|
|
108
|
-
r.versions,
|
|
109
|
-
*[c.result_versions() for c in self._children],
|
|
110
|
-
)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
#
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
_CURRENT_CACHEABLE_CONTEXT: CacheableContext | None = None
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
@contextlib.contextmanager
|
|
120
|
-
def cacheable_context(
|
|
121
|
-
cacheable: Cacheable,
|
|
122
|
-
key: CacheKey,
|
|
123
|
-
) -> ta.Iterator[CacheableContext]:
|
|
124
|
-
global _CURRENT_CACHEABLE_CONTEXT
|
|
125
|
-
prev = _CURRENT_CACHEABLE_CONTEXT
|
|
126
|
-
ctx = CacheableContext(
|
|
127
|
-
cacheable,
|
|
128
|
-
key,
|
|
129
|
-
parent=prev,
|
|
130
|
-
)
|
|
131
|
-
try:
|
|
132
|
-
_CURRENT_CACHEABLE_CONTEXT = ctx
|
|
133
|
-
yield ctx
|
|
134
|
-
finally:
|
|
135
|
-
check.is_(_CURRENT_CACHEABLE_CONTEXT, ctx)
|
|
136
|
-
_CURRENT_CACHEABLE_CONTEXT = prev
|
omdev/cache/comp/fns.py
DELETED
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
import functools
|
|
2
|
-
import importlib
|
|
3
|
-
import typing as ta
|
|
4
|
-
|
|
5
|
-
from omlish import cached
|
|
6
|
-
from omlish import check
|
|
7
|
-
from omlish import collections as col
|
|
8
|
-
from omlish import dataclasses as dc
|
|
9
|
-
from omlish import lang
|
|
10
|
-
|
|
11
|
-
from .contexts import cacheable_context
|
|
12
|
-
from .contexts import get_current_cache
|
|
13
|
-
from .types import Cacheable
|
|
14
|
-
from .types import CacheableName
|
|
15
|
-
from .types import CacheableResolver
|
|
16
|
-
from .types import CacheKey
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
T = ta.TypeVar('T')
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
##
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
@dc.dataclass(frozen=True)
|
|
26
|
-
class FnCacheableName(CacheableName, lang.Final):
|
|
27
|
-
module: str
|
|
28
|
-
qualname: str
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
@dc.dataclass(frozen=True)
|
|
32
|
-
class FnCacheable(Cacheable, lang.Final):
|
|
33
|
-
fn: ta.Callable
|
|
34
|
-
version: int = dc.xfield(override=True)
|
|
35
|
-
|
|
36
|
-
@cached.property
|
|
37
|
-
def name(self) -> FnCacheableName:
|
|
38
|
-
return FnCacheableName(self.fn.__module__, self.fn.__qualname__) # noqa
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
class FnCacheableResolver(CacheableResolver):
|
|
42
|
-
def resolve(self, name: CacheableName) -> Cacheable:
|
|
43
|
-
fname = check.isinstance(name, FnCacheableName)
|
|
44
|
-
|
|
45
|
-
mod = importlib.import_module(fname.module)
|
|
46
|
-
obj = mod
|
|
47
|
-
for a in fname.qualname.split('.'):
|
|
48
|
-
obj = getattr(obj, a)
|
|
49
|
-
|
|
50
|
-
check.callable(obj)
|
|
51
|
-
fc = check.isinstance(obj.__cacheable__, FnCacheable)
|
|
52
|
-
|
|
53
|
-
return fc
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
@dc.dataclass(frozen=True)
|
|
57
|
-
class FnCacheKey(CacheKey[FnCacheableName], lang.Final):
|
|
58
|
-
args: tuple
|
|
59
|
-
kwargs: col.frozendict[str, ta.Any]
|
|
60
|
-
|
|
61
|
-
@dc.validate
|
|
62
|
-
def _check_fn_types(self) -> bool:
|
|
63
|
-
return (
|
|
64
|
-
isinstance(self.name, FnCacheableName) and
|
|
65
|
-
isinstance(self.args, tuple) and
|
|
66
|
-
isinstance(self.kwargs, col.frozendict)
|
|
67
|
-
)
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
##
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def cached_fn(version: int) -> ta.Callable[[T], T]:
|
|
74
|
-
def outer(fn):
|
|
75
|
-
@functools.wraps(fn)
|
|
76
|
-
def inner(*args, **kwargs):
|
|
77
|
-
# NOTE: just for testing :x allows updating
|
|
78
|
-
# TODO: proper wrapper obj probably (enforce name resolution)
|
|
79
|
-
cacheable = inner.__cacheable__ # type: ignore
|
|
80
|
-
|
|
81
|
-
if (cache := get_current_cache()) is not None:
|
|
82
|
-
key = FnCacheKey(
|
|
83
|
-
cacheable.name,
|
|
84
|
-
args,
|
|
85
|
-
col.frozendict(kwargs),
|
|
86
|
-
)
|
|
87
|
-
|
|
88
|
-
with cacheable_context(
|
|
89
|
-
cacheable,
|
|
90
|
-
key,
|
|
91
|
-
) as ctx:
|
|
92
|
-
if (hit := cache.get(key)) is not None:
|
|
93
|
-
ctx.set_hit(hit)
|
|
94
|
-
return hit.value
|
|
95
|
-
|
|
96
|
-
val = fn(*args, **kwargs)
|
|
97
|
-
ctx.set_miss(val)
|
|
98
|
-
cache.put(
|
|
99
|
-
key,
|
|
100
|
-
ctx.result_versions(),
|
|
101
|
-
val,
|
|
102
|
-
)
|
|
103
|
-
return val
|
|
104
|
-
|
|
105
|
-
else:
|
|
106
|
-
return fn(*args, **kwargs)
|
|
107
|
-
|
|
108
|
-
inner.__cacheable__ = FnCacheable( # type: ignore
|
|
109
|
-
fn,
|
|
110
|
-
version,
|
|
111
|
-
)
|
|
112
|
-
|
|
113
|
-
return inner
|
|
114
|
-
|
|
115
|
-
return outer # noqa
|
omdev/cache/comp/resolvers.py
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
from .types import Cacheable
|
|
2
|
-
from .types import CacheableName
|
|
3
|
-
from .types import CacheableResolver
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class CachingCacheableResolver(CacheableResolver):
|
|
7
|
-
def __init__(self, child: CacheableResolver) -> None:
|
|
8
|
-
super().__init__()
|
|
9
|
-
|
|
10
|
-
self._child = child
|
|
11
|
-
self._dct: dict[CacheableName, Cacheable] = {}
|
|
12
|
-
|
|
13
|
-
def clear(self) -> None:
|
|
14
|
-
self._dct.clear()
|
|
15
|
-
|
|
16
|
-
def resolve(self, name: CacheableName) -> Cacheable:
|
|
17
|
-
try:
|
|
18
|
-
return self._dct[name]
|
|
19
|
-
except KeyError:
|
|
20
|
-
pass
|
|
21
|
-
ret = self._child.resolve(name)
|
|
22
|
-
self._dct[name] = ret
|
|
23
|
-
return ret
|