omdev 0.0.0.dev222__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. omdev/ci/cache.py +148 -23
  2. omdev/ci/ci.py +50 -110
  3. omdev/ci/cli.py +24 -23
  4. omdev/ci/docker/__init__.py +0 -0
  5. omdev/ci/docker/buildcaching.py +69 -0
  6. omdev/ci/docker/cache.py +57 -0
  7. omdev/ci/docker/cacheserved.py +262 -0
  8. omdev/ci/{docker.py → docker/cmds.py} +1 -44
  9. omdev/ci/docker/dataserver.py +204 -0
  10. omdev/ci/docker/imagepulling.py +65 -0
  11. omdev/ci/docker/inject.py +37 -0
  12. omdev/ci/docker/packing.py +72 -0
  13. omdev/ci/docker/repositories.py +40 -0
  14. omdev/ci/docker/utils.py +48 -0
  15. omdev/ci/github/cache.py +35 -6
  16. omdev/ci/github/client.py +9 -2
  17. omdev/ci/github/inject.py +30 -0
  18. omdev/ci/inject.py +61 -0
  19. omdev/ci/utils.py +0 -49
  20. omdev/dataserver/__init__.py +1 -0
  21. omdev/dataserver/handlers.py +198 -0
  22. omdev/dataserver/http.py +69 -0
  23. omdev/dataserver/routes.py +49 -0
  24. omdev/dataserver/server.py +90 -0
  25. omdev/dataserver/targets.py +121 -0
  26. omdev/oci/building.py +107 -9
  27. omdev/oci/compression.py +8 -0
  28. omdev/oci/data.py +43 -0
  29. omdev/oci/datarefs.py +90 -50
  30. omdev/oci/dataserver.py +64 -0
  31. omdev/oci/loading.py +20 -0
  32. omdev/oci/media.py +20 -0
  33. omdev/oci/pack/__init__.py +0 -0
  34. omdev/oci/pack/packing.py +185 -0
  35. omdev/oci/pack/repositories.py +162 -0
  36. omdev/oci/pack/unpacking.py +204 -0
  37. omdev/oci/repositories.py +84 -2
  38. omdev/oci/tars.py +144 -0
  39. omdev/pyproject/resources/python.sh +1 -1
  40. omdev/scripts/ci.py +2137 -512
  41. omdev/scripts/interp.py +119 -22
  42. omdev/scripts/pyproject.py +141 -28
  43. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/METADATA +2 -2
  44. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/RECORD +48 -23
  45. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/LICENSE +0 -0
  46. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/WHEEL +0 -0
  47. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/entry_points.txt +0 -0
  48. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/top_level.txt +0 -0
omdev/scripts/ci.py CHANGED
@@ -21,6 +21,7 @@ import asyncio.base_subprocess
21
21
  import asyncio.subprocess
22
22
  import collections
23
23
  import contextlib
24
+ import contextvars
24
25
  import dataclasses as dc
25
26
  import datetime
26
27
  import functools
@@ -44,6 +45,7 @@ import types
44
45
  import typing as ta
45
46
  import urllib.parse
46
47
  import urllib.request
48
+ import weakref
47
49
 
48
50
 
49
51
  ########################################
@@ -80,6 +82,13 @@ ArgparseCmdFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
80
82
  ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
81
83
  AsyncExitStackedT = ta.TypeVar('AsyncExitStackedT', bound='AsyncExitStacked')
82
84
 
85
+ # ../../omlish/lite/inject.py
86
+ U = ta.TypeVar('U')
87
+ InjectorKeyCls = ta.Union[type, ta.NewType]
88
+ InjectorProviderFn = ta.Callable[['Injector'], ta.Any]
89
+ InjectorProviderFnMap = ta.Mapping['InjectorKey', 'InjectorProviderFn']
90
+ InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
91
+
83
92
  # ../../omlish/subprocesses.py
84
93
  SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
85
94
 
@@ -150,6 +159,27 @@ class ShellCmd:
150
159
  )
151
160
 
152
161
 
162
+ ########################################
163
+ # ../utils.py
164
+
165
+
166
+ ##
167
+
168
+
169
+ def read_yaml_file(yaml_file: str) -> ta.Any:
170
+ yaml = __import__('yaml')
171
+
172
+ with open(yaml_file) as f:
173
+ return yaml.safe_load(f)
174
+
175
+
176
+ ##
177
+
178
+
179
+ def sha256_str(s: str) -> str:
180
+ return hashlib.sha256(s.encode('utf-8')).hexdigest()
181
+
182
+
153
183
  ########################################
154
184
  # ../../../omlish/asyncs/asyncio/asyncio.py
155
185
 
@@ -797,6 +827,50 @@ json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON
797
827
  log = logging.getLogger(__name__)
798
828
 
799
829
 
830
+ ########################################
831
+ # ../../../omlish/lite/maybes.py
832
+
833
+
834
+ class Maybe(ta.Generic[T]):
835
+ @property
836
+ @abc.abstractmethod
837
+ def present(self) -> bool:
838
+ raise NotImplementedError
839
+
840
+ @abc.abstractmethod
841
+ def must(self) -> T:
842
+ raise NotImplementedError
843
+
844
+ @classmethod
845
+ def just(cls, v: T) -> 'Maybe[T]':
846
+ return tuple.__new__(_Maybe, (v,)) # noqa
847
+
848
+ _empty: ta.ClassVar['Maybe']
849
+
850
+ @classmethod
851
+ def empty(cls) -> 'Maybe[T]':
852
+ return Maybe._empty
853
+
854
+
855
+ class _Maybe(Maybe[T], tuple):
856
+ __slots__ = ()
857
+
858
+ def __init_subclass__(cls, **kwargs):
859
+ raise TypeError
860
+
861
+ @property
862
+ def present(self) -> bool:
863
+ return bool(self)
864
+
865
+ def must(self) -> T:
866
+ if not self:
867
+ raise ValueError
868
+ return self[0]
869
+
870
+
871
+ Maybe._empty = tuple.__new__(_Maybe, ()) # noqa
872
+
873
+
800
874
  ########################################
801
875
  # ../../../omlish/lite/reflect.py
802
876
 
@@ -1093,6 +1167,63 @@ class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
1093
1167
  self._underlying.handleError(record)
1094
1168
 
1095
1169
 
1170
+ ########################################
1171
+ # ../../../omlish/logs/timing.py
1172
+
1173
+
1174
+ ##
1175
+
1176
+
1177
+ class LogTimingContext:
1178
+ DEFAULT_LOG: ta.ClassVar[ta.Optional[logging.Logger]] = None
1179
+
1180
+ class _NOT_SPECIFIED: # noqa
1181
+ def __new__(cls, *args, **kwargs): # noqa
1182
+ raise TypeError
1183
+
1184
+ def __init__(
1185
+ self,
1186
+ description: str,
1187
+ *,
1188
+ log: ta.Union[logging.Logger, ta.Type[_NOT_SPECIFIED], None] = _NOT_SPECIFIED, # noqa
1189
+ level: int = logging.DEBUG,
1190
+ ) -> None:
1191
+ super().__init__()
1192
+
1193
+ self._description = description
1194
+ if log is self._NOT_SPECIFIED:
1195
+ log = self.DEFAULT_LOG # noqa
1196
+ self._log: ta.Optional[logging.Logger] = log # type: ignore
1197
+ self._level = level
1198
+
1199
+ def set_description(self, description: str) -> 'LogTimingContext':
1200
+ self._description = description
1201
+ return self
1202
+
1203
+ _begin_time: float
1204
+ _end_time: float
1205
+
1206
+ def __enter__(self) -> 'LogTimingContext':
1207
+ self._begin_time = time.time()
1208
+
1209
+ if self._log is not None:
1210
+ self._log.log(self._level, f'Begin : {self._description}') # noqa
1211
+
1212
+ return self
1213
+
1214
+ def __exit__(self, exc_type, exc_val, exc_tb):
1215
+ self._end_time = time.time()
1216
+
1217
+ if self._log is not None:
1218
+ self._log.log(
1219
+ self._level,
1220
+ f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
1221
+ )
1222
+
1223
+
1224
+ log_timing_context = LogTimingContext
1225
+
1226
+
1096
1227
  ########################################
1097
1228
  # ../../../omlish/os/files.py
1098
1229
 
@@ -1130,143 +1261,47 @@ def unlinking_if_exists(path: str) -> ta.Iterator[None]:
1130
1261
 
1131
1262
 
1132
1263
  ########################################
1133
- # ../cache.py
1264
+ # ../docker/utils.py
1265
+ """
1266
+ TODO:
1267
+ - some less stupid Dockerfile hash
1268
+ - doesn't change too much though
1269
+ """
1134
1270
 
1135
1271
 
1136
1272
  ##
1137
1273
 
1138
1274
 
1139
- @abc.abstractmethod
1140
- class FileCache(abc.ABC):
1141
- def __init__(
1142
- self,
1143
- *,
1144
- version: int = CI_CACHE_VERSION,
1145
- ) -> None:
1146
- super().__init__()
1147
-
1148
- check.isinstance(version, int)
1149
- check.arg(version >= 0)
1150
- self._version = version
1151
-
1152
- @property
1153
- def version(self) -> int:
1154
- return self._version
1155
-
1156
- #
1157
-
1158
- @abc.abstractmethod
1159
- def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
1160
- raise NotImplementedError
1161
-
1162
- @abc.abstractmethod
1163
- def put_file(
1164
- self,
1165
- key: str,
1166
- file_path: str,
1167
- *,
1168
- steal: bool = False,
1169
- ) -> ta.Awaitable[str]:
1170
- raise NotImplementedError
1171
-
1172
-
1173
- #
1174
-
1175
-
1176
- class DirectoryFileCache(FileCache):
1177
- def __init__(
1178
- self,
1179
- dir: str, # noqa
1180
- *,
1181
- no_create: bool = False,
1182
- no_purge: bool = False,
1183
- **kwargs: ta.Any,
1184
- ) -> None: # noqa
1185
- super().__init__(**kwargs)
1186
-
1187
- self._dir = dir
1188
- self._no_create = no_create
1189
- self._no_purge = no_purge
1190
-
1191
- #
1192
-
1193
- VERSION_FILE_NAME = '.ci-cache-version'
1194
-
1195
- @cached_nullary
1196
- def setup_dir(self) -> None:
1197
- version_file = os.path.join(self._dir, self.VERSION_FILE_NAME)
1198
-
1199
- if self._no_create:
1200
- check.state(os.path.isdir(self._dir))
1201
-
1202
- elif not os.path.isdir(self._dir):
1203
- os.makedirs(self._dir)
1204
- with open(version_file, 'w') as f:
1205
- f.write(str(self._version))
1206
- return
1207
-
1208
- with open(version_file) as f:
1209
- dir_version = int(f.read().strip())
1210
-
1211
- if dir_version == self._version:
1212
- return
1213
-
1214
- if self._no_purge:
1215
- raise RuntimeError(f'{dir_version=} != {self._version=}')
1216
-
1217
- dirs = [n for n in sorted(os.listdir(self._dir)) if os.path.isdir(os.path.join(self._dir, n))]
1218
- if dirs:
1219
- raise RuntimeError(
1220
- f'Refusing to remove stale cache dir {self._dir!r} '
1221
- f'due to present directories: {", ".join(dirs)}',
1222
- )
1275
+ def build_docker_file_hash(docker_file: str) -> str:
1276
+ with open(docker_file) as f:
1277
+ contents = f.read()
1223
1278
 
1224
- for n in sorted(os.listdir(self._dir)):
1225
- if n.startswith('.'):
1226
- continue
1227
- fp = os.path.join(self._dir, n)
1228
- check.state(os.path.isfile(fp))
1229
- log.debug('Purging stale cache file: %s', fp)
1230
- os.unlink(fp)
1279
+ return sha256_str(contents)
1231
1280
 
1232
- os.unlink(version_file)
1233
1281
 
1234
- with open(version_file, 'w') as f:
1235
- f.write(str(self._version))
1282
+ ##
1236
1283
 
1237
- #
1238
1284
 
1239
- def get_cache_file_path(
1240
- self,
1241
- key: str,
1242
- ) -> str:
1243
- self.setup_dir()
1244
- return os.path.join(self._dir, key)
1285
+ def read_docker_tar_image_tag(tar_file: str) -> str:
1286
+ with tarfile.open(tar_file) as tf:
1287
+ with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
1288
+ m = mf.read()
1245
1289
 
1246
- def format_incomplete_file(self, f: str) -> str:
1247
- return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
1290
+ manifests = json.loads(m.decode('utf-8'))
1291
+ manifest = check.single(manifests)
1292
+ tag = check.non_empty_str(check.single(manifest['RepoTags']))
1293
+ return tag
1248
1294
 
1249
- #
1250
1295
 
1251
- async def get_file(self, key: str) -> ta.Optional[str]:
1252
- cache_file_path = self.get_cache_file_path(key)
1253
- if not os.path.exists(cache_file_path):
1254
- return None
1255
- return cache_file_path
1296
+ def read_docker_tar_image_id(tar_file: str) -> str:
1297
+ with tarfile.open(tar_file) as tf:
1298
+ with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
1299
+ i = mf.read()
1256
1300
 
1257
- async def put_file(
1258
- self,
1259
- key: str,
1260
- file_path: str,
1261
- *,
1262
- steal: bool = False,
1263
- ) -> str:
1264
- cache_file_path = self.get_cache_file_path(key)
1265
- if steal:
1266
- shutil.move(file_path, cache_file_path)
1267
- else:
1268
- shutil.copyfile(file_path, cache_file_path)
1269
- return cache_file_path
1301
+ index = json.loads(i.decode('utf-8'))
1302
+ manifest = check.single(index['manifests'])
1303
+ image_id = check.non_empty_str(manifest['digest'])
1304
+ return image_id
1270
1305
 
1271
1306
 
1272
1307
  ########################################
@@ -1491,111 +1526,45 @@ def is_in_github_actions() -> bool:
1491
1526
 
1492
1527
 
1493
1528
  ########################################
1494
- # ../utils.py
1529
+ # ../../../omlish/argparse/cli.py
1530
+ """
1531
+ TODO:
1532
+ - default command
1533
+ - auto match all underscores to hyphens
1534
+ - pre-run, post-run hooks
1535
+ - exitstack?
1536
+ """
1495
1537
 
1496
1538
 
1497
1539
  ##
1498
1540
 
1499
1541
 
1500
- def read_yaml_file(yaml_file: str) -> ta.Any:
1501
- yaml = __import__('yaml')
1502
-
1503
- with open(yaml_file) as f:
1504
- return yaml.safe_load(f)
1542
+ @dc.dataclass(eq=False)
1543
+ class ArgparseArg:
1544
+ args: ta.Sequence[ta.Any]
1545
+ kwargs: ta.Mapping[str, ta.Any]
1546
+ dest: ta.Optional[str] = None
1505
1547
 
1548
+ def __get__(self, instance, owner=None):
1549
+ if instance is None:
1550
+ return self
1551
+ return getattr(instance.args, self.dest) # type: ignore
1506
1552
 
1507
- ##
1508
1553
 
1554
+ def argparse_arg(*args, **kwargs) -> ArgparseArg:
1555
+ return ArgparseArg(args, kwargs)
1509
1556
 
1510
- def sha256_str(s: str) -> str:
1511
- return hashlib.sha256(s.encode('utf-8')).hexdigest()
1512
1557
 
1558
+ #
1513
1559
 
1514
- ##
1515
1560
 
1561
+ @dc.dataclass(eq=False)
1562
+ class ArgparseCmd:
1563
+ name: str
1564
+ fn: ArgparseCmdFn
1565
+ args: ta.Sequence[ArgparseArg] = () # noqa
1516
1566
 
1517
- class LogTimingContext:
1518
- DEFAULT_LOG: ta.ClassVar[logging.Logger] = log
1519
-
1520
- def __init__(
1521
- self,
1522
- description: str,
1523
- *,
1524
- log: ta.Optional[logging.Logger] = None, # noqa
1525
- level: int = logging.DEBUG,
1526
- ) -> None:
1527
- super().__init__()
1528
-
1529
- self._description = description
1530
- self._log = log if log is not None else self.DEFAULT_LOG
1531
- self._level = level
1532
-
1533
- def set_description(self, description: str) -> 'LogTimingContext':
1534
- self._description = description
1535
- return self
1536
-
1537
- _begin_time: float
1538
- _end_time: float
1539
-
1540
- def __enter__(self) -> 'LogTimingContext':
1541
- self._begin_time = time.time()
1542
-
1543
- self._log.log(self._level, f'Begin : {self._description}') # noqa
1544
-
1545
- return self
1546
-
1547
- def __exit__(self, exc_type, exc_val, exc_tb):
1548
- self._end_time = time.time()
1549
-
1550
- self._log.log(
1551
- self._level,
1552
- f'End : {self._description} - {self._end_time - self._begin_time:0.2f} s elapsed',
1553
- )
1554
-
1555
-
1556
- log_timing_context = LogTimingContext
1557
-
1558
-
1559
- ########################################
1560
- # ../../../omlish/argparse/cli.py
1561
- """
1562
- TODO:
1563
- - default command
1564
- - auto match all underscores to hyphens
1565
- - pre-run, post-run hooks
1566
- - exitstack?
1567
- """
1568
-
1569
-
1570
- ##
1571
-
1572
-
1573
- @dc.dataclass(eq=False)
1574
- class ArgparseArg:
1575
- args: ta.Sequence[ta.Any]
1576
- kwargs: ta.Mapping[str, ta.Any]
1577
- dest: ta.Optional[str] = None
1578
-
1579
- def __get__(self, instance, owner=None):
1580
- if instance is None:
1581
- return self
1582
- return getattr(instance.args, self.dest) # type: ignore
1583
-
1584
-
1585
- def argparse_arg(*args, **kwargs) -> ArgparseArg:
1586
- return ArgparseArg(args, kwargs)
1587
-
1588
-
1589
- #
1590
-
1591
-
1592
- @dc.dataclass(eq=False)
1593
- class ArgparseCmd:
1594
- name: str
1595
- fn: ArgparseCmdFn
1596
- args: ta.Sequence[ArgparseArg] = () # noqa
1597
-
1598
- # _: dc.KW_ONLY
1567
+ # _: dc.KW_ONLY
1599
1568
 
1600
1569
  aliases: ta.Optional[ta.Sequence[str]] = None
1601
1570
  parent: ta.Optional['ArgparseCmd'] = None
@@ -1902,173 +1871,1516 @@ class AsyncExitStacked:
1902
1871
  return await es.enter_async_context(cm)
1903
1872
 
1904
1873
 
1905
- ##
1874
+ ##
1875
+
1876
+
1877
+ @contextlib.contextmanager
1878
+ def defer(fn: ta.Callable) -> ta.Generator[ta.Callable, None, None]:
1879
+ try:
1880
+ yield fn
1881
+ finally:
1882
+ fn()
1883
+
1884
+
1885
+ @contextlib.asynccontextmanager
1886
+ async def adefer(fn: ta.Callable) -> ta.AsyncGenerator[ta.Callable, None]:
1887
+ try:
1888
+ yield fn
1889
+ finally:
1890
+ await fn()
1891
+
1892
+
1893
+ ##
1894
+
1895
+
1896
+ @contextlib.contextmanager
1897
+ def attr_setting(obj, attr, val, *, default=None): # noqa
1898
+ not_set = object()
1899
+ orig = getattr(obj, attr, not_set)
1900
+ try:
1901
+ setattr(obj, attr, val)
1902
+ if orig is not not_set:
1903
+ yield orig
1904
+ else:
1905
+ yield default
1906
+ finally:
1907
+ if orig is not_set:
1908
+ delattr(obj, attr)
1909
+ else:
1910
+ setattr(obj, attr, orig)
1911
+
1912
+
1913
+ ##
1914
+
1915
+
1916
+ class aclosing(contextlib.AbstractAsyncContextManager): # noqa
1917
+ def __init__(self, thing):
1918
+ self.thing = thing
1919
+
1920
+ async def __aenter__(self):
1921
+ return self.thing
1922
+
1923
+ async def __aexit__(self, *exc_info):
1924
+ await self.thing.aclose()
1925
+
1926
+
1927
+ ########################################
1928
+ # ../../../omlish/lite/inject.py
1929
+
1930
+
1931
+ ###
1932
+ # types
1933
+
1934
+
1935
+ @dc.dataclass(frozen=True)
1936
+ class InjectorKey(ta.Generic[T]):
1937
+ # Before PEP-560 typing.Generic was a metaclass with a __new__ that takes a 'cls' arg, so instantiating a dataclass
1938
+ # with kwargs (such as through dc.replace) causes `TypeError: __new__() got multiple values for argument 'cls'`.
1939
+ # See:
1940
+ # - https://github.com/python/cpython/commit/d911e40e788fb679723d78b6ea11cabf46caed5a
1941
+ # - https://gist.github.com/wrmsr/4468b86efe9f373b6b114bfe85b98fd3
1942
+ cls_: InjectorKeyCls
1943
+
1944
+ tag: ta.Any = None
1945
+ array: bool = False
1946
+
1947
+
1948
+ def is_valid_injector_key_cls(cls: ta.Any) -> bool:
1949
+ return isinstance(cls, type) or is_new_type(cls)
1950
+
1951
+
1952
+ def check_valid_injector_key_cls(cls: T) -> T:
1953
+ if not is_valid_injector_key_cls(cls):
1954
+ raise TypeError(cls)
1955
+ return cls
1956
+
1957
+
1958
+ ##
1959
+
1960
+
1961
+ class InjectorProvider(abc.ABC):
1962
+ @abc.abstractmethod
1963
+ def provider_fn(self) -> InjectorProviderFn:
1964
+ raise NotImplementedError
1965
+
1966
+
1967
+ ##
1968
+
1969
+
1970
+ @dc.dataclass(frozen=True)
1971
+ class InjectorBinding:
1972
+ key: InjectorKey
1973
+ provider: InjectorProvider
1974
+
1975
+ def __post_init__(self) -> None:
1976
+ check.isinstance(self.key, InjectorKey)
1977
+ check.isinstance(self.provider, InjectorProvider)
1978
+
1979
+
1980
+ class InjectorBindings(abc.ABC):
1981
+ @abc.abstractmethod
1982
+ def bindings(self) -> ta.Iterator[InjectorBinding]:
1983
+ raise NotImplementedError
1984
+
1985
+ ##
1986
+
1987
+
1988
+ class Injector(abc.ABC):
1989
+ @abc.abstractmethod
1990
+ def try_provide(self, key: ta.Any) -> Maybe[ta.Any]:
1991
+ raise NotImplementedError
1992
+
1993
+ @abc.abstractmethod
1994
+ def provide(self, key: ta.Any) -> ta.Any:
1995
+ raise NotImplementedError
1996
+
1997
+ @abc.abstractmethod
1998
+ def provide_kwargs(
1999
+ self,
2000
+ obj: ta.Any,
2001
+ *,
2002
+ skip_args: int = 0,
2003
+ skip_kwargs: ta.Optional[ta.Iterable[ta.Any]] = None,
2004
+ ) -> ta.Mapping[str, ta.Any]:
2005
+ raise NotImplementedError
2006
+
2007
+ @abc.abstractmethod
2008
+ def inject(
2009
+ self,
2010
+ obj: ta.Any,
2011
+ *,
2012
+ args: ta.Optional[ta.Sequence[ta.Any]] = None,
2013
+ kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None,
2014
+ ) -> ta.Any:
2015
+ raise NotImplementedError
2016
+
2017
+ def __getitem__(
2018
+ self,
2019
+ target: ta.Union[InjectorKey[T], ta.Type[T]],
2020
+ ) -> T:
2021
+ return self.provide(target)
2022
+
2023
+
2024
+ ###
2025
+ # exceptions
2026
+
2027
+
2028
+ class InjectorError(Exception):
2029
+ pass
2030
+
2031
+
2032
+ @dc.dataclass()
2033
+ class InjectorKeyError(InjectorError):
2034
+ key: InjectorKey
2035
+
2036
+ source: ta.Any = None
2037
+ name: ta.Optional[str] = None
2038
+
2039
+
2040
+ class UnboundInjectorKeyError(InjectorKeyError):
2041
+ pass
2042
+
2043
+
2044
+ class DuplicateInjectorKeyError(InjectorKeyError):
2045
+ pass
2046
+
2047
+
2048
+ class CyclicDependencyInjectorKeyError(InjectorKeyError):
2049
+ pass
2050
+
2051
+
2052
+ ###
2053
+ # keys
2054
+
2055
+
2056
+ def as_injector_key(o: ta.Any) -> InjectorKey:
2057
+ if o is inspect.Parameter.empty:
2058
+ raise TypeError(o)
2059
+ if isinstance(o, InjectorKey):
2060
+ return o
2061
+ if is_valid_injector_key_cls(o):
2062
+ return InjectorKey(o)
2063
+ raise TypeError(o)
2064
+
2065
+
2066
+ ###
2067
+ # providers
2068
+
2069
+
2070
+ @dc.dataclass(frozen=True)
2071
+ class FnInjectorProvider(InjectorProvider):
2072
+ fn: ta.Any
2073
+
2074
+ def __post_init__(self) -> None:
2075
+ check.not_isinstance(self.fn, type)
2076
+
2077
+ def provider_fn(self) -> InjectorProviderFn:
2078
+ def pfn(i: Injector) -> ta.Any:
2079
+ return i.inject(self.fn)
2080
+
2081
+ return pfn
2082
+
2083
+
2084
+ @dc.dataclass(frozen=True)
2085
+ class CtorInjectorProvider(InjectorProvider):
2086
+ cls_: type
2087
+
2088
+ def __post_init__(self) -> None:
2089
+ check.isinstance(self.cls_, type)
2090
+
2091
+ def provider_fn(self) -> InjectorProviderFn:
2092
+ def pfn(i: Injector) -> ta.Any:
2093
+ return i.inject(self.cls_)
2094
+
2095
+ return pfn
2096
+
2097
+
2098
+ @dc.dataclass(frozen=True)
2099
+ class ConstInjectorProvider(InjectorProvider):
2100
+ v: ta.Any
2101
+
2102
+ def provider_fn(self) -> InjectorProviderFn:
2103
+ return lambda _: self.v
2104
+
2105
+
2106
+ @dc.dataclass(frozen=True)
2107
+ class SingletonInjectorProvider(InjectorProvider):
2108
+ p: InjectorProvider
2109
+
2110
+ def __post_init__(self) -> None:
2111
+ check.isinstance(self.p, InjectorProvider)
2112
+
2113
+ def provider_fn(self) -> InjectorProviderFn:
2114
+ v = not_set = object()
2115
+
2116
+ def pfn(i: Injector) -> ta.Any:
2117
+ nonlocal v
2118
+ if v is not_set:
2119
+ v = ufn(i)
2120
+ return v
2121
+
2122
+ ufn = self.p.provider_fn()
2123
+ return pfn
2124
+
2125
+
2126
+ @dc.dataclass(frozen=True)
2127
+ class LinkInjectorProvider(InjectorProvider):
2128
+ k: InjectorKey
2129
+
2130
+ def __post_init__(self) -> None:
2131
+ check.isinstance(self.k, InjectorKey)
2132
+
2133
+ def provider_fn(self) -> InjectorProviderFn:
2134
+ def pfn(i: Injector) -> ta.Any:
2135
+ return i.provide(self.k)
2136
+
2137
+ return pfn
2138
+
2139
+
2140
+ @dc.dataclass(frozen=True)
2141
+ class ArrayInjectorProvider(InjectorProvider):
2142
+ ps: ta.Sequence[InjectorProvider]
2143
+
2144
+ def provider_fn(self) -> InjectorProviderFn:
2145
+ ps = [p.provider_fn() for p in self.ps]
2146
+
2147
+ def pfn(i: Injector) -> ta.Any:
2148
+ rv = []
2149
+ for ep in ps:
2150
+ o = ep(i)
2151
+ rv.append(o)
2152
+ return rv
2153
+
2154
+ return pfn
2155
+
2156
+
2157
+ ###
2158
+ # bindings
2159
+
2160
+
2161
+ @dc.dataclass(frozen=True)
2162
+ class _InjectorBindings(InjectorBindings):
2163
+ bs: ta.Optional[ta.Sequence[InjectorBinding]] = None
2164
+ ps: ta.Optional[ta.Sequence[InjectorBindings]] = None
2165
+
2166
+ def bindings(self) -> ta.Iterator[InjectorBinding]:
2167
+ if self.bs is not None:
2168
+ yield from self.bs
2169
+ if self.ps is not None:
2170
+ for p in self.ps:
2171
+ yield from p.bindings()
2172
+
2173
+
2174
+ def as_injector_bindings(*args: InjectorBindingOrBindings) -> InjectorBindings:
2175
+ bs: ta.List[InjectorBinding] = []
2176
+ ps: ta.List[InjectorBindings] = []
2177
+
2178
+ for a in args:
2179
+ if isinstance(a, InjectorBindings):
2180
+ ps.append(a)
2181
+ elif isinstance(a, InjectorBinding):
2182
+ bs.append(a)
2183
+ else:
2184
+ raise TypeError(a)
2185
+
2186
+ return _InjectorBindings(
2187
+ bs or None,
2188
+ ps or None,
2189
+ )
2190
+
2191
+
2192
+ ##
2193
+
2194
+
2195
+ def build_injector_provider_map(bs: InjectorBindings) -> ta.Mapping[InjectorKey, InjectorProvider]:
2196
+ pm: ta.Dict[InjectorKey, InjectorProvider] = {}
2197
+ am: ta.Dict[InjectorKey, ta.List[InjectorProvider]] = {}
2198
+
2199
+ for b in bs.bindings():
2200
+ if b.key.array:
2201
+ al = am.setdefault(b.key, [])
2202
+ if isinstance(b.provider, ArrayInjectorProvider):
2203
+ al.extend(b.provider.ps)
2204
+ else:
2205
+ al.append(b.provider)
2206
+ else:
2207
+ if b.key in pm:
2208
+ raise KeyError(b.key)
2209
+ pm[b.key] = b.provider
2210
+
2211
+ if am:
2212
+ for k, aps in am.items():
2213
+ pm[k] = ArrayInjectorProvider(aps)
2214
+
2215
+ return pm
2216
+
2217
+
2218
+ ###
2219
+ # overrides
2220
+
2221
+
2222
+ @dc.dataclass(frozen=True)
2223
+ class OverridesInjectorBindings(InjectorBindings):
2224
+ p: InjectorBindings
2225
+ m: ta.Mapping[InjectorKey, InjectorBinding]
2226
+
2227
+ def bindings(self) -> ta.Iterator[InjectorBinding]:
2228
+ for b in self.p.bindings():
2229
+ yield self.m.get(b.key, b)
2230
+
2231
+
2232
+ def injector_override(p: InjectorBindings, *args: InjectorBindingOrBindings) -> InjectorBindings:
2233
+ m: ta.Dict[InjectorKey, InjectorBinding] = {}
2234
+
2235
+ for b in as_injector_bindings(*args).bindings():
2236
+ if b.key in m:
2237
+ raise DuplicateInjectorKeyError(b.key)
2238
+ m[b.key] = b
2239
+
2240
+ return OverridesInjectorBindings(p, m)
2241
+
2242
+
2243
+ ###
2244
+ # scopes
2245
+
2246
+
2247
+ class InjectorScope(abc.ABC): # noqa
2248
+ def __init__(
2249
+ self,
2250
+ *,
2251
+ _i: Injector,
2252
+ ) -> None:
2253
+ check.not_in(abc.ABC, type(self).__bases__)
2254
+
2255
+ super().__init__()
2256
+
2257
+ self._i = _i
2258
+
2259
+ all_seeds: ta.Iterable[_InjectorScopeSeed] = self._i.provide(InjectorKey(_InjectorScopeSeed, array=True))
2260
+ self._sks = {s.k for s in all_seeds if s.sc is type(self)}
2261
+
2262
+ #
2263
+
2264
+ @dc.dataclass(frozen=True)
2265
+ class State:
2266
+ seeds: ta.Dict[InjectorKey, ta.Any]
2267
+ provisions: ta.Dict[InjectorKey, ta.Any] = dc.field(default_factory=dict)
2268
+
2269
+ def new_state(self, vs: ta.Mapping[InjectorKey, ta.Any]) -> State:
2270
+ vs = dict(vs)
2271
+ check.equal(set(vs.keys()), self._sks)
2272
+ return InjectorScope.State(vs)
2273
+
2274
+ #
2275
+
2276
+ @abc.abstractmethod
2277
+ def state(self) -> State:
2278
+ raise NotImplementedError
2279
+
2280
+ @abc.abstractmethod
2281
+ def enter(self, vs: ta.Mapping[InjectorKey, ta.Any]) -> ta.ContextManager[None]:
2282
+ raise NotImplementedError
2283
+
2284
+
2285
+ class ExclusiveInjectorScope(InjectorScope, abc.ABC):
2286
+ _st: ta.Optional[InjectorScope.State] = None
2287
+
2288
+ def state(self) -> InjectorScope.State:
2289
+ return check.not_none(self._st)
2290
+
2291
+ @contextlib.contextmanager
2292
+ def enter(self, vs: ta.Mapping[InjectorKey, ta.Any]) -> ta.Iterator[None]:
2293
+ check.none(self._st)
2294
+ self._st = self.new_state(vs)
2295
+ try:
2296
+ yield
2297
+ finally:
2298
+ self._st = None
2299
+
2300
+
2301
+ class ContextvarInjectorScope(InjectorScope, abc.ABC):
2302
+ _cv: contextvars.ContextVar
2303
+
2304
+ def __init_subclass__(cls, **kwargs: ta.Any) -> None:
2305
+ super().__init_subclass__(**kwargs)
2306
+ check.not_in(abc.ABC, cls.__bases__)
2307
+ check.state(not hasattr(cls, '_cv'))
2308
+ cls._cv = contextvars.ContextVar(f'{cls.__name__}_cv')
2309
+
2310
+ def state(self) -> InjectorScope.State:
2311
+ return self._cv.get()
2312
+
2313
+ @contextlib.contextmanager
2314
+ def enter(self, vs: ta.Mapping[InjectorKey, ta.Any]) -> ta.Iterator[None]:
2315
+ try:
2316
+ self._cv.get()
2317
+ except LookupError:
2318
+ pass
2319
+ else:
2320
+ raise RuntimeError(f'Scope already entered: {self}')
2321
+ st = self.new_state(vs)
2322
+ tok = self._cv.set(st)
2323
+ try:
2324
+ yield
2325
+ finally:
2326
+ self._cv.reset(tok)
2327
+
2328
+
2329
+ #
2330
+
2331
+
2332
+ @dc.dataclass(frozen=True)
2333
+ class ScopedInjectorProvider(InjectorProvider):
2334
+ p: InjectorProvider
2335
+ k: InjectorKey
2336
+ sc: ta.Type[InjectorScope]
2337
+
2338
+ def __post_init__(self) -> None:
2339
+ check.isinstance(self.p, InjectorProvider)
2340
+ check.isinstance(self.k, InjectorKey)
2341
+ check.issubclass(self.sc, InjectorScope)
2342
+
2343
+ def provider_fn(self) -> InjectorProviderFn:
2344
+ def pfn(i: Injector) -> ta.Any:
2345
+ st = i[self.sc].state()
2346
+ try:
2347
+ return st.provisions[self.k]
2348
+ except KeyError:
2349
+ pass
2350
+ v = ufn(i)
2351
+ st.provisions[self.k] = v
2352
+ return v
2353
+
2354
+ ufn = self.p.provider_fn()
2355
+ return pfn
2356
+
2357
+
2358
+ @dc.dataclass(frozen=True)
2359
+ class _ScopeSeedInjectorProvider(InjectorProvider):
2360
+ k: InjectorKey
2361
+ sc: ta.Type[InjectorScope]
2362
+
2363
+ def __post_init__(self) -> None:
2364
+ check.isinstance(self.k, InjectorKey)
2365
+ check.issubclass(self.sc, InjectorScope)
2366
+
2367
+ def provider_fn(self) -> InjectorProviderFn:
2368
+ def pfn(i: Injector) -> ta.Any:
2369
+ st = i[self.sc].state()
2370
+ return st.seeds[self.k]
2371
+ return pfn
2372
+
2373
+
2374
+ def bind_injector_scope(sc: ta.Type[InjectorScope]) -> InjectorBindingOrBindings:
2375
+ return InjectorBinder.bind(sc, singleton=True)
2376
+
2377
+
2378
+ #
2379
+
2380
+
2381
+ @dc.dataclass(frozen=True)
2382
+ class _InjectorScopeSeed:
2383
+ sc: ta.Type['InjectorScope']
2384
+ k: InjectorKey
2385
+
2386
+ def __post_init__(self) -> None:
2387
+ check.issubclass(self.sc, InjectorScope)
2388
+ check.isinstance(self.k, InjectorKey)
2389
+
2390
+
2391
+ def bind_injector_scope_seed(k: ta.Any, sc: ta.Type[InjectorScope]) -> InjectorBindingOrBindings:
2392
+ kk = as_injector_key(k)
2393
+ return as_injector_bindings(
2394
+ InjectorBinding(kk, _ScopeSeedInjectorProvider(kk, sc)),
2395
+ InjectorBinder.bind(_InjectorScopeSeed(sc, kk), array=True),
2396
+ )
2397
+
2398
+
2399
+ ###
2400
+ # inspection
2401
+
2402
+
2403
+ class _InjectionInspection(ta.NamedTuple):
2404
+ signature: inspect.Signature
2405
+ type_hints: ta.Mapping[str, ta.Any]
2406
+ args_offset: int
2407
+
2408
+
2409
+ _INJECTION_INSPECTION_CACHE: ta.MutableMapping[ta.Any, _InjectionInspection] = weakref.WeakKeyDictionary()
2410
+
2411
+
2412
+ def _do_injection_inspect(obj: ta.Any) -> _InjectionInspection:
2413
+ tgt = obj
2414
+ if isinstance(tgt, type) and tgt.__init__ is not object.__init__: # type: ignore[misc]
2415
+ # Python 3.8's inspect.signature can't handle subclasses overriding __new__, always generating *args/**kwargs.
2416
+ # - https://bugs.python.org/issue40897
2417
+ # - https://github.com/python/cpython/commit/df7c62980d15acd3125dfbd81546dad359f7add7
2418
+ tgt = tgt.__init__ # type: ignore[misc]
2419
+ has_generic_base = True
2420
+ else:
2421
+ has_generic_base = False
2422
+
2423
+ # inspect.signature(eval_str=True) was added in 3.10 and we have to support 3.8, so we have to get_type_hints to
2424
+ # eval str annotations *in addition to* getting the signature for parameter information.
2425
+ uw = tgt
2426
+ has_partial = False
2427
+ while True:
2428
+ if isinstance(uw, functools.partial):
2429
+ has_partial = True
2430
+ uw = uw.func
2431
+ else:
2432
+ if (uw2 := inspect.unwrap(uw)) is uw:
2433
+ break
2434
+ uw = uw2
2435
+
2436
+ if has_generic_base and has_partial:
2437
+ raise InjectorError(
2438
+ 'Injector inspection does not currently support both a typing.Generic base and a functools.partial: '
2439
+ f'{obj}',
2440
+ )
2441
+
2442
+ return _InjectionInspection(
2443
+ inspect.signature(tgt),
2444
+ ta.get_type_hints(uw),
2445
+ 1 if has_generic_base else 0,
2446
+ )
2447
+
2448
+
2449
+ def _injection_inspect(obj: ta.Any) -> _InjectionInspection:
2450
+ try:
2451
+ return _INJECTION_INSPECTION_CACHE[obj]
2452
+ except TypeError:
2453
+ return _do_injection_inspect(obj)
2454
+ except KeyError:
2455
+ pass
2456
+ insp = _do_injection_inspect(obj)
2457
+ _INJECTION_INSPECTION_CACHE[obj] = insp
2458
+ return insp
2459
+
2460
+
2461
+ class InjectionKwarg(ta.NamedTuple):
2462
+ name: str
2463
+ key: InjectorKey
2464
+ has_default: bool
2465
+
2466
+
2467
+ class InjectionKwargsTarget(ta.NamedTuple):
2468
+ obj: ta.Any
2469
+ kwargs: ta.Sequence[InjectionKwarg]
2470
+
2471
+
2472
+ def build_injection_kwargs_target(
2473
+ obj: ta.Any,
2474
+ *,
2475
+ skip_args: int = 0,
2476
+ skip_kwargs: ta.Optional[ta.Iterable[str]] = None,
2477
+ raw_optional: bool = False,
2478
+ ) -> InjectionKwargsTarget:
2479
+ insp = _injection_inspect(obj)
2480
+
2481
+ params = list(insp.signature.parameters.values())
2482
+
2483
+ skip_names: ta.Set[str] = set()
2484
+ if skip_kwargs is not None:
2485
+ skip_names.update(check.not_isinstance(skip_kwargs, str))
2486
+
2487
+ seen: ta.Set[InjectorKey] = set()
2488
+ kws: ta.List[InjectionKwarg] = []
2489
+ for p in params[insp.args_offset + skip_args:]:
2490
+ if p.name in skip_names:
2491
+ continue
2492
+
2493
+ if p.annotation is inspect.Signature.empty:
2494
+ if p.default is not inspect.Parameter.empty:
2495
+ raise KeyError(f'{obj}, {p.name}')
2496
+ continue
2497
+
2498
+ if p.kind not in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY):
2499
+ raise TypeError(insp)
2500
+
2501
+ # 3.8 inspect.signature doesn't eval_str but typing.get_type_hints does, so prefer that.
2502
+ ann = insp.type_hints.get(p.name, p.annotation)
2503
+ if (
2504
+ not raw_optional and
2505
+ is_optional_alias(ann)
2506
+ ):
2507
+ ann = get_optional_alias_arg(ann)
2508
+
2509
+ k = as_injector_key(ann)
2510
+
2511
+ if k in seen:
2512
+ raise DuplicateInjectorKeyError(k)
2513
+ seen.add(k)
2514
+
2515
+ kws.append(InjectionKwarg(
2516
+ p.name,
2517
+ k,
2518
+ p.default is not inspect.Parameter.empty,
2519
+ ))
2520
+
2521
+ return InjectionKwargsTarget(
2522
+ obj,
2523
+ kws,
2524
+ )
2525
+
2526
+
2527
+ ###
2528
+ # injector
2529
+
2530
+
2531
+ _INJECTOR_INJECTOR_KEY: InjectorKey[Injector] = InjectorKey(Injector)
2532
+
2533
+
2534
+ @dc.dataclass(frozen=True)
2535
+ class _InjectorEager:
2536
+ key: InjectorKey
2537
+
2538
+
2539
+ _INJECTOR_EAGER_ARRAY_KEY: InjectorKey[_InjectorEager] = InjectorKey(_InjectorEager, array=True)
2540
+
2541
+
2542
+ class _Injector(Injector):
2543
+ _DEFAULT_BINDINGS: ta.ClassVar[ta.List[InjectorBinding]] = []
2544
+
2545
+ def __init__(self, bs: InjectorBindings, p: ta.Optional[Injector] = None) -> None:
2546
+ super().__init__()
2547
+
2548
+ self._bs = check.isinstance(bs, InjectorBindings)
2549
+ self._p: ta.Optional[Injector] = check.isinstance(p, (Injector, type(None)))
2550
+
2551
+ self._pfm = {
2552
+ k: v.provider_fn()
2553
+ for k, v in build_injector_provider_map(as_injector_bindings(
2554
+ *self._DEFAULT_BINDINGS,
2555
+ bs,
2556
+ )).items()
2557
+ }
2558
+
2559
+ if _INJECTOR_INJECTOR_KEY in self._pfm:
2560
+ raise DuplicateInjectorKeyError(_INJECTOR_INJECTOR_KEY)
2561
+
2562
+ self.__cur_req: ta.Optional[_Injector._Request] = None
2563
+
2564
+ if _INJECTOR_EAGER_ARRAY_KEY in self._pfm:
2565
+ for e in self.provide(_INJECTOR_EAGER_ARRAY_KEY):
2566
+ self.provide(e.key)
2567
+
2568
+ class _Request:
2569
+ def __init__(self, injector: '_Injector') -> None:
2570
+ super().__init__()
2571
+ self._injector = injector
2572
+ self._provisions: ta.Dict[InjectorKey, Maybe] = {}
2573
+ self._seen_keys: ta.Set[InjectorKey] = set()
2574
+
2575
+ def handle_key(self, key: InjectorKey) -> Maybe[Maybe]:
2576
+ try:
2577
+ return Maybe.just(self._provisions[key])
2578
+ except KeyError:
2579
+ pass
2580
+ if key in self._seen_keys:
2581
+ raise CyclicDependencyInjectorKeyError(key)
2582
+ self._seen_keys.add(key)
2583
+ return Maybe.empty()
2584
+
2585
+ def handle_provision(self, key: InjectorKey, mv: Maybe) -> Maybe:
2586
+ check.in_(key, self._seen_keys)
2587
+ check.not_in(key, self._provisions)
2588
+ self._provisions[key] = mv
2589
+ return mv
2590
+
2591
+ @contextlib.contextmanager
2592
+ def _current_request(self) -> ta.Generator[_Request, None, None]:
2593
+ if (cr := self.__cur_req) is not None:
2594
+ yield cr
2595
+ return
2596
+
2597
+ cr = self._Request(self)
2598
+ try:
2599
+ self.__cur_req = cr
2600
+ yield cr
2601
+ finally:
2602
+ self.__cur_req = None
2603
+
2604
+ def try_provide(self, key: ta.Any) -> Maybe[ta.Any]:
2605
+ key = as_injector_key(key)
2606
+
2607
+ cr: _Injector._Request
2608
+ with self._current_request() as cr:
2609
+ if (rv := cr.handle_key(key)).present:
2610
+ return rv.must()
2611
+
2612
+ if key == _INJECTOR_INJECTOR_KEY:
2613
+ return cr.handle_provision(key, Maybe.just(self))
2614
+
2615
+ fn = self._pfm.get(key)
2616
+ if fn is not None:
2617
+ return cr.handle_provision(key, Maybe.just(fn(self)))
2618
+
2619
+ if self._p is not None:
2620
+ pv = self._p.try_provide(key)
2621
+ if pv is not None:
2622
+ return cr.handle_provision(key, Maybe.empty())
2623
+
2624
+ return cr.handle_provision(key, Maybe.empty())
2625
+
2626
+ def provide(self, key: ta.Any) -> ta.Any:
2627
+ v = self.try_provide(key)
2628
+ if v.present:
2629
+ return v.must()
2630
+ raise UnboundInjectorKeyError(key)
2631
+
2632
+ def provide_kwargs(
2633
+ self,
2634
+ obj: ta.Any,
2635
+ *,
2636
+ skip_args: int = 0,
2637
+ skip_kwargs: ta.Optional[ta.Iterable[ta.Any]] = None,
2638
+ ) -> ta.Mapping[str, ta.Any]:
2639
+ kt = build_injection_kwargs_target(
2640
+ obj,
2641
+ skip_args=skip_args,
2642
+ skip_kwargs=skip_kwargs,
2643
+ )
2644
+
2645
+ ret: ta.Dict[str, ta.Any] = {}
2646
+ for kw in kt.kwargs:
2647
+ if kw.has_default:
2648
+ if not (mv := self.try_provide(kw.key)).present:
2649
+ continue
2650
+ v = mv.must()
2651
+ else:
2652
+ v = self.provide(kw.key)
2653
+ ret[kw.name] = v
2654
+ return ret
2655
+
2656
+ def inject(
2657
+ self,
2658
+ obj: ta.Any,
2659
+ *,
2660
+ args: ta.Optional[ta.Sequence[ta.Any]] = None,
2661
+ kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None,
2662
+ ) -> ta.Any:
2663
+ provided = self.provide_kwargs(
2664
+ obj,
2665
+ skip_args=len(args) if args is not None else 0,
2666
+ skip_kwargs=kwargs if kwargs is not None else None,
2667
+ )
2668
+
2669
+ return obj(
2670
+ *(args if args is not None else ()),
2671
+ **(kwargs if kwargs is not None else {}),
2672
+ **provided,
2673
+ )
2674
+
2675
+
2676
+ ###
2677
+ # binder
2678
+
2679
+
2680
+ class InjectorBinder:
2681
+ def __new__(cls, *args, **kwargs): # noqa
2682
+ raise TypeError
2683
+
2684
+ _FN_TYPES: ta.ClassVar[ta.Tuple[type, ...]] = (
2685
+ types.FunctionType,
2686
+ types.MethodType,
2687
+
2688
+ classmethod,
2689
+ staticmethod,
2690
+
2691
+ functools.partial,
2692
+ functools.partialmethod,
2693
+ )
2694
+
2695
+ @classmethod
2696
+ def _is_fn(cls, obj: ta.Any) -> bool:
2697
+ return isinstance(obj, cls._FN_TYPES)
2698
+
2699
+ @classmethod
2700
+ def bind_as_fn(cls, icls: ta.Type[T]) -> ta.Type[T]:
2701
+ check.isinstance(icls, type)
2702
+ if icls not in cls._FN_TYPES:
2703
+ cls._FN_TYPES = (*cls._FN_TYPES, icls)
2704
+ return icls
2705
+
2706
+ _BANNED_BIND_TYPES: ta.ClassVar[ta.Tuple[type, ...]] = (
2707
+ InjectorProvider,
2708
+ )
2709
+
2710
+ @classmethod
2711
+ def bind(
2712
+ cls,
2713
+ obj: ta.Any,
2714
+ *,
2715
+ key: ta.Any = None,
2716
+ tag: ta.Any = None,
2717
+ array: ta.Optional[bool] = None, # noqa
2718
+
2719
+ to_fn: ta.Any = None,
2720
+ to_ctor: ta.Any = None,
2721
+ to_const: ta.Any = None,
2722
+ to_key: ta.Any = None,
2723
+
2724
+ in_: ta.Optional[ta.Type[InjectorScope]] = None,
2725
+ singleton: bool = False,
2726
+
2727
+ eager: bool = False,
2728
+ ) -> InjectorBindingOrBindings:
2729
+ if obj is None or obj is inspect.Parameter.empty:
2730
+ raise TypeError(obj)
2731
+ if isinstance(obj, cls._BANNED_BIND_TYPES):
2732
+ raise TypeError(obj)
2733
+
2734
+ #
2735
+
2736
+ if key is not None:
2737
+ key = as_injector_key(key)
2738
+
2739
+ #
2740
+
2741
+ has_to = (
2742
+ to_fn is not None or
2743
+ to_ctor is not None or
2744
+ to_const is not None or
2745
+ to_key is not None
2746
+ )
2747
+ if isinstance(obj, InjectorKey):
2748
+ if key is None:
2749
+ key = obj
2750
+ elif isinstance(obj, type):
2751
+ if not has_to:
2752
+ to_ctor = obj
2753
+ if key is None:
2754
+ key = InjectorKey(obj)
2755
+ elif cls._is_fn(obj) and not has_to:
2756
+ to_fn = obj
2757
+ if key is None:
2758
+ insp = _injection_inspect(obj)
2759
+ key_cls: ta.Any = check_valid_injector_key_cls(check.not_none(insp.type_hints.get('return')))
2760
+ key = InjectorKey(key_cls)
2761
+ else:
2762
+ if to_const is not None:
2763
+ raise TypeError('Cannot bind instance with to_const')
2764
+ to_const = obj
2765
+ if key is None:
2766
+ key = InjectorKey(type(obj))
2767
+ del has_to
2768
+
2769
+ #
2770
+
2771
+ if tag is not None:
2772
+ if key.tag is not None:
2773
+ raise TypeError('Tag already set')
2774
+ key = dc.replace(key, tag=tag)
2775
+
2776
+ if array is not None:
2777
+ key = dc.replace(key, array=array)
2778
+
2779
+ #
2780
+
2781
+ providers: ta.List[InjectorProvider] = []
2782
+ if to_fn is not None:
2783
+ providers.append(FnInjectorProvider(to_fn))
2784
+ if to_ctor is not None:
2785
+ providers.append(CtorInjectorProvider(to_ctor))
2786
+ if to_const is not None:
2787
+ providers.append(ConstInjectorProvider(to_const))
2788
+ if to_key is not None:
2789
+ providers.append(LinkInjectorProvider(as_injector_key(to_key)))
2790
+ if not providers:
2791
+ raise TypeError('Must specify provider')
2792
+ if len(providers) > 1:
2793
+ raise TypeError('May not specify multiple providers')
2794
+ provider = check.single(providers)
2795
+
2796
+ #
2797
+
2798
+ pws: ta.List[ta.Any] = []
2799
+ if in_ is not None:
2800
+ check.issubclass(in_, InjectorScope)
2801
+ check.not_in(abc.ABC, in_.__bases__)
2802
+ pws.append(functools.partial(ScopedInjectorProvider, k=key, sc=in_))
2803
+ if singleton:
2804
+ pws.append(SingletonInjectorProvider)
2805
+ if len(pws) > 1:
2806
+ raise TypeError('May not specify multiple provider wrappers')
2807
+ elif pws:
2808
+ provider = check.single(pws)(provider)
2809
+
2810
+ #
2811
+
2812
+ binding = InjectorBinding(key, provider)
2813
+
2814
+ #
2815
+
2816
+ extras: ta.List[InjectorBinding] = []
2817
+
2818
+ if eager:
2819
+ extras.append(bind_injector_eager_key(key))
2820
+
2821
+ #
2822
+
2823
+ if extras:
2824
+ return as_injector_bindings(binding, *extras)
2825
+ else:
2826
+ return binding
2827
+
2828
+
2829
+ ###
2830
+ # injection helpers
2831
+
2832
+
2833
+ def make_injector_factory(
2834
+ fn: ta.Callable[..., T],
2835
+ cls: U,
2836
+ ann: ta.Any = None,
2837
+ ) -> ta.Callable[..., U]:
2838
+ if ann is None:
2839
+ ann = cls
2840
+
2841
+ def outer(injector: Injector) -> ann:
2842
+ def inner(*args, **kwargs):
2843
+ return injector.inject(fn, args=args, kwargs=kwargs)
2844
+ return cls(inner) # type: ignore
2845
+
2846
+ return outer
2847
+
2848
+
2849
+ def bind_injector_array(
2850
+ obj: ta.Any = None,
2851
+ *,
2852
+ tag: ta.Any = None,
2853
+ ) -> InjectorBindingOrBindings:
2854
+ key = as_injector_key(obj)
2855
+ if tag is not None:
2856
+ if key.tag is not None:
2857
+ raise ValueError('Must not specify multiple tags')
2858
+ key = dc.replace(key, tag=tag)
2859
+
2860
+ if key.array:
2861
+ raise ValueError('Key must not be array')
2862
+
2863
+ return InjectorBinding(
2864
+ dc.replace(key, array=True),
2865
+ ArrayInjectorProvider([]),
2866
+ )
2867
+
2868
+
2869
+ def make_injector_array_type(
2870
+ ele: ta.Union[InjectorKey, InjectorKeyCls],
2871
+ cls: U,
2872
+ ann: ta.Any = None,
2873
+ ) -> ta.Callable[..., U]:
2874
+ if isinstance(ele, InjectorKey):
2875
+ if not ele.array:
2876
+ raise InjectorError('Provided key must be array', ele)
2877
+ key = ele
2878
+ else:
2879
+ key = dc.replace(as_injector_key(ele), array=True)
2880
+
2881
+ if ann is None:
2882
+ ann = cls
2883
+
2884
+ def inner(injector: Injector) -> ann:
2885
+ return cls(injector.provide(key)) # type: ignore[operator]
2886
+
2887
+ return inner
2888
+
2889
+
2890
+ def bind_injector_eager_key(key: ta.Any) -> InjectorBinding:
2891
+ return InjectorBinding(_INJECTOR_EAGER_ARRAY_KEY, ConstInjectorProvider(_InjectorEager(as_injector_key(key))))
2892
+
2893
+
2894
+ ###
2895
+ # api
2896
+
2897
+
2898
+ class InjectionApi:
2899
+ # keys
2900
+
2901
+ def as_key(self, o: ta.Any) -> InjectorKey:
2902
+ return as_injector_key(o)
2903
+
2904
+ def array(self, o: ta.Any) -> InjectorKey:
2905
+ return dc.replace(as_injector_key(o), array=True)
2906
+
2907
+ def tag(self, o: ta.Any, t: ta.Any) -> InjectorKey:
2908
+ return dc.replace(as_injector_key(o), tag=t)
2909
+
2910
+ # bindings
2911
+
2912
+ def as_bindings(self, *args: InjectorBindingOrBindings) -> InjectorBindings:
2913
+ return as_injector_bindings(*args)
2914
+
2915
+ # overrides
2916
+
2917
+ def override(self, p: InjectorBindings, *args: InjectorBindingOrBindings) -> InjectorBindings:
2918
+ return injector_override(p, *args)
2919
+
2920
+ # scopes
2921
+
2922
+ def bind_scope(self, sc: ta.Type[InjectorScope]) -> InjectorBindingOrBindings:
2923
+ return bind_injector_scope(sc)
2924
+
2925
+ def bind_scope_seed(self, k: ta.Any, sc: ta.Type[InjectorScope]) -> InjectorBindingOrBindings:
2926
+ return bind_injector_scope_seed(k, sc)
2927
+
2928
+ # injector
2929
+
2930
+ def create_injector(self, *args: InjectorBindingOrBindings, parent: ta.Optional[Injector] = None) -> Injector:
2931
+ return _Injector(as_injector_bindings(*args), parent)
2932
+
2933
+ # binder
2934
+
2935
+ def bind(
2936
+ self,
2937
+ obj: ta.Any,
2938
+ *,
2939
+ key: ta.Any = None,
2940
+ tag: ta.Any = None,
2941
+ array: ta.Optional[bool] = None, # noqa
2942
+
2943
+ to_fn: ta.Any = None,
2944
+ to_ctor: ta.Any = None,
2945
+ to_const: ta.Any = None,
2946
+ to_key: ta.Any = None,
2947
+
2948
+ in_: ta.Optional[ta.Type[InjectorScope]] = None,
2949
+ singleton: bool = False,
2950
+
2951
+ eager: bool = False,
2952
+ ) -> InjectorBindingOrBindings:
2953
+ return InjectorBinder.bind(
2954
+ obj,
2955
+
2956
+ key=key,
2957
+ tag=tag,
2958
+ array=array,
2959
+
2960
+ to_fn=to_fn,
2961
+ to_ctor=to_ctor,
2962
+ to_const=to_const,
2963
+ to_key=to_key,
2964
+
2965
+ in_=in_,
2966
+ singleton=singleton,
2967
+
2968
+ eager=eager,
2969
+ )
2970
+
2971
+ # helpers
2972
+
2973
+ def bind_factory(
2974
+ self,
2975
+ fn: ta.Callable[..., T],
2976
+ cls_: U,
2977
+ ann: ta.Any = None,
2978
+ ) -> InjectorBindingOrBindings:
2979
+ return self.bind(make_injector_factory(fn, cls_, ann))
2980
+
2981
+ def bind_array(
2982
+ self,
2983
+ obj: ta.Any = None,
2984
+ *,
2985
+ tag: ta.Any = None,
2986
+ ) -> InjectorBindingOrBindings:
2987
+ return bind_injector_array(obj, tag=tag)
2988
+
2989
+ def bind_array_type(
2990
+ self,
2991
+ ele: ta.Union[InjectorKey, InjectorKeyCls],
2992
+ cls_: U,
2993
+ ann: ta.Any = None,
2994
+ ) -> InjectorBindingOrBindings:
2995
+ return self.bind(make_injector_array_type(ele, cls_, ann))
2996
+
2997
+
2998
+ inj = InjectionApi()
2999
+
3000
+
3001
+ ########################################
3002
+ # ../../../omlish/lite/runtime.py
3003
+
3004
+
3005
+ @cached_nullary
3006
+ def is_debugger_attached() -> bool:
3007
+ return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
3008
+
3009
+
3010
+ LITE_REQUIRED_PYTHON_VERSION = (3, 8)
3011
+
3012
+
3013
+ def check_lite_runtime_version() -> None:
3014
+ if sys.version_info < LITE_REQUIRED_PYTHON_VERSION:
3015
+ raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
3016
+
3017
+
3018
+ ########################################
3019
+ # ../../../omlish/lite/timing.py
3020
+
3021
+
3022
+ LogTimingContext.DEFAULT_LOG = log
3023
+
3024
+ log_timing_context = log_timing_context # noqa
3025
+
3026
+
3027
+ ########################################
3028
+ # ../../../omlish/logs/json.py
3029
+ """
3030
+ TODO:
3031
+ - translate json keys
3032
+ """
3033
+
3034
+
3035
+ class JsonLogFormatter(logging.Formatter):
3036
+ KEYS: ta.Mapping[str, bool] = {
3037
+ 'name': False,
3038
+ 'msg': False,
3039
+ 'args': False,
3040
+ 'levelname': False,
3041
+ 'levelno': False,
3042
+ 'pathname': False,
3043
+ 'filename': False,
3044
+ 'module': False,
3045
+ 'exc_info': True,
3046
+ 'exc_text': True,
3047
+ 'stack_info': True,
3048
+ 'lineno': False,
3049
+ 'funcName': False,
3050
+ 'created': False,
3051
+ 'msecs': False,
3052
+ 'relativeCreated': False,
3053
+ 'thread': False,
3054
+ 'threadName': False,
3055
+ 'processName': False,
3056
+ 'process': False,
3057
+ }
3058
+
3059
+ def __init__(
3060
+ self,
3061
+ *args: ta.Any,
3062
+ json_dumps: ta.Optional[ta.Callable[[ta.Any], str]] = None,
3063
+ **kwargs: ta.Any,
3064
+ ) -> None:
3065
+ super().__init__(*args, **kwargs)
3066
+
3067
+ if json_dumps is None:
3068
+ json_dumps = json_dumps_compact
3069
+ self._json_dumps = json_dumps
3070
+
3071
+ def format(self, record: logging.LogRecord) -> str:
3072
+ dct = {
3073
+ k: v
3074
+ for k, o in self.KEYS.items()
3075
+ for v in [getattr(record, k)]
3076
+ if not (o and v is None)
3077
+ }
3078
+ return self._json_dumps(dct)
3079
+
3080
+
3081
+ ########################################
3082
+ # ../../../omlish/os/temp.py
3083
+
3084
+
3085
+ def make_temp_file(**kwargs: ta.Any) -> str:
3086
+ file_fd, file = tempfile.mkstemp(**kwargs)
3087
+ os.close(file_fd)
3088
+ return file
3089
+
3090
+
3091
+ @contextlib.contextmanager
3092
+ def temp_file_context(**kwargs: ta.Any) -> ta.Iterator[str]:
3093
+ path = make_temp_file(**kwargs)
3094
+ try:
3095
+ yield path
3096
+ finally:
3097
+ unlink_if_exists(path)
3098
+
3099
+
3100
+ @contextlib.contextmanager
3101
+ def temp_dir_context(
3102
+ root_dir: ta.Optional[str] = None,
3103
+ **kwargs: ta.Any,
3104
+ ) -> ta.Iterator[str]:
3105
+ path = tempfile.mkdtemp(dir=root_dir, **kwargs)
3106
+ try:
3107
+ yield path
3108
+ finally:
3109
+ shutil.rmtree(path, ignore_errors=True)
3110
+
3111
+
3112
+ @contextlib.contextmanager
3113
+ def temp_named_file_context(
3114
+ root_dir: ta.Optional[str] = None,
3115
+ cleanup: bool = True,
3116
+ **kwargs: ta.Any,
3117
+ ) -> ta.Iterator[tempfile._TemporaryFileWrapper]: # noqa
3118
+ with tempfile.NamedTemporaryFile(dir=root_dir, delete=False, **kwargs) as f:
3119
+ try:
3120
+ yield f
3121
+ finally:
3122
+ if cleanup:
3123
+ shutil.rmtree(f.name, ignore_errors=True)
3124
+
3125
+
3126
+ ########################################
3127
+ # ../cache.py
3128
+
3129
+
3130
+ CacheVersion = ta.NewType('CacheVersion', int)
3131
+
3132
+
3133
+ ##
3134
+
3135
+
3136
+ class FileCache(abc.ABC):
3137
+ DEFAULT_CACHE_VERSION: ta.ClassVar[CacheVersion] = CacheVersion(CI_CACHE_VERSION)
3138
+
3139
+ def __init__(
3140
+ self,
3141
+ *,
3142
+ version: ta.Optional[CacheVersion] = None,
3143
+ ) -> None:
3144
+ super().__init__()
3145
+
3146
+ if version is None:
3147
+ version = self.DEFAULT_CACHE_VERSION
3148
+ check.isinstance(version, int)
3149
+ check.arg(version >= 0)
3150
+ self._version: CacheVersion = version
3151
+
3152
+ @property
3153
+ def version(self) -> CacheVersion:
3154
+ return self._version
3155
+
3156
+ #
3157
+
3158
+ @abc.abstractmethod
3159
+ def get_file(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
3160
+ raise NotImplementedError
3161
+
3162
+ @abc.abstractmethod
3163
+ def put_file(
3164
+ self,
3165
+ key: str,
3166
+ file_path: str,
3167
+ *,
3168
+ steal: bool = False,
3169
+ ) -> ta.Awaitable[str]:
3170
+ raise NotImplementedError
3171
+
3172
+
3173
+ #
3174
+
3175
+
3176
+ class DirectoryFileCache(FileCache):
3177
+ @dc.dataclass(frozen=True)
3178
+ class Config:
3179
+ dir: str
3180
+
3181
+ no_create: bool = False
3182
+ no_purge: bool = False
3183
+
3184
+ def __init__(
3185
+ self,
3186
+ config: Config,
3187
+ *,
3188
+ version: ta.Optional[CacheVersion] = None,
3189
+ ) -> None: # noqa
3190
+ super().__init__(
3191
+ version=version,
3192
+ )
3193
+
3194
+ self._config = config
3195
+
3196
+ @property
3197
+ def dir(self) -> str:
3198
+ return self._config.dir
3199
+
3200
+ #
3201
+
3202
+ VERSION_FILE_NAME = '.ci-cache-version'
3203
+
3204
+ @cached_nullary
3205
+ def setup_dir(self) -> None:
3206
+ version_file = os.path.join(self.dir, self.VERSION_FILE_NAME)
3207
+
3208
+ if self._config.no_create:
3209
+ check.state(os.path.isdir(self.dir))
3210
+
3211
+ elif not os.path.isdir(self.dir):
3212
+ os.makedirs(self.dir)
3213
+ with open(version_file, 'w') as f:
3214
+ f.write(str(self._version))
3215
+ return
3216
+
3217
+ # NOTE: intentionally raises FileNotFoundError to refuse to use an existing non-cache dir as a cache dir.
3218
+ with open(version_file) as f:
3219
+ dir_version = int(f.read().strip())
3220
+
3221
+ if dir_version == self._version:
3222
+ return
3223
+
3224
+ if self._config.no_purge:
3225
+ raise RuntimeError(f'{dir_version=} != {self._version=}')
3226
+
3227
+ dirs = [n for n in sorted(os.listdir(self.dir)) if os.path.isdir(os.path.join(self.dir, n))]
3228
+ if dirs:
3229
+ raise RuntimeError(
3230
+ f'Refusing to remove stale cache dir {self.dir!r} '
3231
+ f'due to present directories: {", ".join(dirs)}',
3232
+ )
3233
+
3234
+ for n in sorted(os.listdir(self.dir)):
3235
+ if n.startswith('.'):
3236
+ continue
3237
+ fp = os.path.join(self.dir, n)
3238
+ check.state(os.path.isfile(fp))
3239
+ log.debug('Purging stale cache file: %s', fp)
3240
+ os.unlink(fp)
3241
+
3242
+ os.unlink(version_file)
3243
+
3244
+ with open(version_file, 'w') as f:
3245
+ f.write(str(self._version))
3246
+
3247
+ #
3248
+
3249
+ def get_cache_file_path(
3250
+ self,
3251
+ key: str,
3252
+ ) -> str:
3253
+ self.setup_dir()
3254
+ return os.path.join(self.dir, key)
3255
+
3256
+ def format_incomplete_file(self, f: str) -> str:
3257
+ return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
3258
+
3259
+ #
1906
3260
 
3261
+ async def get_file(self, key: str) -> ta.Optional[str]:
3262
+ cache_file_path = self.get_cache_file_path(key)
3263
+ if not os.path.exists(cache_file_path):
3264
+ return None
3265
+ return cache_file_path
1907
3266
 
1908
- @contextlib.contextmanager
1909
- def defer(fn: ta.Callable) -> ta.Generator[ta.Callable, None, None]:
1910
- try:
1911
- yield fn
1912
- finally:
1913
- fn()
3267
+ async def put_file(
3268
+ self,
3269
+ key: str,
3270
+ file_path: str,
3271
+ *,
3272
+ steal: bool = False,
3273
+ ) -> str:
3274
+ cache_file_path = self.get_cache_file_path(key)
3275
+ if steal:
3276
+ shutil.move(file_path, cache_file_path)
3277
+ else:
3278
+ shutil.copyfile(file_path, cache_file_path)
3279
+ return cache_file_path
1914
3280
 
1915
3281
 
1916
- @contextlib.asynccontextmanager
1917
- async def adefer(fn: ta.Callable) -> ta.AsyncGenerator[ta.Callable, None]:
1918
- try:
1919
- yield fn
1920
- finally:
1921
- await fn()
3282
+ ##
1922
3283
 
1923
3284
 
1924
- ##
3285
+ class DataCache:
3286
+ @dc.dataclass(frozen=True)
3287
+ class Data(abc.ABC): # noqa
3288
+ pass
1925
3289
 
3290
+ @dc.dataclass(frozen=True)
3291
+ class BytesData(Data):
3292
+ data: bytes
1926
3293
 
1927
- @contextlib.contextmanager
1928
- def attr_setting(obj, attr, val, *, default=None): # noqa
1929
- not_set = object()
1930
- orig = getattr(obj, attr, not_set)
1931
- try:
1932
- setattr(obj, attr, val)
1933
- if orig is not not_set:
1934
- yield orig
1935
- else:
1936
- yield default
1937
- finally:
1938
- if orig is not_set:
1939
- delattr(obj, attr)
1940
- else:
1941
- setattr(obj, attr, orig)
3294
+ @dc.dataclass(frozen=True)
3295
+ class FileData(Data):
3296
+ file_path: str
1942
3297
 
3298
+ @dc.dataclass(frozen=True)
3299
+ class UrlData(Data):
3300
+ url: str
1943
3301
 
1944
- ##
3302
+ #
1945
3303
 
3304
+ @abc.abstractmethod
3305
+ def get_data(self, key: str) -> ta.Awaitable[ta.Optional[Data]]:
3306
+ raise NotImplementedError
1946
3307
 
1947
- class aclosing(contextlib.AbstractAsyncContextManager): # noqa
1948
- def __init__(self, thing):
1949
- self.thing = thing
3308
+ @abc.abstractmethod
3309
+ def put_data(self, key: str, data: Data) -> ta.Awaitable[None]:
3310
+ raise NotImplementedError
1950
3311
 
1951
- async def __aenter__(self):
1952
- return self.thing
1953
3312
 
1954
- async def __aexit__(self, *exc_info):
1955
- await self.thing.aclose()
3313
+ #
1956
3314
 
1957
3315
 
1958
- ########################################
1959
- # ../../../omlish/lite/runtime.py
3316
+ @functools.singledispatch
3317
+ async def read_data_cache_data(data: DataCache.Data) -> bytes:
3318
+ raise TypeError(data)
1960
3319
 
1961
3320
 
1962
- @cached_nullary
1963
- def is_debugger_attached() -> bool:
1964
- return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
3321
+ @read_data_cache_data.register
3322
+ async def _(data: DataCache.BytesData) -> bytes:
3323
+ return data.data
1965
3324
 
1966
3325
 
1967
- LITE_REQUIRED_PYTHON_VERSION = (3, 8)
3326
+ @read_data_cache_data.register
3327
+ async def _(data: DataCache.FileData) -> bytes:
3328
+ with open(data.file_path, 'rb') as f: # noqa
3329
+ return f.read()
1968
3330
 
1969
3331
 
1970
- def check_lite_runtime_version() -> None:
1971
- if sys.version_info < LITE_REQUIRED_PYTHON_VERSION:
1972
- raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
3332
+ @read_data_cache_data.register
3333
+ async def _(data: DataCache.UrlData) -> bytes:
3334
+ def inner() -> bytes:
3335
+ with urllib.request.urlopen(urllib.request.Request( # noqa
3336
+ data.url,
3337
+ )) as resp:
3338
+ return resp.read()
1973
3339
 
3340
+ return await asyncio.get_running_loop().run_in_executor(None, inner)
1974
3341
 
1975
- ########################################
1976
- # ../../../omlish/logs/json.py
1977
- """
1978
- TODO:
1979
- - translate json keys
1980
- """
1981
3342
 
3343
+ #
1982
3344
 
1983
- class JsonLogFormatter(logging.Formatter):
1984
- KEYS: ta.Mapping[str, bool] = {
1985
- 'name': False,
1986
- 'msg': False,
1987
- 'args': False,
1988
- 'levelname': False,
1989
- 'levelno': False,
1990
- 'pathname': False,
1991
- 'filename': False,
1992
- 'module': False,
1993
- 'exc_info': True,
1994
- 'exc_text': True,
1995
- 'stack_info': True,
1996
- 'lineno': False,
1997
- 'funcName': False,
1998
- 'created': False,
1999
- 'msecs': False,
2000
- 'relativeCreated': False,
2001
- 'thread': False,
2002
- 'threadName': False,
2003
- 'processName': False,
2004
- 'process': False,
2005
- }
2006
3345
 
3346
+ class FileCacheDataCache(DataCache):
2007
3347
  def __init__(
2008
3348
  self,
2009
- *args: ta.Any,
2010
- json_dumps: ta.Optional[ta.Callable[[ta.Any], str]] = None,
2011
- **kwargs: ta.Any,
3349
+ file_cache: FileCache,
2012
3350
  ) -> None:
2013
- super().__init__(*args, **kwargs)
2014
-
2015
- if json_dumps is None:
2016
- json_dumps = json_dumps_compact
2017
- self._json_dumps = json_dumps
2018
-
2019
- def format(self, record: logging.LogRecord) -> str:
2020
- dct = {
2021
- k: v
2022
- for k, o in self.KEYS.items()
2023
- for v in [getattr(record, k)]
2024
- if not (o and v is None)
2025
- }
2026
- return self._json_dumps(dct)
2027
-
3351
+ super().__init__()
2028
3352
 
2029
- ########################################
2030
- # ../../../omlish/os/temp.py
3353
+ self._file_cache = file_cache
2031
3354
 
3355
+ async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
3356
+ if (file_path := await self._file_cache.get_file(key)) is None:
3357
+ return None
2032
3358
 
2033
- def make_temp_file(**kwargs: ta.Any) -> str:
2034
- file_fd, file = tempfile.mkstemp(**kwargs)
2035
- os.close(file_fd)
2036
- return file
3359
+ return DataCache.FileData(file_path)
2037
3360
 
3361
+ async def put_data(self, key: str, data: DataCache.Data) -> None:
3362
+ steal = False
2038
3363
 
2039
- @contextlib.contextmanager
2040
- def temp_file_context(**kwargs: ta.Any) -> ta.Iterator[str]:
2041
- path = make_temp_file(**kwargs)
2042
- try:
2043
- yield path
2044
- finally:
2045
- unlink_if_exists(path)
3364
+ if isinstance(data, DataCache.BytesData):
3365
+ file_path = make_temp_file()
3366
+ with open(file_path, 'wb') as f: # noqa
3367
+ f.write(data.data)
3368
+ steal = True
2046
3369
 
3370
+ elif isinstance(data, DataCache.FileData):
3371
+ file_path = data.file_path
2047
3372
 
2048
- @contextlib.contextmanager
2049
- def temp_dir_context(
2050
- root_dir: ta.Optional[str] = None,
2051
- **kwargs: ta.Any,
2052
- ) -> ta.Iterator[str]:
2053
- path = tempfile.mkdtemp(dir=root_dir, **kwargs)
2054
- try:
2055
- yield path
2056
- finally:
2057
- shutil.rmtree(path, ignore_errors=True)
3373
+ elif isinstance(data, DataCache.UrlData):
3374
+ raise NotImplementedError
2058
3375
 
3376
+ else:
3377
+ raise TypeError(data)
2059
3378
 
2060
- @contextlib.contextmanager
2061
- def temp_named_file_context(
2062
- root_dir: ta.Optional[str] = None,
2063
- cleanup: bool = True,
2064
- **kwargs: ta.Any,
2065
- ) -> ta.Iterator[tempfile._TemporaryFileWrapper]: # noqa
2066
- with tempfile.NamedTemporaryFile(dir=root_dir, delete=False, **kwargs) as f:
2067
- try:
2068
- yield f
2069
- finally:
2070
- if cleanup:
2071
- shutil.rmtree(f.name, ignore_errors=True)
3379
+ await self._file_cache.put_file(
3380
+ key,
3381
+ file_path,
3382
+ steal=steal,
3383
+ )
2072
3384
 
2073
3385
 
2074
3386
  ########################################
@@ -2086,6 +3398,9 @@ class GithubCacheClient(abc.ABC):
2086
3398
  def get_entry(self, key: str) -> ta.Awaitable[ta.Optional[Entry]]:
2087
3399
  raise NotImplementedError
2088
3400
 
3401
+ def get_entry_url(self, entry: Entry) -> ta.Optional[str]:
3402
+ return None
3403
+
2089
3404
  @abc.abstractmethod
2090
3405
  def download_file(self, entry: Entry, out_file: str) -> ta.Awaitable[None]:
2091
3406
  raise NotImplementedError
@@ -2152,7 +3467,7 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
2152
3467
  def _get_loop(self) -> asyncio.AbstractEventLoop:
2153
3468
  if (loop := self._given_loop) is not None:
2154
3469
  return loop
2155
- return asyncio.get_event_loop()
3470
+ return asyncio.get_running_loop()
2156
3471
 
2157
3472
  #
2158
3473
 
@@ -2280,6 +3595,10 @@ class GithubCacheServiceV1BaseClient(GithubCacheClient, abc.ABC):
2280
3595
  class Entry(GithubCacheClient.Entry):
2281
3596
  artifact: GithubCacheServiceV1.ArtifactCacheEntry
2282
3597
 
3598
+ def get_entry_url(self, entry: GithubCacheClient.Entry) -> ta.Optional[str]:
3599
+ entry1 = check.isinstance(entry, self.Entry)
3600
+ return entry1.artifact.cache_key
3601
+
2283
3602
  #
2284
3603
 
2285
3604
  def build_get_entry_url_path(self, *keys: str) -> str:
@@ -2911,7 +4230,70 @@ class BaseSubprocesses(abc.ABC): # noqa
2911
4230
  ##
2912
4231
 
2913
4232
 
4233
+ @dc.dataclass(frozen=True)
4234
+ class SubprocessRun:
4235
+ cmd: ta.Sequence[str]
4236
+ input: ta.Any = None
4237
+ timeout: ta.Optional[float] = None
4238
+ check: bool = False
4239
+ capture_output: ta.Optional[bool] = None
4240
+ kwargs: ta.Optional[ta.Mapping[str, ta.Any]] = None
4241
+
4242
+ @classmethod
4243
+ def of(
4244
+ cls,
4245
+ *cmd: str,
4246
+ input: ta.Any = None, # noqa
4247
+ timeout: ta.Optional[float] = None,
4248
+ check: bool = False,
4249
+ capture_output: ta.Optional[bool] = None,
4250
+ **kwargs: ta.Any,
4251
+ ) -> 'SubprocessRun':
4252
+ return cls(
4253
+ cmd=cmd,
4254
+ input=input,
4255
+ timeout=timeout,
4256
+ check=check,
4257
+ capture_output=capture_output,
4258
+ kwargs=kwargs,
4259
+ )
4260
+
4261
+
4262
+ @dc.dataclass(frozen=True)
4263
+ class SubprocessRunOutput(ta.Generic[T]):
4264
+ proc: T
4265
+
4266
+ returncode: int # noqa
4267
+
4268
+ stdout: ta.Optional[bytes] = None
4269
+ stderr: ta.Optional[bytes] = None
4270
+
4271
+
2914
4272
  class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
4273
+ @abc.abstractmethod
4274
+ def run_(self, run: SubprocessRun) -> SubprocessRunOutput:
4275
+ raise NotImplementedError
4276
+
4277
+ def run(
4278
+ self,
4279
+ *cmd: str,
4280
+ input: ta.Any = None, # noqa
4281
+ timeout: ta.Optional[float] = None,
4282
+ check: bool = False,
4283
+ capture_output: ta.Optional[bool] = None,
4284
+ **kwargs: ta.Any,
4285
+ ) -> SubprocessRunOutput:
4286
+ return self.run_(SubprocessRun(
4287
+ cmd=cmd,
4288
+ input=input,
4289
+ timeout=timeout,
4290
+ check=check,
4291
+ capture_output=capture_output,
4292
+ kwargs=kwargs,
4293
+ ))
4294
+
4295
+ #
4296
+
2915
4297
  @abc.abstractmethod
2916
4298
  def check_call(
2917
4299
  self,
@@ -2975,6 +4357,25 @@ class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
2975
4357
 
2976
4358
 
2977
4359
  class Subprocesses(AbstractSubprocesses):
4360
+ def run_(self, run: SubprocessRun) -> SubprocessRunOutput[subprocess.CompletedProcess]:
4361
+ proc = subprocess.run(
4362
+ run.cmd,
4363
+ input=run.input,
4364
+ timeout=run.timeout,
4365
+ check=run.check,
4366
+ capture_output=run.capture_output or False,
4367
+ **(run.kwargs or {}),
4368
+ )
4369
+
4370
+ return SubprocessRunOutput(
4371
+ proc=proc,
4372
+
4373
+ returncode=proc.returncode,
4374
+
4375
+ stdout=proc.stdout, # noqa
4376
+ stderr=proc.stderr, # noqa
4377
+ )
4378
+
2978
4379
  def check_call(
2979
4380
  self,
2980
4381
  *cmd: str,
@@ -3000,6 +4401,30 @@ subprocesses = Subprocesses()
3000
4401
 
3001
4402
 
3002
4403
  class AbstractAsyncSubprocesses(BaseSubprocesses):
4404
+ @abc.abstractmethod
4405
+ async def run_(self, run: SubprocessRun) -> SubprocessRunOutput:
4406
+ raise NotImplementedError
4407
+
4408
+ def run(
4409
+ self,
4410
+ *cmd: str,
4411
+ input: ta.Any = None, # noqa
4412
+ timeout: ta.Optional[float] = None,
4413
+ check: bool = False,
4414
+ capture_output: ta.Optional[bool] = None,
4415
+ **kwargs: ta.Any,
4416
+ ) -> ta.Awaitable[SubprocessRunOutput]:
4417
+ return self.run_(SubprocessRun(
4418
+ cmd=cmd,
4419
+ input=input,
4420
+ timeout=timeout,
4421
+ check=check,
4422
+ capture_output=capture_output,
4423
+ kwargs=kwargs,
4424
+ ))
4425
+
4426
+ #
4427
+
3003
4428
  @abc.abstractmethod
3004
4429
  async def check_call(
3005
4430
  self,
@@ -3066,17 +4491,23 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
3066
4491
  ##
3067
4492
 
3068
4493
 
3069
- class GithubFileCache(FileCache):
4494
+ class GithubCache(FileCache, DataCache):
4495
+ @dc.dataclass(frozen=True)
4496
+ class Config:
4497
+ dir: str
4498
+
3070
4499
  def __init__(
3071
4500
  self,
3072
- dir: str, # noqa
4501
+ config: Config,
3073
4502
  *,
3074
4503
  client: ta.Optional[GithubCacheClient] = None,
3075
- **kwargs: ta.Any,
4504
+ version: ta.Optional[CacheVersion] = None,
3076
4505
  ) -> None:
3077
- super().__init__(**kwargs)
4506
+ super().__init__(
4507
+ version=version,
4508
+ )
3078
4509
 
3079
- self._dir = check.not_none(dir)
4510
+ self._config = config
3080
4511
 
3081
4512
  if client is None:
3082
4513
  client = GithubCacheServiceV1Client(
@@ -3085,10 +4516,14 @@ class GithubFileCache(FileCache):
3085
4516
  self._client: GithubCacheClient = client
3086
4517
 
3087
4518
  self._local = DirectoryFileCache(
3088
- self._dir,
4519
+ DirectoryFileCache.Config(
4520
+ dir=check.non_empty_str(config.dir),
4521
+ ),
3089
4522
  version=self._version,
3090
4523
  )
3091
4524
 
4525
+ #
4526
+
3092
4527
  async def get_file(self, key: str) -> ta.Optional[str]:
3093
4528
  local_file = self._local.get_cache_file_path(key)
3094
4529
  if os.path.exists(local_file):
@@ -3122,6 +4557,21 @@ class GithubFileCache(FileCache):
3122
4557
 
3123
4558
  return cache_file_path
3124
4559
 
4560
+ #
4561
+
4562
+ async def get_data(self, key: str) -> ta.Optional[DataCache.Data]:
4563
+ local_file = self._local.get_cache_file_path(key)
4564
+ if os.path.exists(local_file):
4565
+ return DataCache.FileData(local_file)
4566
+
4567
+ if (entry := await self._client.get_entry(key)) is None:
4568
+ return None
4569
+
4570
+ return DataCache.UrlData(check.non_empty_str(self._client.get_entry_url(entry)))
4571
+
4572
+ async def put_data(self, key: str, data: DataCache.Data) -> None:
4573
+ await FileCacheDataCache(self).put_data(key, data)
4574
+
3125
4575
 
3126
4576
  ########################################
3127
4577
  # ../github/cli.py
@@ -3388,41 +4838,32 @@ class AsyncioSubprocesses(AbstractAsyncSubprocesses):
3388
4838
 
3389
4839
  #
3390
4840
 
3391
- @dc.dataclass(frozen=True)
3392
- class RunOutput:
3393
- proc: asyncio.subprocess.Process
3394
- stdout: ta.Optional[bytes]
3395
- stderr: ta.Optional[bytes]
4841
+ async def run_(self, run: SubprocessRun) -> SubprocessRunOutput[asyncio.subprocess.Process]:
4842
+ kwargs = dict(run.kwargs or {})
3396
4843
 
3397
- async def run(
3398
- self,
3399
- *cmd: str,
3400
- input: ta.Any = None, # noqa
3401
- timeout: ta.Optional[float] = None,
3402
- check: bool = False, # noqa
3403
- capture_output: ta.Optional[bool] = None,
3404
- **kwargs: ta.Any,
3405
- ) -> RunOutput:
3406
- if capture_output:
4844
+ if run.capture_output:
3407
4845
  kwargs.setdefault('stdout', subprocess.PIPE)
3408
4846
  kwargs.setdefault('stderr', subprocess.PIPE)
3409
4847
 
3410
4848
  proc: asyncio.subprocess.Process
3411
- async with self.popen(*cmd, **kwargs) as proc:
3412
- stdout, stderr = await self.communicate(proc, input, timeout)
4849
+ async with self.popen(*run.cmd, **kwargs) as proc:
4850
+ stdout, stderr = await self.communicate(proc, run.input, run.timeout)
3413
4851
 
3414
4852
  if check and proc.returncode:
3415
4853
  raise subprocess.CalledProcessError(
3416
4854
  proc.returncode,
3417
- cmd,
4855
+ run.cmd,
3418
4856
  output=stdout,
3419
4857
  stderr=stderr,
3420
4858
  )
3421
4859
 
3422
- return self.RunOutput(
3423
- proc,
3424
- stdout,
3425
- stderr,
4860
+ return SubprocessRunOutput(
4861
+ proc=proc,
4862
+
4863
+ returncode=check.isinstance(proc.returncode, int),
4864
+
4865
+ stdout=stdout,
4866
+ stderr=stderr,
3426
4867
  )
3427
4868
 
3428
4869
  #
@@ -3615,47 +5056,7 @@ class DockerComposeRun(AsyncExitStacked):
3615
5056
 
3616
5057
 
3617
5058
  ########################################
3618
- # ../docker.py
3619
- """
3620
- TODO:
3621
- - some less stupid Dockerfile hash
3622
- - doesn't change too much though
3623
- """
3624
-
3625
-
3626
- ##
3627
-
3628
-
3629
- def build_docker_file_hash(docker_file: str) -> str:
3630
- with open(docker_file) as f:
3631
- contents = f.read()
3632
-
3633
- return sha256_str(contents)
3634
-
3635
-
3636
- ##
3637
-
3638
-
3639
- def read_docker_tar_image_tag(tar_file: str) -> str:
3640
- with tarfile.open(tar_file) as tf:
3641
- with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
3642
- m = mf.read()
3643
-
3644
- manifests = json.loads(m.decode('utf-8'))
3645
- manifest = check.single(manifests)
3646
- tag = check.non_empty_str(check.single(manifest['RepoTags']))
3647
- return tag
3648
-
3649
-
3650
- def read_docker_tar_image_id(tar_file: str) -> str:
3651
- with tarfile.open(tar_file) as tf:
3652
- with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
3653
- i = mf.read()
3654
-
3655
- index = json.loads(i.decode('utf-8'))
3656
- manifest = check.single(index['manifests'])
3657
- image_id = check.non_empty_str(manifest['digest'])
3658
- return image_id
5059
+ # ../docker/cmds.py
3659
5060
 
3660
5061
 
3661
5062
  ##
@@ -3772,75 +5173,58 @@ async def load_docker_tar(
3772
5173
 
3773
5174
 
3774
5175
  ########################################
3775
- # ../ci.py
5176
+ # ../github/inject.py
3776
5177
 
3777
5178
 
3778
- class Ci(AsyncExitStacked):
3779
- KEY_HASH_LEN = 16
5179
+ ##
3780
5180
 
3781
- @dc.dataclass(frozen=True)
3782
- class Config:
3783
- project_dir: str
3784
5181
 
3785
- docker_file: str
5182
+ def bind_github(
5183
+ *,
5184
+ cache_dir: ta.Optional[str] = None,
5185
+ ) -> InjectorBindings:
5186
+ lst: ta.List[InjectorBindingOrBindings] = []
5187
+
5188
+ if cache_dir is not None:
5189
+ lst.extend([
5190
+ inj.bind(GithubCache.Config(
5191
+ dir=cache_dir,
5192
+ )),
5193
+ inj.bind(GithubCache, singleton=True),
5194
+ inj.bind(FileCache, to_key=GithubCache),
5195
+ ])
3786
5196
 
3787
- compose_file: str
3788
- service: str
5197
+ return inj.as_bindings(*lst)
3789
5198
 
3790
- cmd: ShellCmd
3791
5199
 
3792
- #
5200
+ ########################################
5201
+ # ../docker/cache.py
3793
5202
 
3794
- requirements_txts: ta.Optional[ta.Sequence[str]] = None
3795
5203
 
3796
- always_pull: bool = False
3797
- always_build: bool = False
5204
+ ##
3798
5205
 
3799
- no_dependencies: bool = False
3800
5206
 
3801
- run_options: ta.Optional[ta.Sequence[str]] = None
5207
+ class DockerCache(abc.ABC):
5208
+ @abc.abstractmethod
5209
+ def load_cache_docker_image(self, key: str) -> ta.Awaitable[ta.Optional[str]]:
5210
+ raise NotImplementedError
3802
5211
 
3803
- #
5212
+ @abc.abstractmethod
5213
+ def save_cache_docker_image(self, key: str, image: str) -> ta.Awaitable[None]:
5214
+ raise NotImplementedError
3804
5215
 
3805
- def __post_init__(self) -> None:
3806
- check.not_isinstance(self.requirements_txts, str)
3807
5216
 
5217
+ class DockerCacheImpl(DockerCache):
3808
5218
  def __init__(
3809
5219
  self,
3810
- cfg: Config,
3811
5220
  *,
3812
5221
  file_cache: ta.Optional[FileCache] = None,
3813
5222
  ) -> None:
3814
5223
  super().__init__()
3815
5224
 
3816
- self._cfg = cfg
3817
5225
  self._file_cache = file_cache
3818
5226
 
3819
- #
3820
-
3821
- async def _load_docker_image(self, image: str) -> None:
3822
- if not self._cfg.always_pull and (await is_docker_image_present(image)):
3823
- return
3824
-
3825
- dep_suffix = image
3826
- for c in '/:.-_':
3827
- dep_suffix = dep_suffix.replace(c, '-')
3828
-
3829
- cache_key = f'docker-{dep_suffix}'
3830
- if (await self._load_cache_docker_image(cache_key)) is not None:
3831
- return
3832
-
3833
- await pull_docker_image(image)
3834
-
3835
- await self._save_cache_docker_image(cache_key, image)
3836
-
3837
- async def load_docker_image(self, image: str) -> None:
3838
- with log_timing_context(f'Load docker image: {image}'):
3839
- await self._load_docker_image(image)
3840
-
3841
- #
3842
-
3843
- async def _load_cache_docker_image(self, key: str) -> ta.Optional[str]:
5227
+ async def load_cache_docker_image(self, key: str) -> ta.Optional[str]:
3844
5228
  if self._file_cache is None:
3845
5229
  return None
3846
5230
 
@@ -3852,7 +5236,7 @@ class Ci(AsyncExitStacked):
3852
5236
 
3853
5237
  return await load_docker_tar_cmd(get_cache_cmd)
3854
5238
 
3855
- async def _save_cache_docker_image(self, key: str, image: str) -> None:
5239
+ async def save_cache_docker_image(self, key: str, image: str) -> None:
3856
5240
  if self._file_cache is None:
3857
5241
  return
3858
5242
 
@@ -3863,19 +5247,58 @@ class Ci(AsyncExitStacked):
3863
5247
 
3864
5248
  await self._file_cache.put_file(key, tmp_file, steal=True)
3865
5249
 
3866
- #
3867
5250
 
3868
- async def _resolve_docker_image(
5251
+ ########################################
5252
+ # ../docker/buildcaching.py
5253
+
5254
+
5255
+ ##
5256
+
5257
+
5258
+ class DockerBuildCaching(abc.ABC):
5259
+ @abc.abstractmethod
5260
+ def cached_build_docker_image(
5261
+ self,
5262
+ cache_key: str,
5263
+ build_and_tag: ta.Callable[[str], ta.Awaitable[str]], # image_tag -> image_id
5264
+ ) -> ta.Awaitable[str]:
5265
+ raise NotImplementedError
5266
+
5267
+
5268
+ class DockerBuildCachingImpl(DockerBuildCaching):
5269
+ @dc.dataclass(frozen=True)
5270
+ class Config:
5271
+ service: str
5272
+
5273
+ always_build: bool = False
5274
+
5275
+ def __init__(
5276
+ self,
5277
+ *,
5278
+ config: Config,
5279
+
5280
+ docker_cache: ta.Optional[DockerCache] = None,
5281
+ ) -> None:
5282
+ super().__init__()
5283
+
5284
+ self._config = config
5285
+
5286
+ self._docker_cache = docker_cache
5287
+
5288
+ async def cached_build_docker_image(
3869
5289
  self,
3870
5290
  cache_key: str,
3871
5291
  build_and_tag: ta.Callable[[str], ta.Awaitable[str]],
3872
5292
  ) -> str:
3873
- image_tag = f'{self._cfg.service}:{cache_key}'
5293
+ image_tag = f'{self._config.service}:{cache_key}'
3874
5294
 
3875
- if not self._cfg.always_build and (await is_docker_image_present(image_tag)):
5295
+ if not self._config.always_build and (await is_docker_image_present(image_tag)):
3876
5296
  return image_tag
3877
5297
 
3878
- if (cache_image_id := await self._load_cache_docker_image(cache_key)) is not None:
5298
+ if (
5299
+ self._docker_cache is not None and
5300
+ (cache_image_id := await self._docker_cache.load_cache_docker_image(cache_key)) is not None
5301
+ ):
3879
5302
  await tag_docker_image(
3880
5303
  cache_image_id,
3881
5304
  image_tag,
@@ -3884,27 +5307,143 @@ class Ci(AsyncExitStacked):
3884
5307
 
3885
5308
  image_id = await build_and_tag(image_tag)
3886
5309
 
3887
- await self._save_cache_docker_image(cache_key, image_id)
5310
+ if self._docker_cache is not None:
5311
+ await self._docker_cache.save_cache_docker_image(cache_key, image_id)
3888
5312
 
3889
5313
  return image_tag
3890
5314
 
5315
+
5316
+ ########################################
5317
+ # ../docker/imagepulling.py
5318
+
5319
+
5320
+ ##
5321
+
5322
+
5323
+ class DockerImagePulling(abc.ABC):
5324
+ @abc.abstractmethod
5325
+ def pull_docker_image(self, image: str) -> ta.Awaitable[None]:
5326
+ raise NotImplementedError
5327
+
5328
+
5329
+ class DockerImagePullingImpl(DockerImagePulling):
5330
+ @dc.dataclass(frozen=True)
5331
+ class Config:
5332
+ always_pull: bool = False
5333
+
5334
+ def __init__(
5335
+ self,
5336
+ *,
5337
+ config: Config = Config(),
5338
+
5339
+ file_cache: ta.Optional[FileCache] = None,
5340
+ docker_cache: ta.Optional[DockerCache] = None,
5341
+ ) -> None:
5342
+ super().__init__()
5343
+
5344
+ self._config = config
5345
+
5346
+ self._file_cache = file_cache
5347
+ self._docker_cache = docker_cache
5348
+
5349
+ async def _pull_docker_image(self, image: str) -> None:
5350
+ if not self._config.always_pull and (await is_docker_image_present(image)):
5351
+ return
5352
+
5353
+ dep_suffix = image
5354
+ for c in '/:.-_':
5355
+ dep_suffix = dep_suffix.replace(c, '-')
5356
+
5357
+ cache_key = f'docker-{dep_suffix}'
5358
+ if (
5359
+ self._docker_cache is not None and
5360
+ (await self._docker_cache.load_cache_docker_image(cache_key)) is not None
5361
+ ):
5362
+ return
5363
+
5364
+ await pull_docker_image(image)
5365
+
5366
+ if self._docker_cache is not None:
5367
+ await self._docker_cache.save_cache_docker_image(cache_key, image)
5368
+
5369
+ async def pull_docker_image(self, image: str) -> None:
5370
+ with log_timing_context(f'Load docker image: {image}'):
5371
+ await self._pull_docker_image(image)
5372
+
5373
+
5374
+ ########################################
5375
+ # ../ci.py
5376
+
5377
+
5378
+ ##
5379
+
5380
+
5381
+ class Ci(AsyncExitStacked):
5382
+ KEY_HASH_LEN = 16
5383
+
5384
+ @dc.dataclass(frozen=True)
5385
+ class Config:
5386
+ project_dir: str
5387
+
5388
+ docker_file: str
5389
+
5390
+ compose_file: str
5391
+ service: str
5392
+
5393
+ cmd: ShellCmd
5394
+
5395
+ #
5396
+
5397
+ requirements_txts: ta.Optional[ta.Sequence[str]] = None
5398
+
5399
+ always_pull: bool = False
5400
+ always_build: bool = False
5401
+
5402
+ no_dependencies: bool = False
5403
+
5404
+ run_options: ta.Optional[ta.Sequence[str]] = None
5405
+
5406
+ #
5407
+
5408
+ def __post_init__(self) -> None:
5409
+ check.not_isinstance(self.requirements_txts, str)
5410
+
5411
+ def __init__(
5412
+ self,
5413
+ config: Config,
5414
+ *,
5415
+ docker_build_caching: DockerBuildCaching,
5416
+ docker_image_pulling: DockerImagePulling,
5417
+ ) -> None:
5418
+ super().__init__()
5419
+
5420
+ self._config = config
5421
+
5422
+ self._docker_build_caching = docker_build_caching
5423
+ self._docker_image_pulling = docker_image_pulling
5424
+
3891
5425
  #
3892
5426
 
3893
5427
  @cached_nullary
3894
5428
  def docker_file_hash(self) -> str:
3895
- return build_docker_file_hash(self._cfg.docker_file)[:self.KEY_HASH_LEN]
5429
+ return build_docker_file_hash(self._config.docker_file)[:self.KEY_HASH_LEN]
5430
+
5431
+ @cached_nullary
5432
+ def ci_base_image_cache_key(self) -> str:
5433
+ return f'ci-base-{self.docker_file_hash()}'
3896
5434
 
3897
5435
  async def _resolve_ci_base_image(self) -> str:
3898
5436
  async def build_and_tag(image_tag: str) -> str:
3899
5437
  return await build_docker_image(
3900
- self._cfg.docker_file,
5438
+ self._config.docker_file,
3901
5439
  tag=image_tag,
3902
- cwd=self._cfg.project_dir,
5440
+ cwd=self._config.project_dir,
3903
5441
  )
3904
5442
 
3905
- cache_key = f'ci-base-{self.docker_file_hash()}'
3906
-
3907
- return await self._resolve_docker_image(cache_key, build_and_tag)
5443
+ return await self._docker_build_caching.cached_build_docker_image(
5444
+ self.ci_base_image_cache_key(),
5445
+ build_and_tag,
5446
+ )
3908
5447
 
3909
5448
  @async_cached_nullary
3910
5449
  async def resolve_ci_base_image(self) -> str:
@@ -3918,14 +5457,18 @@ class Ci(AsyncExitStacked):
3918
5457
  @cached_nullary
3919
5458
  def requirements_txts(self) -> ta.Sequence[str]:
3920
5459
  return [
3921
- os.path.join(self._cfg.project_dir, rf)
3922
- for rf in check.not_none(self._cfg.requirements_txts)
5460
+ os.path.join(self._config.project_dir, rf)
5461
+ for rf in check.not_none(self._config.requirements_txts)
3923
5462
  ]
3924
5463
 
3925
5464
  @cached_nullary
3926
5465
  def requirements_hash(self) -> str:
3927
5466
  return build_requirements_hash(self.requirements_txts())[:self.KEY_HASH_LEN]
3928
5467
 
5468
+ @cached_nullary
5469
+ def ci_image_cache_key(self) -> str:
5470
+ return f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
5471
+
3929
5472
  async def _resolve_ci_image(self) -> str:
3930
5473
  async def build_and_tag(image_tag: str) -> str:
3931
5474
  base_image = await self.resolve_ci_base_image()
@@ -3942,7 +5485,7 @@ class Ci(AsyncExitStacked):
3942
5485
  '--no-cache',
3943
5486
  '--index-strategy unsafe-best-match',
3944
5487
  '--system',
3945
- *[f'-r /project/{rf}' for rf in self._cfg.requirements_txts or []],
5488
+ *[f'-r /project/{rf}' for rf in self._config.requirements_txts or []],
3946
5489
  ]),
3947
5490
  ]
3948
5491
  setup_cmd = ' && '.join(setup_cmds)
@@ -3950,7 +5493,7 @@ class Ci(AsyncExitStacked):
3950
5493
  docker_file_lines = [
3951
5494
  f'FROM {base_image}',
3952
5495
  'RUN mkdir /project',
3953
- *[f'COPY {rf} /project/{rf}' for rf in self._cfg.requirements_txts or []],
5496
+ *[f'COPY {rf} /project/{rf}' for rf in self._config.requirements_txts or []],
3954
5497
  f'RUN {setup_cmd}',
3955
5498
  'RUN rm /project/*',
3956
5499
  'WORKDIR /project',
@@ -3963,12 +5506,13 @@ class Ci(AsyncExitStacked):
3963
5506
  return await build_docker_image(
3964
5507
  docker_file,
3965
5508
  tag=image_tag,
3966
- cwd=self._cfg.project_dir,
5509
+ cwd=self._config.project_dir,
3967
5510
  )
3968
5511
 
3969
- cache_key = f'ci-{self.docker_file_hash()}-{self.requirements_hash()}'
3970
-
3971
- return await self._resolve_docker_image(cache_key, build_and_tag)
5512
+ return await self._docker_build_caching.cached_build_docker_image(
5513
+ self.ci_image_cache_key(),
5514
+ build_and_tag,
5515
+ )
3972
5516
 
3973
5517
  @async_cached_nullary
3974
5518
  async def resolve_ci_image(self) -> str:
@@ -3980,34 +5524,34 @@ class Ci(AsyncExitStacked):
3980
5524
  #
3981
5525
 
3982
5526
  @async_cached_nullary
3983
- async def load_dependencies(self) -> None:
5527
+ async def pull_dependencies(self) -> None:
3984
5528
  deps = get_compose_service_dependencies(
3985
- self._cfg.compose_file,
3986
- self._cfg.service,
5529
+ self._config.compose_file,
5530
+ self._config.service,
3987
5531
  )
3988
5532
 
3989
5533
  for dep_image in deps.values():
3990
- await self.load_docker_image(dep_image)
5534
+ await self._docker_image_pulling.pull_docker_image(dep_image)
3991
5535
 
3992
5536
  #
3993
5537
 
3994
5538
  async def _run_compose_(self) -> None:
3995
5539
  async with DockerComposeRun(DockerComposeRun.Config(
3996
- compose_file=self._cfg.compose_file,
3997
- service=self._cfg.service,
5540
+ compose_file=self._config.compose_file,
5541
+ service=self._config.service,
3998
5542
 
3999
5543
  image=await self.resolve_ci_image(),
4000
5544
 
4001
- cmd=self._cfg.cmd,
5545
+ cmd=self._config.cmd,
4002
5546
 
4003
5547
  run_options=[
4004
- '-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
4005
- *(self._cfg.run_options or []),
5548
+ '-v', f'{os.path.abspath(self._config.project_dir)}:/project',
5549
+ *(self._config.run_options or []),
4006
5550
  ],
4007
5551
 
4008
- cwd=self._cfg.project_dir,
5552
+ cwd=self._config.project_dir,
4009
5553
 
4010
- no_dependencies=self._cfg.no_dependencies,
5554
+ no_dependencies=self._config.no_dependencies,
4011
5555
  )) as ci_compose_run:
4012
5556
  await ci_compose_run.run()
4013
5557
 
@@ -4020,11 +5564,90 @@ class Ci(AsyncExitStacked):
4020
5564
  async def run(self) -> None:
4021
5565
  await self.resolve_ci_image()
4022
5566
 
4023
- await self.load_dependencies()
5567
+ await self.pull_dependencies()
4024
5568
 
4025
5569
  await self._run_compose()
4026
5570
 
4027
5571
 
5572
+ ########################################
5573
+ # ../docker/inject.py
5574
+
5575
+
5576
+ ##
5577
+
5578
+
5579
+ def bind_docker(
5580
+ *,
5581
+ build_caching_config: DockerBuildCachingImpl.Config,
5582
+ image_pulling_config: DockerImagePullingImpl.Config = DockerImagePullingImpl.Config(),
5583
+ ) -> InjectorBindings:
5584
+ lst: ta.List[InjectorBindingOrBindings] = [
5585
+ inj.bind(build_caching_config),
5586
+ inj.bind(DockerBuildCachingImpl, singleton=True),
5587
+ inj.bind(DockerBuildCaching, to_key=DockerBuildCachingImpl),
5588
+
5589
+ inj.bind(DockerCacheImpl, singleton=True),
5590
+ inj.bind(DockerCache, to_key=DockerCacheImpl),
5591
+
5592
+ inj.bind(image_pulling_config),
5593
+ inj.bind(DockerImagePullingImpl, singleton=True),
5594
+ inj.bind(DockerImagePulling, to_key=DockerImagePullingImpl),
5595
+ ]
5596
+
5597
+ return inj.as_bindings(*lst)
5598
+
5599
+
5600
+ ########################################
5601
+ # ../inject.py
5602
+
5603
+
5604
+ ##
5605
+
5606
+
5607
+ def bind_ci(
5608
+ *,
5609
+ config: Ci.Config,
5610
+
5611
+ github: bool = False,
5612
+
5613
+ cache_dir: ta.Optional[str] = None,
5614
+ ) -> InjectorBindings:
5615
+ lst: ta.List[InjectorBindingOrBindings] = [ # noqa
5616
+ inj.bind(config),
5617
+ inj.bind(Ci, singleton=True),
5618
+ ]
5619
+
5620
+ lst.append(bind_docker(
5621
+ build_caching_config=DockerBuildCachingImpl.Config(
5622
+ service=config.service,
5623
+
5624
+ always_build=config.always_build,
5625
+ ),
5626
+
5627
+ image_pulling_config=DockerImagePullingImpl.Config(
5628
+ always_pull=config.always_pull,
5629
+ ),
5630
+ ))
5631
+
5632
+ if cache_dir is not None:
5633
+ if github:
5634
+ lst.append(bind_github(
5635
+ cache_dir=cache_dir,
5636
+ ))
5637
+
5638
+ else:
5639
+ lst.extend([
5640
+ inj.bind(DirectoryFileCache.Config(
5641
+ dir=cache_dir,
5642
+ )),
5643
+ inj.bind(DirectoryFileCache, singleton=True),
5644
+ inj.bind(FileCache, to_key=DirectoryFileCache),
5645
+
5646
+ ])
5647
+
5648
+ return inj.as_bindings(*lst)
5649
+
5650
+
4028
5651
  ########################################
4029
5652
  # cli.py
4030
5653
 
@@ -4159,14 +5782,9 @@ class CiCli(ArgparseCli):
4159
5782
 
4160
5783
  #
4161
5784
 
4162
- file_cache: ta.Optional[FileCache] = None
4163
5785
  if cache_dir is not None:
4164
5786
  cache_dir = os.path.abspath(cache_dir)
4165
5787
  log.debug('Using cache dir %s', cache_dir)
4166
- if github:
4167
- file_cache = GithubFileCache(cache_dir)
4168
- else:
4169
- file_cache = DirectoryFileCache(cache_dir)
4170
5788
 
4171
5789
  #
4172
5790
 
@@ -4182,28 +5800,35 @@ class CiCli(ArgparseCli):
4182
5800
 
4183
5801
  #
4184
5802
 
4185
- async with Ci(
4186
- Ci.Config(
4187
- project_dir=project_dir,
5803
+ config = Ci.Config(
5804
+ project_dir=project_dir,
4188
5805
 
4189
- docker_file=docker_file,
5806
+ docker_file=docker_file,
4190
5807
 
4191
- compose_file=compose_file,
4192
- service=self.args.service,
5808
+ compose_file=compose_file,
5809
+ service=self.args.service,
4193
5810
 
4194
- requirements_txts=requirements_txts,
5811
+ requirements_txts=requirements_txts,
4195
5812
 
4196
- cmd=ShellCmd(cmd),
5813
+ cmd=ShellCmd(cmd),
4197
5814
 
4198
- always_pull=self.args.always_pull,
4199
- always_build=self.args.always_build,
5815
+ always_pull=self.args.always_pull,
5816
+ always_build=self.args.always_build,
4200
5817
 
4201
- no_dependencies=self.args.no_dependencies,
5818
+ no_dependencies=self.args.no_dependencies,
4202
5819
 
4203
- run_options=run_options,
4204
- ),
4205
- file_cache=file_cache,
4206
- ) as ci:
5820
+ run_options=run_options,
5821
+ )
5822
+
5823
+ injector = inj.create_injector(bind_ci(
5824
+ config=config,
5825
+
5826
+ github=github,
5827
+
5828
+ cache_dir=cache_dir,
5829
+ ))
5830
+
5831
+ async with injector[Ci] as ci:
4207
5832
  await ci.run()
4208
5833
 
4209
5834