omdev 0.0.0.dev209__py3-none-any.whl → 0.0.0.dev210__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omdev/.manifests.json +1 -1
- omdev/amalg/main.py +10 -1
- omdev/cc/cdeps.py +38 -0
- omdev/cc/cdeps.toml +4 -1
- omdev/cc/cli.py +2 -25
- omdev/ci/__init__.py +0 -0
- omdev/ci/__main__.py +4 -0
- omdev/ci/cache.py +41 -0
- omdev/ci/ci.py +214 -0
- omdev/ci/cli.py +155 -0
- omdev/ci/compose.py +198 -0
- omdev/ci/dockertars.py +138 -0
- omdev/ci/requirements.py +79 -0
- omdev/ci/utils.py +32 -0
- omdev/interp/cli.py +0 -1
- omdev/interp/providers/system.py +2 -1
- omdev/pycharm/cli.py +1 -1
- omdev/pyproject/cli.py +0 -1
- omdev/revisions.py +0 -1
- omdev/scripts/ci.py +2183 -0
- omdev/scripts/interp.py +19 -3
- omdev/scripts/pyproject.py +19 -3
- omdev/tools/pawk/pawk.py +0 -1
- {omdev-0.0.0.dev209.dist-info → omdev-0.0.0.dev210.dist-info}/METADATA +2 -2
- {omdev-0.0.0.dev209.dist-info → omdev-0.0.0.dev210.dist-info}/RECORD +29 -18
- {omdev-0.0.0.dev209.dist-info → omdev-0.0.0.dev210.dist-info}/LICENSE +0 -0
- {omdev-0.0.0.dev209.dist-info → omdev-0.0.0.dev210.dist-info}/WHEEL +0 -0
- {omdev-0.0.0.dev209.dist-info → omdev-0.0.0.dev210.dist-info}/entry_points.txt +0 -0
- {omdev-0.0.0.dev209.dist-info → omdev-0.0.0.dev210.dist-info}/top_level.txt +0 -0
omdev/scripts/ci.py
ADDED
@@ -0,0 +1,2183 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# noinspection DuplicatedCode
|
3
|
+
# @omlish-lite
|
4
|
+
# @omlish-script
|
5
|
+
# @omlish-amalg-output ../ci/cli.py
|
6
|
+
# ruff: noqa: UP006 UP007 UP036
|
7
|
+
"""
|
8
|
+
Inputs:
|
9
|
+
- requirements.txt
|
10
|
+
- ci.Dockerfile
|
11
|
+
- compose.yml
|
12
|
+
|
13
|
+
==
|
14
|
+
|
15
|
+
./python -m ci run --cache-dir ci/cache ci/project omlish-ci
|
16
|
+
"""
|
17
|
+
import abc
|
18
|
+
import argparse
|
19
|
+
import asyncio
|
20
|
+
import collections
|
21
|
+
import contextlib
|
22
|
+
import dataclasses as dc
|
23
|
+
import functools
|
24
|
+
import hashlib
|
25
|
+
import inspect
|
26
|
+
import itertools
|
27
|
+
import json
|
28
|
+
import logging
|
29
|
+
import os
|
30
|
+
import os.path
|
31
|
+
import shlex
|
32
|
+
import shutil
|
33
|
+
import subprocess
|
34
|
+
import sys
|
35
|
+
import tarfile
|
36
|
+
import tempfile
|
37
|
+
import threading
|
38
|
+
import time
|
39
|
+
import types
|
40
|
+
import typing as ta
|
41
|
+
|
42
|
+
|
43
|
+
########################################
|
44
|
+
|
45
|
+
|
46
|
+
if sys.version_info < (3, 8):
|
47
|
+
raise OSError(f'Requires python (3, 8), got {sys.version_info} from {sys.executable}') # noqa
|
48
|
+
|
49
|
+
|
50
|
+
########################################
|
51
|
+
|
52
|
+
|
53
|
+
# ../../omlish/lite/cached.py
|
54
|
+
T = ta.TypeVar('T')
|
55
|
+
CallableT = ta.TypeVar('CallableT', bound=ta.Callable)
|
56
|
+
|
57
|
+
# ../../omlish/lite/check.py
|
58
|
+
SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
|
59
|
+
CheckMessage = ta.Union[str, ta.Callable[..., ta.Optional[str]], None] # ta.TypeAlias
|
60
|
+
CheckLateConfigureFn = ta.Callable[['Checks'], None] # ta.TypeAlias
|
61
|
+
CheckOnRaiseFn = ta.Callable[[Exception], None] # ta.TypeAlias
|
62
|
+
CheckExceptionFactory = ta.Callable[..., Exception] # ta.TypeAlias
|
63
|
+
CheckArgsRenderer = ta.Callable[..., ta.Optional[str]] # ta.TypeAlias
|
64
|
+
|
65
|
+
# ../../omlish/argparse/cli.py
|
66
|
+
ArgparseCmdFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
|
67
|
+
|
68
|
+
# ../../omlish/lite/contextmanagers.py
|
69
|
+
ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
|
70
|
+
|
71
|
+
# ../../omlish/subprocesses.py
|
72
|
+
SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
|
73
|
+
|
74
|
+
|
75
|
+
########################################
|
76
|
+
# ../cache.py
|
77
|
+
|
78
|
+
|
79
|
+
#
|
80
|
+
|
81
|
+
|
82
|
+
@abc.abstractmethod
|
83
|
+
class FileCache(abc.ABC):
|
84
|
+
@abc.abstractmethod
|
85
|
+
def get_file(self, name: str) -> ta.Optional[str]:
|
86
|
+
raise NotImplementedError
|
87
|
+
|
88
|
+
@abc.abstractmethod
|
89
|
+
def put_file(self, name: str) -> ta.Optional[str]:
|
90
|
+
raise NotImplementedError
|
91
|
+
|
92
|
+
|
93
|
+
#
|
94
|
+
|
95
|
+
|
96
|
+
class DirectoryFileCache(FileCache):
|
97
|
+
def __init__(self, dir: str) -> None: # noqa
|
98
|
+
super().__init__()
|
99
|
+
|
100
|
+
self._dir = dir
|
101
|
+
|
102
|
+
def get_file(self, name: str) -> ta.Optional[str]:
|
103
|
+
file_path = os.path.join(self._dir, name)
|
104
|
+
if not os.path.exists(file_path):
|
105
|
+
return None
|
106
|
+
return file_path
|
107
|
+
|
108
|
+
def put_file(self, file_path: str) -> None:
|
109
|
+
os.makedirs(self._dir, exist_ok=True)
|
110
|
+
cache_file_path = os.path.join(self._dir, os.path.basename(file_path))
|
111
|
+
shutil.copyfile(file_path, cache_file_path)
|
112
|
+
|
113
|
+
|
114
|
+
########################################
|
115
|
+
# ../utils.py
|
116
|
+
|
117
|
+
|
118
|
+
##
|
119
|
+
|
120
|
+
|
121
|
+
def make_temp_file() -> str:
|
122
|
+
file_fd, file = tempfile.mkstemp()
|
123
|
+
os.close(file_fd)
|
124
|
+
return file
|
125
|
+
|
126
|
+
|
127
|
+
##
|
128
|
+
|
129
|
+
|
130
|
+
def read_yaml_file(yaml_file: str) -> ta.Any:
|
131
|
+
yaml = __import__('yaml')
|
132
|
+
|
133
|
+
with open(yaml_file) as f:
|
134
|
+
return yaml.safe_load(f)
|
135
|
+
|
136
|
+
|
137
|
+
##
|
138
|
+
|
139
|
+
|
140
|
+
def sha256_str(s: str) -> str:
|
141
|
+
return hashlib.sha256(s.encode('utf-8')).hexdigest()
|
142
|
+
|
143
|
+
|
144
|
+
########################################
|
145
|
+
# ../../../omlish/lite/cached.py
|
146
|
+
|
147
|
+
|
148
|
+
##
|
149
|
+
|
150
|
+
|
151
|
+
class _AbstractCachedNullary:
|
152
|
+
def __init__(self, fn):
|
153
|
+
super().__init__()
|
154
|
+
self._fn = fn
|
155
|
+
self._value = self._missing = object()
|
156
|
+
functools.update_wrapper(self, fn)
|
157
|
+
|
158
|
+
def __call__(self, *args, **kwargs): # noqa
|
159
|
+
raise TypeError
|
160
|
+
|
161
|
+
def __get__(self, instance, owner): # noqa
|
162
|
+
bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
|
163
|
+
return bound
|
164
|
+
|
165
|
+
|
166
|
+
##
|
167
|
+
|
168
|
+
|
169
|
+
class _CachedNullary(_AbstractCachedNullary):
|
170
|
+
def __call__(self, *args, **kwargs): # noqa
|
171
|
+
if self._value is self._missing:
|
172
|
+
self._value = self._fn()
|
173
|
+
return self._value
|
174
|
+
|
175
|
+
|
176
|
+
def cached_nullary(fn: CallableT) -> CallableT:
|
177
|
+
return _CachedNullary(fn) # type: ignore
|
178
|
+
|
179
|
+
|
180
|
+
def static_init(fn: CallableT) -> CallableT:
|
181
|
+
fn = cached_nullary(fn)
|
182
|
+
fn()
|
183
|
+
return fn
|
184
|
+
|
185
|
+
|
186
|
+
##
|
187
|
+
|
188
|
+
|
189
|
+
class _AsyncCachedNullary(_AbstractCachedNullary):
|
190
|
+
async def __call__(self, *args, **kwargs):
|
191
|
+
if self._value is self._missing:
|
192
|
+
self._value = await self._fn()
|
193
|
+
return self._value
|
194
|
+
|
195
|
+
|
196
|
+
def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
197
|
+
return _AsyncCachedNullary(fn)
|
198
|
+
|
199
|
+
|
200
|
+
########################################
|
201
|
+
# ../../../omlish/lite/check.py
|
202
|
+
"""
|
203
|
+
TODO:
|
204
|
+
- def maybe(v: lang.Maybe[T])
|
205
|
+
- def not_ ?
|
206
|
+
- ** class @dataclass Raise - user message should be able to be an exception type or instance or factory
|
207
|
+
"""
|
208
|
+
|
209
|
+
|
210
|
+
##
|
211
|
+
|
212
|
+
|
213
|
+
class Checks:
|
214
|
+
def __init__(self) -> None:
|
215
|
+
super().__init__()
|
216
|
+
|
217
|
+
self._config_lock = threading.RLock()
|
218
|
+
self._on_raise_fns: ta.Sequence[CheckOnRaiseFn] = []
|
219
|
+
self._exception_factory: CheckExceptionFactory = Checks.default_exception_factory
|
220
|
+
self._args_renderer: ta.Optional[CheckArgsRenderer] = None
|
221
|
+
self._late_configure_fns: ta.Sequence[CheckLateConfigureFn] = []
|
222
|
+
|
223
|
+
@staticmethod
|
224
|
+
def default_exception_factory(exc_cls: ta.Type[Exception], *args, **kwargs) -> Exception:
|
225
|
+
return exc_cls(*args, **kwargs) # noqa
|
226
|
+
|
227
|
+
#
|
228
|
+
|
229
|
+
def register_on_raise(self, fn: CheckOnRaiseFn) -> None:
|
230
|
+
with self._config_lock:
|
231
|
+
self._on_raise_fns = [*self._on_raise_fns, fn]
|
232
|
+
|
233
|
+
def unregister_on_raise(self, fn: CheckOnRaiseFn) -> None:
|
234
|
+
with self._config_lock:
|
235
|
+
self._on_raise_fns = [e for e in self._on_raise_fns if e != fn]
|
236
|
+
|
237
|
+
#
|
238
|
+
|
239
|
+
def set_exception_factory(self, factory: CheckExceptionFactory) -> None:
|
240
|
+
self._exception_factory = factory
|
241
|
+
|
242
|
+
def set_args_renderer(self, renderer: ta.Optional[CheckArgsRenderer]) -> None:
|
243
|
+
self._args_renderer = renderer
|
244
|
+
|
245
|
+
#
|
246
|
+
|
247
|
+
def register_late_configure(self, fn: CheckLateConfigureFn) -> None:
|
248
|
+
with self._config_lock:
|
249
|
+
self._late_configure_fns = [*self._late_configure_fns, fn]
|
250
|
+
|
251
|
+
def _late_configure(self) -> None:
|
252
|
+
if not self._late_configure_fns:
|
253
|
+
return
|
254
|
+
|
255
|
+
with self._config_lock:
|
256
|
+
if not (lc := self._late_configure_fns):
|
257
|
+
return
|
258
|
+
|
259
|
+
for fn in lc:
|
260
|
+
fn(self)
|
261
|
+
|
262
|
+
self._late_configure_fns = []
|
263
|
+
|
264
|
+
#
|
265
|
+
|
266
|
+
class _ArgsKwargs:
|
267
|
+
def __init__(self, *args, **kwargs):
|
268
|
+
self.args = args
|
269
|
+
self.kwargs = kwargs
|
270
|
+
|
271
|
+
def _raise(
|
272
|
+
self,
|
273
|
+
exception_type: ta.Type[Exception],
|
274
|
+
default_message: str,
|
275
|
+
message: CheckMessage,
|
276
|
+
ak: _ArgsKwargs = _ArgsKwargs(),
|
277
|
+
*,
|
278
|
+
render_fmt: ta.Optional[str] = None,
|
279
|
+
) -> ta.NoReturn:
|
280
|
+
exc_args = ()
|
281
|
+
if callable(message):
|
282
|
+
message = ta.cast(ta.Callable, message)(*ak.args, **ak.kwargs)
|
283
|
+
if isinstance(message, tuple):
|
284
|
+
message, *exc_args = message # type: ignore
|
285
|
+
|
286
|
+
if message is None:
|
287
|
+
message = default_message
|
288
|
+
|
289
|
+
self._late_configure()
|
290
|
+
|
291
|
+
if render_fmt is not None and (af := self._args_renderer) is not None:
|
292
|
+
rendered_args = af(render_fmt, *ak.args)
|
293
|
+
if rendered_args is not None:
|
294
|
+
message = f'{message} : {rendered_args}'
|
295
|
+
|
296
|
+
exc = self._exception_factory(
|
297
|
+
exception_type,
|
298
|
+
message,
|
299
|
+
*exc_args,
|
300
|
+
*ak.args,
|
301
|
+
**ak.kwargs,
|
302
|
+
)
|
303
|
+
|
304
|
+
for fn in self._on_raise_fns:
|
305
|
+
fn(exc)
|
306
|
+
|
307
|
+
raise exc
|
308
|
+
|
309
|
+
#
|
310
|
+
|
311
|
+
def _unpack_isinstance_spec(self, spec: ta.Any) -> tuple:
|
312
|
+
if isinstance(spec, type):
|
313
|
+
return (spec,)
|
314
|
+
if not isinstance(spec, tuple):
|
315
|
+
spec = (spec,)
|
316
|
+
if None in spec:
|
317
|
+
spec = tuple(filter(None, spec)) + (None.__class__,) # noqa
|
318
|
+
if ta.Any in spec:
|
319
|
+
spec = (object,)
|
320
|
+
return spec
|
321
|
+
|
322
|
+
def isinstance(self, v: ta.Any, spec: ta.Union[ta.Type[T], tuple], msg: CheckMessage = None) -> T: # noqa
|
323
|
+
if not isinstance(v, self._unpack_isinstance_spec(spec)):
|
324
|
+
self._raise(
|
325
|
+
TypeError,
|
326
|
+
'Must be instance',
|
327
|
+
msg,
|
328
|
+
Checks._ArgsKwargs(v, spec),
|
329
|
+
render_fmt='not isinstance(%s, %s)',
|
330
|
+
)
|
331
|
+
|
332
|
+
return v
|
333
|
+
|
334
|
+
def of_isinstance(self, spec: ta.Union[ta.Type[T], tuple], msg: CheckMessage = None) -> ta.Callable[[ta.Any], T]:
|
335
|
+
def inner(v):
|
336
|
+
return self.isinstance(v, self._unpack_isinstance_spec(spec), msg)
|
337
|
+
|
338
|
+
return inner
|
339
|
+
|
340
|
+
def cast(self, v: ta.Any, cls: ta.Type[T], msg: CheckMessage = None) -> T: # noqa
|
341
|
+
if not isinstance(v, cls):
|
342
|
+
self._raise(
|
343
|
+
TypeError,
|
344
|
+
'Must be instance',
|
345
|
+
msg,
|
346
|
+
Checks._ArgsKwargs(v, cls),
|
347
|
+
)
|
348
|
+
|
349
|
+
return v
|
350
|
+
|
351
|
+
def of_cast(self, cls: ta.Type[T], msg: CheckMessage = None) -> ta.Callable[[T], T]:
|
352
|
+
def inner(v):
|
353
|
+
return self.cast(v, cls, msg)
|
354
|
+
|
355
|
+
return inner
|
356
|
+
|
357
|
+
def not_isinstance(self, v: T, spec: ta.Any, msg: CheckMessage = None) -> T: # noqa
|
358
|
+
if isinstance(v, self._unpack_isinstance_spec(spec)):
|
359
|
+
self._raise(
|
360
|
+
TypeError,
|
361
|
+
'Must not be instance',
|
362
|
+
msg,
|
363
|
+
Checks._ArgsKwargs(v, spec),
|
364
|
+
render_fmt='isinstance(%s, %s)',
|
365
|
+
)
|
366
|
+
|
367
|
+
return v
|
368
|
+
|
369
|
+
def of_not_isinstance(self, spec: ta.Any, msg: CheckMessage = None) -> ta.Callable[[T], T]:
|
370
|
+
def inner(v):
|
371
|
+
return self.not_isinstance(v, self._unpack_isinstance_spec(spec), msg)
|
372
|
+
|
373
|
+
return inner
|
374
|
+
|
375
|
+
##
|
376
|
+
|
377
|
+
def issubclass(self, v: ta.Type[T], spec: ta.Any, msg: CheckMessage = None) -> ta.Type[T]: # noqa
|
378
|
+
if not issubclass(v, spec):
|
379
|
+
self._raise(
|
380
|
+
TypeError,
|
381
|
+
'Must be subclass',
|
382
|
+
msg,
|
383
|
+
Checks._ArgsKwargs(v, spec),
|
384
|
+
render_fmt='not issubclass(%s, %s)',
|
385
|
+
)
|
386
|
+
|
387
|
+
return v
|
388
|
+
|
389
|
+
def not_issubclass(self, v: ta.Type[T], spec: ta.Any, msg: CheckMessage = None) -> ta.Type[T]: # noqa
|
390
|
+
if issubclass(v, spec):
|
391
|
+
self._raise(
|
392
|
+
TypeError,
|
393
|
+
'Must not be subclass',
|
394
|
+
msg,
|
395
|
+
Checks._ArgsKwargs(v, spec),
|
396
|
+
render_fmt='issubclass(%s, %s)',
|
397
|
+
)
|
398
|
+
|
399
|
+
return v
|
400
|
+
|
401
|
+
#
|
402
|
+
|
403
|
+
def in_(self, v: T, c: ta.Container[T], msg: CheckMessage = None) -> T:
|
404
|
+
if v not in c:
|
405
|
+
self._raise(
|
406
|
+
ValueError,
|
407
|
+
'Must be in',
|
408
|
+
msg,
|
409
|
+
Checks._ArgsKwargs(v, c),
|
410
|
+
render_fmt='%s not in %s',
|
411
|
+
)
|
412
|
+
|
413
|
+
return v
|
414
|
+
|
415
|
+
def not_in(self, v: T, c: ta.Container[T], msg: CheckMessage = None) -> T:
|
416
|
+
if v in c:
|
417
|
+
self._raise(
|
418
|
+
ValueError,
|
419
|
+
'Must not be in',
|
420
|
+
msg,
|
421
|
+
Checks._ArgsKwargs(v, c),
|
422
|
+
render_fmt='%s in %s',
|
423
|
+
)
|
424
|
+
|
425
|
+
return v
|
426
|
+
|
427
|
+
def empty(self, v: SizedT, msg: CheckMessage = None) -> SizedT:
|
428
|
+
if len(v) != 0:
|
429
|
+
self._raise(
|
430
|
+
ValueError,
|
431
|
+
'Must be empty',
|
432
|
+
msg,
|
433
|
+
Checks._ArgsKwargs(v),
|
434
|
+
render_fmt='%s',
|
435
|
+
)
|
436
|
+
|
437
|
+
return v
|
438
|
+
|
439
|
+
def iterempty(self, v: ta.Iterable[T], msg: CheckMessage = None) -> ta.Iterable[T]:
|
440
|
+
it = iter(v)
|
441
|
+
try:
|
442
|
+
next(it)
|
443
|
+
except StopIteration:
|
444
|
+
pass
|
445
|
+
else:
|
446
|
+
self._raise(
|
447
|
+
ValueError,
|
448
|
+
'Must be empty',
|
449
|
+
msg,
|
450
|
+
Checks._ArgsKwargs(v),
|
451
|
+
render_fmt='%s',
|
452
|
+
)
|
453
|
+
|
454
|
+
return v
|
455
|
+
|
456
|
+
def not_empty(self, v: SizedT, msg: CheckMessage = None) -> SizedT:
|
457
|
+
if len(v) == 0:
|
458
|
+
self._raise(
|
459
|
+
ValueError,
|
460
|
+
'Must not be empty',
|
461
|
+
msg,
|
462
|
+
Checks._ArgsKwargs(v),
|
463
|
+
render_fmt='%s',
|
464
|
+
)
|
465
|
+
|
466
|
+
return v
|
467
|
+
|
468
|
+
def unique(self, it: ta.Iterable[T], msg: CheckMessage = None) -> ta.Iterable[T]:
|
469
|
+
dupes = [e for e, c in collections.Counter(it).items() if c > 1]
|
470
|
+
if dupes:
|
471
|
+
self._raise(
|
472
|
+
ValueError,
|
473
|
+
'Must be unique',
|
474
|
+
msg,
|
475
|
+
Checks._ArgsKwargs(it, dupes),
|
476
|
+
)
|
477
|
+
|
478
|
+
return it
|
479
|
+
|
480
|
+
def single(self, obj: ta.Iterable[T], message: CheckMessage = None) -> T:
|
481
|
+
try:
|
482
|
+
[value] = obj
|
483
|
+
except ValueError:
|
484
|
+
self._raise(
|
485
|
+
ValueError,
|
486
|
+
'Must be single',
|
487
|
+
message,
|
488
|
+
Checks._ArgsKwargs(obj),
|
489
|
+
render_fmt='%s',
|
490
|
+
)
|
491
|
+
|
492
|
+
return value
|
493
|
+
|
494
|
+
def opt_single(self, obj: ta.Iterable[T], message: CheckMessage = None) -> ta.Optional[T]:
|
495
|
+
it = iter(obj)
|
496
|
+
try:
|
497
|
+
value = next(it)
|
498
|
+
except StopIteration:
|
499
|
+
return None
|
500
|
+
|
501
|
+
try:
|
502
|
+
next(it)
|
503
|
+
except StopIteration:
|
504
|
+
return value # noqa
|
505
|
+
|
506
|
+
self._raise(
|
507
|
+
ValueError,
|
508
|
+
'Must be empty or single',
|
509
|
+
message,
|
510
|
+
Checks._ArgsKwargs(obj),
|
511
|
+
render_fmt='%s',
|
512
|
+
)
|
513
|
+
|
514
|
+
raise RuntimeError # noqa
|
515
|
+
|
516
|
+
#
|
517
|
+
|
518
|
+
def none(self, v: ta.Any, msg: CheckMessage = None) -> None:
|
519
|
+
if v is not None:
|
520
|
+
self._raise(
|
521
|
+
ValueError,
|
522
|
+
'Must be None',
|
523
|
+
msg,
|
524
|
+
Checks._ArgsKwargs(v),
|
525
|
+
render_fmt='%s',
|
526
|
+
)
|
527
|
+
|
528
|
+
def not_none(self, v: ta.Optional[T], msg: CheckMessage = None) -> T:
|
529
|
+
if v is None:
|
530
|
+
self._raise(
|
531
|
+
ValueError,
|
532
|
+
'Must not be None',
|
533
|
+
msg,
|
534
|
+
Checks._ArgsKwargs(v),
|
535
|
+
render_fmt='%s',
|
536
|
+
)
|
537
|
+
|
538
|
+
return v
|
539
|
+
|
540
|
+
#
|
541
|
+
|
542
|
+
def equal(self, v: T, o: ta.Any, msg: CheckMessage = None) -> T:
|
543
|
+
if o != v:
|
544
|
+
self._raise(
|
545
|
+
ValueError,
|
546
|
+
'Must be equal',
|
547
|
+
msg,
|
548
|
+
Checks._ArgsKwargs(v, o),
|
549
|
+
render_fmt='%s != %s',
|
550
|
+
)
|
551
|
+
|
552
|
+
return v
|
553
|
+
|
554
|
+
def is_(self, v: T, o: ta.Any, msg: CheckMessage = None) -> T:
|
555
|
+
if o is not v:
|
556
|
+
self._raise(
|
557
|
+
ValueError,
|
558
|
+
'Must be the same',
|
559
|
+
msg,
|
560
|
+
Checks._ArgsKwargs(v, o),
|
561
|
+
render_fmt='%s is not %s',
|
562
|
+
)
|
563
|
+
|
564
|
+
return v
|
565
|
+
|
566
|
+
def is_not(self, v: T, o: ta.Any, msg: CheckMessage = None) -> T:
|
567
|
+
if o is v:
|
568
|
+
self._raise(
|
569
|
+
ValueError,
|
570
|
+
'Must not be the same',
|
571
|
+
msg,
|
572
|
+
Checks._ArgsKwargs(v, o),
|
573
|
+
render_fmt='%s is %s',
|
574
|
+
)
|
575
|
+
|
576
|
+
return v
|
577
|
+
|
578
|
+
def callable(self, v: T, msg: CheckMessage = None) -> T: # noqa
|
579
|
+
if not callable(v):
|
580
|
+
self._raise(
|
581
|
+
TypeError,
|
582
|
+
'Must be callable',
|
583
|
+
msg,
|
584
|
+
Checks._ArgsKwargs(v),
|
585
|
+
render_fmt='%s',
|
586
|
+
)
|
587
|
+
|
588
|
+
return v # type: ignore
|
589
|
+
|
590
|
+
def non_empty_str(self, v: ta.Optional[str], msg: CheckMessage = None) -> str:
|
591
|
+
if not isinstance(v, str) or not v:
|
592
|
+
self._raise(
|
593
|
+
ValueError,
|
594
|
+
'Must be non-empty str',
|
595
|
+
msg,
|
596
|
+
Checks._ArgsKwargs(v),
|
597
|
+
render_fmt='%s',
|
598
|
+
)
|
599
|
+
|
600
|
+
return v
|
601
|
+
|
602
|
+
def replacing(self, expected: ta.Any, old: ta.Any, new: T, msg: CheckMessage = None) -> T:
|
603
|
+
if old != expected:
|
604
|
+
self._raise(
|
605
|
+
ValueError,
|
606
|
+
'Must be replacing',
|
607
|
+
msg,
|
608
|
+
Checks._ArgsKwargs(expected, old, new),
|
609
|
+
render_fmt='%s -> %s -> %s',
|
610
|
+
)
|
611
|
+
|
612
|
+
return new
|
613
|
+
|
614
|
+
def replacing_none(self, old: ta.Any, new: T, msg: CheckMessage = None) -> T:
|
615
|
+
if old is not None:
|
616
|
+
self._raise(
|
617
|
+
ValueError,
|
618
|
+
'Must be replacing None',
|
619
|
+
msg,
|
620
|
+
Checks._ArgsKwargs(old, new),
|
621
|
+
render_fmt='%s -> %s',
|
622
|
+
)
|
623
|
+
|
624
|
+
return new
|
625
|
+
|
626
|
+
#
|
627
|
+
|
628
|
+
def arg(self, v: bool, msg: CheckMessage = None) -> None:
|
629
|
+
if not v:
|
630
|
+
self._raise(
|
631
|
+
RuntimeError,
|
632
|
+
'Argument condition not met',
|
633
|
+
msg,
|
634
|
+
Checks._ArgsKwargs(v),
|
635
|
+
render_fmt='%s',
|
636
|
+
)
|
637
|
+
|
638
|
+
def state(self, v: bool, msg: CheckMessage = None) -> None:
|
639
|
+
if not v:
|
640
|
+
self._raise(
|
641
|
+
RuntimeError,
|
642
|
+
'State condition not met',
|
643
|
+
msg,
|
644
|
+
Checks._ArgsKwargs(v),
|
645
|
+
render_fmt='%s',
|
646
|
+
)
|
647
|
+
|
648
|
+
|
649
|
+
check = Checks()
|
650
|
+
|
651
|
+
|
652
|
+
########################################
|
653
|
+
# ../../../omlish/lite/json.py
|
654
|
+
|
655
|
+
|
656
|
+
##
|
657
|
+
|
658
|
+
|
659
|
+
JSON_PRETTY_INDENT = 2
|
660
|
+
|
661
|
+
JSON_PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
|
662
|
+
indent=JSON_PRETTY_INDENT,
|
663
|
+
)
|
664
|
+
|
665
|
+
json_dump_pretty: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_PRETTY_KWARGS) # type: ignore
|
666
|
+
json_dumps_pretty: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_PRETTY_KWARGS)
|
667
|
+
|
668
|
+
|
669
|
+
##
|
670
|
+
|
671
|
+
|
672
|
+
JSON_COMPACT_SEPARATORS = (',', ':')
|
673
|
+
|
674
|
+
JSON_COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
|
675
|
+
indent=None,
|
676
|
+
separators=JSON_COMPACT_SEPARATORS,
|
677
|
+
)
|
678
|
+
|
679
|
+
json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_COMPACT_KWARGS) # type: ignore
|
680
|
+
json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
|
681
|
+
|
682
|
+
|
683
|
+
########################################
|
684
|
+
# ../../../omlish/lite/reflect.py
|
685
|
+
|
686
|
+
|
687
|
+
##
|
688
|
+
|
689
|
+
|
690
|
+
_GENERIC_ALIAS_TYPES = (
|
691
|
+
ta._GenericAlias, # type: ignore # noqa
|
692
|
+
*([ta._SpecialGenericAlias] if hasattr(ta, '_SpecialGenericAlias') else []), # noqa
|
693
|
+
)
|
694
|
+
|
695
|
+
|
696
|
+
def is_generic_alias(obj, *, origin: ta.Any = None) -> bool:
|
697
|
+
return (
|
698
|
+
isinstance(obj, _GENERIC_ALIAS_TYPES) and
|
699
|
+
(origin is None or ta.get_origin(obj) is origin)
|
700
|
+
)
|
701
|
+
|
702
|
+
|
703
|
+
is_union_alias = functools.partial(is_generic_alias, origin=ta.Union)
|
704
|
+
is_callable_alias = functools.partial(is_generic_alias, origin=ta.Callable)
|
705
|
+
|
706
|
+
|
707
|
+
##
|
708
|
+
|
709
|
+
|
710
|
+
def is_optional_alias(spec: ta.Any) -> bool:
|
711
|
+
return (
|
712
|
+
isinstance(spec, _GENERIC_ALIAS_TYPES) and # noqa
|
713
|
+
ta.get_origin(spec) is ta.Union and
|
714
|
+
len(ta.get_args(spec)) == 2 and
|
715
|
+
any(a in (None, type(None)) for a in ta.get_args(spec))
|
716
|
+
)
|
717
|
+
|
718
|
+
|
719
|
+
def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
|
720
|
+
[it] = [it for it in ta.get_args(spec) if it not in (None, type(None))]
|
721
|
+
return it
|
722
|
+
|
723
|
+
|
724
|
+
##
|
725
|
+
|
726
|
+
|
727
|
+
def is_new_type(spec: ta.Any) -> bool:
|
728
|
+
if isinstance(ta.NewType, type):
|
729
|
+
return isinstance(spec, ta.NewType)
|
730
|
+
else:
|
731
|
+
# Before https://github.com/python/cpython/commit/c2f33dfc83ab270412bf243fb21f724037effa1a
|
732
|
+
return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1] # type: ignore # noqa
|
733
|
+
|
734
|
+
|
735
|
+
def get_new_type_supertype(spec: ta.Any) -> ta.Any:
|
736
|
+
return spec.__supertype__
|
737
|
+
|
738
|
+
|
739
|
+
##
|
740
|
+
|
741
|
+
|
742
|
+
def is_literal_type(spec: ta.Any) -> bool:
|
743
|
+
if hasattr(ta, '_LiteralGenericAlias'):
|
744
|
+
return isinstance(spec, ta._LiteralGenericAlias) # noqa
|
745
|
+
else:
|
746
|
+
return (
|
747
|
+
isinstance(spec, ta._GenericAlias) and # type: ignore # noqa
|
748
|
+
spec.__origin__ is ta.Literal
|
749
|
+
)
|
750
|
+
|
751
|
+
|
752
|
+
def get_literal_type_args(spec: ta.Any) -> ta.Iterable[ta.Any]:
|
753
|
+
return spec.__args__
|
754
|
+
|
755
|
+
|
756
|
+
##
|
757
|
+
|
758
|
+
|
759
|
+
def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
|
760
|
+
seen = set()
|
761
|
+
todo = list(reversed(cls.__subclasses__()))
|
762
|
+
while todo:
|
763
|
+
cur = todo.pop()
|
764
|
+
if cur in seen:
|
765
|
+
continue
|
766
|
+
seen.add(cur)
|
767
|
+
yield cur
|
768
|
+
todo.extend(reversed(cur.__subclasses__()))
|
769
|
+
|
770
|
+
|
771
|
+
########################################
|
772
|
+
# ../../../omlish/argparse/cli.py
|
773
|
+
"""
|
774
|
+
TODO:
|
775
|
+
- default command
|
776
|
+
- auto match all underscores to hyphens
|
777
|
+
- pre-run, post-run hooks
|
778
|
+
- exitstack?
|
779
|
+
"""
|
780
|
+
|
781
|
+
|
782
|
+
##
|
783
|
+
|
784
|
+
|
785
|
+
@dc.dataclass(eq=False)
|
786
|
+
class ArgparseArg:
|
787
|
+
args: ta.Sequence[ta.Any]
|
788
|
+
kwargs: ta.Mapping[str, ta.Any]
|
789
|
+
dest: ta.Optional[str] = None
|
790
|
+
|
791
|
+
def __get__(self, instance, owner=None):
|
792
|
+
if instance is None:
|
793
|
+
return self
|
794
|
+
return getattr(instance.args, self.dest) # type: ignore
|
795
|
+
|
796
|
+
|
797
|
+
def argparse_arg(*args, **kwargs) -> ArgparseArg:
|
798
|
+
return ArgparseArg(args, kwargs)
|
799
|
+
|
800
|
+
|
801
|
+
#
|
802
|
+
|
803
|
+
|
804
|
+
@dc.dataclass(eq=False)
|
805
|
+
class ArgparseCmd:
|
806
|
+
name: str
|
807
|
+
fn: ArgparseCmdFn
|
808
|
+
args: ta.Sequence[ArgparseArg] = () # noqa
|
809
|
+
|
810
|
+
# _: dc.KW_ONLY
|
811
|
+
|
812
|
+
aliases: ta.Optional[ta.Sequence[str]] = None
|
813
|
+
parent: ta.Optional['ArgparseCmd'] = None
|
814
|
+
accepts_unknown: bool = False
|
815
|
+
|
816
|
+
def __post_init__(self) -> None:
|
817
|
+
def check_name(s: str) -> None:
|
818
|
+
check.isinstance(s, str)
|
819
|
+
check.not_in('_', s)
|
820
|
+
check.not_empty(s)
|
821
|
+
check_name(self.name)
|
822
|
+
check.not_isinstance(self.aliases, str)
|
823
|
+
for a in self.aliases or []:
|
824
|
+
check_name(a)
|
825
|
+
|
826
|
+
check.arg(callable(self.fn))
|
827
|
+
check.arg(all(isinstance(a, ArgparseArg) for a in self.args))
|
828
|
+
check.isinstance(self.parent, (ArgparseCmd, type(None)))
|
829
|
+
check.isinstance(self.accepts_unknown, bool)
|
830
|
+
|
831
|
+
functools.update_wrapper(self, self.fn)
|
832
|
+
|
833
|
+
def __get__(self, instance, owner=None):
|
834
|
+
if instance is None:
|
835
|
+
return self
|
836
|
+
return dc.replace(self, fn=self.fn.__get__(instance, owner)) # noqa
|
837
|
+
|
838
|
+
def __call__(self, *args, **kwargs) -> ta.Optional[int]:
|
839
|
+
return self.fn(*args, **kwargs)
|
840
|
+
|
841
|
+
|
842
|
+
def argparse_cmd(
|
843
|
+
*args: ArgparseArg,
|
844
|
+
name: ta.Optional[str] = None,
|
845
|
+
aliases: ta.Optional[ta.Iterable[str]] = None,
|
846
|
+
parent: ta.Optional[ArgparseCmd] = None,
|
847
|
+
accepts_unknown: bool = False,
|
848
|
+
) -> ta.Any: # ta.Callable[[ArgparseCmdFn], ArgparseCmd]: # FIXME
|
849
|
+
for arg in args:
|
850
|
+
check.isinstance(arg, ArgparseArg)
|
851
|
+
check.isinstance(name, (str, type(None)))
|
852
|
+
check.isinstance(parent, (ArgparseCmd, type(None)))
|
853
|
+
check.not_isinstance(aliases, str)
|
854
|
+
|
855
|
+
def inner(fn):
|
856
|
+
return ArgparseCmd(
|
857
|
+
(name if name is not None else fn.__name__).replace('_', '-'),
|
858
|
+
fn,
|
859
|
+
args,
|
860
|
+
aliases=tuple(aliases) if aliases is not None else None,
|
861
|
+
parent=parent,
|
862
|
+
accepts_unknown=accepts_unknown,
|
863
|
+
)
|
864
|
+
|
865
|
+
return inner
|
866
|
+
|
867
|
+
|
868
|
+
##
|
869
|
+
|
870
|
+
|
871
|
+
def _get_argparse_arg_ann_kwargs(ann: ta.Any) -> ta.Mapping[str, ta.Any]:
|
872
|
+
if ann is str:
|
873
|
+
return {}
|
874
|
+
elif ann is int:
|
875
|
+
return {'type': int}
|
876
|
+
elif ann is bool:
|
877
|
+
return {'action': 'store_true'}
|
878
|
+
elif ann is list:
|
879
|
+
return {'action': 'append'}
|
880
|
+
elif is_optional_alias(ann):
|
881
|
+
return _get_argparse_arg_ann_kwargs(get_optional_alias_arg(ann))
|
882
|
+
else:
|
883
|
+
raise TypeError(ann)
|
884
|
+
|
885
|
+
|
886
|
+
class _ArgparseCliAnnotationBox:
|
887
|
+
def __init__(self, annotations: ta.Mapping[str, ta.Any]) -> None:
|
888
|
+
super().__init__()
|
889
|
+
self.__annotations__ = annotations # type: ignore
|
890
|
+
|
891
|
+
|
892
|
+
class ArgparseCli:
|
893
|
+
def __init__(self, argv: ta.Optional[ta.Sequence[str]] = None) -> None:
|
894
|
+
super().__init__()
|
895
|
+
|
896
|
+
self._argv = argv if argv is not None else sys.argv[1:]
|
897
|
+
|
898
|
+
self._args, self._unknown_args = self.get_parser().parse_known_args(self._argv)
|
899
|
+
|
900
|
+
#
|
901
|
+
|
902
|
+
def __init_subclass__(cls, **kwargs: ta.Any) -> None:
|
903
|
+
super().__init_subclass__(**kwargs)
|
904
|
+
|
905
|
+
ns = cls.__dict__
|
906
|
+
objs = {}
|
907
|
+
mro = cls.__mro__[::-1]
|
908
|
+
for bns in [bcls.__dict__ for bcls in reversed(mro)] + [ns]:
|
909
|
+
bseen = set() # type: ignore
|
910
|
+
for k, v in bns.items():
|
911
|
+
if isinstance(v, (ArgparseCmd, ArgparseArg)):
|
912
|
+
check.not_in(v, bseen)
|
913
|
+
bseen.add(v)
|
914
|
+
objs[k] = v
|
915
|
+
elif k in objs:
|
916
|
+
del [k]
|
917
|
+
|
918
|
+
#
|
919
|
+
|
920
|
+
anns = ta.get_type_hints(_ArgparseCliAnnotationBox({
|
921
|
+
**{k: v for bcls in reversed(mro) for k, v in getattr(bcls, '__annotations__', {}).items()},
|
922
|
+
**ns.get('__annotations__', {}),
|
923
|
+
}), globalns=ns.get('__globals__', {}))
|
924
|
+
|
925
|
+
#
|
926
|
+
|
927
|
+
if '_parser' in ns:
|
928
|
+
parser = check.isinstance(ns['_parser'], argparse.ArgumentParser)
|
929
|
+
else:
|
930
|
+
parser = argparse.ArgumentParser()
|
931
|
+
setattr(cls, '_parser', parser)
|
932
|
+
|
933
|
+
#
|
934
|
+
|
935
|
+
subparsers = parser.add_subparsers()
|
936
|
+
|
937
|
+
for att, obj in objs.items():
|
938
|
+
if isinstance(obj, ArgparseCmd):
|
939
|
+
if obj.parent is not None:
|
940
|
+
raise NotImplementedError
|
941
|
+
|
942
|
+
for cn in [obj.name, *(obj.aliases or [])]:
|
943
|
+
subparser = subparsers.add_parser(cn)
|
944
|
+
|
945
|
+
for arg in (obj.args or []):
|
946
|
+
if (
|
947
|
+
len(arg.args) == 1 and
|
948
|
+
isinstance(arg.args[0], str) and
|
949
|
+
not (n := check.isinstance(arg.args[0], str)).startswith('-') and
|
950
|
+
'metavar' not in arg.kwargs
|
951
|
+
):
|
952
|
+
subparser.add_argument(
|
953
|
+
n.replace('-', '_'),
|
954
|
+
**arg.kwargs,
|
955
|
+
metavar=n,
|
956
|
+
)
|
957
|
+
else:
|
958
|
+
subparser.add_argument(*arg.args, **arg.kwargs)
|
959
|
+
|
960
|
+
subparser.set_defaults(_cmd=obj)
|
961
|
+
|
962
|
+
elif isinstance(obj, ArgparseArg):
|
963
|
+
if att in anns:
|
964
|
+
ann_kwargs = _get_argparse_arg_ann_kwargs(anns[att])
|
965
|
+
obj.kwargs = {**ann_kwargs, **obj.kwargs}
|
966
|
+
|
967
|
+
if not obj.dest:
|
968
|
+
if 'dest' in obj.kwargs:
|
969
|
+
obj.dest = obj.kwargs['dest']
|
970
|
+
else:
|
971
|
+
obj.dest = obj.kwargs['dest'] = att # type: ignore
|
972
|
+
|
973
|
+
parser.add_argument(*obj.args, **obj.kwargs)
|
974
|
+
|
975
|
+
else:
|
976
|
+
raise TypeError(obj)
|
977
|
+
|
978
|
+
#
|
979
|
+
|
980
|
+
_parser: ta.ClassVar[argparse.ArgumentParser]
|
981
|
+
|
982
|
+
@classmethod
|
983
|
+
def get_parser(cls) -> argparse.ArgumentParser:
|
984
|
+
return cls._parser
|
985
|
+
|
986
|
+
@property
|
987
|
+
def argv(self) -> ta.Sequence[str]:
|
988
|
+
return self._argv
|
989
|
+
|
990
|
+
@property
|
991
|
+
def args(self) -> argparse.Namespace:
|
992
|
+
return self._args
|
993
|
+
|
994
|
+
@property
|
995
|
+
def unknown_args(self) -> ta.Sequence[str]:
|
996
|
+
return self._unknown_args
|
997
|
+
|
998
|
+
#
|
999
|
+
|
1000
|
+
def _bind_cli_cmd(self, cmd: ArgparseCmd) -> ta.Callable:
|
1001
|
+
return cmd.__get__(self, type(self))
|
1002
|
+
|
1003
|
+
def prepare_cli_run(self) -> ta.Optional[ta.Callable]:
|
1004
|
+
cmd = getattr(self.args, '_cmd', None)
|
1005
|
+
|
1006
|
+
if self._unknown_args and not (cmd is not None and cmd.accepts_unknown):
|
1007
|
+
msg = f'unrecognized arguments: {" ".join(self._unknown_args)}'
|
1008
|
+
if (parser := self.get_parser()).exit_on_error: # type: ignore
|
1009
|
+
parser.error(msg)
|
1010
|
+
else:
|
1011
|
+
raise argparse.ArgumentError(None, msg)
|
1012
|
+
|
1013
|
+
if cmd is None:
|
1014
|
+
self.get_parser().print_help()
|
1015
|
+
return None
|
1016
|
+
|
1017
|
+
return self._bind_cli_cmd(cmd)
|
1018
|
+
|
1019
|
+
#
|
1020
|
+
|
1021
|
+
def cli_run(self) -> ta.Optional[int]:
|
1022
|
+
if (fn := self.prepare_cli_run()) is None:
|
1023
|
+
return 0
|
1024
|
+
|
1025
|
+
return fn()
|
1026
|
+
|
1027
|
+
def cli_run_and_exit(self) -> ta.NoReturn:
|
1028
|
+
sys.exit(rc if isinstance(rc := self.cli_run(), int) else 0)
|
1029
|
+
|
1030
|
+
def __call__(self, *, exit: bool = False) -> ta.Optional[int]: # noqa
|
1031
|
+
if exit:
|
1032
|
+
return self.cli_run_and_exit()
|
1033
|
+
else:
|
1034
|
+
return self.cli_run()
|
1035
|
+
|
1036
|
+
#
|
1037
|
+
|
1038
|
+
async def async_cli_run(
|
1039
|
+
self,
|
1040
|
+
*,
|
1041
|
+
force_async: bool = False,
|
1042
|
+
) -> ta.Optional[int]:
|
1043
|
+
if (fn := self.prepare_cli_run()) is None:
|
1044
|
+
return 0
|
1045
|
+
|
1046
|
+
if force_async:
|
1047
|
+
is_async = True
|
1048
|
+
else:
|
1049
|
+
tfn = fn
|
1050
|
+
if isinstance(tfn, ArgparseCmd):
|
1051
|
+
tfn = tfn.fn
|
1052
|
+
is_async = inspect.iscoroutinefunction(tfn)
|
1053
|
+
|
1054
|
+
if is_async:
|
1055
|
+
return await fn()
|
1056
|
+
else:
|
1057
|
+
return fn()
|
1058
|
+
|
1059
|
+
|
1060
|
+
########################################
|
1061
|
+
# ../../../omlish/lite/contextmanagers.py
|
1062
|
+
|
1063
|
+
|
1064
|
+
##
|
1065
|
+
|
1066
|
+
|
1067
|
+
class ExitStacked:
|
1068
|
+
_exit_stack: ta.Optional[contextlib.ExitStack] = None
|
1069
|
+
|
1070
|
+
def __enter__(self: ExitStackedT) -> ExitStackedT:
|
1071
|
+
check.state(self._exit_stack is None)
|
1072
|
+
es = self._exit_stack = contextlib.ExitStack()
|
1073
|
+
es.__enter__()
|
1074
|
+
return self
|
1075
|
+
|
1076
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
1077
|
+
if (es := self._exit_stack) is None:
|
1078
|
+
return None
|
1079
|
+
self._exit_contexts()
|
1080
|
+
return es.__exit__(exc_type, exc_val, exc_tb)
|
1081
|
+
|
1082
|
+
def _exit_contexts(self) -> None:
|
1083
|
+
pass
|
1084
|
+
|
1085
|
+
def _enter_context(self, cm: ta.ContextManager[T]) -> T:
|
1086
|
+
es = check.not_none(self._exit_stack)
|
1087
|
+
return es.enter_context(cm)
|
1088
|
+
|
1089
|
+
|
1090
|
+
##
|
1091
|
+
|
1092
|
+
|
1093
|
+
@contextlib.contextmanager
|
1094
|
+
def defer(fn: ta.Callable) -> ta.Generator[ta.Callable, None, None]:
|
1095
|
+
try:
|
1096
|
+
yield fn
|
1097
|
+
finally:
|
1098
|
+
fn()
|
1099
|
+
|
1100
|
+
|
1101
|
+
@contextlib.contextmanager
|
1102
|
+
def attr_setting(obj, attr, val, *, default=None): # noqa
|
1103
|
+
not_set = object()
|
1104
|
+
orig = getattr(obj, attr, not_set)
|
1105
|
+
try:
|
1106
|
+
setattr(obj, attr, val)
|
1107
|
+
if orig is not not_set:
|
1108
|
+
yield orig
|
1109
|
+
else:
|
1110
|
+
yield default
|
1111
|
+
finally:
|
1112
|
+
if orig is not_set:
|
1113
|
+
delattr(obj, attr)
|
1114
|
+
else:
|
1115
|
+
setattr(obj, attr, orig)
|
1116
|
+
|
1117
|
+
|
1118
|
+
##
|
1119
|
+
|
1120
|
+
|
1121
|
+
class aclosing(contextlib.AbstractAsyncContextManager): # noqa
|
1122
|
+
def __init__(self, thing):
|
1123
|
+
self.thing = thing
|
1124
|
+
|
1125
|
+
async def __aenter__(self):
|
1126
|
+
return self.thing
|
1127
|
+
|
1128
|
+
async def __aexit__(self, *exc_info):
|
1129
|
+
await self.thing.aclose()
|
1130
|
+
|
1131
|
+
|
1132
|
+
########################################
|
1133
|
+
# ../../../omlish/lite/runtime.py
|
1134
|
+
|
1135
|
+
|
1136
|
+
@cached_nullary
|
1137
|
+
def is_debugger_attached() -> bool:
|
1138
|
+
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
1139
|
+
|
1140
|
+
|
1141
|
+
LITE_REQUIRED_PYTHON_VERSION = (3, 8)
|
1142
|
+
|
1143
|
+
|
1144
|
+
def check_lite_runtime_version() -> None:
|
1145
|
+
if sys.version_info < LITE_REQUIRED_PYTHON_VERSION:
|
1146
|
+
raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
|
1147
|
+
|
1148
|
+
|
1149
|
+
########################################
|
1150
|
+
# ../../../omlish/subprocesses.py
|
1151
|
+
|
1152
|
+
|
1153
|
+
##
|
1154
|
+
|
1155
|
+
|
1156
|
+
SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
|
1157
|
+
'pipe': subprocess.PIPE,
|
1158
|
+
'stdout': subprocess.STDOUT,
|
1159
|
+
'devnull': subprocess.DEVNULL,
|
1160
|
+
}
|
1161
|
+
|
1162
|
+
|
1163
|
+
##
|
1164
|
+
|
1165
|
+
|
1166
|
+
_SUBPROCESS_SHELL_WRAP_EXECS = False
|
1167
|
+
|
1168
|
+
|
1169
|
+
def subprocess_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
|
1170
|
+
return ('sh', '-c', ' '.join(map(shlex.quote, cmd)))
|
1171
|
+
|
1172
|
+
|
1173
|
+
def subprocess_maybe_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
|
1174
|
+
if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
|
1175
|
+
return subprocess_shell_wrap_exec(*cmd)
|
1176
|
+
else:
|
1177
|
+
return cmd
|
1178
|
+
|
1179
|
+
|
1180
|
+
##
|
1181
|
+
|
1182
|
+
|
1183
|
+
def subprocess_close(
|
1184
|
+
proc: subprocess.Popen,
|
1185
|
+
timeout: ta.Optional[float] = None,
|
1186
|
+
) -> None:
|
1187
|
+
# TODO: terminate, sleep, kill
|
1188
|
+
if proc.stdout:
|
1189
|
+
proc.stdout.close()
|
1190
|
+
if proc.stderr:
|
1191
|
+
proc.stderr.close()
|
1192
|
+
if proc.stdin:
|
1193
|
+
proc.stdin.close()
|
1194
|
+
|
1195
|
+
proc.wait(timeout)
|
1196
|
+
|
1197
|
+
|
1198
|
+
##
|
1199
|
+
|
1200
|
+
|
1201
|
+
class BaseSubprocesses(abc.ABC): # noqa
|
1202
|
+
DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
|
1203
|
+
|
1204
|
+
def __init__(
|
1205
|
+
self,
|
1206
|
+
*,
|
1207
|
+
log: ta.Optional[logging.Logger] = None,
|
1208
|
+
try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
|
1209
|
+
) -> None:
|
1210
|
+
super().__init__()
|
1211
|
+
|
1212
|
+
self._log = log if log is not None else self.DEFAULT_LOGGER
|
1213
|
+
self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
|
1214
|
+
|
1215
|
+
def set_logger(self, log: ta.Optional[logging.Logger]) -> None:
|
1216
|
+
self._log = log
|
1217
|
+
|
1218
|
+
#
|
1219
|
+
|
1220
|
+
def prepare_args(
|
1221
|
+
self,
|
1222
|
+
*cmd: str,
|
1223
|
+
env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
1224
|
+
extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
|
1225
|
+
quiet: bool = False,
|
1226
|
+
shell: bool = False,
|
1227
|
+
**kwargs: ta.Any,
|
1228
|
+
) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
|
1229
|
+
if self._log:
|
1230
|
+
self._log.debug('Subprocesses.prepare_args: cmd=%r', cmd)
|
1231
|
+
if extra_env:
|
1232
|
+
self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
|
1233
|
+
|
1234
|
+
if extra_env:
|
1235
|
+
env = {**(env if env is not None else os.environ), **extra_env}
|
1236
|
+
|
1237
|
+
if quiet and 'stderr' not in kwargs:
|
1238
|
+
if self._log and not self._log.isEnabledFor(logging.DEBUG):
|
1239
|
+
kwargs['stderr'] = subprocess.DEVNULL
|
1240
|
+
|
1241
|
+
if not shell:
|
1242
|
+
cmd = subprocess_maybe_shell_wrap_exec(*cmd)
|
1243
|
+
|
1244
|
+
return cmd, dict(
|
1245
|
+
env=env,
|
1246
|
+
shell=shell,
|
1247
|
+
**kwargs,
|
1248
|
+
)
|
1249
|
+
|
1250
|
+
@contextlib.contextmanager
|
1251
|
+
def wrap_call(self, *cmd: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
|
1252
|
+
start_time = time.time()
|
1253
|
+
try:
|
1254
|
+
if self._log:
|
1255
|
+
self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
|
1256
|
+
yield
|
1257
|
+
|
1258
|
+
except Exception as exc: # noqa
|
1259
|
+
if self._log:
|
1260
|
+
self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
|
1261
|
+
raise
|
1262
|
+
|
1263
|
+
finally:
|
1264
|
+
end_time = time.time()
|
1265
|
+
elapsed_s = end_time - start_time
|
1266
|
+
if self._log:
|
1267
|
+
self._log.debug('sSubprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
|
1268
|
+
|
1269
|
+
@contextlib.contextmanager
|
1270
|
+
def prepare_and_wrap(
|
1271
|
+
self,
|
1272
|
+
*cmd: ta.Any,
|
1273
|
+
**kwargs: ta.Any,
|
1274
|
+
) -> ta.Iterator[ta.Tuple[
|
1275
|
+
ta.Tuple[ta.Any, ...],
|
1276
|
+
ta.Dict[str, ta.Any],
|
1277
|
+
]]:
|
1278
|
+
cmd, kwargs = self.prepare_args(*cmd, **kwargs)
|
1279
|
+
with self.wrap_call(*cmd, **kwargs):
|
1280
|
+
yield cmd, kwargs
|
1281
|
+
|
1282
|
+
#
|
1283
|
+
|
1284
|
+
DEFAULT_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
|
1285
|
+
FileNotFoundError,
|
1286
|
+
subprocess.CalledProcessError,
|
1287
|
+
)
|
1288
|
+
|
1289
|
+
def try_fn(
|
1290
|
+
self,
|
1291
|
+
fn: ta.Callable[..., T],
|
1292
|
+
*cmd: str,
|
1293
|
+
try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
|
1294
|
+
**kwargs: ta.Any,
|
1295
|
+
) -> ta.Union[T, Exception]:
|
1296
|
+
if try_exceptions is None:
|
1297
|
+
try_exceptions = self._try_exceptions
|
1298
|
+
|
1299
|
+
try:
|
1300
|
+
return fn(*cmd, **kwargs)
|
1301
|
+
|
1302
|
+
except try_exceptions as e: # noqa
|
1303
|
+
if self._log and self._log.isEnabledFor(logging.DEBUG):
|
1304
|
+
self._log.exception('command failed')
|
1305
|
+
return e
|
1306
|
+
|
1307
|
+
async def async_try_fn(
|
1308
|
+
self,
|
1309
|
+
fn: ta.Callable[..., ta.Awaitable[T]],
|
1310
|
+
*cmd: ta.Any,
|
1311
|
+
try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
|
1312
|
+
**kwargs: ta.Any,
|
1313
|
+
) -> ta.Union[T, Exception]:
|
1314
|
+
if try_exceptions is None:
|
1315
|
+
try_exceptions = self._try_exceptions
|
1316
|
+
|
1317
|
+
try:
|
1318
|
+
return await fn(*cmd, **kwargs)
|
1319
|
+
|
1320
|
+
except try_exceptions as e: # noqa
|
1321
|
+
if self._log and self._log.isEnabledFor(logging.DEBUG):
|
1322
|
+
self._log.exception('command failed')
|
1323
|
+
return e
|
1324
|
+
|
1325
|
+
|
1326
|
+
##
|
1327
|
+
|
1328
|
+
|
1329
|
+
class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
|
1330
|
+
@abc.abstractmethod
|
1331
|
+
def check_call(
|
1332
|
+
self,
|
1333
|
+
*cmd: str,
|
1334
|
+
stdout: ta.Any = sys.stderr,
|
1335
|
+
**kwargs: ta.Any,
|
1336
|
+
) -> None:
|
1337
|
+
raise NotImplementedError
|
1338
|
+
|
1339
|
+
@abc.abstractmethod
|
1340
|
+
def check_output(
|
1341
|
+
self,
|
1342
|
+
*cmd: str,
|
1343
|
+
**kwargs: ta.Any,
|
1344
|
+
) -> bytes:
|
1345
|
+
raise NotImplementedError
|
1346
|
+
|
1347
|
+
#
|
1348
|
+
|
1349
|
+
def check_output_str(
|
1350
|
+
self,
|
1351
|
+
*cmd: str,
|
1352
|
+
**kwargs: ta.Any,
|
1353
|
+
) -> str:
|
1354
|
+
return self.check_output(*cmd, **kwargs).decode().strip()
|
1355
|
+
|
1356
|
+
#
|
1357
|
+
|
1358
|
+
def try_call(
|
1359
|
+
self,
|
1360
|
+
*cmd: str,
|
1361
|
+
**kwargs: ta.Any,
|
1362
|
+
) -> bool:
|
1363
|
+
if isinstance(self.try_fn(self.check_call, *cmd, **kwargs), Exception):
|
1364
|
+
return False
|
1365
|
+
else:
|
1366
|
+
return True
|
1367
|
+
|
1368
|
+
def try_output(
|
1369
|
+
self,
|
1370
|
+
*cmd: str,
|
1371
|
+
**kwargs: ta.Any,
|
1372
|
+
) -> ta.Optional[bytes]:
|
1373
|
+
if isinstance(ret := self.try_fn(self.check_output, *cmd, **kwargs), Exception):
|
1374
|
+
return None
|
1375
|
+
else:
|
1376
|
+
return ret
|
1377
|
+
|
1378
|
+
def try_output_str(
|
1379
|
+
self,
|
1380
|
+
*cmd: str,
|
1381
|
+
**kwargs: ta.Any,
|
1382
|
+
) -> ta.Optional[str]:
|
1383
|
+
if (ret := self.try_output(*cmd, **kwargs)) is None:
|
1384
|
+
return None
|
1385
|
+
else:
|
1386
|
+
return ret.decode().strip()
|
1387
|
+
|
1388
|
+
|
1389
|
+
##
|
1390
|
+
|
1391
|
+
|
1392
|
+
class Subprocesses(AbstractSubprocesses):
|
1393
|
+
def check_call(
|
1394
|
+
self,
|
1395
|
+
*cmd: str,
|
1396
|
+
stdout: ta.Any = sys.stderr,
|
1397
|
+
**kwargs: ta.Any,
|
1398
|
+
) -> None:
|
1399
|
+
with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
|
1400
|
+
subprocess.check_call(cmd, **kwargs)
|
1401
|
+
|
1402
|
+
def check_output(
|
1403
|
+
self,
|
1404
|
+
*cmd: str,
|
1405
|
+
**kwargs: ta.Any,
|
1406
|
+
) -> bytes:
|
1407
|
+
with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
|
1408
|
+
return subprocess.check_output(cmd, **kwargs)
|
1409
|
+
|
1410
|
+
|
1411
|
+
subprocesses = Subprocesses()
|
1412
|
+
|
1413
|
+
|
1414
|
+
##
|
1415
|
+
|
1416
|
+
|
1417
|
+
class AbstractAsyncSubprocesses(BaseSubprocesses):
|
1418
|
+
@abc.abstractmethod
|
1419
|
+
async def check_call(
|
1420
|
+
self,
|
1421
|
+
*cmd: str,
|
1422
|
+
stdout: ta.Any = sys.stderr,
|
1423
|
+
**kwargs: ta.Any,
|
1424
|
+
) -> None:
|
1425
|
+
raise NotImplementedError
|
1426
|
+
|
1427
|
+
@abc.abstractmethod
|
1428
|
+
async def check_output(
|
1429
|
+
self,
|
1430
|
+
*cmd: str,
|
1431
|
+
**kwargs: ta.Any,
|
1432
|
+
) -> bytes:
|
1433
|
+
raise NotImplementedError
|
1434
|
+
|
1435
|
+
#
|
1436
|
+
|
1437
|
+
async def check_output_str(
|
1438
|
+
self,
|
1439
|
+
*cmd: str,
|
1440
|
+
**kwargs: ta.Any,
|
1441
|
+
) -> str:
|
1442
|
+
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
1443
|
+
|
1444
|
+
#
|
1445
|
+
|
1446
|
+
async def try_call(
|
1447
|
+
self,
|
1448
|
+
*cmd: str,
|
1449
|
+
**kwargs: ta.Any,
|
1450
|
+
) -> bool:
|
1451
|
+
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
1452
|
+
return False
|
1453
|
+
else:
|
1454
|
+
return True
|
1455
|
+
|
1456
|
+
async def try_output(
|
1457
|
+
self,
|
1458
|
+
*cmd: str,
|
1459
|
+
**kwargs: ta.Any,
|
1460
|
+
) -> ta.Optional[bytes]:
|
1461
|
+
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
1462
|
+
return None
|
1463
|
+
else:
|
1464
|
+
return ret
|
1465
|
+
|
1466
|
+
async def try_output_str(
|
1467
|
+
self,
|
1468
|
+
*cmd: str,
|
1469
|
+
**kwargs: ta.Any,
|
1470
|
+
) -> ta.Optional[str]:
|
1471
|
+
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
1472
|
+
return None
|
1473
|
+
else:
|
1474
|
+
return ret.decode().strip()
|
1475
|
+
|
1476
|
+
|
1477
|
+
########################################
|
1478
|
+
# ../compose.py
|
1479
|
+
"""
|
1480
|
+
TODO:
|
1481
|
+
- fix rmi - only when not referenced anymore
|
1482
|
+
"""
|
1483
|
+
|
1484
|
+
|
1485
|
+
##
|
1486
|
+
|
1487
|
+
|
1488
|
+
def get_compose_service_dependencies(
|
1489
|
+
compose_file: str,
|
1490
|
+
service: str,
|
1491
|
+
) -> ta.Dict[str, str]:
|
1492
|
+
compose_dct = read_yaml_file(compose_file)
|
1493
|
+
|
1494
|
+
services = compose_dct['services']
|
1495
|
+
service_dct = services[service]
|
1496
|
+
|
1497
|
+
out = {}
|
1498
|
+
for dep_service in service_dct.get('depends_on', []):
|
1499
|
+
dep_service_dct = services[dep_service]
|
1500
|
+
out[dep_service] = dep_service_dct['image']
|
1501
|
+
|
1502
|
+
return out
|
1503
|
+
|
1504
|
+
|
1505
|
+
##
|
1506
|
+
|
1507
|
+
|
1508
|
+
class DockerComposeRun(ExitStacked):
|
1509
|
+
@dc.dataclass(frozen=True)
|
1510
|
+
class Config:
|
1511
|
+
compose_file: str
|
1512
|
+
service: str
|
1513
|
+
|
1514
|
+
image: str
|
1515
|
+
|
1516
|
+
run_cmd: ta.Sequence[str]
|
1517
|
+
|
1518
|
+
#
|
1519
|
+
|
1520
|
+
run_options: ta.Optional[ta.Sequence[str]] = None
|
1521
|
+
|
1522
|
+
cwd: ta.Optional[str] = None
|
1523
|
+
|
1524
|
+
#
|
1525
|
+
|
1526
|
+
def __post_init__(self) -> None:
|
1527
|
+
check.not_isinstance(self.run_cmd, str)
|
1528
|
+
|
1529
|
+
check.not_isinstance(self.run_options, str)
|
1530
|
+
|
1531
|
+
def __init__(self, cfg: Config) -> None:
|
1532
|
+
super().__init__()
|
1533
|
+
|
1534
|
+
self._cfg = cfg
|
1535
|
+
|
1536
|
+
self._subprocess_kwargs = {
|
1537
|
+
**(dict(cwd=self._cfg.cwd) if self._cfg.cwd is not None else {}),
|
1538
|
+
}
|
1539
|
+
|
1540
|
+
#
|
1541
|
+
|
1542
|
+
@property
|
1543
|
+
def image_tag(self) -> str:
|
1544
|
+
pfx = 'sha256:'
|
1545
|
+
if (image := self._cfg.image).startswith(pfx):
|
1546
|
+
image = image[len(pfx):]
|
1547
|
+
|
1548
|
+
return f'{self._cfg.service}:{image}'
|
1549
|
+
|
1550
|
+
@cached_nullary
|
1551
|
+
def tag_image(self) -> str:
|
1552
|
+
image_tag = self.image_tag
|
1553
|
+
|
1554
|
+
subprocesses.check_call(
|
1555
|
+
'docker',
|
1556
|
+
'tag',
|
1557
|
+
self._cfg.image,
|
1558
|
+
image_tag,
|
1559
|
+
**self._subprocess_kwargs,
|
1560
|
+
)
|
1561
|
+
|
1562
|
+
def delete_tag() -> None:
|
1563
|
+
subprocesses.check_call(
|
1564
|
+
'docker',
|
1565
|
+
'rmi',
|
1566
|
+
image_tag,
|
1567
|
+
**self._subprocess_kwargs,
|
1568
|
+
)
|
1569
|
+
|
1570
|
+
self._enter_context(defer(delete_tag)) # noqa
|
1571
|
+
|
1572
|
+
return image_tag
|
1573
|
+
|
1574
|
+
#
|
1575
|
+
|
1576
|
+
def _rewrite_compose_dct(self, in_dct: ta.Dict[str, ta.Any]) -> ta.Dict[str, ta.Any]:
|
1577
|
+
out = dict(in_dct)
|
1578
|
+
|
1579
|
+
#
|
1580
|
+
|
1581
|
+
in_services = in_dct['services']
|
1582
|
+
out['services'] = out_services = {}
|
1583
|
+
|
1584
|
+
#
|
1585
|
+
|
1586
|
+
in_service: dict = in_services[self._cfg.service]
|
1587
|
+
out_services[self._cfg.service] = out_service = dict(in_service)
|
1588
|
+
|
1589
|
+
out_service['image'] = self.image_tag
|
1590
|
+
|
1591
|
+
for k in ['build', 'platform']:
|
1592
|
+
if k in out_service:
|
1593
|
+
del out_service[k]
|
1594
|
+
|
1595
|
+
out_service['links'] = [
|
1596
|
+
f'{l}:{l}' if ':' not in l else l
|
1597
|
+
for l in out_service.get('links', [])
|
1598
|
+
]
|
1599
|
+
|
1600
|
+
#
|
1601
|
+
|
1602
|
+
depends_on = in_service.get('depends_on', [])
|
1603
|
+
|
1604
|
+
for dep_service, in_dep_service_dct in list(in_services.items()):
|
1605
|
+
if dep_service not in depends_on:
|
1606
|
+
continue
|
1607
|
+
|
1608
|
+
out_dep_service: dict = dict(in_dep_service_dct)
|
1609
|
+
out_services[dep_service] = out_dep_service
|
1610
|
+
|
1611
|
+
out_dep_service['ports'] = []
|
1612
|
+
|
1613
|
+
#
|
1614
|
+
|
1615
|
+
return out
|
1616
|
+
|
1617
|
+
@cached_nullary
|
1618
|
+
def rewrite_compose_file(self) -> str:
|
1619
|
+
in_dct = read_yaml_file(self._cfg.compose_file)
|
1620
|
+
|
1621
|
+
out_dct = self._rewrite_compose_dct(in_dct)
|
1622
|
+
|
1623
|
+
#
|
1624
|
+
|
1625
|
+
out_compose_file = make_temp_file()
|
1626
|
+
self._enter_context(defer(lambda: os.unlink(out_compose_file))) # noqa
|
1627
|
+
|
1628
|
+
compose_json = json_dumps_pretty(out_dct)
|
1629
|
+
|
1630
|
+
with open(out_compose_file, 'w') as f:
|
1631
|
+
f.write(compose_json)
|
1632
|
+
|
1633
|
+
return out_compose_file
|
1634
|
+
|
1635
|
+
#
|
1636
|
+
|
1637
|
+
def run(self) -> None:
|
1638
|
+
self.tag_image()
|
1639
|
+
|
1640
|
+
compose_file = self.rewrite_compose_file()
|
1641
|
+
|
1642
|
+
try:
|
1643
|
+
subprocesses.check_call(
|
1644
|
+
'docker',
|
1645
|
+
'compose',
|
1646
|
+
'-f', compose_file,
|
1647
|
+
'run',
|
1648
|
+
'--rm',
|
1649
|
+
*self._cfg.run_options or [],
|
1650
|
+
self._cfg.service,
|
1651
|
+
*self._cfg.run_cmd,
|
1652
|
+
**self._subprocess_kwargs,
|
1653
|
+
)
|
1654
|
+
|
1655
|
+
finally:
|
1656
|
+
subprocesses.check_call(
|
1657
|
+
'docker',
|
1658
|
+
'compose',
|
1659
|
+
'-f', compose_file,
|
1660
|
+
'down',
|
1661
|
+
)
|
1662
|
+
|
1663
|
+
|
1664
|
+
########################################
|
1665
|
+
# ../dockertars.py
|
1666
|
+
"""
|
1667
|
+
TODO:
|
1668
|
+
- some less stupid Dockerfile hash
|
1669
|
+
- doesn't change too much though
|
1670
|
+
"""
|
1671
|
+
|
1672
|
+
|
1673
|
+
##
|
1674
|
+
|
1675
|
+
|
1676
|
+
def build_docker_file_hash(docker_file: str) -> str:
|
1677
|
+
with open(docker_file) as f:
|
1678
|
+
contents = f.read()
|
1679
|
+
|
1680
|
+
return sha256_str(contents)
|
1681
|
+
|
1682
|
+
|
1683
|
+
##
|
1684
|
+
|
1685
|
+
|
1686
|
+
def read_docker_tar_image_tag(tar_file: str) -> str:
|
1687
|
+
with tarfile.open(tar_file) as tf:
|
1688
|
+
with contextlib.closing(check.not_none(tf.extractfile('manifest.json'))) as mf:
|
1689
|
+
m = mf.read()
|
1690
|
+
|
1691
|
+
manifests = json.loads(m.decode('utf-8'))
|
1692
|
+
manifest = check.single(manifests)
|
1693
|
+
tag = check.non_empty_str(check.single(manifest['RepoTags']))
|
1694
|
+
return tag
|
1695
|
+
|
1696
|
+
|
1697
|
+
def read_docker_tar_image_id(tar_file: str) -> str:
|
1698
|
+
with tarfile.open(tar_file) as tf:
|
1699
|
+
with contextlib.closing(check.not_none(tf.extractfile('index.json'))) as mf:
|
1700
|
+
i = mf.read()
|
1701
|
+
|
1702
|
+
index = json.loads(i.decode('utf-8'))
|
1703
|
+
manifest = check.single(index['manifests'])
|
1704
|
+
image_id = check.non_empty_str(manifest['digest'])
|
1705
|
+
return image_id
|
1706
|
+
|
1707
|
+
|
1708
|
+
##
|
1709
|
+
|
1710
|
+
|
1711
|
+
def is_docker_image_present(image: str) -> bool:
|
1712
|
+
out = subprocesses.check_output(
|
1713
|
+
'docker',
|
1714
|
+
'images',
|
1715
|
+
'--format', 'json',
|
1716
|
+
image,
|
1717
|
+
)
|
1718
|
+
|
1719
|
+
out_s = out.decode('utf-8').strip()
|
1720
|
+
if not out_s:
|
1721
|
+
return False
|
1722
|
+
|
1723
|
+
json.loads(out_s) # noqa
|
1724
|
+
return True
|
1725
|
+
|
1726
|
+
|
1727
|
+
##
|
1728
|
+
|
1729
|
+
|
1730
|
+
def pull_docker_tar(
|
1731
|
+
image: str,
|
1732
|
+
tar_file: str,
|
1733
|
+
) -> None:
|
1734
|
+
subprocesses.check_call(
|
1735
|
+
'docker',
|
1736
|
+
'pull',
|
1737
|
+
image,
|
1738
|
+
)
|
1739
|
+
|
1740
|
+
subprocesses.check_call(
|
1741
|
+
'docker',
|
1742
|
+
'save',
|
1743
|
+
image,
|
1744
|
+
'-o', tar_file,
|
1745
|
+
)
|
1746
|
+
|
1747
|
+
|
1748
|
+
def build_docker_tar(
|
1749
|
+
docker_file: str,
|
1750
|
+
tar_file: str,
|
1751
|
+
*,
|
1752
|
+
cwd: ta.Optional[str] = None,
|
1753
|
+
) -> str:
|
1754
|
+
id_file = make_temp_file()
|
1755
|
+
with defer(lambda: os.unlink(id_file)):
|
1756
|
+
subprocesses.check_call(
|
1757
|
+
'docker',
|
1758
|
+
'build',
|
1759
|
+
'-f', os.path.abspath(docker_file),
|
1760
|
+
'--iidfile', id_file,
|
1761
|
+
'--squash',
|
1762
|
+
'.',
|
1763
|
+
**(dict(cwd=cwd) if cwd is not None else {}),
|
1764
|
+
)
|
1765
|
+
|
1766
|
+
with open(id_file) as f:
|
1767
|
+
image_id = check.single(f.read().strip().splitlines()).strip()
|
1768
|
+
|
1769
|
+
subprocesses.check_call(
|
1770
|
+
'docker',
|
1771
|
+
'save',
|
1772
|
+
image_id,
|
1773
|
+
'-o', tar_file,
|
1774
|
+
)
|
1775
|
+
|
1776
|
+
return image_id
|
1777
|
+
|
1778
|
+
|
1779
|
+
##
|
1780
|
+
|
1781
|
+
|
1782
|
+
def load_docker_tar(
|
1783
|
+
tar_file: str,
|
1784
|
+
) -> None:
|
1785
|
+
subprocesses.check_call(
|
1786
|
+
'docker',
|
1787
|
+
'load',
|
1788
|
+
'-i', tar_file,
|
1789
|
+
)
|
1790
|
+
|
1791
|
+
|
1792
|
+
########################################
|
1793
|
+
# ../requirements.py
|
1794
|
+
"""
|
1795
|
+
TODO:
|
1796
|
+
- pip compile lol
|
1797
|
+
- but still support git+ stuff
|
1798
|
+
- req.txt format aware hash
|
1799
|
+
- more than just whitespace
|
1800
|
+
- pyproject req rewriting
|
1801
|
+
- download_requirements bootstrap off prev? not worth the dl?
|
1802
|
+
- big deps (torch) change less, probably worth it
|
1803
|
+
- follow embedded -r automatically like pyp
|
1804
|
+
"""
|
1805
|
+
|
1806
|
+
|
1807
|
+
##
|
1808
|
+
|
1809
|
+
|
1810
|
+
def build_requirements_hash(
|
1811
|
+
requirements_txts: ta.Sequence[str],
|
1812
|
+
) -> str:
|
1813
|
+
txt_file_contents: dict = {}
|
1814
|
+
|
1815
|
+
for txt_file in requirements_txts:
|
1816
|
+
txt_file_name = os.path.basename(txt_file)
|
1817
|
+
check.not_in(txt_file_name, txt_file_contents)
|
1818
|
+
with open(txt_file) as f:
|
1819
|
+
txt_contents = f.read()
|
1820
|
+
txt_file_contents[txt_file_name] = txt_contents
|
1821
|
+
|
1822
|
+
#
|
1823
|
+
|
1824
|
+
lines = []
|
1825
|
+
for txt_file, txt_contents in sorted(txt_file_contents.items()):
|
1826
|
+
txt_hash = sha256_str(txt_contents)
|
1827
|
+
lines.append(f'{txt_file}={txt_hash}')
|
1828
|
+
|
1829
|
+
return sha256_str('\n'.join(lines))
|
1830
|
+
|
1831
|
+
|
1832
|
+
##
|
1833
|
+
|
1834
|
+
|
1835
|
+
def download_requirements(
|
1836
|
+
image: str,
|
1837
|
+
requirements_dir: str,
|
1838
|
+
requirements_txts: ta.Sequence[str],
|
1839
|
+
) -> None:
|
1840
|
+
requirements_txt_dir = tempfile.mkdtemp()
|
1841
|
+
with defer(lambda: shutil.rmtree(requirements_txt_dir)):
|
1842
|
+
for rt in requirements_txts:
|
1843
|
+
shutil.copyfile(rt, os.path.join(requirements_txt_dir, os.path.basename(rt)))
|
1844
|
+
|
1845
|
+
subprocesses.check_call(
|
1846
|
+
'docker',
|
1847
|
+
'run',
|
1848
|
+
'-i',
|
1849
|
+
'-v', f'{os.path.abspath(requirements_dir)}:/requirements',
|
1850
|
+
'-v', f'{requirements_txt_dir}:/requirements_txt',
|
1851
|
+
image,
|
1852
|
+
'pip',
|
1853
|
+
'download',
|
1854
|
+
'-d', '/requirements',
|
1855
|
+
*itertools.chain.from_iterable([
|
1856
|
+
['-r', f'/requirements_txt/{os.path.basename(rt)}']
|
1857
|
+
for rt in requirements_txts
|
1858
|
+
]),
|
1859
|
+
)
|
1860
|
+
|
1861
|
+
|
1862
|
+
########################################
|
1863
|
+
# ../ci.py
|
1864
|
+
|
1865
|
+
|
1866
|
+
##
|
1867
|
+
|
1868
|
+
|
1869
|
+
class Ci(ExitStacked):
|
1870
|
+
FILE_NAME_HASH_LEN = 16
|
1871
|
+
|
1872
|
+
@dc.dataclass(frozen=True)
|
1873
|
+
class Config:
|
1874
|
+
project_dir: str
|
1875
|
+
|
1876
|
+
docker_file: str
|
1877
|
+
|
1878
|
+
compose_file: str
|
1879
|
+
service: str
|
1880
|
+
|
1881
|
+
requirements_txts: ta.Optional[ta.Sequence[str]] = None
|
1882
|
+
|
1883
|
+
def __post_init__(self) -> None:
|
1884
|
+
check.not_isinstance(self.requirements_txts, str)
|
1885
|
+
|
1886
|
+
def __init__(
|
1887
|
+
self,
|
1888
|
+
cfg: Config,
|
1889
|
+
*,
|
1890
|
+
file_cache: ta.Optional[FileCache] = None,
|
1891
|
+
) -> None:
|
1892
|
+
super().__init__()
|
1893
|
+
|
1894
|
+
self._cfg = cfg
|
1895
|
+
self._file_cache = file_cache
|
1896
|
+
|
1897
|
+
#
|
1898
|
+
|
1899
|
+
def load_docker_image(self, image: str) -> None:
|
1900
|
+
if is_docker_image_present(image):
|
1901
|
+
return
|
1902
|
+
|
1903
|
+
dep_suffix = image
|
1904
|
+
for c in '/:.-_':
|
1905
|
+
dep_suffix = dep_suffix.replace(c, '-')
|
1906
|
+
|
1907
|
+
tar_file_name = f'docker-{dep_suffix}.tar'
|
1908
|
+
|
1909
|
+
if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
|
1910
|
+
load_docker_tar(cache_tar_file)
|
1911
|
+
return
|
1912
|
+
|
1913
|
+
temp_dir = tempfile.mkdtemp()
|
1914
|
+
with defer(lambda: shutil.rmtree(temp_dir)):
|
1915
|
+
temp_tar_file = os.path.join(temp_dir, tar_file_name)
|
1916
|
+
|
1917
|
+
pull_docker_tar(
|
1918
|
+
image,
|
1919
|
+
temp_tar_file,
|
1920
|
+
)
|
1921
|
+
|
1922
|
+
if self._file_cache is not None:
|
1923
|
+
self._file_cache.put_file(temp_tar_file)
|
1924
|
+
|
1925
|
+
@cached_nullary
|
1926
|
+
def load_compose_service_dependencies(self) -> None:
|
1927
|
+
deps = get_compose_service_dependencies(
|
1928
|
+
self._cfg.compose_file,
|
1929
|
+
self._cfg.service,
|
1930
|
+
)
|
1931
|
+
|
1932
|
+
for dep_image in deps.values():
|
1933
|
+
self.load_docker_image(dep_image)
|
1934
|
+
|
1935
|
+
#
|
1936
|
+
|
1937
|
+
@cached_nullary
|
1938
|
+
def build_ci_image(self) -> str:
|
1939
|
+
docker_file_hash = build_docker_file_hash(self._cfg.docker_file)[:self.FILE_NAME_HASH_LEN]
|
1940
|
+
|
1941
|
+
tar_file_name = f'ci-{docker_file_hash}.tar'
|
1942
|
+
|
1943
|
+
if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
|
1944
|
+
image_id = read_docker_tar_image_id(cache_tar_file)
|
1945
|
+
load_docker_tar(cache_tar_file)
|
1946
|
+
return image_id
|
1947
|
+
|
1948
|
+
temp_dir = tempfile.mkdtemp()
|
1949
|
+
with defer(lambda: shutil.rmtree(temp_dir)):
|
1950
|
+
temp_tar_file = os.path.join(temp_dir, tar_file_name)
|
1951
|
+
|
1952
|
+
image_id = build_docker_tar(
|
1953
|
+
self._cfg.docker_file,
|
1954
|
+
temp_tar_file,
|
1955
|
+
cwd=self._cfg.project_dir,
|
1956
|
+
)
|
1957
|
+
|
1958
|
+
if self._file_cache is not None:
|
1959
|
+
self._file_cache.put_file(temp_tar_file)
|
1960
|
+
|
1961
|
+
return image_id
|
1962
|
+
|
1963
|
+
#
|
1964
|
+
|
1965
|
+
@cached_nullary
|
1966
|
+
def build_requirements_dir(self) -> str:
|
1967
|
+
requirements_txts = check.not_none(self._cfg.requirements_txts)
|
1968
|
+
|
1969
|
+
requirements_hash = build_requirements_hash(requirements_txts)[:self.FILE_NAME_HASH_LEN]
|
1970
|
+
|
1971
|
+
tar_file_name = f'requirements-{requirements_hash}.tar'
|
1972
|
+
|
1973
|
+
temp_dir = tempfile.mkdtemp()
|
1974
|
+
self._enter_context(defer(lambda: shutil.rmtree(temp_dir))) # noqa
|
1975
|
+
|
1976
|
+
if self._file_cache is not None and (cache_tar_file := self._file_cache.get_file(tar_file_name)):
|
1977
|
+
with tarfile.open(cache_tar_file) as tar:
|
1978
|
+
tar.extractall(path=temp_dir) # noqa
|
1979
|
+
|
1980
|
+
return temp_dir
|
1981
|
+
|
1982
|
+
temp_requirements_dir = os.path.join(temp_dir, 'requirements')
|
1983
|
+
os.makedirs(temp_requirements_dir)
|
1984
|
+
|
1985
|
+
download_requirements(
|
1986
|
+
self.build_ci_image(),
|
1987
|
+
temp_requirements_dir,
|
1988
|
+
requirements_txts,
|
1989
|
+
)
|
1990
|
+
|
1991
|
+
if self._file_cache is not None:
|
1992
|
+
temp_tar_file = os.path.join(temp_dir, tar_file_name)
|
1993
|
+
|
1994
|
+
with tarfile.open(temp_tar_file, 'w') as tar:
|
1995
|
+
for requirement_file in os.listdir(temp_requirements_dir):
|
1996
|
+
tar.add(
|
1997
|
+
os.path.join(temp_requirements_dir, requirement_file),
|
1998
|
+
arcname=requirement_file,
|
1999
|
+
)
|
2000
|
+
|
2001
|
+
self._file_cache.put_file(temp_tar_file)
|
2002
|
+
|
2003
|
+
return temp_requirements_dir
|
2004
|
+
|
2005
|
+
#
|
2006
|
+
|
2007
|
+
def run(self) -> None:
|
2008
|
+
self.load_compose_service_dependencies()
|
2009
|
+
|
2010
|
+
ci_image = self.build_ci_image()
|
2011
|
+
|
2012
|
+
requirements_dir = self.build_requirements_dir()
|
2013
|
+
|
2014
|
+
#
|
2015
|
+
|
2016
|
+
setup_cmds = [
|
2017
|
+
'pip install --root-user-action ignore --find-links /requirements --no-index uv',
|
2018
|
+
(
|
2019
|
+
'uv pip install --system --find-links /requirements ' +
|
2020
|
+
' '.join(f'-r /project/{rf}' for rf in self._cfg.requirements_txts or [])
|
2021
|
+
),
|
2022
|
+
]
|
2023
|
+
|
2024
|
+
#
|
2025
|
+
|
2026
|
+
test_cmds = [
|
2027
|
+
'(cd /project && python3 -m pytest -svv test.py)',
|
2028
|
+
]
|
2029
|
+
|
2030
|
+
#
|
2031
|
+
|
2032
|
+
bash_src = ' && '.join([
|
2033
|
+
*setup_cmds,
|
2034
|
+
*test_cmds,
|
2035
|
+
])
|
2036
|
+
|
2037
|
+
with DockerComposeRun(DockerComposeRun.Config(
|
2038
|
+
compose_file=self._cfg.compose_file,
|
2039
|
+
service=self._cfg.service,
|
2040
|
+
|
2041
|
+
image=ci_image,
|
2042
|
+
|
2043
|
+
run_cmd=['bash', '-c', bash_src],
|
2044
|
+
|
2045
|
+
run_options=[
|
2046
|
+
'-v', f'{os.path.abspath(self._cfg.project_dir)}:/project',
|
2047
|
+
'-v', f'{os.path.abspath(requirements_dir)}:/requirements',
|
2048
|
+
],
|
2049
|
+
|
2050
|
+
cwd=self._cfg.project_dir,
|
2051
|
+
)) as ci_compose_run:
|
2052
|
+
ci_compose_run.run()
|
2053
|
+
|
2054
|
+
|
2055
|
+
########################################
|
2056
|
+
# cli.py
|
2057
|
+
|
2058
|
+
|
2059
|
+
##
|
2060
|
+
|
2061
|
+
|
2062
|
+
class CiCli(ArgparseCli):
|
2063
|
+
#
|
2064
|
+
|
2065
|
+
@argparse_cmd(
|
2066
|
+
argparse_arg('requirements-txt', nargs='+'),
|
2067
|
+
)
|
2068
|
+
def print_requirements_hash(self) -> None:
|
2069
|
+
requirements_txts = self.args.requirements_txt
|
2070
|
+
|
2071
|
+
print(build_requirements_hash(requirements_txts))
|
2072
|
+
|
2073
|
+
#
|
2074
|
+
|
2075
|
+
@argparse_cmd(
|
2076
|
+
argparse_arg('compose-file'),
|
2077
|
+
argparse_arg('service'),
|
2078
|
+
)
|
2079
|
+
def dump_compose_deps(self) -> None:
|
2080
|
+
compose_file = self.args.compose_file
|
2081
|
+
service = self.args.service
|
2082
|
+
|
2083
|
+
print(get_compose_service_dependencies(
|
2084
|
+
compose_file,
|
2085
|
+
service,
|
2086
|
+
))
|
2087
|
+
|
2088
|
+
#
|
2089
|
+
|
2090
|
+
@argparse_cmd(
|
2091
|
+
argparse_arg('project-dir'),
|
2092
|
+
argparse_arg('service'),
|
2093
|
+
argparse_arg('--docker-file'),
|
2094
|
+
argparse_arg('--compose-file'),
|
2095
|
+
argparse_arg('-r', '--requirements-txt', action='append'),
|
2096
|
+
argparse_arg('--cache-dir'),
|
2097
|
+
)
|
2098
|
+
async def run(self) -> None:
|
2099
|
+
await asyncio.sleep(1)
|
2100
|
+
|
2101
|
+
project_dir = self.args.project_dir
|
2102
|
+
docker_file = self.args.docker_file
|
2103
|
+
compose_file = self.args.compose_file
|
2104
|
+
service = self.args.service
|
2105
|
+
requirements_txts = self.args.requirements_txt
|
2106
|
+
cache_dir = self.args.cache_dir
|
2107
|
+
|
2108
|
+
#
|
2109
|
+
|
2110
|
+
check.state(os.path.isdir(project_dir))
|
2111
|
+
|
2112
|
+
#
|
2113
|
+
|
2114
|
+
def find_alt_file(*alts: str) -> ta.Optional[str]:
|
2115
|
+
for alt in alts:
|
2116
|
+
alt_file = os.path.join(project_dir, alt)
|
2117
|
+
if os.path.isfile(alt_file):
|
2118
|
+
return alt_file
|
2119
|
+
return None
|
2120
|
+
|
2121
|
+
if docker_file is None:
|
2122
|
+
docker_file = find_alt_file(
|
2123
|
+
'docker/ci/Dockerfile',
|
2124
|
+
'docker/ci.Dockerfile',
|
2125
|
+
'ci.Dockerfile',
|
2126
|
+
'Dockerfile',
|
2127
|
+
)
|
2128
|
+
check.state(os.path.isfile(docker_file))
|
2129
|
+
|
2130
|
+
if compose_file is None:
|
2131
|
+
compose_file = find_alt_file(
|
2132
|
+
'docker/compose.yml',
|
2133
|
+
'compose.yml',
|
2134
|
+
)
|
2135
|
+
check.state(os.path.isfile(compose_file))
|
2136
|
+
|
2137
|
+
if not requirements_txts:
|
2138
|
+
requirements_txts = []
|
2139
|
+
for rf in [
|
2140
|
+
'requirements.txt',
|
2141
|
+
'requirements-dev.txt',
|
2142
|
+
'requirements-ci.txt',
|
2143
|
+
]:
|
2144
|
+
if os.path.exists(os.path.join(project_dir, rf)):
|
2145
|
+
requirements_txts.append(rf)
|
2146
|
+
else:
|
2147
|
+
for rf in requirements_txts:
|
2148
|
+
check.state(os.path.isfile(rf))
|
2149
|
+
|
2150
|
+
#
|
2151
|
+
|
2152
|
+
file_cache: ta.Optional[FileCache] = None
|
2153
|
+
if cache_dir is not None:
|
2154
|
+
if not os.path.exists(cache_dir):
|
2155
|
+
os.makedirs(cache_dir)
|
2156
|
+
check.state(os.path.isdir(cache_dir))
|
2157
|
+
file_cache = DirectoryFileCache(cache_dir)
|
2158
|
+
|
2159
|
+
#
|
2160
|
+
|
2161
|
+
with Ci(
|
2162
|
+
Ci.Config(
|
2163
|
+
project_dir=project_dir,
|
2164
|
+
docker_file=docker_file,
|
2165
|
+
compose_file=compose_file,
|
2166
|
+
service=service,
|
2167
|
+
requirements_txts=requirements_txts,
|
2168
|
+
),
|
2169
|
+
file_cache=file_cache,
|
2170
|
+
) as ci:
|
2171
|
+
ci.run()
|
2172
|
+
|
2173
|
+
|
2174
|
+
async def _async_main() -> ta.Optional[int]:
|
2175
|
+
return await CiCli().async_cli_run()
|
2176
|
+
|
2177
|
+
|
2178
|
+
def _main() -> None:
|
2179
|
+
sys.exit(rc if isinstance(rc := asyncio.run(_async_main()), int) else 0)
|
2180
|
+
|
2181
|
+
|
2182
|
+
if __name__ == '__main__':
|
2183
|
+
_main()
|