ominfra 0.0.0.dev158__py3-none-any.whl → 0.0.0.dev160__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/manage/deploy/apps.py +13 -11
- ominfra/manage/deploy/atomics.py +207 -0
- ominfra/manage/deploy/commands.py +10 -1
- ominfra/manage/deploy/git.py +32 -29
- ominfra/manage/deploy/inject.py +11 -0
- ominfra/manage/deploy/paths.py +46 -37
- ominfra/manage/deploy/specs.py +26 -2
- ominfra/manage/deploy/tmp.py +46 -0
- ominfra/manage/deploy/types.py +1 -0
- ominfra/manage/deploy/venvs.py +6 -1
- ominfra/scripts/journald2aws.py +32 -18
- ominfra/scripts/manage.py +417 -109
- ominfra/scripts/supervisor.py +32 -18
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/RECORD +19 -17
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -24,6 +24,7 @@ import decimal
|
|
24
24
|
import enum
|
25
25
|
import fractions
|
26
26
|
import functools
|
27
|
+
import hashlib
|
27
28
|
import inspect
|
28
29
|
import itertools
|
29
30
|
import json
|
@@ -41,6 +42,7 @@ import string
|
|
41
42
|
import struct
|
42
43
|
import subprocess
|
43
44
|
import sys
|
45
|
+
import tempfile
|
44
46
|
import threading
|
45
47
|
import time
|
46
48
|
import traceback
|
@@ -98,6 +100,10 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
|
|
98
100
|
CommandT = ta.TypeVar('CommandT', bound='Command')
|
99
101
|
CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
|
100
102
|
|
103
|
+
# deploy/atomics.py
|
104
|
+
DeployAtomicPathSwapKind = ta.Literal['dir', 'file']
|
105
|
+
DeployAtomicPathSwapState = ta.Literal['open', 'committed', 'aborted'] # ta.TypeAlias
|
106
|
+
|
101
107
|
# deploy/paths.py
|
102
108
|
DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
|
103
109
|
DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
|
@@ -1382,6 +1388,7 @@ DeployHome = ta.NewType('DeployHome', str)
|
|
1382
1388
|
DeployApp = ta.NewType('DeployApp', str)
|
1383
1389
|
DeployTag = ta.NewType('DeployTag', str)
|
1384
1390
|
DeployRev = ta.NewType('DeployRev', str)
|
1391
|
+
DeployKey = ta.NewType('DeployKey', str)
|
1385
1392
|
|
1386
1393
|
|
1387
1394
|
class DeployAppTag(ta.NamedTuple):
|
@@ -2685,6 +2692,10 @@ def is_new_type(spec: ta.Any) -> bool:
|
|
2685
2692
|
return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1] # type: ignore # noqa
|
2686
2693
|
|
2687
2694
|
|
2695
|
+
def get_new_type_supertype(spec: ta.Any) -> ta.Any:
|
2696
|
+
return spec.__supertype__
|
2697
|
+
|
2698
|
+
|
2688
2699
|
def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
|
2689
2700
|
seen = set()
|
2690
2701
|
todo = list(reversed(cls.__subclasses__()))
|
@@ -4054,41 +4065,212 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
|
4054
4065
|
|
4055
4066
|
|
4056
4067
|
########################################
|
4057
|
-
# ../deploy/
|
4058
|
-
|
4059
|
-
|
4060
|
-
|
4061
|
-
|
4062
|
-
|
4063
|
-
|
4064
|
-
|
4065
|
-
|
4066
|
-
|
4067
|
-
|
4068
|
-
|
4069
|
-
|
4070
|
-
|
4071
|
-
|
4072
|
-
|
4073
|
-
|
4074
|
-
|
4075
|
-
|
4076
|
-
|
4077
|
-
|
4078
|
-
|
4079
|
-
|
4080
|
-
|
4081
|
-
|
4082
|
-
|
4083
|
-
|
4084
|
-
|
4085
|
-
|
4086
|
-
|
4087
|
-
|
4088
|
-
|
4068
|
+
# ../deploy/atomics.py
|
4069
|
+
|
4070
|
+
|
4071
|
+
##
|
4072
|
+
|
4073
|
+
|
4074
|
+
class DeployAtomicPathSwap(abc.ABC):
|
4075
|
+
def __init__(
|
4076
|
+
self,
|
4077
|
+
kind: DeployAtomicPathSwapKind,
|
4078
|
+
dst_path: str,
|
4079
|
+
*,
|
4080
|
+
auto_commit: bool = False,
|
4081
|
+
) -> None:
|
4082
|
+
super().__init__()
|
4083
|
+
|
4084
|
+
self._kind = kind
|
4085
|
+
self._dst_path = dst_path
|
4086
|
+
self._auto_commit = auto_commit
|
4087
|
+
|
4088
|
+
self._state: DeployAtomicPathSwapState = 'open'
|
4089
|
+
|
4090
|
+
def __repr__(self) -> str:
|
4091
|
+
return attr_repr(self, 'kind', 'dst_path', 'tmp_path')
|
4092
|
+
|
4093
|
+
@property
|
4094
|
+
def kind(self) -> DeployAtomicPathSwapKind:
|
4095
|
+
return self._kind
|
4096
|
+
|
4097
|
+
@property
|
4098
|
+
def dst_path(self) -> str:
|
4099
|
+
return self._dst_path
|
4100
|
+
|
4101
|
+
@property
|
4102
|
+
@abc.abstractmethod
|
4103
|
+
def tmp_path(self) -> str:
|
4104
|
+
raise NotImplementedError
|
4105
|
+
|
4106
|
+
#
|
4107
|
+
|
4108
|
+
@property
|
4109
|
+
def state(self) -> DeployAtomicPathSwapState:
|
4110
|
+
return self._state
|
4111
|
+
|
4112
|
+
def _check_state(self, *states: DeployAtomicPathSwapState) -> None:
|
4113
|
+
if self._state not in states:
|
4114
|
+
raise RuntimeError(f'Atomic path swap not in correct state: {self._state}, {states}')
|
4115
|
+
|
4116
|
+
#
|
4117
|
+
|
4118
|
+
@abc.abstractmethod
|
4119
|
+
def _commit(self) -> None:
|
4120
|
+
raise NotImplementedError
|
4121
|
+
|
4122
|
+
def commit(self) -> None:
|
4123
|
+
if self._state == 'committed':
|
4124
|
+
return
|
4125
|
+
self._check_state('open')
|
4126
|
+
try:
|
4127
|
+
self._commit()
|
4128
|
+
except Exception: # noqa
|
4129
|
+
self._abort()
|
4130
|
+
raise
|
4131
|
+
else:
|
4132
|
+
self._state = 'committed'
|
4133
|
+
|
4134
|
+
#
|
4135
|
+
|
4136
|
+
@abc.abstractmethod
|
4137
|
+
def _abort(self) -> None:
|
4138
|
+
raise NotImplementedError
|
4139
|
+
|
4140
|
+
def abort(self) -> None:
|
4141
|
+
if self._state == 'aborted':
|
4142
|
+
return
|
4143
|
+
self._abort()
|
4144
|
+
self._state = 'aborted'
|
4145
|
+
|
4146
|
+
#
|
4147
|
+
|
4148
|
+
def __enter__(self) -> 'DeployAtomicPathSwap':
|
4149
|
+
return self
|
4150
|
+
|
4151
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
4152
|
+
if (
|
4153
|
+
exc_type is None and
|
4154
|
+
self._auto_commit and
|
4155
|
+
self._state == 'open'
|
4156
|
+
):
|
4157
|
+
self.commit()
|
4158
|
+
else:
|
4159
|
+
self.abort()
|
4160
|
+
|
4161
|
+
|
4162
|
+
#
|
4163
|
+
|
4164
|
+
|
4165
|
+
class DeployAtomicPathSwapping(abc.ABC):
|
4166
|
+
@abc.abstractmethod
|
4167
|
+
def begin_atomic_path_swap(
|
4168
|
+
self,
|
4169
|
+
kind: DeployAtomicPathSwapKind,
|
4170
|
+
dst_path: str,
|
4171
|
+
*,
|
4172
|
+
name_hint: ta.Optional[str] = None,
|
4173
|
+
make_dirs: bool = False,
|
4174
|
+
**kwargs: ta.Any,
|
4175
|
+
) -> DeployAtomicPathSwap:
|
4176
|
+
raise NotImplementedError
|
4177
|
+
|
4178
|
+
|
4179
|
+
##
|
4180
|
+
|
4181
|
+
|
4182
|
+
class OsRenameDeployAtomicPathSwap(DeployAtomicPathSwap):
|
4183
|
+
def __init__(
|
4184
|
+
self,
|
4185
|
+
kind: DeployAtomicPathSwapKind,
|
4186
|
+
dst_path: str,
|
4187
|
+
tmp_path: str,
|
4188
|
+
**kwargs: ta.Any,
|
4189
|
+
) -> None:
|
4190
|
+
if kind == 'dir':
|
4191
|
+
check.state(os.path.isdir(tmp_path))
|
4192
|
+
elif kind == 'file':
|
4193
|
+
check.state(os.path.isfile(tmp_path))
|
4194
|
+
else:
|
4195
|
+
raise TypeError(kind)
|
4196
|
+
|
4197
|
+
super().__init__(
|
4198
|
+
kind,
|
4199
|
+
dst_path,
|
4200
|
+
**kwargs,
|
4201
|
+
)
|
4202
|
+
|
4203
|
+
self._tmp_path = tmp_path
|
4204
|
+
|
4205
|
+
@property
|
4206
|
+
def tmp_path(self) -> str:
|
4207
|
+
return self._tmp_path
|
4208
|
+
|
4209
|
+
def _commit(self) -> None:
|
4210
|
+
os.rename(self._tmp_path, self._dst_path)
|
4211
|
+
|
4212
|
+
def _abort(self) -> None:
|
4213
|
+
shutil.rmtree(self._tmp_path, ignore_errors=True)
|
4214
|
+
|
4215
|
+
|
4216
|
+
class TempDirDeployAtomicPathSwapping(DeployAtomicPathSwapping):
|
4217
|
+
def __init__(
|
4218
|
+
self,
|
4219
|
+
*,
|
4220
|
+
temp_dir: ta.Optional[str] = None,
|
4221
|
+
root_dir: ta.Optional[str] = None,
|
4222
|
+
) -> None:
|
4223
|
+
super().__init__()
|
4224
|
+
|
4225
|
+
if root_dir is not None:
|
4226
|
+
root_dir = os.path.abspath(root_dir)
|
4227
|
+
self._root_dir = root_dir
|
4228
|
+
self._temp_dir = temp_dir
|
4229
|
+
|
4230
|
+
def begin_atomic_path_swap(
|
4231
|
+
self,
|
4232
|
+
kind: DeployAtomicPathSwapKind,
|
4233
|
+
dst_path: str,
|
4234
|
+
*,
|
4235
|
+
name_hint: ta.Optional[str] = None,
|
4236
|
+
make_dirs: bool = False,
|
4237
|
+
**kwargs: ta.Any,
|
4238
|
+
) -> DeployAtomicPathSwap:
|
4239
|
+
dst_path = os.path.abspath(dst_path)
|
4240
|
+
if self._root_dir is not None and not dst_path.startswith(check.non_empty_str(self._root_dir)):
|
4241
|
+
raise RuntimeError(f'Atomic path swap dst must be in root dir: {dst_path}, {self._root_dir}')
|
4242
|
+
|
4243
|
+
dst_dir = os.path.dirname(dst_path)
|
4244
|
+
if make_dirs:
|
4245
|
+
os.makedirs(dst_dir, exist_ok=True)
|
4246
|
+
if not os.path.isdir(dst_dir):
|
4247
|
+
raise RuntimeError(f'Atomic path swap dst dir does not exist: {dst_dir}')
|
4248
|
+
|
4249
|
+
if kind == 'dir':
|
4250
|
+
tmp_path = tempfile.mkdtemp(prefix=name_hint, dir=self._temp_dir)
|
4251
|
+
elif kind == 'file':
|
4252
|
+
fd, tmp_path = tempfile.mkstemp(prefix=name_hint, dir=self._temp_dir)
|
4253
|
+
os.close(fd)
|
4254
|
+
else:
|
4255
|
+
raise TypeError(kind)
|
4089
4256
|
|
4090
|
-
|
4257
|
+
return OsRenameDeployAtomicPathSwap(
|
4258
|
+
kind,
|
4259
|
+
dst_path,
|
4260
|
+
tmp_path,
|
4261
|
+
**kwargs,
|
4262
|
+
)
|
4091
4263
|
|
4264
|
+
|
4265
|
+
########################################
|
4266
|
+
# ../deploy/paths.py
|
4267
|
+
"""
|
4268
|
+
TODO:
|
4269
|
+
- run/pidfile
|
4270
|
+
- logs/...
|
4271
|
+
- current symlink
|
4272
|
+
- conf/{nginx,supervisor}
|
4273
|
+
- env/?
|
4092
4274
|
"""
|
4093
4275
|
|
4094
4276
|
|
@@ -4100,7 +4282,7 @@ DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
|
|
4100
4282
|
|
4101
4283
|
DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
|
4102
4284
|
'app',
|
4103
|
-
'tag',
|
4285
|
+
'tag',
|
4104
4286
|
])
|
4105
4287
|
|
4106
4288
|
|
@@ -4227,6 +4409,8 @@ class DeployPath:
|
|
4227
4409
|
parts: ta.Sequence[DeployPathPart]
|
4228
4410
|
|
4229
4411
|
def __post_init__(self) -> None:
|
4412
|
+
hash(self)
|
4413
|
+
|
4230
4414
|
check.not_empty(self.parts)
|
4231
4415
|
for p in self.parts[:-1]:
|
4232
4416
|
check.equal(p.kind, 'dir')
|
@@ -4261,10 +4445,10 @@ class DeployPath:
|
|
4261
4445
|
else:
|
4262
4446
|
tail_parse = FileDeployPathPart.parse
|
4263
4447
|
ps = check.non_empty_str(s).split('/')
|
4264
|
-
return cls(
|
4448
|
+
return cls((
|
4265
4449
|
*([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
|
4266
4450
|
tail_parse(ps[-1]),
|
4267
|
-
|
4451
|
+
))
|
4268
4452
|
|
4269
4453
|
|
4270
4454
|
##
|
@@ -4272,10 +4456,41 @@ class DeployPath:
|
|
4272
4456
|
|
4273
4457
|
class DeployPathOwner(abc.ABC):
|
4274
4458
|
@abc.abstractmethod
|
4275
|
-
def
|
4459
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
4276
4460
|
raise NotImplementedError
|
4277
4461
|
|
4278
4462
|
|
4463
|
+
class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
|
4464
|
+
def __init__(
|
4465
|
+
self,
|
4466
|
+
*args: ta.Any,
|
4467
|
+
owned_dir: str,
|
4468
|
+
deploy_home: ta.Optional[DeployHome],
|
4469
|
+
**kwargs: ta.Any,
|
4470
|
+
) -> None:
|
4471
|
+
super().__init__(*args, **kwargs)
|
4472
|
+
|
4473
|
+
check.not_in('/', owned_dir)
|
4474
|
+
self._owned_dir: str = check.non_empty_str(owned_dir)
|
4475
|
+
|
4476
|
+
self._deploy_home = deploy_home
|
4477
|
+
|
4478
|
+
self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
|
4479
|
+
|
4480
|
+
@cached_nullary
|
4481
|
+
def _dir(self) -> str:
|
4482
|
+
return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
|
4483
|
+
|
4484
|
+
@cached_nullary
|
4485
|
+
def _make_dir(self) -> str:
|
4486
|
+
if not os.path.isdir(d := self._dir()):
|
4487
|
+
os.makedirs(d, exist_ok=True)
|
4488
|
+
return d
|
4489
|
+
|
4490
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
4491
|
+
return self._owned_deploy_paths
|
4492
|
+
|
4493
|
+
|
4279
4494
|
########################################
|
4280
4495
|
# ../deploy/specs.py
|
4281
4496
|
|
@@ -4294,14 +4509,35 @@ class DeployGitRepo:
|
|
4294
4509
|
check.not_in('.', check.non_empty_str(self.path))
|
4295
4510
|
|
4296
4511
|
|
4512
|
+
@dc.dataclass(frozen=True)
|
4513
|
+
class DeployGitCheckout:
|
4514
|
+
repo: DeployGitRepo
|
4515
|
+
rev: DeployRev
|
4516
|
+
|
4517
|
+
subtrees: ta.Optional[ta.Sequence[str]] = None
|
4518
|
+
|
4519
|
+
def __post_init__(self) -> None:
|
4520
|
+
hash(self)
|
4521
|
+
check.non_empty_str(self.rev)
|
4522
|
+
if self.subtrees is not None:
|
4523
|
+
for st in self.subtrees:
|
4524
|
+
check.non_empty_str(st)
|
4525
|
+
|
4526
|
+
|
4297
4527
|
##
|
4298
4528
|
|
4299
4529
|
|
4300
4530
|
@dc.dataclass(frozen=True)
|
4301
4531
|
class DeploySpec:
|
4302
4532
|
app: DeployApp
|
4303
|
-
|
4304
|
-
|
4533
|
+
checkout: DeployGitCheckout
|
4534
|
+
|
4535
|
+
def __post_init__(self) -> None:
|
4536
|
+
hash(self)
|
4537
|
+
|
4538
|
+
@cached_nullary
|
4539
|
+
def key(self) -> DeployKey:
|
4540
|
+
return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
|
4305
4541
|
|
4306
4542
|
|
4307
4543
|
########################################
|
@@ -5761,9 +5997,7 @@ inj = Injection
|
|
5761
5997
|
"""
|
5762
5998
|
TODO:
|
5763
5999
|
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
5764
|
-
- namedtuple
|
5765
6000
|
- literals
|
5766
|
-
- newtypes?
|
5767
6001
|
"""
|
5768
6002
|
|
5769
6003
|
|
@@ -5773,7 +6007,7 @@ TODO:
|
|
5773
6007
|
@dc.dataclass(frozen=True)
|
5774
6008
|
class ObjMarshalOptions:
|
5775
6009
|
raw_bytes: bool = False
|
5776
|
-
|
6010
|
+
non_strict_fields: bool = False
|
5777
6011
|
|
5778
6012
|
|
5779
6013
|
class ObjMarshaler(abc.ABC):
|
@@ -5902,10 +6136,10 @@ class IterableObjMarshaler(ObjMarshaler):
|
|
5902
6136
|
|
5903
6137
|
|
5904
6138
|
@dc.dataclass(frozen=True)
|
5905
|
-
class
|
6139
|
+
class FieldsObjMarshaler(ObjMarshaler):
|
5906
6140
|
ty: type
|
5907
6141
|
fs: ta.Mapping[str, ObjMarshaler]
|
5908
|
-
|
6142
|
+
non_strict: bool = False
|
5909
6143
|
|
5910
6144
|
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5911
6145
|
return {
|
@@ -5917,7 +6151,7 @@ class DataclassObjMarshaler(ObjMarshaler):
|
|
5917
6151
|
return self.ty(**{
|
5918
6152
|
k: self.fs[k].unmarshal(v, ctx)
|
5919
6153
|
for k, v in o.items()
|
5920
|
-
if not (self.
|
6154
|
+
if not (self.non_strict or ctx.options.non_strict_fields) or k in self.fs
|
5921
6155
|
})
|
5922
6156
|
|
5923
6157
|
|
@@ -6049,7 +6283,7 @@ class ObjMarshalerManager:
|
|
6049
6283
|
ty: ta.Any,
|
6050
6284
|
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
6051
6285
|
*,
|
6052
|
-
|
6286
|
+
non_strict_fields: bool = False,
|
6053
6287
|
) -> ObjMarshaler:
|
6054
6288
|
if isinstance(ty, type):
|
6055
6289
|
if abc.ABC in ty.__bases__:
|
@@ -6071,12 +6305,22 @@ class ObjMarshalerManager:
|
|
6071
6305
|
return EnumObjMarshaler(ty)
|
6072
6306
|
|
6073
6307
|
if dc.is_dataclass(ty):
|
6074
|
-
return
|
6308
|
+
return FieldsObjMarshaler(
|
6075
6309
|
ty,
|
6076
6310
|
{f.name: rec(f.type) for f in dc.fields(ty)},
|
6077
|
-
|
6311
|
+
non_strict=non_strict_fields,
|
6078
6312
|
)
|
6079
6313
|
|
6314
|
+
if issubclass(ty, tuple) and hasattr(ty, '_fields'):
|
6315
|
+
return FieldsObjMarshaler(
|
6316
|
+
ty,
|
6317
|
+
{p.name: rec(p.annotation) for p in inspect.signature(ty).parameters.values()},
|
6318
|
+
non_strict=non_strict_fields,
|
6319
|
+
)
|
6320
|
+
|
6321
|
+
if is_new_type(ty):
|
6322
|
+
return rec(get_new_type_supertype(ty))
|
6323
|
+
|
6080
6324
|
if is_generic_alias(ty):
|
6081
6325
|
try:
|
6082
6326
|
mt = self._generic_mapping_types[ta.get_origin(ty)]
|
@@ -6210,12 +6454,12 @@ def is_debugger_attached() -> bool:
|
|
6210
6454
|
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
6211
6455
|
|
6212
6456
|
|
6213
|
-
|
6457
|
+
LITE_REQUIRED_PYTHON_VERSION = (3, 8)
|
6214
6458
|
|
6215
6459
|
|
6216
|
-
def
|
6217
|
-
if sys.version_info <
|
6218
|
-
raise OSError(f'Requires python {
|
6460
|
+
def check_lite_runtime_version() -> None:
|
6461
|
+
if sys.version_info < LITE_REQUIRED_PYTHON_VERSION:
|
6462
|
+
raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
|
6219
6463
|
|
6220
6464
|
|
6221
6465
|
########################################
|
@@ -6503,24 +6747,41 @@ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command
|
|
6503
6747
|
|
6504
6748
|
|
6505
6749
|
########################################
|
6506
|
-
# ../deploy/
|
6507
|
-
|
6508
|
-
|
6509
|
-
##
|
6750
|
+
# ../deploy/tmp.py
|
6510
6751
|
|
6511
6752
|
|
6512
|
-
|
6513
|
-
|
6514
|
-
|
6515
|
-
|
6516
|
-
|
6517
|
-
|
6753
|
+
class DeployTmpManager(
|
6754
|
+
SingleDirDeployPathOwner,
|
6755
|
+
DeployAtomicPathSwapping,
|
6756
|
+
):
|
6757
|
+
def __init__(
|
6758
|
+
self,
|
6759
|
+
*,
|
6760
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
6761
|
+
) -> None:
|
6762
|
+
super().__init__(
|
6763
|
+
owned_dir='tmp',
|
6764
|
+
deploy_home=deploy_home,
|
6765
|
+
)
|
6518
6766
|
|
6519
|
-
|
6520
|
-
|
6521
|
-
|
6767
|
+
@cached_nullary
|
6768
|
+
def _swapping(self) -> DeployAtomicPathSwapping:
|
6769
|
+
return TempDirDeployAtomicPathSwapping(
|
6770
|
+
temp_dir=self._make_dir(),
|
6771
|
+
root_dir=check.non_empty_str(self._deploy_home),
|
6772
|
+
)
|
6522
6773
|
|
6523
|
-
|
6774
|
+
def begin_atomic_path_swap(
|
6775
|
+
self,
|
6776
|
+
kind: DeployAtomicPathSwapKind,
|
6777
|
+
dst_path: str,
|
6778
|
+
**kwargs: ta.Any,
|
6779
|
+
) -> DeployAtomicPathSwap:
|
6780
|
+
return self._swapping().begin_atomic_path_swap(
|
6781
|
+
kind,
|
6782
|
+
dst_path,
|
6783
|
+
**kwargs,
|
6784
|
+
)
|
6524
6785
|
|
6525
6786
|
|
6526
6787
|
########################################
|
@@ -6630,6 +6891,7 @@ TODO:
|
|
6630
6891
|
- structured
|
6631
6892
|
- prefixed
|
6632
6893
|
- debug
|
6894
|
+
- optional noisy? noisy will never be lite - some kinda configure_standard callback mechanism?
|
6633
6895
|
"""
|
6634
6896
|
|
6635
6897
|
|
@@ -6666,8 +6928,9 @@ class StandardLogFormatter(logging.Formatter):
|
|
6666
6928
|
##
|
6667
6929
|
|
6668
6930
|
|
6669
|
-
class
|
6670
|
-
|
6931
|
+
class StandardConfiguredLogHandler(ProxyLogHandler):
|
6932
|
+
def __init_subclass__(cls, **kwargs):
|
6933
|
+
raise TypeError('This class serves only as a marker and should not be subclassed.')
|
6671
6934
|
|
6672
6935
|
|
6673
6936
|
##
|
@@ -6698,7 +6961,7 @@ def configure_standard_logging(
|
|
6698
6961
|
target: ta.Optional[logging.Logger] = None,
|
6699
6962
|
force: bool = False,
|
6700
6963
|
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
6701
|
-
) -> ta.Optional[
|
6964
|
+
) -> ta.Optional[StandardConfiguredLogHandler]:
|
6702
6965
|
with _locking_logging_module_lock():
|
6703
6966
|
if target is None:
|
6704
6967
|
target = logging.root
|
@@ -6706,7 +6969,7 @@ def configure_standard_logging(
|
|
6706
6969
|
#
|
6707
6970
|
|
6708
6971
|
if not force:
|
6709
|
-
if any(isinstance(h,
|
6972
|
+
if any(isinstance(h, StandardConfiguredLogHandler) for h in list(target.handlers)):
|
6710
6973
|
return None
|
6711
6974
|
|
6712
6975
|
#
|
@@ -6740,7 +7003,7 @@ def configure_standard_logging(
|
|
6740
7003
|
|
6741
7004
|
#
|
6742
7005
|
|
6743
|
-
return
|
7006
|
+
return StandardConfiguredLogHandler(handler)
|
6744
7007
|
|
6745
7008
|
|
6746
7009
|
########################################
|
@@ -7907,27 +8170,22 @@ github.com/wrmsr/omlish@rev
|
|
7907
8170
|
##
|
7908
8171
|
|
7909
8172
|
|
7910
|
-
class DeployGitManager(
|
8173
|
+
class DeployGitManager(SingleDirDeployPathOwner):
|
7911
8174
|
def __init__(
|
7912
8175
|
self,
|
7913
8176
|
*,
|
7914
8177
|
deploy_home: ta.Optional[DeployHome] = None,
|
8178
|
+
atomics: DeployAtomicPathSwapping,
|
7915
8179
|
) -> None:
|
7916
|
-
super().__init__(
|
8180
|
+
super().__init__(
|
8181
|
+
owned_dir='git',
|
8182
|
+
deploy_home=deploy_home,
|
8183
|
+
)
|
7917
8184
|
|
7918
|
-
self.
|
8185
|
+
self._atomics = atomics
|
7919
8186
|
|
7920
8187
|
self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
|
7921
8188
|
|
7922
|
-
@cached_nullary
|
7923
|
-
def _dir(self) -> str:
|
7924
|
-
return os.path.join(check.non_empty_str(self._deploy_home), 'git')
|
7925
|
-
|
7926
|
-
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
7927
|
-
return {
|
7928
|
-
DeployPath.parse('git'),
|
7929
|
-
}
|
7930
|
-
|
7931
8189
|
class RepoDir:
|
7932
8190
|
def __init__(
|
7933
8191
|
self,
|
@@ -7939,7 +8197,7 @@ class DeployGitManager(DeployPathOwner):
|
|
7939
8197
|
self._git = git
|
7940
8198
|
self._repo = repo
|
7941
8199
|
self._dir = os.path.join(
|
7942
|
-
self._git.
|
8200
|
+
self._git._make_dir(), # noqa
|
7943
8201
|
check.non_empty_str(repo.host),
|
7944
8202
|
check.non_empty_str(repo.path),
|
7945
8203
|
)
|
@@ -7955,12 +8213,16 @@ class DeployGitManager(DeployPathOwner):
|
|
7955
8213
|
else:
|
7956
8214
|
return f'https://{self._repo.host}/{self._repo.path}'
|
7957
8215
|
|
8216
|
+
#
|
8217
|
+
|
7958
8218
|
async def _call(self, *cmd: str) -> None:
|
7959
8219
|
await asyncio_subprocesses.check_call(
|
7960
8220
|
*cmd,
|
7961
8221
|
cwd=self._dir,
|
7962
8222
|
)
|
7963
8223
|
|
8224
|
+
#
|
8225
|
+
|
7964
8226
|
@async_cached_nullary
|
7965
8227
|
async def init(self) -> None:
|
7966
8228
|
os.makedirs(self._dir, exist_ok=True)
|
@@ -7974,20 +8236,24 @@ class DeployGitManager(DeployPathOwner):
|
|
7974
8236
|
await self.init()
|
7975
8237
|
await self._call('git', 'fetch', '--depth=1', 'origin', rev)
|
7976
8238
|
|
7977
|
-
|
7978
|
-
check.state(not os.path.exists(dst_dir))
|
7979
|
-
|
7980
|
-
await self.fetch(rev)
|
8239
|
+
#
|
7981
8240
|
|
7982
|
-
|
7983
|
-
os.
|
8241
|
+
async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
|
8242
|
+
check.state(not os.path.exists(dst_dir))
|
8243
|
+
with self._git._atomics.begin_atomic_path_swap( # noqa
|
8244
|
+
'dir',
|
8245
|
+
dst_dir,
|
8246
|
+
auto_commit=True,
|
8247
|
+
make_dirs=True,
|
8248
|
+
) as dst_swap:
|
8249
|
+
await self.fetch(checkout.rev)
|
7984
8250
|
|
7985
|
-
|
7986
|
-
|
8251
|
+
dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
|
8252
|
+
await dst_call('git', 'init')
|
7987
8253
|
|
7988
|
-
|
7989
|
-
|
7990
|
-
|
8254
|
+
await dst_call('git', 'remote', 'add', 'local', self._dir)
|
8255
|
+
await dst_call('git', 'fetch', '--depth=1', 'local', checkout.rev)
|
8256
|
+
await dst_call('git', 'checkout', checkout.rev, *(checkout.subtrees or []))
|
7991
8257
|
|
7992
8258
|
def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
|
7993
8259
|
try:
|
@@ -7996,8 +8262,8 @@ class DeployGitManager(DeployPathOwner):
|
|
7996
8262
|
repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
|
7997
8263
|
return repo_dir
|
7998
8264
|
|
7999
|
-
async def checkout(self,
|
8000
|
-
await self.get_repo_dir(repo).checkout(
|
8265
|
+
async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
|
8266
|
+
await self.get_repo_dir(checkout.repo).checkout(checkout, dst_dir)
|
8001
8267
|
|
8002
8268
|
|
8003
8269
|
########################################
|
@@ -8014,16 +8280,18 @@ class DeployVenvManager(DeployPathOwner):
|
|
8014
8280
|
self,
|
8015
8281
|
*,
|
8016
8282
|
deploy_home: ta.Optional[DeployHome] = None,
|
8283
|
+
atomics: DeployAtomicPathSwapping,
|
8017
8284
|
) -> None:
|
8018
8285
|
super().__init__()
|
8019
8286
|
|
8020
8287
|
self._deploy_home = deploy_home
|
8288
|
+
self._atomics = atomics
|
8021
8289
|
|
8022
8290
|
@cached_nullary
|
8023
8291
|
def _dir(self) -> str:
|
8024
8292
|
return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
|
8025
8293
|
|
8026
|
-
def
|
8294
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
8027
8295
|
return {
|
8028
8296
|
DeployPath.parse('venvs/@app/@tag/'),
|
8029
8297
|
}
|
@@ -8037,6 +8305,8 @@ class DeployVenvManager(DeployPathOwner):
|
|
8037
8305
|
) -> None:
|
8038
8306
|
sys_exe = 'python3'
|
8039
8307
|
|
8308
|
+
# !! NOTE: (most) venvs cannot be relocated, so an atomic swap can't be used. it's up to the path manager to
|
8309
|
+
# garbage collect orphaned dirs.
|
8040
8310
|
await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
|
8041
8311
|
|
8042
8312
|
#
|
@@ -8594,13 +8864,15 @@ def bind_commands(
|
|
8594
8864
|
|
8595
8865
|
def make_deploy_tag(
|
8596
8866
|
rev: DeployRev,
|
8597
|
-
|
8867
|
+
key: DeployKey,
|
8868
|
+
*,
|
8869
|
+
utcnow: ta.Optional[datetime.datetime] = None,
|
8598
8870
|
) -> DeployTag:
|
8599
|
-
if
|
8600
|
-
|
8601
|
-
now_fmt = '%Y%m%dT%H%M%
|
8602
|
-
now_str =
|
8603
|
-
return DeployTag('-'.join([now_str, rev]))
|
8871
|
+
if utcnow is None:
|
8872
|
+
utcnow = datetime.datetime.now(tz=datetime.timezone.utc) # noqa
|
8873
|
+
now_fmt = '%Y%m%dT%H%M%SZ'
|
8874
|
+
now_str = utcnow.strftime(now_fmt)
|
8875
|
+
return DeployTag('-'.join([now_str, rev, key]))
|
8604
8876
|
|
8605
8877
|
|
8606
8878
|
class DeployAppManager(DeployPathOwner):
|
@@ -8621,7 +8893,7 @@ class DeployAppManager(DeployPathOwner):
|
|
8621
8893
|
def _dir(self) -> str:
|
8622
8894
|
return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
|
8623
8895
|
|
8624
|
-
def
|
8896
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
8625
8897
|
return {
|
8626
8898
|
DeployPath.parse('apps/@app/@tag'),
|
8627
8899
|
}
|
@@ -8629,15 +8901,14 @@ class DeployAppManager(DeployPathOwner):
|
|
8629
8901
|
async def prepare_app(
|
8630
8902
|
self,
|
8631
8903
|
spec: DeploySpec,
|
8632
|
-
):
|
8633
|
-
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev))
|
8904
|
+
) -> None:
|
8905
|
+
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.checkout.rev, spec.key()))
|
8634
8906
|
app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
|
8635
8907
|
|
8636
8908
|
#
|
8637
8909
|
|
8638
8910
|
await self._git.checkout(
|
8639
|
-
spec.
|
8640
|
-
spec.rev,
|
8911
|
+
spec.checkout,
|
8641
8912
|
app_dir,
|
8642
8913
|
)
|
8643
8914
|
|
@@ -9360,6 +9631,34 @@ class SystemInterpProvider(InterpProvider):
|
|
9360
9631
|
raise KeyError(version)
|
9361
9632
|
|
9362
9633
|
|
9634
|
+
########################################
|
9635
|
+
# ../deploy/commands.py
|
9636
|
+
|
9637
|
+
|
9638
|
+
##
|
9639
|
+
|
9640
|
+
|
9641
|
+
@dc.dataclass(frozen=True)
|
9642
|
+
class DeployCommand(Command['DeployCommand.Output']):
|
9643
|
+
spec: DeploySpec
|
9644
|
+
|
9645
|
+
@dc.dataclass(frozen=True)
|
9646
|
+
class Output(Command.Output):
|
9647
|
+
pass
|
9648
|
+
|
9649
|
+
|
9650
|
+
@dc.dataclass(frozen=True)
|
9651
|
+
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
9652
|
+
_apps: DeployAppManager
|
9653
|
+
|
9654
|
+
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
9655
|
+
log.info('Deploying! %r', cmd.spec)
|
9656
|
+
|
9657
|
+
await self._apps.prepare_app(cmd.spec)
|
9658
|
+
|
9659
|
+
return DeployCommand.Output()
|
9660
|
+
|
9661
|
+
|
9363
9662
|
########################################
|
9364
9663
|
# ../remote/inject.py
|
9365
9664
|
|
@@ -9738,10 +10037,19 @@ def bind_deploy(
|
|
9738
10037
|
lst: ta.List[InjectorBindingOrBindings] = [
|
9739
10038
|
inj.bind(deploy_config),
|
9740
10039
|
|
10040
|
+
#
|
10041
|
+
|
9741
10042
|
inj.bind(DeployAppManager, singleton=True),
|
10043
|
+
|
9742
10044
|
inj.bind(DeployGitManager, singleton=True),
|
10045
|
+
|
10046
|
+
inj.bind(DeployTmpManager, singleton=True),
|
10047
|
+
inj.bind(DeployAtomicPathSwapping, to_key=DeployTmpManager),
|
10048
|
+
|
9743
10049
|
inj.bind(DeployVenvManager, singleton=True),
|
9744
10050
|
|
10051
|
+
#
|
10052
|
+
|
9745
10053
|
bind_command(DeployCommand, DeployCommandExecutor),
|
9746
10054
|
bind_command(InterpCommand, InterpCommandExecutor),
|
9747
10055
|
]
|