ominfra 0.0.0.dev158__py3-none-any.whl → 0.0.0.dev160__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/manage/deploy/apps.py +13 -11
- ominfra/manage/deploy/atomics.py +207 -0
- ominfra/manage/deploy/commands.py +10 -1
- ominfra/manage/deploy/git.py +32 -29
- ominfra/manage/deploy/inject.py +11 -0
- ominfra/manage/deploy/paths.py +46 -37
- ominfra/manage/deploy/specs.py +26 -2
- ominfra/manage/deploy/tmp.py +46 -0
- ominfra/manage/deploy/types.py +1 -0
- ominfra/manage/deploy/venvs.py +6 -1
- ominfra/scripts/journald2aws.py +32 -18
- ominfra/scripts/manage.py +417 -109
- ominfra/scripts/supervisor.py +32 -18
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/RECORD +19 -17
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev158.dist-info → ominfra-0.0.0.dev160.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -24,6 +24,7 @@ import decimal
|
|
24
24
|
import enum
|
25
25
|
import fractions
|
26
26
|
import functools
|
27
|
+
import hashlib
|
27
28
|
import inspect
|
28
29
|
import itertools
|
29
30
|
import json
|
@@ -41,6 +42,7 @@ import string
|
|
41
42
|
import struct
|
42
43
|
import subprocess
|
43
44
|
import sys
|
45
|
+
import tempfile
|
44
46
|
import threading
|
45
47
|
import time
|
46
48
|
import traceback
|
@@ -98,6 +100,10 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
|
|
98
100
|
CommandT = ta.TypeVar('CommandT', bound='Command')
|
99
101
|
CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
|
100
102
|
|
103
|
+
# deploy/atomics.py
|
104
|
+
DeployAtomicPathSwapKind = ta.Literal['dir', 'file']
|
105
|
+
DeployAtomicPathSwapState = ta.Literal['open', 'committed', 'aborted'] # ta.TypeAlias
|
106
|
+
|
101
107
|
# deploy/paths.py
|
102
108
|
DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
|
103
109
|
DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
|
@@ -1382,6 +1388,7 @@ DeployHome = ta.NewType('DeployHome', str)
|
|
1382
1388
|
DeployApp = ta.NewType('DeployApp', str)
|
1383
1389
|
DeployTag = ta.NewType('DeployTag', str)
|
1384
1390
|
DeployRev = ta.NewType('DeployRev', str)
|
1391
|
+
DeployKey = ta.NewType('DeployKey', str)
|
1385
1392
|
|
1386
1393
|
|
1387
1394
|
class DeployAppTag(ta.NamedTuple):
|
@@ -2685,6 +2692,10 @@ def is_new_type(spec: ta.Any) -> bool:
|
|
2685
2692
|
return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1] # type: ignore # noqa
|
2686
2693
|
|
2687
2694
|
|
2695
|
+
def get_new_type_supertype(spec: ta.Any) -> ta.Any:
|
2696
|
+
return spec.__supertype__
|
2697
|
+
|
2698
|
+
|
2688
2699
|
def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
|
2689
2700
|
seen = set()
|
2690
2701
|
todo = list(reversed(cls.__subclasses__()))
|
@@ -4054,41 +4065,212 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
|
4054
4065
|
|
4055
4066
|
|
4056
4067
|
########################################
|
4057
|
-
# ../deploy/
|
4058
|
-
|
4059
|
-
|
4060
|
-
|
4061
|
-
|
4062
|
-
|
4063
|
-
|
4064
|
-
|
4065
|
-
|
4066
|
-
|
4067
|
-
|
4068
|
-
|
4069
|
-
|
4070
|
-
|
4071
|
-
|
4072
|
-
|
4073
|
-
|
4074
|
-
|
4075
|
-
|
4076
|
-
|
4077
|
-
|
4078
|
-
|
4079
|
-
|
4080
|
-
|
4081
|
-
|
4082
|
-
|
4083
|
-
|
4084
|
-
|
4085
|
-
|
4086
|
-
|
4087
|
-
|
4088
|
-
|
4068
|
+
# ../deploy/atomics.py
|
4069
|
+
|
4070
|
+
|
4071
|
+
##
|
4072
|
+
|
4073
|
+
|
4074
|
+
class DeployAtomicPathSwap(abc.ABC):
|
4075
|
+
def __init__(
|
4076
|
+
self,
|
4077
|
+
kind: DeployAtomicPathSwapKind,
|
4078
|
+
dst_path: str,
|
4079
|
+
*,
|
4080
|
+
auto_commit: bool = False,
|
4081
|
+
) -> None:
|
4082
|
+
super().__init__()
|
4083
|
+
|
4084
|
+
self._kind = kind
|
4085
|
+
self._dst_path = dst_path
|
4086
|
+
self._auto_commit = auto_commit
|
4087
|
+
|
4088
|
+
self._state: DeployAtomicPathSwapState = 'open'
|
4089
|
+
|
4090
|
+
def __repr__(self) -> str:
|
4091
|
+
return attr_repr(self, 'kind', 'dst_path', 'tmp_path')
|
4092
|
+
|
4093
|
+
@property
|
4094
|
+
def kind(self) -> DeployAtomicPathSwapKind:
|
4095
|
+
return self._kind
|
4096
|
+
|
4097
|
+
@property
|
4098
|
+
def dst_path(self) -> str:
|
4099
|
+
return self._dst_path
|
4100
|
+
|
4101
|
+
@property
|
4102
|
+
@abc.abstractmethod
|
4103
|
+
def tmp_path(self) -> str:
|
4104
|
+
raise NotImplementedError
|
4105
|
+
|
4106
|
+
#
|
4107
|
+
|
4108
|
+
@property
|
4109
|
+
def state(self) -> DeployAtomicPathSwapState:
|
4110
|
+
return self._state
|
4111
|
+
|
4112
|
+
def _check_state(self, *states: DeployAtomicPathSwapState) -> None:
|
4113
|
+
if self._state not in states:
|
4114
|
+
raise RuntimeError(f'Atomic path swap not in correct state: {self._state}, {states}')
|
4115
|
+
|
4116
|
+
#
|
4117
|
+
|
4118
|
+
@abc.abstractmethod
|
4119
|
+
def _commit(self) -> None:
|
4120
|
+
raise NotImplementedError
|
4121
|
+
|
4122
|
+
def commit(self) -> None:
|
4123
|
+
if self._state == 'committed':
|
4124
|
+
return
|
4125
|
+
self._check_state('open')
|
4126
|
+
try:
|
4127
|
+
self._commit()
|
4128
|
+
except Exception: # noqa
|
4129
|
+
self._abort()
|
4130
|
+
raise
|
4131
|
+
else:
|
4132
|
+
self._state = 'committed'
|
4133
|
+
|
4134
|
+
#
|
4135
|
+
|
4136
|
+
@abc.abstractmethod
|
4137
|
+
def _abort(self) -> None:
|
4138
|
+
raise NotImplementedError
|
4139
|
+
|
4140
|
+
def abort(self) -> None:
|
4141
|
+
if self._state == 'aborted':
|
4142
|
+
return
|
4143
|
+
self._abort()
|
4144
|
+
self._state = 'aborted'
|
4145
|
+
|
4146
|
+
#
|
4147
|
+
|
4148
|
+
def __enter__(self) -> 'DeployAtomicPathSwap':
|
4149
|
+
return self
|
4150
|
+
|
4151
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
4152
|
+
if (
|
4153
|
+
exc_type is None and
|
4154
|
+
self._auto_commit and
|
4155
|
+
self._state == 'open'
|
4156
|
+
):
|
4157
|
+
self.commit()
|
4158
|
+
else:
|
4159
|
+
self.abort()
|
4160
|
+
|
4161
|
+
|
4162
|
+
#
|
4163
|
+
|
4164
|
+
|
4165
|
+
class DeployAtomicPathSwapping(abc.ABC):
|
4166
|
+
@abc.abstractmethod
|
4167
|
+
def begin_atomic_path_swap(
|
4168
|
+
self,
|
4169
|
+
kind: DeployAtomicPathSwapKind,
|
4170
|
+
dst_path: str,
|
4171
|
+
*,
|
4172
|
+
name_hint: ta.Optional[str] = None,
|
4173
|
+
make_dirs: bool = False,
|
4174
|
+
**kwargs: ta.Any,
|
4175
|
+
) -> DeployAtomicPathSwap:
|
4176
|
+
raise NotImplementedError
|
4177
|
+
|
4178
|
+
|
4179
|
+
##
|
4180
|
+
|
4181
|
+
|
4182
|
+
class OsRenameDeployAtomicPathSwap(DeployAtomicPathSwap):
|
4183
|
+
def __init__(
|
4184
|
+
self,
|
4185
|
+
kind: DeployAtomicPathSwapKind,
|
4186
|
+
dst_path: str,
|
4187
|
+
tmp_path: str,
|
4188
|
+
**kwargs: ta.Any,
|
4189
|
+
) -> None:
|
4190
|
+
if kind == 'dir':
|
4191
|
+
check.state(os.path.isdir(tmp_path))
|
4192
|
+
elif kind == 'file':
|
4193
|
+
check.state(os.path.isfile(tmp_path))
|
4194
|
+
else:
|
4195
|
+
raise TypeError(kind)
|
4196
|
+
|
4197
|
+
super().__init__(
|
4198
|
+
kind,
|
4199
|
+
dst_path,
|
4200
|
+
**kwargs,
|
4201
|
+
)
|
4202
|
+
|
4203
|
+
self._tmp_path = tmp_path
|
4204
|
+
|
4205
|
+
@property
|
4206
|
+
def tmp_path(self) -> str:
|
4207
|
+
return self._tmp_path
|
4208
|
+
|
4209
|
+
def _commit(self) -> None:
|
4210
|
+
os.rename(self._tmp_path, self._dst_path)
|
4211
|
+
|
4212
|
+
def _abort(self) -> None:
|
4213
|
+
shutil.rmtree(self._tmp_path, ignore_errors=True)
|
4214
|
+
|
4215
|
+
|
4216
|
+
class TempDirDeployAtomicPathSwapping(DeployAtomicPathSwapping):
|
4217
|
+
def __init__(
|
4218
|
+
self,
|
4219
|
+
*,
|
4220
|
+
temp_dir: ta.Optional[str] = None,
|
4221
|
+
root_dir: ta.Optional[str] = None,
|
4222
|
+
) -> None:
|
4223
|
+
super().__init__()
|
4224
|
+
|
4225
|
+
if root_dir is not None:
|
4226
|
+
root_dir = os.path.abspath(root_dir)
|
4227
|
+
self._root_dir = root_dir
|
4228
|
+
self._temp_dir = temp_dir
|
4229
|
+
|
4230
|
+
def begin_atomic_path_swap(
|
4231
|
+
self,
|
4232
|
+
kind: DeployAtomicPathSwapKind,
|
4233
|
+
dst_path: str,
|
4234
|
+
*,
|
4235
|
+
name_hint: ta.Optional[str] = None,
|
4236
|
+
make_dirs: bool = False,
|
4237
|
+
**kwargs: ta.Any,
|
4238
|
+
) -> DeployAtomicPathSwap:
|
4239
|
+
dst_path = os.path.abspath(dst_path)
|
4240
|
+
if self._root_dir is not None and not dst_path.startswith(check.non_empty_str(self._root_dir)):
|
4241
|
+
raise RuntimeError(f'Atomic path swap dst must be in root dir: {dst_path}, {self._root_dir}')
|
4242
|
+
|
4243
|
+
dst_dir = os.path.dirname(dst_path)
|
4244
|
+
if make_dirs:
|
4245
|
+
os.makedirs(dst_dir, exist_ok=True)
|
4246
|
+
if not os.path.isdir(dst_dir):
|
4247
|
+
raise RuntimeError(f'Atomic path swap dst dir does not exist: {dst_dir}')
|
4248
|
+
|
4249
|
+
if kind == 'dir':
|
4250
|
+
tmp_path = tempfile.mkdtemp(prefix=name_hint, dir=self._temp_dir)
|
4251
|
+
elif kind == 'file':
|
4252
|
+
fd, tmp_path = tempfile.mkstemp(prefix=name_hint, dir=self._temp_dir)
|
4253
|
+
os.close(fd)
|
4254
|
+
else:
|
4255
|
+
raise TypeError(kind)
|
4089
4256
|
|
4090
|
-
|
4257
|
+
return OsRenameDeployAtomicPathSwap(
|
4258
|
+
kind,
|
4259
|
+
dst_path,
|
4260
|
+
tmp_path,
|
4261
|
+
**kwargs,
|
4262
|
+
)
|
4091
4263
|
|
4264
|
+
|
4265
|
+
########################################
|
4266
|
+
# ../deploy/paths.py
|
4267
|
+
"""
|
4268
|
+
TODO:
|
4269
|
+
- run/pidfile
|
4270
|
+
- logs/...
|
4271
|
+
- current symlink
|
4272
|
+
- conf/{nginx,supervisor}
|
4273
|
+
- env/?
|
4092
4274
|
"""
|
4093
4275
|
|
4094
4276
|
|
@@ -4100,7 +4282,7 @@ DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
|
|
4100
4282
|
|
4101
4283
|
DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
|
4102
4284
|
'app',
|
4103
|
-
'tag',
|
4285
|
+
'tag',
|
4104
4286
|
])
|
4105
4287
|
|
4106
4288
|
|
@@ -4227,6 +4409,8 @@ class DeployPath:
|
|
4227
4409
|
parts: ta.Sequence[DeployPathPart]
|
4228
4410
|
|
4229
4411
|
def __post_init__(self) -> None:
|
4412
|
+
hash(self)
|
4413
|
+
|
4230
4414
|
check.not_empty(self.parts)
|
4231
4415
|
for p in self.parts[:-1]:
|
4232
4416
|
check.equal(p.kind, 'dir')
|
@@ -4261,10 +4445,10 @@ class DeployPath:
|
|
4261
4445
|
else:
|
4262
4446
|
tail_parse = FileDeployPathPart.parse
|
4263
4447
|
ps = check.non_empty_str(s).split('/')
|
4264
|
-
return cls(
|
4448
|
+
return cls((
|
4265
4449
|
*([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
|
4266
4450
|
tail_parse(ps[-1]),
|
4267
|
-
|
4451
|
+
))
|
4268
4452
|
|
4269
4453
|
|
4270
4454
|
##
|
@@ -4272,10 +4456,41 @@ class DeployPath:
|
|
4272
4456
|
|
4273
4457
|
class DeployPathOwner(abc.ABC):
|
4274
4458
|
@abc.abstractmethod
|
4275
|
-
def
|
4459
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
4276
4460
|
raise NotImplementedError
|
4277
4461
|
|
4278
4462
|
|
4463
|
+
class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
|
4464
|
+
def __init__(
|
4465
|
+
self,
|
4466
|
+
*args: ta.Any,
|
4467
|
+
owned_dir: str,
|
4468
|
+
deploy_home: ta.Optional[DeployHome],
|
4469
|
+
**kwargs: ta.Any,
|
4470
|
+
) -> None:
|
4471
|
+
super().__init__(*args, **kwargs)
|
4472
|
+
|
4473
|
+
check.not_in('/', owned_dir)
|
4474
|
+
self._owned_dir: str = check.non_empty_str(owned_dir)
|
4475
|
+
|
4476
|
+
self._deploy_home = deploy_home
|
4477
|
+
|
4478
|
+
self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
|
4479
|
+
|
4480
|
+
@cached_nullary
|
4481
|
+
def _dir(self) -> str:
|
4482
|
+
return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
|
4483
|
+
|
4484
|
+
@cached_nullary
|
4485
|
+
def _make_dir(self) -> str:
|
4486
|
+
if not os.path.isdir(d := self._dir()):
|
4487
|
+
os.makedirs(d, exist_ok=True)
|
4488
|
+
return d
|
4489
|
+
|
4490
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
4491
|
+
return self._owned_deploy_paths
|
4492
|
+
|
4493
|
+
|
4279
4494
|
########################################
|
4280
4495
|
# ../deploy/specs.py
|
4281
4496
|
|
@@ -4294,14 +4509,35 @@ class DeployGitRepo:
|
|
4294
4509
|
check.not_in('.', check.non_empty_str(self.path))
|
4295
4510
|
|
4296
4511
|
|
4512
|
+
@dc.dataclass(frozen=True)
|
4513
|
+
class DeployGitCheckout:
|
4514
|
+
repo: DeployGitRepo
|
4515
|
+
rev: DeployRev
|
4516
|
+
|
4517
|
+
subtrees: ta.Optional[ta.Sequence[str]] = None
|
4518
|
+
|
4519
|
+
def __post_init__(self) -> None:
|
4520
|
+
hash(self)
|
4521
|
+
check.non_empty_str(self.rev)
|
4522
|
+
if self.subtrees is not None:
|
4523
|
+
for st in self.subtrees:
|
4524
|
+
check.non_empty_str(st)
|
4525
|
+
|
4526
|
+
|
4297
4527
|
##
|
4298
4528
|
|
4299
4529
|
|
4300
4530
|
@dc.dataclass(frozen=True)
|
4301
4531
|
class DeploySpec:
|
4302
4532
|
app: DeployApp
|
4303
|
-
|
4304
|
-
|
4533
|
+
checkout: DeployGitCheckout
|
4534
|
+
|
4535
|
+
def __post_init__(self) -> None:
|
4536
|
+
hash(self)
|
4537
|
+
|
4538
|
+
@cached_nullary
|
4539
|
+
def key(self) -> DeployKey:
|
4540
|
+
return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
|
4305
4541
|
|
4306
4542
|
|
4307
4543
|
########################################
|
@@ -5761,9 +5997,7 @@ inj = Injection
|
|
5761
5997
|
"""
|
5762
5998
|
TODO:
|
5763
5999
|
- pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
|
5764
|
-
- namedtuple
|
5765
6000
|
- literals
|
5766
|
-
- newtypes?
|
5767
6001
|
"""
|
5768
6002
|
|
5769
6003
|
|
@@ -5773,7 +6007,7 @@ TODO:
|
|
5773
6007
|
@dc.dataclass(frozen=True)
|
5774
6008
|
class ObjMarshalOptions:
|
5775
6009
|
raw_bytes: bool = False
|
5776
|
-
|
6010
|
+
non_strict_fields: bool = False
|
5777
6011
|
|
5778
6012
|
|
5779
6013
|
class ObjMarshaler(abc.ABC):
|
@@ -5902,10 +6136,10 @@ class IterableObjMarshaler(ObjMarshaler):
|
|
5902
6136
|
|
5903
6137
|
|
5904
6138
|
@dc.dataclass(frozen=True)
|
5905
|
-
class
|
6139
|
+
class FieldsObjMarshaler(ObjMarshaler):
|
5906
6140
|
ty: type
|
5907
6141
|
fs: ta.Mapping[str, ObjMarshaler]
|
5908
|
-
|
6142
|
+
non_strict: bool = False
|
5909
6143
|
|
5910
6144
|
def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
|
5911
6145
|
return {
|
@@ -5917,7 +6151,7 @@ class DataclassObjMarshaler(ObjMarshaler):
|
|
5917
6151
|
return self.ty(**{
|
5918
6152
|
k: self.fs[k].unmarshal(v, ctx)
|
5919
6153
|
for k, v in o.items()
|
5920
|
-
if not (self.
|
6154
|
+
if not (self.non_strict or ctx.options.non_strict_fields) or k in self.fs
|
5921
6155
|
})
|
5922
6156
|
|
5923
6157
|
|
@@ -6049,7 +6283,7 @@ class ObjMarshalerManager:
|
|
6049
6283
|
ty: ta.Any,
|
6050
6284
|
rec: ta.Callable[[ta.Any], ObjMarshaler],
|
6051
6285
|
*,
|
6052
|
-
|
6286
|
+
non_strict_fields: bool = False,
|
6053
6287
|
) -> ObjMarshaler:
|
6054
6288
|
if isinstance(ty, type):
|
6055
6289
|
if abc.ABC in ty.__bases__:
|
@@ -6071,12 +6305,22 @@ class ObjMarshalerManager:
|
|
6071
6305
|
return EnumObjMarshaler(ty)
|
6072
6306
|
|
6073
6307
|
if dc.is_dataclass(ty):
|
6074
|
-
return
|
6308
|
+
return FieldsObjMarshaler(
|
6075
6309
|
ty,
|
6076
6310
|
{f.name: rec(f.type) for f in dc.fields(ty)},
|
6077
|
-
|
6311
|
+
non_strict=non_strict_fields,
|
6078
6312
|
)
|
6079
6313
|
|
6314
|
+
if issubclass(ty, tuple) and hasattr(ty, '_fields'):
|
6315
|
+
return FieldsObjMarshaler(
|
6316
|
+
ty,
|
6317
|
+
{p.name: rec(p.annotation) for p in inspect.signature(ty).parameters.values()},
|
6318
|
+
non_strict=non_strict_fields,
|
6319
|
+
)
|
6320
|
+
|
6321
|
+
if is_new_type(ty):
|
6322
|
+
return rec(get_new_type_supertype(ty))
|
6323
|
+
|
6080
6324
|
if is_generic_alias(ty):
|
6081
6325
|
try:
|
6082
6326
|
mt = self._generic_mapping_types[ta.get_origin(ty)]
|
@@ -6210,12 +6454,12 @@ def is_debugger_attached() -> bool:
|
|
6210
6454
|
return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
|
6211
6455
|
|
6212
6456
|
|
6213
|
-
|
6457
|
+
LITE_REQUIRED_PYTHON_VERSION = (3, 8)
|
6214
6458
|
|
6215
6459
|
|
6216
|
-
def
|
6217
|
-
if sys.version_info <
|
6218
|
-
raise OSError(f'Requires python {
|
6460
|
+
def check_lite_runtime_version() -> None:
|
6461
|
+
if sys.version_info < LITE_REQUIRED_PYTHON_VERSION:
|
6462
|
+
raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
|
6219
6463
|
|
6220
6464
|
|
6221
6465
|
########################################
|
@@ -6503,24 +6747,41 @@ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command
|
|
6503
6747
|
|
6504
6748
|
|
6505
6749
|
########################################
|
6506
|
-
# ../deploy/
|
6507
|
-
|
6508
|
-
|
6509
|
-
##
|
6750
|
+
# ../deploy/tmp.py
|
6510
6751
|
|
6511
6752
|
|
6512
|
-
|
6513
|
-
|
6514
|
-
|
6515
|
-
|
6516
|
-
|
6517
|
-
|
6753
|
+
class DeployTmpManager(
|
6754
|
+
SingleDirDeployPathOwner,
|
6755
|
+
DeployAtomicPathSwapping,
|
6756
|
+
):
|
6757
|
+
def __init__(
|
6758
|
+
self,
|
6759
|
+
*,
|
6760
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
6761
|
+
) -> None:
|
6762
|
+
super().__init__(
|
6763
|
+
owned_dir='tmp',
|
6764
|
+
deploy_home=deploy_home,
|
6765
|
+
)
|
6518
6766
|
|
6519
|
-
|
6520
|
-
|
6521
|
-
|
6767
|
+
@cached_nullary
|
6768
|
+
def _swapping(self) -> DeployAtomicPathSwapping:
|
6769
|
+
return TempDirDeployAtomicPathSwapping(
|
6770
|
+
temp_dir=self._make_dir(),
|
6771
|
+
root_dir=check.non_empty_str(self._deploy_home),
|
6772
|
+
)
|
6522
6773
|
|
6523
|
-
|
6774
|
+
def begin_atomic_path_swap(
|
6775
|
+
self,
|
6776
|
+
kind: DeployAtomicPathSwapKind,
|
6777
|
+
dst_path: str,
|
6778
|
+
**kwargs: ta.Any,
|
6779
|
+
) -> DeployAtomicPathSwap:
|
6780
|
+
return self._swapping().begin_atomic_path_swap(
|
6781
|
+
kind,
|
6782
|
+
dst_path,
|
6783
|
+
**kwargs,
|
6784
|
+
)
|
6524
6785
|
|
6525
6786
|
|
6526
6787
|
########################################
|
@@ -6630,6 +6891,7 @@ TODO:
|
|
6630
6891
|
- structured
|
6631
6892
|
- prefixed
|
6632
6893
|
- debug
|
6894
|
+
- optional noisy? noisy will never be lite - some kinda configure_standard callback mechanism?
|
6633
6895
|
"""
|
6634
6896
|
|
6635
6897
|
|
@@ -6666,8 +6928,9 @@ class StandardLogFormatter(logging.Formatter):
|
|
6666
6928
|
##
|
6667
6929
|
|
6668
6930
|
|
6669
|
-
class
|
6670
|
-
|
6931
|
+
class StandardConfiguredLogHandler(ProxyLogHandler):
|
6932
|
+
def __init_subclass__(cls, **kwargs):
|
6933
|
+
raise TypeError('This class serves only as a marker and should not be subclassed.')
|
6671
6934
|
|
6672
6935
|
|
6673
6936
|
##
|
@@ -6698,7 +6961,7 @@ def configure_standard_logging(
|
|
6698
6961
|
target: ta.Optional[logging.Logger] = None,
|
6699
6962
|
force: bool = False,
|
6700
6963
|
handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
|
6701
|
-
) -> ta.Optional[
|
6964
|
+
) -> ta.Optional[StandardConfiguredLogHandler]:
|
6702
6965
|
with _locking_logging_module_lock():
|
6703
6966
|
if target is None:
|
6704
6967
|
target = logging.root
|
@@ -6706,7 +6969,7 @@ def configure_standard_logging(
|
|
6706
6969
|
#
|
6707
6970
|
|
6708
6971
|
if not force:
|
6709
|
-
if any(isinstance(h,
|
6972
|
+
if any(isinstance(h, StandardConfiguredLogHandler) for h in list(target.handlers)):
|
6710
6973
|
return None
|
6711
6974
|
|
6712
6975
|
#
|
@@ -6740,7 +7003,7 @@ def configure_standard_logging(
|
|
6740
7003
|
|
6741
7004
|
#
|
6742
7005
|
|
6743
|
-
return
|
7006
|
+
return StandardConfiguredLogHandler(handler)
|
6744
7007
|
|
6745
7008
|
|
6746
7009
|
########################################
|
@@ -7907,27 +8170,22 @@ github.com/wrmsr/omlish@rev
|
|
7907
8170
|
##
|
7908
8171
|
|
7909
8172
|
|
7910
|
-
class DeployGitManager(
|
8173
|
+
class DeployGitManager(SingleDirDeployPathOwner):
|
7911
8174
|
def __init__(
|
7912
8175
|
self,
|
7913
8176
|
*,
|
7914
8177
|
deploy_home: ta.Optional[DeployHome] = None,
|
8178
|
+
atomics: DeployAtomicPathSwapping,
|
7915
8179
|
) -> None:
|
7916
|
-
super().__init__(
|
8180
|
+
super().__init__(
|
8181
|
+
owned_dir='git',
|
8182
|
+
deploy_home=deploy_home,
|
8183
|
+
)
|
7917
8184
|
|
7918
|
-
self.
|
8185
|
+
self._atomics = atomics
|
7919
8186
|
|
7920
8187
|
self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
|
7921
8188
|
|
7922
|
-
@cached_nullary
|
7923
|
-
def _dir(self) -> str:
|
7924
|
-
return os.path.join(check.non_empty_str(self._deploy_home), 'git')
|
7925
|
-
|
7926
|
-
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
7927
|
-
return {
|
7928
|
-
DeployPath.parse('git'),
|
7929
|
-
}
|
7930
|
-
|
7931
8189
|
class RepoDir:
|
7932
8190
|
def __init__(
|
7933
8191
|
self,
|
@@ -7939,7 +8197,7 @@ class DeployGitManager(DeployPathOwner):
|
|
7939
8197
|
self._git = git
|
7940
8198
|
self._repo = repo
|
7941
8199
|
self._dir = os.path.join(
|
7942
|
-
self._git.
|
8200
|
+
self._git._make_dir(), # noqa
|
7943
8201
|
check.non_empty_str(repo.host),
|
7944
8202
|
check.non_empty_str(repo.path),
|
7945
8203
|
)
|
@@ -7955,12 +8213,16 @@ class DeployGitManager(DeployPathOwner):
|
|
7955
8213
|
else:
|
7956
8214
|
return f'https://{self._repo.host}/{self._repo.path}'
|
7957
8215
|
|
8216
|
+
#
|
8217
|
+
|
7958
8218
|
async def _call(self, *cmd: str) -> None:
|
7959
8219
|
await asyncio_subprocesses.check_call(
|
7960
8220
|
*cmd,
|
7961
8221
|
cwd=self._dir,
|
7962
8222
|
)
|
7963
8223
|
|
8224
|
+
#
|
8225
|
+
|
7964
8226
|
@async_cached_nullary
|
7965
8227
|
async def init(self) -> None:
|
7966
8228
|
os.makedirs(self._dir, exist_ok=True)
|
@@ -7974,20 +8236,24 @@ class DeployGitManager(DeployPathOwner):
|
|
7974
8236
|
await self.init()
|
7975
8237
|
await self._call('git', 'fetch', '--depth=1', 'origin', rev)
|
7976
8238
|
|
7977
|
-
|
7978
|
-
check.state(not os.path.exists(dst_dir))
|
7979
|
-
|
7980
|
-
await self.fetch(rev)
|
8239
|
+
#
|
7981
8240
|
|
7982
|
-
|
7983
|
-
os.
|
8241
|
+
async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
|
8242
|
+
check.state(not os.path.exists(dst_dir))
|
8243
|
+
with self._git._atomics.begin_atomic_path_swap( # noqa
|
8244
|
+
'dir',
|
8245
|
+
dst_dir,
|
8246
|
+
auto_commit=True,
|
8247
|
+
make_dirs=True,
|
8248
|
+
) as dst_swap:
|
8249
|
+
await self.fetch(checkout.rev)
|
7984
8250
|
|
7985
|
-
|
7986
|
-
|
8251
|
+
dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
|
8252
|
+
await dst_call('git', 'init')
|
7987
8253
|
|
7988
|
-
|
7989
|
-
|
7990
|
-
|
8254
|
+
await dst_call('git', 'remote', 'add', 'local', self._dir)
|
8255
|
+
await dst_call('git', 'fetch', '--depth=1', 'local', checkout.rev)
|
8256
|
+
await dst_call('git', 'checkout', checkout.rev, *(checkout.subtrees or []))
|
7991
8257
|
|
7992
8258
|
def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
|
7993
8259
|
try:
|
@@ -7996,8 +8262,8 @@ class DeployGitManager(DeployPathOwner):
|
|
7996
8262
|
repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
|
7997
8263
|
return repo_dir
|
7998
8264
|
|
7999
|
-
async def checkout(self,
|
8000
|
-
await self.get_repo_dir(repo).checkout(
|
8265
|
+
async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
|
8266
|
+
await self.get_repo_dir(checkout.repo).checkout(checkout, dst_dir)
|
8001
8267
|
|
8002
8268
|
|
8003
8269
|
########################################
|
@@ -8014,16 +8280,18 @@ class DeployVenvManager(DeployPathOwner):
|
|
8014
8280
|
self,
|
8015
8281
|
*,
|
8016
8282
|
deploy_home: ta.Optional[DeployHome] = None,
|
8283
|
+
atomics: DeployAtomicPathSwapping,
|
8017
8284
|
) -> None:
|
8018
8285
|
super().__init__()
|
8019
8286
|
|
8020
8287
|
self._deploy_home = deploy_home
|
8288
|
+
self._atomics = atomics
|
8021
8289
|
|
8022
8290
|
@cached_nullary
|
8023
8291
|
def _dir(self) -> str:
|
8024
8292
|
return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
|
8025
8293
|
|
8026
|
-
def
|
8294
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
8027
8295
|
return {
|
8028
8296
|
DeployPath.parse('venvs/@app/@tag/'),
|
8029
8297
|
}
|
@@ -8037,6 +8305,8 @@ class DeployVenvManager(DeployPathOwner):
|
|
8037
8305
|
) -> None:
|
8038
8306
|
sys_exe = 'python3'
|
8039
8307
|
|
8308
|
+
# !! NOTE: (most) venvs cannot be relocated, so an atomic swap can't be used. it's up to the path manager to
|
8309
|
+
# garbage collect orphaned dirs.
|
8040
8310
|
await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
|
8041
8311
|
|
8042
8312
|
#
|
@@ -8594,13 +8864,15 @@ def bind_commands(
|
|
8594
8864
|
|
8595
8865
|
def make_deploy_tag(
|
8596
8866
|
rev: DeployRev,
|
8597
|
-
|
8867
|
+
key: DeployKey,
|
8868
|
+
*,
|
8869
|
+
utcnow: ta.Optional[datetime.datetime] = None,
|
8598
8870
|
) -> DeployTag:
|
8599
|
-
if
|
8600
|
-
|
8601
|
-
now_fmt = '%Y%m%dT%H%M%
|
8602
|
-
now_str =
|
8603
|
-
return DeployTag('-'.join([now_str, rev]))
|
8871
|
+
if utcnow is None:
|
8872
|
+
utcnow = datetime.datetime.now(tz=datetime.timezone.utc) # noqa
|
8873
|
+
now_fmt = '%Y%m%dT%H%M%SZ'
|
8874
|
+
now_str = utcnow.strftime(now_fmt)
|
8875
|
+
return DeployTag('-'.join([now_str, rev, key]))
|
8604
8876
|
|
8605
8877
|
|
8606
8878
|
class DeployAppManager(DeployPathOwner):
|
@@ -8621,7 +8893,7 @@ class DeployAppManager(DeployPathOwner):
|
|
8621
8893
|
def _dir(self) -> str:
|
8622
8894
|
return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
|
8623
8895
|
|
8624
|
-
def
|
8896
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
8625
8897
|
return {
|
8626
8898
|
DeployPath.parse('apps/@app/@tag'),
|
8627
8899
|
}
|
@@ -8629,15 +8901,14 @@ class DeployAppManager(DeployPathOwner):
|
|
8629
8901
|
async def prepare_app(
|
8630
8902
|
self,
|
8631
8903
|
spec: DeploySpec,
|
8632
|
-
):
|
8633
|
-
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev))
|
8904
|
+
) -> None:
|
8905
|
+
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.checkout.rev, spec.key()))
|
8634
8906
|
app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
|
8635
8907
|
|
8636
8908
|
#
|
8637
8909
|
|
8638
8910
|
await self._git.checkout(
|
8639
|
-
spec.
|
8640
|
-
spec.rev,
|
8911
|
+
spec.checkout,
|
8641
8912
|
app_dir,
|
8642
8913
|
)
|
8643
8914
|
|
@@ -9360,6 +9631,34 @@ class SystemInterpProvider(InterpProvider):
|
|
9360
9631
|
raise KeyError(version)
|
9361
9632
|
|
9362
9633
|
|
9634
|
+
########################################
|
9635
|
+
# ../deploy/commands.py
|
9636
|
+
|
9637
|
+
|
9638
|
+
##
|
9639
|
+
|
9640
|
+
|
9641
|
+
@dc.dataclass(frozen=True)
|
9642
|
+
class DeployCommand(Command['DeployCommand.Output']):
|
9643
|
+
spec: DeploySpec
|
9644
|
+
|
9645
|
+
@dc.dataclass(frozen=True)
|
9646
|
+
class Output(Command.Output):
|
9647
|
+
pass
|
9648
|
+
|
9649
|
+
|
9650
|
+
@dc.dataclass(frozen=True)
|
9651
|
+
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
9652
|
+
_apps: DeployAppManager
|
9653
|
+
|
9654
|
+
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
9655
|
+
log.info('Deploying! %r', cmd.spec)
|
9656
|
+
|
9657
|
+
await self._apps.prepare_app(cmd.spec)
|
9658
|
+
|
9659
|
+
return DeployCommand.Output()
|
9660
|
+
|
9661
|
+
|
9363
9662
|
########################################
|
9364
9663
|
# ../remote/inject.py
|
9365
9664
|
|
@@ -9738,10 +10037,19 @@ def bind_deploy(
|
|
9738
10037
|
lst: ta.List[InjectorBindingOrBindings] = [
|
9739
10038
|
inj.bind(deploy_config),
|
9740
10039
|
|
10040
|
+
#
|
10041
|
+
|
9741
10042
|
inj.bind(DeployAppManager, singleton=True),
|
10043
|
+
|
9742
10044
|
inj.bind(DeployGitManager, singleton=True),
|
10045
|
+
|
10046
|
+
inj.bind(DeployTmpManager, singleton=True),
|
10047
|
+
inj.bind(DeployAtomicPathSwapping, to_key=DeployTmpManager),
|
10048
|
+
|
9743
10049
|
inj.bind(DeployVenvManager, singleton=True),
|
9744
10050
|
|
10051
|
+
#
|
10052
|
+
|
9745
10053
|
bind_command(DeployCommand, DeployCommandExecutor),
|
9746
10054
|
bind_command(InterpCommand, InterpCommandExecutor),
|
9747
10055
|
]
|