ominfra 0.0.0.dev167__py3-none-any.whl → 0.0.0.dev169__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/manage/deploy/apps.py +76 -26
- ominfra/manage/deploy/commands.py +3 -3
- ominfra/manage/deploy/conf.py +73 -33
- ominfra/manage/deploy/deploy.py +27 -0
- ominfra/manage/deploy/git.py +1 -1
- ominfra/manage/deploy/inject.py +7 -7
- ominfra/manage/deploy/paths/__init__.py +0 -0
- ominfra/manage/deploy/paths/inject.py +21 -0
- ominfra/manage/deploy/paths/manager.py +36 -0
- ominfra/manage/deploy/paths/owners.py +50 -0
- ominfra/manage/deploy/paths/paths.py +216 -0
- ominfra/manage/deploy/specs.py +1 -2
- ominfra/manage/deploy/tmp.py +1 -1
- ominfra/manage/deploy/types.py +26 -1
- ominfra/scripts/journald2aws.py +24 -0
- ominfra/scripts/manage.py +923 -697
- ominfra/scripts/supervisor.py +24 -0
- {ominfra-0.0.0.dev167.dist-info → ominfra-0.0.0.dev169.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev167.dist-info → ominfra-0.0.0.dev169.dist-info}/RECORD +23 -18
- ominfra/manage/deploy/paths.py +0 -243
- {ominfra-0.0.0.dev167.dist-info → ominfra-0.0.0.dev169.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev167.dist-info → ominfra-0.0.0.dev169.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev167.dist-info → ominfra-0.0.0.dev169.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev167.dist-info → ominfra-0.0.0.dev169.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py
CHANGED
@@ -100,9 +100,9 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
|
|
100
100
|
CommandT = ta.TypeVar('CommandT', bound='Command')
|
101
101
|
CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
|
102
102
|
|
103
|
-
# deploy/
|
103
|
+
# deploy/types.py
|
104
104
|
DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
|
105
|
-
DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
|
105
|
+
DeployPathPlaceholder = ta.Literal['app', 'tag', 'conf'] # ta.TypeAlias
|
106
106
|
|
107
107
|
# ../../omlish/argparse/cli.py
|
108
108
|
ArgparseCommandFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
|
@@ -1379,23 +1379,6 @@ class DeployConfig:
|
|
1379
1379
|
deploy_home: ta.Optional[str] = None
|
1380
1380
|
|
1381
1381
|
|
1382
|
-
########################################
|
1383
|
-
# ../deploy/types.py
|
1384
|
-
|
1385
|
-
|
1386
|
-
DeployHome = ta.NewType('DeployHome', str)
|
1387
|
-
|
1388
|
-
DeployApp = ta.NewType('DeployApp', str)
|
1389
|
-
DeployTag = ta.NewType('DeployTag', str)
|
1390
|
-
DeployRev = ta.NewType('DeployRev', str)
|
1391
|
-
DeployKey = ta.NewType('DeployKey', str)
|
1392
|
-
|
1393
|
-
|
1394
|
-
class DeployAppTag(ta.NamedTuple):
|
1395
|
-
app: DeployApp
|
1396
|
-
tag: DeployTag
|
1397
|
-
|
1398
|
-
|
1399
1382
|
########################################
|
1400
1383
|
# ../../pyremote.py
|
1401
1384
|
"""
|
@@ -2074,6 +2057,8 @@ def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
|
|
2074
2057
|
"""
|
2075
2058
|
TODO:
|
2076
2059
|
- def maybe(v: lang.Maybe[T])
|
2060
|
+
- def not_ ?
|
2061
|
+
- ** class @dataclass Raise - user message should be able to be an exception type or instance or factory
|
2077
2062
|
"""
|
2078
2063
|
|
2079
2064
|
|
@@ -2738,6 +2723,28 @@ def strip_with_newline(s: str) -> str:
|
|
2738
2723
|
return s.strip() + '\n'
|
2739
2724
|
|
2740
2725
|
|
2726
|
+
@ta.overload
|
2727
|
+
def split_keep_delimiter(s: str, d: str) -> str:
|
2728
|
+
...
|
2729
|
+
|
2730
|
+
|
2731
|
+
@ta.overload
|
2732
|
+
def split_keep_delimiter(s: bytes, d: bytes) -> bytes:
|
2733
|
+
...
|
2734
|
+
|
2735
|
+
|
2736
|
+
def split_keep_delimiter(s, d):
|
2737
|
+
ps = []
|
2738
|
+
i = 0
|
2739
|
+
while i < len(s):
|
2740
|
+
if (n := s.find(d, i)) < i:
|
2741
|
+
ps.append(s[i:])
|
2742
|
+
break
|
2743
|
+
ps.append(s[i:n + 1])
|
2744
|
+
i = n + 1
|
2745
|
+
return ps
|
2746
|
+
|
2747
|
+
|
2741
2748
|
##
|
2742
2749
|
|
2743
2750
|
|
@@ -3422,8 +3429,12 @@ def relative_symlink(
|
|
3422
3429
|
*,
|
3423
3430
|
target_is_directory: bool = False,
|
3424
3431
|
dir_fd: ta.Optional[int] = None,
|
3432
|
+
make_dirs: bool = False,
|
3425
3433
|
**kwargs: ta.Any,
|
3426
3434
|
) -> None:
|
3435
|
+
if make_dirs:
|
3436
|
+
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
3437
|
+
|
3427
3438
|
os.symlink(
|
3428
3439
|
os.path.relpath(src, os.path.dirname(dst)),
|
3429
3440
|
dst,
|
@@ -4106,531 +4117,201 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
|
4106
4117
|
|
4107
4118
|
|
4108
4119
|
########################################
|
4109
|
-
# ../deploy/
|
4110
|
-
"""
|
4111
|
-
TODO:
|
4112
|
-
- run/{.pid,.sock}
|
4113
|
-
- logs/...
|
4114
|
-
- current symlink
|
4115
|
-
- conf/{nginx,supervisor}
|
4116
|
-
- env/?
|
4117
|
-
- apps/<app>/shared
|
4118
|
-
"""
|
4120
|
+
# ../deploy/types.py
|
4119
4121
|
|
4120
4122
|
|
4121
4123
|
##
|
4122
4124
|
|
4123
4125
|
|
4124
|
-
|
4125
|
-
DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
|
4126
|
+
DeployHome = ta.NewType('DeployHome', str)
|
4126
4127
|
|
4127
|
-
|
4128
|
-
|
4129
|
-
|
4130
|
-
|
4128
|
+
DeployApp = ta.NewType('DeployApp', str)
|
4129
|
+
DeployTag = ta.NewType('DeployTag', str)
|
4130
|
+
DeployRev = ta.NewType('DeployRev', str)
|
4131
|
+
DeployKey = ta.NewType('DeployKey', str)
|
4131
4132
|
|
4132
4133
|
|
4133
|
-
|
4134
|
-
pass
|
4134
|
+
##
|
4135
4135
|
|
4136
4136
|
|
4137
4137
|
@dc.dataclass(frozen=True)
|
4138
|
-
class
|
4139
|
-
|
4140
|
-
|
4141
|
-
def kind(self) -> DeployPathKind:
|
4142
|
-
raise NotImplementedError
|
4138
|
+
class DeployAppTag:
|
4139
|
+
app: DeployApp
|
4140
|
+
tag: DeployTag
|
4143
4141
|
|
4144
|
-
|
4145
|
-
|
4146
|
-
|
4142
|
+
def __post_init__(self) -> None:
|
4143
|
+
for s in [self.app, self.tag]:
|
4144
|
+
check.non_empty_str(s)
|
4145
|
+
check.equal(s, s.strip())
|
4147
4146
|
|
4147
|
+
def placeholders(self) -> ta.Mapping[DeployPathPlaceholder, str]:
|
4148
|
+
return {
|
4149
|
+
'app': self.app,
|
4150
|
+
'tag': self.tag,
|
4151
|
+
}
|
4148
4152
|
|
4149
|
-
#
|
4150
4153
|
|
4154
|
+
########################################
|
4155
|
+
# ../remote/config.py
|
4151
4156
|
|
4152
|
-
class DirDeployPathPart(DeployPathPart, abc.ABC):
|
4153
|
-
@property
|
4154
|
-
def kind(self) -> DeployPathKind:
|
4155
|
-
return 'dir'
|
4156
4157
|
|
4157
|
-
|
4158
|
-
|
4159
|
-
|
4160
|
-
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
4161
|
-
return PlaceholderDirDeployPathPart(s[1:])
|
4162
|
-
else:
|
4163
|
-
return ConstDirDeployPathPart(s)
|
4158
|
+
@dc.dataclass(frozen=True)
|
4159
|
+
class RemoteConfig:
|
4160
|
+
payload_file: ta.Optional[str] = None
|
4164
4161
|
|
4162
|
+
set_pgid: bool = True
|
4165
4163
|
|
4166
|
-
|
4167
|
-
@property
|
4168
|
-
def kind(self) -> DeployPathKind:
|
4169
|
-
return 'file'
|
4164
|
+
deathsig: ta.Optional[str] = 'KILL'
|
4170
4165
|
|
4171
|
-
|
4172
|
-
def parse(cls, s: str) -> 'FileDeployPathPart':
|
4173
|
-
if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
|
4174
|
-
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
4175
|
-
if not any(c in s for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS):
|
4176
|
-
return PlaceholderFileDeployPathPart(s[1:], '')
|
4177
|
-
else:
|
4178
|
-
p = min(f for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS if (f := s.find(c)) > 0)
|
4179
|
-
return PlaceholderFileDeployPathPart(s[1:p], s[p:])
|
4180
|
-
else:
|
4181
|
-
return ConstFileDeployPathPart(s)
|
4166
|
+
pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
|
4182
4167
|
|
4168
|
+
forward_logging: bool = True
|
4183
4169
|
|
4184
|
-
|
4170
|
+
timebomb_delay_s: ta.Optional[float] = 60 * 60.
|
4185
4171
|
|
4172
|
+
heartbeat_interval_s: float = 3.
|
4186
4173
|
|
4187
|
-
@dc.dataclass(frozen=True)
|
4188
|
-
class ConstDeployPathPart(DeployPathPart, abc.ABC):
|
4189
|
-
name: str
|
4190
4174
|
|
4191
|
-
|
4192
|
-
|
4193
|
-
check.not_in('/', self.name)
|
4194
|
-
check.not_in(DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, self.name)
|
4175
|
+
########################################
|
4176
|
+
# ../remote/payload.py
|
4195
4177
|
|
4196
|
-
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4197
|
-
return self.name
|
4198
4178
|
|
4179
|
+
RemoteExecutionPayloadFile = ta.NewType('RemoteExecutionPayloadFile', str)
|
4199
4180
|
|
4200
|
-
class ConstDirDeployPathPart(ConstDeployPathPart, DirDeployPathPart):
|
4201
|
-
pass
|
4202
4181
|
|
4182
|
+
@cached_nullary
|
4183
|
+
def _get_self_src() -> str:
|
4184
|
+
return inspect.getsource(sys.modules[__name__])
|
4203
4185
|
|
4204
|
-
class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
|
4205
|
-
pass
|
4206
4186
|
|
4187
|
+
def _is_src_amalg(src: str) -> bool:
|
4188
|
+
for l in src.splitlines(): # noqa
|
4189
|
+
if l.startswith('# @omlish-amalg-output '):
|
4190
|
+
return True
|
4191
|
+
return False
|
4207
4192
|
|
4208
|
-
#
|
4209
4193
|
|
4194
|
+
@cached_nullary
|
4195
|
+
def _is_self_amalg() -> bool:
|
4196
|
+
return _is_src_amalg(_get_self_src())
|
4210
4197
|
|
4211
|
-
@dc.dataclass(frozen=True)
|
4212
|
-
class PlaceholderDeployPathPart(DeployPathPart, abc.ABC):
|
4213
|
-
placeholder: str # DeployPathPlaceholder
|
4214
4198
|
|
4215
|
-
|
4216
|
-
|
4217
|
-
|
4218
|
-
|
4219
|
-
|
4199
|
+
def get_remote_payload_src(
|
4200
|
+
*,
|
4201
|
+
file: ta.Optional[RemoteExecutionPayloadFile],
|
4202
|
+
) -> str:
|
4203
|
+
if file is not None:
|
4204
|
+
with open(file) as f:
|
4205
|
+
return f.read()
|
4220
4206
|
|
4221
|
-
|
4222
|
-
|
4223
|
-
return placeholders[self.placeholder] # type: ignore
|
4224
|
-
else:
|
4225
|
-
return DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER + self.placeholder
|
4207
|
+
if _is_self_amalg():
|
4208
|
+
return _get_self_src()
|
4226
4209
|
|
4210
|
+
import importlib.resources
|
4211
|
+
return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
|
4227
4212
|
|
4228
|
-
@dc.dataclass(frozen=True)
|
4229
|
-
class PlaceholderDirDeployPathPart(PlaceholderDeployPathPart, DirDeployPathPart):
|
4230
|
-
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4231
|
-
return self._render_placeholder(placeholders)
|
4232
4213
|
|
4214
|
+
########################################
|
4215
|
+
# ../system/platforms.py
|
4233
4216
|
|
4234
|
-
@dc.dataclass(frozen=True)
|
4235
|
-
class PlaceholderFileDeployPathPart(PlaceholderDeployPathPart, FileDeployPathPart):
|
4236
|
-
suffix: str
|
4237
4217
|
|
4238
|
-
|
4239
|
-
super().__post_init__()
|
4240
|
-
if self.suffix:
|
4241
|
-
for c in [DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
|
4242
|
-
check.not_in(c, self.suffix)
|
4218
|
+
##
|
4243
4219
|
|
4244
|
-
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4245
|
-
return self._render_placeholder(placeholders) + self.suffix
|
4246
4220
|
|
4221
|
+
@dc.dataclass(frozen=True)
|
4222
|
+
class Platform(abc.ABC): # noqa
|
4223
|
+
pass
|
4247
4224
|
|
4248
|
-
##
|
4249
4225
|
|
4226
|
+
class LinuxPlatform(Platform, abc.ABC):
|
4227
|
+
pass
|
4250
4228
|
|
4251
|
-
@dc.dataclass(frozen=True)
|
4252
|
-
class DeployPath:
|
4253
|
-
parts: ta.Sequence[DeployPathPart]
|
4254
4229
|
|
4255
|
-
|
4256
|
-
|
4230
|
+
class UbuntuPlatform(LinuxPlatform):
|
4231
|
+
pass
|
4257
4232
|
|
4258
|
-
check.not_empty(self.parts)
|
4259
|
-
for p in self.parts[:-1]:
|
4260
|
-
check.equal(p.kind, 'dir')
|
4261
4233
|
|
4262
|
-
|
4263
|
-
|
4264
|
-
if isinstance(p, PlaceholderDeployPathPart):
|
4265
|
-
if p.placeholder in pd:
|
4266
|
-
raise DeployPathError('Duplicate placeholders in path', self)
|
4267
|
-
pd[p.placeholder] = i
|
4234
|
+
class AmazonLinuxPlatform(LinuxPlatform):
|
4235
|
+
pass
|
4268
4236
|
|
4269
|
-
if 'tag' in pd:
|
4270
|
-
if 'app' not in pd or pd['app'] >= pd['tag']:
|
4271
|
-
raise DeployPathError('Tag placeholder in path without preceding app', self)
|
4272
4237
|
|
4273
|
-
|
4274
|
-
|
4275
|
-
return self.parts[-1].kind
|
4238
|
+
class GenericLinuxPlatform(LinuxPlatform):
|
4239
|
+
pass
|
4276
4240
|
|
4277
|
-
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
4278
|
-
return os.path.join( # noqa
|
4279
|
-
*[p.render(placeholders) for p in self.parts],
|
4280
|
-
*([''] if self.kind == 'dir' else []),
|
4281
|
-
)
|
4282
4241
|
|
4283
|
-
|
4284
|
-
|
4285
|
-
tail_parse: ta.Callable[[str], DeployPathPart]
|
4286
|
-
if s.endswith('/'):
|
4287
|
-
tail_parse = DirDeployPathPart.parse
|
4288
|
-
s = s[:-1]
|
4289
|
-
else:
|
4290
|
-
tail_parse = FileDeployPathPart.parse
|
4291
|
-
ps = check.non_empty_str(s).split('/')
|
4292
|
-
return cls((
|
4293
|
-
*([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
|
4294
|
-
tail_parse(ps[-1]),
|
4295
|
-
))
|
4242
|
+
class DarwinPlatform(Platform):
|
4243
|
+
pass
|
4296
4244
|
|
4297
4245
|
|
4298
|
-
|
4246
|
+
class UnknownPlatform(Platform):
|
4247
|
+
pass
|
4299
4248
|
|
4300
4249
|
|
4301
|
-
|
4302
|
-
@abc.abstractmethod
|
4303
|
-
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
4304
|
-
raise NotImplementedError
|
4250
|
+
##
|
4305
4251
|
|
4306
4252
|
|
4307
|
-
|
4253
|
+
def _detect_system_platform() -> Platform:
|
4254
|
+
plat = sys.platform
|
4308
4255
|
|
4256
|
+
if plat == 'linux':
|
4257
|
+
if (osr := LinuxOsRelease.read()) is None:
|
4258
|
+
return GenericLinuxPlatform()
|
4309
4259
|
|
4310
|
-
|
4311
|
-
|
4312
|
-
self,
|
4313
|
-
*args: ta.Any,
|
4314
|
-
owned_dir: str,
|
4315
|
-
deploy_home: ta.Optional[DeployHome],
|
4316
|
-
**kwargs: ta.Any,
|
4317
|
-
) -> None:
|
4318
|
-
super().__init__(*args, **kwargs)
|
4260
|
+
if osr.id == 'amzn':
|
4261
|
+
return AmazonLinuxPlatform()
|
4319
4262
|
|
4320
|
-
|
4321
|
-
|
4263
|
+
elif osr.id == 'ubuntu':
|
4264
|
+
return UbuntuPlatform()
|
4322
4265
|
|
4323
|
-
|
4266
|
+
else:
|
4267
|
+
return GenericLinuxPlatform()
|
4324
4268
|
|
4325
|
-
|
4269
|
+
elif plat == 'darwin':
|
4270
|
+
return DarwinPlatform()
|
4326
4271
|
|
4327
|
-
|
4328
|
-
|
4329
|
-
return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
|
4272
|
+
else:
|
4273
|
+
return UnknownPlatform()
|
4330
4274
|
|
4331
|
-
@cached_nullary
|
4332
|
-
def _make_dir(self) -> str:
|
4333
|
-
if not os.path.isdir(d := self._dir()):
|
4334
|
-
os.makedirs(d, exist_ok=True)
|
4335
|
-
return d
|
4336
4275
|
|
4337
|
-
|
4338
|
-
|
4276
|
+
@cached_nullary
|
4277
|
+
def detect_system_platform() -> Platform:
|
4278
|
+
platform = _detect_system_platform()
|
4279
|
+
log.info('Detected platform: %r', platform)
|
4280
|
+
return platform
|
4339
4281
|
|
4340
4282
|
|
4341
4283
|
########################################
|
4342
|
-
# ../
|
4284
|
+
# ../targets/targets.py
|
4285
|
+
"""
|
4286
|
+
It's desugaring. Subprocess and locals are only leafs. Retain an origin?
|
4287
|
+
** TWO LAYERS ** - ManageTarget is user-facing, ConnectorTarget is bound, internal
|
4288
|
+
"""
|
4343
4289
|
|
4344
4290
|
|
4345
4291
|
##
|
4346
4292
|
|
4347
4293
|
|
4348
|
-
|
4349
|
-
|
4350
|
-
|
4351
|
-
check.not_in(c, s)
|
4352
|
-
check.arg(not s.startswith('/'))
|
4353
|
-
return s
|
4294
|
+
class ManageTarget(abc.ABC): # noqa
|
4295
|
+
def __init_subclass__(cls, **kwargs: ta.Any) -> None:
|
4296
|
+
super().__init_subclass__(**kwargs)
|
4354
4297
|
|
4298
|
+
check.state(cls.__name__.endswith('ManageTarget'))
|
4355
4299
|
|
4356
|
-
|
4300
|
+
|
4301
|
+
#
|
4357
4302
|
|
4358
4303
|
|
4359
4304
|
@dc.dataclass(frozen=True)
|
4360
|
-
class
|
4361
|
-
|
4362
|
-
|
4363
|
-
path: ta.Optional[str] = None
|
4305
|
+
class PythonRemoteManageTarget:
|
4306
|
+
DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
|
4307
|
+
python: str = DEFAULT_PYTHON
|
4364
4308
|
|
4365
|
-
def __post_init__(self) -> None:
|
4366
|
-
check.not_in('..', check.non_empty_str(self.host))
|
4367
|
-
check.not_in('.', check.non_empty_str(self.path))
|
4368
4309
|
|
4310
|
+
#
|
4369
4311
|
|
4370
|
-
@dc.dataclass(frozen=True)
|
4371
|
-
class DeployGitSpec:
|
4372
|
-
repo: DeployGitRepo
|
4373
|
-
rev: DeployRev
|
4374
4312
|
|
4375
|
-
|
4376
|
-
|
4377
|
-
def __post_init__(self) -> None:
|
4378
|
-
hash(self)
|
4379
|
-
check.non_empty_str(self.rev)
|
4380
|
-
if self.subtrees is not None:
|
4381
|
-
for st in self.subtrees:
|
4382
|
-
check.non_empty_str(st)
|
4383
|
-
|
4384
|
-
|
4385
|
-
##
|
4386
|
-
|
4387
|
-
|
4388
|
-
@dc.dataclass(frozen=True)
|
4389
|
-
class DeployVenvSpec:
|
4390
|
-
interp: ta.Optional[str] = None
|
4391
|
-
|
4392
|
-
requirements_files: ta.Optional[ta.Sequence[str]] = None
|
4393
|
-
extra_dependencies: ta.Optional[ta.Sequence[str]] = None
|
4394
|
-
|
4395
|
-
use_uv: bool = False
|
4396
|
-
|
4397
|
-
|
4398
|
-
##
|
4399
|
-
|
4400
|
-
|
4401
|
-
@dc.dataclass(frozen=True)
|
4402
|
-
class DeployConfFile:
|
4403
|
-
path: str
|
4404
|
-
body: str
|
4405
|
-
|
4406
|
-
def __post_init__(self) -> None:
|
4407
|
-
check_valid_deploy_spec_path(self.path)
|
4408
|
-
|
4409
|
-
|
4410
|
-
#
|
4411
|
-
|
4412
|
-
|
4413
|
-
@dc.dataclass(frozen=True)
|
4414
|
-
class DeployConfLink(abc.ABC): # noqa
|
4415
|
-
"""
|
4416
|
-
May be either:
|
4417
|
-
- @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
|
4418
|
-
- @conf/file - links a single file in a single subdir to conf/@conf/@dst-file
|
4419
|
-
- @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
|
4420
|
-
"""
|
4421
|
-
|
4422
|
-
src: str
|
4423
|
-
|
4424
|
-
def __post_init__(self) -> None:
|
4425
|
-
check_valid_deploy_spec_path(self.src)
|
4426
|
-
if '/' in self.src:
|
4427
|
-
check.equal(self.src.count('/'), 1)
|
4428
|
-
|
4429
|
-
|
4430
|
-
class AppDeployConfLink(DeployConfLink):
|
4431
|
-
pass
|
4432
|
-
|
4433
|
-
|
4434
|
-
class TagDeployConfLink(DeployConfLink):
|
4435
|
-
pass
|
4436
|
-
|
4437
|
-
|
4438
|
-
#
|
4439
|
-
|
4440
|
-
|
4441
|
-
@dc.dataclass(frozen=True)
|
4442
|
-
class DeployConfSpec:
|
4443
|
-
files: ta.Optional[ta.Sequence[DeployConfFile]] = None
|
4444
|
-
|
4445
|
-
links: ta.Optional[ta.Sequence[DeployConfLink]] = None
|
4446
|
-
|
4447
|
-
def __post_init__(self) -> None:
|
4448
|
-
if self.files:
|
4449
|
-
seen: ta.Set[str] = set()
|
4450
|
-
for f in self.files:
|
4451
|
-
check.not_in(f.path, seen)
|
4452
|
-
seen.add(f.path)
|
4453
|
-
|
4454
|
-
|
4455
|
-
##
|
4456
|
-
|
4457
|
-
|
4458
|
-
@dc.dataclass(frozen=True)
|
4459
|
-
class DeploySpec:
|
4460
|
-
app: DeployApp
|
4461
|
-
|
4462
|
-
git: DeployGitSpec
|
4463
|
-
|
4464
|
-
venv: ta.Optional[DeployVenvSpec] = None
|
4465
|
-
|
4466
|
-
conf: ta.Optional[DeployConfSpec] = None
|
4467
|
-
|
4468
|
-
@cached_nullary
|
4469
|
-
def key(self) -> DeployKey:
|
4470
|
-
return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
|
4471
|
-
|
4472
|
-
|
4473
|
-
########################################
|
4474
|
-
# ../remote/config.py
|
4475
|
-
|
4476
|
-
|
4477
|
-
@dc.dataclass(frozen=True)
|
4478
|
-
class RemoteConfig:
|
4479
|
-
payload_file: ta.Optional[str] = None
|
4480
|
-
|
4481
|
-
set_pgid: bool = True
|
4482
|
-
|
4483
|
-
deathsig: ta.Optional[str] = 'KILL'
|
4484
|
-
|
4485
|
-
pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
|
4486
|
-
|
4487
|
-
forward_logging: bool = True
|
4488
|
-
|
4489
|
-
timebomb_delay_s: ta.Optional[float] = 60 * 60.
|
4490
|
-
|
4491
|
-
heartbeat_interval_s: float = 3.
|
4492
|
-
|
4493
|
-
|
4494
|
-
########################################
|
4495
|
-
# ../remote/payload.py
|
4496
|
-
|
4497
|
-
|
4498
|
-
RemoteExecutionPayloadFile = ta.NewType('RemoteExecutionPayloadFile', str)
|
4499
|
-
|
4500
|
-
|
4501
|
-
@cached_nullary
|
4502
|
-
def _get_self_src() -> str:
|
4503
|
-
return inspect.getsource(sys.modules[__name__])
|
4504
|
-
|
4505
|
-
|
4506
|
-
def _is_src_amalg(src: str) -> bool:
|
4507
|
-
for l in src.splitlines(): # noqa
|
4508
|
-
if l.startswith('# @omlish-amalg-output '):
|
4509
|
-
return True
|
4510
|
-
return False
|
4511
|
-
|
4512
|
-
|
4513
|
-
@cached_nullary
|
4514
|
-
def _is_self_amalg() -> bool:
|
4515
|
-
return _is_src_amalg(_get_self_src())
|
4516
|
-
|
4517
|
-
|
4518
|
-
def get_remote_payload_src(
|
4519
|
-
*,
|
4520
|
-
file: ta.Optional[RemoteExecutionPayloadFile],
|
4521
|
-
) -> str:
|
4522
|
-
if file is not None:
|
4523
|
-
with open(file) as f:
|
4524
|
-
return f.read()
|
4525
|
-
|
4526
|
-
if _is_self_amalg():
|
4527
|
-
return _get_self_src()
|
4528
|
-
|
4529
|
-
import importlib.resources
|
4530
|
-
return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
|
4531
|
-
|
4532
|
-
|
4533
|
-
########################################
|
4534
|
-
# ../system/platforms.py
|
4535
|
-
|
4536
|
-
|
4537
|
-
##
|
4538
|
-
|
4539
|
-
|
4540
|
-
@dc.dataclass(frozen=True)
|
4541
|
-
class Platform(abc.ABC): # noqa
|
4542
|
-
pass
|
4543
|
-
|
4544
|
-
|
4545
|
-
class LinuxPlatform(Platform, abc.ABC):
|
4546
|
-
pass
|
4547
|
-
|
4548
|
-
|
4549
|
-
class UbuntuPlatform(LinuxPlatform):
|
4550
|
-
pass
|
4551
|
-
|
4552
|
-
|
4553
|
-
class AmazonLinuxPlatform(LinuxPlatform):
|
4554
|
-
pass
|
4555
|
-
|
4556
|
-
|
4557
|
-
class GenericLinuxPlatform(LinuxPlatform):
|
4558
|
-
pass
|
4559
|
-
|
4560
|
-
|
4561
|
-
class DarwinPlatform(Platform):
|
4562
|
-
pass
|
4563
|
-
|
4564
|
-
|
4565
|
-
class UnknownPlatform(Platform):
|
4566
|
-
pass
|
4567
|
-
|
4568
|
-
|
4569
|
-
##
|
4570
|
-
|
4571
|
-
|
4572
|
-
def _detect_system_platform() -> Platform:
|
4573
|
-
plat = sys.platform
|
4574
|
-
|
4575
|
-
if plat == 'linux':
|
4576
|
-
if (osr := LinuxOsRelease.read()) is None:
|
4577
|
-
return GenericLinuxPlatform()
|
4578
|
-
|
4579
|
-
if osr.id == 'amzn':
|
4580
|
-
return AmazonLinuxPlatform()
|
4581
|
-
|
4582
|
-
elif osr.id == 'ubuntu':
|
4583
|
-
return UbuntuPlatform()
|
4584
|
-
|
4585
|
-
else:
|
4586
|
-
return GenericLinuxPlatform()
|
4587
|
-
|
4588
|
-
elif plat == 'darwin':
|
4589
|
-
return DarwinPlatform()
|
4590
|
-
|
4591
|
-
else:
|
4592
|
-
return UnknownPlatform()
|
4593
|
-
|
4594
|
-
|
4595
|
-
@cached_nullary
|
4596
|
-
def detect_system_platform() -> Platform:
|
4597
|
-
platform = _detect_system_platform()
|
4598
|
-
log.info('Detected platform: %r', platform)
|
4599
|
-
return platform
|
4600
|
-
|
4601
|
-
|
4602
|
-
########################################
|
4603
|
-
# ../targets/targets.py
|
4604
|
-
"""
|
4605
|
-
It's desugaring. Subprocess and locals are only leafs. Retain an origin?
|
4606
|
-
** TWO LAYERS ** - ManageTarget is user-facing, ConnectorTarget is bound, internal
|
4607
|
-
"""
|
4608
|
-
|
4609
|
-
|
4610
|
-
##
|
4611
|
-
|
4612
|
-
|
4613
|
-
class ManageTarget(abc.ABC): # noqa
|
4614
|
-
def __init_subclass__(cls, **kwargs: ta.Any) -> None:
|
4615
|
-
super().__init_subclass__(**kwargs)
|
4616
|
-
|
4617
|
-
check.state(cls.__name__.endswith('ManageTarget'))
|
4618
|
-
|
4619
|
-
|
4620
|
-
#
|
4621
|
-
|
4622
|
-
|
4623
|
-
@dc.dataclass(frozen=True)
|
4624
|
-
class PythonRemoteManageTarget:
|
4625
|
-
DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
|
4626
|
-
python: str = DEFAULT_PYTHON
|
4627
|
-
|
4628
|
-
|
4629
|
-
#
|
4630
|
-
|
4631
|
-
|
4632
|
-
class RemoteManageTarget(ManageTarget, abc.ABC):
|
4633
|
-
pass
|
4313
|
+
class RemoteManageTarget(ManageTarget, abc.ABC):
|
4314
|
+
pass
|
4634
4315
|
|
4635
4316
|
|
4636
4317
|
class PhysicallyRemoteManageTarget(RemoteManageTarget, abc.ABC):
|
@@ -6859,172 +6540,343 @@ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command
|
|
6859
6540
|
|
6860
6541
|
|
6861
6542
|
########################################
|
6862
|
-
# ../deploy/
|
6543
|
+
# ../deploy/paths/paths.py
|
6863
6544
|
"""
|
6864
6545
|
TODO:
|
6865
|
-
-
|
6866
|
-
-
|
6867
|
-
|
6868
|
-
|
6869
|
-
-
|
6870
|
-
|
6871
|
-
- 2) for each subdir not needing modification, hardlink into temp dir
|
6872
|
-
- 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
|
6873
|
-
- 4) write (or if deleting, omit) new files
|
6874
|
-
- 5) swap top level
|
6875
|
-
- ** whole deploy can be atomic(-ish) - do this for everything **
|
6876
|
-
- just a '/deploy/current' dir
|
6877
|
-
- some things (venvs) cannot be moved, thus the /deploy/venvs dir
|
6878
|
-
- ** ensure (enforce) equivalent relpath nesting
|
6546
|
+
- run/{.pid,.sock}
|
6547
|
+
- logs/...
|
6548
|
+
- current symlink
|
6549
|
+
- conf/{nginx,supervisor}
|
6550
|
+
- env/?
|
6551
|
+
- apps/<app>/shared
|
6879
6552
|
"""
|
6880
6553
|
|
6881
6554
|
|
6882
|
-
|
6883
|
-
def __init__(
|
6884
|
-
self,
|
6885
|
-
*,
|
6886
|
-
deploy_home: ta.Optional[DeployHome] = None,
|
6887
|
-
) -> None:
|
6888
|
-
super().__init__(
|
6889
|
-
owned_dir='conf',
|
6890
|
-
deploy_home=deploy_home,
|
6891
|
-
)
|
6555
|
+
##
|
6892
6556
|
|
6893
|
-
async def _write_conf_file(
|
6894
|
-
self,
|
6895
|
-
cf: DeployConfFile,
|
6896
|
-
conf_dir: str,
|
6897
|
-
) -> None:
|
6898
|
-
conf_file = os.path.join(conf_dir, cf.path)
|
6899
|
-
check.arg(is_path_in_dir(conf_dir, conf_file))
|
6900
6557
|
|
6901
|
-
|
6558
|
+
DEPLOY_PATH_PLACEHOLDER_SIGIL = '@'
|
6559
|
+
DEPLOY_PATH_PLACEHOLDER_SEPARATOR = '--'
|
6902
6560
|
|
6903
|
-
|
6904
|
-
|
6561
|
+
DEPLOY_PATH_PLACEHOLDER_DELIMITERS: ta.AbstractSet[str] = frozenset([
|
6562
|
+
DEPLOY_PATH_PLACEHOLDER_SEPARATOR,
|
6563
|
+
'.',
|
6564
|
+
])
|
6905
6565
|
|
6906
|
-
|
6907
|
-
|
6908
|
-
|
6909
|
-
|
6910
|
-
|
6911
|
-
link_dir: str,
|
6912
|
-
) -> None:
|
6913
|
-
link_src = os.path.join(conf_dir, link.src)
|
6914
|
-
check.arg(is_path_in_dir(conf_dir, link_src))
|
6566
|
+
DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
|
6567
|
+
'app',
|
6568
|
+
'tag',
|
6569
|
+
'conf',
|
6570
|
+
])
|
6915
6571
|
|
6916
|
-
is_link_dir = link.src.endswith('/')
|
6917
|
-
if is_link_dir:
|
6918
|
-
check.arg(link.src.count('/') == 1)
|
6919
|
-
check.arg(os.path.isdir(link_src))
|
6920
|
-
link_dst_pfx = link.src
|
6921
|
-
link_dst_sfx = ''
|
6922
6572
|
|
6923
|
-
|
6924
|
-
|
6925
|
-
|
6926
|
-
|
6927
|
-
|
6928
|
-
|
6573
|
+
class DeployPathError(Exception):
|
6574
|
+
pass
|
6575
|
+
|
6576
|
+
|
6577
|
+
class DeployPathRenderable(abc.ABC):
|
6578
|
+
@abc.abstractmethod
|
6579
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
6580
|
+
raise NotImplementedError
|
6581
|
+
|
6582
|
+
|
6583
|
+
##
|
6929
6584
|
|
6585
|
+
|
6586
|
+
class DeployPathNamePart(DeployPathRenderable, abc.ABC):
|
6587
|
+
@classmethod
|
6588
|
+
def parse(cls, s: str) -> 'DeployPathNamePart':
|
6589
|
+
check.non_empty_str(s)
|
6590
|
+
if s.startswith(DEPLOY_PATH_PLACEHOLDER_SIGIL):
|
6591
|
+
return PlaceholderDeployPathNamePart(s[1:])
|
6592
|
+
elif s in DEPLOY_PATH_PLACEHOLDER_DELIMITERS:
|
6593
|
+
return DelimiterDeployPathNamePart(s)
|
6930
6594
|
else:
|
6931
|
-
|
6932
|
-
|
6933
|
-
|
6934
|
-
|
6935
|
-
|
6936
|
-
|
6937
|
-
|
6938
|
-
|
6595
|
+
return ConstDeployPathNamePart(s)
|
6596
|
+
|
6597
|
+
|
6598
|
+
@dc.dataclass(frozen=True)
|
6599
|
+
class PlaceholderDeployPathNamePart(DeployPathNamePart):
|
6600
|
+
placeholder: str # DeployPathPlaceholder
|
6601
|
+
|
6602
|
+
def __post_init__(self) -> None:
|
6603
|
+
check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
|
6604
|
+
|
6605
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
6606
|
+
if placeholders is not None:
|
6607
|
+
return placeholders[self.placeholder] # type: ignore
|
6608
|
+
else:
|
6609
|
+
return DEPLOY_PATH_PLACEHOLDER_SIGIL + self.placeholder
|
6610
|
+
|
6611
|
+
|
6612
|
+
@dc.dataclass(frozen=True)
|
6613
|
+
class DelimiterDeployPathNamePart(DeployPathNamePart):
|
6614
|
+
delimiter: str
|
6615
|
+
|
6616
|
+
def __post_init__(self) -> None:
|
6617
|
+
check.in_(self.delimiter, DEPLOY_PATH_PLACEHOLDER_DELIMITERS)
|
6618
|
+
|
6619
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
6620
|
+
return self.delimiter
|
6621
|
+
|
6622
|
+
|
6623
|
+
@dc.dataclass(frozen=True)
|
6624
|
+
class ConstDeployPathNamePart(DeployPathNamePart):
|
6625
|
+
const: str
|
6626
|
+
|
6627
|
+
def __post_init__(self) -> None:
|
6628
|
+
check.non_empty_str(self.const)
|
6629
|
+
for c in [*DEPLOY_PATH_PLACEHOLDER_DELIMITERS, DEPLOY_PATH_PLACEHOLDER_SIGIL, '/']:
|
6630
|
+
check.not_in(c, self.const)
|
6631
|
+
|
6632
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
6633
|
+
return self.const
|
6634
|
+
|
6635
|
+
|
6636
|
+
@dc.dataclass(frozen=True)
|
6637
|
+
class DeployPathName(DeployPathRenderable):
|
6638
|
+
parts: ta.Sequence[DeployPathNamePart]
|
6639
|
+
|
6640
|
+
def __post_init__(self) -> None:
|
6641
|
+
hash(self)
|
6642
|
+
check.not_empty(self.parts)
|
6643
|
+
for k, g in itertools.groupby(self.parts, type):
|
6644
|
+
if len(gl := list(g)) > 1:
|
6645
|
+
raise DeployPathError(f'May not have consecutive path name part types: {k} {gl}')
|
6646
|
+
|
6647
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
6648
|
+
return ''.join(p.render(placeholders) for p in self.parts)
|
6649
|
+
|
6650
|
+
@classmethod
|
6651
|
+
def parse(cls, s: str) -> 'DeployPathName':
|
6652
|
+
check.non_empty_str(s)
|
6653
|
+
check.not_in('/', s)
|
6654
|
+
|
6655
|
+
i = 0
|
6656
|
+
ps = []
|
6657
|
+
while i < len(s):
|
6658
|
+
ns = [(n, d) for d in DEPLOY_PATH_PLACEHOLDER_DELIMITERS if (n := s.find(d, i)) >= 0]
|
6659
|
+
if not ns:
|
6660
|
+
ps.append(s[i:])
|
6661
|
+
break
|
6662
|
+
n, d = min(ns)
|
6663
|
+
ps.append(check.non_empty_str(s[i:n]))
|
6664
|
+
ps.append(s[n:n + len(d)])
|
6665
|
+
i = n + len(d)
|
6666
|
+
|
6667
|
+
return cls(tuple(DeployPathNamePart.parse(p) for p in ps))
|
6668
|
+
|
6669
|
+
|
6670
|
+
##
|
6671
|
+
|
6672
|
+
|
6673
|
+
@dc.dataclass(frozen=True)
|
6674
|
+
class DeployPathPart(DeployPathRenderable, abc.ABC): # noqa
|
6675
|
+
name: DeployPathName
|
6676
|
+
|
6677
|
+
@property
|
6678
|
+
@abc.abstractmethod
|
6679
|
+
def kind(self) -> DeployPathKind:
|
6680
|
+
raise NotImplementedError
|
6681
|
+
|
6682
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
6683
|
+
return self.name.render(placeholders) + ('/' if self.kind == 'dir' else '')
|
6684
|
+
|
6685
|
+
@classmethod
|
6686
|
+
def parse(cls, s: str) -> 'DeployPathPart':
|
6687
|
+
if (is_dir := s.endswith('/')):
|
6688
|
+
s = s[:-1]
|
6689
|
+
check.non_empty_str(s)
|
6690
|
+
check.not_in('/', s)
|
6691
|
+
|
6692
|
+
n = DeployPathName.parse(s)
|
6693
|
+
if is_dir:
|
6694
|
+
return DirDeployPathPart(n)
|
6695
|
+
else:
|
6696
|
+
return FileDeployPathPart(n)
|
6697
|
+
|
6698
|
+
|
6699
|
+
class DirDeployPathPart(DeployPathPart):
|
6700
|
+
@property
|
6701
|
+
def kind(self) -> DeployPathKind:
|
6702
|
+
return 'dir'
|
6703
|
+
|
6704
|
+
|
6705
|
+
class FileDeployPathPart(DeployPathPart):
|
6706
|
+
@property
|
6707
|
+
def kind(self) -> DeployPathKind:
|
6708
|
+
return 'file'
|
6709
|
+
|
6710
|
+
|
6711
|
+
#
|
6712
|
+
|
6713
|
+
|
6714
|
+
@dc.dataclass(frozen=True)
|
6715
|
+
class DeployPath:
|
6716
|
+
parts: ta.Sequence[DeployPathPart]
|
6717
|
+
|
6718
|
+
@property
|
6719
|
+
def name_parts(self) -> ta.Iterator[DeployPathNamePart]:
|
6720
|
+
for p in self.parts:
|
6721
|
+
yield from p.name.parts
|
6722
|
+
|
6723
|
+
def __post_init__(self) -> None:
|
6724
|
+
hash(self)
|
6725
|
+
check.not_empty(self.parts)
|
6726
|
+
for p in self.parts[:-1]:
|
6727
|
+
check.equal(p.kind, 'dir')
|
6728
|
+
|
6729
|
+
pd: ta.Dict[DeployPathPlaceholder, ta.List[int]] = {}
|
6730
|
+
for i, np in enumerate(self.name_parts):
|
6731
|
+
if isinstance(np, PlaceholderDeployPathNamePart):
|
6732
|
+
pd.setdefault(ta.cast(DeployPathPlaceholder, np.placeholder), []).append(i)
|
6733
|
+
|
6734
|
+
# if 'tag' in pd and 'app' not in pd:
|
6735
|
+
# raise DeployPathError('Tag placeholder in path without app', self)
|
6736
|
+
|
6737
|
+
@property
|
6738
|
+
def kind(self) -> ta.Literal['file', 'dir']:
|
6739
|
+
return self.parts[-1].kind
|
6740
|
+
|
6741
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
6742
|
+
return ''.join([p.render(placeholders) for p in self.parts])
|
6743
|
+
|
6744
|
+
@classmethod
|
6745
|
+
def parse(cls, s: str) -> 'DeployPath':
|
6746
|
+
check.non_empty_str(s)
|
6747
|
+
ps = split_keep_delimiter(s, '/')
|
6748
|
+
return cls(tuple(DeployPathPart.parse(p) for p in ps))
|
6749
|
+
|
6750
|
+
|
6751
|
+
########################################
|
6752
|
+
# ../deploy/specs.py
|
6753
|
+
|
6754
|
+
|
6755
|
+
##
|
6756
|
+
|
6757
|
+
|
6758
|
+
def check_valid_deploy_spec_path(s: str) -> str:
|
6759
|
+
check.non_empty_str(s)
|
6760
|
+
for c in ['..', '//']:
|
6761
|
+
check.not_in(c, s)
|
6762
|
+
check.arg(not s.startswith('/'))
|
6763
|
+
return s
|
6764
|
+
|
6765
|
+
|
6766
|
+
##
|
6767
|
+
|
6768
|
+
|
6769
|
+
@dc.dataclass(frozen=True)
|
6770
|
+
class DeployGitRepo:
|
6771
|
+
host: ta.Optional[str] = None
|
6772
|
+
username: ta.Optional[str] = None
|
6773
|
+
path: ta.Optional[str] = None
|
6774
|
+
|
6775
|
+
def __post_init__(self) -> None:
|
6776
|
+
check.not_in('..', check.non_empty_str(self.host))
|
6777
|
+
check.not_in('.', check.non_empty_str(self.path))
|
6778
|
+
|
6779
|
+
|
6780
|
+
@dc.dataclass(frozen=True)
|
6781
|
+
class DeployGitSpec:
|
6782
|
+
repo: DeployGitRepo
|
6783
|
+
rev: DeployRev
|
6784
|
+
|
6785
|
+
subtrees: ta.Optional[ta.Sequence[str]] = None
|
6786
|
+
|
6787
|
+
def __post_init__(self) -> None:
|
6788
|
+
check.non_empty_str(self.rev)
|
6789
|
+
if self.subtrees is not None:
|
6790
|
+
for st in self.subtrees:
|
6791
|
+
check.non_empty_str(st)
|
6792
|
+
|
6793
|
+
|
6794
|
+
##
|
6795
|
+
|
6796
|
+
|
6797
|
+
@dc.dataclass(frozen=True)
|
6798
|
+
class DeployVenvSpec:
|
6799
|
+
interp: ta.Optional[str] = None
|
6800
|
+
|
6801
|
+
requirements_files: ta.Optional[ta.Sequence[str]] = None
|
6802
|
+
extra_dependencies: ta.Optional[ta.Sequence[str]] = None
|
6803
|
+
|
6804
|
+
use_uv: bool = False
|
6805
|
+
|
6806
|
+
|
6807
|
+
##
|
6808
|
+
|
6809
|
+
|
6810
|
+
@dc.dataclass(frozen=True)
|
6811
|
+
class DeployConfFile:
|
6812
|
+
path: str
|
6813
|
+
body: str
|
6814
|
+
|
6815
|
+
def __post_init__(self) -> None:
|
6816
|
+
check_valid_deploy_spec_path(self.path)
|
6817
|
+
|
6818
|
+
|
6819
|
+
#
|
6820
|
+
|
6821
|
+
|
6822
|
+
@dc.dataclass(frozen=True)
|
6823
|
+
class DeployConfLink(abc.ABC): # noqa
|
6824
|
+
"""
|
6825
|
+
May be either:
|
6826
|
+
- @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
|
6827
|
+
- @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
|
6828
|
+
- @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
|
6829
|
+
"""
|
6830
|
+
|
6831
|
+
src: str
|
6832
|
+
|
6833
|
+
def __post_init__(self) -> None:
|
6834
|
+
check_valid_deploy_spec_path(self.src)
|
6835
|
+
if '/' in self.src:
|
6836
|
+
check.equal(self.src.count('/'), 1)
|
6837
|
+
|
6838
|
+
|
6839
|
+
class AppDeployConfLink(DeployConfLink):
|
6840
|
+
pass
|
6841
|
+
|
6842
|
+
|
6843
|
+
class TagDeployConfLink(DeployConfLink):
|
6844
|
+
pass
|
6939
6845
|
|
6940
|
-
if isinstance(link, AppDeployConfLink):
|
6941
|
-
link_dst_mid = str(app_tag.app)
|
6942
|
-
sym_root = link_dir
|
6943
|
-
elif isinstance(link, TagDeployConfLink):
|
6944
|
-
link_dst_mid = '-'.join([app_tag.app, app_tag.tag])
|
6945
|
-
sym_root = conf_dir
|
6946
|
-
else:
|
6947
|
-
raise TypeError(link)
|
6948
6846
|
|
6949
|
-
|
6950
|
-
link_dst_pfx,
|
6951
|
-
link_dst_mid,
|
6952
|
-
link_dst_sfx,
|
6953
|
-
])
|
6847
|
+
#
|
6954
6848
|
|
6955
|
-
root_conf_dir = self._make_dir()
|
6956
|
-
sym_src = os.path.join(sym_root, link.src)
|
6957
|
-
sym_dst = os.path.join(root_conf_dir, link_dst)
|
6958
|
-
check.arg(is_path_in_dir(root_conf_dir, sym_dst))
|
6959
6849
|
|
6960
|
-
|
6961
|
-
|
6850
|
+
@dc.dataclass(frozen=True)
|
6851
|
+
class DeployConfSpec:
|
6852
|
+
files: ta.Optional[ta.Sequence[DeployConfFile]] = None
|
6962
6853
|
|
6963
|
-
|
6964
|
-
self,
|
6965
|
-
spec: DeployConfSpec,
|
6966
|
-
conf_dir: str,
|
6967
|
-
app_tag: DeployAppTag,
|
6968
|
-
link_dir: str,
|
6969
|
-
) -> None:
|
6970
|
-
conf_dir = os.path.abspath(conf_dir)
|
6971
|
-
os.makedirs(conf_dir)
|
6854
|
+
links: ta.Optional[ta.Sequence[DeployConfLink]] = None
|
6972
6855
|
|
6973
|
-
|
6856
|
+
def __post_init__(self) -> None:
|
6857
|
+
if self.files:
|
6858
|
+
seen: ta.Set[str] = set()
|
6859
|
+
for f in self.files:
|
6860
|
+
check.not_in(f.path, seen)
|
6861
|
+
seen.add(f.path)
|
6974
6862
|
|
6975
|
-
for cf in spec.files or []:
|
6976
|
-
await self._write_conf_file(
|
6977
|
-
cf,
|
6978
|
-
conf_dir,
|
6979
|
-
)
|
6980
6863
|
|
6981
|
-
|
6864
|
+
##
|
6982
6865
|
|
6983
|
-
for link in spec.links or []:
|
6984
|
-
await self._make_conf_link(
|
6985
|
-
link,
|
6986
|
-
conf_dir,
|
6987
|
-
app_tag,
|
6988
|
-
link_dir,
|
6989
|
-
)
|
6990
6866
|
|
6867
|
+
@dc.dataclass(frozen=True)
|
6868
|
+
class DeploySpec:
|
6869
|
+
app: DeployApp
|
6991
6870
|
|
6992
|
-
|
6993
|
-
# ../deploy/tmp.py
|
6871
|
+
git: DeployGitSpec
|
6994
6872
|
|
6873
|
+
venv: ta.Optional[DeployVenvSpec] = None
|
6995
6874
|
|
6996
|
-
|
6997
|
-
SingleDirDeployPathOwner,
|
6998
|
-
AtomicPathSwapping,
|
6999
|
-
):
|
7000
|
-
def __init__(
|
7001
|
-
self,
|
7002
|
-
*,
|
7003
|
-
deploy_home: ta.Optional[DeployHome] = None,
|
7004
|
-
) -> None:
|
7005
|
-
super().__init__(
|
7006
|
-
owned_dir='tmp',
|
7007
|
-
deploy_home=deploy_home,
|
7008
|
-
)
|
6875
|
+
conf: ta.Optional[DeployConfSpec] = None
|
7009
6876
|
|
7010
6877
|
@cached_nullary
|
7011
|
-
def
|
7012
|
-
return
|
7013
|
-
temp_dir=self._make_dir(),
|
7014
|
-
root_dir=check.non_empty_str(self._deploy_home),
|
7015
|
-
)
|
7016
|
-
|
7017
|
-
def begin_atomic_path_swap(
|
7018
|
-
self,
|
7019
|
-
kind: AtomicPathSwapKind,
|
7020
|
-
dst_path: str,
|
7021
|
-
**kwargs: ta.Any,
|
7022
|
-
) -> AtomicPathSwap:
|
7023
|
-
return self._swapping().begin_atomic_path_swap(
|
7024
|
-
kind,
|
7025
|
-
dst_path,
|
7026
|
-
**kwargs,
|
7027
|
-
)
|
6878
|
+
def key(self) -> DeployKey:
|
6879
|
+
return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
|
7028
6880
|
|
7029
6881
|
|
7030
6882
|
########################################
|
@@ -7537,78 +7389,293 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
|
|
7537
7389
|
|
7538
7390
|
#
|
7539
7391
|
|
7540
|
-
async def check_output_str(
|
7541
|
-
self,
|
7542
|
-
*cmd: str,
|
7543
|
-
**kwargs: ta.Any,
|
7544
|
-
) -> str:
|
7545
|
-
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
7392
|
+
async def check_output_str(
|
7393
|
+
self,
|
7394
|
+
*cmd: str,
|
7395
|
+
**kwargs: ta.Any,
|
7396
|
+
) -> str:
|
7397
|
+
return (await self.check_output(*cmd, **kwargs)).decode().strip()
|
7398
|
+
|
7399
|
+
#
|
7400
|
+
|
7401
|
+
async def try_call(
|
7402
|
+
self,
|
7403
|
+
*cmd: str,
|
7404
|
+
**kwargs: ta.Any,
|
7405
|
+
) -> bool:
|
7406
|
+
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
7407
|
+
return False
|
7408
|
+
else:
|
7409
|
+
return True
|
7410
|
+
|
7411
|
+
async def try_output(
|
7412
|
+
self,
|
7413
|
+
*cmd: str,
|
7414
|
+
**kwargs: ta.Any,
|
7415
|
+
) -> ta.Optional[bytes]:
|
7416
|
+
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
7417
|
+
return None
|
7418
|
+
else:
|
7419
|
+
return ret
|
7420
|
+
|
7421
|
+
async def try_output_str(
|
7422
|
+
self,
|
7423
|
+
*cmd: str,
|
7424
|
+
**kwargs: ta.Any,
|
7425
|
+
) -> ta.Optional[str]:
|
7426
|
+
if (ret := await self.try_output(*cmd, **kwargs)) is None:
|
7427
|
+
return None
|
7428
|
+
else:
|
7429
|
+
return ret.decode().strip()
|
7430
|
+
|
7431
|
+
|
7432
|
+
########################################
|
7433
|
+
# ../bootstrap.py
|
7434
|
+
|
7435
|
+
|
7436
|
+
@dc.dataclass(frozen=True)
|
7437
|
+
class MainBootstrap:
|
7438
|
+
main_config: MainConfig = MainConfig()
|
7439
|
+
|
7440
|
+
deploy_config: DeployConfig = DeployConfig()
|
7441
|
+
|
7442
|
+
remote_config: RemoteConfig = RemoteConfig()
|
7443
|
+
|
7444
|
+
system_config: SystemConfig = SystemConfig()
|
7445
|
+
|
7446
|
+
|
7447
|
+
########################################
|
7448
|
+
# ../commands/local.py
|
7449
|
+
|
7450
|
+
|
7451
|
+
class LocalCommandExecutor(CommandExecutor):
|
7452
|
+
def __init__(
|
7453
|
+
self,
|
7454
|
+
*,
|
7455
|
+
command_executors: CommandExecutorMap,
|
7456
|
+
) -> None:
|
7457
|
+
super().__init__()
|
7458
|
+
|
7459
|
+
self._command_executors = command_executors
|
7460
|
+
|
7461
|
+
async def execute(self, cmd: Command) -> Command.Output:
|
7462
|
+
ce: CommandExecutor = self._command_executors[type(cmd)]
|
7463
|
+
return await ce.execute(cmd)
|
7464
|
+
|
7465
|
+
|
7466
|
+
########################################
|
7467
|
+
# ../deploy/conf.py
|
7468
|
+
"""
|
7469
|
+
TODO:
|
7470
|
+
- @conf DeployPathPlaceholder? :|
|
7471
|
+
- post-deploy: remove any dir_links not present in new spec
|
7472
|
+
- * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
|
7473
|
+
- no such thing as 'previously present'.. build a 'deploy state' and pass it back?
|
7474
|
+
- ** whole thing can be atomic **
|
7475
|
+
- 1) new atomic temp dir
|
7476
|
+
- 2) for each subdir not needing modification, hardlink into temp dir
|
7477
|
+
- 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
|
7478
|
+
- 4) write (or if deleting, omit) new files
|
7479
|
+
- 5) swap top level
|
7480
|
+
- ** whole deploy can be atomic(-ish) - do this for everything **
|
7481
|
+
- just a '/deploy/current' dir
|
7482
|
+
- some things (venvs) cannot be moved, thus the /deploy/venvs dir
|
7483
|
+
- ** ensure (enforce) equivalent relpath nesting
|
7484
|
+
"""
|
7485
|
+
|
7486
|
+
|
7487
|
+
class DeployConfManager:
|
7488
|
+
def __init__(
|
7489
|
+
self,
|
7490
|
+
*,
|
7491
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
7492
|
+
) -> None:
|
7493
|
+
super().__init__()
|
7494
|
+
|
7495
|
+
self._deploy_home = deploy_home
|
7496
|
+
|
7497
|
+
#
|
7498
|
+
|
7499
|
+
async def _write_conf_file(
|
7500
|
+
self,
|
7501
|
+
cf: DeployConfFile,
|
7502
|
+
conf_dir: str,
|
7503
|
+
) -> None:
|
7504
|
+
conf_file = os.path.join(conf_dir, cf.path)
|
7505
|
+
check.arg(is_path_in_dir(conf_dir, conf_file))
|
7506
|
+
|
7507
|
+
os.makedirs(os.path.dirname(conf_file), exist_ok=True)
|
7508
|
+
|
7509
|
+
with open(conf_file, 'w') as f: # noqa
|
7510
|
+
f.write(cf.body)
|
7511
|
+
|
7512
|
+
#
|
7513
|
+
|
7514
|
+
class _ComputedConfLink(ta.NamedTuple):
|
7515
|
+
is_dir: bool
|
7516
|
+
link_src: str
|
7517
|
+
link_dst: str
|
7518
|
+
|
7519
|
+
def _compute_conf_link_dst(
|
7520
|
+
self,
|
7521
|
+
link: DeployConfLink,
|
7522
|
+
app_tag: DeployAppTag,
|
7523
|
+
conf_dir: str,
|
7524
|
+
link_dir: str,
|
7525
|
+
) -> _ComputedConfLink:
|
7526
|
+
link_src = os.path.join(conf_dir, link.src)
|
7527
|
+
check.arg(is_path_in_dir(conf_dir, link_src))
|
7528
|
+
|
7529
|
+
#
|
7530
|
+
|
7531
|
+
if (is_dir := link.src.endswith('/')):
|
7532
|
+
# @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
|
7533
|
+
check.arg(link.src.count('/') == 1)
|
7534
|
+
link_dst_pfx = link.src
|
7535
|
+
link_dst_sfx = ''
|
7536
|
+
|
7537
|
+
elif '/' in link.src:
|
7538
|
+
# @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
|
7539
|
+
d, f = os.path.split(link.src)
|
7540
|
+
# TODO: check filename :|
|
7541
|
+
link_dst_pfx = d + '/'
|
7542
|
+
link_dst_sfx = DEPLOY_PATH_PLACEHOLDER_SEPARATOR + f
|
7543
|
+
|
7544
|
+
else: # noqa
|
7545
|
+
# @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
|
7546
|
+
if '.' in link.src:
|
7547
|
+
l, _, r = link.src.partition('.')
|
7548
|
+
link_dst_pfx = l + '/'
|
7549
|
+
link_dst_sfx = '.' + r
|
7550
|
+
else:
|
7551
|
+
link_dst_pfx = link.src + '/'
|
7552
|
+
link_dst_sfx = ''
|
7553
|
+
|
7554
|
+
#
|
7555
|
+
|
7556
|
+
if isinstance(link, AppDeployConfLink):
|
7557
|
+
link_dst_mid = str(app_tag.app)
|
7558
|
+
elif isinstance(link, TagDeployConfLink):
|
7559
|
+
link_dst_mid = DEPLOY_PATH_PLACEHOLDER_SEPARATOR.join([app_tag.app, app_tag.tag])
|
7560
|
+
else:
|
7561
|
+
raise TypeError(link)
|
7562
|
+
|
7563
|
+
#
|
7564
|
+
|
7565
|
+
link_dst_name = ''.join([
|
7566
|
+
link_dst_pfx,
|
7567
|
+
link_dst_mid,
|
7568
|
+
link_dst_sfx,
|
7569
|
+
])
|
7570
|
+
link_dst = os.path.join(link_dir, link_dst_name)
|
7571
|
+
|
7572
|
+
return DeployConfManager._ComputedConfLink(
|
7573
|
+
is_dir=is_dir,
|
7574
|
+
link_src=link_src,
|
7575
|
+
link_dst=link_dst,
|
7576
|
+
)
|
7577
|
+
|
7578
|
+
async def _make_conf_link(
|
7579
|
+
self,
|
7580
|
+
link: DeployConfLink,
|
7581
|
+
app_tag: DeployAppTag,
|
7582
|
+
conf_dir: str,
|
7583
|
+
link_dir: str,
|
7584
|
+
) -> None:
|
7585
|
+
comp = self._compute_conf_link_dst(
|
7586
|
+
link,
|
7587
|
+
app_tag,
|
7588
|
+
conf_dir,
|
7589
|
+
link_dir,
|
7590
|
+
)
|
7591
|
+
|
7592
|
+
#
|
7546
7593
|
|
7547
|
-
|
7594
|
+
check.arg(is_path_in_dir(conf_dir, comp.link_src))
|
7595
|
+
check.arg(is_path_in_dir(link_dir, comp.link_dst))
|
7548
7596
|
|
7549
|
-
|
7550
|
-
|
7551
|
-
*cmd: str,
|
7552
|
-
**kwargs: ta.Any,
|
7553
|
-
) -> bool:
|
7554
|
-
if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
|
7555
|
-
return False
|
7597
|
+
if comp.is_dir:
|
7598
|
+
check.arg(os.path.isdir(comp.link_src))
|
7556
7599
|
else:
|
7557
|
-
|
7600
|
+
check.arg(os.path.isfile(comp.link_src))
|
7558
7601
|
|
7559
|
-
|
7560
|
-
self,
|
7561
|
-
*cmd: str,
|
7562
|
-
**kwargs: ta.Any,
|
7563
|
-
) -> ta.Optional[bytes]:
|
7564
|
-
if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
|
7565
|
-
return None
|
7566
|
-
else:
|
7567
|
-
return ret
|
7602
|
+
#
|
7568
7603
|
|
7569
|
-
|
7570
|
-
|
7571
|
-
|
7572
|
-
|
7573
|
-
|
7574
|
-
|
7575
|
-
return None
|
7576
|
-
else:
|
7577
|
-
return ret.decode().strip()
|
7604
|
+
relative_symlink( # noqa
|
7605
|
+
comp.link_src,
|
7606
|
+
comp.link_dst,
|
7607
|
+
target_is_directory=comp.is_dir,
|
7608
|
+
make_dirs=True,
|
7609
|
+
)
|
7578
7610
|
|
7611
|
+
#
|
7579
7612
|
|
7580
|
-
|
7581
|
-
|
7613
|
+
async def write_conf(
|
7614
|
+
self,
|
7615
|
+
spec: DeployConfSpec,
|
7616
|
+
app_tag: DeployAppTag,
|
7617
|
+
conf_dir: str,
|
7618
|
+
link_dir: str,
|
7619
|
+
) -> None:
|
7620
|
+
for cf in spec.files or []:
|
7621
|
+
await self._write_conf_file(
|
7622
|
+
cf,
|
7623
|
+
conf_dir,
|
7624
|
+
)
|
7582
7625
|
|
7626
|
+
#
|
7583
7627
|
|
7584
|
-
|
7585
|
-
|
7586
|
-
|
7628
|
+
for link in spec.links or []:
|
7629
|
+
await self._make_conf_link(
|
7630
|
+
link,
|
7631
|
+
app_tag,
|
7632
|
+
conf_dir,
|
7633
|
+
link_dir,
|
7634
|
+
)
|
7587
7635
|
|
7588
|
-
deploy_config: DeployConfig = DeployConfig()
|
7589
7636
|
|
7590
|
-
|
7637
|
+
########################################
|
7638
|
+
# ../deploy/paths/owners.py
|
7591
7639
|
|
7592
|
-
|
7640
|
+
|
7641
|
+
class DeployPathOwner(abc.ABC):
|
7642
|
+
@abc.abstractmethod
|
7643
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
7644
|
+
raise NotImplementedError
|
7593
7645
|
|
7594
7646
|
|
7595
|
-
|
7596
|
-
# ../commands/local.py
|
7647
|
+
DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
|
7597
7648
|
|
7598
7649
|
|
7599
|
-
class
|
7650
|
+
class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
|
7600
7651
|
def __init__(
|
7601
7652
|
self,
|
7602
|
-
|
7603
|
-
|
7653
|
+
*args: ta.Any,
|
7654
|
+
owned_dir: str,
|
7655
|
+
deploy_home: ta.Optional[DeployHome],
|
7656
|
+
**kwargs: ta.Any,
|
7604
7657
|
) -> None:
|
7605
|
-
super().__init__()
|
7658
|
+
super().__init__(*args, **kwargs)
|
7606
7659
|
|
7607
|
-
|
7660
|
+
check.not_in('/', owned_dir)
|
7661
|
+
self._owned_dir: str = check.non_empty_str(owned_dir)
|
7608
7662
|
|
7609
|
-
|
7610
|
-
|
7611
|
-
|
7663
|
+
self._deploy_home = deploy_home
|
7664
|
+
|
7665
|
+
self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
|
7666
|
+
|
7667
|
+
@cached_nullary
|
7668
|
+
def _dir(self) -> str:
|
7669
|
+
return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
|
7670
|
+
|
7671
|
+
@cached_nullary
|
7672
|
+
def _make_dir(self) -> str:
|
7673
|
+
if not os.path.isdir(d := self._dir()):
|
7674
|
+
os.makedirs(d, exist_ok=True)
|
7675
|
+
return d
|
7676
|
+
|
7677
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
7678
|
+
return self._owned_deploy_paths
|
7612
7679
|
|
7613
7680
|
|
7614
7681
|
########################################
|
@@ -8513,6 +8580,74 @@ class DeployGitManager(SingleDirDeployPathOwner):
|
|
8513
8580
|
await self.get_repo_dir(spec.repo).checkout(spec, dst_dir)
|
8514
8581
|
|
8515
8582
|
|
8583
|
+
########################################
|
8584
|
+
# ../deploy/paths/manager.py
|
8585
|
+
|
8586
|
+
|
8587
|
+
class DeployPathsManager:
|
8588
|
+
def __init__(
|
8589
|
+
self,
|
8590
|
+
*,
|
8591
|
+
deploy_home: ta.Optional[DeployHome],
|
8592
|
+
deploy_path_owners: DeployPathOwners,
|
8593
|
+
) -> None:
|
8594
|
+
super().__init__()
|
8595
|
+
|
8596
|
+
self._deploy_home = deploy_home
|
8597
|
+
self._deploy_path_owners = deploy_path_owners
|
8598
|
+
|
8599
|
+
@cached_nullary
|
8600
|
+
def owners_by_path(self) -> ta.Mapping[DeployPath, DeployPathOwner]:
|
8601
|
+
dct: ta.Dict[DeployPath, DeployPathOwner] = {}
|
8602
|
+
for o in self._deploy_path_owners:
|
8603
|
+
for p in o.get_owned_deploy_paths():
|
8604
|
+
if p in dct:
|
8605
|
+
raise DeployPathError(f'Duplicate deploy path owner: {p}')
|
8606
|
+
dct[p] = o
|
8607
|
+
return dct
|
8608
|
+
|
8609
|
+
def validate_deploy_paths(self) -> None:
|
8610
|
+
self.owners_by_path()
|
8611
|
+
|
8612
|
+
|
8613
|
+
########################################
|
8614
|
+
# ../deploy/tmp.py
|
8615
|
+
|
8616
|
+
|
8617
|
+
class DeployTmpManager(
|
8618
|
+
SingleDirDeployPathOwner,
|
8619
|
+
AtomicPathSwapping,
|
8620
|
+
):
|
8621
|
+
def __init__(
|
8622
|
+
self,
|
8623
|
+
*,
|
8624
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
8625
|
+
) -> None:
|
8626
|
+
super().__init__(
|
8627
|
+
owned_dir='tmp',
|
8628
|
+
deploy_home=deploy_home,
|
8629
|
+
)
|
8630
|
+
|
8631
|
+
@cached_nullary
|
8632
|
+
def _swapping(self) -> AtomicPathSwapping:
|
8633
|
+
return TempDirAtomicPathSwapping(
|
8634
|
+
temp_dir=self._make_dir(),
|
8635
|
+
root_dir=check.non_empty_str(self._deploy_home),
|
8636
|
+
)
|
8637
|
+
|
8638
|
+
def begin_atomic_path_swap(
|
8639
|
+
self,
|
8640
|
+
kind: AtomicPathSwapKind,
|
8641
|
+
dst_path: str,
|
8642
|
+
**kwargs: ta.Any,
|
8643
|
+
) -> AtomicPathSwap:
|
8644
|
+
return self._swapping().begin_atomic_path_swap(
|
8645
|
+
kind,
|
8646
|
+
dst_path,
|
8647
|
+
**kwargs,
|
8648
|
+
)
|
8649
|
+
|
8650
|
+
|
8516
8651
|
########################################
|
8517
8652
|
# ../deploy/venvs.py
|
8518
8653
|
"""
|
@@ -9122,20 +9257,44 @@ class DeployAppManager(DeployPathOwner):
|
|
9122
9257
|
self._git = git
|
9123
9258
|
self._venvs = venvs
|
9124
9259
|
|
9125
|
-
|
9126
|
-
|
9127
|
-
|
9260
|
+
#
|
9261
|
+
|
9262
|
+
_APP_TAG_DIR_STR = 'tags/apps/@app/@tag/'
|
9263
|
+
_APP_TAG_DIR = DeployPath.parse(_APP_TAG_DIR_STR)
|
9264
|
+
|
9265
|
+
_CONF_TAG_DIR_STR = 'tags/conf/@tag--@app/'
|
9266
|
+
_CONF_TAG_DIR = DeployPath.parse(_CONF_TAG_DIR_STR)
|
9128
9267
|
|
9268
|
+
_DEPLOY_DIR_STR = 'deploys/@tag--@app/'
|
9269
|
+
_DEPLOY_DIR = DeployPath.parse(_DEPLOY_DIR_STR)
|
9270
|
+
|
9271
|
+
_APP_DEPLOY_LINK = DeployPath.parse(f'{_DEPLOY_DIR_STR}apps/@app')
|
9272
|
+
_CONF_DEPLOY_LINK = DeployPath.parse(f'{_DEPLOY_DIR_STR}conf')
|
9273
|
+
|
9274
|
+
@cached_nullary
|
9129
9275
|
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
9130
9276
|
return {
|
9131
|
-
|
9132
|
-
|
9277
|
+
self._APP_TAG_DIR,
|
9278
|
+
|
9279
|
+
self._CONF_TAG_DIR,
|
9133
9280
|
|
9134
|
-
|
9135
|
-
|
9136
|
-
|
9281
|
+
self._DEPLOY_DIR,
|
9282
|
+
|
9283
|
+
self._APP_DEPLOY_LINK,
|
9284
|
+
self._CONF_DEPLOY_LINK,
|
9285
|
+
|
9286
|
+
*[
|
9287
|
+
DeployPath.parse(f'{self._APP_TAG_DIR_STR}{sfx}/')
|
9288
|
+
for sfx in [
|
9289
|
+
'conf',
|
9290
|
+
'git',
|
9291
|
+
'venv',
|
9292
|
+
]
|
9293
|
+
],
|
9137
9294
|
}
|
9138
9295
|
|
9296
|
+
#
|
9297
|
+
|
9139
9298
|
async def prepare_app(
|
9140
9299
|
self,
|
9141
9300
|
spec: DeploySpec,
|
@@ -9144,26 +9303,52 @@ class DeployAppManager(DeployPathOwner):
|
|
9144
9303
|
|
9145
9304
|
#
|
9146
9305
|
|
9147
|
-
|
9148
|
-
|
9306
|
+
deploy_home = check.non_empty_str(self._deploy_home)
|
9307
|
+
|
9308
|
+
def build_path(pth: DeployPath) -> str:
|
9309
|
+
return os.path.join(deploy_home, pth.render(app_tag.placeholders()))
|
9310
|
+
|
9311
|
+
app_tag_dir = build_path(self._APP_TAG_DIR)
|
9312
|
+
conf_tag_dir = build_path(self._CONF_TAG_DIR)
|
9313
|
+
deploy_dir = build_path(self._DEPLOY_DIR)
|
9314
|
+
app_deploy_link = build_path(self._APP_DEPLOY_LINK)
|
9315
|
+
conf_deploy_link_file = build_path(self._CONF_DEPLOY_LINK)
|
9149
9316
|
|
9150
9317
|
#
|
9151
9318
|
|
9152
|
-
|
9153
|
-
|
9319
|
+
os.makedirs(deploy_dir)
|
9320
|
+
|
9321
|
+
deploying_link = os.path.join(deploy_home, 'deploys/deploying')
|
9322
|
+
relative_symlink(
|
9323
|
+
deploy_dir,
|
9324
|
+
deploying_link,
|
9325
|
+
target_is_directory=True,
|
9326
|
+
make_dirs=True,
|
9327
|
+
)
|
9154
9328
|
|
9155
9329
|
#
|
9156
9330
|
|
9157
|
-
|
9158
|
-
|
9331
|
+
os.makedirs(app_tag_dir)
|
9332
|
+
relative_symlink(
|
9333
|
+
app_tag_dir,
|
9334
|
+
app_deploy_link,
|
9335
|
+
target_is_directory=True,
|
9336
|
+
make_dirs=True,
|
9337
|
+
)
|
9338
|
+
|
9339
|
+
#
|
9159
9340
|
|
9160
|
-
|
9161
|
-
|
9162
|
-
|
9341
|
+
os.makedirs(conf_tag_dir)
|
9342
|
+
relative_symlink(
|
9343
|
+
conf_tag_dir,
|
9344
|
+
conf_deploy_link_file,
|
9345
|
+
target_is_directory=True,
|
9346
|
+
make_dirs=True,
|
9347
|
+
)
|
9163
9348
|
|
9164
9349
|
#
|
9165
9350
|
|
9166
|
-
git_dir = os.path.join(
|
9351
|
+
git_dir = os.path.join(app_tag_dir, 'git')
|
9167
9352
|
await self._git.checkout(
|
9168
9353
|
spec.git,
|
9169
9354
|
git_dir,
|
@@ -9172,7 +9357,7 @@ class DeployAppManager(DeployPathOwner):
|
|
9172
9357
|
#
|
9173
9358
|
|
9174
9359
|
if spec.venv is not None:
|
9175
|
-
venv_dir = os.path.join(
|
9360
|
+
venv_dir = os.path.join(app_tag_dir, 'venv')
|
9176
9361
|
await self._venvs.setup_venv(
|
9177
9362
|
spec.venv,
|
9178
9363
|
git_dir,
|
@@ -9182,18 +9367,33 @@ class DeployAppManager(DeployPathOwner):
|
|
9182
9367
|
#
|
9183
9368
|
|
9184
9369
|
if spec.conf is not None:
|
9185
|
-
conf_dir = os.path.join(
|
9186
|
-
conf_link_dir = os.path.join(current_file, 'conf')
|
9370
|
+
conf_dir = os.path.join(app_tag_dir, 'conf')
|
9187
9371
|
await self._conf.write_conf(
|
9188
9372
|
spec.conf,
|
9189
|
-
conf_dir,
|
9190
9373
|
app_tag,
|
9191
|
-
|
9374
|
+
conf_dir,
|
9375
|
+
conf_tag_dir,
|
9192
9376
|
)
|
9193
9377
|
|
9194
9378
|
#
|
9195
9379
|
|
9196
|
-
os.
|
9380
|
+
current_link = os.path.join(deploy_home, 'deploys/current')
|
9381
|
+
os.replace(deploying_link, current_link)
|
9382
|
+
|
9383
|
+
|
9384
|
+
########################################
|
9385
|
+
# ../deploy/paths/inject.py
|
9386
|
+
|
9387
|
+
|
9388
|
+
def bind_deploy_paths() -> InjectorBindings:
|
9389
|
+
lst: ta.List[InjectorBindingOrBindings] = [
|
9390
|
+
inj.bind_array(DeployPathOwner),
|
9391
|
+
inj.bind_array_type(DeployPathOwner, DeployPathOwners),
|
9392
|
+
|
9393
|
+
inj.bind(DeployPathsManager, singleton=True),
|
9394
|
+
]
|
9395
|
+
|
9396
|
+
return inj.as_bindings(*lst)
|
9197
9397
|
|
9198
9398
|
|
9199
9399
|
########################################
|
@@ -9911,31 +10111,30 @@ class SystemInterpProvider(InterpProvider):
|
|
9911
10111
|
|
9912
10112
|
|
9913
10113
|
########################################
|
9914
|
-
# ../deploy/
|
9915
|
-
|
10114
|
+
# ../deploy/deploy.py
|
9916
10115
|
|
9917
|
-
##
|
9918
|
-
|
9919
|
-
|
9920
|
-
@dc.dataclass(frozen=True)
|
9921
|
-
class DeployCommand(Command['DeployCommand.Output']):
|
9922
|
-
spec: DeploySpec
|
9923
|
-
|
9924
|
-
@dc.dataclass(frozen=True)
|
9925
|
-
class Output(Command.Output):
|
9926
|
-
pass
|
9927
10116
|
|
10117
|
+
class DeployManager:
|
10118
|
+
def __init__(
|
10119
|
+
self,
|
10120
|
+
*,
|
10121
|
+
apps: DeployAppManager,
|
10122
|
+
paths: DeployPathsManager,
|
10123
|
+
):
|
10124
|
+
super().__init__()
|
9928
10125
|
|
9929
|
-
|
9930
|
-
|
9931
|
-
_apps: DeployAppManager
|
10126
|
+
self._apps = apps
|
10127
|
+
self._paths = paths
|
9932
10128
|
|
9933
|
-
async def
|
9934
|
-
|
10129
|
+
async def run_deploy(
|
10130
|
+
self,
|
10131
|
+
spec: DeploySpec,
|
10132
|
+
) -> None:
|
10133
|
+
self._paths.validate_deploy_paths()
|
9935
10134
|
|
9936
|
-
|
10135
|
+
#
|
9937
10136
|
|
9938
|
-
|
10137
|
+
await self._apps.prepare_app(spec)
|
9939
10138
|
|
9940
10139
|
|
9941
10140
|
########################################
|
@@ -10246,6 +10445,34 @@ DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
|
|
10246
10445
|
]])
|
10247
10446
|
|
10248
10447
|
|
10448
|
+
########################################
|
10449
|
+
# ../deploy/commands.py
|
10450
|
+
|
10451
|
+
|
10452
|
+
##
|
10453
|
+
|
10454
|
+
|
10455
|
+
@dc.dataclass(frozen=True)
|
10456
|
+
class DeployCommand(Command['DeployCommand.Output']):
|
10457
|
+
spec: DeploySpec
|
10458
|
+
|
10459
|
+
@dc.dataclass(frozen=True)
|
10460
|
+
class Output(Command.Output):
|
10461
|
+
pass
|
10462
|
+
|
10463
|
+
|
10464
|
+
@dc.dataclass(frozen=True)
|
10465
|
+
class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
|
10466
|
+
_deploy: DeployManager
|
10467
|
+
|
10468
|
+
async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
|
10469
|
+
log.info('Deploying! %r', cmd.spec)
|
10470
|
+
|
10471
|
+
await self._deploy.run_deploy(cmd.spec)
|
10472
|
+
|
10473
|
+
return DeployCommand.Output()
|
10474
|
+
|
10475
|
+
|
10249
10476
|
########################################
|
10250
10477
|
# ../targets/inject.py
|
10251
10478
|
|
@@ -10315,6 +10542,8 @@ def bind_deploy(
|
|
10315
10542
|
) -> InjectorBindings:
|
10316
10543
|
lst: ta.List[InjectorBindingOrBindings] = [
|
10317
10544
|
inj.bind(deploy_config),
|
10545
|
+
|
10546
|
+
bind_deploy_paths(),
|
10318
10547
|
]
|
10319
10548
|
|
10320
10549
|
#
|
@@ -10326,11 +10555,6 @@ def bind_deploy(
|
|
10326
10555
|
*([inj.bind(DeployPathOwner, to_key=cls, array=True)] if issubclass(cls, DeployPathOwner) else []),
|
10327
10556
|
)
|
10328
10557
|
|
10329
|
-
lst.extend([
|
10330
|
-
inj.bind_array(DeployPathOwner),
|
10331
|
-
inj.bind_array_type(DeployPathOwner, DeployPathOwners),
|
10332
|
-
])
|
10333
|
-
|
10334
10558
|
#
|
10335
10559
|
|
10336
10560
|
lst.extend([
|
@@ -10340,6 +10564,8 @@ def bind_deploy(
|
|
10340
10564
|
|
10341
10565
|
bind_manager(DeployGitManager),
|
10342
10566
|
|
10567
|
+
bind_manager(DeployManager),
|
10568
|
+
|
10343
10569
|
bind_manager(DeployTmpManager),
|
10344
10570
|
inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
|
10345
10571
|
|