ominfra 0.0.0.dev167__py3-none-any.whl → 0.0.0.dev168__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
ominfra/scripts/manage.py CHANGED
@@ -100,9 +100,9 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
100
100
  CommandT = ta.TypeVar('CommandT', bound='Command')
101
101
  CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
102
102
 
103
- # deploy/paths.py
103
+ # deploy/types.py
104
104
  DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
105
- DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
105
+ DeployPathPlaceholder = ta.Literal['app', 'tag', 'conf'] # ta.TypeAlias
106
106
 
107
107
  # ../../omlish/argparse/cli.py
108
108
  ArgparseCommandFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
@@ -1379,23 +1379,6 @@ class DeployConfig:
1379
1379
  deploy_home: ta.Optional[str] = None
1380
1380
 
1381
1381
 
1382
- ########################################
1383
- # ../deploy/types.py
1384
-
1385
-
1386
- DeployHome = ta.NewType('DeployHome', str)
1387
-
1388
- DeployApp = ta.NewType('DeployApp', str)
1389
- DeployTag = ta.NewType('DeployTag', str)
1390
- DeployRev = ta.NewType('DeployRev', str)
1391
- DeployKey = ta.NewType('DeployKey', str)
1392
-
1393
-
1394
- class DeployAppTag(ta.NamedTuple):
1395
- app: DeployApp
1396
- tag: DeployTag
1397
-
1398
-
1399
1382
  ########################################
1400
1383
  # ../../pyremote.py
1401
1384
  """
@@ -2074,6 +2057,8 @@ def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
2074
2057
  """
2075
2058
  TODO:
2076
2059
  - def maybe(v: lang.Maybe[T])
2060
+ - def not_ ?
2061
+ - ** class @dataclass Raise - user message should be able to be an exception type or instance or factory
2077
2062
  """
2078
2063
 
2079
2064
 
@@ -2738,6 +2723,28 @@ def strip_with_newline(s: str) -> str:
2738
2723
  return s.strip() + '\n'
2739
2724
 
2740
2725
 
2726
+ @ta.overload
2727
+ def split_keep_delimiter(s: str, d: str) -> str:
2728
+ ...
2729
+
2730
+
2731
+ @ta.overload
2732
+ def split_keep_delimiter(s: bytes, d: bytes) -> bytes:
2733
+ ...
2734
+
2735
+
2736
+ def split_keep_delimiter(s, d):
2737
+ ps = []
2738
+ i = 0
2739
+ while i < len(s):
2740
+ if (n := s.find(d, i)) < i:
2741
+ ps.append(s[i:])
2742
+ break
2743
+ ps.append(s[i:n + 1])
2744
+ i = n + 1
2745
+ return ps
2746
+
2747
+
2741
2748
  ##
2742
2749
 
2743
2750
 
@@ -3422,8 +3429,12 @@ def relative_symlink(
3422
3429
  *,
3423
3430
  target_is_directory: bool = False,
3424
3431
  dir_fd: ta.Optional[int] = None,
3432
+ make_dirs: bool = False,
3425
3433
  **kwargs: ta.Any,
3426
3434
  ) -> None:
3435
+ if make_dirs:
3436
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
3437
+
3427
3438
  os.symlink(
3428
3439
  os.path.relpath(src, os.path.dirname(dst)),
3429
3440
  dst,
@@ -4106,531 +4117,201 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
4106
4117
 
4107
4118
 
4108
4119
  ########################################
4109
- # ../deploy/paths.py
4110
- """
4111
- TODO:
4112
- - run/{.pid,.sock}
4113
- - logs/...
4114
- - current symlink
4115
- - conf/{nginx,supervisor}
4116
- - env/?
4117
- - apps/<app>/shared
4118
- """
4120
+ # ../deploy/types.py
4119
4121
 
4120
4122
 
4121
4123
  ##
4122
4124
 
4123
4125
 
4124
- DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER = '@'
4125
- DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
4126
+ DeployHome = ta.NewType('DeployHome', str)
4126
4127
 
4127
- DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
4128
- 'app',
4129
- 'tag',
4130
- ])
4128
+ DeployApp = ta.NewType('DeployApp', str)
4129
+ DeployTag = ta.NewType('DeployTag', str)
4130
+ DeployRev = ta.NewType('DeployRev', str)
4131
+ DeployKey = ta.NewType('DeployKey', str)
4131
4132
 
4132
4133
 
4133
- class DeployPathError(Exception):
4134
- pass
4134
+ ##
4135
4135
 
4136
4136
 
4137
4137
  @dc.dataclass(frozen=True)
4138
- class DeployPathPart(abc.ABC): # noqa
4139
- @property
4140
- @abc.abstractmethod
4141
- def kind(self) -> DeployPathKind:
4142
- raise NotImplementedError
4138
+ class DeployAppTag:
4139
+ app: DeployApp
4140
+ tag: DeployTag
4143
4141
 
4144
- @abc.abstractmethod
4145
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4146
- raise NotImplementedError
4142
+ def __post_init__(self) -> None:
4143
+ for s in [self.app, self.tag]:
4144
+ check.non_empty_str(s)
4145
+ check.equal(s, s.strip())
4147
4146
 
4147
+ def placeholders(self) -> ta.Mapping[DeployPathPlaceholder, str]:
4148
+ return {
4149
+ 'app': self.app,
4150
+ 'tag': self.tag,
4151
+ }
4148
4152
 
4149
- #
4150
4153
 
4154
+ ########################################
4155
+ # ../remote/config.py
4151
4156
 
4152
- class DirDeployPathPart(DeployPathPart, abc.ABC):
4153
- @property
4154
- def kind(self) -> DeployPathKind:
4155
- return 'dir'
4156
4157
 
4157
- @classmethod
4158
- def parse(cls, s: str) -> 'DirDeployPathPart':
4159
- if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
4160
- check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
4161
- return PlaceholderDirDeployPathPart(s[1:])
4162
- else:
4163
- return ConstDirDeployPathPart(s)
4158
+ @dc.dataclass(frozen=True)
4159
+ class RemoteConfig:
4160
+ payload_file: ta.Optional[str] = None
4164
4161
 
4162
+ set_pgid: bool = True
4165
4163
 
4166
- class FileDeployPathPart(DeployPathPart, abc.ABC):
4167
- @property
4168
- def kind(self) -> DeployPathKind:
4169
- return 'file'
4164
+ deathsig: ta.Optional[str] = 'KILL'
4170
4165
 
4171
- @classmethod
4172
- def parse(cls, s: str) -> 'FileDeployPathPart':
4173
- if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
4174
- check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
4175
- if not any(c in s for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS):
4176
- return PlaceholderFileDeployPathPart(s[1:], '')
4177
- else:
4178
- p = min(f for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS if (f := s.find(c)) > 0)
4179
- return PlaceholderFileDeployPathPart(s[1:p], s[p:])
4180
- else:
4181
- return ConstFileDeployPathPart(s)
4166
+ pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
4182
4167
 
4168
+ forward_logging: bool = True
4183
4169
 
4184
- #
4170
+ timebomb_delay_s: ta.Optional[float] = 60 * 60.
4185
4171
 
4172
+ heartbeat_interval_s: float = 3.
4186
4173
 
4187
- @dc.dataclass(frozen=True)
4188
- class ConstDeployPathPart(DeployPathPart, abc.ABC):
4189
- name: str
4190
4174
 
4191
- def __post_init__(self) -> None:
4192
- check.non_empty_str(self.name)
4193
- check.not_in('/', self.name)
4194
- check.not_in(DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, self.name)
4175
+ ########################################
4176
+ # ../remote/payload.py
4195
4177
 
4196
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4197
- return self.name
4198
4178
 
4179
+ RemoteExecutionPayloadFile = ta.NewType('RemoteExecutionPayloadFile', str)
4199
4180
 
4200
- class ConstDirDeployPathPart(ConstDeployPathPart, DirDeployPathPart):
4201
- pass
4202
4181
 
4182
+ @cached_nullary
4183
+ def _get_self_src() -> str:
4184
+ return inspect.getsource(sys.modules[__name__])
4203
4185
 
4204
- class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
4205
- pass
4206
4186
 
4187
+ def _is_src_amalg(src: str) -> bool:
4188
+ for l in src.splitlines(): # noqa
4189
+ if l.startswith('# @omlish-amalg-output '):
4190
+ return True
4191
+ return False
4207
4192
 
4208
- #
4209
4193
 
4194
+ @cached_nullary
4195
+ def _is_self_amalg() -> bool:
4196
+ return _is_src_amalg(_get_self_src())
4210
4197
 
4211
- @dc.dataclass(frozen=True)
4212
- class PlaceholderDeployPathPart(DeployPathPart, abc.ABC):
4213
- placeholder: str # DeployPathPlaceholder
4214
4198
 
4215
- def __post_init__(self) -> None:
4216
- check.non_empty_str(self.placeholder)
4217
- for c in [*DEPLOY_PATH_PLACEHOLDER_SEPARATORS, DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
4218
- check.not_in(c, self.placeholder)
4219
- check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
4199
+ def get_remote_payload_src(
4200
+ *,
4201
+ file: ta.Optional[RemoteExecutionPayloadFile],
4202
+ ) -> str:
4203
+ if file is not None:
4204
+ with open(file) as f:
4205
+ return f.read()
4220
4206
 
4221
- def _render_placeholder(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4222
- if placeholders is not None:
4223
- return placeholders[self.placeholder] # type: ignore
4224
- else:
4225
- return DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER + self.placeholder
4207
+ if _is_self_amalg():
4208
+ return _get_self_src()
4226
4209
 
4210
+ import importlib.resources
4211
+ return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
4227
4212
 
4228
- @dc.dataclass(frozen=True)
4229
- class PlaceholderDirDeployPathPart(PlaceholderDeployPathPart, DirDeployPathPart):
4230
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4231
- return self._render_placeholder(placeholders)
4232
4213
 
4214
+ ########################################
4215
+ # ../system/platforms.py
4233
4216
 
4234
- @dc.dataclass(frozen=True)
4235
- class PlaceholderFileDeployPathPart(PlaceholderDeployPathPart, FileDeployPathPart):
4236
- suffix: str
4237
4217
 
4238
- def __post_init__(self) -> None:
4239
- super().__post_init__()
4240
- if self.suffix:
4241
- for c in [DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
4242
- check.not_in(c, self.suffix)
4218
+ ##
4243
4219
 
4244
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4245
- return self._render_placeholder(placeholders) + self.suffix
4246
4220
 
4221
+ @dc.dataclass(frozen=True)
4222
+ class Platform(abc.ABC): # noqa
4223
+ pass
4247
4224
 
4248
- ##
4249
4225
 
4226
+ class LinuxPlatform(Platform, abc.ABC):
4227
+ pass
4250
4228
 
4251
- @dc.dataclass(frozen=True)
4252
- class DeployPath:
4253
- parts: ta.Sequence[DeployPathPart]
4254
4229
 
4255
- def __post_init__(self) -> None:
4256
- hash(self)
4230
+ class UbuntuPlatform(LinuxPlatform):
4231
+ pass
4257
4232
 
4258
- check.not_empty(self.parts)
4259
- for p in self.parts[:-1]:
4260
- check.equal(p.kind, 'dir')
4261
4233
 
4262
- pd = {}
4263
- for i, p in enumerate(self.parts):
4264
- if isinstance(p, PlaceholderDeployPathPart):
4265
- if p.placeholder in pd:
4266
- raise DeployPathError('Duplicate placeholders in path', self)
4267
- pd[p.placeholder] = i
4234
+ class AmazonLinuxPlatform(LinuxPlatform):
4235
+ pass
4268
4236
 
4269
- if 'tag' in pd:
4270
- if 'app' not in pd or pd['app'] >= pd['tag']:
4271
- raise DeployPathError('Tag placeholder in path without preceding app', self)
4272
4237
 
4273
- @property
4274
- def kind(self) -> ta.Literal['file', 'dir']:
4275
- return self.parts[-1].kind
4238
+ class GenericLinuxPlatform(LinuxPlatform):
4239
+ pass
4276
4240
 
4277
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4278
- return os.path.join( # noqa
4279
- *[p.render(placeholders) for p in self.parts],
4280
- *([''] if self.kind == 'dir' else []),
4281
- )
4282
4241
 
4283
- @classmethod
4284
- def parse(cls, s: str) -> 'DeployPath':
4285
- tail_parse: ta.Callable[[str], DeployPathPart]
4286
- if s.endswith('/'):
4287
- tail_parse = DirDeployPathPart.parse
4288
- s = s[:-1]
4289
- else:
4290
- tail_parse = FileDeployPathPart.parse
4291
- ps = check.non_empty_str(s).split('/')
4292
- return cls((
4293
- *([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
4294
- tail_parse(ps[-1]),
4295
- ))
4242
+ class DarwinPlatform(Platform):
4243
+ pass
4296
4244
 
4297
4245
 
4298
- ##
4246
+ class UnknownPlatform(Platform):
4247
+ pass
4299
4248
 
4300
4249
 
4301
- class DeployPathOwner(abc.ABC):
4302
- @abc.abstractmethod
4303
- def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
4304
- raise NotImplementedError
4250
+ ##
4305
4251
 
4306
4252
 
4307
- DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
4253
+ def _detect_system_platform() -> Platform:
4254
+ plat = sys.platform
4308
4255
 
4256
+ if plat == 'linux':
4257
+ if (osr := LinuxOsRelease.read()) is None:
4258
+ return GenericLinuxPlatform()
4309
4259
 
4310
- class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
4311
- def __init__(
4312
- self,
4313
- *args: ta.Any,
4314
- owned_dir: str,
4315
- deploy_home: ta.Optional[DeployHome],
4316
- **kwargs: ta.Any,
4317
- ) -> None:
4318
- super().__init__(*args, **kwargs)
4260
+ if osr.id == 'amzn':
4261
+ return AmazonLinuxPlatform()
4319
4262
 
4320
- check.not_in('/', owned_dir)
4321
- self._owned_dir: str = check.non_empty_str(owned_dir)
4263
+ elif osr.id == 'ubuntu':
4264
+ return UbuntuPlatform()
4322
4265
 
4323
- self._deploy_home = deploy_home
4266
+ else:
4267
+ return GenericLinuxPlatform()
4324
4268
 
4325
- self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
4269
+ elif plat == 'darwin':
4270
+ return DarwinPlatform()
4326
4271
 
4327
- @cached_nullary
4328
- def _dir(self) -> str:
4329
- return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
4272
+ else:
4273
+ return UnknownPlatform()
4330
4274
 
4331
- @cached_nullary
4332
- def _make_dir(self) -> str:
4333
- if not os.path.isdir(d := self._dir()):
4334
- os.makedirs(d, exist_ok=True)
4335
- return d
4336
4275
 
4337
- def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
4338
- return self._owned_deploy_paths
4276
+ @cached_nullary
4277
+ def detect_system_platform() -> Platform:
4278
+ platform = _detect_system_platform()
4279
+ log.info('Detected platform: %r', platform)
4280
+ return platform
4339
4281
 
4340
4282
 
4341
4283
  ########################################
4342
- # ../deploy/specs.py
4284
+ # ../targets/targets.py
4285
+ """
4286
+ It's desugaring. Subprocess and locals are only leafs. Retain an origin?
4287
+ ** TWO LAYERS ** - ManageTarget is user-facing, ConnectorTarget is bound, internal
4288
+ """
4343
4289
 
4344
4290
 
4345
4291
  ##
4346
4292
 
4347
4293
 
4348
- def check_valid_deploy_spec_path(s: str) -> str:
4349
- check.non_empty_str(s)
4350
- for c in ['..', '//']:
4351
- check.not_in(c, s)
4352
- check.arg(not s.startswith('/'))
4353
- return s
4294
+ class ManageTarget(abc.ABC): # noqa
4295
+ def __init_subclass__(cls, **kwargs: ta.Any) -> None:
4296
+ super().__init_subclass__(**kwargs)
4354
4297
 
4298
+ check.state(cls.__name__.endswith('ManageTarget'))
4355
4299
 
4356
- ##
4300
+
4301
+ #
4357
4302
 
4358
4303
 
4359
4304
  @dc.dataclass(frozen=True)
4360
- class DeployGitRepo:
4361
- host: ta.Optional[str] = None
4362
- username: ta.Optional[str] = None
4363
- path: ta.Optional[str] = None
4305
+ class PythonRemoteManageTarget:
4306
+ DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
4307
+ python: str = DEFAULT_PYTHON
4364
4308
 
4365
- def __post_init__(self) -> None:
4366
- check.not_in('..', check.non_empty_str(self.host))
4367
- check.not_in('.', check.non_empty_str(self.path))
4368
4309
 
4310
+ #
4369
4311
 
4370
- @dc.dataclass(frozen=True)
4371
- class DeployGitSpec:
4372
- repo: DeployGitRepo
4373
- rev: DeployRev
4374
4312
 
4375
- subtrees: ta.Optional[ta.Sequence[str]] = None
4376
-
4377
- def __post_init__(self) -> None:
4378
- hash(self)
4379
- check.non_empty_str(self.rev)
4380
- if self.subtrees is not None:
4381
- for st in self.subtrees:
4382
- check.non_empty_str(st)
4383
-
4384
-
4385
- ##
4386
-
4387
-
4388
- @dc.dataclass(frozen=True)
4389
- class DeployVenvSpec:
4390
- interp: ta.Optional[str] = None
4391
-
4392
- requirements_files: ta.Optional[ta.Sequence[str]] = None
4393
- extra_dependencies: ta.Optional[ta.Sequence[str]] = None
4394
-
4395
- use_uv: bool = False
4396
-
4397
-
4398
- ##
4399
-
4400
-
4401
- @dc.dataclass(frozen=True)
4402
- class DeployConfFile:
4403
- path: str
4404
- body: str
4405
-
4406
- def __post_init__(self) -> None:
4407
- check_valid_deploy_spec_path(self.path)
4408
-
4409
-
4410
- #
4411
-
4412
-
4413
- @dc.dataclass(frozen=True)
4414
- class DeployConfLink(abc.ABC): # noqa
4415
- """
4416
- May be either:
4417
- - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
4418
- - @conf/file - links a single file in a single subdir to conf/@conf/@dst-file
4419
- - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
4420
- """
4421
-
4422
- src: str
4423
-
4424
- def __post_init__(self) -> None:
4425
- check_valid_deploy_spec_path(self.src)
4426
- if '/' in self.src:
4427
- check.equal(self.src.count('/'), 1)
4428
-
4429
-
4430
- class AppDeployConfLink(DeployConfLink):
4431
- pass
4432
-
4433
-
4434
- class TagDeployConfLink(DeployConfLink):
4435
- pass
4436
-
4437
-
4438
- #
4439
-
4440
-
4441
- @dc.dataclass(frozen=True)
4442
- class DeployConfSpec:
4443
- files: ta.Optional[ta.Sequence[DeployConfFile]] = None
4444
-
4445
- links: ta.Optional[ta.Sequence[DeployConfLink]] = None
4446
-
4447
- def __post_init__(self) -> None:
4448
- if self.files:
4449
- seen: ta.Set[str] = set()
4450
- for f in self.files:
4451
- check.not_in(f.path, seen)
4452
- seen.add(f.path)
4453
-
4454
-
4455
- ##
4456
-
4457
-
4458
- @dc.dataclass(frozen=True)
4459
- class DeploySpec:
4460
- app: DeployApp
4461
-
4462
- git: DeployGitSpec
4463
-
4464
- venv: ta.Optional[DeployVenvSpec] = None
4465
-
4466
- conf: ta.Optional[DeployConfSpec] = None
4467
-
4468
- @cached_nullary
4469
- def key(self) -> DeployKey:
4470
- return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
4471
-
4472
-
4473
- ########################################
4474
- # ../remote/config.py
4475
-
4476
-
4477
- @dc.dataclass(frozen=True)
4478
- class RemoteConfig:
4479
- payload_file: ta.Optional[str] = None
4480
-
4481
- set_pgid: bool = True
4482
-
4483
- deathsig: ta.Optional[str] = 'KILL'
4484
-
4485
- pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
4486
-
4487
- forward_logging: bool = True
4488
-
4489
- timebomb_delay_s: ta.Optional[float] = 60 * 60.
4490
-
4491
- heartbeat_interval_s: float = 3.
4492
-
4493
-
4494
- ########################################
4495
- # ../remote/payload.py
4496
-
4497
-
4498
- RemoteExecutionPayloadFile = ta.NewType('RemoteExecutionPayloadFile', str)
4499
-
4500
-
4501
- @cached_nullary
4502
- def _get_self_src() -> str:
4503
- return inspect.getsource(sys.modules[__name__])
4504
-
4505
-
4506
- def _is_src_amalg(src: str) -> bool:
4507
- for l in src.splitlines(): # noqa
4508
- if l.startswith('# @omlish-amalg-output '):
4509
- return True
4510
- return False
4511
-
4512
-
4513
- @cached_nullary
4514
- def _is_self_amalg() -> bool:
4515
- return _is_src_amalg(_get_self_src())
4516
-
4517
-
4518
- def get_remote_payload_src(
4519
- *,
4520
- file: ta.Optional[RemoteExecutionPayloadFile],
4521
- ) -> str:
4522
- if file is not None:
4523
- with open(file) as f:
4524
- return f.read()
4525
-
4526
- if _is_self_amalg():
4527
- return _get_self_src()
4528
-
4529
- import importlib.resources
4530
- return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
4531
-
4532
-
4533
- ########################################
4534
- # ../system/platforms.py
4535
-
4536
-
4537
- ##
4538
-
4539
-
4540
- @dc.dataclass(frozen=True)
4541
- class Platform(abc.ABC): # noqa
4542
- pass
4543
-
4544
-
4545
- class LinuxPlatform(Platform, abc.ABC):
4546
- pass
4547
-
4548
-
4549
- class UbuntuPlatform(LinuxPlatform):
4550
- pass
4551
-
4552
-
4553
- class AmazonLinuxPlatform(LinuxPlatform):
4554
- pass
4555
-
4556
-
4557
- class GenericLinuxPlatform(LinuxPlatform):
4558
- pass
4559
-
4560
-
4561
- class DarwinPlatform(Platform):
4562
- pass
4563
-
4564
-
4565
- class UnknownPlatform(Platform):
4566
- pass
4567
-
4568
-
4569
- ##
4570
-
4571
-
4572
- def _detect_system_platform() -> Platform:
4573
- plat = sys.platform
4574
-
4575
- if plat == 'linux':
4576
- if (osr := LinuxOsRelease.read()) is None:
4577
- return GenericLinuxPlatform()
4578
-
4579
- if osr.id == 'amzn':
4580
- return AmazonLinuxPlatform()
4581
-
4582
- elif osr.id == 'ubuntu':
4583
- return UbuntuPlatform()
4584
-
4585
- else:
4586
- return GenericLinuxPlatform()
4587
-
4588
- elif plat == 'darwin':
4589
- return DarwinPlatform()
4590
-
4591
- else:
4592
- return UnknownPlatform()
4593
-
4594
-
4595
- @cached_nullary
4596
- def detect_system_platform() -> Platform:
4597
- platform = _detect_system_platform()
4598
- log.info('Detected platform: %r', platform)
4599
- return platform
4600
-
4601
-
4602
- ########################################
4603
- # ../targets/targets.py
4604
- """
4605
- It's desugaring. Subprocess and locals are only leafs. Retain an origin?
4606
- ** TWO LAYERS ** - ManageTarget is user-facing, ConnectorTarget is bound, internal
4607
- """
4608
-
4609
-
4610
- ##
4611
-
4612
-
4613
- class ManageTarget(abc.ABC): # noqa
4614
- def __init_subclass__(cls, **kwargs: ta.Any) -> None:
4615
- super().__init_subclass__(**kwargs)
4616
-
4617
- check.state(cls.__name__.endswith('ManageTarget'))
4618
-
4619
-
4620
- #
4621
-
4622
-
4623
- @dc.dataclass(frozen=True)
4624
- class PythonRemoteManageTarget:
4625
- DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
4626
- python: str = DEFAULT_PYTHON
4627
-
4628
-
4629
- #
4630
-
4631
-
4632
- class RemoteManageTarget(ManageTarget, abc.ABC):
4633
- pass
4313
+ class RemoteManageTarget(ManageTarget, abc.ABC):
4314
+ pass
4634
4315
 
4635
4316
 
4636
4317
  class PhysicallyRemoteManageTarget(RemoteManageTarget, abc.ABC):
@@ -6859,172 +6540,343 @@ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command
6859
6540
 
6860
6541
 
6861
6542
  ########################################
6862
- # ../deploy/conf.py
6543
+ # ../deploy/paths/paths.py
6863
6544
  """
6864
6545
  TODO:
6865
- - @conf DeployPathPlaceholder? :|
6866
- - post-deploy: remove any dir_links not present in new spec
6867
- - * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
6868
- - no such thing as 'previously present'.. build a 'deploy state' and pass it back?
6869
- - ** whole thing can be atomic **
6870
- - 1) new atomic temp dir
6871
- - 2) for each subdir not needing modification, hardlink into temp dir
6872
- - 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
6873
- - 4) write (or if deleting, omit) new files
6874
- - 5) swap top level
6875
- - ** whole deploy can be atomic(-ish) - do this for everything **
6876
- - just a '/deploy/current' dir
6877
- - some things (venvs) cannot be moved, thus the /deploy/venvs dir
6878
- - ** ensure (enforce) equivalent relpath nesting
6546
+ - run/{.pid,.sock}
6547
+ - logs/...
6548
+ - current symlink
6549
+ - conf/{nginx,supervisor}
6550
+ - env/?
6551
+ - apps/<app>/shared
6879
6552
  """
6880
6553
 
6881
6554
 
6882
- class DeployConfManager(SingleDirDeployPathOwner):
6883
- def __init__(
6884
- self,
6885
- *,
6886
- deploy_home: ta.Optional[DeployHome] = None,
6887
- ) -> None:
6888
- super().__init__(
6889
- owned_dir='conf',
6890
- deploy_home=deploy_home,
6891
- )
6555
+ ##
6892
6556
 
6893
- async def _write_conf_file(
6894
- self,
6895
- cf: DeployConfFile,
6896
- conf_dir: str,
6897
- ) -> None:
6898
- conf_file = os.path.join(conf_dir, cf.path)
6899
- check.arg(is_path_in_dir(conf_dir, conf_file))
6900
6557
 
6901
- os.makedirs(os.path.dirname(conf_file), exist_ok=True)
6558
+ DEPLOY_PATH_PLACEHOLDER_SIGIL = '@'
6559
+ DEPLOY_PATH_PLACEHOLDER_SEPARATOR = '--'
6902
6560
 
6903
- with open(conf_file, 'w') as f: # noqa
6904
- f.write(cf.body)
6561
+ DEPLOY_PATH_PLACEHOLDER_DELIMITERS: ta.AbstractSet[str] = frozenset([
6562
+ DEPLOY_PATH_PLACEHOLDER_SEPARATOR,
6563
+ '.',
6564
+ ])
6905
6565
 
6906
- async def _make_conf_link(
6907
- self,
6908
- link: DeployConfLink,
6909
- conf_dir: str,
6910
- app_tag: DeployAppTag,
6911
- link_dir: str,
6912
- ) -> None:
6913
- link_src = os.path.join(conf_dir, link.src)
6914
- check.arg(is_path_in_dir(conf_dir, link_src))
6566
+ DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
6567
+ 'app',
6568
+ 'tag',
6569
+ 'conf',
6570
+ ])
6915
6571
 
6916
- is_link_dir = link.src.endswith('/')
6917
- if is_link_dir:
6918
- check.arg(link.src.count('/') == 1)
6919
- check.arg(os.path.isdir(link_src))
6920
- link_dst_pfx = link.src
6921
- link_dst_sfx = ''
6922
6572
 
6923
- elif '/' in link.src:
6924
- check.arg(os.path.isfile(link_src))
6925
- d, f = os.path.split(link.src)
6926
- # TODO: check filename :|
6927
- link_dst_pfx = d + '/'
6928
- link_dst_sfx = '-' + f
6573
+ class DeployPathError(Exception):
6574
+ pass
6575
+
6576
+
6577
+ class DeployPathRenderable(abc.ABC):
6578
+ @abc.abstractmethod
6579
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6580
+ raise NotImplementedError
6581
+
6582
+
6583
+ ##
6929
6584
 
6585
+
6586
+ class DeployPathNamePart(DeployPathRenderable, abc.ABC):
6587
+ @classmethod
6588
+ def parse(cls, s: str) -> 'DeployPathNamePart':
6589
+ check.non_empty_str(s)
6590
+ if s.startswith(DEPLOY_PATH_PLACEHOLDER_SIGIL):
6591
+ return PlaceholderDeployPathNamePart(s[1:])
6592
+ elif s in DEPLOY_PATH_PLACEHOLDER_DELIMITERS:
6593
+ return DelimiterDeployPathNamePart(s)
6930
6594
  else:
6931
- check.arg(os.path.isfile(link_src))
6932
- if '.' in link.src:
6933
- l, _, r = link.src.partition('.')
6934
- link_dst_pfx = l + '/'
6935
- link_dst_sfx = '.' + r
6936
- else:
6937
- link_dst_pfx = link.src + '/'
6938
- link_dst_sfx = ''
6595
+ return ConstDeployPathNamePart(s)
6596
+
6597
+
6598
+ @dc.dataclass(frozen=True)
6599
+ class PlaceholderDeployPathNamePart(DeployPathNamePart):
6600
+ placeholder: str # DeployPathPlaceholder
6601
+
6602
+ def __post_init__(self) -> None:
6603
+ check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
6604
+
6605
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6606
+ if placeholders is not None:
6607
+ return placeholders[self.placeholder] # type: ignore
6608
+ else:
6609
+ return DEPLOY_PATH_PLACEHOLDER_SIGIL + self.placeholder
6610
+
6611
+
6612
+ @dc.dataclass(frozen=True)
6613
+ class DelimiterDeployPathNamePart(DeployPathNamePart):
6614
+ delimiter: str
6615
+
6616
+ def __post_init__(self) -> None:
6617
+ check.in_(self.delimiter, DEPLOY_PATH_PLACEHOLDER_DELIMITERS)
6618
+
6619
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6620
+ return self.delimiter
6621
+
6622
+
6623
+ @dc.dataclass(frozen=True)
6624
+ class ConstDeployPathNamePart(DeployPathNamePart):
6625
+ const: str
6626
+
6627
+ def __post_init__(self) -> None:
6628
+ check.non_empty_str(self.const)
6629
+ for c in [*DEPLOY_PATH_PLACEHOLDER_DELIMITERS, DEPLOY_PATH_PLACEHOLDER_SIGIL, '/']:
6630
+ check.not_in(c, self.const)
6631
+
6632
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6633
+ return self.const
6634
+
6635
+
6636
+ @dc.dataclass(frozen=True)
6637
+ class DeployPathName(DeployPathRenderable):
6638
+ parts: ta.Sequence[DeployPathNamePart]
6639
+
6640
+ def __post_init__(self) -> None:
6641
+ hash(self)
6642
+ check.not_empty(self.parts)
6643
+ for k, g in itertools.groupby(self.parts, type):
6644
+ if len(gl := list(g)) > 1:
6645
+ raise DeployPathError(f'May not have consecutive path name part types: {k} {gl}')
6646
+
6647
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6648
+ return ''.join(p.render(placeholders) for p in self.parts)
6649
+
6650
+ @classmethod
6651
+ def parse(cls, s: str) -> 'DeployPathName':
6652
+ check.non_empty_str(s)
6653
+ check.not_in('/', s)
6654
+
6655
+ i = 0
6656
+ ps = []
6657
+ while i < len(s):
6658
+ ns = [(n, d) for d in DEPLOY_PATH_PLACEHOLDER_DELIMITERS if (n := s.find(d, i)) >= 0]
6659
+ if not ns:
6660
+ ps.append(s[i:])
6661
+ break
6662
+ n, d = min(ns)
6663
+ ps.append(check.non_empty_str(s[i:n]))
6664
+ ps.append(s[n:n + len(d)])
6665
+ i = n + len(d)
6666
+
6667
+ return cls(tuple(DeployPathNamePart.parse(p) for p in ps))
6668
+
6669
+
6670
+ ##
6671
+
6672
+
6673
+ @dc.dataclass(frozen=True)
6674
+ class DeployPathPart(DeployPathRenderable, abc.ABC): # noqa
6675
+ name: DeployPathName
6676
+
6677
+ @property
6678
+ @abc.abstractmethod
6679
+ def kind(self) -> DeployPathKind:
6680
+ raise NotImplementedError
6681
+
6682
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6683
+ return self.name.render(placeholders) + ('/' if self.kind == 'dir' else '')
6684
+
6685
+ @classmethod
6686
+ def parse(cls, s: str) -> 'DeployPathPart':
6687
+ if (is_dir := s.endswith('/')):
6688
+ s = s[:-1]
6689
+ check.non_empty_str(s)
6690
+ check.not_in('/', s)
6691
+
6692
+ n = DeployPathName.parse(s)
6693
+ if is_dir:
6694
+ return DirDeployPathPart(n)
6695
+ else:
6696
+ return FileDeployPathPart(n)
6697
+
6698
+
6699
+ class DirDeployPathPart(DeployPathPart):
6700
+ @property
6701
+ def kind(self) -> DeployPathKind:
6702
+ return 'dir'
6703
+
6704
+
6705
+ class FileDeployPathPart(DeployPathPart):
6706
+ @property
6707
+ def kind(self) -> DeployPathKind:
6708
+ return 'file'
6709
+
6710
+
6711
+ #
6712
+
6713
+
6714
+ @dc.dataclass(frozen=True)
6715
+ class DeployPath:
6716
+ parts: ta.Sequence[DeployPathPart]
6717
+
6718
+ @property
6719
+ def name_parts(self) -> ta.Iterator[DeployPathNamePart]:
6720
+ for p in self.parts:
6721
+ yield from p.name.parts
6722
+
6723
+ def __post_init__(self) -> None:
6724
+ hash(self)
6725
+ check.not_empty(self.parts)
6726
+ for p in self.parts[:-1]:
6727
+ check.equal(p.kind, 'dir')
6728
+
6729
+ pd: ta.Dict[DeployPathPlaceholder, ta.List[int]] = {}
6730
+ for i, np in enumerate(self.name_parts):
6731
+ if isinstance(np, PlaceholderDeployPathNamePart):
6732
+ pd.setdefault(ta.cast(DeployPathPlaceholder, np.placeholder), []).append(i)
6733
+
6734
+ # if 'tag' in pd and 'app' not in pd:
6735
+ # raise DeployPathError('Tag placeholder in path without app', self)
6736
+
6737
+ @property
6738
+ def kind(self) -> ta.Literal['file', 'dir']:
6739
+ return self.parts[-1].kind
6740
+
6741
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6742
+ return ''.join([p.render(placeholders) for p in self.parts])
6743
+
6744
+ @classmethod
6745
+ def parse(cls, s: str) -> 'DeployPath':
6746
+ check.non_empty_str(s)
6747
+ ps = split_keep_delimiter(s, '/')
6748
+ return cls(tuple(DeployPathPart.parse(p) for p in ps))
6749
+
6750
+
6751
+ ########################################
6752
+ # ../deploy/specs.py
6753
+
6754
+
6755
+ ##
6756
+
6757
+
6758
+ def check_valid_deploy_spec_path(s: str) -> str:
6759
+ check.non_empty_str(s)
6760
+ for c in ['..', '//']:
6761
+ check.not_in(c, s)
6762
+ check.arg(not s.startswith('/'))
6763
+ return s
6764
+
6765
+
6766
+ ##
6767
+
6768
+
6769
+ @dc.dataclass(frozen=True)
6770
+ class DeployGitRepo:
6771
+ host: ta.Optional[str] = None
6772
+ username: ta.Optional[str] = None
6773
+ path: ta.Optional[str] = None
6774
+
6775
+ def __post_init__(self) -> None:
6776
+ check.not_in('..', check.non_empty_str(self.host))
6777
+ check.not_in('.', check.non_empty_str(self.path))
6778
+
6779
+
6780
+ @dc.dataclass(frozen=True)
6781
+ class DeployGitSpec:
6782
+ repo: DeployGitRepo
6783
+ rev: DeployRev
6784
+
6785
+ subtrees: ta.Optional[ta.Sequence[str]] = None
6786
+
6787
+ def __post_init__(self) -> None:
6788
+ check.non_empty_str(self.rev)
6789
+ if self.subtrees is not None:
6790
+ for st in self.subtrees:
6791
+ check.non_empty_str(st)
6792
+
6793
+
6794
+ ##
6795
+
6796
+
6797
+ @dc.dataclass(frozen=True)
6798
+ class DeployVenvSpec:
6799
+ interp: ta.Optional[str] = None
6800
+
6801
+ requirements_files: ta.Optional[ta.Sequence[str]] = None
6802
+ extra_dependencies: ta.Optional[ta.Sequence[str]] = None
6803
+
6804
+ use_uv: bool = False
6805
+
6806
+
6807
+ ##
6808
+
6809
+
6810
+ @dc.dataclass(frozen=True)
6811
+ class DeployConfFile:
6812
+ path: str
6813
+ body: str
6814
+
6815
+ def __post_init__(self) -> None:
6816
+ check_valid_deploy_spec_path(self.path)
6817
+
6818
+
6819
+ #
6820
+
6821
+
6822
+ @dc.dataclass(frozen=True)
6823
+ class DeployConfLink(abc.ABC): # noqa
6824
+ """
6825
+ May be either:
6826
+ - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
6827
+ - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
6828
+ - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
6829
+ """
6830
+
6831
+ src: str
6832
+
6833
+ def __post_init__(self) -> None:
6834
+ check_valid_deploy_spec_path(self.src)
6835
+ if '/' in self.src:
6836
+ check.equal(self.src.count('/'), 1)
6837
+
6838
+
6839
+ class AppDeployConfLink(DeployConfLink):
6840
+ pass
6841
+
6842
+
6843
+ class TagDeployConfLink(DeployConfLink):
6844
+ pass
6939
6845
 
6940
- if isinstance(link, AppDeployConfLink):
6941
- link_dst_mid = str(app_tag.app)
6942
- sym_root = link_dir
6943
- elif isinstance(link, TagDeployConfLink):
6944
- link_dst_mid = '-'.join([app_tag.app, app_tag.tag])
6945
- sym_root = conf_dir
6946
- else:
6947
- raise TypeError(link)
6948
6846
 
6949
- link_dst = ''.join([
6950
- link_dst_pfx,
6951
- link_dst_mid,
6952
- link_dst_sfx,
6953
- ])
6847
+ #
6954
6848
 
6955
- root_conf_dir = self._make_dir()
6956
- sym_src = os.path.join(sym_root, link.src)
6957
- sym_dst = os.path.join(root_conf_dir, link_dst)
6958
- check.arg(is_path_in_dir(root_conf_dir, sym_dst))
6959
6849
 
6960
- os.makedirs(os.path.dirname(sym_dst), exist_ok=True)
6961
- relative_symlink(sym_src, sym_dst, target_is_directory=is_link_dir)
6850
+ @dc.dataclass(frozen=True)
6851
+ class DeployConfSpec:
6852
+ files: ta.Optional[ta.Sequence[DeployConfFile]] = None
6962
6853
 
6963
- async def write_conf(
6964
- self,
6965
- spec: DeployConfSpec,
6966
- conf_dir: str,
6967
- app_tag: DeployAppTag,
6968
- link_dir: str,
6969
- ) -> None:
6970
- conf_dir = os.path.abspath(conf_dir)
6971
- os.makedirs(conf_dir)
6854
+ links: ta.Optional[ta.Sequence[DeployConfLink]] = None
6972
6855
 
6973
- #
6856
+ def __post_init__(self) -> None:
6857
+ if self.files:
6858
+ seen: ta.Set[str] = set()
6859
+ for f in self.files:
6860
+ check.not_in(f.path, seen)
6861
+ seen.add(f.path)
6974
6862
 
6975
- for cf in spec.files or []:
6976
- await self._write_conf_file(
6977
- cf,
6978
- conf_dir,
6979
- )
6980
6863
 
6981
- #
6864
+ ##
6982
6865
 
6983
- for link in spec.links or []:
6984
- await self._make_conf_link(
6985
- link,
6986
- conf_dir,
6987
- app_tag,
6988
- link_dir,
6989
- )
6990
6866
 
6867
+ @dc.dataclass(frozen=True)
6868
+ class DeploySpec:
6869
+ app: DeployApp
6991
6870
 
6992
- ########################################
6993
- # ../deploy/tmp.py
6871
+ git: DeployGitSpec
6994
6872
 
6873
+ venv: ta.Optional[DeployVenvSpec] = None
6995
6874
 
6996
- class DeployTmpManager(
6997
- SingleDirDeployPathOwner,
6998
- AtomicPathSwapping,
6999
- ):
7000
- def __init__(
7001
- self,
7002
- *,
7003
- deploy_home: ta.Optional[DeployHome] = None,
7004
- ) -> None:
7005
- super().__init__(
7006
- owned_dir='tmp',
7007
- deploy_home=deploy_home,
7008
- )
6875
+ conf: ta.Optional[DeployConfSpec] = None
7009
6876
 
7010
6877
  @cached_nullary
7011
- def _swapping(self) -> AtomicPathSwapping:
7012
- return TempDirAtomicPathSwapping(
7013
- temp_dir=self._make_dir(),
7014
- root_dir=check.non_empty_str(self._deploy_home),
7015
- )
7016
-
7017
- def begin_atomic_path_swap(
7018
- self,
7019
- kind: AtomicPathSwapKind,
7020
- dst_path: str,
7021
- **kwargs: ta.Any,
7022
- ) -> AtomicPathSwap:
7023
- return self._swapping().begin_atomic_path_swap(
7024
- kind,
7025
- dst_path,
7026
- **kwargs,
7027
- )
6878
+ def key(self) -> DeployKey:
6879
+ return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
7028
6880
 
7029
6881
 
7030
6882
  ########################################
@@ -7537,78 +7389,293 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
7537
7389
 
7538
7390
  #
7539
7391
 
7540
- async def check_output_str(
7541
- self,
7542
- *cmd: str,
7543
- **kwargs: ta.Any,
7544
- ) -> str:
7545
- return (await self.check_output(*cmd, **kwargs)).decode().strip()
7392
+ async def check_output_str(
7393
+ self,
7394
+ *cmd: str,
7395
+ **kwargs: ta.Any,
7396
+ ) -> str:
7397
+ return (await self.check_output(*cmd, **kwargs)).decode().strip()
7398
+
7399
+ #
7400
+
7401
+ async def try_call(
7402
+ self,
7403
+ *cmd: str,
7404
+ **kwargs: ta.Any,
7405
+ ) -> bool:
7406
+ if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
7407
+ return False
7408
+ else:
7409
+ return True
7410
+
7411
+ async def try_output(
7412
+ self,
7413
+ *cmd: str,
7414
+ **kwargs: ta.Any,
7415
+ ) -> ta.Optional[bytes]:
7416
+ if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
7417
+ return None
7418
+ else:
7419
+ return ret
7420
+
7421
+ async def try_output_str(
7422
+ self,
7423
+ *cmd: str,
7424
+ **kwargs: ta.Any,
7425
+ ) -> ta.Optional[str]:
7426
+ if (ret := await self.try_output(*cmd, **kwargs)) is None:
7427
+ return None
7428
+ else:
7429
+ return ret.decode().strip()
7430
+
7431
+
7432
+ ########################################
7433
+ # ../bootstrap.py
7434
+
7435
+
7436
+ @dc.dataclass(frozen=True)
7437
+ class MainBootstrap:
7438
+ main_config: MainConfig = MainConfig()
7439
+
7440
+ deploy_config: DeployConfig = DeployConfig()
7441
+
7442
+ remote_config: RemoteConfig = RemoteConfig()
7443
+
7444
+ system_config: SystemConfig = SystemConfig()
7445
+
7446
+
7447
+ ########################################
7448
+ # ../commands/local.py
7449
+
7450
+
7451
+ class LocalCommandExecutor(CommandExecutor):
7452
+ def __init__(
7453
+ self,
7454
+ *,
7455
+ command_executors: CommandExecutorMap,
7456
+ ) -> None:
7457
+ super().__init__()
7458
+
7459
+ self._command_executors = command_executors
7460
+
7461
+ async def execute(self, cmd: Command) -> Command.Output:
7462
+ ce: CommandExecutor = self._command_executors[type(cmd)]
7463
+ return await ce.execute(cmd)
7464
+
7465
+
7466
+ ########################################
7467
+ # ../deploy/conf.py
7468
+ """
7469
+ TODO:
7470
+ - @conf DeployPathPlaceholder? :|
7471
+ - post-deploy: remove any dir_links not present in new spec
7472
+ - * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
7473
+ - no such thing as 'previously present'.. build a 'deploy state' and pass it back?
7474
+ - ** whole thing can be atomic **
7475
+ - 1) new atomic temp dir
7476
+ - 2) for each subdir not needing modification, hardlink into temp dir
7477
+ - 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
7478
+ - 4) write (or if deleting, omit) new files
7479
+ - 5) swap top level
7480
+ - ** whole deploy can be atomic(-ish) - do this for everything **
7481
+ - just a '/deploy/current' dir
7482
+ - some things (venvs) cannot be moved, thus the /deploy/venvs dir
7483
+ - ** ensure (enforce) equivalent relpath nesting
7484
+ """
7485
+
7486
+
7487
+ class DeployConfManager:
7488
+ def __init__(
7489
+ self,
7490
+ *,
7491
+ deploy_home: ta.Optional[DeployHome] = None,
7492
+ ) -> None:
7493
+ super().__init__()
7494
+
7495
+ self._deploy_home = deploy_home
7496
+
7497
+ #
7498
+
7499
+ async def _write_conf_file(
7500
+ self,
7501
+ cf: DeployConfFile,
7502
+ conf_dir: str,
7503
+ ) -> None:
7504
+ conf_file = os.path.join(conf_dir, cf.path)
7505
+ check.arg(is_path_in_dir(conf_dir, conf_file))
7506
+
7507
+ os.makedirs(os.path.dirname(conf_file), exist_ok=True)
7508
+
7509
+ with open(conf_file, 'w') as f: # noqa
7510
+ f.write(cf.body)
7511
+
7512
+ #
7513
+
7514
+ class _ComputedConfLink(ta.NamedTuple):
7515
+ is_dir: bool
7516
+ link_src: str
7517
+ link_dst: str
7518
+
7519
+ def _compute_conf_link_dst(
7520
+ self,
7521
+ link: DeployConfLink,
7522
+ app_tag: DeployAppTag,
7523
+ conf_dir: str,
7524
+ link_dir: str,
7525
+ ) -> _ComputedConfLink:
7526
+ link_src = os.path.join(conf_dir, link.src)
7527
+ check.arg(is_path_in_dir(conf_dir, link_src))
7528
+
7529
+ #
7530
+
7531
+ if (is_dir := link.src.endswith('/')):
7532
+ # @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
7533
+ check.arg(link.src.count('/') == 1)
7534
+ link_dst_pfx = link.src
7535
+ link_dst_sfx = ''
7536
+
7537
+ elif '/' in link.src:
7538
+ # @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
7539
+ d, f = os.path.split(link.src)
7540
+ # TODO: check filename :|
7541
+ link_dst_pfx = d + '/'
7542
+ link_dst_sfx = DEPLOY_PATH_PLACEHOLDER_SEPARATOR + f
7543
+
7544
+ else: # noqa
7545
+ # @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
7546
+ if '.' in link.src:
7547
+ l, _, r = link.src.partition('.')
7548
+ link_dst_pfx = l + '/'
7549
+ link_dst_sfx = '.' + r
7550
+ else:
7551
+ link_dst_pfx = link.src + '/'
7552
+ link_dst_sfx = ''
7553
+
7554
+ #
7555
+
7556
+ if isinstance(link, AppDeployConfLink):
7557
+ link_dst_mid = str(app_tag.app)
7558
+ elif isinstance(link, TagDeployConfLink):
7559
+ link_dst_mid = DEPLOY_PATH_PLACEHOLDER_SEPARATOR.join([app_tag.app, app_tag.tag])
7560
+ else:
7561
+ raise TypeError(link)
7562
+
7563
+ #
7564
+
7565
+ link_dst_name = ''.join([
7566
+ link_dst_pfx,
7567
+ link_dst_mid,
7568
+ link_dst_sfx,
7569
+ ])
7570
+ link_dst = os.path.join(link_dir, link_dst_name)
7571
+
7572
+ return DeployConfManager._ComputedConfLink(
7573
+ is_dir=is_dir,
7574
+ link_src=link_src,
7575
+ link_dst=link_dst,
7576
+ )
7577
+
7578
+ async def _make_conf_link(
7579
+ self,
7580
+ link: DeployConfLink,
7581
+ app_tag: DeployAppTag,
7582
+ conf_dir: str,
7583
+ link_dir: str,
7584
+ ) -> None:
7585
+ comp = self._compute_conf_link_dst(
7586
+ link,
7587
+ app_tag,
7588
+ conf_dir,
7589
+ link_dir,
7590
+ )
7591
+
7592
+ #
7546
7593
 
7547
- #
7594
+ check.arg(is_path_in_dir(conf_dir, comp.link_src))
7595
+ check.arg(is_path_in_dir(link_dir, comp.link_dst))
7548
7596
 
7549
- async def try_call(
7550
- self,
7551
- *cmd: str,
7552
- **kwargs: ta.Any,
7553
- ) -> bool:
7554
- if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
7555
- return False
7597
+ if comp.is_dir:
7598
+ check.arg(os.path.isdir(comp.link_src))
7556
7599
  else:
7557
- return True
7600
+ check.arg(os.path.isfile(comp.link_src))
7558
7601
 
7559
- async def try_output(
7560
- self,
7561
- *cmd: str,
7562
- **kwargs: ta.Any,
7563
- ) -> ta.Optional[bytes]:
7564
- if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
7565
- return None
7566
- else:
7567
- return ret
7602
+ #
7568
7603
 
7569
- async def try_output_str(
7570
- self,
7571
- *cmd: str,
7572
- **kwargs: ta.Any,
7573
- ) -> ta.Optional[str]:
7574
- if (ret := await self.try_output(*cmd, **kwargs)) is None:
7575
- return None
7576
- else:
7577
- return ret.decode().strip()
7604
+ relative_symlink( # noqa
7605
+ comp.link_src,
7606
+ comp.link_dst,
7607
+ target_is_directory=comp.is_dir,
7608
+ make_dirs=True,
7609
+ )
7578
7610
 
7611
+ #
7579
7612
 
7580
- ########################################
7581
- # ../bootstrap.py
7613
+ async def write_conf(
7614
+ self,
7615
+ spec: DeployConfSpec,
7616
+ app_tag: DeployAppTag,
7617
+ conf_dir: str,
7618
+ link_dir: str,
7619
+ ) -> None:
7620
+ for cf in spec.files or []:
7621
+ await self._write_conf_file(
7622
+ cf,
7623
+ conf_dir,
7624
+ )
7582
7625
 
7626
+ #
7583
7627
 
7584
- @dc.dataclass(frozen=True)
7585
- class MainBootstrap:
7586
- main_config: MainConfig = MainConfig()
7628
+ for link in spec.links or []:
7629
+ await self._make_conf_link(
7630
+ link,
7631
+ app_tag,
7632
+ conf_dir,
7633
+ link_dir,
7634
+ )
7587
7635
 
7588
- deploy_config: DeployConfig = DeployConfig()
7589
7636
 
7590
- remote_config: RemoteConfig = RemoteConfig()
7637
+ ########################################
7638
+ # ../deploy/paths/owners.py
7591
7639
 
7592
- system_config: SystemConfig = SystemConfig()
7640
+
7641
+ class DeployPathOwner(abc.ABC):
7642
+ @abc.abstractmethod
7643
+ def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
7644
+ raise NotImplementedError
7593
7645
 
7594
7646
 
7595
- ########################################
7596
- # ../commands/local.py
7647
+ DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
7597
7648
 
7598
7649
 
7599
- class LocalCommandExecutor(CommandExecutor):
7650
+ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
7600
7651
  def __init__(
7601
7652
  self,
7602
- *,
7603
- command_executors: CommandExecutorMap,
7653
+ *args: ta.Any,
7654
+ owned_dir: str,
7655
+ deploy_home: ta.Optional[DeployHome],
7656
+ **kwargs: ta.Any,
7604
7657
  ) -> None:
7605
- super().__init__()
7658
+ super().__init__(*args, **kwargs)
7606
7659
 
7607
- self._command_executors = command_executors
7660
+ check.not_in('/', owned_dir)
7661
+ self._owned_dir: str = check.non_empty_str(owned_dir)
7608
7662
 
7609
- async def execute(self, cmd: Command) -> Command.Output:
7610
- ce: CommandExecutor = self._command_executors[type(cmd)]
7611
- return await ce.execute(cmd)
7663
+ self._deploy_home = deploy_home
7664
+
7665
+ self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
7666
+
7667
+ @cached_nullary
7668
+ def _dir(self) -> str:
7669
+ return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
7670
+
7671
+ @cached_nullary
7672
+ def _make_dir(self) -> str:
7673
+ if not os.path.isdir(d := self._dir()):
7674
+ os.makedirs(d, exist_ok=True)
7675
+ return d
7676
+
7677
+ def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
7678
+ return self._owned_deploy_paths
7612
7679
 
7613
7680
 
7614
7681
  ########################################
@@ -8513,6 +8580,74 @@ class DeployGitManager(SingleDirDeployPathOwner):
8513
8580
  await self.get_repo_dir(spec.repo).checkout(spec, dst_dir)
8514
8581
 
8515
8582
 
8583
+ ########################################
8584
+ # ../deploy/paths/manager.py
8585
+
8586
+
8587
+ class DeployPathsManager:
8588
+ def __init__(
8589
+ self,
8590
+ *,
8591
+ deploy_home: ta.Optional[DeployHome],
8592
+ deploy_path_owners: DeployPathOwners,
8593
+ ) -> None:
8594
+ super().__init__()
8595
+
8596
+ self._deploy_home = deploy_home
8597
+ self._deploy_path_owners = deploy_path_owners
8598
+
8599
+ @cached_nullary
8600
+ def owners_by_path(self) -> ta.Mapping[DeployPath, DeployPathOwner]:
8601
+ dct: ta.Dict[DeployPath, DeployPathOwner] = {}
8602
+ for o in self._deploy_path_owners:
8603
+ for p in o.get_owned_deploy_paths():
8604
+ if p in dct:
8605
+ raise DeployPathError(f'Duplicate deploy path owner: {p}')
8606
+ dct[p] = o
8607
+ return dct
8608
+
8609
+ def validate_deploy_paths(self) -> None:
8610
+ self.owners_by_path()
8611
+
8612
+
8613
+ ########################################
8614
+ # ../deploy/tmp.py
8615
+
8616
+
8617
+ class DeployTmpManager(
8618
+ SingleDirDeployPathOwner,
8619
+ AtomicPathSwapping,
8620
+ ):
8621
+ def __init__(
8622
+ self,
8623
+ *,
8624
+ deploy_home: ta.Optional[DeployHome] = None,
8625
+ ) -> None:
8626
+ super().__init__(
8627
+ owned_dir='tmp',
8628
+ deploy_home=deploy_home,
8629
+ )
8630
+
8631
+ @cached_nullary
8632
+ def _swapping(self) -> AtomicPathSwapping:
8633
+ return TempDirAtomicPathSwapping(
8634
+ temp_dir=self._make_dir(),
8635
+ root_dir=check.non_empty_str(self._deploy_home),
8636
+ )
8637
+
8638
+ def begin_atomic_path_swap(
8639
+ self,
8640
+ kind: AtomicPathSwapKind,
8641
+ dst_path: str,
8642
+ **kwargs: ta.Any,
8643
+ ) -> AtomicPathSwap:
8644
+ return self._swapping().begin_atomic_path_swap(
8645
+ kind,
8646
+ dst_path,
8647
+ **kwargs,
8648
+ )
8649
+
8650
+
8516
8651
  ########################################
8517
8652
  # ../deploy/venvs.py
8518
8653
  """
@@ -9122,20 +9257,44 @@ class DeployAppManager(DeployPathOwner):
9122
9257
  self._git = git
9123
9258
  self._venvs = venvs
9124
9259
 
9125
- @cached_nullary
9126
- def _dir(self) -> str:
9127
- return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
9260
+ #
9261
+
9262
+ _APP_TAG_DIR_STR = 'tags/apps/@app/@tag/'
9263
+ _APP_TAG_DIR = DeployPath.parse(_APP_TAG_DIR_STR)
9264
+
9265
+ _CONF_TAG_DIR_STR = 'tags/conf/@tag--@app/'
9266
+ _CONF_TAG_DIR = DeployPath.parse(_CONF_TAG_DIR_STR)
9128
9267
 
9268
+ _DEPLOY_DIR_STR = 'deploys/@tag--@app/'
9269
+ _DEPLOY_DIR = DeployPath.parse(_DEPLOY_DIR_STR)
9270
+
9271
+ _APP_DEPLOY_LINK = DeployPath.parse(f'{_DEPLOY_DIR_STR}apps/@app')
9272
+ _CONF_DEPLOY_LINK = DeployPath.parse(f'{_DEPLOY_DIR_STR}conf')
9273
+
9274
+ @cached_nullary
9129
9275
  def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
9130
9276
  return {
9131
- DeployPath.parse('apps/@app/current'),
9132
- DeployPath.parse('apps/@app/deploying'),
9277
+ self._APP_TAG_DIR,
9278
+
9279
+ self._CONF_TAG_DIR,
9133
9280
 
9134
- DeployPath.parse('apps/@app/tags/@tag/conf/'),
9135
- DeployPath.parse('apps/@app/tags/@tag/git/'),
9136
- DeployPath.parse('apps/@app/tags/@tag/venv/'),
9281
+ self._DEPLOY_DIR,
9282
+
9283
+ self._APP_DEPLOY_LINK,
9284
+ self._CONF_DEPLOY_LINK,
9285
+
9286
+ *[
9287
+ DeployPath.parse(f'{self._APP_TAG_DIR_STR}{sfx}/')
9288
+ for sfx in [
9289
+ 'conf',
9290
+ 'git',
9291
+ 'venv',
9292
+ ]
9293
+ ],
9137
9294
  }
9138
9295
 
9296
+ #
9297
+
9139
9298
  async def prepare_app(
9140
9299
  self,
9141
9300
  spec: DeploySpec,
@@ -9144,26 +9303,52 @@ class DeployAppManager(DeployPathOwner):
9144
9303
 
9145
9304
  #
9146
9305
 
9147
- app_dir = os.path.join(self._dir(), spec.app)
9148
- os.makedirs(app_dir, exist_ok=True)
9306
+ deploy_home = check.non_empty_str(self._deploy_home)
9307
+
9308
+ def build_path(pth: DeployPath) -> str:
9309
+ return os.path.join(deploy_home, pth.render(app_tag.placeholders()))
9310
+
9311
+ app_tag_dir = build_path(self._APP_TAG_DIR)
9312
+ conf_tag_dir = build_path(self._CONF_TAG_DIR)
9313
+ deploy_dir = build_path(self._DEPLOY_DIR)
9314
+ app_deploy_link = build_path(self._APP_DEPLOY_LINK)
9315
+ conf_deploy_link_file = build_path(self._CONF_DEPLOY_LINK)
9149
9316
 
9150
9317
  #
9151
9318
 
9152
- tag_dir = os.path.join(app_dir, 'tags', app_tag.tag)
9153
- os.makedirs(tag_dir)
9319
+ os.makedirs(deploy_dir)
9320
+
9321
+ deploying_link = os.path.join(deploy_home, 'deploys/deploying')
9322
+ relative_symlink(
9323
+ deploy_dir,
9324
+ deploying_link,
9325
+ target_is_directory=True,
9326
+ make_dirs=True,
9327
+ )
9154
9328
 
9155
9329
  #
9156
9330
 
9157
- deploying_file = os.path.join(app_dir, 'deploying')
9158
- current_file = os.path.join(app_dir, 'current')
9331
+ os.makedirs(app_tag_dir)
9332
+ relative_symlink(
9333
+ app_tag_dir,
9334
+ app_deploy_link,
9335
+ target_is_directory=True,
9336
+ make_dirs=True,
9337
+ )
9338
+
9339
+ #
9159
9340
 
9160
- if os.path.exists(deploying_file):
9161
- os.unlink(deploying_file)
9162
- relative_symlink(tag_dir, deploying_file, target_is_directory=True)
9341
+ os.makedirs(conf_tag_dir)
9342
+ relative_symlink(
9343
+ conf_tag_dir,
9344
+ conf_deploy_link_file,
9345
+ target_is_directory=True,
9346
+ make_dirs=True,
9347
+ )
9163
9348
 
9164
9349
  #
9165
9350
 
9166
- git_dir = os.path.join(tag_dir, 'git')
9351
+ git_dir = os.path.join(app_tag_dir, 'git')
9167
9352
  await self._git.checkout(
9168
9353
  spec.git,
9169
9354
  git_dir,
@@ -9172,7 +9357,7 @@ class DeployAppManager(DeployPathOwner):
9172
9357
  #
9173
9358
 
9174
9359
  if spec.venv is not None:
9175
- venv_dir = os.path.join(tag_dir, 'venv')
9360
+ venv_dir = os.path.join(app_tag_dir, 'venv')
9176
9361
  await self._venvs.setup_venv(
9177
9362
  spec.venv,
9178
9363
  git_dir,
@@ -9182,18 +9367,33 @@ class DeployAppManager(DeployPathOwner):
9182
9367
  #
9183
9368
 
9184
9369
  if spec.conf is not None:
9185
- conf_dir = os.path.join(tag_dir, 'conf')
9186
- conf_link_dir = os.path.join(current_file, 'conf')
9370
+ conf_dir = os.path.join(app_tag_dir, 'conf')
9187
9371
  await self._conf.write_conf(
9188
9372
  spec.conf,
9189
- conf_dir,
9190
9373
  app_tag,
9191
- conf_link_dir,
9374
+ conf_dir,
9375
+ conf_tag_dir,
9192
9376
  )
9193
9377
 
9194
9378
  #
9195
9379
 
9196
- os.replace(deploying_file, current_file)
9380
+ current_link = os.path.join(deploy_home, 'deploys/current')
9381
+ os.replace(deploying_link, current_link)
9382
+
9383
+
9384
+ ########################################
9385
+ # ../deploy/paths/inject.py
9386
+
9387
+
9388
+ def bind_deploy_paths() -> InjectorBindings:
9389
+ lst: ta.List[InjectorBindingOrBindings] = [
9390
+ inj.bind_array(DeployPathOwner),
9391
+ inj.bind_array_type(DeployPathOwner, DeployPathOwners),
9392
+
9393
+ inj.bind(DeployPathsManager, singleton=True),
9394
+ ]
9395
+
9396
+ return inj.as_bindings(*lst)
9197
9397
 
9198
9398
 
9199
9399
  ########################################
@@ -9911,31 +10111,30 @@ class SystemInterpProvider(InterpProvider):
9911
10111
 
9912
10112
 
9913
10113
  ########################################
9914
- # ../deploy/commands.py
9915
-
10114
+ # ../deploy/deploy.py
9916
10115
 
9917
- ##
9918
-
9919
-
9920
- @dc.dataclass(frozen=True)
9921
- class DeployCommand(Command['DeployCommand.Output']):
9922
- spec: DeploySpec
9923
-
9924
- @dc.dataclass(frozen=True)
9925
- class Output(Command.Output):
9926
- pass
9927
10116
 
10117
+ class DeployManager:
10118
+ def __init__(
10119
+ self,
10120
+ *,
10121
+ apps: DeployAppManager,
10122
+ paths: DeployPathsManager,
10123
+ ):
10124
+ super().__init__()
9928
10125
 
9929
- @dc.dataclass(frozen=True)
9930
- class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
9931
- _apps: DeployAppManager
10126
+ self._apps = apps
10127
+ self._paths = paths
9932
10128
 
9933
- async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
9934
- log.info('Deploying! %r', cmd.spec)
10129
+ async def run_deploy(
10130
+ self,
10131
+ spec: DeploySpec,
10132
+ ) -> None:
10133
+ self._paths.validate_deploy_paths()
9935
10134
 
9936
- await self._apps.prepare_app(cmd.spec)
10135
+ #
9937
10136
 
9938
- return DeployCommand.Output()
10137
+ await self._apps.prepare_app(spec)
9939
10138
 
9940
10139
 
9941
10140
  ########################################
@@ -10246,6 +10445,34 @@ DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
10246
10445
  ]])
10247
10446
 
10248
10447
 
10448
+ ########################################
10449
+ # ../deploy/commands.py
10450
+
10451
+
10452
+ ##
10453
+
10454
+
10455
+ @dc.dataclass(frozen=True)
10456
+ class DeployCommand(Command['DeployCommand.Output']):
10457
+ spec: DeploySpec
10458
+
10459
+ @dc.dataclass(frozen=True)
10460
+ class Output(Command.Output):
10461
+ pass
10462
+
10463
+
10464
+ @dc.dataclass(frozen=True)
10465
+ class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
10466
+ _deploy: DeployManager
10467
+
10468
+ async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
10469
+ log.info('Deploying! %r', cmd.spec)
10470
+
10471
+ await self._deploy.run_deploy(cmd.spec)
10472
+
10473
+ return DeployCommand.Output()
10474
+
10475
+
10249
10476
  ########################################
10250
10477
  # ../targets/inject.py
10251
10478
 
@@ -10315,6 +10542,8 @@ def bind_deploy(
10315
10542
  ) -> InjectorBindings:
10316
10543
  lst: ta.List[InjectorBindingOrBindings] = [
10317
10544
  inj.bind(deploy_config),
10545
+
10546
+ bind_deploy_paths(),
10318
10547
  ]
10319
10548
 
10320
10549
  #
@@ -10326,11 +10555,6 @@ def bind_deploy(
10326
10555
  *([inj.bind(DeployPathOwner, to_key=cls, array=True)] if issubclass(cls, DeployPathOwner) else []),
10327
10556
  )
10328
10557
 
10329
- lst.extend([
10330
- inj.bind_array(DeployPathOwner),
10331
- inj.bind_array_type(DeployPathOwner, DeployPathOwners),
10332
- ])
10333
-
10334
10558
  #
10335
10559
 
10336
10560
  lst.extend([
@@ -10340,6 +10564,8 @@ def bind_deploy(
10340
10564
 
10341
10565
  bind_manager(DeployGitManager),
10342
10566
 
10567
+ bind_manager(DeployManager),
10568
+
10343
10569
  bind_manager(DeployTmpManager),
10344
10570
  inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
10345
10571